How to synchronize data between stSoftware server and an external system.

Summary

A client program calls the stSoftware server with the /v6/sync/{className} ReST service for the class name that is required to be sync'd to an external system. The since parameter controls which record changes will returned. Initially the program starts with a default value for since, once the result set of the changed records has been scrolled through a new value of since will be returned from the server. The returned since is the server time of the last change returned.

For each record changed there will be:-

Action:-

'D' deleted

'U' updated

'I' created

_href which points to the data for record. This URI is used to call the v8/class/{className}/{key} ReST service. The results of this ReST service call can be cached as the key will always change if the record is changed in anyway.

ReST web serices

The two main web services used are /v6/sync and /v8/class

Sample Program

Download here

package com.aspc;

import org.apache.commons.logging.Log;
import com.aspc.remote.rest.ReST;
import com.aspc.remote.rest.Response;
import com.aspc.remote.rest.Status;
import com.aspc.remote.rest.errors.ReSTException;
import com.aspc.remote.util.misc.CLogger;
import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import org.json.JSONArray;
import org.json.JSONObject;

/**
 * SampleSync
 *
 * The SampleSync program implements an application that simply make ReST call
 * to Job class and log the results.
 *
 * @author parminder
 */
public class SampleSync {

    private final String host;
    private final String username;
    private final String password;

    private static final Log LOGGER = CLogger.getLog("src.main.java.com.aspc.SampleSync");//#LOGGER-NOPMD

    public SampleSync(final @Nonnull String host, final @Nonnull String username, final @Nonnull String password) {
        this.host = host;
        this.username = username;
        this.password = password;
    }

    /**
     * Loop forever scanning for new or changed jobs.
     *
     * @param scanFrom from what time should we scan
     *
     * @throws Exception a non temporary issue has occurred.
     */
    @SuppressWarnings("SleepWhileInLoop")
    public void scan(final @Nonnegative String scanFrom) throws Exception {

        String since = scanFrom;
        while (true) {

            try {
                since = process(since);
            } catch (ReSTException re) {
                switch (re.status) {
                    case C500_SERVER_INTERNAL_ERROR:
                    case C503_SERVICE_UNAVAILABLE:
                    case C521_WEB_SERVER_IS_DOWN:
                    case C599_TIMED_OUT_SERVER_NETWORK_CONNECT:
                        LOGGER.warn("scan failed, retrying", re);
                        Thread.sleep((long) (120000 * Math.random()) + 1);

                    default:
                        LOGGER.error("Permanent issue", re);
                        throw re;
                }
            }
        }
    }

    /**
     *
     * @param since the time to scan from.
     *
     * @return the last transaction time of any record on the server.
     *
     * @throws Exception a issue as occurred.
     */
    public @Nonnull String process(final @Nonnull String since) throws Exception {

        long nextSince;
        String callURL = "/ReST/v6/sync/Job";

        while (true) {

            ReST.Builder b = ReST.builder(host + callURL)
                    .setAuthorization(username, password)
                    .setParameter("since", since)
                    .setParameter("block", "2 minute");

            JSONObject json = b.getResponseAndCheck().getContentAsJSON();

            JSONArray resultsArray = json.getJSONArray("results");

            int arrayLength = resultsArray.length();

            for (int pos = 0; pos < arrayLength; pos++) {

                JSONObject transaction = resultsArray.getJSONObject(pos);

                processJob(transaction.getString("_href"));
            }

            if (json.has("next")) {
                callURL = json.getString("next");
            } else {
                nextSince = json.getLong("since");
                break;
            }
        }
        assert nextSince > 0;
        return Long.toString(nextSince);

    }

    /**
     *
     * @param call ReST URL
     *
     * @throws Exception
     */
    private void processJob(final @Nonnull String call) throws Exception {

        JSONObject json;
        Response r = ReST
                .builder(host + call)
                .setAuthorization(username, password)
                .getResponse();

        if (r.status == Status.C404_ERROR_NOT_FOUND) {
            LOGGER.info(r.status);
        }
        r.checkStatus();

        json = r.getContentAsJSON();

        // DO STUFF 
        LOGGER.info(json.toString(2));
    }

}