feat: signal update success to api

.github/workflows/docker.test.yml

@@ -23,8 +23,8 @@ jobs:
       - name: Wait for some things to be inserted
         uses: nick-invision/retry@v2
         with:
-          timeout_minutes: 5
-          max_attempts: 50
+          timeout_minutes: 10
+          max_attempts: 100
           retry_wait_seconds: 2
           warning_on_retry: false
           command: if (($(docker-compose ${{ env.COMPOSE_ARGS }} logs wdqs-updater | grep "org.wikidata.query.rdf.tool.Updater - Polled" | wc -l) >= 10)); then exit 0; else exit 1; fi

Dockerfile

@@ -1,8 +1,11 @@
 FROM maven:3.6.3-jdk-8 as jarjar
 
-COPY ./ /tmp
 WORKDIR /tmp
-RUN mvn clean compile assembly:single
+COPY ./pom.xml /tmp/pom.xml
+RUN mvn clean install
+
+COPY ./ /tmp
+RUN mvn compile assembly:single
 
 
 FROM openjdk:8-jdk-alpine

docker-compose.ci.yml

@@ -28,6 +28,8 @@ services:
       - WBSTACK_WIKIBASE_SCHEME=http
       - WBSTACK_LOOP_LIMIT=100
       - WBSTACK_API_ENDPOINT=http://api.svc:3030
+      - WBSTACK_API_ENDPOINT_MARK_FAILED=http://api.svc:3030
+      - WBSTACK_API_ENDPOINT_MARK_DONE=http://api.svc:3030
       - WBSTACK_BATCH_SLEEP=1
       - WIKIBASE_HOST=wikibase.svc
       - HEAP_SIZE=32m

src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java

@@ -7,6 +7,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.OutputStream;
 import java.lang.management.ManagementFactory;
 import java.net.HttpURLConnection;
 import java.net.URI;
@@ -71,6 +73,8 @@ public final class WbStackUpdate {
 
     // Static configuration, primarily from environment variables
     private static String wbStackApiEndpoint;
+    private static String wbStackApiEndpointMarkDone;
+    private static String wbStackApiEndpointMarkFailed;
     private static long wbStackSleepBetweenApiCalls;
     private static int wbStackUpdaterThreadCount;
     private static String wbStackUpdaterNamespaces;
@@ -93,6 +97,8 @@ private WbStackUpdate() {
 
     private static void setValuesFromEnvOrDie() {
         if (System.getenv("WBSTACK_API_ENDPOINT") == null
+                || System.getenv("WBSTACK_API_ENDPOINT_MARK_FAILED") == null
+                || System.getenv("WBSTACK_API_ENDPOINT_MARK_DONE") == null
                 || System.getenv("WBSTACK_BATCH_SLEEP") == null
                 || System.getenv("WBSTACK_LOOP_LIMIT") == null) {
             System.err.println("WBSTACK_API_ENDPOINT, WBSTACK_BATCH_SLEEP and WBSTACK_LOOP_LIMIT environment variables must be set.");
@@ -101,6 +107,8 @@ private static void setValuesFromEnvOrDie() {
 
         wbStackProxyMapIngress = System.getenv("WBSTACK_PROXYMAP_INGRESS");
         wbStackApiEndpoint = System.getenv("WBSTACK_API_ENDPOINT");
+        wbStackApiEndpointMarkFailed = System.getenv("WBSTACK_API_ENDPOINT_MARK_FAILED");
+        wbStackApiEndpointMarkDone = System.getenv("WBSTACK_API_ENDPOINT_MARK_DONE");
         wbStackSleepBetweenApiCalls = Long.parseLong(System.getenv("WBSTACK_BATCH_SLEEP"));
         wbStackUpdaterThreadCount = Integer.parseInt(System.getenv().getOrDefault("WBSTACK_THREAD_COUNT", "10"));
         wbStackUpdaterNamespaces = System.getenv().getOrDefault("WBSTACK_UPDATER_NAMESPACES", "120,122,146");
@@ -193,18 +201,45 @@ private static JsonArray getBatchesFromApi() throws IOException {
         }
     }
 
+    private static void updateRemoteBatchStatus(int batchId, boolean success) throws IOException {
+        URL obj = new URL(success ? wbStackApiEndpointMarkDone : wbStackApiEndpointMarkFailed);
+        HttpURLConnection con = (HttpURLConnection) obj.openConnection();
+        con.setRequestMethod("POST");
+        con.setRequestProperty("User-Agent", USER_AGENT);
+        con.setDoOutput(true);
+
+        JsonObject body = new JsonObject();
+        JsonArray batches = new JsonArray();
+        batches.add(batchId);
+        body.add("batches", batches);
+
+        OutputStream os = con.getOutputStream();
+        OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8");
+        osw.write(body.toString());
+        osw.flush();
+        osw.close();
+        os.close();
+        con.connect();
+
+        int responseCode = con.getResponseCode();
+        if (responseCode != 200) {
+            throw new IOException("Got non 200 response code from API: " + responseCode);
+        }
+    }
+
     private static void updateBatch(JsonElement batchElement) {
         // Get the values for the batch from the JSON
         JsonObject batch = batchElement.getAsJsonObject();
         String entityIDs = batch.get("entityIds").getAsString();
+        int batchId = batch.get("id").getAsInt();
         JsonObject wiki = batch.get("wiki").getAsJsonObject();
         String domain = wiki.get("domain").getAsString();
         JsonObject wikiQsNamespace = wiki.get("wiki_queryservice_namespace").getAsJsonObject();
         String qsBackend = wikiQsNamespace.get("backend").getAsString();
         String qsNamespace = wikiQsNamespace.get("namespace").getAsString();
 
         // Run the main Update class with our altered args
-        runUpdaterWithArgs(new String[]{
+        boolean updateWasSuccessful = runUpdaterWithArgs(new String[]{
                 "--wikibaseHost", domain,
                 "--ids", entityIDs,
                 "--entityNamespaces", wbStackUpdaterNamespaces,
@@ -213,11 +248,16 @@ private static void updateBatch(JsonElement batchElement) {
                 "--conceptUri", "https://" + domain
         });
 
-        // TODO on success maybe report back?
+        try {
+            updateRemoteBatchStatus(batchId, updateWasSuccessful);
+        } catch (Exception ex) {
+            System.err.println("Failed to update remote batch status.");
+            ex.printStackTrace();
+        }
     }
 
     @SuppressFBWarnings(value = "IMC_IMMATURE_CLASS_PRINTSTACKTRACE", justification = "We should introduce proper logging framework")
-    private static void runUpdaterWithArgs(String[] args) {
+    private static boolean runUpdaterWithArgs(String[] args) {
         try {
             Closer closer = Closer.create();
 
@@ -242,7 +282,9 @@ private static void runUpdaterWithArgs(String[] args) {
         } catch (Exception e) {
             System.err.println("Failed batch!");
             e.printStackTrace();
+            return false;
         }
+        return true;
     }
 
     private static String getProxyMapString( UpdateOptions options ) {