Merge pull request #157 from aclavio/develop

Release 1.1.0

CONTRIBUTING.md

@@ -28,7 +28,7 @@ Looking to build the code from source?
 #### Prerequisites
 You need:
 
-- MarkLogic 9.0-10 or later
+- MarkLogic 10.0-6 or later
 - Gradle 4.6 or later
 - Java JDK 8 or later
 - A decent IDE. (Recommended: Visual Studio Code)

README.md

@@ -40,19 +40,14 @@ Should you use MLSC for your project? See [Applicability](#applicability)
 
 Prerequisites:
 
-> 1. MarkLogic 9+
+> 1. MarkLogic 10.0-6+
 > 2. [ml-gradle](https://github.com/marklogic-community/ml-gradle) 3.14.0+
 
 The _State Conductor_ is distributed as an [mlBundle](https://github.com/marklogic-community/ml-gradle/wiki/Bundles) for `ml-gradle` projects. To add the State Conductor to your project, add the following dependency to your ml-gradle project:
 
 ```groovy
-repositories {
-  maven {
-    url {"https://dl.bintray.com/aclavio/maven"}
-  }
-}
 dependencies {
-  mlBundle "com.marklogic:marklogic-state-conductor:1.0.0"
+  mlBundle "com.marklogic:marklogic-state-conductor:1.1.0"
 }
 ```
 

build.gradle

@@ -5,11 +5,12 @@ allprojects {
   }
 
   repositories {
-    jcenter()
     mavenLocal()
+    mavenCentral()
+    jcenter() // remove after dependencies have been migrated
     maven {url "http://developer.marklogic.com/maven2/"}
   }
 
   group = "com.marklogic"
-  version = "1.0.0"
+  version = "1.1.0"
 }

settings.gradle

@@ -1 +1 @@
-include "state-conductor-modules", "state-conductor-dataservices", "state-conductor-example", "state-conductor-dhf5-example"
+include "state-conductor-modules", "state-conductor-dataservices", "state-conductor-example", "state-conductor-dhf5-example", "state-conductor-corb-example"

state-conductor-corb-example/bin/corbDriver.properties

@@ -1,11 +1,11 @@
 #=========== Do not change ==========
 URIS-MODULE=/state-conductor/corb2/uris-module.sjs
-PROCESS-MODULE=state-conductor/corb2/process-module.sjs
+PROCESS-MODULE=/state-conductor/corb2/process-module.sjs
 EXIT-CODE-NO-URIS=3
 #====================================
 
 #=== User Configurable Properties ===
 XCC-CONNECTION-URI=
-URIS-MODULE.jobCount=500
+URIS-MODULE.executionCount=500
 URIS-MODULE.resumeWait=true
 #URIS-MODULE.flowNames=

state-conductor-corb-example/build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   repositories {
-    jcenter()
     mavenLocal()
+    mavenCentral()
   }
   dependencies {
     classpath "com.marklogic:marklogic-unit-test-client:1.0.0"
@@ -25,7 +25,7 @@ dependencies {
   corb "com.marklogic:marklogic-corb:2.4.0"
 }
 
-task runCorbDriver(type: com.marklogic.gradle.task.CorbTask, dependsOn: ["mlLoadModules"]) {
+task runCorbDriver(type: com.marklogic.gradle.task.CorbTask) {
   urisModule="/state-conductor/corb2/uris-module.sjs"
   transformModule="/state-conductor/corb2/process-module.sjs"
   threadCount = 5

state-conductor-dataservices/README.md

@@ -18,7 +18,7 @@ The `com.marklogic.stateconductor.StateConductorDriver` class provides an implem
 Run the stand-alone jar file:
 
 ```
-java -jar state-conductor-dataservices-1.0.0.jar
+java -jar state-conductor-dataservices-1.1.0.jar
 ```
 
 ## Status WebServer

state-conductor-dataservices/application.properties

@@ -2,3 +2,4 @@ mlHost=localhost
 mlPort=9010
 username=admin
 password=admin
+server.port=9000

state-conductor-dataservices/build.gradle

@@ -1,7 +1,8 @@
 buildscript {
   repositories {
-    jcenter()
     mavenLocal()
+    mavenCentral()
+    jcenter()
   }
   dependencies {
     classpath "com.marklogic:marklogic-unit-test-client:1.0.0"
@@ -28,7 +29,7 @@ dependencies {
   mlBundle "com.marklogic:marklogic-unit-test-modules:1.0.0"
 
   implementation 'org.slf4j:slf4j-api:1.7.21'
-	implementation 'com.marklogic:marklogic-client-api:4.2.0'
+  implementation 'com.marklogic:marklogic-client-api:4.2.0'
   implementation 'com.marklogic:ml-javaclient-util:3.13.4'
   implementation 'com.google.guava:guava:28.1-jre'
 

state-conductor-dhf5-example/build.gradle

@@ -1,12 +1,12 @@
 buildscript {
-	repositories {
-		jcenter()
-		mavenLocal()
-	}
-	dependencies {
-		classpath "com.marklogic:marklogic-unit-test-client:1.0.0"
-		classpath "com.marklogic:ml-gradle:${mlgradleVersion}"
-	}
+  repositories {
+    mavenLocal()
+    mavenCentral()
+  }
+  dependencies {
+    classpath "com.marklogic:marklogic-unit-test-client:1.0.0"
+    classpath "com.marklogic:ml-gradle:${mlgradleVersion}"
+  }
 }
 
 plugins {
@@ -16,6 +16,6 @@ plugins {
 }
 
 dependencies {
-	mlBundle "com.marklogic:marklogic-state-conductor:${version}"
+  mlBundle "com.marklogic:marklogic-state-conductor:${version}"
   mlBundle "com.marklogic:marklogic-unit-test-modules:1.0.0"
 }

state-conductor-dhf5-example/flows/CustomFlow.flow.json

@@ -8,22 +8,23 @@
   },
   "steps": {
     "1": {
-      "name" : "MyCustomStep",
-      "stepDefinitionName" : "MyCustomStep",
-      "stepDefinitionType" : "CUSTOM",
-      "description" : "Adds some custom test data",
-      "options" : {
-        "collections" : [ "MyCustomStep" ],
-        "sourceDatabase" : "sce-dh5-STAGING",
-        "permissions" : "data-hub-operator,read,data-hub-operator,update",
-        "outputFormat" : "json",
-        "targetDatabase" : "sce-dh5-STAGING"
+      "name": "MyCustomStep",
+      "stepDefinitionName": "MyCustomStep",
+      "stepDefinitionType": "CUSTOM",
+      "description": "Adds some custom test data",
+      "options": {
+        "collections": ["MyCustomStep"],
+        "sourceQuery": "cts.collectionQuery([\"test\"])",
+        "sourceDatabase": "sce-dh5-STAGING",
+        "permissions": "data-hub-operator,read,data-hub-operator,update",
+        "outputFormat": "json",
+        "targetDatabase": "sce-dh5-STAGING"
       },
-      "customHook" : { },
-      "modulePath" : "/custom-modules/custom/MyCustomStep/main.sjs",
-      "retryLimit" : 0,
-      "batchSize" : 100,
-      "threadCount" : 4
+      "customHook": {},
+      "modulePath": "/custom-modules/custom/MyCustomStep/main.sjs",
+      "retryLimit": 0,
+      "batchSize": 100,
+      "threadCount": 4
     }
   }
 }

state-conductor-dhf5-example/gradle.properties

@@ -25,9 +25,9 @@ mlFinalTriggersDbName=sce-dh5-FINAL-TRIGGERS
 mlFinalSchemasDbName=sce-dh5-FINAL-SCHEMAS
 mlFinalForestsPerHost=1
 
-mlJobsPort=8313
-mlJobsAppserverName=sce-dh5-JOBS-server
-mlJobsDbName=sce-dh5-JOBS
-mlJobsForestsPerHost=1
+mlJobPort=8313
+mlJobAppserverName=sce-dh5-JOBS-server
+mlJobDbName=sce-dh5-JOBS
+mlJobForestsPerHost=1
 
 mlModulesDbName=sce-dh5-MODULES

state-conductor-dhf5-example/src/test/ml-modules/root/test/suites/StateConductorDHF5Suite/test-data/stateMachines/missing-dhf-state-machine.asl.json

@@ -16,6 +16,17 @@
       },
       "Next": "Success"
     },
+    "runStep99": {
+      "Type": "Task",
+      "Comment": "runs the dhf 5 flow step",
+      "Resource": "/state-conductor/actions/common/dhf/dhf5RunFlowStepAction.sjs",
+      "Parameters": {
+        "flowName": "PersonFlow",
+        "step": 99,
+        "flowOptions": {}
+      },
+      "Next": "Success"
+    },
     "Success": {
       "Type": "Succeed"
     }

state-conductor-dhf5-example/src/test/ml-modules/root/test/suites/StateConductorDHF5Suite/test-dhf5RunFlowStepActionSQ.sjs

@@ -0,0 +1,93 @@
+'use strict';
+const test = require('/test/test-helper.xqy');
+const dhf5RunFlowStepAction = require('/state-conductor/actions/common/dhf/dhf5RunFlowStepAction.sjs');
+
+const uri1 = '/data/janedoe.json';
+const uri2 = '/data/johndoe.json';
+
+// helper to allow processing in seperate transactions
+function isolate(func, db) {
+  db = db || xdmp.database();
+  return fn.head(
+    xdmp.invokeFunction(
+      () => {
+        declareUpdate();
+        return func();
+      },
+      {
+        isolation: 'different-transaction',
+        commit: 'auto',
+        database: db,
+      }
+    )
+  );
+}
+
+const assertions = [];
+
+// run a test documents through the CustomFlow flow step 1 using it's sourceQuery
+let flowName = 'CustomFlow';
+let step = 1;
+
+let original1 = isolate(() => cts.doc(uri1).toObject());
+let original2 = isolate(() => cts.doc(uri2).toObject());
+let resp = isolate(() =>
+  dhf5RunFlowStepAction.performAction(null, {
+    flowName: flowName,
+    step: step,
+    useSourceQuery: true,
+  })
+);
+let transformed1 = isolate(() => cts.doc(uri1).toObject());
+let transformed2 = isolate(() => cts.doc(uri2).toObject());
+
+xdmp.log(resp);
+xdmp.log(transformed1);
+xdmp.log(transformed2);
+
+let flowResult = resp[flowName];
+assertions.push(test.assertTrue(flowResult !== null));
+
+let stepResult = flowResult['' + step];
+assertions.push(test.assertTrue(stepResult !== null));
+
+assertions.push(
+  test.assertTrue(stepResult.jobId.length > 0),
+  test.assertEqual(2, stepResult.totalCount, 'found souceQuery docs'),
+  test.assertEqual(0, stepResult.errorCount),
+  test.assertTrue(stepResult.completedItems.includes(uri1)),
+  test.assertTrue(stepResult.completedItems.includes(uri2)),
+
+  test.assertTrue(transformed1.envelope != null),
+  test.assertTrue(transformed1.envelope.headers != null),
+  test.assertTrue(transformed1.envelope.headers.createdOn.length > 0),
+  test.assertTrue(transformed1.envelope.headers.randomUuid.length > 0),
+  test.assertTrue(transformed1.envelope.triples != null),
+  test.assertTrue(transformed1.envelope.instance != null),
+  test.assertEqual('Jane', transformed1.envelope.instance.firstName),
+  test.assertEqual('Doe', transformed1.envelope.instance.lastName),
+  test.assertEqual(JSON.stringify(original1), JSON.stringify(transformed1.envelope.instance)),
+  test.assertTrue(isolate(() => xdmp.documentGetCollections(uri1).includes('MyCustomStep'))),
+  test.assertTrue(isolate(() => xdmp.documentGetMetadata(uri1).datahubCreatedInFlow === flowName)),
+  test.assertTrue(
+    isolate(() => xdmp.documentGetMetadata(uri1).datahubCreatedByStep === 'MyCustomStep')
+  ),
+
+  test.assertTrue(transformed2.envelope != null),
+  test.assertTrue(transformed2.envelope.headers != null),
+  test.assertTrue(transformed2.envelope.headers.createdOn.length > 0),
+  test.assertTrue(transformed2.envelope.headers.randomUuid.length > 0),
+  test.assertTrue(transformed2.envelope.triples != null),
+  test.assertTrue(transformed2.envelope.instance != null),
+  test.assertEqual('John', transformed2.envelope.instance.firstName),
+  test.assertEqual('Doe', transformed2.envelope.instance.lastName),
+  test.assertEqual(JSON.stringify(original2), JSON.stringify(transformed2.envelope.instance)),
+  test.assertTrue(isolate(() => xdmp.documentGetCollections(uri2).includes('MyCustomStep'))),
+  test.assertTrue(isolate(() => xdmp.documentGetMetadata(uri2).datahubCreatedInFlow === flowName)),
+  test.assertTrue(
+    isolate(() => xdmp.documentGetMetadata(uri2).datahubCreatedByStep === 'MyCustomStep')
+  )
+);
+
+// return
+assertions;

state-conductor-dhf5-example/src/test/ml-modules/root/test/suites/StateConductorDHF5Suite/test-scDhf5Integration.sjs

@@ -41,10 +41,35 @@ respDoc = sc.executeStateByExecutionDoc(executionDoc, false);
 xdmp.log(respDoc);
 assertions.push(
   test.assertEqual('failed', respDoc.status, 'status check'),
-  test.assertEqual('JS-JAVASCRIPT', respDoc.errors['runStep1'].name, 'handled missing dhf flow'),
+  test.assertEqual('MISSING-FLOW', respDoc.errors['runStep1'].name, 'handled missing dhf flow'),
   test.assertTrue(
     respDoc.errors['runStep1'].data.includes(
-      'Error: The flow with the name MissingFlow could not be found.'
+      'The flow with the name MissingFlow could not be found.'
+    ),
+    'handled missing dhf flow'
+  )
+);
+
+// test reference to wrong DHF step
+executionDoc = xdmp.toJSON({
+  id: '0405536f-dd84-4ca6-8de8-c57062b2252d',
+  name: 'missing-dhf-state-machine',
+  status: 'working',
+  state: 'runStep99',
+  uri: '/data/johndoe.json',
+  database: xdmp.database(),
+  modules: xdmp.modulesDatabase(),
+  provenance: [],
+});
+
+respDoc = sc.executeStateByExecutionDoc(executionDoc, false);
+xdmp.log(respDoc);
+assertions.push(
+  test.assertEqual('failed', respDoc.status, 'status check'),
+  test.assertEqual('MISSING-STEP', respDoc.errors['runStep99'].name, 'handled missing dhf step'),
+  test.assertTrue(
+    respDoc.errors['runStep99'].data.includes(
+      'Step 99 for the flow: PersonFlow could not be found.'
     ),
     'handled missing dhf flow'
   )

state-conductor-example/README.md

@@ -1,3 +1,3 @@
-# MarkLogic State Conductor Example Project
-
-_TODO_
+# MarkLogic State Conductor Example Project
+
+State Conductor examples and unit tests.

state-conductor-example/build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   repositories {
-    jcenter()
     mavenLocal()
+    mavenCentral()
   }
   dependencies {
     classpath "com.marklogic:marklogic-unit-test-client:1.0.0"