Merge pull request #70 from marklogic-community/develop

For version 1.2.0

build.gradle

@@ -68,6 +68,7 @@ test {
     systemProperty "feature.user", mlUsername
     systemProperty "feature.password", mlPassword
     systemProperty "feature.service", serviceName
+    systemProperty "gds.version", project.version
 }
 
 

examples/sample-project/build.gradle

@@ -11,7 +11,7 @@ buildscript {
     mavenLocal()
   }
   dependencies {
-    classpath "com.marklogic:marklogic-geo-data-services-modules:1.1.2"
+    classpath "com.marklogic:marklogic-geo-data-services-modules:1.2.0"
     classpath "com.google.code.gson:gson:2.8.6"
   }
 }

gradle-test.properties

@@ -11,6 +11,7 @@ mlUsername=admin
 mlPassword=admin
 
 mlConfigPaths=src/main/ml-config,src/test/ml-config
+mlModulePaths=src/main/ml-modules,src/test/ml-modules
 mlDataPaths=src/test/ml-data
 mlSchemaPaths=src/test/ml-schemas
 

gradle.properties

@@ -1,5 +1,5 @@
 mlAppName=geo-data-services
-version=1.1.2
+version=1.2.0
 
 mlHost=localhost
 mlRestPort=8095

src/main/ml-modules/ext/search/geo-search-util.xqy

@@ -4,14 +4,55 @@ module namespace gsu = "http://marklogic.com/geo-data-services/geo-search-util";
 
 import module namespace sut = "http://marklogic.com/rest-api/lib/search-util" at "/MarkLogic/rest-api/lib/search-util.xqy";
 import module namespace search = "http://marklogic.com/appservices/search" at "/MarkLogic/appservices/search/search.xqy";
+import module namespace ast = "http://marklogic.com/appservices/search-ast" at "/MarkLogic/appservices/search/ast.xqy";
 
 declare private variable $VALUES_OPTION_PREFIX as xs:string := "__generated_values_";
 
+declare function gsu:parse-combined(
+  $json as item(),
+  $options-name as xs:string?
+) as cts:query
+{
+  (: convert to xml :)
+  let $search := gsu:search-from-json($json)
+
+  (: get options from request body, or fall back to named options :)
+  let $resolved-options := gsu:resolve-options($search, ($options-name, "all")[1])
+
+  (: include any additional query :)
+  let $additional-queries := $resolved-options/search:additional-query/*/cts:query(.)
+
+  (: include qtext :)
+  let $text-query :=
+    if ($search/search:qtext) then
+      search:parse($search/search:qtext, $resolved-options, "cts:query")
+    else ()
+
+  (: include structured query :)
+  let $structured-query :=
+    if ($search/search:query) then
+      gsu:structured-to-cts($search/search:query, $resolved-options)
+    else ()
+
+  (: combine all into large andQuery :)
+  return cts:and-query((
+    $additional-queries,
+    $text-query,
+    $structured-query
+  ))
+};
+
 declare function gsu:search-from-json(
-  $json as node()
+  $json as item()
 ) as element(search:search)
 {
-  sut:search-from-json($json)
+  let $json :=
+    if ($json instance of json:object) then
+      xdmp:to-json($json)
+    else
+      $json
+  return
+    sut:search-from-json($json)
 };
 
 declare function gsu:search-to-json(
@@ -21,9 +62,35 @@ declare function gsu:search-to-json(
   sut:search-to-json($search)
 };
 
-declare function gsu:get-search-options(
+declare function gsu:structured-to-cts(
+  $sq as element(search:query),
+  $options as element(search:options)
+) as cts:query?
+{
+  map:get(ast:to-query($sq, $options), "query")
+};
+
+declare function gsu:resolve-options(
+  $search as element(search:search),
   $options-name as xs:string
 ) as element(search:options)
+{
+  let $delta-options := $search/search:options
+  let $saved-options := gsu:get-search-options($options-name)
+  let $empty-options := <options xmlns="http://marklogic.com/appservices/search"/>
+  return
+    if (exists($delta-options) and exists($saved-options)) then
+      sut:merge-options($saved-options, $delta-options)
+    else if (empty($delta-options) and empty($saved-options)) then
+      $empty-options
+    else
+      (: only one of these contains a value :)
+      ($delta-options, $saved-options)
+};
+
+declare function gsu:get-search-options(
+  $options-name as xs:string
+) as element(search:options)?
 {
   sut:options(map:entry("options", $options-name))
 };
@@ -54,7 +121,7 @@ declare function gsu:get-geometry-type(
   else ()
 };
 
-declare function gsu:create-heatmapped-constraint(  
+declare function gsu:create-heatmapped-constraint(
   $base-geo-constraint as element(search:constraint),
   $options as object-node()
 ) as element(search:constraint)
@@ -106,7 +173,7 @@ declare function gsu:create-search-criteria(
   let $return-values := $options/returnValues eq fn:true()
   let $values-limit := xs:unsignedLong($options/valuesLimit)
   let $base-options := gsu:get-search-options($stored-options-name)
-  
+
   (: qtext to structured queries :)
   let $qtext-queries := search:parse($options/fullQueryText, $base-options, "search:query")
 
@@ -125,18 +192,18 @@ declare function gsu:create-search-criteria(
       $base-geo-constraint-index,
       if ($values-limit gt 0) then element search:values-option { fn:concat("limit=", $values-limit) } else ()
     }
-  
+
   (: create modified stored options :)
   let $new-options := element search:options {
     $base-options/@*,
     $base-options/* except $base-geo-constraints,
     $new-geo-constraints,
     $values-options
   }
-  
+
   (: merge with delta options :)
   let $merged-options := sut:merge-options($new-options, $delta-search/search:options)
-  
+
   (: create search:search :)
   return element search:search {
     element search:query {
@@ -154,8 +221,8 @@ declare function gsu:get-search-suggestions(
 ) as xs:string*
 {
   search:suggest(
-    $qtext, 
-    $search/search:options, 
+    $qtext,
+    $search/search:options,
     ($limit, 10)[1],
     (),
     (),
@@ -224,7 +291,7 @@ declare private function gsu:add-constraint-clusters(
   let $aggregate-values := $options/aggregateValues eq fn:true()
   let $geo-constraint-boxes := $search-response/search:boxes[@name = $geo-constraint-name]
   return if ($aggregate-values and $geo-constraint-boxes)
-  then 
+  then
     let $point-clusters := json:array(), $points := json:array()
     let $_ := (
       for $box in $geo-constraint-boxes/search:box
@@ -260,7 +327,7 @@ declare private function gsu:add-constraint-values(
   let $geo-constraint-values := $search/search:options/search:values[@name = $geo-constraint-values-name]
   let $aggregate-values := $options/aggregateValues eq fn:true()
   return if (fn:not($aggregate-values) and $geo-constraint-values)
-  then 
+  then
     let $is-longlat := fn:not(fn:empty($geo-constraint-values//search:geo-option[fn:string(.) eq "type=long-lat-point"]))
     let $lat-index := if ($is-longlat) then 2 else 1
     let $lon-index := if ($is-longlat) then 1 else 2
@@ -277,4 +344,4 @@ declare private function gsu:add-constraint-values(
         })
     )
   else $json
-};
+};

src/main/ml-modules/ext/version.sjs

@@ -1 +1 @@
-exports.version = "1.1.2"; // TODO: replace with "%%version%%" token, but resolve how to render this before packaging it as a JAR
+exports.version = "1.2.0"; // TODO: replace with "%%version%%" token, but resolve how to render this before packaging it as a JAR

src/main/ml-modules/services/geoQueryService.sjs

@@ -11,6 +11,7 @@ const sql2optic = require('/ext/sql/sql2optic.sjs');
 const geostats = require('/ext/geo/geostats.js');
 const geoextractor = require('/ext/geo/extractor.sjs');
 const qd = require('/ext/query/ctsQueryDeserialize.sjs').qd;
+const gsu = require('/ext/search/geo-search-util.xqy');
 
 const MAX_RECORD_COUNT = 5000;
 
@@ -23,6 +24,7 @@ function post(context, params, input) {
     xdmp.trace("GDS-DEBUG", JSON.stringify(geoJson));
     return {
       "$version": require('/ext/version.sjs').version,
+      "$timestamp": new Date().toISOString(),
       ...geoJson
     };
   } catch (err) {
@@ -551,9 +553,15 @@ function parseWhere(query) {
 
   const where = query.where;
   let whereQuery = null;
+  if (where && typeof where !== 'string' && where.search) {
+    // where.search contains Combined Query
+    whereQuery = gsu.parseCombined(where, query.optionsName);
+
+  } else
   if (!where || where === "1=1" || where === "1 = 1" || where === "") {
     //whereQuery = cts.trueQuery();
     whereQuery = op.eq(1, 1)
+
   } else {
     whereQuery = sql2optic.where(where);
   }
@@ -1040,12 +1048,12 @@ function getObjects(req, exportPlan=false) {
     return exported;
   }
   else {
-    // GeoJSON features must always have a "geometry" property; for cases where the feature has no 
+    // GeoJSON features must always have a "geometry" property; for cases where the feature has no
     // associated geometry data or "returnGeometry" is set to false, set "geometry" property to null.
     // See GeoJSON RFC: https://tools.ietf.org/html/rfc7946#section-3.2
     pipeline = pipeline.map((feature) => {
       var outFeature = feature;
-      
+
       if (returnGeometry && extractor.hasExtractFunction()) {
         xdmp.trace("GDS-DEBUG", "Getting Extractor function");
         outFeature = extractor.extract(feature);

src/test/java/ServiceDescriptorTest.java

@@ -8,6 +8,28 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 public class ServiceDescriptorTest extends AbstractFeatureServiceTest {
+    @Test
+    public void versionCheck() {
+        // Sanity check the version number output of GDS endpoint(s)
+        // If this test fails, it means the src/main/ml-modules/version.sjs doesn't match the version in gradle.properties, and should be updated.
+        String expectedVersion = System.getProperty("gds.version");
+
+        JsonPath postBody = getJson("testServiceDescriptor.json");
+
+        RestAssured
+            .given()
+                .contentType(ContentType.JSON)
+                .body(postBody.prettyPrint())
+            .when()
+                .log().uri()
+                .post(url)
+            .then()
+                .log().ifError()
+                .statusCode(200)
+                .log().ifValidationFails()
+                .body("$version", is(expectedVersion))
+        ;
+    }
 
     @Test
     public void testServiceDescriptor() {

src/test/java/WhereTest.java

@@ -321,5 +321,45 @@ public void testLike() {
                 .body("features.properties.domain", everyItem(containsString("journal")))
         ;
     }
+
+    @Test
+    public void testCombinedSearch() {
+        JsonPath postBody = getJson("testCombinedSearch.json");
+
+        RestAssured
+            .given()
+                .contentType(ContentType.JSON)
+                .body(postBody.prettyPrint())
+            .when()
+                .log().uri()
+                .post(url)
+            .then()
+                .log().ifError()
+                .statusCode(200)
+                .log().ifValidationFails()
+                .body(isValidFeatureCollection())
+                .body("features.size()", is(29))
+        ;
+    }
+
+    @Test
+    public void testCombinedSearchFalse() {
+        JsonPath postBody = getJson("testCombinedSearchFalse.json");
+
+        RestAssured
+            .given()
+                .contentType(ContentType.JSON)
+                .body(postBody.prettyPrint())
+            .when()
+                .log().uri()
+                .post(url)
+            .then()
+                .log().ifError()
+                .statusCode(200)
+                .log().ifValidationFails()
+                .body(isValidFeatureCollection())
+                .body("features.size()", is(0))
+        ;
+    }
 }
 

src/test/ml-modules/options/test.xml

@@ -0,0 +1,15 @@
+<options xmlns="http://marklogic.com/appservices/search">
+  <search-option>unfiltered</search-option>
+  <page-length>10</page-length>
+
+  <additional-query>
+    <cts:collection-query xmlns:cts="http://marklogic.com/cts">
+      <cts:uri>test-data</cts:uri>
+    </cts:collection-query>
+  </additional-query>
+
+  <constraint name="collection">
+    <collection facet="false"/>
+  </constraint>
+
+</options>

src/test/resources/WhereTest/testCombinedSearch.json

@@ -0,0 +1,19 @@
+{
+  "params" : {
+    "id" : "GDeltGKG",
+    "layer" : 0,
+    "method" : "query"
+  },
+  "query" : {
+    "optionsName": "test",
+    "where" : {
+      "search": {
+        "qtext": "nikkei",
+        "query": {
+          "true-query": ""
+        }
+      }
+    },
+    "returnCountsOnly": true
+  }
+}

src/test/resources/WhereTest/testCombinedSearchFalse.json

@@ -0,0 +1,19 @@
+{
+  "params" : {
+    "id" : "GDeltGKG",
+    "layer" : 0,
+    "method" : "query"
+  },
+  "query" : {
+    "optionsName": "test",
+    "where" : {
+      "search": {
+        "qtext": "nikkei",
+        "query": {
+          "false-query": ""
+        }
+      }
+    },
+    "returnCountsOnly": true
+  }
+}