Feature/static filter for the TREE specification

.eslintrc.js

@@ -75,6 +75,7 @@ module.exports = {
     'unicorn/consistent-destructuring': 'off',
     'unicorn/no-array-callback-reference': 'off',
     'unicorn/no-new-array': 'off',
+    'unicorn/prefer-at': 'off', // not compatible with node v14
 
     // TS
     '@typescript-eslint/lines-between-class-members': ['error', { exceptAfterSingleLine: true }],

engines/config-query-sparql-link-traversal/config/extract-links/actors/tree-not-guided.json

@@ -0,0 +1,17 @@
+{
+    "@context": [
+      "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/runner/^2.0.0/components/context.jsonld",
+
+      "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-extract-links-tree/^0.0.0/components/context.jsonld"  
+    ],
+    "@id": "urn:comunica:default:Runner",
+    "@type": "Runner",
+    "actors": [
+      {
+        "@id": "urn:comunica:default:extract-links/actors#extract-links-tree",
+        "@type": "ActorExtractLinksTree",
+        "reachabilityCriterionUseSPARQLFilter": false
+      }
+    ]
+  }
+

engines/config-query-sparql-link-traversal/config/extract-links/actors/tree.json

@@ -9,7 +9,8 @@
     "actors": [
       {
         "@id": "urn:comunica:default:extract-links/actors#extract-links-tree",
-        "@type": "ActorExtractLinksTree"
+        "@type": "ActorExtractLinksTree",
+        "reachabilityCriterionUseSPARQLFilter": true
       }
     ]
   }

packages/actor-extract-links-extract-tree/README.md

@@ -2,7 +2,13 @@
 
 [![npm version](https://badge.fury.io/js/%40comunica%2Factor-extract-links-tree.svg)](https://www.npmjs.com/package/@comunica/actor-extract-links-tree)
 
-A comunica [Extract Links](https://github.com/comunica/comunica-feature-link-traversal/tree/master/packages/bus-extract-links) [TREE](https://treecg.github.io/specification/) Actor.
+A comunica [Extract Links Actor](https://github.com/comunica/comunica-feature-link-traversal/tree/master/packages/bus-extract-links) for the [TREE](https://treecg.github.io/specification/).
+
+The [Guided Linked Traversal Query Processing](https://arxiv.org/abs/2005.02239)
+option that can be enabled using the `reachabilityCriterionUseSPARQLFilte` flag. The traversal algorithm will consider the solvability of the query filter expression
+combined with the [`tree:relation`](https://treecg.github.io/specification/#Relation) of each data source encountered. 
+A more thorough explanation is available in the poster article
+["How TREE hypermedia can speed up Link Traversal-based Query Processing for SPARQL queries with filters"](https://constraintautomaton.github.io/How-TREE-hypermedia-can-speed-up-Link-Traversal-based-Query-Processing-queries/) 
 
 This module is part of the [Comunica framework](https://github.com/comunica/comunica),
 and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/).
@@ -22,13 +28,15 @@ After installing, this package can be added to your engine's configuration as fo
 {
   "@context": [
     ...
-    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-extract-links-tree/^2.0.0/components/context.jsonld"  
+    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-extract-links-tree/^0.0.1/components/context.jsonld"  
   ],
   "actors": [
     ...
     {
       "@id": "urn:comunica:default:extract-links/actors#extract-links-tree",
-      "@type": "ActorExtractLinksTree"
+      "@type": "ActorExtractLinksTree",
+      "reachabilityCriterionUseSPARQLFilter": true
+
     }
   ]
 }

packages/actor-extract-links-extract-tree/lib/ActorExtractLinksTree.ts

@@ -1,84 +1,149 @@
 import type {
   IActionExtractLinks,
-  IActorExtractLinksOutput, IActorExtractLinksArgs,
+  IActorExtractLinksOutput,
 } from '@comunica/bus-extract-links';
 import { ActorExtractLinks } from '@comunica/bus-extract-links';
-import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links';
-import type { IActorTest } from '@comunica/core';
-import { DataFactory } from 'rdf-data-factory';
-import type * as RDF from 'rdf-js';
 
-const DF = new DataFactory<RDF.BaseQuad>();
+import type { IActorArgs, IActorTest } from '@comunica/core';
+import type { IActionContext } from '@comunica/types';
+import type * as RDF from 'rdf-js';
+import { termToString } from 'rdf-string';
+import { FilterNode } from './FilterNode';
+import { TreeNodes } from './TreeMetadata';
+import type { ITreeRelationRaw, ITreeRelation, ITreeNode } from './TreeMetadata';
+import { buildRelationElement, materializeTreeRelation, addRelationDescription } from './treeMetadataExtraction';
 
 /**
  * A comunica Extract Links Tree Extract Links Actor.
  */
 export class ActorExtractLinksTree extends ActorExtractLinks {
-  public static readonly aNodeType = DF.namedNode('https://w3id.org/tree#node');
-  public static readonly aRelation = DF.namedNode('https://w3id.org/tree#relation');
-  private static readonly rdfTypeNode = DF.namedNode('http://www.w3.org/1999/02/22-rdf-syntax-ns#type');
-
-  public constructor(args: IActorExtractLinksArgs) {
+  private readonly reachabilityCriterionUseSPARQLFilter: boolean = true;
+  public constructor(args: IActorExtractLinksTreeArgs) {
     super(args);
+    this.reachabilityCriterionUseSPARQLFilter = args.reachabilityCriterionUseSPARQLFilter === undefined ?
+      true :
+      args.reachabilityCriterionUseSPARQLFilter;
   }
 
   public async test(action: IActionExtractLinks): Promise<IActorTest> {
     return true;
   }
 
+  public isUsingReachabilitySPARQLFilter(): boolean {
+    return this.reachabilityCriterionUseSPARQLFilter;
+  }
+
   public async run(action: IActionExtractLinks): Promise<IActorExtractLinksOutput> {
     return new Promise((resolve, reject) => {
       const metadata = action.metadata;
-      const currentNodeUrl = action.url;
-      const pageRelationNodes: Set<string> = new Set();
+      const currentPageUrl = action.url;
+      // Identifiers of the relationships defined by the TREE document, represented as stringified RDF terms.
+      const relationIdentifiers: Set<string> = new Set();
+      // Maps relationship identifiers to their description.
+      // At this point, there's no guarantee yet that these relationships are linked to the current TREE document.
+      const relationDescriptions: Map<string, ITreeRelationRaw> = new Map();
+      const relations: ITreeRelation[] = [];
+      // An array of pairs of relationship identifiers and next page link to another TREE document,
+      // represented as stringified RDF terms.
       const nodeLinks: [string, string][] = [];
-      const links: ILink[] = [];
 
       // Forward errors
       metadata.on('error', reject);
 
-      // Invoke callback on each metadata quad
+      // Collect information about relationships spread over quads, so that we can accumulate them afterwards.
       metadata.on('data', (quad: RDF.Quad) =>
-        this.getTreeQuadsRawRelations(quad,
-          currentNodeUrl,
-          pageRelationNodes,
-          nodeLinks));
+        this.interpretQuad(quad,
+          currentPageUrl,
+          relationIdentifiers,
+          nodeLinks,
+          relationDescriptions));
 
-      // Resolve to discovered links
-      metadata.on('end', () => {
-        // Validate if the node forward have the current node as implicit subject
-        for (const [ nodeValue, link ] of nodeLinks) {
-          if (pageRelationNodes.has(nodeValue)) {
-            links.push({ url: link });
+      // Accumulate collected relationship information.
+      metadata.on('end', async() => {
+        // Validate if the potential relation node are linked with the current page
+        // and add the relation description if it is connected.
+        for (const [ identifier, link ] of nodeLinks) {
+          // Check if the identifier is the object of a relation of the current page
+          if (relationIdentifiers.has(identifier)) {
+            const relationDescription = relationDescriptions.get(identifier);
+            // Add the relation to the relation array
+            relations.push(materializeTreeRelation(relationDescription || {}, link));
           }
         }
-        resolve({ links });
+
+        // Create a ITreeNode object
+        const node: ITreeNode = { relation: relations, identifier: currentPageUrl };
+        let acceptedRelation = relations;
+        if (this.reachabilityCriterionUseSPARQLFilter) {
+          // Filter the relation based on the query
+          const filters = await this.applyFilter(node, action.context);
+          acceptedRelation = this.handleFilter(filters, acceptedRelation);
+        }
+        resolve({ links: acceptedRelation.map(el => ({ url: el.node })) });
       });
     });
   }
 
+  /**
+   * @param {ITreeNode} node - TREE metadata
+   * @param {IActionContext} context - context of the action; containing the query
+   * @returns {Promise<Map<string, boolean>>} a map containing the filter
+   */
+  public async applyFilter(node: ITreeNode, context: IActionContext): Promise<Map<string, boolean>> {
+    return await new FilterNode().run(node, context);
+  }
+
+  /**
+   * @param { Map<string, boolean>} filters
+   * @param {ITreeRelation[]} acceptedRelation - the current accepted relation
+   * @returns {ITreeRelation[]} the relation when the nodes has been filtered
+   */
+  private handleFilter(filters: Map<string, boolean>, acceptedRelation: ITreeRelation[]): ITreeRelation[] {
+    return filters.size > 0 ?
+      acceptedRelation.filter(relation => filters?.get(relation.node)) :
+      acceptedRelation;
+  }
+
   /**
    * A helper function to find all the relations of a TREE document and the possible next nodes to visit.
    * The next nodes are not guaranteed to have as subject the URL of the current page,
    * so filtering is necessary afterward.
-   * @param quad the current quad.
-   * @param url url of the page
-   * @param pageRelationNodes the url of the relation node of the page that have as subject the URL of the page
-   * @param nodeLinks the url of the next potential page that has to be visited,
-   *  regardless if the implicit subject is the node of the page
+   * @param {RDF.Quad} quad - The current quad.
+   * @param {string} currentPageUrl - The url of the page.
+   * @param {Set<string>} relationIdentifiers - Identifiers of the relationships defined by the TREE document,
+   *                                            represented as stringified RDF terms.
+   * @param {[string, string][]} nodeLinks - An array of pairs of relationship identifiers and next page link to another
+   *                                         TREE document, represented as stringified RDF terms.
+   * @param {Map<string, ITreeRelationRaw>} relationDescriptions - Maps relationship identifiers to their description.
    */
-  private getTreeQuadsRawRelations(
+  private interpretQuad(
     quad: RDF.Quad,
-    url: string,
-    pageRelationNodes: Set<string>,
+    currentPageUrl: string,
+    relationIdentifiers: Set<string>,
     nodeLinks: [string, string][],
+    relationDescriptions: Map<string, ITreeRelationRaw>,
   ): void {
     // If it's a relation of the current node
-    if (quad.subject.value === url && quad.predicate.equals(ActorExtractLinksTree.aRelation)) {
-      pageRelationNodes.add(quad.object.value);
+    if (quad.subject.value === currentPageUrl && quad.predicate.value === TreeNodes.Relation) {
+      relationIdentifiers.add(termToString(quad.object));
       // If it's a node forward
-    } else if (quad.predicate.equals(ActorExtractLinksTree.aNodeType)) {
-      nodeLinks.push([ quad.subject.value, quad.object.value ]);
+    } else if (quad.predicate.value === TreeNodes.Node) {
+      nodeLinks.push([ termToString(quad.subject), quad.object.value ]);
+    }
+    const descriptionElement = buildRelationElement(quad);
+    if (descriptionElement) {
+      const [ value, key ] = descriptionElement;
+      addRelationDescription(relationDescriptions, quad, value, key);
     }
   }
 }
+
+export interface IActorExtractLinksTreeArgs
+  extends IActorArgs<IActionExtractLinks, IActorTest, IActorExtractLinksOutput> {
+  /**
+   * If true (default), then we use a reachability criterion that prune links that don't respect the
+   * SPARQL filter
+   * @default {true}
+   */
+  reachabilityCriterionUseSPARQLFilter: boolean;
+}

packages/actor-extract-links-extract-tree/lib/FilterNode.ts

@@ -0,0 +1,139 @@
+import { BindingsFactory } from '@comunica/bindings-factory';
+import { KeysInitQuery } from '@comunica/context-entries';
+import type { IActionContext } from '@comunica/types';
+import { Algebra, Factory as AlgebraFactory, Util } from 'sparqlalgebrajs';
+import { isBooleanExpressionTreeRelationFilterSolvable } from './solver';
+import type { Variable } from './solverInterfaces';
+import type { ITreeRelation, ITreeNode } from './TreeMetadata';
+
+const AF = new AlgebraFactory();
+const BF = new BindingsFactory();
+
+/**
+ * A class to apply [SPAQL filters](https://www.w3.org/TR/sparql11-query/#evaluation)
+ * to the [TREE specification](https://treecg.github.io/specification/).
+ * It use [sparqlee](https://github.com/comunica/sparqlee) to evaluate the filter where
+ * the binding are remplace by the [value of TREE relation](https://treecg.github.io/specification/#traversing).
+ */
+export class FilterNode {
+  /**
+   * Return the filter expression if the TREE node has relations
+   * @param {ITreeNode} node - The current TREE node
+   * @param {IActionContext} context - The context
+   * @returns {Algebra.Expression | undefined} The filter expression or undefined if the TREE node has no relations
+   */
+  public getFilterExpressionIfTreeNodeHasConstraint(node: ITreeNode,
+    context: IActionContext): Algebra.Expression | undefined {
+    if (!node.relation) {
+      return undefined;
+    }
+
+    if (node.relation.length === 0) {
+      return undefined;
+    }
+
+    const query: Algebra.Operation = context.get(KeysInitQuery.query)!;
+    const filterExpression = FilterNode.findNode(query, Algebra.types.FILTER);
+    if (!filterExpression) {
+      return undefined;
+    }
+
+    return filterExpression.expression;
+  }
+
+  /**
+   * Analyze if the tree:relation(s) of a tree:Node should be followed and return a map
+   * where if the value of the key representing the URL to follow is true than the link must be followed
+   * if it is false then it should be ignored.
+   * @param {ITreeNode} node - The current TREE node
+   * @param {IActionContext} context - The context
+   * @returns {Promise<Map<string, boolean>>} A map of the indicating if a tree:relation should be follow
+   */
+  public async run(node: ITreeNode, context: IActionContext): Promise<Map<string, boolean>> {
+    const filterMap: Map<string, boolean> = new Map();
+
+    const filterOperation: Algebra.Expression | undefined =
+      this.getFilterExpressionIfTreeNodeHasConstraint(node, context);
+
+    if (!filterOperation) {
+      return new Map();
+    }
+
+    // Extract the bgp of the query.
+    const queryBody: Algebra.Operation = context.get(KeysInitQuery.query)!;
+
+    // Capture the relation from the function argument.
+    const relations: ITreeRelation[] = node.relation!;
+
+    for (const relation of relations) {
+      // Accept the relation if the relation does't specify a condition.
+      if (!relation.path || !relation.value) {
+        filterMap.set(relation.node, true);
+        continue;
+      }
+      // Find the quad from the bgp that are related to the TREE relation.
+      const variables = FilterNode.findRelevantVariableFromBgp(queryBody, relation.path);
+
+      // Accept the relation if no variable are linked with the relation.
+      if (variables.length === 0) {
+        filterMap.set(relation.node, true);
+        continue;
+      }
+      let filtered = false;
+      // For all the variable check if one is has a possible solutions.
+      for (const variable of variables) {
+        filtered = filtered || isBooleanExpressionTreeRelationFilterSolvable(
+          { relation, filterExpression: filterOperation, variable },
+        );
+      }
+
+      filterMap.set(relation.node, filtered);
+    }
+    return filterMap;
+  }
+
+  /**
+   * Find the variables from the BGP that match the predicate defined by the TREE:path from a TREE relation.
+   *  The subject can be anyting.
+   * @param {Algebra.Operation} queryBody - the body of the query
+   * @param {string} path - TREE path
+   * @returns {Variable[]} the variables of the Quad objects that contain the TREE path as predicate
+   */
+  private static findRelevantVariableFromBgp(queryBody: Algebra.Operation, path: string): Variable[] {
+    const resp: Variable[] = [];
+    const addVariable = (quad: any): boolean => {
+      if (quad.predicate.value === path && quad.object.termType === 'Variable') {
+        resp.push(quad.object.value);
+      }
+      return true;
+    };
+
+    Util.recurseOperation(queryBody, {
+      [Algebra.types.PATH]: addVariable,
+      [Algebra.types.PATTERN]: addVariable,
+
+    });
+    return resp;
+  }
+
+  /**
+   * Find the first node of type `nodeType`, if it doesn't exist
+   * it return undefined.
+   * @param {Algebra.Operation} query - the original query
+   * @param {string} nodeType - the type of node requested
+   * @returns {any}
+   */
+  private static findNode(query: Algebra.Operation, nodeType: string): any {
+    let currentNode = query;
+    do {
+      if (currentNode.type === nodeType) {
+        return currentNode;
+      }
+      if ('input' in currentNode) {
+        currentNode = currentNode.input;
+      }
+    } while ('input' in currentNode);
+    return undefined;
+  }
+}
+