NGSTACK-836: descendant indexing

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
     tests:
-        name: ${{ matrix.php }} ${{ matrix.coverage }} ${{ matrix.engine }} ${{ matrix.solr_version }} ${{ matrix.regression }} ${{ matrix.core_setup }}
+        name: ${{ matrix.php }} ${{ matrix.coverage }} ${{ matrix.engine }} ${{ matrix.solr_version }} ${{ matrix.regression }} ${{ matrix.asynchronous }} ${{ matrix.core_setup }}
         runs-on: ubuntu-latest
         env:
             extensions: pdo_sqlite
@@ -53,6 +53,37 @@ jobs:
                       solr_cloud: 'yes'
                       core_setup: 'cloud'
 
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      solr_cores: 'collection1'
+                      core_setup: 'single'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      core_setup: 'dedicated'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      core_setup: 'shared'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      solr_cloud: 'yes'
+                      core_setup: 'cloud'
+                      asynchronous: 'asynchronous'
+
                     - php: '8.1'
                       coverage: 'integration'
                       engine: 'solr'
@@ -84,6 +115,41 @@ jobs:
                       core_setup: 'cloud'
                       regression: 'regression'
 
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      solr_cores: 'collection1'
+                      core_setup: 'single'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      core_setup: 'dedicated'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      core_setup: 'shared'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '7.7.3'
+                      solr_cloud: 'yes'
+                      core_setup: 'cloud'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+
                     - php: '8.1'
                       coverage: 'integration'
                       engine: 'solr'
@@ -111,6 +177,37 @@ jobs:
                       solr_cloud: 'yes'
                       core_setup: 'cloud'
 
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      solr_cores: 'collection1'
+                      core_setup: 'single'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      core_setup: 'dedicated'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      core_setup: 'shared'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-integration-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      solr_cloud: 'yes'
+                      core_setup: 'cloud'
+                      asynchronous: 'asynchronous'
+
                     - php: '8.1'
                       coverage: 'integration'
                       engine: 'solr'
@@ -142,6 +239,41 @@ jobs:
                       core_setup: 'cloud'
                       regression: 'regression'
 
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      solr_cores: 'collection1'
+                      core_setup: 'single'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      core_setup: 'dedicated'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      core_setup: 'shared'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+                    - php: '8.1'
+                      coverage: 'integration'
+                      engine: 'solr'
+                      config: 'phpunit-core-integration-legacy-solr-asynchronous.xml'
+                      solr_version: '8.11.2'
+                      solr_cloud: 'yes'
+                      core_setup: 'cloud'
+                      regression: 'regression'
+                      asynchronous: 'asynchronous'
+
         steps:
             - uses: actions/checkout@v2
             - uses: actions/setup-java@v1

bundle/DependencyInjection/Configuration.php

@@ -7,6 +7,7 @@
 use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
+use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
 
 class Configuration implements ConfigurationInterface
 {
@@ -25,6 +26,7 @@ public function getConfigTreeBuilder(): TreeBuilder
         $this->addIndexableFieldTypeSection($rootNode);
         $this->addSearchResultExtractorSection($rootNode);
         $this->addAsynchronousIndexingSection($rootNode);
+        $this->addHierarchicalIndexingSection($rootNode);
 
         return $treeBuilder;
     }
@@ -45,7 +47,7 @@ private function addIndexableFieldTypeSection(ArrayNodeDefinition $nodeDefinitio
                                     ->info("Maximum number of characters for the indexed short text ('value' string type field)")
                                     ->defaultValue(256)
                                 ->end()
-                            ->end()
+                            ?->end()
                         ->end()
                     ->end()
                 ->end()
@@ -60,7 +62,7 @@ private function addSearchResultExtractorSection(ArrayNodeDefinition $nodeDefini
                     ->info('Get search result objects by loading them from the persistence layer, instead of reconstructing them from the returned Solr data')
                     ->defaultTrue()
                 ->end()
-            ->end();
+            ?->end();
     }
 
     private function addAsynchronousIndexingSection(ArrayNodeDefinition $nodeDefinition): void
@@ -71,6 +73,110 @@ private function addAsynchronousIndexingSection(ArrayNodeDefinition $nodeDefinit
                     ->info('Use asynchronous mechanism to handle repository content indexing')
                     ->defaultFalse()
                 ->end()
-            ->end();
+            ?->end();
+    }
+
+    private function addHierarchicalIndexingSection(ArrayNodeDefinition $nodeDefinition): void
+    {
+        $childrenNodeDefinition = $nodeDefinition
+            ->children()
+                ->arrayNode('hierarchical_indexing')
+                    ->info('Hierarchical indexing configuration')
+                    ->addDefaultsIfNotSet()
+                    ->children()
+                        ->arrayNode('descendant_indexing')
+                            ->info('Descendant indexing configuration')
+                            ->addDefaultsIfNotSet()
+                            ->children()
+                                ->booleanNode('enabled')
+                                    ->info('Enable/disable descendant indexing')
+                                    ->defaultFalse()
+                                ->end()
+                                ?->arrayNode('map')
+                                    ->useAttributeAsKey('name')
+                                    ->normalizeKeys(false)
+                                    ->arrayPrototype()
+                                        ->children()
+                                            ->arrayNode('handlers')
+                                                ->info('List of indexing handlers to execute')
+                                                ->example([
+                                                    'handler_identifier_1',
+                                                    'handler_identifier_2',
+                                                ])
+                                                ->scalarPrototype()
+                                                    ->defaultValue([])
+                                                    ->validate()
+                                                        ->ifTrue(fn ($v) => !is_string($v))
+                                                        ->thenInvalid('Handler identifier must be a string.')
+                                                    ->end()
+                                                ->end()
+                                            ?->end()
+                                            ?->arrayNode('children')
+                                                ->useAttributeAsKey('name')
+                                                ->normalizeKeys(false)
+                                                ->arrayPrototype()
+        ;
+
+        $this->buildChildrenNode($childrenNodeDefinition);
+    }
+
+    private function evaluateChildren(&$child, $name): void
+    {
+        $builder = new TreeBuilder($name, 'array');
+        $root = $builder->getRootNode();
+
+        $this->buildChildrenNode($root);
+
+        $root->getNode(true)->finalize($child);
+    }
+
+    private function buildChildrenNode(ArrayNodeDefinition $node): void
+    {
+        $node
+            ->addDefaultsIfNotSet()
+            ->children()
+                ->booleanNode('indexed')
+                    ->info('Whether the node should be indexed')
+                    ->defaultTrue()
+                ->end()
+                ?->variableNode('children')
+                    ->defaultValue([])
+                    ->validate()
+                        ->ifTrue(fn ($v) => !is_array($v))
+                        ->thenInvalid('The children element must be an array.')
+                    ->end()
+                    ->validate()
+                        ->always(
+                            function ($children) {
+                                array_walk($children, $this->evaluateChildren(...));
+
+                                return $children;
+                            }
+                        )
+                    ->end()
+                ->end()
+            ?->end()
+            ->validate()
+                ->always(
+                    function ($children) {
+                        foreach (array_keys($children) as $key) {
+                            $allowedOptions = ['indexed', 'children'];
+
+                            if (!in_array($key, $allowedOptions, true)) {
+                                throw new InvalidConfigurationException(
+                                    sprintf(
+                                        'Unrecognized option "%s". Available options are "%s".',
+                                        $key,
+                                        implode('", "', $allowedOptions),
+                                    ),
+                                );
+                            }
+                        }
+
+                        return $children;
+                    }
+                )
+            ->end()
+        ;
     }
 }

bundle/DependencyInjection/NetgenIbexaSearchExtraExtension.php

@@ -88,6 +88,7 @@ private function processExtensionConfiguration(array $configs, ContainerBuilder
         $this->processIndexableFieldTypeConfiguration($configuration, $container);
         $this->processSearchResultExtractorConfiguration($configuration, $container);
         $this->processAsynchronousIndexingConfiguration($configuration, $container);
+        $this->processDescendantIndexingConfiguration($configuration, $container);
     }
 
     private function processSearchResultExtractorConfiguration(array $configuration, ContainerBuilder $container): void
@@ -117,4 +118,12 @@ private function processAsynchronousIndexingConfiguration(array $configuration,
             $configuration['use_asynchronous_indexing'],
         );
     }
+
+    private function processDescendantIndexingConfiguration(array $configuration, ContainerBuilder $container): void
+    {
+        $container->setParameter(
+            'netgen.ibexa_search_extra.descendant_indexing.configuration',
+            $configuration['hierarchical_indexing']['descendant_indexing'],
+        );
+    }
 }

bundle/NetgenIbexaSearchExtraBundle.php

@@ -5,6 +5,7 @@
 namespace Netgen\Bundle\IbexaSearchExtraBundle;
 
 use Netgen\IbexaSearchExtra\Container\Compiler;
+use Symfony\Component\DependencyInjection\Compiler\PassConfig;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\HttpKernel\Bundle\Bundle;
 
@@ -21,6 +22,7 @@ public function build(ContainerBuilder $container): void
         $container->addCompilerPass(new Compiler\AggregateFacetBuilderVisitorPass());
         $container->addCompilerPass(new Compiler\AggregateSubdocumentQueryCriterionVisitorPass());
         $container->addCompilerPass(new Compiler\AsynchronousIndexingPass());
+        $container->addCompilerPass(new Compiler\DescendantIndexingPass(), PassConfig::TYPE_BEFORE_OPTIMIZATION, 100);
         $container->addCompilerPass(new Compiler\FieldType\RichTextIndexablePass());
         $container->addCompilerPass(new Compiler\SearchResultExtractorPass());
         $container->addCompilerPass(new Compiler\RawFacetBuilderDomainVisitorPass());