From 7e3354d77a57088b2196bd3c3f7af03127e9a7a9 Mon Sep 17 00:00:00 2001 From: Toni Rudolf Date: Tue, 11 Feb 2020 10:50:48 +0100 Subject: [PATCH 1/3] [POC] Implemtend working config with multiple indexes per document class --- DependencyInjection/Compiler/MappingPass.php | 208 ++++++++++++------- Exception/DocumentIndexParserException.php | 16 ++ Mapping/IndexSettings.php | 36 ++-- 3 files changed, 170 insertions(+), 90 deletions(-) create mode 100644 Exception/DocumentIndexParserException.php diff --git a/DependencyInjection/Compiler/MappingPass.php b/DependencyInjection/Compiler/MappingPass.php index e0b8ee0f..5d3ebcb9 100644 --- a/DependencyInjection/Compiler/MappingPass.php +++ b/DependencyInjection/Compiler/MappingPass.php @@ -13,11 +13,13 @@ use ONGR\ElasticsearchBundle\Annotation\Index; use ONGR\ElasticsearchBundle\DependencyInjection\Configuration; +use ONGR\ElasticsearchBundle\Exception\DocumentIndexParserException; use ONGR\ElasticsearchBundle\Mapping\Converter; use ONGR\ElasticsearchBundle\Mapping\DocumentParser; use ONGR\ElasticsearchBundle\Mapping\IndexSettings; use ONGR\ElasticsearchBundle\Service\IndexService; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\Container; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; @@ -36,9 +38,65 @@ class MappingPass implements CompilerPassInterface public function process(ContainerBuilder $container) { $kernelDir = $container->getParameter('kernel.project_dir'); + $parser = $container->get(DocumentParser::class); + $indexClasses = []; + $indexSettingsArray = []; foreach ($container->getParameter(Configuration::ONGR_SOURCE_DIR) as $dir) { - $this->handleDirectoryMapping($container, $kernelDir . $dir); + foreach ($this->getNamespaces($kernelDir . $dir) as $namespace) { + $indexClasses[$namespace] = $namespace; + } + } + + $overwrittenClasses = []; + $indexOverrides = $container->getParameter(Configuration::ONGR_INDEXES_OVERRIDE); + + foreach ($indexOverrides as $name => $indexOverride) { + $class = isset($indexOverride['class']) ? $indexOverride['class'] : $name; + + if (!isset($indexClasses[$class])) { + throw new \RuntimeException( + sprintf( + 'Document `%s` defined in ongr_elasticsearch.indexes config could not been found', + $class, + ) + ); + } + + $indexSettings = $this->parseIndexSettingsFromClass($parser, $class); + + if ($class !== $name) { + $indexSettings->setIndexName('ongr.es.index.'.$name); + } + + if (isset($indexOverride['alias'])) { + $indexSettings->setAlias($indexOverride['alias']); + } + + if (isset($indexOverride['settings'])) { + $indexSettings->setIndexMetadata($indexOverride['settings']); + } + + if (isset($indexOverride['hosts'])) { + $indexSettings->setHosts($indexOverride['hosts']); + } + + if (isset($indexOverride['default'])) { + $indexSettings->setDefaultIndex($indexOverride['default']); + } + + $indexSettingsArray[$name] = $indexSettings; + $overwrittenClasses[$class] = $class; + } + + foreach (array_diff($indexClasses, $overwrittenClasses) as $indexClass) { + try { + $indexSettingsArray[$indexClass] = $this->parseIndexSettingsFromClass($parser, $indexClass); + } catch (DocumentIndexParserException $e) {} + } + + foreach($indexSettingsArray as $indexSettings) { + $this->createIndex($container, $indexSettings); } $container->setParameter(Configuration::ONGR_INDEXES, $this->indexes); @@ -48,88 +106,84 @@ public function process(ContainerBuilder $container) ); } - /** - * @param ContainerBuilder $container - * @param string $dir - * - * @throws \ReflectionException - */ - private function handleDirectoryMapping(ContainerBuilder $container, string $dir): void + private function parseIndexSettingsFromClass(DocumentParser $parser, string $className) : IndexSettings { - /** @var DocumentParser $parser */ - $parser = $container->get(DocumentParser::class); - $indexesOverride = $container->getParameter(Configuration::ONGR_INDEXES_OVERRIDE); - $converterDefinition = $container->getDefinition(Converter::class); + $class = new \ReflectionClass($className); - foreach ($this->getNamespaces($dir) as $namespace) { - $class = new \ReflectionClass($namespace); + /** @var Index $document */ + $document = $parser->getIndexAnnotation($class); - if (isset($indexesOverride[$namespace]['alias']) && $indexesOverride[$namespace]['alias']) { - $indexAlias = $indexesOverride[$namespace]['alias']; - } else { - $indexAlias = $parser->getIndexAliasName($class); - } + if ($document === null) { + throw new DocumentIndexParserException(); + } - /** @var Index $document */ - $document = $parser->getIndexAnnotation($class); - $indexMetadata = $parser->getIndexMetadata($class); - - if (!empty($indexMetadata)) { - $indexMetadata['settings'] = array_filter(array_merge_recursive( - $indexMetadata['settings'] ?? [], - [ - 'number_of_replicas' => $document->numberOfReplicas, - 'number_of_shards' => $document->numberOfShards, - ], - $indexesOverride[$namespace]['settings'] ?? [] - )); - - $indexSettings = new Definition( - IndexSettings::class, - [ - $namespace, - $indexAlias, - $indexAlias, - $indexMetadata, - $indexesOverride[$namespace]['hosts'] ?? $document->hosts, - $indexesOverride[$namespace]['default'] ?? $document->default, - ] - ); + $indexSettings = new IndexSettings( + $className, + $className, + $parser->getIndexAliasName($class), + $parser->getIndexMetadata($class), + $parser->getPropertyMetadata($class), + $document->hosts, + $parser->isDefaultIndex($class) + ); - $indexServiceDefinition = new Definition(IndexService::class, [ - $namespace, - $converterDefinition, - $container->getDefinition('event_dispatcher'), - $indexSettings, - $container->getParameter(Configuration::ONGR_PROFILER_CONFIG) - ? $container->getDefinition('ongr.esb.tracer') : null - ]); - $indexServiceDefinition->setPublic(true); - $converterDefinition->addMethodCall( - 'addClassMetadata', - [ - $namespace, - $parser->getPropertyMetadata($class) - ] - ); + $indexSettings->setIndexMetadata(['settings' => [ + 'number_of_replicas' => $document->numberOfReplicas, + 'number_of_shards' => $document->numberOfShards, + ]]); - $container->setDefinition($namespace, $indexServiceDefinition); - $this->indexes[$indexAlias] = $namespace; - $isCurrentIndexDefault = $parser->isDefaultIndex($class); - if ($this->defaultIndex && $isCurrentIndexDefault) { - throw new \RuntimeException( - sprintf( - 'Only one index can be set as default. We found 2 indexes as default ones `%s` and `%s`', - $this->defaultIndex, - $indexAlias - ) - ); - } - - if ($isCurrentIndexDefault) { - $this->defaultIndex = $indexAlias; - } - } + return $indexSettings; + } + + private function createIndex(Container $container, IndexSettings $indexSettings) { + $converterDefinition = $container->getDefinition(Converter::class); + + $indexSettingsDefinition = new Definition( + IndexSettings::class, + [ + $indexSettings->getNamespace(), + $indexSettings->getAlias(), + $indexSettings->getAlias(), + $indexSettings->getIndexMetadata(), + $indexSettings->getPropertyMetadata(), + $indexSettings->getHosts(), + $indexSettings->isDefaultIndex(), + ] + ); + + $indexServiceDefinition = new Definition(IndexService::class, [ + $indexSettings->getNamespace(), + $converterDefinition, + $container->getDefinition('event_dispatcher'), + $indexSettingsDefinition, + $container->getParameter(Configuration::ONGR_PROFILER_CONFIG) + ? $container->getDefinition('ongr.esb.tracer') : null + ]); + + $indexServiceDefinition->setPublic(true); + $converterDefinition->addMethodCall( + 'addClassMetadata', + [ + $indexSettings->getNamespace(), + $indexSettings->getPropertyMetadata() + ] + ); + + $container->setDefinition($indexSettings->getIndexName(), $indexServiceDefinition); + $this->indexes[$indexSettings->getAlias()] = $indexSettings->getIndexName(); + $isCurrentIndexDefault = $indexSettings->isDefaultIndex(); + if ($this->defaultIndex && $isCurrentIndexDefault) { + throw new \RuntimeException( + sprintf( + 'Only one index can be set as default. We found 2 indexes as default ones `%s` and `%s`', + $this->defaultIndex, + $indexSettings->getAlias() + ) + ); + } + + if ($isCurrentIndexDefault) { + $this->defaultIndex = $indexSettings->getAlias(); } } diff --git a/Exception/DocumentIndexParserException.php b/Exception/DocumentIndexParserException.php new file mode 100644 index 00000000..9f456c68 --- /dev/null +++ b/Exception/DocumentIndexParserException.php @@ -0,0 +1,16 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace ONGR\ElasticsearchBundle\Exception; + +class DocumentIndexParserException extends DocumentParserException +{ +} diff --git a/Mapping/IndexSettings.php b/Mapping/IndexSettings.php index 81296fc9..b37694c7 100644 --- a/Mapping/IndexSettings.php +++ b/Mapping/IndexSettings.php @@ -16,6 +16,7 @@ class IndexSettings private $indexName; private $alias; private $indexMetadata; + private $propertyMetadata; private $hosts; private $defaultIndex = false; @@ -24,6 +25,7 @@ public function __construct( string $indexName, string $alias, array $indexMetadata = [], + array $propertyMetadata = [], array $hosts = [], bool $defaultIndex = false ) { @@ -31,6 +33,7 @@ public function __construct( $this->indexName = $indexName; $this->alias = $alias; $this->indexMetadata = $indexMetadata; + $this->propertyMetadata = $propertyMetadata; $this->hosts = $hosts; $this->defaultIndex = $defaultIndex; } @@ -40,10 +43,9 @@ public function getNamespace() return $this->namespace; } - public function setNamespace($namespace): self + public function setNamespace($namespace): void { $this->namespace = $namespace; - return $this; } public function getIndexName() @@ -51,10 +53,9 @@ public function getIndexName() return $this->indexName; } - public function setIndexName($indexName): self + public function setIndexName($indexName): void { $this->indexName = $indexName; - return $this; } public function getAlias() @@ -62,10 +63,9 @@ public function getAlias() return $this->alias; } - public function setAlias($alias): self + public function setAlias($alias): void { $this->alias = $alias; - return $this; } public function getIndexMetadata() @@ -73,10 +73,22 @@ public function getIndexMetadata() return $this->indexMetadata; } - public function setIndexMetadata($indexMetadata): self + public function setIndexMetadata($indexMetadata): void { - $this->indexMetadata = $indexMetadata; - return $this; + $this->indexMetadata = array_filter(array_merge_recursive( + $this->indexMetadata, + $indexMetadata + )); + } + + public function getPropertyMetadata(): array + { + return $this->propertyMetadata; + } + + public function setPropertyMetadata(array $propertyMetadata): void + { + $this->propertyMetadata = $propertyMetadata; } public function getHosts() @@ -84,10 +96,9 @@ public function getHosts() return $this->hosts; } - public function setHosts($hosts): self + public function setHosts($hosts): void { $this->hosts = $hosts; - return $this; } public function isDefaultIndex(): bool @@ -95,9 +106,8 @@ public function isDefaultIndex(): bool return $this->defaultIndex; } - public function setDefaultIndex(bool $defaultIndex): self + public function setDefaultIndex(bool $defaultIndex): void { $this->defaultIndex = $defaultIndex; - return $this; } } From c56878a7cd4728ecd037d2681867011da080aed9 Mon Sep 17 00:00:00 2001 From: Toni Rudolf Date: Tue, 11 Feb 2020 11:21:49 +0100 Subject: [PATCH 2/3] Fixed code for PHP <7.4 --- DependencyInjection/Compiler/MappingPass.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DependencyInjection/Compiler/MappingPass.php b/DependencyInjection/Compiler/MappingPass.php index 5d3ebcb9..e75e7d14 100644 --- a/DependencyInjection/Compiler/MappingPass.php +++ b/DependencyInjection/Compiler/MappingPass.php @@ -58,7 +58,7 @@ public function process(ContainerBuilder $container) throw new \RuntimeException( sprintf( 'Document `%s` defined in ongr_elasticsearch.indexes config could not been found', - $class, + $class ) ); } From 02be3d3268915f2b2064de2d189546cea6a2ad3f Mon Sep 17 00:00:00 2001 From: Toni Rudolf Date: Tue, 11 Feb 2020 12:59:51 +0100 Subject: [PATCH 3/3] CS Fixes --- DependencyInjection/Compiler/MappingPass.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DependencyInjection/Compiler/MappingPass.php b/DependencyInjection/Compiler/MappingPass.php index e75e7d14..1e169a8f 100644 --- a/DependencyInjection/Compiler/MappingPass.php +++ b/DependencyInjection/Compiler/MappingPass.php @@ -92,10 +92,11 @@ public function process(ContainerBuilder $container) foreach (array_diff($indexClasses, $overwrittenClasses) as $indexClass) { try { $indexSettingsArray[$indexClass] = $this->parseIndexSettingsFromClass($parser, $indexClass); - } catch (DocumentIndexParserException $e) {} + } catch (DocumentIndexParserException $e) { + } } - foreach($indexSettingsArray as $indexSettings) { + foreach ($indexSettingsArray as $indexSettings) { $this->createIndex($container, $indexSettings); } @@ -135,7 +136,8 @@ private function parseIndexSettingsFromClass(DocumentParser $parser, string $cla return $indexSettings; } - private function createIndex(Container $container, IndexSettings $indexSettings) { + private function createIndex(Container $container, IndexSettings $indexSettings) + { $converterDefinition = $container->getDefinition(Converter::class); $indexSettingsDefinition = new Definition(