-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
NGSTACK-834 page indexing implementation from fina + FieldMapper impl…
…ementation for elasticsearch
- Loading branch information
Katarina Miočić
committed
Apr 15, 2024
1 parent
9af5bf6
commit bee2cf8
Showing
31 changed files
with
1,491 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace Netgen\IbexaSearchExtra\Command; | ||
|
|
||
| use Ibexa\Contracts\Core\Persistence\Handler as PersistenceHandler; | ||
| use Ibexa\Contracts\Core\Repository\ContentService; | ||
| use Ibexa\Contracts\Core\Repository\Exceptions\InvalidArgumentException; | ||
| use Ibexa\Contracts\Core\Repository\Exceptions\NotFoundException; | ||
| use Ibexa\Contracts\Core\Repository\Exceptions\UnauthorizedException; | ||
| use Ibexa\Contracts\Core\Repository\Values\Content\Content; | ||
| use Ibexa\Contracts\Core\Repository\Values\Content\Query; | ||
| use Ibexa\Contracts\Core\Repository\Values\Content\Query\Criterion; | ||
| use Ibexa\Contracts\Core\Repository\Values\Filter\Filter; | ||
| use Ibexa\Contracts\Core\Search\Handler as SearchHandler; | ||
| use Netgen\IbexaSearchExtra\Exception\IndexPageUnavailableException; | ||
| use Symfony\Component\Console\Command\Command; | ||
| use Symfony\Component\Console\Helper\ProgressBar; | ||
| use Symfony\Component\Console\Input\InputInterface; | ||
| use Symfony\Component\Console\Input\InputOption; | ||
| use Symfony\Component\Console\Output\OutputInterface; | ||
| use Ibexa\Contracts\Core\Repository\Values\Content\ContentList; | ||
|
|
||
| use function count; | ||
| use function explode; | ||
|
|
||
| class IndexPageContentCommand extends Command | ||
| { | ||
| protected static $defaultName = 'netgen-search-extra:index-page-content'; | ||
|
|
||
| /** | ||
| * @param ContentService $contentService | ||
| * @param SearchHandler $searchHandler | ||
| * @param PersistenceHandler $persistenceHandler | ||
| * @param array<string> $allowedContentTypes | ||
| */ | ||
| public function __construct( | ||
| private readonly ContentService $contentService, | ||
| private readonly SearchHandler $searchHandler, | ||
| private readonly PersistenceHandler $persistenceHandler, | ||
| private readonly array $allowedContentTypes, | ||
| ) { | ||
| parent::__construct($this::$defaultName); | ||
| } | ||
|
|
||
| protected function configure(): void | ||
| { | ||
| $this | ||
| ->setDescription('Index content related through layouts') | ||
| ->addOption( | ||
| 'content-ids', | ||
| null, | ||
| InputOption::VALUE_REQUIRED, | ||
| 'Comma separated list of content id\'s of content to index.', | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * @throws NotFoundException | ||
| * @throws InvalidArgumentException | ||
| * @throws UnauthorizedException | ||
| */ | ||
| protected function execute(InputInterface $input, OutputInterface $output): int | ||
| { | ||
| $contentIds = $input->getOption('content-ids'); | ||
| if ($contentIds !== null) { | ||
| $contentIds = explode(',', $contentIds); | ||
|
|
||
| $totalCount = count($contentIds); | ||
| $output->writeln("Number of objects to index: {$totalCount}"); | ||
|
|
||
| $progressBar = new ProgressBar($output, $totalCount); | ||
| $progressBar->start(); | ||
| foreach ($contentIds as $contentId) { | ||
| $content = $this->contentService->loadContent((int) $contentId); | ||
| $this->indexContentWithLocations($content); | ||
| $progressBar->advance(); | ||
| } | ||
| } else { | ||
| $query = new Query(); | ||
| $offset = 0; | ||
| $limit = 50; | ||
| $query->query = new Criterion\ContentTypeIdentifier($this->allowedContentTypes); | ||
| $totalCount = $this->getTotalCount($query); | ||
| $progressBar = new ProgressBar($output, $totalCount); | ||
|
|
||
| if ($totalCount <= 0) { | ||
| $output->writeln('No content found to index, exiting.'); | ||
|
|
||
| return Command::SUCCESS; | ||
| } | ||
|
|
||
| $output->writeln('Found ' . $totalCount . ' content objects...'); | ||
| $output->writeln(''); | ||
|
|
||
| $progressBar->start($totalCount); | ||
|
|
||
| while ($offset < $totalCount) { | ||
| $chunk = $this->getChunk($query, $limit, $offset); | ||
|
|
||
| $this->processChunk($chunk, $output, $progressBar); | ||
|
|
||
| $offset += $limit; | ||
| } | ||
|
|
||
| $progressBar->finish(); | ||
|
|
||
| $output->writeln(''); | ||
| $output->writeln(''); | ||
| $output->writeln('Finished.'); | ||
| } | ||
|
|
||
| return Command::SUCCESS; | ||
| } | ||
|
|
||
| /** | ||
| * @throws InvalidArgumentException | ||
| */ | ||
| private function getTotalCount(Query $query): int | ||
| { | ||
| $filter = new Filter(); | ||
| $filter | ||
| ->withCriterion( | ||
| new Query\Criterion\ContentTypeIdentifier($this->allowedContentTypes) | ||
| ) | ||
| ->withLimit(0) | ||
| ->withOffset(0) | ||
| ; | ||
|
|
||
| return $this->contentService->find($filter)->getTotalCount() ?? 0; | ||
| } | ||
|
|
||
| /** | ||
| * @throws InvalidArgumentException | ||
| */ | ||
| private function getChunk(Query $query, int $limit, int $offset): ContentList | ||
| { | ||
| $filter = new Filter(); | ||
| $filter | ||
| ->withLimit($limit) | ||
| ->withOffset($offset) | ||
| ; | ||
| return $this->contentService->find($filter); | ||
| } | ||
|
|
||
| private function processChunk(ContentList $contentList, OutputInterface $output, ProgressBar $progressBar): void | ||
| { | ||
| foreach ($contentList->getIterator() as $content) { | ||
| try { | ||
| //$this->indexContentWithLocations($content); | ||
| $progressBar->advance(); | ||
| } catch (IndexPageUnavailableException $exception) { | ||
| $output->writeln($exception->getMessage()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private function indexContentWithLocations(Content $content): void | ||
| { | ||
| $this->searchHandler->indexContent( | ||
| $this->persistenceHandler->contentHandler()->load($content->id, $content->versionInfo->versionNo), | ||
| ); | ||
|
|
||
| $locations = $this->persistenceHandler->locationHandler()->loadLocationsByContent($content->id); | ||
| foreach ($locations as $location) { | ||
| $this->searchHandler->indexLocation($location); | ||
| } | ||
| } | ||
| } |
50 changes: 50 additions & 0 deletions
50
lib/Container/Compiler/AggregateElasticsearchContentFieldMapperMapperPass.php
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace Netgen\IbexaSearchExtra\Container\Compiler; | ||
|
|
||
| use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; | ||
| use Symfony\Component\DependencyInjection\ContainerBuilder; | ||
| use Symfony\Component\DependencyInjection\Definition; | ||
| use Symfony\Component\DependencyInjection\Reference; | ||
| use function array_keys; | ||
|
|
||
| /** | ||
| * This compiler pass will register Content translation subdocument mappers. | ||
| * | ||
| * @see \Netgen\IbexaSearchExtra\Core\Search\Solr\SubdocumentMapper\ContentTranslationSubdocumentMapper | ||
| * @see \Netgen\IbexaSearchExtra\Core\Search\Solr\SubdocumentMapper\ContentTranslationSubdocumentMapper\Aggregate | ||
| */ | ||
| final class AggregateElasticsearchContentFieldMapperMapperPass implements CompilerPassInterface | ||
| { | ||
| public function process(ContainerBuilder $container): void | ||
| { | ||
| $this->processVisitors($container, 'block_translation'); | ||
| $this->processVisitors($container, 'block'); | ||
| $this->processVisitors($container, 'content'); | ||
| $this->processVisitors($container, 'content_translation'); | ||
| $this->processVisitors($container, 'location'); | ||
| $this->processVisitors($container, 'location_translation'); | ||
| } | ||
|
|
||
| private function processVisitors(ContainerBuilder $container, string $name): void | ||
| { | ||
| if (!$container->hasDefinition(sprintf('netgen.ibexa_search_extra.elasticsearch.field_mapper.%s.aggregate', $name))) { | ||
| return; | ||
| } | ||
|
|
||
| $aggregateDefinition = $container->getDefinition( | ||
| sprintf('netgen.ibexa_search_extra.elasticsearch.field_mapper.%s.aggregate', $name), | ||
| ); | ||
|
|
||
| $this->registerMappers($aggregateDefinition, $container->findTaggedServiceIds(sprintf('netgen.ibexa_search_extra.elasticsearch.field_mapper.%s', $name))); | ||
| } | ||
|
|
||
| private function registerMappers(Definition $definition, array $mapperIds): void | ||
| { | ||
| foreach (array_keys($mapperIds) as $id) { | ||
| $definition->addMethodCall('addMapper', [new Reference($id)]); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.