diff --git a/src/Jobs/AlgoliaReindexAllJob.php b/src/Jobs/AlgoliaReindexAllJob.php index bbe0103..c0ca026 100644 --- a/src/Jobs/AlgoliaReindexAllJob.php +++ b/src/Jobs/AlgoliaReindexAllJob.php @@ -28,6 +28,11 @@ class AlgoliaReindexAllJob extends AbstractQueuedJob implements QueuedJob */ private static $reindexing_default_filters = []; + /** + * @config + */ + private static $use_batching = true; + public function __construct($params = array()) { } @@ -96,18 +101,42 @@ public function process() } $task = new AlgoliaReindex(); + $batchSize = $task->config()->get('batch_size'); + $batching = $this->config()->get('use_batching'); foreach ($remainingChildren as $class => $ids) { $take = array_slice($ids, 0, $batchSize); $this->indexData[$class] = array_slice($ids, $batchSize); - $take = array_slice($ids, 0, $batchSize); - if (!empty($take)) { $this->currentStep += count($take); - $task->indexItems($class, '', DataObject::get($class)->filter('ID', $take), false); - $this->addMessage('Indexing '. $class . ' ['. implode(', ', $take) . ']'); + + if ($batching) { + if ($task->indexItems($class, '', DataObject::get($class)->filter('ID', $take), false)) { + $this->addMessage('Successfully indexing '. $class . ' ['. implode(', ', $take) . ']'); + } else { + $this->addMessage('Error indexing '. $class . ' ['. implode(', ', $take) . ']'); + } + } else { + $items = DataObject::get($class)->filter('ID', $take); + + foreach ($items as $item) { + if ($task->indexItem($item)) { + $this->addMessage('Successfully indexed '. $class . ' ['. $item->ID . ']'); + } else { + $this->addMessage('Error indexing '. $class . ' ['. $item->ID . ']'); + } + } + } + + $errors = $task->getErrors(); + + if (!empty($errors)) { + $this->addMessage(implode(', ', $errors)); + + $task->clearErrors(); + } } else { unset($this->indexData[$class]); } diff --git a/src/Tasks/AlgoliaReindex.php b/src/Tasks/AlgoliaReindex.php index 24fede5..d7f9e73 100644 --- a/src/Tasks/AlgoliaReindex.php +++ b/src/Tasks/AlgoliaReindex.php @@ -28,6 +28,8 @@ class AlgoliaReindex extends BuildTask private static $batch_size = 20; + protected $errors = []; + public function run($request) { Environment::increaseMemoryLimitTo(); @@ -113,11 +115,38 @@ public function getItems($targetClass, $filter = '') return $items; } + /** + * @param DataObject $obj + * + * @return bool + */ + public function indexItem($obj = null): bool + { + if (!$obj) { + return false; + } elseif (!$obj->canIndexInAlgolia()) { + return false; + } else { + if (!$obj->AlgoliaUUID) { + $obj->assignAlgoliaUUID(); + } + + if ($obj->doImmediateIndexInAlgolia()) { + return true; + } else { + return false; + } + } + } + + /** * @param string $targetClass * @param string $filter * @param DataList? $items * @param bool $output; + * + * @return bool|string */ public function indexItems($targetClass, $filter = '', $items = null, $output = true) { @@ -125,7 +154,7 @@ public function indexItems($targetClass, $filter = '', $items = null, $output = $count = 0; $skipped = 0; $total = ($items) ? $items->count() : 0; - $batchSize = $this->config()->get('batch_size'); + $batchSize = $this->config()->get('batch_size') ?? 25; $batchesTotal = ($total > 0) ? (ceil($total / $batchSize)) : 0; $indexer = Injector::inst()->get(AlgoliaIndexer::class); @@ -144,7 +173,7 @@ public function indexItems($targetClass, $filter = '', $items = null, $output = $pos = 0; if ($total < 1) { - return; + return false; } $currentBatches = []; @@ -209,14 +238,14 @@ public function indexItems($targetClass, $filter = '', $items = null, $output = } } + $summary = sprintf( + "Number of objects indexed: %s, Skipped %s", + $count, + $skipped + ); + if ($output) { - Debug::message( - sprintf( - "Number of objects indexed: %s, Skipped %s", - $count, - $skipped - ) - ); + Debug::message($summary); Debug::message( sprintf( @@ -227,6 +256,8 @@ public function indexItems($targetClass, $filter = '', $items = null, $output = ) ); } + + return $summary; } /** @@ -236,13 +267,20 @@ public function indexItems($targetClass, $filter = '', $items = null, $output = * * @return bool */ - public function indexBatch($items) + public function indexBatch($items): bool { $indexes = Injector::inst()->create(AlgoliaService::class)->initIndexes($items[0]); try { + foreach ($indexes as $index) { - $index->saveObjects($items); + $result = $index->saveObjects($items, [ + 'autoGenerateObjectIDIfNotExist' => true + ]); + + if (!$result->valid()) { + return false; + } } return true; @@ -253,7 +291,28 @@ public function indexBatch($items) Debug::message($e->getMessage()); } + $this->errors[] = $e->getMessage(); + return false; } } + + /** + * @return string[] + */ + public function getErrors() + { + return $this->errors; + } + + /** + * @return $this + */ + public function clearErrors() + { + $this->errors = []; + + return $this; + } + }