Better reporting and surfacing of errors while batch indexing

wilr · Aug 3, 2021 · 71f21ad · 71f21ad
1 parent 564cfdb
commit 71f21ad
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 15 deletions.
diff --git a/src/Jobs/AlgoliaReindexAllJob.php b/src/Jobs/AlgoliaReindexAllJob.php
@@ -28,6 +28,11 @@ class AlgoliaReindexAllJob extends AbstractQueuedJob implements QueuedJob
      */
     private static $reindexing_default_filters = [];
 
+    /**
+     * @config
+     */
+    private static $use_batching = true;
+
     public function __construct($params = array())
     {
     }
@@ -96,18 +101,42 @@ public function process()
         }
 
         $task = new AlgoliaReindex();
+
         $batchSize = $task->config()->get('batch_size');
+        $batching = $this->config()->get('use_batching');
 
         foreach ($remainingChildren as $class => $ids) {
             $take = array_slice($ids, 0, $batchSize);
             $this->indexData[$class] = array_slice($ids, $batchSize);
 
-            $take = array_slice($ids, 0, $batchSize);
-
             if (!empty($take)) {
                 $this->currentStep += count($take);
-                $task->indexItems($class, '', DataObject::get($class)->filter('ID', $take), false);
-                $this->addMessage('Indexing '. $class . ' ['. implode(', ', $take) . ']');
+
+                if ($batching) {
+                    if ($task->indexItems($class, '', DataObject::get($class)->filter('ID', $take), false)) {
+                        $this->addMessage('Successfully indexing '. $class . ' ['. implode(', ', $take) . ']');
+                    } else {
+                        $this->addMessage('Error indexing '. $class . ' ['. implode(', ', $take) . ']');
+                    }
+                } else {
+                    $items = DataObject::get($class)->filter('ID', $take);
+
+                    foreach ($items as $item) {
+                        if ($task->indexItem($item)) {
+                            $this->addMessage('Successfully indexed '. $class . ' ['. $item->ID . ']');
+                        } else {
+                            $this->addMessage('Error indexing '. $class . ' ['. $item->ID . ']');
+                        }
+                    }
+                }
+
+                $errors = $task->getErrors();
+
+                if (!empty($errors)) {
+                    $this->addMessage(implode(', ', $errors));
+
+                    $task->clearErrors();
+                }
             } else {
                 unset($this->indexData[$class]);
             }

diff --git a/src/Tasks/AlgoliaReindex.php b/src/Tasks/AlgoliaReindex.php
@@ -28,6 +28,8 @@ class AlgoliaReindex extends BuildTask
 
     private static $batch_size = 20;
 
+    protected $errors = [];
+
     public function run($request)
     {
         Environment::increaseMemoryLimitTo();
@@ -113,19 +115,46 @@ public function getItems($targetClass, $filter = '')
         return $items;
     }
 
+    /**
+     * @param DataObject $obj
+     *
+     * @return bool
+     */
+    public function indexItem($obj = null): bool
+    {
+        if (!$obj) {
+            return false;
+        } elseif (!$obj->canIndexInAlgolia()) {
+            return false;
+        } else {
+            if (!$obj->AlgoliaUUID) {
+                $obj->assignAlgoliaUUID();
+            }
+
+            if ($obj->doImmediateIndexInAlgolia()) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
+
     /**
      * @param string $targetClass
      * @param string $filter
      * @param DataList? $items
      * @param bool $output;
+     *
+     * @return bool|string
      */
     public function indexItems($targetClass, $filter = '', $items = null, $output = true)
     {
         $algoliaService = Injector::inst()->get(AlgoliaService::class);
         $count = 0;
         $skipped = 0;
         $total = ($items) ? $items->count() : 0;
-        $batchSize = $this->config()->get('batch_size');
+        $batchSize = $this->config()->get('batch_size') ?? 25;
         $batchesTotal = ($total > 0) ? (ceil($total / $batchSize)) : 0;
         $indexer = Injector::inst()->get(AlgoliaIndexer::class);
 
@@ -144,7 +173,7 @@ public function indexItems($targetClass, $filter = '', $items = null, $output =
         $pos = 0;
 
         if ($total < 1) {
-            return;
+            return false;
         }
 
         $currentBatches = [];
@@ -209,14 +238,14 @@ public function indexItems($targetClass, $filter = '', $items = null, $output =
             }
         }
 
+        $summary = sprintf(
+            "Number of objects indexed: %s, Skipped %s",
+            $count,
+            $skipped
+        );
+
         if ($output) {
-            Debug::message(
-                sprintf(
-                    "Number of objects indexed: %s, Skipped %s",
-                    $count,
-                    $skipped
-                )
-            );
+            Debug::message($summary);
 
             Debug::message(
                 sprintf(
@@ -227,6 +256,8 @@ public function indexItems($targetClass, $filter = '', $items = null, $output =
                 )
             );
         }
+
+        return $summary;
     }
 
     /**
@@ -236,13 +267,20 @@ public function indexItems($targetClass, $filter = '', $items = null, $output =
      *
      * @return bool
      */
-    public function indexBatch($items)
+    public function indexBatch($items): bool
     {
         $indexes = Injector::inst()->create(AlgoliaService::class)->initIndexes($items[0]);
 
         try {
+
             foreach ($indexes as $index) {
-                $index->saveObjects($items);
+                $result = $index->saveObjects($items, [
+                    'autoGenerateObjectIDIfNotExist' => true
+                ]);
+
+                if (!$result->valid()) {
+                    return false;
+                }
             }
 
             return true;
@@ -253,7 +291,28 @@ public function indexBatch($items)
                 Debug::message($e->getMessage());
             }
 
+            $this->errors[] = $e->getMessage();
+
             return false;
         }
     }
+
+    /**
+     * @return string[]
+     */
+    public function getErrors()
+    {
+        return $this->errors;
+    }
+
+    /**
+     * @return $this
+     */
+    public function clearErrors()
+    {
+        $this->errors = [];
+
+        return $this;
+    }
+
 }