Lock search index while reindexing (#6934)

* Lock search index while reindexing

* Set delay 5 minutes between retries of search reindex queue jobs

* Update SearchJobService.php

---------

Co-authored-by: Lucas Bartholemy <luke-@users.noreply.github.com>

CHANGELOG-DEV.md

@@ -7,6 +7,7 @@ HumHub Changelog
 - Fix #5629: Better handling of legacy configuration file options
 - Fix #6931: Fix visibility of private spaces in the user notification settings
 - Enh #6757: Allow changing visibility of global content
+- Fix #5419: Lock search index while reindexing
 
 1.16.0-beta.1 (April 5, 2024)
 -----------------------------

protected/humhub/modules/content/jobs/SearchDeleteDocument.php

@@ -1,15 +1,19 @@
 <?php
+/**
+ * @link https://www.humhub.org/
+ * @copyright Copyright (c) HumHub GmbH & Co. KG
+ * @license https://www.humhub.com/licences
+ */
 
 namespace humhub\modules\content\jobs;
 
 use humhub\modules\content\models\Content;
-use humhub\modules\content\Module;
 use humhub\modules\content\services\ContentSearchService;
-use humhub\modules\queue\ActiveJob;
+use humhub\modules\content\services\SearchJobService;
 use humhub\modules\queue\interfaces\ExclusiveJobInterface;
-use Yii;
+use humhub\modules\queue\LongRunningActiveJob;
 
-class SearchDeleteDocument extends ActiveJob implements ExclusiveJobInterface
+class SearchDeleteDocument extends LongRunningActiveJob implements ExclusiveJobInterface
 {
     public $contentId;
 
@@ -26,10 +30,24 @@ class SearchDeleteDocument extends ActiveJob implements ExclusiveJobInterface
      */
     public function run()
     {
-        $content = Content::findOne(['id' => $this->contentId]);
-        if ($content) {
-            (new ContentSearchService($content))->delete(false);
-        }
+        return $this->getService()->run(function () {
+            $content = Content::findOne(['id' => $this->contentId]);
+            if ($content) {
+                (new ContentSearchService($content))->delete(false);
+            }
+        });
     }
 
+    /**
+     * @inheritdoc
+     */
+    public function canRetry($attempt, $error)
+    {
+        return $this->getService()->canRetry($attempt);
+    }
+
+    public function getService(): SearchJobService
+    {
+        return new SearchJobService();
+    }
 }

protected/humhub/modules/content/jobs/SearchRebuildIndex.php

@@ -10,6 +10,7 @@ namespace humhub\modules\content\jobs;
 
 use humhub\modules\content\models\Content;
 use humhub\modules\content\services\ContentSearchService;
+use humhub\modules\content\services\SearchJobService;
 use humhub\modules\queue\interfaces\ExclusiveJobInterface;
 use humhub\modules\queue\LongRunningActiveJob;
 
@@ -28,8 +29,23 @@ class SearchRebuildIndex extends LongRunningActiveJob implements ExclusiveJobInt
      */
     public function run()
     {
-        foreach (Content::find()->each() as $content) {
-            (new ContentSearchService($content))->update(false);
-        }
+        return $this->getService()->run(function () {
+            foreach (Content::find()->each() as $content) {
+                (new ContentSearchService($content))->update(false);
+            }
+        });
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function canRetry($attempt, $error)
+    {
+        return $this->getService()->canRetry($attempt);
+    }
+
+    public function getService(): SearchJobService
+    {
+        return new SearchJobService();
     }
 }

protected/humhub/modules/content/jobs/SearchUpdateDocument.php

@@ -1,15 +1,19 @@
 <?php
+/**
+ * @link https://www.humhub.org/
+ * @copyright Copyright (c) HumHub GmbH & Co. KG
+ * @license https://www.humhub.com/licences
+ */
 
 namespace humhub\modules\content\jobs;
 
 use humhub\modules\content\models\Content;
-use humhub\modules\content\Module;
 use humhub\modules\content\services\ContentSearchService;
-use humhub\modules\queue\ActiveJob;
+use humhub\modules\content\services\SearchJobService;
 use humhub\modules\queue\interfaces\ExclusiveJobInterface;
-use Yii;
+use humhub\modules\queue\LongRunningActiveJob;
 
-class SearchUpdateDocument extends ActiveJob implements ExclusiveJobInterface
+class SearchUpdateDocument extends LongRunningActiveJob implements ExclusiveJobInterface
 {
     public $contentId;
 
@@ -26,10 +30,24 @@ class SearchUpdateDocument extends ActiveJob implements ExclusiveJobInterface
      */
     public function run()
     {
-        $content = Content::findOne(['id' => $this->contentId]);
-        if ($content) {
-            (new ContentSearchService($content))->update(false);
-        }
+        return $this->getService()->run(function () {
+            $content = Content::findOne(['id' => $this->contentId]);
+            if ($content) {
+                (new ContentSearchService($content))->update(false);
+            }
+        });
     }
 
+    /**
+     * @inheritdoc
+     */
+    public function canRetry($attempt, $error)
+    {
+        return $this->getService()->canRetry($attempt);
+    }
+
+    public function getService(): SearchJobService
+    {
+        return new SearchJobService();
+    }
 }

protected/humhub/modules/content/services/SearchJobService.php

@@ -0,0 +1,43 @@
+<?php
+/**
+ * @link https://www.humhub.org/
+ * @copyright Copyright (c) HumHub GmbH & Co. KG
+ * @license https://www.humhub.com/licences
+ */
+
+namespace humhub\modules\content\services;
+
+use Yii;
+
+class SearchJobService
+{
+    public const MUTEX_ID = 'SearchQueueJob';
+    public int $retryAttemptNum = 10;
+    public int $retryDelayInSeconds = 5 * 60;
+
+    public function run(callable $callable): bool
+    {
+        if (!Yii::$app->mutex->acquire(self::MUTEX_ID, $this->retryDelayInSeconds)) {
+            return false;
+        }
+
+        try {
+            call_user_func($callable);
+            Yii::$app->mutex->release(self::MUTEX_ID);
+        } catch (\Exception $e) {
+            Yii::$app->mutex->release(self::MUTEX_ID);
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * @param $attempt Number of the current attempt
+     * @return bool
+     */
+    public function canRetry($attempt): bool
+    {
+        return $attempt < $this->retryAttemptNum;
+    }
+}