Do not delete and recreate index on class definitions save (#135)

* add command to reindex class definitions with deployment * Apply php-cs-fixer changes * apply feedback from review * fix: qodana * Apply php-cs-fixer changes * first adaptations * use native reindex * use native reindex * fix qodana errors * Apply php-cs-fixer changes * fix errors * fix errors * fix errors * reindex command should not queue items * Apply php-cs-fixer changes * add different class definition for mapping tests * Apply php-cs-fixer changes --------- Co-authored-by: lukmzig <lukmzig@users.noreply.github.com>
pimcore · Apr 16, 2024 · 2ebe0eb · 2ebe0eb
1 parent 83aec8e
commit 2ebe0eb
Show file tree

Hide file tree

Showing 21 changed files with 491 additions and 69 deletions.
diff --git a/config/pimcore/messenger.yaml b/config/pimcore/messenger.yaml
@@ -10,6 +10,7 @@ framework:
             Pimcore\Bundle\GenericDataIndexBundle\Message\IndexUpdateQueueMessage: pimcore_generic_data_index_queue
             Pimcore\Bundle\GenericDataIndexBundle\Message\DispatchQueueMessagesMessage: pimcore_generic_data_index_queue
             Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateLanguageSettingsMessage: pimcore_generic_data_index_queue
+            Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateClassMappingMessage: pimcore_generic_data_index_queue
         buses:
             messenger.bus.pimcore-generic-data-index:
                 middleware:

diff --git a/config/services/search/index.yaml b/config/services/search/index.yaml
@@ -10,6 +10,9 @@ services:
     Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateServiceInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateService
 
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\ReindexServiceInterface:
+        class: Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\ReindexService
+
     Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueueServiceInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueueService
 

diff --git a/doc/02_Configuration/03_Index_Management.md b/doc/02_Configuration/03_Index_Management.md
@@ -80,6 +80,17 @@ Do this with the index update command and pass -r option.
 bin/console generic-data-index:update:index -r
 ```
 
+### Updating Index Structure for Data Indices
+
+Index mapping is updated automatically e.g. when adding system languages or new fields to the class definition. 
+Sometimes it might be necessary to update the index structure manually.
+
+Do this with the reindex command.
+```
+# updates index mapping with native reindexing
+bin/console generic-data-index:reindex
+```
+
 ### Handling Failed Messages
 
 By default, the messenger will retry failed messages 3 times and then send them into the failed queue `pimcore_generic_data_index_failed`.

diff --git a/src/Command/Update/DeploymentReindex.php → src/Command/DeploymentReindexCommand.php b/src/Command/Update/DeploymentReindex.php → src/Command/DeploymentReindexCommand.php
@@ -11,13 +11,13 @@
  *  @license    http://www.pimcore.org/license     PCL
  */
 
-namespace Pimcore\Bundle\GenericDataIndexBundle\Command\Update;
+namespace Pimcore\Bundle\GenericDataIndexBundle\Command;
 
 use Exception;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\CommandAlreadyRunningException;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\EnqueueServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\IndexHandler\DataObjectIndexHandler;
-use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\ReindexServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SettingsStoreServiceInterface;
 use Pimcore\Console\AbstractCommand;
 use Pimcore\Model\DataObject\ClassDefinition;
@@ -28,14 +28,14 @@
 /**
  * @internal
  */
-final class DeploymentReindex extends AbstractCommand
+final class DeploymentReindexCommand extends AbstractCommand
 {
     use LockableTrait;
 
     public function __construct(
         private readonly DataObjectIndexHandler $indexHandler,
         private readonly EnqueueServiceInterface $enqueueService,
-        private readonly IndexUpdateServiceInterface $indexUpdateService,
+        private readonly ReindexServiceInterface $reindexService,
         private readonly SettingsStoreServiceInterface $settingsStoreService,
         string $name = null
     ) {
@@ -80,8 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                     $updatedIds[] = $classDefinitionId;
 
                     $this
-                        ->indexUpdateService
-                        ->updateClassDefinition($classDefinition);
+                        ->reindexService
+                        ->reindexClassDefinition($classDefinition);
                 }
             }
 

diff --git a/src/Command/ReindexItemsCommand.php b/src/Command/ReindexItemsCommand.php
@@ -0,0 +1,69 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Pimcore
+ *
+ * This source file is available under following license:
+ * - Pimcore Commercial License (PCL)
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\Command;
+
+use Exception;
+use Pimcore\Bundle\GenericDataIndexBundle\Exception\CommandAlreadyRunningException;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\ReindexServiceInterface;
+use Pimcore\Console\AbstractCommand;
+use Symfony\Component\Console\Command\LockableTrait;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * @internal
+ */
+final class ReindexItemsCommand extends AbstractCommand
+{
+    use LockableTrait;
+
+    public function __construct(
+        private readonly ReindexServiceInterface $reindexService,
+        string $name = null
+    ) {
+        parent::__construct($name);
+    }
+
+    protected function configure(): void
+    {
+        $this
+            ->setName('generic-data-index:reindex')
+            ->setDescription(
+                'Triggers native reindexing of existing indices.'
+            );
+    }
+
+    /**
+     * @throws CommandAlreadyRunningException
+     *
+     */
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        if (!$this->lock()) {
+            throw new CommandAlreadyRunningException(
+                'The command is already running in another process.'
+            );
+        }
+
+        try {
+            $this->reindexService->reindexAllIndices();
+        } catch (Exception $e) {
+            $output->writeln('<error>' . $e->getMessage() . '</error>');
+        } finally {
+            $this->release();
+        }
+
+        return self::SUCCESS;
+    }
+}
diff --git a/src/EventSubscriber/AdminSettingsSubscriber.php b/src/EventSubscriber/AdminSettingsSubscriber.php
@@ -16,15 +16,19 @@
 use Exception;
 use Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateLanguageSettingsMessage;
 use Pimcore\Event\SystemEvents;
+use Pimcore\Helper\StopMessengerWorkersTrait;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 use Symfony\Component\EventDispatcher\GenericEvent;
 use Symfony\Component\Messenger\MessageBusInterface;
+use Symfony\Component\Messenger\Stamp\DelayStamp;
 
 /**
  * @internal
  */
 final readonly class AdminSettingsSubscriber implements EventSubscriberInterface
 {
+    use StopMessengerWorkersTrait;
+
     public function __construct(
         private MessageBusInterface $messageBus
     ) {
@@ -43,12 +47,14 @@ public static function getSubscribedEvents(): array
     public function updateSearchIndex(GenericEvent $event): void
     {
         $arguments = $event->getArguments();
+        $this->stopMessengerWorkers();
 
         $this->messageBus->dispatch(
             new UpdateLanguageSettingsMessage(
                 currentLanguages: $arguments['existingValues']['general']['valid_languages'],
                 validLanguages: explode(',', $arguments['values']['general.validLanguages']),
-            )
+            ),
+            [new DelayStamp(2000)]
         );
     }
 }
diff --git a/src/EventSubscriber/ClassDefinitionIndexSubscriber.php b/src/EventSubscriber/ClassDefinitionIndexSubscriber.php
@@ -15,26 +15,29 @@
 
 use Exception;
 use Pimcore\Bundle\GenericDataIndexBundle\Installer;
-use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\EnqueueServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateClassMappingMessage;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\IndexHandler\DataObjectIndexHandler;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SettingsStoreServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Traits\LoggerAwareTrait;
 use Pimcore\Event\DataObjectClassDefinitionEvents;
 use Pimcore\Event\Model\DataObject\ClassDefinitionEvent;
-use Pimcore\Model\DataObject\ClassDefinition;
+use Pimcore\Helper\StopMessengerWorkersTrait;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+use Symfony\Component\Messenger\MessageBusInterface;
+use Symfony\Component\Messenger\Stamp\DelayStamp;
 
 /**
  * @internal
  */
 final class ClassDefinitionIndexSubscriber implements EventSubscriberInterface
 {
     use LoggerAwareTrait;
+    use StopMessengerWorkersTrait;
 
     public function __construct(
         private readonly Installer $installer,
         private readonly DataObjectIndexHandler $dataObjectMappingHandler,
-        private readonly EnqueueServiceInterface $enqueueService,
+        private readonly MessageBusInterface $messageBus,
         private readonly SettingsStoreServiceInterface $settingsStoreService,
     ) {
     }
@@ -53,31 +56,16 @@ public static function getSubscribedEvents(): array
      */
     public function addClassDefinitionMapping(ClassDefinitionEvent $event): void
     {
-        if (!$this->installer->isInstalled()) {
-            return;
-        }
 
-        $classDefinition = $event->getClassDefinition();
-        $this->updateClassMapping($classDefinition);
+        $this->handleMappingUpdate($event, false);
     }
 
     /**
      * @throws Exception
      */
     public function updateClassDefinitionMapping(ClassDefinitionEvent $event): void
     {
-        if (!$this->installer->isInstalled()) {
-            return;
-        }
-
-        $classDefinition = $event->getClassDefinition();
-        $mappingUpdated = $this->updateClassMapping($classDefinition);
-
-        if ($mappingUpdated) {
-            $this->enqueueService
-                ->enqueueByClassDefinition($classDefinition)
-                ->dispatchQueueMessages();
-        }
+        $this->handleMappingUpdate($event, true);
     }
 
     public function deleteClassDefinitionIndex(ClassDefinitionEvent $event): void
@@ -104,30 +92,21 @@ classDefinitionId: $classDefinition->getId()
     /**
      * @throws Exception
      */
-    private function updateClassMapping(ClassDefinition $classDefinition): bool
+    private function handleMappingUpdate(ClassDefinitionEvent $event, bool $dispatch): void
     {
-        $mappingProperties = $this->dataObjectMappingHandler->getMappingProperties($classDefinition);
-        $currentCheckSum = $this->dataObjectMappingHandler->getClassMappingCheckSum($mappingProperties);
-        $storedCheckSum = $this->settingsStoreService->getClassMappingCheckSum($classDefinition->getId());
-
-        if ($storedCheckSum === $currentCheckSum) {
-            return false;
+        if (!$this->installer->isInstalled()) {
+            return;
         }
 
-        $this->dataObjectMappingHandler
-            ->updateMapping(
-                context: $classDefinition,
-                forceCreateIndex: true,
-                mappingProperties: $mappingProperties
-            );
+        $this->stopMessengerWorkers();
+        $classDefinition = $event->getClassDefinition();
 
-        $this->settingsStoreService->storeClassMapping(
-            classDefinitionId: $classDefinition->getId(),
-            data: $this->dataObjectMappingHandler->getClassMappingCheckSum(
-                $mappingProperties
-            )
+        $this->messageBus->dispatch(
+            new UpdateClassMappingMessage(
+                classDefinition: $classDefinition,
+                dispatchQueueMessages: $dispatch
+            ),
+            [new DelayStamp(2000)]
         );
-
-        return true;
     }
 }
diff --git a/src/Message/UpdateClassMappingMessage.php b/src/Message/UpdateClassMappingMessage.php
@@ -0,0 +1,38 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Pimcore
+ *
+ * This source file is available under following license:
+ * - Pimcore Commercial License (PCL)
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\Message;
+
+use Pimcore\Model\DataObject\ClassDefinition;
+
+/**
+ * @internal
+ */
+final readonly class UpdateClassMappingMessage
+{
+    public function __construct(
+        private ClassDefinition $classDefinition,
+        private bool $dispatchQueueMessages = false
+    ) {
+    }
+
+    public function getClassDefinition(): ClassDefinition
+    {
+        return $this->classDefinition;
+    }
+
+    public function isDispatchQueueMessages(): bool
+    {
+        return $this->dispatchQueueMessages;
+    }
+}