Prepare first version of an OpenSearch index queue mechanism similar …

…to portal engine

config/doctrine.yaml

@@ -0,0 +1,12 @@
+doctrine:
+    orm:
+        entity_managers:
+            pimcore_generic_data_index:
+                connection: default
+                mappings:
+                    PimcoreGenericDataIndexBundle:
+                        type: attribute
+                        is_bundle: true
+                        dir: "src/Entity"
+                        prefix: Pimcore\Bundle\GenericDataIndexBundle\Entity
+                        alias: PimcoreGenericDataIndex

config/pimcore/config.yaml

@@ -0,0 +1,2 @@
+imports:
+    - { resource: messenger.yaml }

config/pimcore/messenger.yaml

@@ -0,0 +1,11 @@
+framework:
+    messenger:
+        enabled: true
+        transports:
+            pimcore_index_queues: 'doctrine://default?queue_name=pimcore_index_queues'
+        routing:
+            Pimcore\Bundle\GenericDataIndexBundle\Message\IndexUpdateQueueMessage: pimcore_index_queues
+        buses:
+            messenger.bus.pimcore-generic-data-index:
+                middleware:
+                    - 'Pimcore\Bundle\GenericDataIndexBundle\Service\Messenger\Middleware\CollectGarbageMiddleware'

config/pimcore/routing.yaml

@@ -0,0 +1,3 @@
+pimcore_generic_data_index_homepage:
+    path:     /pimcore_generic_data_index
+    defaults: { _controller: Pimcore\Bundle\GenericDataIndexBundle\Controller\DefaultController:indexAction }

config/services.yaml

@@ -1,5 +1,27 @@
+imports:
+    - { resource: 'services/search/index.yaml' }
+    - { resource: 'services/search/data-object/*' }
+    - { resource: 'services/*' }
 services:
     _defaults:
         autowire: true
         autoconfigure: true
-        public: false
+        public: false
+
+    # controllers are imported separately to make sure they're public
+    # and have a tag that allows actions to type-hint services
+    Pimcore\Bundle\GenericDataIndexBundle\Controller\:
+        resource: '../src/Controller'
+        public: true
+        tags: ['controller.service_arguments']
+
+    Pimcore\Bundle\GenericDataIndexBundle\MessageHandler\:
+        resource: '../src/MessageHandler'
+        tags: [ { name: messenger.message_handler, bus: messenger.bus.pimcore-generic-data-index } ]
+
+# add more services, or override services that need manual wiring
+#    Pimcore\_generic-data-index-bundle\ExampleClass:
+#        arguments:
+#            - "@service_id"
+#            - "plain_value"
+#            - "%parameter%"

config/services/commands.yaml

@@ -0,0 +1,10 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    # auto-register all commands as services
+    Pimcore\Bundle\GenericDataIndexBundle\Command\:
+        resource: '../../src/Command/*'
+        tags: ['console.command']

config/services/data-object.yaml

@@ -0,0 +1,7 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\DataObject\SelectOptionsService: ~

config/services/messenger.yaml

@@ -0,0 +1,7 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\Messenger\Middleware\CollectGarbageMiddleware: ~

config/services/search/data-object/field-definition-adapters.yml

@@ -0,0 +1,104 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\DefaultAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "wysiwyg" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\TextAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "input" }
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "textarea" }
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "calculatedValue" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\NumericAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "numeric" }
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "slider" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\DateAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "date" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\DateTimeAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "datetime" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\LocalizedFieldsAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "localizedfields" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\QuantityValueAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "quantityValue" }
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "inputQuantityValue" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\BooleanAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "checkbox" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\SelectAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "select" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\MultiSelectAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "multiselect" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\CountryAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "country" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\LanguageAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "language" }
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\CountryMultiSelectAdapter:
+        shared: false
+        tags:
+            - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "countrymultiselect" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\LanguageMultiSelectAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "languagemultiselect" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\ManyToOneRelationAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "manyToOneRelation" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\ManyToManyObjectRelationAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "manyToManyObjectRelation" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\AdvancedManyToManyRelationAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "advancedManyToManyRelation" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\AdvancedManyToManyObjectRelationAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "advancedManyToManyObjectRelation" }
+
+    #Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionAdapter\ManyToManyRelationAdapter:
+    #    shared: false
+    #    tags:
+    #        - { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "manyToManyRelation" }

config/services/search/index.yaml

@@ -0,0 +1,24 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\LanguageService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\AssetIndexService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\DataObjectIndexService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueueService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\OpenSearch\OpenSearchService: ~
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\OpenSearch\BulkOperationService: ~
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\DataObject\FieldDefinitionService:
+        arguments:
+            - '@pimcore.generic_data_index.object.search_index_field_definition_locator'
+
+    pimcore.generic_data_index.object.search_index_field_definition_locator:
+        class: Symfony\Component\DependencyInjection\ServiceLocator
+        tags: [ 'container.service_locator' ]
+        arguments:
+            - [ ]

config/services/workflow.yaml

@@ -0,0 +1,7 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\Service\Workflow\WorkflowService: ~

src/Command/Update/IndexUpdateCommand.php

@@ -0,0 +1,130 @@
+<?php
+
+/**
+ * Pimcore
+ *
+ * This source file is available under following license:
+ * - Pimcore Commercial License (PCL)
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\Command\Update;
+
+use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueueService;
+use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexUpdateService;
+use Pimcore\Console\AbstractCommand;
+use Pimcore\Model\DataObject\ClassDefinition;
+use Symfony\Component\Console\Command\LockableTrait;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Contracts\Service\Attribute\Required;
+
+class IndexUpdateCommand extends AbstractCommand
+{
+    use LockableTrait;
+
+    const OPTION_CLASS_DEFINITION_ID = 'class-definition-id';
+    const OPTION_UPDATE_ASSET_INDEX = 'update-asset-index';
+    const OPTION_RECREATE_INDEX = 'recreate_index';
+
+    protected IndexUpdateService $indexUpdateService;
+    protected IndexQueueService $indexQueueService;
+
+
+    protected function configure()
+    {
+        $this
+            ->setName('generic-data-index:update:index')
+            ->addOption(self::OPTION_CLASS_DEFINITION_ID, 'cid', InputOption::VALUE_OPTIONAL, 'Update mapping and data for specific data object classDefinition', null)
+            ->addOption(self::OPTION_UPDATE_ASSET_INDEX, 'a', InputOption::VALUE_NONE, 'Update mapping and data for asset index', null)
+            ->addOption(self::OPTION_RECREATE_INDEX, 'r', InputOption::VALUE_NONE, 'Delete OpenSearch index and recreate it', null)
+            ->setDescription('Updates index/mapping for all classDefinitions/asset without deleting them. Adds there elements to index queue.');
+    }
+
+    /**
+     * @param InputInterface $input
+     * @param OutputInterface $output
+     *
+     * @throws \Exception
+     *
+     * @return int
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        if (!$this->lock()) {
+            throw new \Exception('The command is already running in another process.');
+        }
+
+        $this->indexUpdateService->setReCreateIndex($input->getOption(self::OPTION_RECREATE_INDEX));
+
+        $updateAll = true;
+
+        /** @var string|null $classDefinitionId */
+        $classDefinitionId = $input->getOption(self::OPTION_CLASS_DEFINITION_ID);
+
+        if ($classDefinitionId) {
+            $updateAll = false;
+
+            try {
+                $classDefinition = ClassDefinition::getById($classDefinitionId);
+                if (!$classDefinition) {
+                    throw new \Exception(sprintf('ClassDefinition with id %s not found', $classDefinitionId));
+                }
+
+                $this->output->writeln(sprintf('<info>Update index and indices for ClassDefinition with id %s</info>', $classDefinitionId), OutputInterface::VERBOSITY_VERBOSE);
+
+                $this
+                    ->indexUpdateService
+                    ->updateClassDefinition($classDefinition);
+            } catch (\Exception $e) {
+                $this->output->writeln('<error>' . $e->getMessage() . '</error>');
+            }
+        }
+
+        if ($input->getOption(self::OPTION_UPDATE_ASSET_INDEX)) {
+            $updateAll = false;
+
+            try {
+                $output->writeln('<info>Update asset index</info>', OutputInterface::VERBOSITY_VERBOSE);
+
+                $this
+                    ->indexUpdateService
+                    ->updateAssets();
+            } catch (\Exception $e) {
+                $this->output->writeln($e->getMessage());
+            }
+        }
+
+        if ($updateAll) {
+            $this->output->writeln('<info>Update all mappings and indices for objects/assets</info>', OutputInterface::VERBOSITY_VERBOSE);
+
+            $this
+                ->indexUpdateService
+                ->updateAll();
+        }
+
+        $this->indexQueueService->dispatchQueueMessages($output);
+
+        $this->release();
+
+        $this->output->writeln('<info>Finished</info>', OutputInterface::VERBOSITY_VERBOSE);
+
+        return self::SUCCESS;
+    }
+
+    #[Required]
+    public function setIndexUpdateService(IndexUpdateService $indexUpdateService): void
+    {
+        $this->indexUpdateService = $indexUpdateService;
+    }
+
+    #[Required]
+    public function setIndexQueueService(IndexQueueService $indexQueueService): void
+    {
+        $this->indexQueueService = $indexQueueService;
+    }
+
+}

src/Controller/DefaultController.php

@@ -11,7 +11,7 @@
  *  @license    http://www.pimcore.org/license     PCL
  */
 
-namespace Pimcore\Bundle\xTemplateBundlex\Controller;
+namespace Pimcore\Bundle\GenericDataIndexBundle\Controller;
 
 use Symfony\Component\HttpFoundation\Response;