From 702b9d221fe02fce3a7d9038f58a7e7f5ff2db94 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 29 Nov 2023 10:47:47 +0100 Subject: [PATCH 001/145] feat: initial implementation --- bin/console | 20 ++ config/services.yml | 15 + src/Console/Application.php | 18 ++ src/Console/Command/Indexer.php | 132 ++++++++ .../Command/Io/IndexerProgressProgressBar.php | 64 ++++ src/Console/Command/MoreLikeThis.php | 114 +++++++ src/Console/Command/Search.php | 140 +++++++++ src/Console/Command/Suggest.php | 95 ++++++ src/Dto/Indexer/IndexerParameter.php | 16 + src/Dto/Search/Query/Facet/CategoryFacet.php | 13 + .../Query/Facet/ContentSectionTypeFacet.php | 13 + src/Dto/Search/Query/Facet/Facet.php | 10 + src/Dto/Search/Query/Facet/FacetField.php | 36 +++ .../Search/Query/Facet/FacetMultiQuery.php | 29 ++ src/Dto/Search/Query/Facet/FacetQuery.php | 22 ++ src/Dto/Search/Query/Facet/GroupFacet.php | 13 + .../Search/Query/Facet/ObjectTypeFacet.php | 13 + src/Dto/Search/Query/Facet/SiteFacet.php | 13 + src/Dto/Search/Query/Filter/ArchiveFilter.php | 16 + .../Search/Query/Filter/CategoryFilter.php | 17 + .../Query/Filter/ContentSectionTypeFilter.php | 17 + src/Dto/Search/Query/Filter/FieldFilter.php | 57 ++++ src/Dto/Search/Query/Filter/Filter.php | 36 +++ src/Dto/Search/Query/Filter/GroupFilter.php | 17 + .../Search/Query/Filter/ObjectTypeFilter.php | 17 + src/Dto/Search/Query/Filter/SiteFilter.php | 17 + src/Dto/Search/Query/MoreLikeThisQuery.php | 54 ++++ src/Dto/Search/Query/QueryDefaultOperator.php | 9 + src/Dto/Search/Query/SelectQuery.php | 83 +++++ src/Dto/Search/Query/SelectQueryBuilder.php | 171 +++++++++++ src/Dto/Search/Query/SuggestQuery.php | 52 ++++ src/Dto/Search/Result/Facet.php | 24 ++ src/Dto/Search/Result/FacetGroup.php | 30 ++ .../Search/Result/ResourceSearchResult.php | 65 ++++ src/Dto/Search/Result/SuggestResult.php | 30 ++ src/Dto/Search/Result/Suggestion.php | 24 ++ .../MissMatchingResourceFactoryException.php | 28 ++ src/Exception/UnexpectedResultException.php | 28 ++ src/Indexer.php | 21 ++ src/MoreLikeThisSearcher.php | 26 ++ src/SelectSearcher.php | 16 + src/Service/Indexer/DocumentEnricher.php | 21 ++ .../Indexer/IndexerProgressHandler.php | 16 + src/Service/Indexer/LocationFinder.php | 61 ++++ .../DefaultSchema21DocumentEnricher.php | 290 ++++++++++++++++++ src/Service/Indexer/SolrIndexer.php | 215 +++++++++++++ .../Search/ExternalResourceFactory.php | 38 +++ .../Search/InternalMediaResourceFactory.php | 42 +++ .../Search/InternalResourceFactory.php | 33 ++ src/Service/Search/ResourceFactory.php | 27 ++ .../Search/SiteKit/DefaultBoostModifier.php | 32 ++ src/Service/Search/SolrMoreLikeThis.php | 77 +++++ src/Service/Search/SolrQueryModifier.php | 17 + .../Search/SolrResultToResourceResolver.php | 58 ++++ src/Service/Search/SolrSelect.php | 260 ++++++++++++++++ src/Service/Search/SolrSuggest.php | 127 ++++++++ src/Service/SolrClientFactory.php | 16 + src/Service/SolrParameterClientFactory.php | 46 +++ src/SuggestSearcher.php | 19 ++ test/Console/ApplicationTest.php | 25 ++ test/Console/Command/IndexerTest.php | 41 +++ test/Service/IndexerTest.php | 19 ++ 62 files changed, 3111 insertions(+) create mode 100755 bin/console create mode 100644 config/services.yml create mode 100644 src/Console/Application.php create mode 100644 src/Console/Command/Indexer.php create mode 100644 src/Console/Command/Io/IndexerProgressProgressBar.php create mode 100644 src/Console/Command/MoreLikeThis.php create mode 100644 src/Console/Command/Search.php create mode 100644 src/Console/Command/Suggest.php create mode 100644 src/Dto/Indexer/IndexerParameter.php create mode 100644 src/Dto/Search/Query/Facet/CategoryFacet.php create mode 100644 src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php create mode 100644 src/Dto/Search/Query/Facet/Facet.php create mode 100644 src/Dto/Search/Query/Facet/FacetField.php create mode 100644 src/Dto/Search/Query/Facet/FacetMultiQuery.php create mode 100644 src/Dto/Search/Query/Facet/FacetQuery.php create mode 100644 src/Dto/Search/Query/Facet/GroupFacet.php create mode 100644 src/Dto/Search/Query/Facet/ObjectTypeFacet.php create mode 100644 src/Dto/Search/Query/Facet/SiteFacet.php create mode 100644 src/Dto/Search/Query/Filter/ArchiveFilter.php create mode 100644 src/Dto/Search/Query/Filter/CategoryFilter.php create mode 100644 src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php create mode 100644 src/Dto/Search/Query/Filter/FieldFilter.php create mode 100644 src/Dto/Search/Query/Filter/Filter.php create mode 100644 src/Dto/Search/Query/Filter/GroupFilter.php create mode 100644 src/Dto/Search/Query/Filter/ObjectTypeFilter.php create mode 100644 src/Dto/Search/Query/Filter/SiteFilter.php create mode 100644 src/Dto/Search/Query/MoreLikeThisQuery.php create mode 100644 src/Dto/Search/Query/QueryDefaultOperator.php create mode 100644 src/Dto/Search/Query/SelectQuery.php create mode 100644 src/Dto/Search/Query/SelectQueryBuilder.php create mode 100644 src/Dto/Search/Query/SuggestQuery.php create mode 100644 src/Dto/Search/Result/Facet.php create mode 100644 src/Dto/Search/Result/FacetGroup.php create mode 100644 src/Dto/Search/Result/ResourceSearchResult.php create mode 100644 src/Dto/Search/Result/SuggestResult.php create mode 100644 src/Dto/Search/Result/Suggestion.php create mode 100644 src/Exception/MissMatchingResourceFactoryException.php create mode 100644 src/Exception/UnexpectedResultException.php create mode 100644 src/Indexer.php create mode 100644 src/MoreLikeThisSearcher.php create mode 100644 src/SelectSearcher.php create mode 100644 src/Service/Indexer/DocumentEnricher.php create mode 100644 src/Service/Indexer/IndexerProgressHandler.php create mode 100644 src/Service/Indexer/LocationFinder.php create mode 100644 src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php create mode 100644 src/Service/Indexer/SolrIndexer.php create mode 100644 src/Service/Search/ExternalResourceFactory.php create mode 100644 src/Service/Search/InternalMediaResourceFactory.php create mode 100644 src/Service/Search/InternalResourceFactory.php create mode 100644 src/Service/Search/ResourceFactory.php create mode 100644 src/Service/Search/SiteKit/DefaultBoostModifier.php create mode 100644 src/Service/Search/SolrMoreLikeThis.php create mode 100644 src/Service/Search/SolrQueryModifier.php create mode 100644 src/Service/Search/SolrResultToResourceResolver.php create mode 100644 src/Service/Search/SolrSelect.php create mode 100644 src/Service/Search/SolrSuggest.php create mode 100644 src/Service/SolrClientFactory.php create mode 100644 src/Service/SolrParameterClientFactory.php create mode 100644 src/SuggestSearcher.php create mode 100644 test/Console/ApplicationTest.php create mode 100644 test/Console/Command/IndexerTest.php create mode 100644 test/Service/IndexerTest.php diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..089d452 --- /dev/null +++ b/bin/console @@ -0,0 +1,20 @@ +#!/usr/bin/env php +load('services.yml'); +$container->compile(); + +$application = $container->get(Application::class); + +$application->run(); \ No newline at end of file diff --git a/config/services.yml b/config/services.yml new file mode 100644 index 0000000..1b29748 --- /dev/null +++ b/config/services.yml @@ -0,0 +1,15 @@ +services: + _defaults: + autowire: true + autoconfigure: true + _instanceof: + Symfony\Component\Console\Command\Command: + tags: ['command'] + + Atoolo\Search\Console\: + resource: '../src/Console' + + Atoolo\Search\Console\Application: + public: true + arguments: + - !tagged command \ No newline at end of file diff --git a/src/Console/Application.php b/src/Console/Application.php new file mode 100644 index 0000000..0fcfbbe --- /dev/null +++ b/src/Console/Application.php @@ -0,0 +1,18 @@ +add($command); + } + } +} diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php new file mode 100644 index 0000000..7ef089b --- /dev/null +++ b/src/Console/Command/Indexer.php @@ -0,0 +1,132 @@ +setHelp('Command to fill a search index') + ->addArgument( + 'solr-core', + InputArgument::REQUIRED, + 'Solr core to be used.' + ) + ->addArgument( + 'resource-dir', + InputArgument::REQUIRED, + 'Resource directory whose data is to be indexed.' + ) + ->addArgument( + 'directories', + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Resources or directories of the resource to be indexed.' + ) + ->addOption( + 'cleanup-threshold', + null, + InputArgument::OPTIONAL, + 'Specifies the number of indexed documents from ' . + 'which indexing is considered successful and old entries ' . + 'can be deleted. Is only used for full indexing.', + 0 + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->io = new SymfonyStyle($input, $output); + $this->progressBar = new IndexerProgressProgressBar($output); + $this->resourceDir = $input->getArgument('resource-dir'); + $directories = $input->getArgument('directories'); + + $cleanupThreshold = empty($directories) + ? $input->getArgument('cleanup-threshold') + : 0; + + if (empty($directories)) { + $this->io->title('Index all resources'); + } else { + $this->io->title('Index resources subdirectories'); + $this->io->listing($directories); + } + + $parameter = new IndexerParameter( + $input->getArgument('solr-core'), + $this->resourceDir, + $cleanupThreshold, + $directories + ); + + $indexer = $this->createIndexer(); + $indexer->index($parameter); + + $this->errorReport(); + + return Command::SUCCESS; + } + + protected function errorReport(): void + { + foreach ($this->progressBar->getErrors() as $error) { + if ($error instanceof InvalidResourceException) { + $this->io->error( + $error->getLocation() . ': ' . + $error->getMessage() + ); + } else { + $this->io->error($error->getMessage()); + } + } + } + + protected function createIndexer(): SolrIndexer + { + $resourceLoader = new SiteKitLoader($this->resourceDir); + $navigationLoader = new SiteKitNavigationHierarchyLoader( + $resourceLoader + ); + $schema21 = new DefaultSchema21DocumentEnricher( + $navigationLoader + ); + + $clientFactory = new SolrParameterClientFactory(); + return new SolrIndexer( + [$schema21], + $this->progressBar, + $resourceLoader, + $clientFactory, + 'internal' + ); + } +} diff --git a/src/Console/Command/Io/IndexerProgressProgressBar.php b/src/Console/Command/Io/IndexerProgressProgressBar.php new file mode 100644 index 0000000..13b156c --- /dev/null +++ b/src/Console/Command/Io/IndexerProgressProgressBar.php @@ -0,0 +1,64 @@ +output = $output; + } + + public function start(int $total): void + { + $this->progressBar = new ProgressBar($this->output, $total); + $this->formatProgressBar('green'); + } + + public function advance(int $step): void + { + $this->progressBar->advance($step); + } + + private function formatProgressBar(string $color): void + { + $this->progressBar->setBarCharacter('•'); + $this->progressBar->setEmptyBarCharacter('⚬'); + $this->progressBar->setProgressCharacter('➤'); + $this->progressBar->setFormat( + "%current%/%max% [%bar%] %percent:3s%%\n" . + " %estimated:-20s% %memory:20s%" + ); + } + + public function error(Exception $exception): void + { + $this->formatProgressBar('red'); + $this->errors[] = $exception; + } + + public function finish(): void + { + $this->progressBar->finish(); + } + + /** + * @return array + */ + public function getErrors(): array + { + return $this->errors; + } +} diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php new file mode 100644 index 0000000..ab8cc33 --- /dev/null +++ b/src/Console/Command/MoreLikeThis.php @@ -0,0 +1,114 @@ +setHelp('Command to performs a more-like-this search') + ->addArgument( + 'solr-core', + InputArgument::REQUIRED, + 'Solr core to be used.' + ) + ->addArgument( + 'resource-dir', + InputArgument::REQUIRED, + 'Resource directory whose data is to be indexed.' + ) + ->addArgument( + 'location', + InputArgument::REQUIRED, + 'Resource directory whose data is to be indexed.' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->io = new SymfonyStyle($input, $output); + + $this->solrCore = $input->getArgument('solr-core'); + $this->resourceDir = $input->getArgument('resource-dir'); + $location = $input->getArgument('location'); + + $searcher = $this->createSearcher(); + $query = $this->buildQuery($location); + $result = $searcher->moreLikeThis($query); + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function createSearcher(): SolrMoreLikeThis + { + $resourceLoader = new SiteKitLoader($this->resourceDir); + $clientFactory = new SolrParameterClientFactory(); + $resourceFactoryList = [ + new ExternalResourceFactory(), + new InternalResourceFactory($resourceLoader), + new InternalMediaResourceFactory($resourceLoader) + ]; + $solrResultToResourceResolver = new SolrResultToResourceResolver( + $resourceFactoryList + ); + + return new SolrMoreLikeThis( + $clientFactory, + $solrResultToResourceResolver + ); + } + + protected function buildQuery(string $location): MoreLikeThisQuery + { + $filterList = []; + return new MoreLikeThisQuery( + $this->solrCore, + $location, + $filterList, + 5, + ['content'] + ); + } + + protected function outputResult(ResourceSearchResult $result): void + { + $this->io->text($result->getTotal() . " Results:"); + foreach ($result as $resource) { + $this->io->text($resource->getLocation()); + } + $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + } +} diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php new file mode 100644 index 0000000..999715d --- /dev/null +++ b/src/Console/Command/Search.php @@ -0,0 +1,140 @@ +setHelp('Command to performs a search') + ->addArgument( + 'solr-core', + InputArgument::REQUIRED, + 'Solr core to be used.' + ) + ->addArgument( + 'resource-dir', + InputArgument::REQUIRED, + 'Resource directory whose data is to be indexed.' + ) + ->addArgument( + 'text', + InputArgument::IS_ARRAY, + 'Text with which to search.' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->io = new SymfonyStyle($input, $output); + $this->resourceDir = $input->getArgument('resource-dir'); + $this->solrCore = $input->getArgument('solr-core'); + + $searcher = $this->createSearch(); + $query = $this->buildQuery($input); + + $result = $searcher->select($query); + + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function createSearch(): SolrSelect + { + $resourceLoader = new SiteKitLoader($this->resourceDir); + $clientFactory = new SolrParameterClientFactory(); + $defaultBoosting = new DefaultBoostModifier(); + + $resourceFactoryList = [ + new ExternalResourceFactory(), + new InternalResourceFactory($resourceLoader), + new InternalMediaResourceFactory($resourceLoader) + ]; + + $solrResultToResourceResolver = new SolrResultToResourceResolver( + $resourceFactoryList + ); + + return new SolrSelect( + $clientFactory, + [$defaultBoosting], + $solrResultToResourceResolver + ); + } + + protected function buildQuery(InputInterface $input): SelectQuery + { + $builder = SelectQuery::builder(); + $builder->core($this->solrCore); + + $text = $input->getArgument('text'); + if (is_array($text)) { + $builder->text(implode(' ', $text)); + } + + // TODO: filter + + // TODO: facet + + return $builder->build(); + } + + protected function outputResult( + ResourceSearchResult $result + ) { + $this->io->title('Results (' . $result->getTotal() . ')'); + foreach ($result as $resource) { + $this->io->text($resource->getLocation()); + } + + if (count($result->getFacetGroupList()) > 0) { + $this->io->title('Facets'); + foreach ($result->getFacetGroupList() as $facetGroup) { + $this->io->section($facetGroup->getKey()); + $listing = []; + foreach ($facetGroup->getFacetList() as $facet) { + $listing[] = + $facet->getKey() . + ' (' . $facet->getHits() . ')'; + } + $this->io->listing($listing); + } + } + + $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + } +} diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php new file mode 100644 index 0000000..a431467 --- /dev/null +++ b/src/Console/Command/Suggest.php @@ -0,0 +1,95 @@ +setHelp('Command to performs a suggest search') + ->addArgument( + 'solr-core', + InputArgument::REQUIRED, + 'Solr core to be used.' + ) + ->addArgument( + 'terms', + InputArgument::REQUIRED | InputArgument::IS_ARRAY, + 'Suggest terms.' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->io = new SymfonyStyle($input, $output); + $this->solrCore = $input->getArgument('solr-core'); + $terms = $input->getArgument('terms'); + + $search = $this->createSearcher(); + $query = $this->buildQuery($terms); + + $result = $search->suggest($query); + + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function createSearcher(): SolrSuggest + { + $clientFactory = new SolrParameterClientFactory(); + return new SolrSuggest($clientFactory); + } + + protected function buildQuery(array $terms): SuggestQuery + { + $excludeMedia = new ObjectTypeFilter('media', 'media'); + $excludeMedia = $excludeMedia->exclude(); + return new SuggestQuery( + $this->solrCore, + $terms, + [ + new ArchiveFilter(), + $excludeMedia + ] + ); + } + + protected function outputResult(SuggestResult $result): void + { + foreach ($result as $suggest) { + $this->io->text( + $suggest->getTerm() . + ' (' . $suggest->getHits() . ')' + ); + } + $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + } +} diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php new file mode 100644 index 0000000..b1ce85e --- /dev/null +++ b/src/Dto/Indexer/IndexerParameter.php @@ -0,0 +1,16 @@ +key; + } + + public function getField(): string + { + return $this->field; + } + + /** + * @return string[] + */ + public function getTerms(): array + { + return $this->terms; + } +} diff --git a/src/Dto/Search/Query/Facet/FacetMultiQuery.php b/src/Dto/Search/Query/Facet/FacetMultiQuery.php new file mode 100644 index 0000000..e93d210 --- /dev/null +++ b/src/Dto/Search/Query/Facet/FacetMultiQuery.php @@ -0,0 +1,29 @@ +key; + } + /** + * @return FacetQuery[] + */ + public function getQueryList(): array + { + return $this->queryList; + } +} diff --git a/src/Dto/Search/Query/Facet/FacetQuery.php b/src/Dto/Search/Query/Facet/FacetQuery.php new file mode 100644 index 0000000..8c02fe3 --- /dev/null +++ b/src/Dto/Search/Query/Facet/FacetQuery.php @@ -0,0 +1,22 @@ +key; + } + public function getQuery(): string + { + return $this->query; + } +} diff --git a/src/Dto/Search/Query/Facet/GroupFacet.php b/src/Dto/Search/Query/Facet/GroupFacet.php new file mode 100644 index 0000000..39dbcdb --- /dev/null +++ b/src/Dto/Search/Query/Facet/GroupFacet.php @@ -0,0 +1,13 @@ +values = $values; + parent::__construct( + $key, + $this->toQuery(), + [$key] + ); + } + + /** + * @param string[] $values + */ + private function toQuery(): string + { + $filterValue = count($this->values) === 1 + ? $this->values[0] + : '(' . implode(' ', $this->values) . ')'; + return $this->field . ':' . $filterValue; + } + + public function exclude(): FieldFilter + { + $field = $this->field; + if (!str_starts_with($field, '-')) { + $field = '-' . $field; + } + return new FieldFilter( + $this->getKey(), + $field, + ...$this->values + ); + } +} diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php new file mode 100644 index 0000000..6d26936 --- /dev/null +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -0,0 +1,36 @@ +key; + } + + public function getQuery(): string + { + return $this->query; + } + + /** + * @return string[] + */ + public function getTags(): array + { + return $this->tags; + } +} diff --git a/src/Dto/Search/Query/Filter/GroupFilter.php b/src/Dto/Search/Query/Filter/GroupFilter.php new file mode 100644 index 0000000..e5d22e6 --- /dev/null +++ b/src/Dto/Search/Query/Filter/GroupFilter.php @@ -0,0 +1,17 @@ +core; + } + + public function getLocation(): string + { + return $this->location; + } + + /** + * @return Filter[] + */ + public function getFilterList(): array + { + return $this->filterList; + } + + public function getLimit(): int + { + return $this->limit; + } + + /** + * @return array + */ + public function getFieldList(): array + { + return $this->fieldList; + } +} diff --git a/src/Dto/Search/Query/QueryDefaultOperator.php b/src/Dto/Search/Query/QueryDefaultOperator.php new file mode 100644 index 0000000..b485d5d --- /dev/null +++ b/src/Dto/Search/Query/QueryDefaultOperator.php @@ -0,0 +1,9 @@ +core = $builder->getCore(); + $this->text = $builder->getText(); + $this->offset = $builder->getOffset(); + $this->limit = $builder->getLimit(); + $this->filterList = $builder->getFilterList(); + $this->facetList = $builder->getFacetList(); + $this->queryDefaultOperator = $builder->getQueryDefaultOperator(); + } + + public static function builder(): SelectQueryBuilder + { + return new SelectQueryBuilder(); + } + + public function getCore(): string + { + return $this->core; + } + + public function getText(): string + { + return $this->text; + } + + public function getOffset(): int + { + return $this->offset; + } + + public function getLimit(): int + { + return $this->limit; + } + + /** + * @return Filter[] + */ + public function getFilterList(): array + { + return $this->filterList; + } + /** + * @return Facet[] + */ + public function getFacetList(): array + { + return $this->facetList; + } + public function getQueryDefaultOperator(): QueryDefaultOperator + { + return $this->queryDefaultOperator; + } +} diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php new file mode 100644 index 0000000..38033b5 --- /dev/null +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -0,0 +1,171 @@ + + */ + private array $filterList = []; + + /** + * @var array + */ + private array $facetList = []; + + private QueryDefaultOperator $queryDefaultOperator = QueryDefaultOperator::AND; + + /** + * @internal + */ + public function __construct() + { + } + + public function core(string $core): SelectQueryBuilder + { + if (empty($core)) { + throw new \InvalidArgumentException('core is empty'); + } + $this->core = $core; + return $this; + } + + /** + * @internal + */ + public function getCore(): string + { + return $this->core; + } + + public function text(string $text): SelectQueryBuilder + { + $this->text = $text; + return $this; + } + + /** + * @internal + */ + public function getText(): string + { + return $this->text; + } + + public function offset(int $offset): SelectQueryBuilder + { + if ($offset < 0) { + throw new \InvalidArgumentException('offset is lower then 0'); + } + $this->offset = $offset; + return $this; + } + + /** + * @internal + */ + public function getOffset(): int + { + return $this->offset; + } + + public function limit(int $limit): SelectQueryBuilder + { + if ($limit < 0) { + throw new \InvalidArgumentException('offset is lower then 0'); + } + $this->limit = $limit; + return $this; + } + + /** + * @internal + */ + public function getLimit(): int + { + return $this->limit; + } + + /** + * @param Filter[] $filterList + */ + public function filter(Filter ...$filterList): SelectQueryBuilder + { + foreach ($filterList as $filter) { + if (isset($this->filterList[$filter->getKey()])) { + throw new \InvalidArgumentException( + 'filter key "' . $filter->getKey() . + '" already exists' + ); + } + $this->filterList[$filter->getKey()] = $filter; + } + return $this; + } + + /** + * @internal + * @return Filter[] + */ + public function getFilterList(): array + { + return array_values($this->filterList); + } + + /** + * @param Filter[] $filterList + */ + public function facet(Facet ...$facetList): SelectQueryBuilder + { + foreach ($facetList as $facet) { + if (isset($this->facetList[$facet->getKey()])) { + throw new \InvalidArgumentException( + 'facet key "' . $facet->getKey() . + '" already exists' + ); + } + $this->facetList[$facet->getKey()] = $facet; + } + return $this; + } + + /** + * @internal + * @return Facet[] + */ + public function getFacetList(): array + { + return array_values($this->facetList); + } + + public function queryDefaultOperator( + QueryDefaultOperator $queryDefaultOperator + ): SelectQueryBuilder { + $this->queryDefaultOperator = $queryDefaultOperator; + return $this; + } + + public function getQueryDefaultOperator(): QueryDefaultOperator + { + return $this->queryDefaultOperator; + } + + public function build(): SelectQuery + { + if (empty($this->core)) { + throw new \InvalidArgumentException('core is not set'); + } + return new SelectQuery($this); + } +} diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php new file mode 100644 index 0000000..144c168 --- /dev/null +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -0,0 +1,52 @@ +core; + } + /** + * @return string[] + */ + public function getTermList(): array + { + return $this->termList; + } + + public function getLimit(): int + { + return $this->limit; + } + + /** + * @return Filter[] + */ + public function getFilterList(): array + { + return $this->filterList; + } + public function getField(): string + { + return $this->field; + } +} diff --git a/src/Dto/Search/Result/Facet.php b/src/Dto/Search/Result/Facet.php new file mode 100644 index 0000000..8e19cd0 --- /dev/null +++ b/src/Dto/Search/Result/Facet.php @@ -0,0 +1,24 @@ +key; + } + + public function getHits(): int + { + return $this->hits; + } +} \ No newline at end of file diff --git a/src/Dto/Search/Result/FacetGroup.php b/src/Dto/Search/Result/FacetGroup.php new file mode 100644 index 0000000..300b2ca --- /dev/null +++ b/src/Dto/Search/Result/FacetGroup.php @@ -0,0 +1,30 @@ +key; + } + + /** + * @return Facet[] + */ + public function getFacetList(): array + { + return $this->facetList; + } +} diff --git a/src/Dto/Search/Result/ResourceSearchResult.php b/src/Dto/Search/Result/ResourceSearchResult.php new file mode 100644 index 0000000..9f3cd68 --- /dev/null +++ b/src/Dto/Search/Result/ResourceSearchResult.php @@ -0,0 +1,65 @@ + + */ +class ResourceSearchResult implements IteratorAggregate +{ + /** + * @param Resource[] $resourceList + * @param FacetGroup[] $facetGroupList + */ + public function __construct( + private readonly int $total, + private readonly int $offset, + private readonly array $resourceList, + private readonly array $facetGroupList, + private readonly int $queryTime + ) { + } + + public function getIterator(): Traversable + { + return new ArrayIterator($this->resourceList); + } + + /** + * @return Resource[] + */ + public function getResourceList(): array + { + return $this->resourceList; + } + + public function getTotal(): int + { + return $this->total; + } + + public function getOffset(): int + { + return $this->offset; + } + + /** + * @return FacetGroup[] + */ + public function getFacetGroupList(): array + { + return $this->facetGroupList; + } + + public function getQueryTime(): int + { + return $this->queryTime; + } +} diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php new file mode 100644 index 0000000..9b4770a --- /dev/null +++ b/src/Dto/Search/Result/SuggestResult.php @@ -0,0 +1,30 @@ + + */ +class SuggestResult implements IteratorAggregate +{ + public function __construct( + private readonly array $suggestions, + private readonly int $queryTime + ) { + } + + public function getIterator(): ArrayIterator + { + return new ArrayIterator($this->suggestions); + } + + public function getQueryTime(): int + { + return $this->queryTime; + } +} diff --git a/src/Dto/Search/Result/Suggestion.php b/src/Dto/Search/Result/Suggestion.php new file mode 100644 index 0000000..bfe9293 --- /dev/null +++ b/src/Dto/Search/Result/Suggestion.php @@ -0,0 +1,24 @@ +term; + } + + public function getHits(): int + { + return $this->hits; + } +} diff --git a/src/Exception/MissMatchingResourceFactoryException.php b/src/Exception/MissMatchingResourceFactoryException.php new file mode 100644 index 0000000..e01833a --- /dev/null +++ b/src/Exception/MissMatchingResourceFactoryException.php @@ -0,0 +1,28 @@ +location; + } +} diff --git a/src/Exception/UnexpectedResultException.php b/src/Exception/UnexpectedResultException.php new file mode 100644 index 0000000..0a5ffda --- /dev/null +++ b/src/Exception/UnexpectedResultException.php @@ -0,0 +1,28 @@ +result, + $code, + $previous + ); + } + + public function getResult(): string + { + return $this->result; + } +} diff --git a/src/Indexer.php b/src/Indexer.php new file mode 100644 index 0000000..55d4f58 --- /dev/null +++ b/src/Indexer.php @@ -0,0 +1,21 @@ +basePath = rtrim($basePath, '/'); + } + + /** + * @return string[] + */ + public function findAll(): array + { + + $finder = new Finder(); + $finder->in($this->basePath); + $finder->name('*.php'); + $finder->files(); + + $pathList = []; + foreach ($finder as $file) { + $pathList[] = $this->toRelativePath($file->getPathname()); + } + + return $pathList; + } + + /** + * @param string[] $directories + */ + public function findInSubdirectories(array $directories): array + { + $finder = new Finder(); + foreach ($directories as $directory) { + $finder->in($this->basePath . '/' . $directory); + } + $finder->name('*.php'); + $finder->files(); + + $pathList = []; + foreach ($finder as $file) { + $pathList[] = $this->toRelativePath($file->getPathname()); + } + + return $pathList; + } + + private function toRelativePath(string $path): string + { + return substr($path, strlen($this->basePath)); + } +} diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php new file mode 100644 index 0000000..129e367 --- /dev/null +++ b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php @@ -0,0 +1,290 @@ +sp_id = $resource->getId(); + $doc->sp_name = $resource->getName(); + $doc->sp_anchor = $resource->getData('init.anchor'); + $doc->title = $resource->getData('base.title'); + $doc->description = $resource->getData('metadata.description'); + $doc->sp_objecttype = $resource->getObjectType(); + $doc->sp_canonical = true; + $doc->crawl_process_id = $processId; + + $mediaUrl = $resource->getData('init.mediaUrl'); + if ($mediaUrl !== null) { + $doc->id = $mediaUrl; + $doc->url = $mediaUrl; + } else { + $doc->id = $resource->getLocation(); + $doc->url = $resource->getLocation(); + } + + $spContentType = [$resource->getObjectType()]; + if ($resource->getData('init.media') !== true) { + $spContentType[] = 'article'; + } + $contentSectionTypes = $resource->getData('init.contentSectionTypes'); + if (is_array($contentSectionTypes)) { + $spContentType = array_merge($spContentType, $contentSectionTypes); + } + if ($resource->getData('base.teaser.image') !== null) { + $spContentType[] = 'teaserImage'; + } + if ($resource->getData('base.teaser.image.copyright') !== null) { + $spContentType[] = 'teaserImageCopyright'; + } + if ($resource->getData('base.teaser.headline') !== null) { + $spContentType[] = 'teaserHeadline'; + } + if ($resource->getData('base.teaser.text') !== null) { + $spContentType[] = 'teaserText'; + } + $doc->sp_contenttype = $spContentType; + + $locale = $this->getLocaleFromResource($resource); + $lang = $this->toLangFromLocale($locale); + $doc->sp_language = $lang; + $doc->meta_content_language = $lang; + + $doc->sp_changed = $this->toDateTime( + $resource->getData('init.changed') + ); + $doc->sp_date = $this->toDateTime( + $resource->getData('base.date') + ); + + $doc->sp_archive = $resource->getData('base.archive') ?? false; + + $headline = $resource->getData('metadata.headline'); + if (empty($headline)) { + $headline = $resource->getData('base.teaser.headline'); + } + if (empty($headline)) { + $headline = $resource->getData('base.title'); + } + $doc->sp_title = $headline; + + // However, the teaser heading, if specified, must be used for sorting + $sortHeadline = $resource->getData('base.teaser.headline'); + if (empty($sortHeadline)) { + $sortHeadline = $resource->getData('metadata.headline'); + } + if (empty($sortHeadline)) { + $sortHeadline = $resource->getData('base.title'); + } + $doc->sp_sortvalue = $sortHeadline; + + $doc->sp_boost_keywords = $resource->getData('metadata.boostKeywords'); + + $sites = $this->getParentSiteGroupIdList($resource); + + $navigationRoot = $this->navigationLoader->loadRoot( + $resource->getLocation() + ); + $siteGroupId = $navigationRoot->getData('init.siteGroup.id'); + if ($siteGroupId !== null) { + $sites[] = $siteGroupId; + } + $doc->sp_site = array_unique($sites); + + $wktPrimaryList = $resource->getData('base.geo.wkt.primary'); + if (is_array($wktPrimaryList)) { + $allWkt = []; + foreach ($wktPrimaryList as $wkt) { + $allWkt[] = $wkt; + } + if (count($allWkt) > 0) { + $doc->sp_geo_points = $allWkt; + } + } + + $categoryList = $resource->getData('metadata.categories'); + if (is_array($categoryList)) { + $categoryIdList = []; + foreach ($categoryList as $category) { + $categoryIdList[] = $category['id']; + } + $doc->sp_category = $categoryIdList; + } + + $categoryPath = $resource->getData('metadata.categoriesPath'); + if (is_array($categoryPath)) { + $categoryIdPath = []; + foreach ($categoryPath as $category) { + $categoryIdPath[] = $category['id']; + } + $doc->sp_category_path = $categoryIdPath; + } + + $groupPath = $resource->getData('init.groupPath'); + $groupPathAsIdList = []; + if (is_array($groupPath)) { + foreach ($groupPath as $group) { + $groupPathAsIdList[] = $group['id']; + } + } + $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; + $doc->sp_group_path = $groupPathAsIdList; + + $schedulingList = $resource->getData('metadata.scheduling'); + if (is_array($schedulingList) && count($schedulingList) > 0) { + $doc->sp_date = $this->toDateTime($schedulingList[0]['from']); + $dateList = []; + $contentTypeList = []; + foreach ($schedulingList as $scheduling) { + $contentTypeList[] = explode(' ', $scheduling['contentType']); + $dateList[] = $this->toDateTime($scheduling['from']); + } + $doc->sp_contenttype = array_merge( + $doc->sp_contenttype, + ...$contentTypeList + ); + $doc->sp_contenttype = array_unique($doc->sp_contenttype); + + $doc->sp_date_list = $dateList; + } + + $contentType = $resource->getData('base.mime'); + if ($contentType === null) { + $contentType = 'text/html; charset=UTF-8'; + } + $doc->meta_content_type = $contentType; + $doc->content = $resource->getData('searchindexdata.content'); + + $accessType = $resource->getData('init.access.type'); + $groups = $resource->getData('init.access.groups'); + + + if ($accessType === 'allow' && is_array($groups)) { + $doc->include_groups = array_map( + fn($id): int => $this->idWithoutSignature($id), + $groups + ); + } elseif ($accessType === 'deny' && is_array($groups)) { + $doc->exclude_groups = array_map( + fn($id): int => $this->idWithoutSignature($id), + $groups + ); + } else { + $doc->exclude_groups = ['none']; + $doc->include_groups = ['all']; + } + + $doc->sp_source = ['internal']; + + return $doc; + } + + private function idWithoutSignature(string $id): int + { + $s = substr($id, -11); + return (int)$s; + } + + /* Customization + * - https://gitlab.sitepark.com/customer-projects/fhdo/blob/develop/fhdo-module/src/publish/php/SP/Fhdo/Component/Content/DetailPage/StartletterIndexSupplier.php#L31 + * - https://gitlab.sitepark.com/apis/sitekit-php/blob/develop/php/SP/SiteKit/Component/Content/NewsdeskRss.php#L235 + * - https://gitlab.sitepark.com/customer-projects/fhdo/blob/develop/fhdo-module/src/publish/php/SP/Fhdo/Component/SearchMetadataExtension.php#L41 + * - https://gitlab.sitepark.com/customer-projects/paderborn/blob/develop/paderborn-module/src/publish/php/SP/Paderborn/Component/FscEntity.php#L67 + * - https://gitlab.sitepark.com/customer-projects/paderborn/blob/develop/paderborn-module/src/publish/php/SP/Paderborn/Component/FscContactPerson.php#L24 + * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/ParkingSpaceExpose.php#L38 + * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/Expose.php#L38 + * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 + * - https://gitlab.sitepark.com/customer-projects/stuttgart/blob/develop/stuttgart-module/src/publish/php/SP/Stuttgart/Component/JobOffer.php#L29 + * - https://gitlab.sitepark.com/customer-projects/stuttgart/blob/develop/stuttgart-module/src/publish/php/SP/Stuttgart/Component/EventsCalendarExtension.php#L124 + * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 + * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Controller/Environment.php#L76 + * - https://gitlab.sitepark.com/ies-modules/sitekit-real-estate/blob/develop/src/publish/php/SP/RealEstate/Component/Expose.php#L47 + */ + + private function getLocaleFromResource(Resource $resource): string + { + + $locale = $resource->getData('init.locale'); + if ($locale !== null) { + return $locale; + } + $groupPath = $resource->getData('init.groupPath'); + $len = count($groupPath); + if (is_array($groupPath)) { + for ($i = $len - 1; $i >= 0; $i--) { + $group = $groupPath[$i]; + if (isset($group['locale'])) { + return $group['locale']; + } + } + } + + return 'de_DE'; + } + + private function toLangFromLocale(string $locale): string + { + if (str_contains($locale, '_')) { + $parts = explode('_', $locale); + return $parts[0]; + } + return $locale; + } + + private function toDateTime(?int $timestamp): ?DateTime + { + if ($timestamp === null) { + return null; + } + if ($timestamp <= 0) { + return null; + } + + $dateTime = new DateTime(); + $dateTime->setTimestamp($timestamp); + return $dateTime; + } + + private function getParentSiteGroupIdList(Resource $resource): array + { + $parents = $this->getNavigationParents($resource); + if (empty($parents)) { + return []; + } + + $siteGroupIdList = []; + foreach ($parents as $parent) { + if (isset($parent['siteGroup']['id'])) { + $siteGroupIdList[] = $parent['siteGroup']['id']; + } + } + + return $siteGroupIdList; + } + + /** + * @return array + */ + private function getNavigationParents(Resource $resource): array + { + $parents = $resource->getData('base.trees.navigation.parents'); + return $parents ?? []; + } +} diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php new file mode 100644 index 0000000..4bf37ee --- /dev/null +++ b/src/Service/Indexer/SolrIndexer.php @@ -0,0 +1,215 @@ + $documentEnricherList + */ + public function __construct( + private readonly iterable $documentEnricherList, + private readonly IndexerProgressHandler $indexerProgressHandler, + private readonly ResourceLoader $resourceLoader, + private readonly SolrClientFactory $clientFactory, + private readonly string $source + ) { + } + + public function index(IndexerParameter $parameter): void + { + $finder = new LocationFinder($parameter->basePath); + if (empty($parameter->directories)) { + $pathList = $finder->findAll(); + } else { + $pathList = $finder->findInSubdirectories($parameter->directories); + } + $this->indexResources($parameter, $pathList); + } + + /** + * @param array $pathList + */ + private function indexResources( + IndexerParameter $parameter, + array $pathList + ): void { + if (count($pathList) === 0) { + return; + } + + $total = count($pathList); + $this->indexerProgressHandler->start($total); + + $processId = uniqid('', true); + $offset = 0; + $chunkSize = 500; + $successCount = 0; + + try { + while (true) { + $indexedCount = $this->indexChunks( + $processId, + $parameter->coreId, + $pathList, + $offset, + $chunkSize + ); + if ($indexedCount === false) { + break; + } + $successCount += $indexedCount; + $offset += $chunkSize; + } + + if ( + $parameter->cleanupThreshold > 0 && + $successCount >= $parameter->cleanupThreshold + ) { + $this->deleteByProcessId($parameter->coreId, $processId); + } + $this->commit($parameter->coreId); + } finally { + $this->indexerProgressHandler->finish(); + } + } + + /** + * @param string[] $pathList + */ + private function indexChunks( + string $processId, + string $solrCore, + array $pathList, + int $offset, + int $length + ): int|false { + $resourceList = $this->loadResources( + $pathList, + $offset, + $length + ); + if ($resourceList === false) { + return false; + } + $this->indexerProgressHandler->advance(count($resourceList)); + $result = $this->add($solrCore, $processId, $resourceList); + + if ($result->getStatus() !== 0) { + $this->indexerProgressHandler->error(new Exception( + $result->getResponse()->getStatusMessage() + )); + return 0; + } + + return count($resourceList); + } + + /** + * @param string[] $pathList + * @return Resource[]|false + */ + private function loadResources( + array $pathList, + int $offset, + int $length + ): array | false { + + $maxLength = (count($pathList) ?? 0) - $offset; + if ($maxLength <= 0) { + return false; + } + + if ($length > $maxLength) { + $length = $maxLength; + } + + $resourceList = []; + for ($i = $offset; $i < ($length + $offset); $i++) { + $path = $pathList[$i]; + try { + $resource = $this->resourceLoader->load($path); + $resourceList[] = $resource; + } catch (InvalidResourceException $e) { + $this->indexerProgressHandler->error($e); + } + } + return $resourceList; + } + + /** + * @param string $solrCore + * @param string $processId + * @param array $resources + * @return ResultInterface|Result + */ + private function add( + string $solrCore, + string $processId, + array $resources + ): ResultInterface|Result { + $client = $this->clientFactory->create($solrCore); + + $update = $client->createUpdate(); + + $documents = []; + foreach ($resources as $resource) { + $doc = $update->createDocument(); + foreach ($this->documentEnricherList as $enricher) { + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + $documents[] = $doc; + } + + // add the documents and a commit command to the update query + $update->addDocuments($documents); + + // this executes the query and returns the result + return $client->update($update); + } + + private function deleteByProcessId(string $core, string $processId): void + { + $this->deleteByQuery( + $core, + '-crawl_process_id:' . $processId . ' AND ' . + ' sp_source:' . $this->source + ); + } + + private function deleteByQuery(string $core, string $query): void + { + $client = $this->clientFactory->create($core); + $update = $client->createUpdate(); + $update->addDeleteQuery($query); + $client->update($update); + } + + private function commit(string $core): void + { + $client = $this->clientFactory->create($core); + $update = $client->createUpdate(); + $update->addCommit(); + $update->addOptimize(); + $client->update($update); + } +} diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php new file mode 100644 index 0000000..9d09cf9 --- /dev/null +++ b/src/Service/Search/ExternalResourceFactory.php @@ -0,0 +1,38 @@ +url; + return ( + str_starts_with($location, 'http://') || + str_starts_with($location, 'https://') + ); + } + + public function create(Document $document): Resource + { + return new Resource( + $document->url, + '', + $document->title, + 'external', + [] + ); + } +} diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php new file mode 100644 index 0000000..753e138 --- /dev/null +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -0,0 +1,42 @@ +getMetaLocation($document); + return $this->resourceLoader->exists($metaLocation); + } + + public function create(Document $document): Resource + { + $metaLocation = $this->getMetaLocation($document); + return $this->resourceLoader->load($metaLocation); + } + + private function getMetaLocation(Document $document): string + { + return $document->url . '.meta.php'; + } +} diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php new file mode 100644 index 0000000..0217adc --- /dev/null +++ b/src/Service/Search/InternalResourceFactory.php @@ -0,0 +1,33 @@ +url, '.php'); + } + + public function create(Document $document): Resource + { + return $this->resourceLoader->load($document->url); + } +} diff --git a/src/Service/Search/ResourceFactory.php b/src/Service/Search/ResourceFactory.php new file mode 100644 index 0000000..6fbaea0 --- /dev/null +++ b/src/Service/Search/ResourceFactory.php @@ -0,0 +1,27 @@ +getEDisMax(); + $edismax->setQueryFields(implode(' ', [ + 'sp_title^1.4', + 'keywords^1.2', + 'description^1.0', + 'title^1.0', + 'url^0.9', + 'content^0.8' + ])); + $edismax->setPhraseFields(implode(' ', [ + 'sp_title^1.5', + 'description^1', + 'content^0.8' + ])); + $edismax->setBoostQuery('sp_objecttype:searchTip^100'); + + return $query; + } +} diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php new file mode 100644 index 0000000..11e87b8 --- /dev/null +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -0,0 +1,77 @@ + $resourceFactoryList + */ + public function __construct( + private readonly SolrClientFactory $clientFactory, + private readonly SolrResultToResourceResolver $resultToResourceResolver + ) { + } + + public function moreLikeThis(MoreLikeThisQuery $query): ResourceSearchResult + { + $client = $this->clientFactory->create($query->getCore()); + $solrQuery = $this->buildSolrQuery($client, $query); + $result = $client->execute($solrQuery); + return $this->buildResult($result); + } + + private function buildSolrQuery( + Client $client, + MoreLikeThisQuery $query + ): SolrMoreLikeThisQuery { + + $solrQuery = $client->createMoreLikeThis(); + $solrQuery->setOmitHeader(false); + $solrQuery->setQuery('url:"' . $query->getLocation() . '"'); + $solrQuery->setMltFields($query->getFieldList()); + $solrQuery->setRows($query->getLimit()); + $solrQuery->setMinimumTermFrequency(2); + $solrQuery->setMatchInclude(true); + $solrQuery->createFilterQuery('nomedia') + ->setQuery('-sp_objecttype:media'); + + // Filter + foreach ($query->getFilterList() as $filter) { + $solrQuery->createFilterQuery($filter->getKey()) + ->setQuery($filter->getQuery()) + ->setTags($filter->getTags()); + } + + return $solrQuery; + } + + private function buildResult( + ResultInterface $result + ): ResourceSearchResult { + + $resourceList = $this->resultToResourceResolver + ->loadResourceList($result); + + return new ResourceSearchResult( + $result->getNumFound(), + 0, + $resourceList, + [], + $result->getQueryTime() + ); + } +} diff --git a/src/Service/Search/SolrQueryModifier.php b/src/Service/Search/SolrQueryModifier.php new file mode 100644 index 0000000..ab51430 --- /dev/null +++ b/src/Service/Search/SolrQueryModifier.php @@ -0,0 +1,17 @@ + $resourceFactoryList + */ + public function __construct( + private readonly iterable $resourceFactoryList, + private readonly LoggerInterface $logger = new NullLogger() + ) { + } + + /** + * @return array + */ + public function loadResourceList(ResultInterface $result): array + { + $resourceList = []; + foreach ($result as $document) { + try { + $resourceList[] = $this->loadResource($document); + } catch (\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + } + } + return $resourceList; + } + + private function loadResource(Document $document): Resource + { + + foreach ($this->resourceFactoryList as $resourceFactory) { + if ($resourceFactory->accept($document)) { + return $resourceFactory->create($document); + } + } + + throw new MissMatchingResourceFactoryException($document->url); + } +} diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php new file mode 100644 index 0000000..b5b289a --- /dev/null +++ b/src/Service/Search/SolrSelect.php @@ -0,0 +1,260 @@ + $solrQueryModifierList + */ + public function __construct( + private readonly SolrClientFactory $clientFactory, + private readonly iterable $solrQueryModifierList, + private readonly SolrResultToResourceResolver $resultToResourceResolver + ) { + } + + public function select(SelectQuery $query): ResourceSearchResult + { + $client = $this->clientFactory->create($query->getCore()); + + $solrQuery = $this->buildSolrQuery($client, $query); + $result = $client->execute($solrQuery); + return $this->buildResult($query, $result); + } + + private function buildSolrQuery( + Client $client, + SelectQuery $query + ): SolrSelectQuery { + + $solrQuery = $client->createSelect(); + + // supplements the query with standard values, e.g. for boosting + foreach ($this->solrQueryModifierList as $solrQueryModifier) { + $solrQuery = $solrQueryModifier->modify($solrQuery); + } + + $solrQuery->setStart($query->getOffset()); + $solrQuery->setRows($query->getLimit()); + + // to get query-time + $solrQuery->setOmitHeader(false); + + $this->addRequiredFieldListToSolrQuery($solrQuery); + $this->addTextFilterToSolrQuery($solrQuery, $query->getText()); + $this->addQueryDefaultOperatorToSolrQuery( + $solrQuery, + $query->getQueryDefaultOperator() + ); + $this->addFilterQueriesToSolrQuery( + $solrQuery, + $query->getFilterList() + ); + $this->addFacetListToSolrQuery( + $solrQuery, + $query->getFacetList() + ); + + return $solrQuery; + } + + private function addRequiredFieldListToSolrQuery( + SolrSelectQuery $solrQuery + ): void { + $fields = $solrQuery->getFields(); + if (in_array('url', $fields, true)) { + return; + } + $fields[] = 'url'; + $solrQuery->setFields($fields); + } + + private function addTextFilterToSolrQuery( + SolrSelectQuery $solrQuery, + string $text + ): void { + if (empty($text)) { + return; + } + $terms = explode(' ', $text); + $terms = array_map(function ($term) use ($solrQuery) { + $term = trim($term); + return $solrQuery->getHelper()->escapeTerm($term); + }, + $terms); + $text = implode(' ', $terms); + $solrQuery->setQuery($text); + } + + private function addQueryDefaultOperatorToSolrQuery( + SolrSelectQuery $solrQuery, + QueryDefaultOperator $operator + ): void { + if ($operator === QueryDefaultOperator::OR) { + $solrQuery->setQueryDefaultOperator( + SolrSelectQuery::QUERY_OPERATOR_OR + ); + } else { + $solrQuery->setQueryDefaultOperator( + SolrSelectQuery::QUERY_OPERATOR_AND + ); + } + } + + /** + * @param Filter[] $filterList + */ + private function addFilterQueriesToSolrQuery( + SolrSelectQuery $solrQuery, + array $filterList + ): void { + + foreach ($filterList as $filter) { + $solrQuery->createFilterQuery($filter->getKey()) + ->setQuery($filter->getQuery()) + ->setTags($filter->getTags()); + } + } + + /** + * @param \Atoolo\Search\Dto\Search\Query\Facet\Facet[] $filterList + */ + private function addFacetListToSolrQuery( + SolrSelectQuery $solrQuery, + array $facetList + ): void { + foreach ($facetList as $facet) { + if ($facet instanceof FacetField) { + $this->addFacetFieldToSolrQuery($solrQuery, $facet); + } elseif ($facet instanceof FacetQuery) { + $this->addFacetQueryToSolrQuery($solrQuery, $facet); + } elseif ($facet instanceof FacetMultiQuery) { + $this->addFacetMultiQueryToSolrQuery($solrQuery, $facet); + } else { + throw new \InvalidArgumentException( + 'Unsupported facet-class ' . get_class($facet) + ); + } + } + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-field/ + */ + private function addFacetFieldToSolrQuery( + SolrSelectQuery $solrQuery, + FacetField $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + // https://solr.apache.org/guide/solr/latest/query-guide/faceting.html#tagging-and-excluding-filters + $fieldWithExclude = '{!ex=' . $facet->getKey() . '}' . + $facet->getField(); + $facetSet->createFacetField($facet->getKey()) + ->setField($fieldWithExclude) + ->setTerms($facet->getTerms()); + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-query/ + */ + private function addFacetQueryToSolrQuery( + SolrSelectQuery $solrQuery, + FacetQuery $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + $facetSet->createFacetQuery($facet->getKey()) + ->setQuery($facet->getQuery()); + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-multiquery/ + */ + private function addFacetMultiQueryToSolrQuery( + SolrSelectQuery $solrQuery, + FacetMultiQuery $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + $solrFacet = $facetSet->createFacetMultiQuery($facet->getKey()); + foreach ($facet->getQueryList() as $facetQuery) { + $solrFacet->createQuery( + $facetQuery->getKey(), + $facetQuery->getQuery() + ); + } + } + + private function buildResult( + SelectQuery $query, + ResultInterface $result + ): ResourceSearchResult { + + $resourceList = $this->resultToResourceResolver + ->loadResourceList($result); + $facetGroupList = $this->buildFacetGroupList($query, $result); + + return new ResourceSearchResult( + $result->getNumFound(), + $query->getOffset(), + $resourceList, + $facetGroupList, + $result->getQueryTime() + ); + } + + /** + * @param ResultInterface $result + * @return FacetGroup[] + */ + private function buildFacetGroupList( + SelectQuery $query, + ResultInterface $result + ): array { + + $facetSet = $result->getFacetSet(); + if ($facetSet === null) { + return []; + } + + $facetGroupList = []; + foreach ($query->getFacetList() as $facet) { + $facetGroupList[] = $this->buildFacetGroup( + $facet->getKey(), + $facetSet->getFacet($facet->getKey()) + ); + } + return $facetGroupList; + } + + private function buildFacetGroup( + string $key, + FacetResultInterface $solrFacet + ): FacetGroup { + $facetList = []; + foreach ($solrFacet as $value => $count) { + $facetList[] = new Facet((string)$value, $count); + } + return new FacetGroup($key, $facetList); + } +} diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php new file mode 100644 index 0000000..17b9a48 --- /dev/null +++ b/src/Service/Search/SolrSuggest.php @@ -0,0 +1,127 @@ +clientFactory->create($query->getCore()); + + $solrQuery = $this->buildSolrQuery($client, $query); + $solrResult = $client->select($solrQuery); + return $this->buildResult($solrResult, $query->getField()); + } + + private function buildSolrQuery( + Client $client, + SuggestQuery $query + ): SolrSelectQuery { + $solrQuery = $client->createSelect(); + $solrQuery->addParam("spellcheck", "true"); + $solrQuery->addParam("spellcheck.accuracy", "0.6"); + $solrQuery->addParam("spellcheck.onlyMorePopular", "false"); + $solrQuery->addParam("spellcheck.count", "15"); + $solrQuery->addParam("spellcheck.maxCollations", "5"); + $solrQuery->addParam("spellcheck.maxCollationTries", "15"); + $solrQuery->addParam("spellcheck.collate", "true"); + $solrQuery->addParam("spellcheck.collateExtendedResults", "true"); + $solrQuery->addParam("spellcheck.extendedResults", "true"); + $solrQuery->addParam("facet", "true"); + $solrQuery->addParam("facet.sort", "count"); + $solrQuery->addParam("facet.method", "enum"); + $solrQuery->addParam( + "facet.prefix", + implode(' ', $query->getTermList()) + ); + $solrQuery->addParam("facet.limit", $query->getLimit()); + $solrQuery->addParam("facet.field", $query->getField()); + + $solrQuery->setOmitHeader(false); + $solrQuery->setStart(0); + $solrQuery->setRows(0); + + // Filter + foreach ($query->getFilterList() as $filter) { + $solrQuery->createFilterQuery($filter->getKey()) + ->setQuery($filter->getQuery()) + ->setTags($filter->getTags()); + } + + return $solrQuery; + } + + private function buildResult( + SolrSelectResult $solrResult, + string $resultField + ): SuggestResult { + $suggestions = $this->parseSuggestion( + $solrResult->getResponse()->getBody(), + $resultField + ); + return new SuggestResult($suggestions, $solrResult->getQueryTime()); + } + + /** + * @throws UnexpectedResultException + * @return Suggestion[] + */ + private function parseSuggestion( + string $responseBody, + string $facetField + ): array { + try { + $json = json_decode( + $responseBody, + true, + 5, + JSON_THROW_ON_ERROR + ); + $facets = + $json['facet_counts']['facet_fields'][$facetField] + ?? []; + + $len = count($facets); + + $suggestions = []; + for ($i = 0; $i < $len; $i += 2) { + $term = $facets[$i]; + $hits = $facets[$i + 1]; + $suggestions[] = new Suggestion($term, $hits); + } + + return $suggestions; + } catch (JsonException $e) { + throw new UnexpectedResultException( + $responseBody, + "Invalid JSON for suggest result", + 0, + $e + ); + } + } +} diff --git a/src/Service/SolrClientFactory.php b/src/Service/SolrClientFactory.php new file mode 100644 index 0000000..ce26af6 --- /dev/null +++ b/src/Service/SolrClientFactory.php @@ -0,0 +1,16 @@ +setTimeout(30); + //$adapter->setProxy('http://localhost:8889'); + $eventDispatcher = new EventDispatcher(); + $config = [ + 'endpoint' => [ + $host => [ + 'scheme' => 'https', + 'host' => $host, + 'port' => 443, + 'path' => '', + 'core' => $core, + ] + ] + ]; + + // create a client instance + return new Client( + $adapter, + $eventDispatcher, + $config + ); + } +} diff --git a/src/SuggestSearcher.php b/src/SuggestSearcher.php new file mode 100644 index 0000000..e5ff274 --- /dev/null +++ b/src/SuggestSearcher.php @@ -0,0 +1,19 @@ +get('atoolo:indexer'); + $this->assertInstanceOf( + Indexer::class, + $command, + 'unexpected indexer command' + ); + } +} diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php new file mode 100644 index 0000000..855b898 --- /dev/null +++ b/test/Console/Command/IndexerTest.php @@ -0,0 +1,41 @@ +find('atoolo:indexer'); + $commandTester = new CommandTester($command); + $commandTester->execute([ + // pass arguments to the helper + 'resource-dir' => 'abc', + + // prefix the key with two dashes when passing options, + // e.g: '--some-option' => 'option_value', + // use brackets for testing array value, + // e.g: '--some-option' => ['option_value'], + ]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString('Whoa!', $output); + + // ... + } + +} diff --git a/test/Service/IndexerTest.php b/test/Service/IndexerTest.php new file mode 100644 index 0000000..ae1a00d --- /dev/null +++ b/test/Service/IndexerTest.php @@ -0,0 +1,19 @@ +index(); + */ + } +} From e1439a7b2b3f08204e5972e82c9c3365cb7f653a Mon Sep 17 00:00:00 2001 From: veltrup Date: Thu, 30 Nov 2023 17:53:53 +0100 Subject: [PATCH 002/145] refactor: Various changes following internal coordination --- src/Console/Application.php | 4 ++ src/Console/Command/Indexer.php | 39 +++++++++++++++++-- src/Console/Command/Search.php | 8 ++-- src/Dto/Search/Query/Facet/CategoryFacet.php | 17 ++++++-- .../Query/Facet/ContentSectionTypeFacet.php | 17 ++++++-- src/Dto/Search/Query/Facet/Facet.php | 3 +- src/Dto/Search/Query/Facet/FacetField.php | 8 +++- .../Search/Query/Facet/FacetMultiQuery.php | 9 ++++- src/Dto/Search/Query/Facet/FacetQuery.php | 7 +++- src/Dto/Search/Query/Facet/GroupFacet.php | 17 ++++++-- .../Search/Query/Facet/ObjectTypeFacet.php | 17 ++++++-- src/Dto/Search/Query/Facet/SiteFacet.php | 17 ++++++-- src/Dto/Search/Query/SelectQuery.php | 8 ++-- src/Dto/Search/Query/SelectQueryBuilder.php | 21 +++++----- src/Dto/Search/Result/FacetGroup.php | 6 +-- .../Search/Result/ResourceSearchResult.php | 24 +++++++----- src/Service/Search/SolrSelect.php | 11 ++++-- 17 files changed, 176 insertions(+), 57 deletions(-) diff --git a/src/Console/Application.php b/src/Console/Application.php index 0fcfbbe..96a4b3e 100644 --- a/src/Console/Application.php +++ b/src/Console/Application.php @@ -5,9 +5,13 @@ namespace Atoolo\Search\Console; use Symfony\Component\Console\Application as BaseApplication; +use Symfony\Component\Console\Command\Command; class Application extends BaseApplication { + /** + * @param iterable $commands + */ public function __construct(iterable $commands = []) { parent::__construct(); diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 7ef089b..491a400 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -12,6 +12,7 @@ use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema21DocumentEnricher; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\SolrParameterClientFactory; +use http\Exception\InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -67,11 +68,14 @@ protected function execute( $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); - $this->resourceDir = $input->getArgument('resource-dir'); - $directories = $input->getArgument('directories'); + $this->resourceDir = $this->getStringArgument( + $input, + 'resource-dir' + ); + $directories = (array)$input->getArgument('directories'); $cleanupThreshold = empty($directories) - ? $input->getArgument('cleanup-threshold') + ? $this->getIntArgument($input, 'cleanup-threshold') : 0; if (empty($directories)) { @@ -82,7 +86,7 @@ protected function execute( } $parameter = new IndexerParameter( - $input->getArgument('solr-core'), + $this->getStringArgument($input, 'solr-core'), $this->resourceDir, $cleanupThreshold, $directories @@ -96,6 +100,33 @@ protected function execute( return Command::SUCCESS; } + private function getStringArgument( + InputInterface $input, + string $name + ): string { + $value = $input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + $name . ' must be a string' + ); + } + return strval($value); + return (string)$value; + } + + private function getIntArgument( + InputInterface $input, + string $name + ): int { + $value = $input->getArgument($name); + if (!is_int($value)) { + throw new InvalidArgumentException( + $name . ' must be a integer' + ); + } + return (int)$value; + } + protected function errorReport(): void { foreach ($this->progressBar->getErrors() as $error) { diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 999715d..cf8b0e2 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -28,7 +28,7 @@ class Search extends Command { private SymfonyStyle $io; - private string $solrCore; + private string $index; private string $resourceDir; @@ -37,7 +37,7 @@ protected function configure(): void $this ->setHelp('Command to performs a search') ->addArgument( - 'solr-core', + 'index', InputArgument::REQUIRED, 'Solr core to be used.' ) @@ -61,7 +61,7 @@ protected function execute( $this->io = new SymfonyStyle($input, $output); $this->resourceDir = $input->getArgument('resource-dir'); - $this->solrCore = $input->getArgument('solr-core'); + $this->index = $input->getArgument('index'); $searcher = $this->createSearch(); $query = $this->buildQuery($input); @@ -99,7 +99,7 @@ protected function createSearch(): SolrSelect protected function buildQuery(InputInterface $input): SelectQuery { $builder = SelectQuery::builder(); - $builder->core($this->solrCore); + $builder->index($this->index); $text = $input->getArgument('text'); if (is_array($text)) { diff --git a/src/Dto/Search/Query/Facet/CategoryFacet.php b/src/Dto/Search/Query/Facet/CategoryFacet.php index d501b6b..fc03bf0 100644 --- a/src/Dto/Search/Query/Facet/CategoryFacet.php +++ b/src/Dto/Search/Query/Facet/CategoryFacet.php @@ -6,8 +6,19 @@ class CategoryFacet extends FacetField { - public function __construct(string $key, string ...$category) - { - parent::__construct($key, 'sp_category_path', $category); + /** + * @param string[] $groups + */ + public function __construct( + string $key, + array $categories, + ?string $excludeFilter + ) { + parent::__construct( + $key, + 'sp_category_path', + $categories, + $excludeFilter + ); } } diff --git a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php index 5245211..54f2987 100644 --- a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php @@ -6,8 +6,19 @@ class ContentSectionTypeFacet extends FacetField { - public function __construct(string $key, string ...$terms) - { - parent::__construct($key, 'sp_contenttype', $terms); + /** + * @param string[] $contentSectionTypes + */ + public function __construct( + string $key, + array $contentSectionTypes, + ?string $excludeFilter + ) { + parent::__construct( + $key, + 'sp_contenttype', + $contentSectionTypes, + $excludeFilter + ); } } diff --git a/src/Dto/Search/Query/Facet/Facet.php b/src/Dto/Search/Query/Facet/Facet.php index 3cf4060..56eb375 100644 --- a/src/Dto/Search/Query/Facet/Facet.php +++ b/src/Dto/Search/Query/Facet/Facet.php @@ -6,5 +6,6 @@ interface Facet { - public function getKey(): string; + public function getKey(): ?string; + public function getExcludeFilter(): ?string; } diff --git a/src/Dto/Search/Query/Facet/FacetField.php b/src/Dto/Search/Query/Facet/FacetField.php index 74f7c9d..e4bab7e 100644 --- a/src/Dto/Search/Query/Facet/FacetField.php +++ b/src/Dto/Search/Query/Facet/FacetField.php @@ -12,7 +12,8 @@ class FacetField implements Facet public function __construct( private readonly string $key, private readonly string $field, - private readonly array $terms + private readonly array $terms, + private readonly ?string $excludeFilter ) { } @@ -33,4 +34,9 @@ public function getTerms(): array { return $this->terms; } + + public function getExcludeFilter(): ?string + { + return $this->excludeFilter; + } } diff --git a/src/Dto/Search/Query/Facet/FacetMultiQuery.php b/src/Dto/Search/Query/Facet/FacetMultiQuery.php index e93d210..4da4569 100644 --- a/src/Dto/Search/Query/Facet/FacetMultiQuery.php +++ b/src/Dto/Search/Query/Facet/FacetMultiQuery.php @@ -12,9 +12,11 @@ class FacetMultiQuery implements Facet */ public function __construct( private readonly string $key, - private readonly array $queryList + private readonly array $queryList, + private readonly ?string $excludeFilter ) { } + public function getKey(): string { return $this->key; @@ -26,4 +28,9 @@ public function getQueryList(): array { return $this->queryList; } + + public function getExcludeFilter(): ?string + { + return $this->excludeFilter; + } } diff --git a/src/Dto/Search/Query/Facet/FacetQuery.php b/src/Dto/Search/Query/Facet/FacetQuery.php index 8c02fe3..a1b5deb 100644 --- a/src/Dto/Search/Query/Facet/FacetQuery.php +++ b/src/Dto/Search/Query/Facet/FacetQuery.php @@ -8,7 +8,8 @@ class FacetQuery implements Facet { public function __construct( private readonly string $key, - private readonly string $query + private readonly string $query, + private readonly ?string $excludeFilter ) { } public function getKey(): string @@ -19,4 +20,8 @@ public function getQuery(): string { return $this->query; } + public function getExcludeFilter(): ?string + { + return $this->excludeFilter; + } } diff --git a/src/Dto/Search/Query/Facet/GroupFacet.php b/src/Dto/Search/Query/Facet/GroupFacet.php index 39dbcdb..88606d9 100644 --- a/src/Dto/Search/Query/Facet/GroupFacet.php +++ b/src/Dto/Search/Query/Facet/GroupFacet.php @@ -6,8 +6,19 @@ class GroupFacet extends FacetField { - public function __construct(string $key, string ...$group) - { - parent::__construct($key, 'sp_group_path', $group); + /** + * @param string[] $groups + */ + public function __construct( + string $key, + array $groups, + ?string $excludeFilter + ) { + parent::__construct( + $key, + 'sp_group_path', + $groups, + $excludeFilter + ); } } diff --git a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php index 555c38b..da3fffa 100644 --- a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php @@ -6,8 +6,19 @@ class ObjectTypeFacet extends FacetField { - public function __construct(string $key, string ...$terms) - { - parent::__construct($key, 'sp_objecttype', $terms); + /** + * @param string[] $objectTypes + */ + public function __construct( + string $key, + array $objectTypes, + ?string $excludeFilter + ) { + parent::__construct( + $key, + 'sp_objecttype', + $objectTypes, + $excludeFilter + ); } } diff --git a/src/Dto/Search/Query/Facet/SiteFacet.php b/src/Dto/Search/Query/Facet/SiteFacet.php index 367df95..6ea341e 100644 --- a/src/Dto/Search/Query/Facet/SiteFacet.php +++ b/src/Dto/Search/Query/Facet/SiteFacet.php @@ -6,8 +6,19 @@ class SiteFacet extends FacetField { - public function __construct(string $key, string ...$site) - { - parent::__construct($key, 'sp_site', $site); + /** + * @param string[] $sites + */ + public function __construct( + string $key, + array $sites, + ?string $excludeFilter + ) { + parent::__construct( + $key, + 'sp_site', + $sites, + $excludeFilter + ); } } diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index d1ba7c5..2eea4a4 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -9,7 +9,7 @@ class SelectQuery { - private readonly string $core; + private readonly string $index; private readonly string $text; private readonly int $offset; private readonly int $limit; @@ -28,7 +28,7 @@ class SelectQuery */ public function __construct(SelectQueryBuilder $builder) { - $this->core = $builder->getCore(); + $this->index = $builder->getIndex(); $this->text = $builder->getText(); $this->offset = $builder->getOffset(); $this->limit = $builder->getLimit(); @@ -42,9 +42,9 @@ public static function builder(): SelectQueryBuilder return new SelectQueryBuilder(); } - public function getCore(): string + public function getIndex(): string { - return $this->core; + return $this->index; } public function getText(): string diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 38033b5..b24971e 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -9,7 +9,7 @@ class SelectQueryBuilder { - private string $core = ''; + private string $index = ''; private string $text = ''; private int $offset = 0; private int $limit = 10; @@ -23,7 +23,8 @@ class SelectQueryBuilder */ private array $facetList = []; - private QueryDefaultOperator $queryDefaultOperator = QueryDefaultOperator::AND; + private QueryDefaultOperator $queryDefaultOperator = + QueryDefaultOperator::AND; /** * @internal @@ -32,21 +33,21 @@ public function __construct() { } - public function core(string $core): SelectQueryBuilder + public function index(string $index): SelectQueryBuilder { - if (empty($core)) { - throw new \InvalidArgumentException('core is empty'); + if (empty($index)) { + throw new \InvalidArgumentException('index is empty'); } - $this->core = $core; + $this->index = $index; return $this; } /** * @internal */ - public function getCore(): string + public function getIndex(): string { - return $this->core; + return $this->index; } public function text(string $text): SelectQueryBuilder @@ -163,8 +164,8 @@ public function getQueryDefaultOperator(): QueryDefaultOperator public function build(): SelectQuery { - if (empty($this->core)) { - throw new \InvalidArgumentException('core is not set'); + if (empty($this->index)) { + throw new \InvalidArgumentException('index is not set'); } return new SelectQuery($this); } diff --git a/src/Dto/Search/Result/FacetGroup.php b/src/Dto/Search/Result/FacetGroup.php index 300b2ca..35740a8 100644 --- a/src/Dto/Search/Result/FacetGroup.php +++ b/src/Dto/Search/Result/FacetGroup.php @@ -11,7 +11,7 @@ class FacetGroup */ public function __construct( private readonly string $key, - private readonly array $facetList + private readonly array $facets ) { } @@ -23,8 +23,8 @@ public function getKey(): string /** * @return Facet[] */ - public function getFacetList(): array + public function getFacets(): array { - return $this->facetList; + return $this->facets; } } diff --git a/src/Dto/Search/Result/ResourceSearchResult.php b/src/Dto/Search/Result/ResourceSearchResult.php index 9f3cd68..1d1def5 100644 --- a/src/Dto/Search/Result/ResourceSearchResult.php +++ b/src/Dto/Search/Result/ResourceSearchResult.php @@ -15,29 +15,30 @@ class ResourceSearchResult implements IteratorAggregate { /** - * @param Resource[] $resourceList - * @param FacetGroup[] $facetGroupList + * @param Resource[] $results + * @param FacetGroup[] $facetGroups */ public function __construct( private readonly int $total, + private readonly int $limit, private readonly int $offset, - private readonly array $resourceList, - private readonly array $facetGroupList, + private readonly array $results, + private readonly array $facetGroups, private readonly int $queryTime ) { } public function getIterator(): Traversable { - return new ArrayIterator($this->resourceList); + return new ArrayIterator($this->results); } /** * @return Resource[] */ - public function getResourceList(): array + public function getResults(): array { - return $this->resourceList; + return $this->results; } public function getTotal(): int @@ -45,6 +46,11 @@ public function getTotal(): int return $this->total; } + public function getLimit(): int + { + return $this->limit; + } + public function getOffset(): int { return $this->offset; @@ -53,9 +59,9 @@ public function getOffset(): int /** * @return FacetGroup[] */ - public function getFacetGroupList(): array + public function getFacetGroups(): array { - return $this->facetGroupList; + return $this->facetGroups; } public function getQueryTime(): int diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index b5b289a..2950abc 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -37,7 +37,7 @@ public function __construct( public function select(SelectQuery $query): ResourceSearchResult { - $client = $this->clientFactory->create($query->getCore()); + $client = $this->clientFactory->create($query->getIndex()); $solrQuery = $this->buildSolrQuery($client, $query); $result = $client->execute($solrQuery); @@ -168,11 +168,13 @@ private function addFacetFieldToSolrQuery( FacetField $facet ): void { $facetSet = $solrQuery->getFacetSet(); + $field = $facet->getField(); // https://solr.apache.org/guide/solr/latest/query-guide/faceting.html#tagging-and-excluding-filters - $fieldWithExclude = '{!ex=' . $facet->getKey() . '}' . - $facet->getField(); + if ($facet->getExcludeFilter() !== null) { + $field = '{!ex=' . $facet->getExcludeFilter() . '}' . $field; + } $facetSet->createFacetField($facet->getKey()) - ->setField($fieldWithExclude) + ->setField($field) ->setTerms($facet->getTerms()); } @@ -216,6 +218,7 @@ private function buildResult( return new ResourceSearchResult( $result->getNumFound(), + $query->getLimit(), $query->getOffset(), $resourceList, $facetGroupList, From ec432bad9447e52bd8f727934047d62fca5a52e3 Mon Sep 17 00:00:00 2001 From: veltrup Date: Fri, 1 Dec 2023 14:44:29 +0100 Subject: [PATCH 003/145] feat: filter key is optional --- .../Search/Query/Filter/CategoryFilter.php | 6 ++++-- .../Query/Filter/ContentSectionTypeFilter.php | 6 ++++-- src/Dto/Search/Query/Filter/FieldFilter.php | 2 +- src/Dto/Search/Query/Filter/Filter.php | 4 ++-- src/Dto/Search/Query/Filter/GroupFilter.php | 6 ++++-- .../Search/Query/Filter/ObjectTypeFilter.php | 6 ++++-- src/Dto/Search/Query/Filter/SiteFilter.php | 6 ++++-- src/Dto/Search/Query/SuggestQuery.php | 21 +++++++++---------- src/Dto/Search/Result/SuggestResult.php | 8 +++++++ src/Service/Search/SolrSuggest.php | 6 +++--- 10 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/Dto/Search/Query/Filter/CategoryFilter.php b/src/Dto/Search/Query/Filter/CategoryFilter.php index 0882d4b..9d6ad53 100644 --- a/src/Dto/Search/Query/Filter/CategoryFilter.php +++ b/src/Dto/Search/Query/Filter/CategoryFilter.php @@ -6,8 +6,10 @@ class CategoryFilter extends FieldFilter { - public function __construct(string $key, string ...$category) - { + public function __construct( + ?string $key, + string ...$category + ) { parent::__construct( $key, 'sp_category_path', diff --git a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php index 29c29c9..65b92d9 100644 --- a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php @@ -6,8 +6,10 @@ class ContentSectionTypeFilter extends FieldFilter { - public function __construct(string $key, string ...$contentTypes) - { + public function __construct( + ?string $key, + string ...$contentTypes + ) { parent::__construct( $key, 'sp_contenttype', diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index 673d01b..71d2a6b 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -14,7 +14,7 @@ class FieldFilter extends Filter * @param string[] $values */ public function __construct( - string $key, + ?string $key, private readonly string $field, string ...$values ) { diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php index 6d26936..4ae4e88 100644 --- a/src/Dto/Search/Query/Filter/Filter.php +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -10,13 +10,13 @@ class Filter * @param string[] $tags */ public function __construct( - private readonly string $key, + private readonly ?string $key, private readonly string $query, private readonly array $tags = [] ) { } - public function getKey(): string + public function getKey(): ?string { return $this->key; } diff --git a/src/Dto/Search/Query/Filter/GroupFilter.php b/src/Dto/Search/Query/Filter/GroupFilter.php index e5d22e6..4c0b558 100644 --- a/src/Dto/Search/Query/Filter/GroupFilter.php +++ b/src/Dto/Search/Query/Filter/GroupFilter.php @@ -6,8 +6,10 @@ class GroupFilter extends FieldFilter { - public function __construct(string $key, string ...$group) - { + public function __construct( + ?string $key, + string ...$group + ) { parent::__construct( $key, 'sp_group_path', diff --git a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php index da16963..fbee75d 100644 --- a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php @@ -6,8 +6,10 @@ class ObjectTypeFilter extends FieldFilter { - public function __construct(string $key, string ...$objectTypes) - { + public function __construct( + ?string $key, + string ...$objectTypes + ) { parent::__construct( $key, 'sp_objecttype', diff --git a/src/Dto/Search/Query/Filter/SiteFilter.php b/src/Dto/Search/Query/Filter/SiteFilter.php index 4fa4565..565a494 100644 --- a/src/Dto/Search/Query/Filter/SiteFilter.php +++ b/src/Dto/Search/Query/Filter/SiteFilter.php @@ -6,8 +6,10 @@ class SiteFilter extends FieldFilter { - public function __construct(string $key, string ...$site) - { + public function __construct( + ?string $key, + string ...$site + ) { parent::__construct( $key, 'sp_site', diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php index 144c168..8998bdf 100644 --- a/src/Dto/Search/Query/SuggestQuery.php +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -9,28 +9,27 @@ class SuggestQuery { /** - * @param string[] $termList - * @param Filter[] $filterList + * @param Filter[] $filter */ public function __construct( - private readonly string $core, - private readonly array $termList, - private readonly array $filterList = [], + private readonly string $index, + private readonly string $text, + private readonly array $filter = [], private readonly int $limit = 10, private readonly string $field = 'raw_content' ) { } - public function getCore(): string + public function getIndex(): string { - return $this->core; + return $this->index; } /** * @return string[] */ - public function getTermList(): array + public function getText(): string { - return $this->termList; + return $this->text; } public function getLimit(): int @@ -41,9 +40,9 @@ public function getLimit(): int /** * @return Filter[] */ - public function getFilterList(): array + public function getFilter(): array { - return $this->filterList; + return $this->filter; } public function getField(): string { diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php index 9b4770a..6408c77 100644 --- a/src/Dto/Search/Result/SuggestResult.php +++ b/src/Dto/Search/Result/SuggestResult.php @@ -23,6 +23,14 @@ public function getIterator(): ArrayIterator return new ArrayIterator($this->suggestions); } + /** + * @return Suggestion[] + */ + public function getSuggestions(): array + { + return $this->suggestions; + } + public function getQueryTime(): int { return $this->queryTime; diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 17b9a48..6efe272 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -30,7 +30,7 @@ public function __construct( */ public function suggest(SuggestQuery $query): SuggestResult { - $client = $this->clientFactory->create($query->getCore()); + $client = $this->clientFactory->create($query->getIndex()); $solrQuery = $this->buildSolrQuery($client, $query); $solrResult = $client->select($solrQuery); @@ -56,7 +56,7 @@ private function buildSolrQuery( $solrQuery->addParam("facet.method", "enum"); $solrQuery->addParam( "facet.prefix", - implode(' ', $query->getTermList()) + $query->getText() ); $solrQuery->addParam("facet.limit", $query->getLimit()); $solrQuery->addParam("facet.field", $query->getField()); @@ -66,7 +66,7 @@ private function buildSolrQuery( $solrQuery->setRows(0); // Filter - foreach ($query->getFilterList() as $filter) { + foreach ($query->getFilter() as $filter) { $solrQuery->createFilterQuery($filter->getKey()) ->setQuery($filter->getQuery()) ->setTags($filter->getTags()); From 124de81497f8e12b053839e7267e5b2b41018e82 Mon Sep 17 00:00:00 2001 From: veltrup Date: Mon, 4 Dec 2023 08:44:10 +0100 Subject: [PATCH 004/145] fix: only url field required --- src/Service/Search/SolrSelect.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 2950abc..cc9ed8f 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -83,12 +83,7 @@ private function buildSolrQuery( private function addRequiredFieldListToSolrQuery( SolrSelectQuery $solrQuery ): void { - $fields = $solrQuery->getFields(); - if (in_array('url', $fields, true)) { - return; - } - $fields[] = 'url'; - $solrQuery->setFields($fields); + $solrQuery->setFields(['url']); } private function addTextFilterToSolrQuery( From 68352eb2ec20019e75517c7b225529f9e951b716 Mon Sep 17 00:00:00 2001 From: veltrup Date: Thu, 7 Dec 2023 09:42:20 +0100 Subject: [PATCH 005/145] feat: add background indexer --- composer.json | 29 +++++-- src/Console/Command/Indexer.php | 59 ++++++++----- src/Console/Command/MoreLikeThis.php | 23 ++++- src/Console/Command/Search.php | 18 +++- src/Console/Command/Suggest.php | 14 +++- src/Dto/Indexer/IndexerParameter.php | 3 +- src/Indexer.php | 5 +- src/Service/Indexer/BackgroundIndexer.php | 82 ++++++++++++++++++ .../BackgroundIndexerProgressState.php | 59 +++++++++++++ .../Indexer/BackgroundIndexerStatus.php | 83 +++++++++++++++++++ src/Service/Indexer/SolrIndexer.php | 20 +++-- src/Service/Search/SolrSelect.php | 3 +- src/Service/SolrParameterClientFactory.php | 22 +++-- 13 files changed, 368 insertions(+), 52 deletions(-) create mode 100644 src/Service/Indexer/BackgroundIndexer.php create mode 100644 src/Service/Indexer/BackgroundIndexerProgressState.php create mode 100644 src/Service/Indexer/BackgroundIndexerStatus.php diff --git a/composer.json b/composer.json index cb78212..aef39f8 100644 --- a/composer.json +++ b/composer.json @@ -21,14 +21,15 @@ "prefer-stable": true, "require": { "php": ">=8.1 <8.4.0", - "atoolo/resource": "dev-feature/hierarchy-loader", + "atoolo/resource": "dev-feature/resource-base-locator", "solarium/solarium": "^6.3", - "symfony/config": "^6.3", - "symfony/console": "^6.3", - "symfony/dependency-injection": "^6.3", - "symfony/event-dispatcher": "^6.3", - "symfony/finder": "^6.3", - "symfony/yaml": "^6.3" + "symfony/config": "^6.3 | ^7.0", + "symfony/console": "^6.3 | ^7.0", + "symfony/dependency-injection": "^6.3 | ^7.0", + "symfony/event-dispatcher": "^6.3 | ^7.0", + "symfony/finder": "^6.3 | ^7.0", + "symfony/yaml": "^6.3 | ^7.0", + "symfony/lock": "^6.3 | ^7.0" }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^1.0", @@ -77,5 +78,19 @@ "*": "dist" }, "sort-packages": true + }, + "extra": { + "composer-link": { + "atoolo/resource": { + "dev": false, + "version": "dev-feature/hierarchy-loader" + } + } + }, + "repositories": { + "atoolo/resource": { + "type": "path", + "url": "/home/veltrup/.cache/composer/link/atoolo/resource" + } } } diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 491a400..8629b38 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -7,6 +7,7 @@ use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; +use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema21DocumentEnricher; @@ -28,12 +29,19 @@ class Indexer extends Command { private IndexerProgressProgressBar $progressBar; private SymfonyStyle $io; + + private InputInterface $input; private string $resourceDir; protected function configure(): void { $this ->setHelp('Command to fill a search index') + ->addArgument( + 'solr-connection-url', + InputArgument::REQUIRED, + 'Solr connection url.' + ) ->addArgument( 'solr-core', InputArgument::REQUIRED, @@ -66,16 +74,14 @@ protected function execute( OutputInterface $output ): int { + $this->input = $input; $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); - $this->resourceDir = $this->getStringArgument( - $input, - 'resource-dir' - ); + $this->resourceDir = $this->getStringArgument('resource-dir'); $directories = (array)$input->getArgument('directories'); $cleanupThreshold = empty($directories) - ? $this->getIntArgument($input, 'cleanup-threshold') + ? $this->getIntArgument('cleanup-threshold', 0) : 0; if (empty($directories)) { @@ -86,8 +92,7 @@ protected function execute( } $parameter = new IndexerParameter( - $this->getStringArgument($input, 'solr-core'), - $this->resourceDir, + $this->getStringArgument('solr-core'), $cleanupThreshold, $directories ); @@ -100,25 +105,23 @@ protected function execute( return Command::SUCCESS; } - private function getStringArgument( - InputInterface $input, - string $name - ): string { - $value = $input->getArgument($name); + private function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); if (!is_string($value)) { throw new InvalidArgumentException( $name . ' must be a string' ); } - return strval($value); - return (string)$value; + return $value; } - private function getIntArgument( - InputInterface $input, - string $name - ): int { - $value = $input->getArgument($name); + private function getIntArgument(string $name, int $default): int + { + if (!$this->input->hasArgument($name)) { + return $default; + } + $value = $this->input->getArgument($name); if (!is_int($value)) { throw new InvalidArgumentException( $name . ' must be a integer' @@ -143,7 +146,10 @@ protected function errorReport(): void protected function createIndexer(): SolrIndexer { - $resourceLoader = new SiteKitLoader($this->resourceDir); + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); $navigationLoader = new SiteKitNavigationHierarchyLoader( $resourceLoader ); @@ -151,10 +157,21 @@ protected function createIndexer(): SolrIndexer $navigationLoader ); - $clientFactory = new SolrParameterClientFactory(); + $url = parse_url($this->getStringArgument('solr-connection-url')); + + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382), + $url['path'] ?? '', + null, + 0 + ); + return new SolrIndexer( [$schema21], $this->progressBar, + $resourceBaseLocator, $resourceLoader, $clientFactory, 'internal' diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index ab8cc33..475a3fb 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; +use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; use Atoolo\Search\Service\Search\ExternalResourceFactory; @@ -28,6 +29,7 @@ class MoreLikeThis extends Command { private SymfonyStyle $io; + private InputInterface $input; private string $solrCore; private string $resourceDir; @@ -35,6 +37,11 @@ protected function configure(): void { $this ->setHelp('Command to performs a more-like-this search') + ->addArgument( + 'solr-connection-url', + InputArgument::REQUIRED, + 'Solr connection url.' + ) ->addArgument( 'solr-core', InputArgument::REQUIRED, @@ -58,6 +65,7 @@ protected function execute( OutputInterface $output ): int { + $this->input = $input; $this->io = new SymfonyStyle($input, $output); $this->solrCore = $input->getArgument('solr-core'); @@ -74,8 +82,19 @@ protected function execute( protected function createSearcher(): SolrMoreLikeThis { - $resourceLoader = new SiteKitLoader($this->resourceDir); - $clientFactory = new SolrParameterClientFactory(); + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + $url = parse_url($this->input->getArgument('solr-connection-url')); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + $url['path'] ?? '', + null, + 0 + ); $resourceFactoryList = [ new ExternalResourceFactory(), new InternalResourceFactory($resourceLoader), diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index cf8b0e2..49fd4fe 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; +use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; use Atoolo\Search\Service\Search\ExternalResourceFactory; @@ -28,6 +29,7 @@ class Search extends Command { private SymfonyStyle $io; + private InputInterface $input; private string $index; private string $resourceDir; @@ -59,6 +61,7 @@ protected function execute( OutputInterface $output ): int { + $this->input = $input; $this->io = new SymfonyStyle($input, $output); $this->resourceDir = $input->getArgument('resource-dir'); $this->index = $input->getArgument('index'); @@ -75,8 +78,19 @@ protected function execute( protected function createSearch(): SolrSelect { - $resourceLoader = new SiteKitLoader($this->resourceDir); - $clientFactory = new SolrParameterClientFactory(); + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + $url = parse_url($this->input->getArgument('solr-connection-url')); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + $url['path'] ?? '', + null, + 0 + ); $defaultBoosting = new DefaultBoostModifier(); $resourceFactoryList = [ diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index a431467..e40638b 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -23,6 +23,7 @@ )] class Suggest extends Command { + private InputInterface $input; private SymfonyStyle $io; private string $solrCore; @@ -47,7 +48,7 @@ protected function execute( InputInterface $input, OutputInterface $output ): int { - + $this->input = $input; $this->io = new SymfonyStyle($input, $output); $this->solrCore = $input->getArgument('solr-core'); $terms = $input->getArgument('terms'); @@ -65,6 +66,15 @@ protected function execute( protected function createSearcher(): SolrSuggest { $clientFactory = new SolrParameterClientFactory(); + $url = parse_url($this->input->getArgument('solr-connection-url')); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + $url['path'] ?? '', + null, + 0 + ); return new SolrSuggest($clientFactory); } @@ -74,7 +84,7 @@ protected function buildQuery(array $terms): SuggestQuery $excludeMedia = $excludeMedia->exclude(); return new SuggestQuery( $this->solrCore, - $terms, + implode(' ', $terms), [ new ArchiveFilter(), $excludeMedia diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index b1ce85e..e343f40 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -7,8 +7,7 @@ class IndexerParameter { public function __construct( - public readonly string $coreId, - public readonly string $basePath, + public readonly string $index, public readonly int $cleanupThreshold = 0, public readonly array $directories = [] ) { diff --git a/src/Indexer.php b/src/Indexer.php index 55d4f58..54a38ff 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -17,5 +17,8 @@ */ interface Indexer { - public function index(IndexerParameter $parameter): void; + /** + * @return string process id + */ + public function index(IndexerParameter $parameter): string; } diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php new file mode 100644 index 0000000..52c9465 --- /dev/null +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -0,0 +1,82 @@ + $documentEnricherList + */ + public function __construct( + private readonly iterable $documentEnricherList, + private readonly ResourceBaseLocator $resourceBaseLocator, + private readonly ResourceLoader $resourceLoader, + private readonly SolrClientFactory $clientFactory, + private readonly string $source, + private readonly string $statusCacheDir + ) { + $this->lockFactory = new LockFactory(new SemaphoreStore()); + if ( + !is_dir($concurrentDirectory = $this->statusCacheDir) && + !mkdir($concurrentDirectory) && + !is_dir($concurrentDirectory) + ) { + throw new \RuntimeException(sprintf( + 'Directory "%s" was not created', + $concurrentDirectory + )); + } + } + + public function index(IndexerParameter $parameter): string + { + $lock = $this->lockFactory->createLock($parameter->index); + if (!$lock->acquire()) { + return ''; + } + try { + return $this->getIndexer($parameter->index)->index($parameter); + } finally { + $lock->release(); + } + } + + public function getStatus(string $index): ?BackgroundIndexerStatus + { + $file = $this->getStatusFile($index); + return BackgroundIndexerStatus::load($file); + } + + private function getIndexer(string $index): SolrIndexer + { + $progressHandler = new BackgroundIndexerProgressState( + $this->getStatusFile($index) + ); + return new SolrIndexer( + $this->documentEnricherList, + $progressHandler, + $this->resourceBaseLocator, + $this->resourceLoader, + $this->clientFactory, + $this->source, + ); + } + + private function getStatusFile(string $index): string + { + return $this->statusCacheDir . + '/atoolo.search.index.' . $index . ".status.json"; + } +} diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php new file mode 100644 index 0000000..a500ab9 --- /dev/null +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -0,0 +1,59 @@ +status = new BackgroundIndexerStatus( + new \DateTime(), + null, + $total, + 0, + 0 + ); + } + + public function advance(int $step): void + { + $this->status->processed += $step; + $this->status->store($this->file); + } + + public function error(Exception $exception): void + { + $this->status->errors++; + } + + public function finish(): void + { + $this->status->endTime = new DateTime(); + $this->status->store($this->file); + } + + /** + * @return array + */ + public function getErrors(): array + { + return []; + } + + private function getStatusLine(): string + { + return $this->status->getStatusLine(); + } +} diff --git a/src/Service/Indexer/BackgroundIndexerStatus.php b/src/Service/Indexer/BackgroundIndexerStatus.php new file mode 100644 index 0000000..a14fe89 --- /dev/null +++ b/src/Service/Indexer/BackgroundIndexerStatus.php @@ -0,0 +1,83 @@ +endTime; + if ($endTime === null) { + $endTime = new DateTime(); + } + $duration = $this->startTime->diff($endTime); + return + 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . + 'processed: ' . $this->processed . "/" . $this->total . ', ' . + 'errors: ' . $this->errors; + } + + /** + * @throws \JsonException + */ + public static function load(string $file): ?BackgroundIndexerStatus + { + if (!file_exists($file)) { + return null; + } + $content = file_get_contents($file); + $data = json_decode( + $content, + true, + 512, + JSON_THROW_ON_ERROR + ); + + $startTime = new DateTime(); + $startTime->setTimestamp($data['startTime']); + + $endTime = null; + if ($data['endTime'] !== null) { + $endTime = new DateTime(); + $endTime->setTimestamp($data['endTime']); + } + + return new BackgroundIndexerStatus( + $startTime, + $endTime, + $data['total'], + $data['processed'], + $data['errors'] + ); + } + + /** + * @throws \JsonException + */ + public function store(string $file): void + { + $jsonString = json_encode([ + 'statusline' => $this->getStatusLine(), + 'startTime' => $this->startTime->getTimestamp(), + 'endTime' => $this->endTime?->getTimestamp(), + 'total' => $this->total, + 'processed' => $this->processed, + 'errors' => $this->errors + ], JSON_THROW_ON_ERROR); + file_put_contents($file, $jsonString); + } +} diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 4bf37ee..c9888c2 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Indexer; @@ -25,21 +26,22 @@ class SolrIndexer implements Indexer public function __construct( private readonly iterable $documentEnricherList, private readonly IndexerProgressHandler $indexerProgressHandler, + private readonly ResourceBaseLocator $resourceBaseLocator, private readonly ResourceLoader $resourceLoader, private readonly SolrClientFactory $clientFactory, private readonly string $source ) { } - public function index(IndexerParameter $parameter): void + public function index(IndexerParameter $parameter): string { - $finder = new LocationFinder($parameter->basePath); + $finder = new LocationFinder($this->resourceBaseLocator->locate()); if (empty($parameter->directories)) { $pathList = $finder->findAll(); } else { $pathList = $finder->findInSubdirectories($parameter->directories); } - $this->indexResources($parameter, $pathList); + return $this->indexResources($parameter, $pathList); } /** @@ -48,9 +50,9 @@ public function index(IndexerParameter $parameter): void private function indexResources( IndexerParameter $parameter, array $pathList - ): void { + ): string { if (count($pathList) === 0) { - return; + return ''; } $total = count($pathList); @@ -65,7 +67,7 @@ private function indexResources( while (true) { $indexedCount = $this->indexChunks( $processId, - $parameter->coreId, + $parameter->index, $pathList, $offset, $chunkSize @@ -81,9 +83,11 @@ private function indexResources( $parameter->cleanupThreshold > 0 && $successCount >= $parameter->cleanupThreshold ) { - $this->deleteByProcessId($parameter->coreId, $processId); + $this->deleteByProcessId($parameter->index, $processId); } - $this->commit($parameter->coreId); + $this->commit($parameter->index); + + return $processId; } finally { $this->indexerProgressHandler->finish(); } diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index cc9ed8f..d00be49 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -127,7 +127,8 @@ private function addFilterQueriesToSolrQuery( ): void { foreach ($filterList as $filter) { - $solrQuery->createFilterQuery($filter->getKey()) + $key = $filter->getKey() ?? uniqid('', true); + $solrQuery->createFilterQuery($key) ->setQuery($filter->getQuery()) ->setTags($filter->getTags()); } diff --git a/src/Service/SolrParameterClientFactory.php b/src/Service/SolrParameterClientFactory.php index 207c0b0..38902fd 100644 --- a/src/Service/SolrParameterClientFactory.php +++ b/src/Service/SolrParameterClientFactory.php @@ -16,21 +16,31 @@ */ class SolrParameterClientFactory implements SolrClientFactory { + public function __construct( + private readonly string $scheme, + private readonly string $host, + private readonly int $port, + private readonly string $path = '', + private readonly ?string $proxy = null, + private readonly ?int $timeout = 0 + ) { + } + public function create(string $core): Client { $host = 'solr-neu-isenburg-whinchat.veltrup.sitepark.de'; $adapter = new Curl(); - $adapter->setTimeout(30); - //$adapter->setProxy('http://localhost:8889'); + $adapter->setTimeout($this->timeout); + $adapter->setProxy($this->proxy); $eventDispatcher = new EventDispatcher(); $config = [ 'endpoint' => [ $host => [ - 'scheme' => 'https', - 'host' => $host, - 'port' => 443, - 'path' => '', + 'scheme' => $this->scheme, + 'host' => $this->host, + 'port' => $this->port, + 'path' => $this->path, 'core' => $core, ] ] From 97b92c7bbede10011dbb1231e20b5658a094fa02 Mon Sep 17 00:00:00 2001 From: veltrup Date: Mon, 11 Dec 2023 09:32:10 +0100 Subject: [PATCH 006/145] feat: indexer extended --- composer.json | 2 +- .../Command/Io/IndexerProgressProgressBar.php | 26 ++++++++++++++--- .../Indexer/IndexerStatus.php} | 29 ++++++++++++++----- src/Indexer.php | 3 +- src/Service/Indexer/BackgroundIndexer.php | 16 ++++++---- .../BackgroundIndexerProgressState.php | 25 ++++++++++------ .../Indexer/IndexerProgressHandler.php | 7 +++-- .../DefaultSchema21DocumentEnricher.php | 3 ++ src/Service/Indexer/SolrIndexer.php | 11 ++++--- 9 files changed, 87 insertions(+), 35 deletions(-) rename src/{Service/Indexer/BackgroundIndexerStatus.php => Dto/Indexer/IndexerStatus.php} (77%) diff --git a/composer.json b/composer.json index aef39f8..0785604 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,7 @@ "prefer-stable": true, "require": { "php": ">=8.1 <8.4.0", - "atoolo/resource": "dev-feature/resource-base-locator", + "atoolo/resource": "dev-main", "solarium/solarium": "^6.3", "symfony/config": "^6.3 | ^7.0", "symfony/console": "^6.3 | ^7.0", diff --git a/src/Console/Command/Io/IndexerProgressProgressBar.php b/src/Console/Command/Io/IndexerProgressProgressBar.php index 13b156c..0ddff5a 100644 --- a/src/Console/Command/Io/IndexerProgressProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressProgressBar.php @@ -4,8 +4,10 @@ namespace Atoolo\Search\Console\Command\Io; +use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; -use Exception; +use DateTime; +use Throwable; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\OutputInterface; @@ -13,6 +15,7 @@ class IndexerProgressProgressBar implements IndexerProgressHandler { private OutputInterface $output; private ProgressBar $progressBar; + private IndexerStatus $status; private array $errors = []; @@ -25,11 +28,19 @@ public function start(int $total): void { $this->progressBar = new ProgressBar($this->output, $total); $this->formatProgressBar('green'); + $this->status = new IndexerStatus( + new DateTime(), + null, + $total, + 0, + 0 + ); } public function advance(int $step): void { $this->progressBar->advance($step); + $this->status->processed += $step; } private function formatProgressBar(string $color): void @@ -43,22 +54,29 @@ private function formatProgressBar(string $color): void ); } - public function error(Exception $exception): void + public function error(Throwable $throwable): void { $this->formatProgressBar('red'); - $this->errors[] = $exception; + $this->errors[] = $throwable; + $this->status->errors++; } public function finish(): void { $this->progressBar->finish(); + $this->status->endTime = new DateTime(); } /** - * @return array + * @return array */ public function getErrors(): array { return $this->errors; } + + public function getStatus(): IndexerStatus + { + return $this->status; + } } diff --git a/src/Service/Indexer/BackgroundIndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php similarity index 77% rename from src/Service/Indexer/BackgroundIndexerStatus.php rename to src/Dto/Indexer/IndexerStatus.php index a14fe89..599444f 100644 --- a/src/Service/Indexer/BackgroundIndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -2,11 +2,12 @@ declare(strict_types=1); -namespace Atoolo\Search\Service\Indexer; +namespace Atoolo\Search\Dto\Indexer; use DateTime; +use JsonException; -class BackgroundIndexerStatus +class IndexerStatus { public function __construct( public readonly DateTime $startTime, @@ -17,6 +18,18 @@ public function __construct( ) { } + public static function empty(): IndexerStatus + { + $now = new DateTime(); + return new IndexerStatus( + $now, + $now, + 0, + 0, + 0 + ); + } + public function getStatusLine(): string { $endTime = $this->endTime; @@ -32,12 +45,12 @@ public function getStatusLine(): string } /** - * @throws \JsonException + * @throws JsonException */ - public static function load(string $file): ?BackgroundIndexerStatus + public static function load(string $file): IndexerStatus { if (!file_exists($file)) { - return null; + return self::empty(); } $content = file_get_contents($file); $data = json_decode( @@ -56,7 +69,7 @@ public static function load(string $file): ?BackgroundIndexerStatus $endTime->setTimestamp($data['endTime']); } - return new BackgroundIndexerStatus( + return new IndexerStatus( $startTime, $endTime, $data['total'], @@ -66,12 +79,12 @@ public static function load(string $file): ?BackgroundIndexerStatus } /** - * @throws \JsonException + * @throws JsonException */ public function store(string $file): void { $jsonString = json_encode([ - 'statusline' => $this->getStatusLine(), + 'statusLine' => $this->getStatusLine(), 'startTime' => $this->startTime->getTimestamp(), 'endTime' => $this->endTime?->getTimestamp(), 'total' => $this->total, diff --git a/src/Indexer.php b/src/Indexer.php index 54a38ff..279eb44 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -5,6 +5,7 @@ namespace Atoolo\Search; use Atoolo\Search\Dto\Indexer\IndexerParameter; +use Atoolo\Search\Dto\Indexer\IndexerStatus; /** * The service interface for indexing a search index. @@ -20,5 +21,5 @@ interface Indexer /** * @return string process id */ - public function index(IndexerParameter $parameter): string; + public function index(IndexerParameter $parameter): IndexerStatus; } diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 52c9465..ddc69ff 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -7,8 +7,11 @@ use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; +use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; +use RuntimeException; +use JsonException; use Symfony\Component\Lock\LockFactory; use Symfony\Component\Lock\Store\SemaphoreStore; @@ -33,18 +36,18 @@ public function __construct( !mkdir($concurrentDirectory) && !is_dir($concurrentDirectory) ) { - throw new \RuntimeException(sprintf( + throw new RuntimeException(sprintf( 'Directory "%s" was not created', $concurrentDirectory )); } } - public function index(IndexerParameter $parameter): string + public function index(IndexerParameter $parameter): IndexerStatus { $lock = $this->lockFactory->createLock($parameter->index); if (!$lock->acquire()) { - return ''; + return IndexerStatus::empty(); } try { return $this->getIndexer($parameter->index)->index($parameter); @@ -53,10 +56,13 @@ public function index(IndexerParameter $parameter): string } } - public function getStatus(string $index): ?BackgroundIndexerStatus + /** + * @throws JsonException + */ + public function getStatus(string $index): IndexerStatus { $file = $this->getStatusFile($index); - return BackgroundIndexerStatus::load($file); + return IndexerStatus::load($file); } private function getIndexer(string $index): SolrIndexer diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index a500ab9..5d1ed74 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -4,13 +4,14 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Search\Dto\Indexer\IndexerStatus; use DateTime; -use Exception; -use SP\Util\Date; +use Throwable; +use JsonException; class BackgroundIndexerProgressState implements IndexerProgressHandler { - private BackgroundIndexerStatus $status; + private IndexerStatus $status; public function __construct(private readonly string $file) { @@ -18,8 +19,8 @@ public function __construct(private readonly string $file) public function start(int $total): void { - $this->status = new BackgroundIndexerStatus( - new \DateTime(), + $this->status = new IndexerStatus( + new DateTime(), null, $total, 0, @@ -27,17 +28,23 @@ public function start(int $total): void ); } + /** + * @throws JsonException + */ public function advance(int $step): void { $this->status->processed += $step; $this->status->store($this->file); } - public function error(Exception $exception): void + public function error(Throwable $throwable): void { $this->status->errors++; } + /** + * @throws JsonException + */ public function finish(): void { $this->status->endTime = new DateTime(); @@ -45,15 +52,15 @@ public function finish(): void } /** - * @return array + * @return array */ public function getErrors(): array { return []; } - private function getStatusLine(): string + public function getStatus(): IndexerStatus { - return $this->status->getStatusLine(); + return $this->status; } } diff --git a/src/Service/Indexer/IndexerProgressHandler.php b/src/Service/Indexer/IndexerProgressHandler.php index 0e65499..ef05652 100644 --- a/src/Service/Indexer/IndexerProgressHandler.php +++ b/src/Service/Indexer/IndexerProgressHandler.php @@ -4,13 +4,14 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Resource\Resource; -use Solarium\QueryType\Update\Result; +use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Throwable; interface IndexerProgressHandler { public function start(int $total): void; public function advance(int $step): void; - public function error(\Exception $exception): void; + public function error(Throwable $throwable): void; public function finish(): void; + public function getStatus(): IndexerStatus; } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php index 129e367..79a3166 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php @@ -70,6 +70,9 @@ public function enrichDocument( $doc->sp_changed = $this->toDateTime( $resource->getData('init.changed') ); + $doc->sp_generated = $this->toDateTime( + $resource->getData('init.generated') + ); $doc->sp_date = $this->toDateTime( $resource->getData('base.date') ); diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index c9888c2..2cb95c6 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -9,6 +9,8 @@ use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; +use Atoolo\Search\Dto\Indexer\IndexerResult; +use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; use Exception; @@ -33,7 +35,7 @@ public function __construct( ) { } - public function index(IndexerParameter $parameter): string + public function index(IndexerParameter $parameter): IndexerStatus { $finder = new LocationFinder($this->resourceBaseLocator->locate()); if (empty($parameter->directories)) { @@ -50,9 +52,9 @@ public function index(IndexerParameter $parameter): string private function indexResources( IndexerParameter $parameter, array $pathList - ): string { + ): IndexerStatus { if (count($pathList) === 0) { - return ''; + return IndexerStatus::empty(); } $total = count($pathList); @@ -87,10 +89,11 @@ private function indexResources( } $this->commit($parameter->index); - return $processId; } finally { $this->indexerProgressHandler->finish(); } + + return $this->indexerProgressHandler->getStatus(); } /** From 2b7fcc927ef9ae89055718cd9400493ccd885d8f Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 13 Dec 2023 13:33:20 +0100 Subject: [PATCH 007/145] refactor: indexing by paths, aborter --- src/Console/Command/Indexer.php | 24 ++++---- .../Command/Io/IndexerProgressProgressBar.php | 22 +++++++ src/Dto/Indexer/IndexerParameter.php | 2 +- src/Dto/Indexer/IndexerStatus.php | 38 ++++++++++++- src/Dto/Indexer/IndexerStatusState.php | 25 ++++++++ src/Indexer.php | 4 ++ src/Service/Indexer/BackgroundIndexer.php | 14 ++++- .../BackgroundIndexerProgressState.php | 46 ++++++++++++++- src/Service/Indexer/DocumentEnricher.php | 2 + .../Indexer/IndexerProgressHandler.php | 4 ++ src/Service/Indexer/IndexingAborter.php | 35 ++++++++++++ src/Service/Indexer/LocationFinder.php | 22 ++++++- .../DefaultSchema21DocumentEnricher.php | 5 ++ src/Service/Indexer/SolrIndexer.php | 57 ++++++++++++++++--- 14 files changed, 275 insertions(+), 25 deletions(-) create mode 100644 src/Dto/Indexer/IndexerStatusState.php create mode 100644 src/Service/Indexer/IndexingAborter.php diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 8629b38..44290d9 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -10,10 +10,11 @@ use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; use Atoolo\Search\Dto\Indexer\IndexerParameter; +use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema21DocumentEnricher; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\SolrParameterClientFactory; -use http\Exception\InvalidArgumentException; +use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -53,9 +54,9 @@ protected function configure(): void 'Resource directory whose data is to be indexed.' ) ->addArgument( - 'directories', + 'paths', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, - 'Resources or directories of the resource to be indexed.' + 'Resources paths or directories of resources to be indexed.' ) ->addOption( 'cleanup-threshold', @@ -78,23 +79,23 @@ protected function execute( $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); $this->resourceDir = $this->getStringArgument('resource-dir'); - $directories = (array)$input->getArgument('directories'); + $paths = (array)$input->getArgument('paths'); - $cleanupThreshold = empty($directories) + $cleanupThreshold = empty($paths) ? $this->getIntArgument('cleanup-threshold', 0) : 0; - if (empty($directories)) { + if (empty($paths)) { $this->io->title('Index all resources'); } else { - $this->io->title('Index resources subdirectories'); - $this->io->listing($directories); + $this->io->title('Index resource paths'); + $this->io->listing($paths); } $parameter = new IndexerParameter( $this->getStringArgument('solr-core'), $cleanupThreshold, - $directories + $paths ); $indexer = $this->createIndexer(); @@ -127,7 +128,7 @@ private function getIntArgument(string $name, int $default): int $name . ' must be a integer' ); } - return (int)$value; + return $value; } protected function errorReport(): void @@ -168,12 +169,15 @@ protected function createIndexer(): SolrIndexer 0 ); + $aborter = new IndexingAborter('.'); + return new SolrIndexer( [$schema21], $this->progressBar, $resourceBaseLocator, $resourceLoader, $clientFactory, + $aborter, 'internal' ); } diff --git a/src/Console/Command/Io/IndexerProgressProgressBar.php b/src/Console/Command/Io/IndexerProgressProgressBar.php index 0ddff5a..a7214b0 100644 --- a/src/Console/Command/Io/IndexerProgressProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressProgressBar.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Console\Command\Io; use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use DateTime; use Throwable; @@ -29,20 +30,33 @@ public function start(int $total): void $this->progressBar = new ProgressBar($this->output, $total); $this->formatProgressBar('green'); $this->status = new IndexerStatus( + IndexerStatusState::RUNNING, new DateTime(), null, $total, 0, + 0, + new DateTime(), + 0, 0 ); } + public function startUpdate(int $total): void + { + $this->start($total); + } + public function advance(int $step): void { $this->progressBar->advance($step); $this->status->processed += $step; } + public function skip(int $step): void + { + } + private function formatProgressBar(string $color): void { $this->progressBar->setBarCharacter('•'); @@ -64,6 +78,7 @@ public function error(Throwable $throwable): void public function finish(): void { $this->progressBar->finish(); + $this->status->state = IndexerStatusState::INDEXED; $this->status->endTime = new DateTime(); } @@ -79,4 +94,11 @@ public function getStatus(): IndexerStatus { return $this->status; } + + public function abort(): void + { + $this->progressBar->finish(); + $this->status->state = IndexerStatusState::ABORTED; + $this->status->endTime = new DateTime(); + } } diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index e343f40..7744fda 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -9,7 +9,7 @@ class IndexerParameter public function __construct( public readonly string $index, public readonly int $cleanupThreshold = 0, - public readonly array $directories = [] + public readonly array $paths = [] ) { } } diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php index 599444f..414d83d 100644 --- a/src/Dto/Indexer/IndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -10,10 +10,14 @@ class IndexerStatus { public function __construct( + public IndexerStatusState $state, public readonly DateTime $startTime, public ?DateTime $endTime, public int $total, public int $processed, + public int $skipped, + public DateTime $lastUpdate, + public int $updated, public int $errors ) { } @@ -22,10 +26,14 @@ public static function empty(): IndexerStatus { $now = new DateTime(); return new IndexerStatus( + IndexerStatusState::UNKNOWN, $now, $now, 0, 0, + 0, + $now, + 0, 0 ); } @@ -38,9 +46,13 @@ public function getStatusLine(): string } $duration = $this->startTime->diff($endTime); return + '[' . $this->state->name . '] ' . 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . 'processed: ' . $this->processed . "/" . $this->total . ', ' . + 'skipped: ' . $this->skipped . ', ' . + 'lastUpdate: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'updated: ' . $this->updated . ', ' . 'errors: ' . $this->errors; } @@ -60,21 +72,37 @@ public static function load(string $file): IndexerStatus JSON_THROW_ON_ERROR ); + $state = isset($data['state']) + ? IndexerStatusState::valueOf($data['state']) + : IndexerStatusState::UNKNOWN; + $startTime = new DateTime(); $startTime->setTimestamp($data['startTime']); - $endTime = null; + $endTime = new DateTime(); if ($data['endTime'] !== null) { - $endTime = new DateTime(); $endTime->setTimestamp($data['endTime']); + } else { + $endTime->setTimestamp(0); + } + + $lastUpdate = new DateTime(); + if ($data['lastUpdate'] !== null) { + $lastUpdate->setTimestamp($data['lastUpdate']); + } else { + $lastUpdate->setTimestamp(0); } return new IndexerStatus( + $state, $startTime, $endTime, $data['total'], $data['processed'], - $data['errors'] + $data['skipped'] ?? 0, + $lastUpdate, + $data['updated'] ?? 0, + $data['errors'] ?? 0 ); } @@ -84,11 +112,15 @@ public static function load(string $file): IndexerStatus public function store(string $file): void { $jsonString = json_encode([ + 'state' => $this->state->name, 'statusLine' => $this->getStatusLine(), 'startTime' => $this->startTime->getTimestamp(), 'endTime' => $this->endTime?->getTimestamp(), 'total' => $this->total, 'processed' => $this->processed, + 'skipped' => $this->skipped, + 'lastUpdate' => $this->lastUpdate->getTimestamp(), + 'updated' => $this->updated, 'errors' => $this->errors ], JSON_THROW_ON_ERROR); file_put_contents($file, $jsonString); diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php new file mode 100644 index 0000000..48f1964 --- /dev/null +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -0,0 +1,25 @@ +name) { + return $status; + } + } + throw new \ValueError( + "$name is not a valid backing value for enum " . self::class + ); + } +} diff --git a/src/Indexer.php b/src/Indexer.php index 279eb44..2308899 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -22,4 +22,8 @@ interface Indexer * @return string process id */ public function index(IndexerParameter $parameter): IndexerStatus; + + public function abort($index): void; + + public function remove(string $index, string $id): void; } diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index ddc69ff..60d1e7b 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -27,6 +27,7 @@ public function __construct( private readonly ResourceBaseLocator $resourceBaseLocator, private readonly ResourceLoader $resourceLoader, private readonly SolrClientFactory $clientFactory, + private readonly IndexingAborter $aborter, private readonly string $source, private readonly string $statusCacheDir ) { @@ -43,6 +44,16 @@ public function __construct( } } + public function remove(string $index, string $id): void + { + $this->getIndexer($index)->remove($index, $id); + } + + public function abort($index): void + { + $this->getIndexer($index)->abort($index); + } + public function index(IndexerParameter $parameter): IndexerStatus { $lock = $this->lockFactory->createLock($parameter->index); @@ -76,7 +87,8 @@ private function getIndexer(string $index): SolrIndexer $this->resourceBaseLocator, $this->resourceLoader, $this->clientFactory, - $this->source, + $this->aborter, + $this->source ); } diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index 5d1ed74..f3cb3d5 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Atoolo\Search\Dto\Indexer\IndexerStatusState; use DateTime; use Throwable; use JsonException; @@ -13,6 +14,8 @@ class BackgroundIndexerProgressState implements IndexerProgressHandler { private IndexerStatus $status; + private bool $isUpdate = false; + public function __construct(private readonly string $file) { } @@ -20,23 +23,54 @@ public function __construct(private readonly string $file) public function start(int $total): void { $this->status = new IndexerStatus( + IndexerStatusState::RUNNING, new DateTime(), null, $total, 0, + 0, + new DateTime(), + 0, 0 ); } + public function startUpdate(int $total): void + { + $this->isUpdate = true; + $storedStatus = IndexerStatus::load($this->file); + $this->status = new IndexerStatus( + IndexerStatusState::RUNNING, + $storedStatus->startTime, + $storedStatus->endTime, + $storedStatus->total + $total, + $storedStatus->processed, + $storedStatus->skipped, + new DateTime(), + $storedStatus->updated, + $storedStatus->errors, + ); + } + /** * @throws JsonException */ public function advance(int $step): void { $this->status->processed += $step; + $this->status->lastUpdate = new DateTime(); + if ($this->isUpdate) { + $this->status->updated += $step; + } $this->status->store($this->file); } + + public function skip(int $step): void + { + $this->status->skipped += $step; + } + public function error(Throwable $throwable): void { $this->status->errors++; @@ -47,10 +81,20 @@ public function error(Throwable $throwable): void */ public function finish(): void { - $this->status->endTime = new DateTime(); + if (!$this->isUpdate) { + $this->status->endTime = new DateTime(); + } + if ($this->status->state === IndexerStatusState::RUNNING) { + $this->status->state = IndexerStatusState::INDEXED; + } $this->status->store($this->file); } + public function abort(): void + { + $this->status->state = IndexerStatusState::ABORTED; + } + /** * @return array */ diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php index e360252..268db3a 100644 --- a/src/Service/Indexer/DocumentEnricher.php +++ b/src/Service/Indexer/DocumentEnricher.php @@ -13,6 +13,8 @@ */ interface DocumentEnricher { + public function isIndexable(Resource $resource): bool; + public function enrichDocument( Resource $resource, DocumentInterface $doc, diff --git a/src/Service/Indexer/IndexerProgressHandler.php b/src/Service/Indexer/IndexerProgressHandler.php index ef05652..49449e8 100644 --- a/src/Service/Indexer/IndexerProgressHandler.php +++ b/src/Service/Indexer/IndexerProgressHandler.php @@ -10,8 +10,12 @@ interface IndexerProgressHandler { public function start(int $total): void; + public function startUpdate(int $total): void; public function advance(int $step): void; + public function skip(int $step): void; public function error(Throwable $throwable): void; public function finish(): void; + public function abort(): void; + public function getStatus(): IndexerStatus; } diff --git a/src/Service/Indexer/IndexingAborter.php b/src/Service/Indexer/IndexingAborter.php new file mode 100644 index 0000000..31e7cba --- /dev/null +++ b/src/Service/Indexer/IndexingAborter.php @@ -0,0 +1,35 @@ +getAbortMarkerFile($index)); + } + + public function abort(string $index): void + { + touch($this->getAbortMarkerFile($index)); + } + + public function aborted(string $index): void + { + unlink($this->getAbortMarkerFile($index)); + } + + private function getAbortMarkerFile(string $index): string + { + return $this->workdir . '/background-indexer-' . $index . ".abort"; + } +} diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 974ad50..4ecf8d4 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -37,16 +37,34 @@ public function findAll(): array /** * @param string[] $directories */ - public function findInSubdirectories(array $directories): array + public function findPaths(array $paths): array { + $pathList = []; + + $directories = []; + $finder = new Finder(); + foreach ($paths as $path) { + $absolutePath = $this->basePath . '/' . $path; + if (is_file($absolutePath)) { + $pathList[] = $path; + continue; + } + if (is_dir($absolutePath)) { + $directories[] = $path; + } + } + + if (empty($directories)) { + return $pathList; + } + foreach ($directories as $directory) { $finder->in($this->basePath . '/' . $directory); } $finder->name('*.php'); $finder->files(); - $pathList = []; foreach ($finder as $file) { $pathList[] = $this->toRelativePath($file->getPathname()); } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php index 79a3166..873a994 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php @@ -17,6 +17,11 @@ public function __construct( ) { } + public function isIndexable(Resource $resource): bool + { + $noIndex = $resource->getData('init.noIndex'); + return $noIndex !== true; + } public function enrichDocument( Resource $resource, DocumentInterface $doc, diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 2cb95c6..259a210 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -9,7 +9,6 @@ use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; -use Atoolo\Search\Dto\Indexer\IndexerResult; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; @@ -31,22 +30,34 @@ public function __construct( private readonly ResourceBaseLocator $resourceBaseLocator, private readonly ResourceLoader $resourceLoader, private readonly SolrClientFactory $clientFactory, + private readonly IndexingAborter $aborter, private readonly string $source ) { } + public function remove(string $index, string $id): void + { + $this->deleteById($index, $id); + $this->commit($index); + } + + public function abort($index): void + { + $this->aborter->abort($index); + } + public function index(IndexerParameter $parameter): IndexerStatus { $finder = new LocationFinder($this->resourceBaseLocator->locate()); - if (empty($parameter->directories)) { + if (empty($parameter->paths)) { $pathList = $finder->findAll(); } else { - $pathList = $finder->findInSubdirectories($parameter->directories); + $pathList = $finder->findPaths($parameter->paths); } return $this->indexResources($parameter, $pathList); } - /** + /** * @param array $pathList */ private function indexResources( @@ -58,7 +69,12 @@ private function indexResources( } $total = count($pathList); - $this->indexerProgressHandler->start($total); + if (empty($parameter->paths)) { + $this->deleteErrorProtocol($parameter->index); + $this->indexerProgressHandler->start($total); + } else { + $this->indexerProgressHandler->startUpdate($total); + } $processId = uniqid('', true); $offset = 0; @@ -88,7 +104,6 @@ private function indexResources( $this->deleteByProcessId($parameter->index, $processId); } $this->commit($parameter->index); - } finally { $this->indexerProgressHandler->finish(); } @@ -114,6 +129,11 @@ private function indexChunks( if ($resourceList === false) { return false; } + if ($this->aborter->shouldAborted($solrCore)) { + $this->aborter->aborted($solrCore); + $this->indexerProgressHandler->abort(); + return false; + } $this->indexerProgressHandler->advance(count($resourceList)); $result = $this->add($solrCore, $processId, $resourceList); @@ -135,7 +155,7 @@ private function loadResources( array $pathList, int $offset, int $length - ): array | false { + ): array|false { $maxLength = (count($pathList) ?? 0) - $offset; if ($maxLength <= 0) { @@ -176,6 +196,12 @@ private function add( $documents = []; foreach ($resources as $resource) { + foreach ($this->documentEnricherList as $enricher) { + if (!$enricher->isIndexable($resource)) { + $this->indexerProgressHandler->skip(1); + continue 2; + } + } $doc = $update->createDocument(); foreach ($this->documentEnricherList as $enricher) { $doc = $enricher->enrichDocument( @@ -203,6 +229,23 @@ private function deleteByProcessId(string $core, string $processId): void ); } + private function deleteById(string $core, string $id): void + { + $this->deleteByQuery( + $core, + 'sp_id:' . $id . ' AND ' . + 'sp_source:' . $this->source + ); + } + + private function deleteErrorProtocol(string $core): void + { + $this->deleteByQuery( + $core, + 'crawl_status:error OR crawl_status:warning' + ); + } + private function deleteByQuery(string $core, string $query): void { $client = $this->clientFactory->create($core); From b49edc76e1cdd5f19af4a33d3ea0c83dcf350ea1 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 13 Dec 2023 13:37:24 +0100 Subject: [PATCH 008/145] chore: composer update --- composer.json | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index 0785604..34bff57 100644 --- a/composer.json +++ b/composer.json @@ -28,8 +28,8 @@ "symfony/dependency-injection": "^6.3 | ^7.0", "symfony/event-dispatcher": "^6.3 | ^7.0", "symfony/finder": "^6.3 | ^7.0", - "symfony/yaml": "^6.3 | ^7.0", - "symfony/lock": "^6.3 | ^7.0" + "symfony/lock": "^6.3 | ^7.0", + "symfony/yaml": "^6.3 | ^7.0" }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^1.0", @@ -79,18 +79,6 @@ }, "sort-packages": true }, - "extra": { - "composer-link": { - "atoolo/resource": { - "dev": false, - "version": "dev-feature/hierarchy-loader" - } - } - }, "repositories": { - "atoolo/resource": { - "type": "path", - "url": "/home/veltrup/.cache/composer/link/atoolo/resource" - } } } From d65b54c10c8079271f6f8ca533965212a30068ca Mon Sep 17 00:00:00 2001 From: veltrup Date: Mon, 18 Dec 2023 13:46:53 +0100 Subject: [PATCH 009/145] feat: add chunkSize parameter, remove by idList --- composer.json | 12 +++++++++++ src/Console/Command/Indexer.php | 26 +++++++++++++++++------ src/Dto/Indexer/IndexerParameter.php | 6 ++++++ src/Indexer.php | 5 ++++- src/Service/Indexer/BackgroundIndexer.php | 7 ++++-- src/Service/Indexer/LocationFinder.php | 2 +- src/Service/Indexer/SolrIndexer.php | 24 +++++++++++++++------ 7 files changed, 65 insertions(+), 17 deletions(-) diff --git a/composer.json b/composer.json index 34bff57..5e34a26 100644 --- a/composer.json +++ b/composer.json @@ -80,5 +80,17 @@ "sort-packages": true }, "repositories": { + "atoolo/resource": { + "type": "path", + "url": "/home/veltrup/.cache/composer/link/atoolo/resource" + } + }, + "extra": { + "composer-link": { + "atoolo/resource": { + "dev": false, + "version": "dev-main" + } + } } } diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 44290d9..44c9701 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -67,6 +67,16 @@ protected function configure(): void 'can be deleted. Is only used for full indexing.', 0 ) + ->addOption( + 'chunk-size', + null, + InputArgument::OPTIONAL, + 'The chunk size determines how many documents are ' . + 'indexed in an update request. The default value is 500. ' . + 'Higher values no longer have a positive effect. Smaller ' . + 'values can be selected if the memory limit is reached.', + 500 + ) ; } @@ -82,7 +92,7 @@ protected function execute( $paths = (array)$input->getArgument('paths'); $cleanupThreshold = empty($paths) - ? $this->getIntArgument('cleanup-threshold', 0) + ? $this->getIntOption('cleanup-threshold') : 0; if (empty($paths)) { @@ -95,12 +105,19 @@ protected function execute( $parameter = new IndexerParameter( $this->getStringArgument('solr-core'), $cleanupThreshold, + $this->getIntOption('chunk-size'), $paths ); $indexer = $this->createIndexer(); $indexer->index($parameter); + $this->io->text(print_r(gc_status(), true)); + gc_collect_cycles(); + $this->io->text(print_r(gc_status(), true)); + $this->io->text(('memory_get_usage: ' . memory_get_usage())); + $this->io->text(('memory_get_peak_usage: ' . memory_get_peak_usage())); + $this->errorReport(); return Command::SUCCESS; @@ -117,12 +134,9 @@ private function getStringArgument(string $name): string return $value; } - private function getIntArgument(string $name, int $default): int + private function getIntOption(string $name): int { - if (!$this->input->hasArgument($name)) { - return $default; - } - $value = $this->input->getArgument($name); + $value = $this->input->getOption($name); if (!is_int($value)) { throw new InvalidArgumentException( $name . ' must be a integer' diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index 7744fda..ac68625 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -9,7 +9,13 @@ class IndexerParameter public function __construct( public readonly string $index, public readonly int $cleanupThreshold = 0, + public readonly int $chunkSize = 500, public readonly array $paths = [] ) { + if ($this->chunkSize < 10) { + throw new \InvalidArgumentException( + 'chunk Size must be greater than 9' + ); + } } } diff --git a/src/Indexer.php b/src/Indexer.php index 2308899..045d0f6 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -25,5 +25,8 @@ public function index(IndexerParameter $parameter): IndexerStatus; public function abort($index): void; - public function remove(string $index, string $id): void; + /** + * @param string[] $idList + */ + public function remove(string $index, array $idList): void; } diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 60d1e7b..0c6f868 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -44,9 +44,12 @@ public function __construct( } } - public function remove(string $index, string $id): void + /** + * @param string[] $idList + */ + public function remove(string $index, array $idList): void { - $this->getIndexer($index)->remove($index, $id); + $this->getIndexer($index)->remove($index, $idList); } public function abort($index): void diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 4ecf8d4..9b8c008 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -22,7 +22,7 @@ public function findAll(): array { $finder = new Finder(); - $finder->in($this->basePath); + $finder->in($this->basePath)->exclude('WEB-IES'); $finder->name('*.php'); $finder->files(); diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 259a210..aa76983 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -35,9 +35,16 @@ public function __construct( ) { } - public function remove(string $index, string $id): void + /** + * @param string[] $idList + */ + public function remove(string $index, array $idList): void { - $this->deleteById($index, $id); + if (empty($idList)) { + return; + } + + $this->deleteByIdList($index, $idList); $this->commit($index); } @@ -78,7 +85,6 @@ private function indexResources( $processId = uniqid('', true); $offset = 0; - $chunkSize = 500; $successCount = 0; try { @@ -88,13 +94,14 @@ private function indexResources( $parameter->index, $pathList, $offset, - $chunkSize + $parameter->chunkSize ); if ($indexedCount === false) { break; } $successCount += $indexedCount; - $offset += $chunkSize; + $offset += $parameter->chunkSize; + gc_collect_cycles(); } if ( @@ -229,11 +236,14 @@ private function deleteByProcessId(string $core, string $processId): void ); } - private function deleteById(string $core, string $id): void + /** + * @param string[] $idList + */ + private function deleteByIdList(string $core, array $idList): void { $this->deleteByQuery( $core, - 'sp_id:' . $id . ' AND ' . + 'sp_id:(' . implode(' ', $idList) . ') AND ' . 'sp_source:' . $this->source ); } From 1bcebac6ac5ae69a451d84c114ea01e430e82407 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 19 Dec 2023 14:10:59 +0100 Subject: [PATCH 010/145] refactor: Rename ResourceSearchResult to SearchResult --- src/Console/Command/MoreLikeThis.php | 4 ++-- src/Console/Command/Search.php | 4 ++-- .../Result/{ResourceSearchResult.php => SearchResult.php} | 2 +- src/MoreLikeThisSearcher.php | 4 ++-- src/SelectSearcher.php | 4 ++-- src/Service/Search/SolrMoreLikeThis.php | 8 ++++---- src/Service/Search/SolrSelect.php | 8 ++++---- 7 files changed, 17 insertions(+), 17 deletions(-) rename src/Dto/Search/Result/{ResourceSearchResult.php => SearchResult.php} (95%) diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 475a3fb..1709bbc 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -7,7 +7,7 @@ use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; @@ -122,7 +122,7 @@ protected function buildQuery(string $location): MoreLikeThisQuery ); } - protected function outputResult(ResourceSearchResult $result): void + protected function outputResult(SearchResult $result): void { $this->io->text($result->getTotal() . " Results:"); foreach ($result as $resource) { diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 49fd4fe..039e47f 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -7,7 +7,7 @@ use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Dto\Search\Query\SelectQuery; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; @@ -128,7 +128,7 @@ protected function buildQuery(InputInterface $input): SelectQuery } protected function outputResult( - ResourceSearchResult $result + SearchResult $result ) { $this->io->title('Results (' . $result->getTotal() . ')'); foreach ($result as $resource) { diff --git a/src/Dto/Search/Result/ResourceSearchResult.php b/src/Dto/Search/Result/SearchResult.php similarity index 95% rename from src/Dto/Search/Result/ResourceSearchResult.php rename to src/Dto/Search/Result/SearchResult.php index 1d1def5..6da8229 100644 --- a/src/Dto/Search/Result/ResourceSearchResult.php +++ b/src/Dto/Search/Result/SearchResult.php @@ -12,7 +12,7 @@ /** * @implements IteratorAggregate */ -class ResourceSearchResult implements IteratorAggregate +class SearchResult implements IteratorAggregate { /** * @param Resource[] $results diff --git a/src/MoreLikeThisSearcher.php b/src/MoreLikeThisSearcher.php index 3721e25..cbe0d22 100644 --- a/src/MoreLikeThisSearcher.php +++ b/src/MoreLikeThisSearcher.php @@ -5,7 +5,7 @@ namespace Atoolo\Search; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; /** * The service interface for a "More-Like-This" search. @@ -22,5 +22,5 @@ interface MoreLikeThisSearcher { public function moreLikeThis( MoreLikeThisQuery $query - ): ResourceSearchResult; + ): SearchResult; } diff --git a/src/SelectSearcher.php b/src/SelectSearcher.php index e478d22..f2a91c6 100644 --- a/src/SelectSearcher.php +++ b/src/SelectSearcher.php @@ -5,12 +5,12 @@ namespace Atoolo\Search; use Atoolo\Search\Dto\Search\Query\SelectQuery; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; /** * The service interface for a search with full-text, filter and facet support. */ interface SelectSearcher { - public function select(SelectQuery $query): ResourceSearchResult; + public function select(SelectQuery $query): SearchResult; } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 11e87b8..612c81b 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -5,7 +5,7 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\MoreLikeThisSearcher; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Core\Client\Client; @@ -26,7 +26,7 @@ public function __construct( ) { } - public function moreLikeThis(MoreLikeThisQuery $query): ResourceSearchResult + public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { $client = $this->clientFactory->create($query->getCore()); $solrQuery = $this->buildSolrQuery($client, $query); @@ -61,12 +61,12 @@ private function buildSolrQuery( private function buildResult( ResultInterface $result - ): ResourceSearchResult { + ): SearchResult { $resourceList = $this->resultToResourceResolver ->loadResourceList($result); - return new ResourceSearchResult( + return new SearchResult( $result->getNumFound(), 0, $resourceList, diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index d00be49..d7c49d8 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -12,7 +12,7 @@ use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Result\Facet; use Atoolo\Search\Dto\Search\Result\FacetGroup; -use Atoolo\Search\Dto\Search\Result\ResourceSearchResult; +use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\SelectSearcher; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Component\Result\Facet\FacetResultInterface; @@ -35,7 +35,7 @@ public function __construct( ) { } - public function select(SelectQuery $query): ResourceSearchResult + public function select(SelectQuery $query): SearchResult { $client = $this->clientFactory->create($query->getIndex()); @@ -206,13 +206,13 @@ private function addFacetMultiQueryToSolrQuery( private function buildResult( SelectQuery $query, ResultInterface $result - ): ResourceSearchResult { + ): SearchResult { $resourceList = $this->resultToResourceResolver ->loadResourceList($result); $facetGroupList = $this->buildFacetGroupList($query, $result); - return new ResourceSearchResult( + return new SearchResult( $result->getNumFound(), $query->getLimit(), $query->getOffset(), From 2de4c023e08f963949fd3d3f5da9c81255648f15 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 9 Jan 2024 07:53:38 +0100 Subject: [PATCH 011/145] feat: sorting --- src/Dto/Search/Query/SelectQuery.php | 14 ++++++++ src/Dto/Search/Query/SelectQueryBuilder.php | 26 ++++++++++++++ src/Dto/Search/Query/Sort/Criteria.php | 18 ++++++++++ src/Dto/Search/Query/Sort/Date.php | 9 +++++ src/Dto/Search/Query/Sort/Direction.php | 9 +++++ src/Dto/Search/Query/Sort/Headline.php | 9 +++++ src/Dto/Search/Query/Sort/Name.php | 11 ++++++ src/Dto/Search/Query/Sort/Natural.php | 9 +++++ src/Dto/Search/Query/Sort/Score.php | 9 +++++ src/Service/Search/SolrSelect.php | 39 +++++++++++++++++++++ 10 files changed, 153 insertions(+) create mode 100644 src/Dto/Search/Query/Sort/Criteria.php create mode 100644 src/Dto/Search/Query/Sort/Date.php create mode 100644 src/Dto/Search/Query/Sort/Direction.php create mode 100644 src/Dto/Search/Query/Sort/Headline.php create mode 100644 src/Dto/Search/Query/Sort/Name.php create mode 100644 src/Dto/Search/Query/Sort/Natural.php create mode 100644 src/Dto/Search/Query/Sort/Score.php diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index 2eea4a4..a902eac 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -6,6 +6,7 @@ use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; +use Atoolo\Search\Dto\Search\Query\Sort\Criteria; class SelectQuery { @@ -13,6 +14,10 @@ class SelectQuery private readonly string $text; private readonly int $offset; private readonly int $limit; + /** + * @var Criteria[] + */ + private readonly array $sort; /** * @var Filter[] */ @@ -32,6 +37,7 @@ public function __construct(SelectQueryBuilder $builder) $this->text = $builder->getText(); $this->offset = $builder->getOffset(); $this->limit = $builder->getLimit(); + $this->sort = $builder->getSort(); $this->filterList = $builder->getFilterList(); $this->facetList = $builder->getFacetList(); $this->queryDefaultOperator = $builder->getQueryDefaultOperator(); @@ -62,6 +68,14 @@ public function getLimit(): int return $this->limit; } + /** + * @return Criteria[] + */ + public function getSort(): array + { + return $this->sort; + } + /** * @return Filter[] */ diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index b24971e..15c754f 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -6,6 +6,8 @@ use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; +use Atoolo\Search\Dto\Search\Query\Sort\Criteria; +use GuzzleHttp\Promise\Create; class SelectQueryBuilder { @@ -13,6 +15,10 @@ class SelectQueryBuilder private string $text = ''; private int $offset = 0; private int $limit = 10; + /** + * @var Criteria[] + */ + private array $sort = []; /** * @var array */ @@ -98,6 +104,26 @@ public function getLimit(): int return $this->limit; } + /** + * @param Criteria[] $criteriaList + */ + public function sort(Criteria ...$criteriaList): SelectQueryBuilder + { + foreach ($criteriaList as $criteria) { + $this->sort[] = $criteria; + } + return $this; + } + + /** + * @internal + * @return Criteria[] + */ + public function getSort(): array + { + return $this->sort; + } + /** * @param Filter[] $filterList */ diff --git a/src/Dto/Search/Query/Sort/Criteria.php b/src/Dto/Search/Query/Sort/Criteria.php new file mode 100644 index 0000000..8daf3fa --- /dev/null +++ b/src/Dto/Search/Query/Sort/Criteria.php @@ -0,0 +1,18 @@ +direction; + } +} diff --git a/src/Dto/Search/Query/Sort/Date.php b/src/Dto/Search/Query/Sort/Date.php new file mode 100644 index 0000000..467cf86 --- /dev/null +++ b/src/Dto/Search/Query/Sort/Date.php @@ -0,0 +1,9 @@ +setOmitHeader(false); + $this->addSortToSolrQuery($solrQuery, $query->getSort()); $this->addRequiredFieldListToSolrQuery($solrQuery); $this->addTextFilterToSolrQuery($solrQuery, $query->getText()); $this->addQueryDefaultOperatorToSolrQuery( @@ -80,6 +87,38 @@ private function buildSolrQuery( return $solrQuery; } + /** + * @param Criteria[] $criteriaList + */ + private function addSortToSolrQuery( + SolrSelectQuery $solrQuery, + array $criteriaList + ): void { + $sorts = []; + foreach ($criteriaList as $criteria) { + if ($criteria instanceof Name) { + $field = 'sp_name'; + } elseif ($criteria instanceof Headline) { + $field = 'sp_headline'; + } elseif ($criteria instanceof Date) { + $field = 'sp_date'; + } elseif ($criteria instanceof Natural) { + $field = 'sp_sortvalue'; + } elseif ($criteria instanceof Score) { + $field = 'score'; + } else { + throw new \InvalidArgumentException( + 'unsupported sort criteria: ' . get_class($criteria) + ); + } + + $direction = strtolower($criteria->getDirection()->name); + + $sorts[$field] = $direction; + } + $solrQuery->setSorts($sorts); + } + private function addRequiredFieldListToSolrQuery( SolrSelectQuery $solrQuery ): void { From 8c29ad8020edb67fb565eae8cfdb3363be8b2164 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 9 Jan 2024 07:57:00 +0100 Subject: [PATCH 012/145] chore: composer update --- composer.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/composer.json b/composer.json index 5e34a26..34bff57 100644 --- a/composer.json +++ b/composer.json @@ -80,17 +80,5 @@ "sort-packages": true }, "repositories": { - "atoolo/resource": { - "type": "path", - "url": "/home/veltrup/.cache/composer/link/atoolo/resource" - } - }, - "extra": { - "composer-link": { - "atoolo/resource": { - "dev": false, - "version": "dev-main" - } - } } } From 3bb7e1714e484ffe758bfbe21e1289fb97f7ba1a Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 9 Jan 2024 16:24:37 +0100 Subject: [PATCH 013/145] feat: and, or, not filter --- src/Dto/Search/Query/Filter/AndFilter.php | 31 +++++++++++++++++++ src/Dto/Search/Query/Filter/ArchiveFilter.php | 8 +++-- src/Dto/Search/Query/Filter/FieldFilter.php | 3 +- src/Dto/Search/Query/Filter/Filter.php | 8 ++--- src/Dto/Search/Query/Filter/NotFilter.php | 21 +++++++++++++ src/Dto/Search/Query/Filter/OrFilter.php | 29 +++++++++++++++++ 6 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 src/Dto/Search/Query/Filter/AndFilter.php create mode 100644 src/Dto/Search/Query/Filter/NotFilter.php create mode 100644 src/Dto/Search/Query/Filter/OrFilter.php diff --git a/src/Dto/Search/Query/Filter/AndFilter.php b/src/Dto/Search/Query/Filter/AndFilter.php new file mode 100644 index 0000000..de63492 --- /dev/null +++ b/src/Dto/Search/Query/Filter/AndFilter.php @@ -0,0 +1,31 @@ +filter as $filter) { + $query[] = $filter->getQuery(); + } + + return '(' . implode(' AND ', $query) . ')'; + } +} diff --git a/src/Dto/Search/Query/Filter/ArchiveFilter.php b/src/Dto/Search/Query/Filter/ArchiveFilter.php index 8527623..2b12b39 100644 --- a/src/Dto/Search/Query/Filter/ArchiveFilter.php +++ b/src/Dto/Search/Query/Filter/ArchiveFilter.php @@ -9,8 +9,12 @@ class ArchiveFilter extends Filter public function __construct() { parent::__construct( - 'archive', - '-sp_archive:true' + 'archive' ); } + + public function getQuery(): string + { + return '-sp_archive:true'; + } } diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index 71d2a6b..e0bef8f 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -26,7 +26,6 @@ public function __construct( $this->values = $values; parent::__construct( $key, - $this->toQuery(), [$key] ); } @@ -34,7 +33,7 @@ public function __construct( /** * @param string[] $values */ - private function toQuery(): string + public function getQuery(): string { $filterValue = count($this->values) === 1 ? $this->values[0] diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php index 4ae4e88..73fcc0c 100644 --- a/src/Dto/Search/Query/Filter/Filter.php +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -4,14 +4,13 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; -class Filter +abstract class Filter { /** * @param string[] $tags */ public function __construct( private readonly ?string $key, - private readonly string $query, private readonly array $tags = [] ) { } @@ -21,10 +20,7 @@ public function getKey(): ?string return $this->key; } - public function getQuery(): string - { - return $this->query; - } + abstract public function getQuery(): string; /** * @return string[] diff --git a/src/Dto/Search/Query/Filter/NotFilter.php b/src/Dto/Search/Query/Filter/NotFilter.php new file mode 100644 index 0000000..381ecfe --- /dev/null +++ b/src/Dto/Search/Query/Filter/NotFilter.php @@ -0,0 +1,21 @@ +filter->getQuery(); + } +} diff --git a/src/Dto/Search/Query/Filter/OrFilter.php b/src/Dto/Search/Query/Filter/OrFilter.php new file mode 100644 index 0000000..3282fb9 --- /dev/null +++ b/src/Dto/Search/Query/Filter/OrFilter.php @@ -0,0 +1,29 @@ +filter as $filter) { + $query[] = $filter->getQuery(); + } + + return '(' . implode(' OR ', $query) . ')'; + } +} From 28aee3a07073b8822fa0f684f4106561703df082 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 9 Jan 2024 16:56:36 +0100 Subject: [PATCH 014/145] feat: query-filter --- src/Dto/Search/Query/Filter/QueryFilter.php | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/Dto/Search/Query/Filter/QueryFilter.php diff --git a/src/Dto/Search/Query/Filter/QueryFilter.php b/src/Dto/Search/Query/Filter/QueryFilter.php new file mode 100644 index 0000000..2ff8133 --- /dev/null +++ b/src/Dto/Search/Query/Filter/QueryFilter.php @@ -0,0 +1,22 @@ +query; + } +} From 1f9d75cea24dca22e80774e66cc52566056adc51 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 17 Jan 2024 09:12:31 +0100 Subject: [PATCH 015/145] feat: translation-splitter --- config/services.yml | 4 + src/Console/Command/Indexer.php | 32 +- src/Service/Indexer/BackgroundIndexer.php | 6 +- src/Service/Indexer/LocationFinder.php | 25 +- .../DefaultSchema21DocumentEnricher.php | 136 +++---- .../SiteKit/SubDirTranslationSplitter.php | 47 +++ src/Service/Indexer/SolrIndexer.php | 122 +++++-- src/Service/Indexer/TranslationSplitter.php | 10 + .../Indexer/TranslationSplitterResult.php | 44 +++ test/Service/Indexer/LocationFinderTest.php | 17 + .../SiteKit/SubDirTranslationSplitterTest.php | 65 ++++ test/Service/Indexer/SolrIndexerTest.php | 338 ++++++++++++++++++ .../Indexer/TranslationSplitterResultTest.php | 78 ++++ 13 files changed, 809 insertions(+), 115 deletions(-) create mode 100644 src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php create mode 100644 src/Service/Indexer/TranslationSplitter.php create mode 100644 src/Service/Indexer/TranslationSplitterResult.php create mode 100644 test/Service/Indexer/LocationFinderTest.php create mode 100644 test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php create mode 100644 test/Service/Indexer/SolrIndexerTest.php create mode 100644 test/Service/Indexer/TranslationSplitterResultTest.php diff --git a/config/services.yml b/config/services.yml index 1b29748..5fdc303 100644 --- a/config/services.yml +++ b/config/services.yml @@ -9,6 +9,10 @@ services: Atoolo\Search\Console\: resource: '../src/Console' + Atoolo\Search\Console\Command\Indexer: + arguments: + - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher' } + Atoolo\Search\Console\Application: public: true arguments: diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 44c9701..0b367fb 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -11,7 +11,9 @@ use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\IndexingAborter; +use Atoolo\Search\Service\Indexer\LocationFinder; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema21DocumentEnricher; +use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\SolrParameterClientFactory; use InvalidArgumentException; @@ -34,6 +36,11 @@ class Indexer extends Command private InputInterface $input; private string $resourceDir; + public function __construct(private readonly iterable $documentEnricherList) + { + parent::__construct(); + } + protected function configure(): void { $this @@ -89,6 +96,7 @@ protected function execute( $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); $this->resourceDir = $this->getStringArgument('resource-dir'); + $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; $paths = (array)$input->getArgument('paths'); $cleanupThreshold = empty($paths) @@ -112,12 +120,6 @@ protected function execute( $indexer = $this->createIndexer(); $indexer->index($parameter); - $this->io->text(print_r(gc_status(), true)); - gc_collect_cycles(); - $this->io->text(print_r(gc_status(), true)); - $this->io->text(('memory_get_usage: ' . memory_get_usage())); - $this->io->text(('memory_get_peak_usage: ' . memory_get_peak_usage())); - $this->errorReport(); return Command::SUCCESS; @@ -137,12 +139,12 @@ private function getStringArgument(string $name): string private function getIntOption(string $name): int { $value = $this->input->getOption($name); - if (!is_int($value)) { + if (!is_numeric($value)) { throw new InvalidArgumentException( - $name . ' must be a integer' + $name . ' must be a integer: ' . $value ); } - return $value; + return (int)$value; } protected function errorReport(): void @@ -164,6 +166,7 @@ protected function createIndexer(): SolrIndexer $resourceBaseLocator = new StaticResourceBaseLocator( $this->resourceDir ); + $finder = new LocationFinder($resourceBaseLocator); $resourceLoader = new SiteKitLoader($resourceBaseLocator); $navigationLoader = new SiteKitNavigationHierarchyLoader( $resourceLoader @@ -172,6 +175,10 @@ protected function createIndexer(): SolrIndexer $navigationLoader ); + $documentEnricherList = [$schema21]; + foreach ($this->documentEnricherList as $enricher) { + $documentEnricherList[] = $enricher; + } $url = parse_url($this->getStringArgument('solr-connection-url')); $clientFactory = new SolrParameterClientFactory( @@ -183,13 +190,16 @@ protected function createIndexer(): SolrIndexer 0 ); + $translationSplitter = new SubDirTranslationSplitter(); + $aborter = new IndexingAborter('.'); return new SolrIndexer( - [$schema21], + $documentEnricherList, $this->progressBar, - $resourceBaseLocator, + $finder, $resourceLoader, + $translationSplitter, $clientFactory, $aborter, 'internal' diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 0c6f868..2411f42 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -24,8 +24,9 @@ class BackgroundIndexer implements Indexer */ public function __construct( private readonly iterable $documentEnricherList, - private readonly ResourceBaseLocator $resourceBaseLocator, + private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, + private readonly TranslationSplitter $translationSplitter, private readonly SolrClientFactory $clientFactory, private readonly IndexingAborter $aborter, private readonly string $source, @@ -87,8 +88,9 @@ private function getIndexer(string $index): SolrIndexer return new SolrIndexer( $this->documentEnricherList, $progressHandler, - $this->resourceBaseLocator, + $this->finder, $this->resourceLoader, + $this->translationSplitter, $this->clientFactory, $this->aborter, $this->source diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 9b8c008..3ca0458 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -4,15 +4,14 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Resource\ResourceBaseLocator; use Symfony\Component\Finder\Finder; class LocationFinder { - private readonly string $basePath; - - public function __construct(string $basePath) - { - $this->basePath = rtrim($basePath, '/'); + public function __construct( + private readonly ResourceBaseLocator $baseLocator + ) { } /** @@ -22,7 +21,7 @@ public function findAll(): array { $finder = new Finder(); - $finder->in($this->basePath)->exclude('WEB-IES'); + $finder->in($this->getBasePath())->exclude('WEB-IES'); $finder->name('*.php'); $finder->files(); @@ -35,7 +34,8 @@ public function findAll(): array } /** - * @param string[] $directories + * @param string[] $paths + * @return string[] */ public function findPaths(array $paths): array { @@ -45,7 +45,7 @@ public function findPaths(array $paths): array $finder = new Finder(); foreach ($paths as $path) { - $absolutePath = $this->basePath . '/' . $path; + $absolutePath = $this->getBasePath() . '/' . $path; if (is_file($absolutePath)) { $pathList[] = $path; continue; @@ -60,7 +60,7 @@ public function findPaths(array $paths): array } foreach ($directories as $directory) { - $finder->in($this->basePath . '/' . $directory); + $finder->in($this->getBasePath() . '/' . $directory); } $finder->name('*.php'); $finder->files(); @@ -72,8 +72,13 @@ public function findPaths(array $paths): array return $pathList; } + private function getBasePath(): string + { + return rtrim($this->baseLocator->locate(), '/'); + } + private function toRelativePath(string $path): string { - return substr($path, strlen($this->basePath)); + return substr($path, strlen($this->getBasePath())); } } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php index 873a994..fe6b4e8 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php @@ -19,7 +19,7 @@ public function __construct( public function isIndexable(Resource $resource): bool { - $noIndex = $resource->getData('init.noIndex'); + $noIndex = $resource->getData()->getBool('init.noIndex'); return $noIndex !== true; } public function enrichDocument( @@ -29,40 +29,44 @@ public function enrichDocument( ): DocumentInterface { $doc->sp_id = $resource->getId(); $doc->sp_name = $resource->getName(); - $doc->sp_anchor = $resource->getData('init.anchor'); - $doc->title = $resource->getData('base.title'); - $doc->description = $resource->getData('metadata.description'); + $doc->sp_anchor = $resource->getData()->getString('init.anchor'); + $doc->title = $resource->getData()->getString('base.title'); + $doc->description = $resource->getData()->getString( + 'metadata.description' + ); $doc->sp_objecttype = $resource->getObjectType(); $doc->sp_canonical = true; $doc->crawl_process_id = $processId; - $mediaUrl = $resource->getData('init.mediaUrl'); - if ($mediaUrl !== null) { + $mediaUrl = $resource->getData()->getString('init.mediaUrl'); + if (!empty($mediaUrl)) { $doc->id = $mediaUrl; $doc->url = $mediaUrl; } else { - $doc->id = $resource->getLocation(); - $doc->url = $resource->getLocation(); + $url = $resource->getData()->getString('init.url'); + $doc->id = $url; + $doc->url = $url; } $spContentType = [$resource->getObjectType()]; - if ($resource->getData('init.media') !== true) { + if ($resource->getData()->getBool('init.media') !== true) { $spContentType[] = 'article'; } - $contentSectionTypes = $resource->getData('init.contentSectionTypes'); - if (is_array($contentSectionTypes)) { - $spContentType = array_merge($spContentType, $contentSectionTypes); - } - if ($resource->getData('base.teaser.image') !== null) { + $contentSectionTypes = $resource->getData()->getArray( + 'init.contentSectionTypes' + ); + $spContentType = array_merge($spContentType, $contentSectionTypes); + + if ($resource->getData()->has('base.teaser.image')) { $spContentType[] = 'teaserImage'; } - if ($resource->getData('base.teaser.image.copyright') !== null) { + if ($resource->getData()->has('base.teaser.image.copyright')) { $spContentType[] = 'teaserImageCopyright'; } - if ($resource->getData('base.teaser.headline') !== null) { + if ($resource->getData()->has('base.teaser.headline')) { $spContentType[] = 'teaserHeadline'; } - if ($resource->getData('base.teaser.text') !== null) { + if ($resource->getData()->has('base.teaser.text')) { $spContentType[] = 'teaserText'; } $doc->sp_contenttype = $spContentType; @@ -73,51 +77,59 @@ public function enrichDocument( $doc->meta_content_language = $lang; $doc->sp_changed = $this->toDateTime( - $resource->getData('init.changed') + $resource->getData()->getInt('init.changed') ); $doc->sp_generated = $this->toDateTime( - $resource->getData('init.generated') + $resource->getData()->getInt('init.generated') ); $doc->sp_date = $this->toDateTime( - $resource->getData('base.date') + $resource->getData()->getInt('base.date') ); - $doc->sp_archive = $resource->getData('base.archive') ?? false; + $doc->sp_archive = $resource->getData()->getBool('base.archive'); - $headline = $resource->getData('metadata.headline'); + $headline = $resource->getData()->getString('metadata.headline'); if (empty($headline)) { - $headline = $resource->getData('base.teaser.headline'); + $headline = $resource->getData()->getString('base.teaser.headline'); } if (empty($headline)) { - $headline = $resource->getData('base.title'); + $headline = $resource->getData()->getString('base.title'); } $doc->sp_title = $headline; // However, the teaser heading, if specified, must be used for sorting - $sortHeadline = $resource->getData('base.teaser.headline'); + $sortHeadline = $resource->getData()->getString('base.teaser.headline'); if (empty($sortHeadline)) { - $sortHeadline = $resource->getData('metadata.headline'); + $sortHeadline = $resource->getData()->getString( + 'metadata.headline' + ); } if (empty($sortHeadline)) { - $sortHeadline = $resource->getData('base.title'); + $sortHeadline = $resource->getData()->getString('base.title'); } $doc->sp_sortvalue = $sortHeadline; - $doc->sp_boost_keywords = $resource->getData('metadata.boostKeywords'); + $doc->keywords = $resource->getData()->getArray('metadata.keywords'); + + $doc->sp_boost_keywords = $resource->getData()->getArray( + 'metadata.boostKeywords' + ); $sites = $this->getParentSiteGroupIdList($resource); $navigationRoot = $this->navigationLoader->loadRoot( $resource->getLocation() ); - $siteGroupId = $navigationRoot->getData('init.siteGroup.id'); - if ($siteGroupId !== null) { + $siteGroupId = $navigationRoot->getData()->getInt('init.siteGroup.id'); + if ($siteGroupId !== 0) { $sites[] = $siteGroupId; } $doc->sp_site = array_unique($sites); - $wktPrimaryList = $resource->getData('base.geo.wkt.primary'); - if (is_array($wktPrimaryList)) { + $wktPrimaryList = $resource->getData()->getArray( + 'base.geo.wkt.primary' + ); + if (!empty($wktPrimaryList)) { $allWkt = []; foreach ($wktPrimaryList as $wkt) { $allWkt[] = $wkt; @@ -127,8 +139,8 @@ public function enrichDocument( } } - $categoryList = $resource->getData('metadata.categories'); - if (is_array($categoryList)) { + $categoryList = $resource->getData()->getArray('metadata.categories'); + if (!empty($categoryList)) { $categoryIdList = []; foreach ($categoryList as $category) { $categoryIdList[] = $category['id']; @@ -136,8 +148,10 @@ public function enrichDocument( $doc->sp_category = $categoryIdList; } - $categoryPath = $resource->getData('metadata.categoriesPath'); - if (is_array($categoryPath)) { + $categoryPath = $resource->getData()->getArray( + 'metadata.categoriesPath' + ); + if (!empty($categoryPath)) { $categoryIdPath = []; foreach ($categoryPath as $category) { $categoryIdPath[] = $category['id']; @@ -145,18 +159,17 @@ public function enrichDocument( $doc->sp_category_path = $categoryIdPath; } - $groupPath = $resource->getData('init.groupPath'); + $groupPath = $resource->getData()->getArray('init.groupPath'); $groupPathAsIdList = []; - if (is_array($groupPath)) { - foreach ($groupPath as $group) { - $groupPathAsIdList[] = $group['id']; - } + foreach ($groupPath as $group) { + $groupPathAsIdList[] = $group['id']; } + $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; $doc->sp_group_path = $groupPathAsIdList; - $schedulingList = $resource->getData('metadata.scheduling'); - if (is_array($schedulingList) && count($schedulingList) > 0) { + $schedulingList = $resource->getData()->getArray('metadata.scheduling'); + if (!empty($schedulingList)) { $doc->sp_date = $this->toDateTime($schedulingList[0]['from']); $dateList = []; $contentTypeList = []; @@ -173,23 +186,25 @@ public function enrichDocument( $doc->sp_date_list = $dateList; } - $contentType = $resource->getData('base.mime'); + $contentType = $resource->getData()->getString('base.mime'); if ($contentType === null) { $contentType = 'text/html; charset=UTF-8'; } $doc->meta_content_type = $contentType; - $doc->content = $resource->getData('searchindexdata.content'); + $doc->content = $resource->getData()->getString( + 'searchindexdata.content' + ); - $accessType = $resource->getData('init.access.type'); - $groups = $resource->getData('init.access.groups'); + $accessType = $resource->getData()->getString('init.access.type'); + $groups = $resource->getData()->getArray('init.access.groups'); - if ($accessType === 'allow' && is_array($groups)) { + if ($accessType === 'allow' && !empty($groups)) { $doc->include_groups = array_map( fn($id): int => $this->idWithoutSignature($id), $groups ); - } elseif ($accessType === 'deny' && is_array($groups)) { + } elseif ($accessType === 'deny' && !empty($groups)) { $doc->exclude_groups = array_map( fn($id): int => $this->idWithoutSignature($id), $groups @@ -219,7 +234,6 @@ private function idWithoutSignature(string $id): int * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/ParkingSpaceExpose.php#L38 * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/Expose.php#L38 * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 - * - https://gitlab.sitepark.com/customer-projects/stuttgart/blob/develop/stuttgart-module/src/publish/php/SP/Stuttgart/Component/JobOffer.php#L29 * - https://gitlab.sitepark.com/customer-projects/stuttgart/blob/develop/stuttgart-module/src/publish/php/SP/Stuttgart/Component/EventsCalendarExtension.php#L124 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Controller/Environment.php#L76 @@ -229,13 +243,13 @@ private function idWithoutSignature(string $id): int private function getLocaleFromResource(Resource $resource): string { - $locale = $resource->getData('init.locale'); - if ($locale !== null) { + $locale = $resource->getData()->getString('init.locale'); + if ($locale !== '') { return $locale; } - $groupPath = $resource->getData('init.groupPath'); - $len = count($groupPath); - if (is_array($groupPath)) { + $groupPath = $resource->getData()->getArray('init.groupPath'); + if (!empty($groupPath)) { + $len = count($groupPath); for ($i = $len - 1; $i >= 0; $i--) { $group = $groupPath[$i]; if (isset($group['locale'])) { @@ -256,11 +270,8 @@ private function toLangFromLocale(string $locale): string return $locale; } - private function toDateTime(?int $timestamp): ?DateTime + private function toDateTime(int $timestamp): ?DateTime { - if ($timestamp === null) { - return null; - } if ($timestamp <= 0) { return null; } @@ -288,11 +299,12 @@ private function getParentSiteGroupIdList(Resource $resource): array } /** - * @return array + * @return array */ private function getNavigationParents(Resource $resource): array { - $parents = $resource->getData('base.trees.navigation.parents'); - return $parents ?? []; + return $resource->getData()->getAssociativeArray( + 'base.trees.navigation.parents' + ); } } diff --git a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php new file mode 100644 index 0000000..e7ac074 --- /dev/null +++ b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php @@ -0,0 +1,47 @@ +extractLocaleFromPath($path); + if ($locale === 'default') { + $bases[] = $path; + continue; + } + if (!isset($translations[$locale])) { + $translations[$locale] = []; + } + $translations[$locale][] = $path; + } + + return new TranslationSplitterResult($bases, $translations); + } + + private function extractLocaleFromPath(string $path): string + { + $filename = basename($path); + $parentDirName = basename(dirname($path)); + + if (!str_ends_with($parentDirName, '.php.translations')) { + return 'default'; + } + + return basename($filename, '.php'); + } +} diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index aa76983..2318877 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -6,7 +6,6 @@ use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Resource; -use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; @@ -27,8 +26,9 @@ class SolrIndexer implements Indexer public function __construct( private readonly iterable $documentEnricherList, private readonly IndexerProgressHandler $indexerProgressHandler, - private readonly ResourceBaseLocator $resourceBaseLocator, + private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, + private readonly TranslationSplitter $translationSplitter, private readonly SolrClientFactory $clientFactory, private readonly IndexingAborter $aborter, private readonly string $source @@ -55,12 +55,12 @@ public function abort($index): void public function index(IndexerParameter $parameter): IndexerStatus { - $finder = new LocationFinder($this->resourceBaseLocator->locate()); if (empty($parameter->paths)) { - $pathList = $finder->findAll(); + $pathList = $this->finder->findAll(); } else { - $pathList = $finder->findPaths($parameter->paths); + $pathList = $this->finder->findPaths($parameter->paths); } + return $this->indexResources($parameter, $pathList); } @@ -83,42 +83,82 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } + + $availableIndexes = $this->getAvailableIndexes(); + $processId = uniqid('', true); - $offset = 0; - $successCount = 0; - try { - while (true) { - $indexedCount = $this->indexChunks( + $splitterResult = $this->translationSplitter->split($pathList); + + if (in_array($parameter->index, $availableIndexes)) { + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + $parameter->index, + $splitterResult->getBases() + ); + } else { + $this->indexerProgressHandler->error(new Exception( + 'Index "' . $parameter->index . '" not found' + )); + } + + foreach ($splitterResult->getLocales() as $locale) { + $localeIndex = $parameter->index . '-' . $locale; + if (in_array($localeIndex, $availableIndexes)) { + $this->indexResourcesPerLanguageIndex( $processId, - $parameter->index, - $pathList, - $offset, - $parameter->chunkSize + $parameter, + $localeIndex, + $splitterResult->getTranslations($locale) ); - if ($indexedCount === false) { - break; - } - $successCount += $indexedCount; - $offset += $parameter->chunkSize; - gc_collect_cycles(); + } else { + $this->indexerProgressHandler->error(new Exception( + 'Index "' . $localeIndex . '" not found' + )); } + } - if ( - $parameter->cleanupThreshold > 0 && - $successCount >= $parameter->cleanupThreshold - ) { - $this->deleteByProcessId($parameter->index, $processId); + $this->indexerProgressHandler->finish(); + + return $this->indexerProgressHandler->getStatus(); + } + + private function indexResourcesPerLanguageIndex( + string $processId, + IndexerParameter $parameter, + string $index, + array $pathList + ): void { + $offset = 0; + $successCount = 0; + + while (true) { + $indexedCount = $this->indexChunks( + $processId, + $index, + $pathList, + $offset, + $parameter->chunkSize + ); + if ($indexedCount === false) { + break; } - $this->commit($parameter->index); - } finally { - $this->indexerProgressHandler->finish(); + $successCount += $indexedCount; + $offset += $parameter->chunkSize; + gc_collect_cycles(); } - return $this->indexerProgressHandler->getStatus(); + if ( + $parameter->cleanupThreshold > 0 && + $successCount >= $parameter->cleanupThreshold + ) { + $this->deleteByProcessId($index, $processId); + } + $this->commit($index); } - /** + /** * @param string[] $pathList */ private function indexChunks( @@ -141,6 +181,9 @@ private function indexChunks( $this->indexerProgressHandler->abort(); return false; } + if (empty($resourceList)) { + return 0; + } $this->indexerProgressHandler->advance(count($resourceList)); $result = $this->add($solrCore, $processId, $resourceList); @@ -272,4 +315,23 @@ private function commit(string $core): void $update->addOptimize(); $client->update($update); } + + /** + * @return string[] + */ + private function getAvailableIndexes(): array + { + $client = $this->clientFactory->create(''); + $coreAdminQuery = $client->createCoreAdmin(); + $statusAction = $coreAdminQuery->createStatus(); + $coreAdminQuery->setAction($statusAction); + + $availableIndexes = []; + $response = $client->coreAdmin($coreAdminQuery); + foreach ($response->getStatusResults() as $statusResult) { + $availableIndexes[] = $statusResult->getCoreName(); + } + + return $availableIndexes; + } } diff --git a/src/Service/Indexer/TranslationSplitter.php b/src/Service/Indexer/TranslationSplitter.php new file mode 100644 index 0000000..3b0acfd --- /dev/null +++ b/src/Service/Indexer/TranslationSplitter.php @@ -0,0 +1,10 @@ +> $translations + */ + public function __construct( + private readonly array $bases, + private readonly array $translations + ) { + } + + /** + * @return string[] + */ + public function getLocales(): array + { + $locales = array_keys($this->translations); + sort($locales); + return $locales; + } + + /** + * @return string[] + */ + public function getBases(): array + { + return $this->bases; + } + + /** + * @return string[] + */ + public function getTranslations(string $locale): array + { + return $this->translations[$locale] ?? []; + } +} diff --git a/test/Service/Indexer/LocationFinderTest.php b/test/Service/Indexer/LocationFinderTest.php new file mode 100644 index 0000000..dbefa31 --- /dev/null +++ b/test/Service/Indexer/LocationFinderTest.php @@ -0,0 +1,17 @@ +split(['/a/b.php']); + + $this->assertEquals( + ['/a/b.php'], + $result->getBases(), + 'unexpected bases' + ); + } + + public function testLocales(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php', + '/a/b.php.translations/it_IT.php', + '/a/b.php.translations/en_US.php' + ]); + + $this->assertEquals( + ['en_US', 'it_IT'], + $result->getLocales(), + 'unexpected locales' + ); + } + + public function testGetTranlsations(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php', + '/a/b.php.translations/it_IT.php', + '/a/b.php.translations/en_US.php', + '/c/d.php', + '/c/d.php.translations/it_IT.php', + '/c/d.php.translations/en_US.php' + ]); + + $translations = $result->getTranslations('it_IT'); + + $expected = [ + '/a/b.php.translations/it_IT.php', + '/c/d.php.translations/it_IT.php' + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } +} diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/SolrIndexerTest.php new file mode 100644 index 0000000..df7864b --- /dev/null +++ b/test/Service/Indexer/SolrIndexerTest.php @@ -0,0 +1,338 @@ +indexerProgressHandler = $this->createMock( + IndexerProgressHandler::class + ); + $this->finder = $this->createStub(LocationFinder::class); + $this->documentEnricher = $this->createMock(DocumentEnricher::class); + $this->translationSplitter = new SubDirTranslationSplitter(); + $this->resourceLoader = $this->createStub(ResourceLoader::class); + $this->resourceLoader->method('load') + ->willReturnCallback(function ($path) { + $resource = $this->createStub(Resource::class); + $resource->method('getLocation') + ->willReturn($path); + return $resource; + }); + $solrClientFactory = $this->createStub(SolrClientFactory::class); + $this->updateResult = $this->createStub(UpdateResult::class); + $this->solrClient = $this->createMock(Client::class); + $this->updateQuery = $this->createMock(UpdateQuery::class); + $this->solrClient->method('createUpdate') + ->willReturn($this->updateQuery); + $this->solrClient->method('update') + ->willReturn($this->updateResult); + $solrClientFactory->method('create') + ->willReturn($this->solrClient); + $this->aborter = $this->createMock(IndexingAborter::class); + + $this->indexer = new SolrIndexer( + [ $this->documentEnricher ], + $this->indexerProgressHandler, + $this->finder, + $this->resourceLoader, + $this->translationSplitter, + $solrClientFactory, + $this->aborter, + 'test' + ); + } + + public function testAbort(): void + { + $this->aborter->expects($this->once()) + ->method('abort') + ->with('test'); + + $this->indexer->abort('test'); + } + + public function testRemove(): void + { + $this->solrClient->expects($this->exactly(2)) + ->method('update'); + + $this->indexer->remove('test', ['123']); + } + + public function testRemoveEmpty(): void + { + $this->solrClient->expects($this->exactly(0)) + ->method('update'); + + $this->indexer->remove('test', []); + } + + public function testIndexAllWithChunks(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php', + '/a/d.php', + '/a/e.php', + '/a/f.php', + '/a/g.php', + '/a/h.php', + '/a/i.php', + '/a/j.php', + '/a/k.php', + '/a/l.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->documentEnricher->method('isIndexable') + ->willReturn(true); + + $this->documentEnricher->expects($this->exactly(11)) + ->method('enrichDocument'); + + $addDocumentsCalls = 0; + $this->updateQuery->expects($this->exactly(2)) + ->method('addDocuments') + ->willReturnCallback( + function ($documents) use (&$addDocumentsCalls) { + if ($addDocumentsCalls === 0) { + $this->assertCount( + 10, + $documents, + "10 documents expected" + ); + } elseif ($addDocumentsCalls === 1) { + $this->assertCount( + 1, + $documents, + "1 document expected" + ); + } + $addDocumentsCalls++; + return $this->updateQuery; + } + ); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexer->index($parameter); + } + + public function testIndexSkipResource(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->documentEnricher->method('isIndexable') + ->willReturnCallback(function (Resource $resource) { + $location = $resource->getLocation(); + return ($location !== '/a/b.php'); + }); + + $this->documentEnricher->expects($this->exactly(1)) + ->method('enrichDocument'); + + $this->updateQuery->expects($this->exactly(1)) + ->method('addDocuments') + ->willReturnCallback(function ($documents) { + $this->assertCount( + 1, + $documents, + "one document exprected" + ); + return $this->updateQuery; + }); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexer->index($parameter); + } + public function testAborted(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->aborter->method('shouldAborted') + ->willReturn(true); + + $this->aborter->expects($this->once()) + ->method('aborted'); + + $this->indexerProgressHandler->expects($this->once()) + ->method('abort'); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexer->index($parameter); + } + + public function testWithUnsuccessfulStatus(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(500); + + $this->documentEnricher->method('isIndexable') + ->willReturn(true); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexer->index($parameter); + } + + public function testWithInvalidResource(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + ]); + + $this->resourceLoader->method('load') + ->willThrowException(new InvalidResourceException('/a/b.php')); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexer->index($parameter); + } + + public function testEmptyStatus(): void + { + $this->finder->method('findAll') + ->willReturn([]); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $status = $this->indexer->index($parameter); + $this->assertEquals(0, $status->total, 'total should be 0'); + } + + public function testIndexPaths(): void + { + $this->finder->method('findPaths') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->documentEnricher->method('isIndexable') + ->willReturn(true); + + $this->updateQuery->expects($this->exactly(1)) + ->method('addDocuments') + ->willReturnCallback(function ($documents) { + $this->assertCount( + 2, + $documents, + "two documents exprected" + ); + return $this->updateQuery; + }); + + $parameter = new IndexerParameter( + 'test', + 10, + 10, + [ + '/a/b.php', + '/a/c.php' + ] + ); + + $this->indexer->index($parameter); + } +} diff --git a/test/Service/Indexer/TranslationSplitterResultTest.php b/test/Service/Indexer/TranslationSplitterResultTest.php new file mode 100644 index 0000000..69790bc --- /dev/null +++ b/test/Service/Indexer/TranslationSplitterResultTest.php @@ -0,0 +1,78 @@ +assertEquals( + ['/a/b.php'], + $result->getBases(), + 'unexpected bases' + ); + } + + public function testLocales(): void + { + $result = new TranslationSplitterResult( + [], + [ + 'it_IT' => ['/a/b.php.translations/it_IT.php'], + 'en_US' => ['/a/b.php.translations/en_US.php'] + ] + ); + + $this->assertEquals( + ['en_US', 'it_IT'], + $result->getLocales(), + 'unexpected locales' + ); + } + + public function testGetTranslations(): void + { + $result = new TranslationSplitterResult( + [], + [ + 'it_IT' => [ + '/a/b.php.translations/it_IT.php', + '/c/d.php.translations/it_IT.php', + ], + 'en_US' => [ + '/a/b.php.translations/en_US.php', + '/a/b.php.translations/en_US.php', + ] + ] + ); + + $translations = $result->getTranslations('it_IT'); + + $expected = [ + '/a/b.php.translations/it_IT.php', + '/c/d.php.translations/it_IT.php' + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } + public function testGetMissingTranslations(): void + { + $result = new TranslationSplitterResult([], []); + + $this->assertEquals( + [], + $result->getTranslations('en_US'), + 'empty array expected' + ); + } +} From 183eab0b2f7a2bc789d8a6ae47273a8325ae2e4d Mon Sep 17 00:00:00 2001 From: veltrup Date: Thu, 18 Jan 2024 09:35:10 +0100 Subject: [PATCH 016/145] feat: IndexSchema2xDocument implementation --- src/Service/Indexer/DocumentEnricher.php | 11 +- src/Service/Indexer/IndexDocument.php | 14 ++ src/Service/Indexer/IndexSchema2xDocument.php | 132 ++++++++++++++++++ .../DefaultSchema21DocumentEnricher.php | 10 +- src/Service/Indexer/SolrIndexer.php | 1 + .../Indexer/IndexSchema21DocumentTest.php | 23 +++ 6 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 src/Service/Indexer/IndexDocument.php create mode 100644 src/Service/Indexer/IndexSchema2xDocument.php create mode 100644 test/Service/Indexer/IndexSchema21DocumentTest.php diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php index 268db3a..60f75e1 100644 --- a/src/Service/Indexer/DocumentEnricher.php +++ b/src/Service/Indexer/DocumentEnricher.php @@ -5,19 +5,24 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\Resource; -use Solarium\Core\Query\DocumentInterface; /** * This interface can be used to implement enricher with the help of which a * Solr document can be enriched on the basis of a resource. + * + * @template T of IndexDocument */ interface DocumentEnricher { public function isIndexable(Resource $resource): bool; + /** + * @param T $doc + * @return T + */ public function enrichDocument( Resource $resource, - DocumentInterface $doc, + IndexDocument $doc, string $processId - ): DocumentInterface; + ): IndexDocument; } diff --git a/src/Service/Indexer/IndexDocument.php b/src/Service/Indexer/IndexDocument.php new file mode 100644 index 0000000..0d8955d --- /dev/null +++ b/src/Service/Indexer/IndexDocument.php @@ -0,0 +1,14 @@ + !is_null($value)); + } +} diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php index fe6b4e8..eec98c6 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php @@ -7,9 +7,14 @@ use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Resource; use Atoolo\Search\Service\Indexer\DocumentEnricher; +use Atoolo\Search\Service\Indexer\IndexDocument; +use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use DateTime; use Solarium\Core\Query\DocumentInterface; +/** + * @implements DocumentEnricher + */ class DefaultSchema21DocumentEnricher implements DocumentEnricher { public function __construct( @@ -22,11 +27,12 @@ public function isIndexable(Resource $resource): bool $noIndex = $resource->getData()->getBool('init.noIndex'); return $noIndex !== true; } + public function enrichDocument( Resource $resource, - DocumentInterface $doc, + IndexDocument $doc, string $processId - ): DocumentInterface { + ): IndexDocument { $doc->sp_id = $resource->getId(); $doc->sp_name = $resource->getName(); $doc->sp_anchor = $resource->getData()->getString('init.anchor'); diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 2318877..2e29ca0 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -243,6 +243,7 @@ private function add( $client = $this->clientFactory->create($solrCore); $update = $client->createUpdate(); + $update->setDocumentClass(IndexSchema2xDocument::class); $documents = []; foreach ($resources as $resource) { diff --git a/test/Service/Indexer/IndexSchema21DocumentTest.php b/test/Service/Indexer/IndexSchema21DocumentTest.php new file mode 100644 index 0000000..fc1d2a7 --- /dev/null +++ b/test/Service/Indexer/IndexSchema21DocumentTest.php @@ -0,0 +1,23 @@ +sp_id = '123'; + + $this->assertEquals( + ['sp_id' => '123'], + $doc->getFields(), + 'unexpected fields' + ); + } +} From 242c169323e5e0db0ea86661acaceb595eccb6d2 Mon Sep 17 00:00:00 2001 From: veltrup Date: Thu, 18 Jan 2024 09:51:55 +0100 Subject: [PATCH 017/145] refactor: DefaultSchema21DocumentEnricher -> DocumentEnricherSiteKitSchema2x --- config/services.yml | 2 +- src/Console/Command/Indexer.php | 4 ++-- ...cumentEnricher.php => DefaultSchema2xDocumentEnricher.php} | 3 +-- ...Schema21DocumentTest.php => IndexSchema2xDocumentTest.php} | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) rename src/Service/Indexer/SiteKit/{DefaultSchema21DocumentEnricher.php => DefaultSchema2xDocumentEnricher.php} (99%) rename test/Service/Indexer/{IndexSchema21DocumentTest.php => IndexSchema2xDocumentTest.php} (89%) diff --git a/config/services.yml b/config/services.yml index 5fdc303..3048dea 100644 --- a/config/services.yml +++ b/config/services.yml @@ -11,7 +11,7 @@ services: Atoolo\Search\Console\Command\Indexer: arguments: - - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher' } + - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } Atoolo\Search\Console\Application: public: true diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 0b367fb..c8eaaf9 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -12,7 +12,7 @@ use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\LocationFinder; -use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema21DocumentEnricher; +use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\SolrParameterClientFactory; @@ -171,7 +171,7 @@ protected function createIndexer(): SolrIndexer $navigationLoader = new SiteKitNavigationHierarchyLoader( $resourceLoader ); - $schema21 = new DefaultSchema21DocumentEnricher( + $schema21 = new DefaultSchema2xDocumentEnricher( $navigationLoader ); diff --git a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php similarity index 99% rename from src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php rename to src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index eec98c6..e3b0175 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema21DocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -10,12 +10,11 @@ use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use DateTime; -use Solarium\Core\Query\DocumentInterface; /** * @implements DocumentEnricher */ -class DefaultSchema21DocumentEnricher implements DocumentEnricher +class DefaultSchema2xDocumentEnricher implements DocumentEnricher { public function __construct( private readonly SiteKitNavigationHierarchyLoader $navigationLoader diff --git a/test/Service/Indexer/IndexSchema21DocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php similarity index 89% rename from test/Service/Indexer/IndexSchema21DocumentTest.php rename to test/Service/Indexer/IndexSchema2xDocumentTest.php index fc1d2a7..570079d 100644 --- a/test/Service/Indexer/IndexSchema21DocumentTest.php +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -7,7 +7,7 @@ use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use PHPUnit\Framework\TestCase; -class IndexSchema21DocumentTest extends TestCase +class IndexSchema2xDocumentTest extends TestCase { public function testGetFields(): void { From 5ba12f4ce808d93d8229b7e821fb687bee1cdc60 Mon Sep 17 00:00:00 2001 From: veltrup Date: Thu, 18 Jan 2024 10:59:00 +0100 Subject: [PATCH 018/145] feat: nullable strings for index-documents --- src/Service/Indexer/IndexSchema2xDocument.php | 68 ++++++++++++------- .../Indexer/IndexSchema2xDocumentTest.php | 12 ++++ 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 54b43ab..ee03398 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -10,32 +10,32 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface { public string $sp_id; - public string $sp_name; - public string $sp_anchor; - public string $title; - public string $description; - public string $sp_objecttype; + public ?string $sp_name; + public ?string $sp_anchor; + public ?string $title; + public ?string $description; + public ?string $sp_objecttype; public bool $sp_canonical; - public string $crawl_process_id; - public string $id; - public string $url; + public ?string $crawl_process_id; + public ?string $id; + public ?string $url; /** * @var string[] */ public array $sp_contenttype; - public string $sp_language; - public string $meta_content_language; - public string $meta_content_type; - public DateTime $sp_changed; - public DateTime $sp_generated; - public DateTime $sp_date; + public ?string $sp_language; + public ?string $meta_content_language; + public ?string $meta_content_type; + public ?DateTime $sp_changed; + public ?DateTime $sp_generated; + public ?DateTime $sp_date; /** * @var DateTime[] */ public array $sp_date_list; public bool $sp_archive; - public string $sp_title; - public string $sp_sortvalue; + public ?string $sp_title; + public ?string $sp_sortvalue; /** * @var string[] */ @@ -65,7 +65,7 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface * @var int[] */ public array $sp_group_path; - public string $content; + public ?string $content; /** * @var string[] */ @@ -89,9 +89,9 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface */ public array $sp_citygov_email; - public string $sp_citygov_address; + public ?string $sp_citygov_address; - public string $sp_citygov_startletter; + public ?string $sp_citygov_startletter; /** * @var string[] @@ -100,9 +100,9 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface public int $sp_organisation; - public string $sp_citygov_firstname; + public ?string $sp_citygov_firstname; - public string $sp_citygov_lastname; + public ?string $sp_citygov_lastname; /** * List of Organisation Id's @@ -122,11 +122,33 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface */ public array $sp_citygov_product; - public string $sp_citygov_function; + public ?string $sp_citygov_function; + /** + * @var array + */ + private array $metaString = []; + + public function setMetaString(string $name, string $value): void + { + $this->metaString[$name] = $value; + } public function getFields(): array { $fields = get_object_vars($this); - return array_filter($fields, fn($value) => !is_null($value)); + $fields = array_filter($fields, function ($value, $key) { + if (is_null($value)) { + return false; + } + if ($key === 'metaString') { + return false; + } + return true; + }, ARRAY_FILTER_USE_BOTH); + foreach ($this->metaString as $key => $value) { + $fields['sp_meta_string_' . $key] = $value; + } + + return $fields; } } diff --git a/test/Service/Indexer/IndexSchema2xDocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php index 570079d..e6f0f2a 100644 --- a/test/Service/Indexer/IndexSchema2xDocumentTest.php +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -20,4 +20,16 @@ public function testGetFields(): void 'unexpected fields' ); } + + public function testSetMetaString(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaString('myname', 'myvalue'); + + $this->assertEquals( + ['sp_meta_string_myname' => 'myvalue'], + $doc->getFields(), + 'unexpected meta fields' + ); + } } From e7021d61d7159713c7231992ffe5fa9e9c50552e Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 23 Jan 2024 09:06:05 +0100 Subject: [PATCH 019/145] feat: stability --- src/Dto/Indexer/IndexerStatus.php | 2 +- src/Exception/DocumentEnrichingException.php | 28 +++++++++++++++++++ src/Service/Indexer/BackgroundIndexer.php | 8 ++++-- .../BackgroundIndexerProgressState.php | 19 +++++++++++-- src/Service/Indexer/DocumentEnricher.php | 2 ++ src/Service/Indexer/IndexSchema2xDocument.php | 17 +++++++---- src/Service/Indexer/LocationFinder.php | 2 ++ .../DefaultSchema2xDocumentEnricher.php | 11 ++++++-- src/Service/Indexer/SolrIndexer.php | 24 +++++++++------- 9 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 src/Exception/DocumentEnrichingException.php diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php index 414d83d..2de5136 100644 --- a/src/Dto/Indexer/IndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -41,7 +41,7 @@ public static function empty(): IndexerStatus public function getStatusLine(): string { $endTime = $this->endTime; - if ($endTime === null) { + if ($endTime === null || $endTime->getTimestamp() === 0) { $endTime = new DateTime(); } $duration = $this->startTime->diff($endTime); diff --git a/src/Exception/DocumentEnrichingException.php b/src/Exception/DocumentEnrichingException.php new file mode 100644 index 0000000..8e8392b --- /dev/null +++ b/src/Exception/DocumentEnrichingException.php @@ -0,0 +1,28 @@ +location; + } +} diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 2411f42..7dfcc54 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -10,6 +10,8 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use RuntimeException; use JsonException; use Symfony\Component\Lock\LockFactory; @@ -30,7 +32,8 @@ public function __construct( private readonly SolrClientFactory $clientFactory, private readonly IndexingAborter $aborter, private readonly string $source, - private readonly string $statusCacheDir + private readonly string $statusCacheDir, + private readonly LoggerInterface $logger = new NullLogger() ) { $this->lockFactory = new LockFactory(new SemaphoreStore()); if ( @@ -83,7 +86,8 @@ public function getStatus(string $index): IndexerStatus private function getIndexer(string $index): SolrIndexer { $progressHandler = new BackgroundIndexerProgressState( - $this->getStatusFile($index) + $this->getStatusFile($index), + $this->logger ); return new SolrIndexer( $this->documentEnricherList, diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index f3cb3d5..55fad3d 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -7,6 +7,8 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Dto\Indexer\IndexerStatusState; use DateTime; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use Throwable; use JsonException; @@ -16,8 +18,10 @@ class BackgroundIndexerProgressState implements IndexerProgressHandler private bool $isUpdate = false; - public function __construct(private readonly string $file) - { + public function __construct( + private readonly string $file, + private readonly LoggerInterface $logger = new NullLogger() + ) { } public function start(int $total): void @@ -74,6 +78,12 @@ public function skip(int $step): void public function error(Throwable $throwable): void { $this->status->errors++; + $this->logger->error( + $throwable->getMessage(), + [ + 'exception' => $throwable, + ] + ); } /** @@ -107,4 +117,9 @@ public function getStatus(): IndexerStatus { return $this->status; } + + public function getStatusFile(): string + { + return $this->file; + } } diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php index 60f75e1..6b3b06f 100644 --- a/src/Service/Indexer/DocumentEnricher.php +++ b/src/Service/Indexer/DocumentEnricher.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\Resource; +use Atoolo\Search\Exception\DocumentEnrichingException; /** * This interface can be used to implement enricher with the help of which a @@ -19,6 +20,7 @@ public function isIndexable(Resource $resource): bool; /** * @param T $doc * @return T + * @throws DocumentEnrichingException */ public function enrichDocument( Resource $resource, diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index ee03398..4ce811a 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -5,9 +5,10 @@ namespace Atoolo\Search\Service\Indexer; use DateTime; -use Solarium\Core\Query\DocumentInterface; +use DateTimeInterface; +use Solarium\QueryType\Update\Query\Document; -class IndexSchema2xDocument implements IndexDocument, DocumentInterface +class IndexSchema2xDocument extends Document implements IndexDocument { public string $sp_id; public ?string $sp_name; @@ -40,10 +41,7 @@ class IndexSchema2xDocument implements IndexDocument, DocumentInterface * @var string[] */ public array $keywords; - /** - * @var string[] - */ - public array $sp_boost_keywords; + public string $sp_boost_keywords; /** * @var string[] */ @@ -149,6 +147,13 @@ public function getFields(): array $fields['sp_meta_string_' . $key] = $value; } + $fields = array_map(function ($value) { + if ($value instanceof DateTime) { + return $value->format(DateTimeInterface::ATOM); + } + return $value; + }, $fields); + return $fields; } } diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 3ca0458..fd8a9da 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -23,6 +23,7 @@ public function findAll(): array $finder = new Finder(); $finder->in($this->getBasePath())->exclude('WEB-IES'); $finder->name('*.php'); + $finder->notPath('*-1015t.php*'); // preview files $finder->files(); $pathList = []; @@ -63,6 +64,7 @@ public function findPaths(array $paths): array $finder->in($this->getBasePath() . '/' . $directory); } $finder->name('*.php'); + $finder->notPath('*-1015t.php*'); // preview files $finder->files(); foreach ($finder as $file) { diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index e3b0175..43ac7b6 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Resource; +use Atoolo\Search\Exception\DocumentEnrichingException; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; @@ -27,6 +28,9 @@ public function isIndexable(Resource $resource): bool return $noIndex !== true; } + /** + * @throws DocumentEnrichingException + */ public function enrichDocument( Resource $resource, IndexDocument $doc, @@ -116,8 +120,11 @@ public function enrichDocument( $doc->keywords = $resource->getData()->getArray('metadata.keywords'); - $doc->sp_boost_keywords = $resource->getData()->getArray( - 'metadata.boostKeywords' + $doc->sp_boost_keywords = implode( + ' ', + $resource->getData()->getArray( + 'metadata.boostKeywords' + ) ); $sites = $this->getParentSiteGroupIdList($resource); diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 2e29ca0..58ab24b 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -14,6 +14,7 @@ use Exception; use Solarium\Core\Query\Result\ResultInterface; use Solarium\QueryType\Update\Result; +use Throwable; /** * Implementation of the indexer on the basis of a Solr index. @@ -222,7 +223,7 @@ private function loadResources( try { $resource = $this->resourceLoader->load($path); $resourceList[] = $resource; - } catch (InvalidResourceException $e) { + } catch (Throwable $e) { $this->indexerProgressHandler->error($e); } } @@ -253,17 +254,20 @@ private function add( continue 2; } } - $doc = $update->createDocument(); - foreach ($this->documentEnricherList as $enricher) { - $doc = $enricher->enrichDocument( - $resource, - $doc, - $processId - ); + try { + $doc = $update->createDocument(); + foreach ($this->documentEnricherList as $enricher) { + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + $documents[] = $doc; + } catch (Throwable $e) { + $this->indexerProgressHandler->error($e); } - $documents[] = $doc; } - // add the documents and a commit command to the update query $update->addDocuments($documents); From 5f16d14d63242703d10f8fc848ea59d1b479bfd5 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 23 Jan 2024 11:22:45 +0100 Subject: [PATCH 020/145] fix: error message --- src/Console/Command/Indexer.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index c8eaaf9..ee20e8c 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -150,14 +150,7 @@ private function getIntOption(string $name): int protected function errorReport(): void { foreach ($this->progressBar->getErrors() as $error) { - if ($error instanceof InvalidResourceException) { - $this->io->error( - $error->getLocation() . ': ' . - $error->getMessage() - ); - } else { - $this->io->error($error->getMessage()); - } + $this->io->error($error->getMessage()); } } From 086e90be57b8aa4c96f7e38d35f915371e11a1c7 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 23 Jan 2024 11:23:31 +0100 Subject: [PATCH 021/145] feat: cache error, to see the right location --- .../DefaultSchema2xDocumentEnricher.php | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 43ac7b6..acb5b56 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -11,6 +11,7 @@ use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use DateTime; +use Exception; /** * @implements DocumentEnricher @@ -127,16 +128,28 @@ public function enrichDocument( ) ); - $sites = $this->getParentSiteGroupIdList($resource); + try { + $sites = $this->getParentSiteGroupIdList($resource); - $navigationRoot = $this->navigationLoader->loadRoot( - $resource->getLocation() - ); - $siteGroupId = $navigationRoot->getData()->getInt('init.siteGroup.id'); - if ($siteGroupId !== 0) { - $sites[] = $siteGroupId; + $navigationRoot = $this->navigationLoader->loadRoot( + $resource->getLocation() + ); + + $siteGroupId = $navigationRoot->getData()->getInt( + 'init.siteGroup.id' + ); + if ($siteGroupId !== 0) { + $sites[] = $siteGroupId; + } + $doc->sp_site = array_unique($sites); + } catch (Exception $e) { + throw new DocumentEnrichingException( + $resource->getLocation(), + 'Unable to set sp_site', + 0, + $e + ); } - $doc->sp_site = array_unique($sites); $wktPrimaryList = $resource->getData()->getArray( 'base.geo.wkt.primary' From d7353031a96c06df564a65c66aa9b54e33c2afdf Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 23 Jan 2024 13:35:03 +0100 Subject: [PATCH 022/145] fix: indexing --- src/Console/Command/Indexer.php | 11 +++++++++-- src/Service/Indexer/IndexSchema2xDocument.php | 7 ------- .../SiteKit/DefaultSchema2xDocumentEnricher.php | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index ee20e8c..f9ec666 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Exception\InvalidResourceException; +use Atoolo\Resource\Loader\ServerVarResourceBaseLocator; use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Loader\StaticResourceBaseLocator; @@ -156,8 +157,14 @@ protected function errorReport(): void protected function createIndexer(): SolrIndexer { - $resourceBaseLocator = new StaticResourceBaseLocator( - $this->resourceDir + $subDirectory = null; + if (is_dir($this->resourceDir . '/objects')) { + $subDirectory = 'objects'; + } + $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; + $resourceBaseLocator = new ServerVarResourceBaseLocator( + 'RESOURCE_ROOT', + $subDirectory ); $finder = new LocationFinder($resourceBaseLocator); $resourceLoader = new SiteKitLoader($resourceBaseLocator); diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 4ce811a..232f1c2 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -147,13 +147,6 @@ public function getFields(): array $fields['sp_meta_string_' . $key] = $value; } - $fields = array_map(function ($value) { - if ($value instanceof DateTime) { - return $value->format(DateTimeInterface::ATOM); - } - return $value; - }, $fields); - return $fields; } } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index acb5b56..9071b4d 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -145,7 +145,7 @@ public function enrichDocument( } catch (Exception $e) { throw new DocumentEnrichingException( $resource->getLocation(), - 'Unable to set sp_site', + 'Unable to set sp_site: ' . $e->getMessage(), 0, $e ); From 07074101a0c984cd45ebcff4d48a97f21ad2bea8 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 24 Jan 2024 13:11:17 +0100 Subject: [PATCH 023/145] feat: Support translation information such as /path?loc=en_US --- src/Service/Indexer/SolrIndexer.php | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 58ab24b..496222b 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -59,12 +59,30 @@ public function index(IndexerParameter $parameter): IndexerStatus if (empty($parameter->paths)) { $pathList = $this->finder->findAll(); } else { - $pathList = $this->finder->findPaths($parameter->paths); + $mappedPaths = $this->mapTranslationPaths($parameter->paths); + $pathList = $this->finder->findPaths($mappedPaths); } return $this->indexResources($parameter, $pathList); } + private function mapTranslationPaths(array $pathList): array + { + return array_map(function ($path) { + $queryString = parse_url($path, PHP_URL_QUERY); + if ($queryString === null) { + return $path; + } + $path = parse_url($path, PHP_URL_PATH); + parse_str($queryString, $params); + if (!isset($params['loc'])) { + return $path; + } + $loc = $params['loc']; + return $path . '.translations/' . $loc . ".php"; + }, $pathList); + } + /** * @param array $pathList */ From 14e0d3de3e30e9d473557c7b9f82cde2911a4fb6 Mon Sep 17 00:00:00 2001 From: veltrup Date: Mon, 29 Jan 2024 13:55:17 +0100 Subject: [PATCH 024/145] test: tests made runnable again --- src/Console/Command/Indexer.php | 82 +++----------- src/Console/Command/SolrIndexerBuilder.php | 102 ++++++++++++++++++ src/Service/Indexer/IndexSchema2xDocument.php | 39 +++++-- src/Service/Indexer/LocationFinder.php | 11 +- test/Console/ApplicationTest.php | 10 +- test/Console/Command/IndexerTest.php | 32 ++++-- test/Service/Indexer/LocationFinderTest.php | 55 +++++++++- test/Service/Indexer/SolrIndexerTest.php | 15 ++- test/Service/IndexerTest.php | 19 ---- .../Indexer/LocationFinder/WEB-IES/d.php | 3 + .../Indexer/LocationFinder/a-1015t.php | 3 + .../a-1015t.php.media/a.pdf.media.php | 3 + .../Service/Indexer/LocationFinder/a.php | 3 + .../Indexer/LocationFinder/b/c-1015t.php | 3 + .../Service/Indexer/LocationFinder/b/c.php | 3 + .../Indexer/LocationFinder/f.pdf.media.php | 3 + .../Service/Indexer/LocationFinder/x.txt | 1 + 17 files changed, 273 insertions(+), 114 deletions(-) create mode 100644 src/Console/Command/SolrIndexerBuilder.php delete mode 100644 test/Service/IndexerTest.php create mode 100644 test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php create mode 100644 test/resources/Service/Indexer/LocationFinder/a-1015t.php create mode 100644 test/resources/Service/Indexer/LocationFinder/a-1015t.php.media/a.pdf.media.php create mode 100644 test/resources/Service/Indexer/LocationFinder/a.php create mode 100644 test/resources/Service/Indexer/LocationFinder/b/c-1015t.php create mode 100644 test/resources/Service/Indexer/LocationFinder/b/c.php create mode 100644 test/resources/Service/Indexer/LocationFinder/f.pdf.media.php create mode 100644 test/resources/Service/Indexer/LocationFinder/x.txt diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index f9ec666..4c0897c 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -4,19 +4,8 @@ namespace Atoolo\Search\Console\Command; -use Atoolo\Resource\Exception\InvalidResourceException; -use Atoolo\Resource\Loader\ServerVarResourceBaseLocator; -use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; use Atoolo\Search\Dto\Indexer\IndexerParameter; -use Atoolo\Search\Service\Indexer\IndexingAborter; -use Atoolo\Search\Service\Indexer\LocationFinder; -use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; -use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; -use Atoolo\Search\Service\Indexer\SolrIndexer; -use Atoolo\Search\Service\SolrParameterClientFactory; use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -35,10 +24,11 @@ class Indexer extends Command private SymfonyStyle $io; private InputInterface $input; - private string $resourceDir; - public function __construct(private readonly iterable $documentEnricherList) - { + public function __construct( + private readonly iterable $documentEnricherList, + private readonly SolrIndexerBuilder $solrIndexerBuilder + ) { parent::__construct(); } @@ -96,8 +86,7 @@ protected function execute( $this->input = $input; $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); - $this->resourceDir = $this->getStringArgument('resource-dir'); - $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; + $paths = (array)$input->getArgument('paths'); $cleanupThreshold = empty($paths) @@ -118,7 +107,15 @@ protected function execute( $paths ); - $indexer = $this->createIndexer(); + $this->solrIndexerBuilder + ->resourceDir($this->getStringArgument('resource-dir')) + ->progressBar($this->progressBar) + ->documentEnricherList($this->documentEnricherList) + ->solrConnectionUrl( + $this->getStringArgument('solr-connection-url') + ); + + $indexer = $this->solrIndexerBuilder->build(); $indexer->index($parameter); $this->errorReport(); @@ -154,55 +151,4 @@ protected function errorReport(): void $this->io->error($error->getMessage()); } } - - protected function createIndexer(): SolrIndexer - { - $subDirectory = null; - if (is_dir($this->resourceDir . '/objects')) { - $subDirectory = 'objects'; - } - $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; - $resourceBaseLocator = new ServerVarResourceBaseLocator( - 'RESOURCE_ROOT', - $subDirectory - ); - $finder = new LocationFinder($resourceBaseLocator); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - $navigationLoader = new SiteKitNavigationHierarchyLoader( - $resourceLoader - ); - $schema21 = new DefaultSchema2xDocumentEnricher( - $navigationLoader - ); - - $documentEnricherList = [$schema21]; - foreach ($this->documentEnricherList as $enricher) { - $documentEnricherList[] = $enricher; - } - $url = parse_url($this->getStringArgument('solr-connection-url')); - - $clientFactory = new SolrParameterClientFactory( - $url['scheme'], - $url['host'], - $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382), - $url['path'] ?? '', - null, - 0 - ); - - $translationSplitter = new SubDirTranslationSplitter(); - - $aborter = new IndexingAborter('.'); - - return new SolrIndexer( - $documentEnricherList, - $this->progressBar, - $finder, - $resourceLoader, - $translationSplitter, - $clientFactory, - $aborter, - 'internal' - ); - } } diff --git a/src/Console/Command/SolrIndexerBuilder.php b/src/Console/Command/SolrIndexerBuilder.php new file mode 100644 index 0000000..6aa9883 --- /dev/null +++ b/src/Console/Command/SolrIndexerBuilder.php @@ -0,0 +1,102 @@ +resourceDir = $resourceDir; + return $this; + } + + public function documentEnricherList( + iterable $documentEnricherList + ): SolrIndexerBuilder { + $this->documentEnricherList = $documentEnricherList; + return $this; + } + + public function progressBar( + IndexerProgressProgressBar $progressBar + ): SolrIndexerBuilder { + $this->progressBar = $progressBar; + return $this; + } + + public function solrConnectionUrl( + string $solrConnectionUrl + ): SolrIndexerBuilder { + $this->solrConnectionUrl = $solrConnectionUrl; + return $this; + } + + public function build(): SolrIndexer + { + $subDirectory = null; + if (is_dir($this->resourceDir . '/objects')) { + $subDirectory = 'objects'; + } + $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; + $resourceBaseLocator = new ServerVarResourceBaseLocator( + 'RESOURCE_ROOT', + $subDirectory + ); + $finder = new LocationFinder($resourceBaseLocator); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + $navigationLoader = new SiteKitNavigationHierarchyLoader( + $resourceLoader + ); + $schema21 = new DefaultSchema2xDocumentEnricher( + $navigationLoader + ); + + $documentEnricherList = [$schema21]; + foreach ($this->documentEnricherList as $enricher) { + $documentEnricherList[] = $enricher; + } + $url = parse_url($this->solrConnectionUrl); + + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382), + $url['path'] ?? '', + null, + 0 + ); + + $translationSplitter = new SubDirTranslationSplitter(); + + $aborter = new IndexingAborter('.'); + + return new SolrIndexer( + $documentEnricherList, + $this->progressBar, + $finder, + $resourceLoader, + $translationSplitter, + $clientFactory, + $aborter, + 'internal' + ); + } +} diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 232f1c2..a0c4ff1 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -5,7 +5,6 @@ namespace Atoolo\Search\Service\Indexer; use DateTime; -use DateTimeInterface; use Solarium\QueryType\Update\Query\Document; class IndexSchema2xDocument extends Document implements IndexDocument @@ -103,7 +102,7 @@ class IndexSchema2xDocument extends Document implements IndexDocument public ?string $sp_citygov_lastname; /** - * List of Organisation Id's + * List of Organisation Ids * @var int[] */ public array $sp_organisation_path; @@ -134,15 +133,33 @@ public function setMetaString(string $name, string $value): void public function getFields(): array { $fields = get_object_vars($this); - $fields = array_filter($fields, function ($value, $key) { - if (is_null($value)) { - return false; - } - if ($key === 'metaString') { - return false; - } - return true; - }, ARRAY_FILTER_USE_BOTH); + + // filter out inherited fields + + $fields = array_filter( + $fields, + static function ($value, $key) { + + if (is_null($value)) { + return false; + } + + $inheritedFields = [ + 'fields', + 'modifiers', + 'fieldBoosts' + ]; + if (in_array($key, $inheritedFields, true)) { + return false; + } + + if ($key === 'metaString') { + return false; + } + return true; + }, + ARRAY_FILTER_USE_BOTH + ); foreach ($this->metaString as $key => $value) { $fields['sp_meta_string_' . $key] = $value; } diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index fd8a9da..5800a4a 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -23,7 +23,7 @@ public function findAll(): array $finder = new Finder(); $finder->in($this->getBasePath())->exclude('WEB-IES'); $finder->name('*.php'); - $finder->notPath('*-1015t.php*'); // preview files + $finder->notPath('#.*-1015t.php.*#'); // preview files $finder->files(); $pathList = []; @@ -46,7 +46,10 @@ public function findPaths(array $paths): array $finder = new Finder(); foreach ($paths as $path) { - $absolutePath = $this->getBasePath() . '/' . $path; + if (!str_starts_with($path, '/')) { + $path = '/' . $path; + } + $absolutePath = $this->getBasePath() . $path; if (is_file($absolutePath)) { $pathList[] = $path; continue; @@ -61,10 +64,10 @@ public function findPaths(array $paths): array } foreach ($directories as $directory) { - $finder->in($this->getBasePath() . '/' . $directory); + $finder->in($this->getBasePath() . $directory); } $finder->name('*.php'); - $finder->notPath('*-1015t.php*'); // preview files + $finder->notPath('#.*-1015t.php.*#'); // preview files $finder->files(); foreach ($finder as $file) { diff --git a/test/Console/ApplicationTest.php b/test/Console/ApplicationTest.php index 3e174ac..804ef8d 100644 --- a/test/Console/ApplicationTest.php +++ b/test/Console/ApplicationTest.php @@ -6,14 +6,22 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\SolrIndexerBuilder; +use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; class ApplicationTest extends TestCase { + /** + * @throws Exception + */ public function testConstruct(): void { + $indexBuilder = $this->createStub( + SolrIndexerBuilder::class + ); $application = new Application([ - new Indexer() + new Indexer([], $indexBuilder) ]); $command = $application->get('atoolo:indexer'); $this->assertInstanceOf( diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 855b898..2189b32 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -6,15 +6,26 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\SolrIndexerBuilder; +use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Tester\CommandTester; class IndexerTest extends TestCase { + /** + * @throws Exception + */ public function testExecute(): void { + $indexBuilder = $this->createStub( + SolrIndexerBuilder::class + ); $application = new Application([ - new Indexer() + new Indexer( + [], + $indexBuilder + ) ]); $command = $application->find('atoolo:indexer'); @@ -22,20 +33,23 @@ public function testExecute(): void $commandTester->execute([ // pass arguments to the helper 'resource-dir' => 'abc', - - // prefix the key with two dashes when passing options, - // e.g: '--some-option' => 'option_value', - // use brackets for testing array value, - // e.g: '--some-option' => ['option_value'], + 'solr-connection-url' => 'http://localhost:8080', + 'solr-core' => 'test' ]); $commandTester->assertCommandIsSuccessful(); // the output of the command in the console $output = $commandTester->getDisplay(); - $this->assertStringContainsString('Whoa!', $output); + $this->assertEquals( + <<locationFinder = new LocationFinder( + new StaticResourceBaseLocator($base) + ); + } + public function testFindAll(): void { - $baseLocator = new StaticResourceBaseLocator(''); + $locations = $this->locationFinder->findAll(); + $this->assertEquals( + [ + '/a.php', + '/b/c.php', + '/f.pdf.media.php' + ], + $locations, + 'unexpected locations' + ); + } + + public function testFindPathsWithFile(): void + { + $locations = $this->locationFinder->findPaths( + [ + 'a.php' + ] + ); + $this->assertEquals( + [ + '/a.php', + ], + $locations, + 'unexpected locations' + ); + } + + public function testFindPathsWithDirectory(): void + { + $locations = $this->locationFinder->findPaths( + [ + '/b', + ] + ); + $this->assertEquals( + [ + '/b/c.php', + ], + $locations, + 'unexpected locations' + ); } } diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/SolrIndexerTest.php index df7864b..36f3c30 100644 --- a/test/Service/Indexer/SolrIndexerTest.php +++ b/test/Service/Indexer/SolrIndexerTest.php @@ -9,15 +9,18 @@ use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; +use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use Atoolo\Search\Service\Indexer\LocationFinder; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\Indexer\TranslationSplitter; use Atoolo\Search\Service\SolrClientFactory; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\Attributes\CoversClass; use Solarium\Client; +use Solarium\QueryType\Server\CoreAdmin\Result\Result as CoreAdminResult; +use Solarium\QueryType\Server\CoreAdmin\Result\StatusResult; use Solarium\QueryType\Update\Query\Query as UpdateQuery; use Solarium\QueryType\Update\Result as UpdateResult; @@ -67,10 +70,20 @@ public function setUp(): void $this->updateResult = $this->createStub(UpdateResult::class); $this->solrClient = $this->createMock(Client::class); $this->updateQuery = $this->createMock(UpdateQuery::class); + $this->updateQuery->method('createDocument') + ->willReturn($this->createStub(IndexSchema2xDocument::class)); + $coreAdminResult = $this->createStub(CoreAdminResult::class); + $coreAdminResultStatus = $this->createStub(StatusResult::class); + $coreAdminResultStatus->method('getCoreName') + ->willReturn('test'); + $coreAdminResult->method('getStatusResults') + ->willReturn([$coreAdminResultStatus]); $this->solrClient->method('createUpdate') ->willReturn($this->updateQuery); $this->solrClient->method('update') ->willReturn($this->updateResult); + $this->solrClient->method('coreAdmin') + ->willReturn($coreAdminResult); $solrClientFactory->method('create') ->willReturn($this->solrClient); $this->aborter = $this->createMock(IndexingAborter::class); diff --git a/test/Service/IndexerTest.php b/test/Service/IndexerTest.php deleted file mode 100644 index ae1a00d..0000000 --- a/test/Service/IndexerTest.php +++ /dev/null @@ -1,19 +0,0 @@ -index(); - */ - } -} diff --git a/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php b/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php new file mode 100644 index 0000000..174d7fd --- /dev/null +++ b/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php @@ -0,0 +1,3 @@ + Date: Tue, 30 Jan 2024 09:03:07 +0100 Subject: [PATCH 025/145] fix: phpstan errors --- src/Console/Command/Indexer.php | 22 +++++++- .../Command/Io/IndexerProgressProgressBar.php | 5 +- src/Console/Command/MoreLikeThis.php | 29 +++++++--- src/Console/Command/Search.php | 29 +++++++--- src/Console/Command/SolrIndexerBuilder.php | 14 ++++- src/Console/Command/Suggest.php | 39 ++++++++++++-- src/Dto/Indexer/IndexerParameter.php | 3 ++ src/Dto/Indexer/IndexerStatus.php | 19 +++++++ src/Dto/Search/Query/Facet/CategoryFacet.php | 2 +- src/Dto/Search/Query/Facet/Facet.php | 2 +- src/Dto/Search/Query/Filter/FieldFilter.php | 8 +-- src/Dto/Search/Query/MoreLikeThisQuery.php | 2 +- src/Dto/Search/Query/SelectQueryBuilder.php | 10 ---- src/Dto/Search/Query/SuggestQuery.php | 3 -- src/Dto/Search/Result/FacetGroup.php | 2 +- src/Dto/Search/Result/SuggestResult.php | 9 +++- src/Indexer.php | 5 +- src/Service/Indexer/BackgroundIndexer.php | 7 ++- src/Service/Indexer/IndexSchema2xDocument.php | 3 ++ .../DefaultSchema2xDocumentEnricher.php | 54 +++++++++++-------- .../SiteKit/SubDirTranslationSplitter.php | 3 ++ src/Service/Indexer/SolrIndexer.php | 41 ++++++++------ src/Service/Indexer/TranslationSplitter.php | 3 ++ .../Search/ExternalResourceFactory.php | 24 +++++++-- .../Search/InternalMediaResourceFactory.php | 13 ++++- .../Search/InternalResourceFactory.php | 16 ++++-- src/Service/Search/SolrMoreLikeThis.php | 13 +++-- .../Search/SolrResultToResourceResolver.php | 20 +++++-- src/Service/Search/SolrSelect.php | 51 +++++++++++------- src/Service/Search/SolrSuggest.php | 16 ++++-- src/Service/SolrParameterClientFactory.php | 6 +-- 31 files changed, 334 insertions(+), 139 deletions(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 4c0897c..20b4aeb 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -6,6 +6,8 @@ use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; use Atoolo\Search\Dto\Indexer\IndexerParameter; +use Atoolo\Search\Service\Indexer\DocumentEnricher; +use Atoolo\Search\Service\Indexer\IndexDocument; use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -25,6 +27,10 @@ class Indexer extends Command private InputInterface $input; + /** + * phpcs:ignore + * @param iterable> $documentEnricherList + */ public function __construct( private readonly iterable $documentEnricherList, private readonly SolrIndexerBuilder $solrIndexerBuilder @@ -87,7 +93,7 @@ protected function execute( $this->io = new SymfonyStyle($input, $output); $this->progressBar = new IndexerProgressProgressBar($output); - $paths = (array)$input->getArgument('paths'); + $paths = $this->getArrayArgument('paths'); $cleanupThreshold = empty($paths) ? $this->getIntOption('cleanup-threshold') @@ -145,6 +151,20 @@ private function getIntOption(string $name): int return (int)$value; } + /** + * @return string[] + */ + private function getArrayArgument(string $name): array + { + $value = $this->input->getArgument($name); + if (!is_array($value)) { + throw new InvalidArgumentException( + $name . ' must be a array' + ); + } + return $value; + } + protected function errorReport(): void { foreach ($this->progressBar->getErrors() as $error) { diff --git a/src/Console/Command/Io/IndexerProgressProgressBar.php b/src/Console/Command/Io/IndexerProgressProgressBar.php index a7214b0..f166fba 100644 --- a/src/Console/Command/Io/IndexerProgressProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressProgressBar.php @@ -8,9 +8,9 @@ use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use DateTime; -use Throwable; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\OutputInterface; +use Throwable; class IndexerProgressProgressBar implements IndexerProgressHandler { @@ -18,6 +18,9 @@ class IndexerProgressProgressBar implements IndexerProgressHandler private ProgressBar $progressBar; private IndexerStatus $status; + /** + * @var array + */ private array $errors = []; public function __construct(OutputInterface $output) diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 1709bbc..7e07ca2 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -14,7 +14,7 @@ use Atoolo\Search\Service\Search\SolrMoreLikeThis; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\SolrParameterClientFactory; -use Psr\Log\NullLogger; +use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -50,12 +50,13 @@ protected function configure(): void ->addArgument( 'resource-dir', InputArgument::REQUIRED, - 'Resource directory whose data is to be indexed.' + 'Resource directory where the resources can be found.' ) ->addArgument( 'location', InputArgument::REQUIRED, - 'Resource directory whose data is to be indexed.' + 'Location of the resource to which the MoreLikeThis ' . + 'search is to be applied..' ) ; } @@ -68,9 +69,9 @@ protected function execute( $this->input = $input; $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $input->getArgument('solr-core'); - $this->resourceDir = $input->getArgument('resource-dir'); - $location = $input->getArgument('location'); + $this->solrCore = $this->getStringArgument('solr-core'); + $this->resourceDir = $this->getStringArgument('resource-dir'); + $location = $this->getStringArgument('location'); $searcher = $this->createSearcher(); $query = $this->buildQuery($location); @@ -80,17 +81,29 @@ protected function execute( return Command::SUCCESS; } + private function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + $name . ' must be a string' + ); + } + return $value; + } + protected function createSearcher(): SolrMoreLikeThis { $resourceBaseLocator = new StaticResourceBaseLocator( $this->resourceDir ); $resourceLoader = new SiteKitLoader($resourceBaseLocator); - $url = parse_url($this->input->getArgument('solr-connection-url')); + /** @var string[] */ + $url = parse_url($this->getStringArgument('solr-connection-url')); $clientFactory = new SolrParameterClientFactory( $url['scheme'], $url['host'], - $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), $url['path'] ?? '', null, 0 diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 039e47f..c06cd09 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -15,6 +15,7 @@ use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\Search\SolrSelect; use Atoolo\Search\Service\SolrParameterClientFactory; +use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -63,8 +64,8 @@ protected function execute( $this->input = $input; $this->io = new SymfonyStyle($input, $output); - $this->resourceDir = $input->getArgument('resource-dir'); - $this->index = $input->getArgument('index'); + $this->resourceDir = $this->getStringArgument('resource-dir'); + $this->index = $this->getStringArgument('index'); $searcher = $this->createSearch(); $query = $this->buildQuery($input); @@ -76,17 +77,29 @@ protected function execute( return Command::SUCCESS; } + private function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + $name . ' must be a string' + ); + } + return $value; + } + protected function createSearch(): SolrSelect { $resourceBaseLocator = new StaticResourceBaseLocator( $this->resourceDir ); $resourceLoader = new SiteKitLoader($resourceBaseLocator); - $url = parse_url($this->input->getArgument('solr-connection-url')); + /** @var string[] */ + $url = parse_url($this->getStringArgument('solr-connection-url')); $clientFactory = new SolrParameterClientFactory( $url['scheme'], $url['host'], - $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), $url['path'] ?? '', null, 0 @@ -129,18 +142,18 @@ protected function buildQuery(InputInterface $input): SelectQuery protected function outputResult( SearchResult $result - ) { + ): void { $this->io->title('Results (' . $result->getTotal() . ')'); foreach ($result as $resource) { $this->io->text($resource->getLocation()); } - if (count($result->getFacetGroupList()) > 0) { + if (count($result->getFacetGroups()) > 0) { $this->io->title('Facets'); - foreach ($result->getFacetGroupList() as $facetGroup) { + foreach ($result->getFacetGroups() as $facetGroup) { $this->io->section($facetGroup->getKey()); $listing = []; - foreach ($facetGroup->getFacetList() as $facet) { + foreach ($facetGroup->getFacets() as $facet) { $listing[] = $facet->getKey() . ' (' . $facet->getHits() . ')'; diff --git a/src/Console/Command/SolrIndexerBuilder.php b/src/Console/Command/SolrIndexerBuilder.php index 6aa9883..c9525b0 100644 --- a/src/Console/Command/SolrIndexerBuilder.php +++ b/src/Console/Command/SolrIndexerBuilder.php @@ -8,6 +8,8 @@ use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; +use Atoolo\Search\Service\Indexer\DocumentEnricher; +use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\LocationFinder; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; @@ -18,6 +20,10 @@ class SolrIndexerBuilder { private string $resourceDir; + /** + * phpcs:ignore + * @var iterable> + */ private iterable $documentEnricherList; private IndexerProgressProgressBar $progressBar; private string $solrConnectionUrl; @@ -28,6 +34,10 @@ public function resourceDir(string $resourceDir): SolrIndexerBuilder return $this; } + /** + * phpcs:ignore + * @param iterable> $documentEnricherList + */ public function documentEnricherList( iterable $documentEnricherList ): SolrIndexerBuilder { @@ -69,16 +79,18 @@ public function build(): SolrIndexer $navigationLoader ); + /** @var array> $documentEnricherList */ $documentEnricherList = [$schema21]; foreach ($this->documentEnricherList as $enricher) { $documentEnricherList[] = $enricher; } + /** @var string[] $url */ $url = parse_url($this->solrConnectionUrl); $clientFactory = new SolrParameterClientFactory( $url['scheme'], $url['host'], - $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382), + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382)), $url['path'] ?? '', null, 0 diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index e40638b..edc8ff0 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -10,6 +10,7 @@ use Atoolo\Search\Dto\Search\Result\SuggestResult; use Atoolo\Search\Service\Search\SolrSuggest; use Atoolo\Search\Service\SolrParameterClientFactory; +use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -50,8 +51,8 @@ protected function execute( ): int { $this->input = $input; $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $input->getArgument('solr-core'); - $terms = $input->getArgument('terms'); + $this->solrCore = $this->getStringArgument('solr-core'); + $terms = $this->getArrayArgument('terms'); $search = $this->createSearcher(); $query = $this->buildQuery($terms); @@ -65,12 +66,12 @@ protected function execute( protected function createSearcher(): SolrSuggest { - $clientFactory = new SolrParameterClientFactory(); - $url = parse_url($this->input->getArgument('solr-connection-url')); + /** @var string[] $url */ + $url = parse_url($this->getStringArgument('solr-connection-url')); $clientFactory = new SolrParameterClientFactory( $url['scheme'], $url['host'], - $url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983), + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), $url['path'] ?? '', null, 0 @@ -78,6 +79,34 @@ protected function createSearcher(): SolrSuggest return new SolrSuggest($clientFactory); } + private function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + $name . ' must be a string' + ); + } + return $value; + } + + /** + * @return string[] + */ + private function getArrayArgument(string $name): array + { + $value = $this->input->getArgument($name); + if (!is_array($value)) { + throw new InvalidArgumentException( + $name . ' must be a array' + ); + } + return $value; + } + + /** + * @param string[] $terms + */ protected function buildQuery(array $terms): SuggestQuery { $excludeMedia = new ObjectTypeFilter('media', 'media'); diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index ac68625..456926d 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -6,6 +6,9 @@ class IndexerParameter { + /** + * @param string[] $paths + */ public function __construct( public readonly string $index, public readonly int $cleanupThreshold = 0, diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php index 2de5136..4b2ce3d 100644 --- a/src/Dto/Indexer/IndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -5,8 +5,23 @@ namespace Atoolo\Search\Dto\Indexer; use DateTime; +use InvalidArgumentException; use JsonException; +/** + * @phpstan-type JsonStatus array{ + * state: ?string, + * statusLine: ?string, + * startTime: int, + * endTime: ?int, + * total: int, + * processed: int, + * skipped: ?int, + * lastUpdate: ?int, + * updated: ?int, + * errors: ?int + * } + */ class IndexerStatus { public function __construct( @@ -65,6 +80,10 @@ public static function load(string $file): IndexerStatus return self::empty(); } $content = file_get_contents($file); + if ($content === false) { + throw new InvalidArgumentException('Cannot read file ' . $file); + } + /** @var JsonStatus $data */ $data = json_decode( $content, true, diff --git a/src/Dto/Search/Query/Facet/CategoryFacet.php b/src/Dto/Search/Query/Facet/CategoryFacet.php index fc03bf0..90d13c4 100644 --- a/src/Dto/Search/Query/Facet/CategoryFacet.php +++ b/src/Dto/Search/Query/Facet/CategoryFacet.php @@ -7,7 +7,7 @@ class CategoryFacet extends FacetField { /** - * @param string[] $groups + * @param string[] $categories */ public function __construct( string $key, diff --git a/src/Dto/Search/Query/Facet/Facet.php b/src/Dto/Search/Query/Facet/Facet.php index 56eb375..e71f2ac 100644 --- a/src/Dto/Search/Query/Facet/Facet.php +++ b/src/Dto/Search/Query/Facet/Facet.php @@ -6,6 +6,6 @@ interface Facet { - public function getKey(): ?string; + public function getKey(): string; public function getExcludeFilter(): ?string; } diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index e0bef8f..524b591 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -10,9 +10,6 @@ class FieldFilter extends Filter * @var string[] */ private readonly array $values; - /** - * @param string[] $values - */ public function __construct( ?string $key, private readonly string $field, @@ -26,13 +23,10 @@ public function __construct( $this->values = $values; parent::__construct( $key, - [$key] + $key !== null ? [$key] : [] ); } - /** - * @param string[] $values - */ public function getQuery(): string { $filterValue = count($this->values) === 1 diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index 2757a24..2cc391a 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -9,7 +9,7 @@ class MoreLikeThisQuery { /** - * @param string[] $fields + * @param string[] $fieldList * @param Filter[] $filterList */ public function __construct( diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 15c754f..60b759d 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -7,7 +7,6 @@ use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; -use GuzzleHttp\Promise\Create; class SelectQueryBuilder { @@ -104,9 +103,6 @@ public function getLimit(): int return $this->limit; } - /** - * @param Criteria[] $criteriaList - */ public function sort(Criteria ...$criteriaList): SelectQueryBuilder { foreach ($criteriaList as $criteria) { @@ -124,9 +120,6 @@ public function getSort(): array return $this->sort; } - /** - * @param Filter[] $filterList - */ public function filter(Filter ...$filterList): SelectQueryBuilder { foreach ($filterList as $filter) { @@ -150,9 +143,6 @@ public function getFilterList(): array return array_values($this->filterList); } - /** - * @param Filter[] $filterList - */ public function facet(Facet ...$facetList): SelectQueryBuilder { foreach ($facetList as $facet) { diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php index 8998bdf..231ed88 100644 --- a/src/Dto/Search/Query/SuggestQuery.php +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -24,9 +24,6 @@ public function getIndex(): string { return $this->index; } - /** - * @return string[] - */ public function getText(): string { return $this->text; diff --git a/src/Dto/Search/Result/FacetGroup.php b/src/Dto/Search/Result/FacetGroup.php index 35740a8..1d6ae52 100644 --- a/src/Dto/Search/Result/FacetGroup.php +++ b/src/Dto/Search/Result/FacetGroup.php @@ -7,7 +7,7 @@ class FacetGroup { /** - * @param Facet[] $facetList + * @param Facet[] $facets */ public function __construct( private readonly string $key, diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php index 6408c77..90ed1e4 100644 --- a/src/Dto/Search/Result/SuggestResult.php +++ b/src/Dto/Search/Result/SuggestResult.php @@ -8,16 +8,23 @@ use IteratorAggregate; /** - * @implements IteratorAggregate + * @implements IteratorAggregate */ class SuggestResult implements IteratorAggregate { + /** + * @param Suggestion[] $suggestions + * @param int $queryTime + */ public function __construct( private readonly array $suggestions, private readonly int $queryTime ) { } + /** + * @return ArrayIterator + */ public function getIterator(): ArrayIterator { return new ArrayIterator($this->suggestions); diff --git a/src/Indexer.php b/src/Indexer.php index 045d0f6..7760eed 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -18,12 +18,9 @@ */ interface Indexer { - /** - * @return string process id - */ public function index(IndexerParameter $parameter): IndexerStatus; - public function abort($index): void; + public function abort(string $index): void; /** * @param string[] $idList diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 7dfcc54..a44f1cc 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -4,16 +4,15 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Resource\ResourceBaseLocator; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; +use JsonException; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use RuntimeException; -use JsonException; use Symfony\Component\Lock\LockFactory; use Symfony\Component\Lock\Store\SemaphoreStore; @@ -22,7 +21,7 @@ class BackgroundIndexer implements Indexer private LockFactory $lockFactory; /** - * @param iterable $documentEnricherList + * @param iterable> $documentEnricherList */ public function __construct( private readonly iterable $documentEnricherList, @@ -56,7 +55,7 @@ public function remove(string $index, array $idList): void $this->getIndexer($index)->remove($index, $idList); } - public function abort($index): void + public function abort(string $index): void { $this->getIndexer($index)->abort($index); } diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index a0c4ff1..e2a152d 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -130,6 +130,9 @@ public function setMetaString(string $name, string $value): void $this->metaString[$name] = $value; } + /** + * @return array + */ public function getFields(): array { $fields = get_object_vars($this); diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 9071b4d..d6436d6 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -58,6 +58,7 @@ public function enrichDocument( $doc->url = $url; } + /** @var string[] $spContentType */ $spContentType = [$resource->getObjectType()]; if ($resource->getData()->getBool('init.media') !== true) { $spContentType[] = 'article'; @@ -119,7 +120,9 @@ public function enrichDocument( } $doc->sp_sortvalue = $sortHeadline; - $doc->keywords = $resource->getData()->getArray('metadata.keywords'); + /** @var string[] $keyword */ + $keyword = $resource->getData()->getArray('metadata.keywords'); + $doc->keywords = $keyword; $doc->sp_boost_keywords = implode( ' ', @@ -139,9 +142,9 @@ public function enrichDocument( 'init.siteGroup.id' ); if ($siteGroupId !== 0) { - $sites[] = $siteGroupId; + $sites[] = (string)$siteGroupId; } - $doc->sp_site = array_unique($sites); + $doc->sp_site = array_unique($sites, SORT_STRING); } catch (Exception $e) { throw new DocumentEnrichingException( $resource->getLocation(), @@ -151,39 +154,37 @@ public function enrichDocument( ); } + /** @var string[] $wktPrimaryList */ $wktPrimaryList = $resource->getData()->getArray( 'base.geo.wkt.primary' ); if (!empty($wktPrimaryList)) { - $allWkt = []; - foreach ($wktPrimaryList as $wkt) { - $allWkt[] = $wkt; - } - if (count($allWkt) > 0) { - $doc->sp_geo_points = $allWkt; - } + $doc->sp_geo_points = $wktPrimaryList; } + /** @var array $categoryList */ $categoryList = $resource->getData()->getArray('metadata.categories'); if (!empty($categoryList)) { $categoryIdList = []; foreach ($categoryList as $category) { - $categoryIdList[] = $category['id']; + $categoryIdList[] = (string)$category['id']; } $doc->sp_category = $categoryIdList; } + /** @var array $categoryPath */ $categoryPath = $resource->getData()->getArray( 'metadata.categoriesPath' ); if (!empty($categoryPath)) { $categoryIdPath = []; foreach ($categoryPath as $category) { - $categoryIdPath[] = $category['id']; + $categoryIdPath[] = (string)$category['id']; } $doc->sp_category_path = $categoryIdPath; } + /** @var array $groupPath */ $groupPath = $resource->getData()->getArray('init.groupPath'); $groupPathAsIdList = []; foreach ($groupPath as $group) { @@ -193,6 +194,7 @@ public function enrichDocument( $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; $doc->sp_group_path = $groupPathAsIdList; + /** @var array $schedulingList */ $schedulingList = $resource->getData()->getArray('metadata.scheduling'); if (!empty($schedulingList)) { $doc->sp_date = $this->toDateTime($schedulingList[0]['from']); @@ -200,7 +202,10 @@ public function enrichDocument( $contentTypeList = []; foreach ($schedulingList as $scheduling) { $contentTypeList[] = explode(' ', $scheduling['contentType']); - $dateList[] = $this->toDateTime($scheduling['from']); + $from = $this->toDateTime($scheduling['from']); + if ($from !== null) { + $dateList[] = $from; + } } $doc->sp_contenttype = array_merge( $doc->sp_contenttype, @@ -211,27 +216,28 @@ public function enrichDocument( $doc->sp_date_list = $dateList; } - $contentType = $resource->getData()->getString('base.mime'); - if ($contentType === null) { - $contentType = 'text/html; charset=UTF-8'; - } + $contentType = $resource->getData()->getString( + 'base.mime', + 'text/html; charset=UTF-8' + ); $doc->meta_content_type = $contentType; $doc->content = $resource->getData()->getString( 'searchindexdata.content' ); $accessType = $resource->getData()->getString('init.access.type'); - $groups = $resource->getData()->getArray('init.access.groups'); + /** @var string[] $groups */ + $groups = $resource->getData()->getArray('init.access.groups'); if ($accessType === 'allow' && !empty($groups)) { $doc->include_groups = array_map( - fn($id): int => $this->idWithoutSignature($id), + fn($id): string => (string)$this->idWithoutSignature($id), $groups ); } elseif ($accessType === 'deny' && !empty($groups)) { $doc->exclude_groups = array_map( - fn($id): int => $this->idWithoutSignature($id), + fn($id): string => (string)$this->idWithoutSignature($id), $groups ); } else { @@ -259,7 +265,6 @@ private function idWithoutSignature(string $id): int * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/ParkingSpaceExpose.php#L38 * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/Expose.php#L38 * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 - * - https://gitlab.sitepark.com/customer-projects/stuttgart/blob/develop/stuttgart-module/src/publish/php/SP/Stuttgart/Component/EventsCalendarExtension.php#L124 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Controller/Environment.php#L76 * - https://gitlab.sitepark.com/ies-modules/sitekit-real-estate/blob/develop/src/publish/php/SP/RealEstate/Component/Expose.php#L47 @@ -272,6 +277,8 @@ private function getLocaleFromResource(Resource $resource): string if ($locale !== '') { return $locale; } + + /** @var array $groupPath */ $groupPath = $resource->getData()->getArray('init.groupPath'); if (!empty($groupPath)) { $len = count($groupPath); @@ -306,8 +313,13 @@ private function toDateTime(int $timestamp): ?DateTime return $dateTime; } + /** + * @param Resource $resource + * @return string[] + */ private function getParentSiteGroupIdList(Resource $resource): array { + /** @var array $parents */ $parents = $this->getNavigationParents($resource); if (empty($parents)) { return []; diff --git a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php index e7ac074..1dbceef 100644 --- a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php +++ b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php @@ -14,6 +14,9 @@ */ class SubDirTranslationSplitter implements TranslationSplitter { + /** + * @param string[] $pathList + */ public function split(array $pathList): TranslationSplitterResult { $bases = []; diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 496222b..72e25ad 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -4,7 +4,6 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; @@ -12,8 +11,7 @@ use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; use Exception; -use Solarium\Core\Query\Result\ResultInterface; -use Solarium\QueryType\Update\Result; +use Solarium\QueryType\Update\Result as UpdateResult; use Throwable; /** @@ -22,7 +20,7 @@ class SolrIndexer implements Indexer { /** - * @param iterable $documentEnricherList + * @param iterable> $documentEnricherList */ public function __construct( private readonly iterable $documentEnricherList, @@ -49,7 +47,7 @@ public function remove(string $index, array $idList): void $this->commit($index); } - public function abort($index): void + public function abort(string $index): void { $this->aborter->abort($index); } @@ -66,20 +64,27 @@ public function index(IndexerParameter $parameter): IndexerStatus return $this->indexResources($parameter, $pathList); } + /** + * @param string[] $pathList + * @return string[] + */ private function mapTranslationPaths(array $pathList): array { - return array_map(function ($path) { + return array_map(static function ($path) { $queryString = parse_url($path, PHP_URL_QUERY); - if ($queryString === null) { + if (!is_string($queryString)) { return $path; } - $path = parse_url($path, PHP_URL_PATH); - parse_str($queryString, $params); - if (!isset($params['loc'])) { + $urlPath = parse_url($path, PHP_URL_PATH); + if (!is_string($urlPath)) { return $path; } + parse_str($queryString, $params); + if (!isset($params['loc']) || !is_string($params['loc'])) { + return $urlPath; + } $loc = $params['loc']; - return $path . '.translations/' . $loc . ".php"; + return $urlPath . '.translations/' . $loc . ".php"; }, $pathList); } @@ -143,6 +148,9 @@ private function indexResources( return $this->indexerProgressHandler->getStatus(); } + /** + * @param string[] $pathList + */ private function indexResourcesPerLanguageIndex( string $processId, IndexerParameter $parameter, @@ -226,7 +234,7 @@ private function loadResources( int $length ): array|false { - $maxLength = (count($pathList) ?? 0) - $offset; + $maxLength = (count($pathList) - $offset); if ($maxLength <= 0) { return false; } @@ -249,16 +257,13 @@ private function loadResources( } /** - * @param string $solrCore - * @param string $processId * @param array $resources - * @return ResultInterface|Result */ private function add( string $solrCore, string $processId, array $resources - ): ResultInterface|Result { + ): UpdateResult { $client = $this->clientFactory->create($solrCore); $update = $client->createUpdate(); @@ -273,6 +278,7 @@ private function add( } } try { + /** @var IndexSchema2xDocument $doc */ $doc = $update->createDocument(); foreach ($this->documentEnricherList as $enricher) { $doc = $enricher->enrichDocument( @@ -351,7 +357,8 @@ private function getAvailableIndexes(): array $availableIndexes = []; $response = $client->coreAdmin($coreAdminQuery); - foreach ($response->getStatusResults() as $statusResult) { + $statusResults = $response->getStatusResults() ?? []; + foreach ($statusResults as $statusResult) { $availableIndexes[] = $statusResult->getCoreName(); } diff --git a/src/Service/Indexer/TranslationSplitter.php b/src/Service/Indexer/TranslationSplitter.php index 3b0acfd..c9b757e 100644 --- a/src/Service/Indexer/TranslationSplitter.php +++ b/src/Service/Indexer/TranslationSplitter.php @@ -6,5 +6,8 @@ interface TranslationSplitter { + /** + * @param string[] $pathList + */ public function split(array $pathList): TranslationSplitterResult; } diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index 9d09cf9..20c25ea 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; +use LogicException; use Solarium\QueryType\Select\Result\Document; /** @@ -18,7 +19,10 @@ class ExternalResourceFactory implements ResourceFactory { public function accept(Document $document): bool { - $location = $document->url; + $location = $this->getField($document, 'url'); + if ($location === null) { + return false; + } return ( str_starts_with($location, 'http://') || str_starts_with($location, 'https://') @@ -27,12 +31,26 @@ public function accept(Document $document): bool public function create(Document $document): Resource { + $location = $this->getField($document, 'url'); + if ($location === null) { + throw new LogicException('document should contains a url'); + } + return new Resource( - $document->url, + $location, '', - $document->title, + $this->getField($document, 'title') ?? '', 'external', [] ); } + + private function getField(Document $document, string $name): ?string + { + $fields = $document->getFields(); + if (!isset($fields[$name])) { + return null; + } + return $fields[$name]; + } } diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php index 753e138..bdf7a2f 100644 --- a/src/Service/Search/InternalMediaResourceFactory.php +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -6,7 +6,6 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; -use Atoolo\Search\Service\Search\ResourceFactory; use Solarium\QueryType\Select\Result\Document; /** @@ -37,6 +36,16 @@ public function create(Document $document): Resource private function getMetaLocation(Document $document): string { - return $document->url . '.meta.php'; + $location = $this->getField($document, 'url') ?? ''; + return $location . '.meta.php'; + } + + private function getField(Document $document, string $name): ?string + { + $fields = $document->getFields(); + if (!isset($fields[$name])) { + return null; + } + return $fields[$name]; } } diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php index 0217adc..0568e7e 100644 --- a/src/Service/Search/InternalResourceFactory.php +++ b/src/Service/Search/InternalResourceFactory.php @@ -6,7 +6,6 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; -use Atoolo\Search\Service\Search\ResourceFactory; use Solarium\QueryType\Select\Result\Document; /** @@ -23,11 +22,22 @@ public function __construct( public function accept(Document $document): bool { - return str_ends_with($document->url, '.php'); + $location = $this->getField($document, 'url') ?? ''; + return str_ends_with($location, '.php'); } public function create(Document $document): Resource { - return $this->resourceLoader->load($document->url); + $location = $this->getField($document, 'url') ?? ''; + return $this->resourceLoader->load($location); + } + + private function getField(Document $document, string $name): ?string + { + $fields = $document->getFields(); + if (!isset($fields[$name])) { + return null; + } + return $fields[$name]; } } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 612c81b..5d2f2b9 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -9,17 +9,14 @@ use Atoolo\Search\MoreLikeThisSearcher; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Core\Client\Client; -use Solarium\Core\Query\Result\ResultInterface; use Solarium\QueryType\MoreLikeThis\Query as SolrMoreLikeThisQuery; +use Solarium\QueryType\Select\Result\Result as SelectResult; /** * Implementation of the "More-Like-This" on the basis of a Solr index. */ class SolrMoreLikeThis implements MoreLikeThisSearcher { - /** - * @param iterable $resourceFactoryList - */ public function __construct( private readonly SolrClientFactory $clientFactory, private readonly SolrResultToResourceResolver $resultToResourceResolver @@ -30,6 +27,7 @@ public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { $client = $this->clientFactory->create($query->getCore()); $solrQuery = $this->buildSolrQuery($client, $query); + /** @var SelectResult $result */ $result = $client->execute($solrQuery); return $this->buildResult($result); } @@ -60,18 +58,19 @@ private function buildSolrQuery( } private function buildResult( - ResultInterface $result + SelectResult $result ): SearchResult { $resourceList = $this->resultToResourceResolver ->loadResourceList($result); return new SearchResult( - $result->getNumFound(), + $result->getNumFound() ?? -1, + 0, 0, $resourceList, [], - $result->getQueryTime() + $result->getQueryTime() ?? -1 ); } } diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index aa23465..a97c2cd 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -5,12 +5,11 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; -use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Exception\MissMatchingResourceFactoryException; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; -use Solarium\Core\Query\Result\ResultInterface; use Solarium\QueryType\Select\Result\Document; +use Solarium\QueryType\Select\Result\Result as SelectResult; /** * The loadResourceList() method receives a Solr result and generates the @@ -31,9 +30,10 @@ public function __construct( /** * @return array */ - public function loadResourceList(ResultInterface $result): array + public function loadResourceList(SelectResult $result): array { $resourceList = []; + /** @var Document $document */ foreach ($result as $document) { try { $resourceList[] = $this->loadResource($document); @@ -53,6 +53,18 @@ private function loadResource(Document $document): Resource } } - throw new MissMatchingResourceFactoryException($document->url); + $location = $this->getField($document, 'url') ?? ''; + + throw new MissMatchingResourceFactoryException($location); + } + + private function getField(Document $document, string $name): ?string + { + $fields = $document->getFields(); + if (!isset($fields[$name])) { + return null; + } + return $fields[$name]; } + } diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 544c815..4815811 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -21,10 +21,11 @@ use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\SelectSearcher; use Atoolo\Search\Service\SolrClientFactory; -use Solarium\Component\Result\Facet\FacetResultInterface; +use InvalidArgumentException; +use Solarium\Component\Facet\Field; use Solarium\Core\Client\Client; -use Solarium\Core\Query\Result\ResultInterface; use Solarium\QueryType\Select\Query\Query as SolrSelectQuery; +use Solarium\QueryType\Select\Result\Result as SelectResult; /** * Implementation of the searcher on the basis of a Solr index. @@ -46,6 +47,7 @@ public function select(SelectQuery $query): SearchResult $client = $this->clientFactory->create($query->getIndex()); $solrQuery = $this->buildSolrQuery($client, $query); + /** @var SelectResult $result */ $result = $client->execute($solrQuery); return $this->buildResult($query, $result); } @@ -107,7 +109,7 @@ private function addSortToSolrQuery( } elseif ($criteria instanceof Score) { $field = 'score'; } else { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( 'unsupported sort criteria: ' . get_class($criteria) ); } @@ -133,11 +135,13 @@ private function addTextFilterToSolrQuery( return; } $terms = explode(' ', $text); - $terms = array_map(function ($term) use ($solrQuery) { - $term = trim($term); - return $solrQuery->getHelper()->escapeTerm($term); - }, - $terms); + $terms = array_map( + static function ($term) use ($solrQuery) { + $term = trim($term); + return $solrQuery->getHelper()->escapeTerm($term); + }, + $terms + ); $text = implode(' ', $terms); $solrQuery->setQuery($text); } @@ -174,7 +178,7 @@ private function addFilterQueriesToSolrQuery( } /** - * @param \Atoolo\Search\Dto\Search\Query\Facet\Facet[] $filterList + * @param \Atoolo\Search\Dto\Search\Query\Facet\Facet[] $facetList */ private function addFacetListToSolrQuery( SolrSelectQuery $solrQuery, @@ -188,7 +192,7 @@ private function addFacetListToSolrQuery( } elseif ($facet instanceof FacetMultiQuery) { $this->addFacetMultiQueryToSolrQuery($solrQuery, $facet); } else { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( 'Unsupported facet-class ' . get_class($facet) ); } @@ -208,7 +212,9 @@ private function addFacetFieldToSolrQuery( if ($facet->getExcludeFilter() !== null) { $field = '{!ex=' . $facet->getExcludeFilter() . '}' . $field; } - $facetSet->createFacetField($facet->getKey()) + /** @var Field $solariumFacet */ + $solariumFacet = $facetSet->createFacetField($facet->getKey()); + $solariumFacet ->setField($field) ->setTerms($facet->getTerms()); } @@ -244,7 +250,7 @@ private function addFacetMultiQueryToSolrQuery( private function buildResult( SelectQuery $query, - ResultInterface $result + SelectResult $result ): SearchResult { $resourceList = $this->resultToResourceResolver @@ -252,22 +258,21 @@ private function buildResult( $facetGroupList = $this->buildFacetGroupList($query, $result); return new SearchResult( - $result->getNumFound(), + $result->getNumFound() ?? 0, $query->getLimit(), $query->getOffset(), $resourceList, $facetGroupList, - $result->getQueryTime() + $result->getQueryTime() ?? 0 ); } /** - * @param ResultInterface $result * @return FacetGroup[] */ private function buildFacetGroupList( SelectQuery $query, - ResultInterface $result + SelectResult $result ): array { $facetSet = $result->getFacetSet(); @@ -277,9 +282,14 @@ private function buildFacetGroupList( $facetGroupList = []; foreach ($query->getFacetList() as $facet) { + /** @var ?\Solarium\Component\Result\Facet\Field $resultFacet */ + $resultFacet = $facetSet->getFacet($facet->getKey()); + if ($resultFacet === null) { + continue; + } $facetGroupList[] = $this->buildFacetGroup( $facet->getKey(), - $facetSet->getFacet($facet->getKey()) + $resultFacet ); } return $facetGroupList; @@ -287,10 +297,15 @@ private function buildFacetGroupList( private function buildFacetGroup( string $key, - FacetResultInterface $solrFacet + \Solarium\Component\Result\Facet\Field $solrFacet ): FacetGroup { $facetList = []; foreach ($solrFacet as $value => $count) { + if (!is_int($count)) { + throw new InvalidArgumentException( + 'facet count should be a int: ' . $count + ); + } $facetList[] = new Facet((string)$value, $count); } return new FacetGroup($key, $facetList); diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 6efe272..33b0fcb 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -10,12 +10,18 @@ use Atoolo\Search\Exception\UnexpectedResultException; use Atoolo\Search\Service\SolrParameterClientFactory; use Atoolo\Search\SuggestSearcher; +use JsonException; use Solarium\Core\Client\Client; use Solarium\QueryType\Select\Query\Query as SolrSelectQuery; use Solarium\QueryType\Select\Result\Result as SolrSelectResult; -use JsonException; /** + * @phpstan-type SolariumResponse array{ + * facet_counts: array{ + * facet_fields:array> + * } + * } + * * Implementation of the "suggest search" based on a Solr index. */ class SolrSuggest implements SuggestSearcher @@ -83,7 +89,10 @@ private function buildResult( $solrResult->getResponse()->getBody(), $resultField ); - return new SuggestResult($suggestions, $solrResult->getQueryTime()); + return new SuggestResult( + $suggestions, + $solrResult->getQueryTime() ?? 0 + ); } /** @@ -95,6 +104,7 @@ private function parseSuggestion( string $facetField ): array { try { + /** @var SolariumResponse $json */ $json = json_decode( $responseBody, true, @@ -110,7 +120,7 @@ private function parseSuggestion( $suggestions = []; for ($i = 0; $i < $len; $i += 2) { $term = $facets[$i]; - $hits = $facets[$i + 1]; + $hits = (int)$facets[$i + 1]; $suggestions[] = new Suggestion($term, $hits); } diff --git a/src/Service/SolrParameterClientFactory.php b/src/Service/SolrParameterClientFactory.php index 38902fd..1ea2e19 100644 --- a/src/Service/SolrParameterClientFactory.php +++ b/src/Service/SolrParameterClientFactory.php @@ -22,21 +22,19 @@ public function __construct( private readonly int $port, private readonly string $path = '', private readonly ?string $proxy = null, - private readonly ?int $timeout = 0 + private readonly int $timeout = 0 ) { } public function create(string $core): Client { - $host = 'solr-neu-isenburg-whinchat.veltrup.sitepark.de'; - $adapter = new Curl(); $adapter->setTimeout($this->timeout); $adapter->setProxy($this->proxy); $eventDispatcher = new EventDispatcher(); $config = [ 'endpoint' => [ - $host => [ + $this->host => [ 'scheme' => $this->scheme, 'host' => $this->host, 'port' => $this->port, From 96116626c2ec6a657d70d4084ddf03fa7b87fc1c Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:09:03 +0100 Subject: [PATCH 026/145] fix: phpcs errors --- src/Dto/Search/Result/Facet.php | 2 +- src/Service/Search/SolrResultToResourceResolver.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Dto/Search/Result/Facet.php b/src/Dto/Search/Result/Facet.php index 8e19cd0..ad31aa1 100644 --- a/src/Dto/Search/Result/Facet.php +++ b/src/Dto/Search/Result/Facet.php @@ -21,4 +21,4 @@ public function getHits(): int { return $this->hits; } -} \ No newline at end of file +} diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index a97c2cd..68da172 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -66,5 +66,4 @@ private function getField(Document $document, string $name): ?string } return $fields[$name]; } - } From 2af8c75d9f140f6bbc149f4b28690cb7a4bcf566 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:09:27 +0100 Subject: [PATCH 027/145] docs: add phpstan badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c93069c..c0e6fca 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![codecov](https://codecov.io/gh/sitepark/atoolo-resource/graph/badge.svg?token=QwvDRxKEa2)](https://codecov.io/gh/sitepark/atoolo-resource) +![phpstan](https://img.shields.io/badge/PHPStan-level%209-brightgreen) # Atoolo search From 06ccb044b7228ab7bb466228a5a56b97a578f5e1 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:13:43 +0100 Subject: [PATCH 028/145] docs: add php badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c0e6fca..fb8a9c8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ [![codecov](https://codecov.io/gh/sitepark/atoolo-resource/graph/badge.svg?token=QwvDRxKEa2)](https://codecov.io/gh/sitepark/atoolo-resource) ![phpstan](https://img.shields.io/badge/PHPStan-level%209-brightgreen) +![php](https://img.shields.io/badge/PHP-8.1-brightgreen) +![php](https://img.shields.io/badge/PHP-8.2-brightgreen) +![php](https://img.shields.io/badge/PHP-8.3-brightgreen) # Atoolo search From 4d09f3a8392cea999c7288134c8f3c51d7c1d2f3 Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:14:22 +0100 Subject: [PATCH 029/145] docs: add php badge --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fb8a9c8..6f87596 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ [![codecov](https://codecov.io/gh/sitepark/atoolo-resource/graph/badge.svg?token=QwvDRxKEa2)](https://codecov.io/gh/sitepark/atoolo-resource) ![phpstan](https://img.shields.io/badge/PHPStan-level%209-brightgreen) -![php](https://img.shields.io/badge/PHP-8.1-brightgreen) -![php](https://img.shields.io/badge/PHP-8.2-brightgreen) -![php](https://img.shields.io/badge/PHP-8.3-brightgreen) +![php](https://img.shields.io/badge/PHP-8.1-blue) +![php](https://img.shields.io/badge/PHP-8.2-blue) +![php](https://img.shields.io/badge/PHP-8.3-blue) # Atoolo search From 0800eaf9d4c35d7b340fc59cfd8ef3931073d9fd Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:16:20 +0100 Subject: [PATCH 030/145] docs: update codecov badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f87596..3c91cb6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![codecov](https://codecov.io/gh/sitepark/atoolo-resource/graph/badge.svg?token=QwvDRxKEa2)](https://codecov.io/gh/sitepark/atoolo-resource) +[![codecov](https://codecov.io/gh/sitepark/atoolo-search/graph/badge.svg?token=xBMwUzm34b)](https://codecov.io/gh/sitepark/atoolo-search) ![phpstan](https://img.shields.io/badge/PHPStan-level%209-brightgreen) ![php](https://img.shields.io/badge/PHP-8.1-blue) ![php](https://img.shields.io/badge/PHP-8.2-blue) From aa54d36bd27a07ab83833732da0e20d71f4e2b0b Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 30 Jan 2024 09:30:13 +0100 Subject: [PATCH 031/145] test: fix test --- src/Service/Indexer/LocationFinder.php | 4 ++++ test/Service/Indexer/LocationFinderTest.php | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 5800a4a..1560216 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -31,6 +31,8 @@ public function findAll(): array $pathList[] = $this->toRelativePath($file->getPathname()); } + sort($pathList); + return $pathList; } @@ -74,6 +76,8 @@ public function findPaths(array $paths): array $pathList[] = $this->toRelativePath($file->getPathname()); } + sort($pathList); + return $pathList; } diff --git a/test/Service/Indexer/LocationFinderTest.php b/test/Service/Indexer/LocationFinderTest.php index c3f0ca3..7acceca 100644 --- a/test/Service/Indexer/LocationFinderTest.php +++ b/test/Service/Indexer/LocationFinderTest.php @@ -4,6 +4,7 @@ use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Service\Indexer\LocationFinder; +use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -13,10 +14,12 @@ class LocationFinderTest extends TestCase private LocationFinder $locationFinder; protected function setUp(): void { - $file = __DIR__ . '/../../resources/Service/Indexer/LocationFinder'; $base = realpath( __DIR__ . '/../../resources/Service/Indexer/LocationFinder' ); + if ($base === false) { + throw new InvalidArgumentException('basepath not found'); + } $this->locationFinder = new LocationFinder( new StaticResourceBaseLocator($base) ); From f877a2d2200a06b8faaa3d743a87230a26063ba3 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 31 Jan 2024 10:08:34 +0100 Subject: [PATCH 032/145] test: new tests for Index-Command --- src/Console/Command/Indexer.php | 65 +++------- ...ProgressBar.php => IndexerProgressBar.php} | 2 +- .../Command/Io/IndexerProgressBarFactory.php | 15 +++ src/Console/Command/Io/TypifiedInput.php | 55 ++++++++ src/Console/Command/SolrIndexerBuilder.php | 6 +- test/Console/ApplicationTest.php | 5 +- test/Console/Command/IndexerTest.php | 117 +++++++++++++++++- test/Console/Command/Io/TypifiedInputTest.php | 101 +++++++++++++++ 8 files changed, 306 insertions(+), 60 deletions(-) rename src/Console/Command/Io/{IndexerProgressProgressBar.php => IndexerProgressBar.php} (97%) create mode 100644 src/Console/Command/Io/IndexerProgressBarFactory.php create mode 100644 src/Console/Command/Io/TypifiedInput.php create mode 100644 test/Console/Command/Io/TypifiedInputTest.php diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 20b4aeb..f7cc21e 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -4,11 +4,12 @@ namespace Atoolo\Search\Console\Command; -use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; +use Atoolo\Search\Console\Command\Io\IndexerProgressBar; +use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; +use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; -use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -22,10 +23,9 @@ )] class Indexer extends Command { - private IndexerProgressProgressBar $progressBar; + private IndexerProgressBar $progressBar; private SymfonyStyle $io; - - private InputInterface $input; + private TypifiedInput $input; /** * phpcs:ignore @@ -33,7 +33,8 @@ class Indexer extends Command */ public function __construct( private readonly iterable $documentEnricherList, - private readonly SolrIndexerBuilder $solrIndexerBuilder + private readonly SolrIndexerBuilder $solrIndexerBuilder, + private readonly IndexerProgressBarFactory $progressBarFactory ) { parent::__construct(); } @@ -89,14 +90,14 @@ protected function execute( OutputInterface $output ): int { - $this->input = $input; + $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->progressBar = new IndexerProgressProgressBar($output); + $this->progressBar = $this->progressBarFactory->create($output); - $paths = $this->getArrayArgument('paths'); + $paths = $this->input->getArrayArgument('paths'); $cleanupThreshold = empty($paths) - ? $this->getIntOption('cleanup-threshold') + ? $this->input->getIntOption('cleanup-threshold') : 0; if (empty($paths)) { @@ -107,18 +108,18 @@ protected function execute( } $parameter = new IndexerParameter( - $this->getStringArgument('solr-core'), + $this->input->getStringArgument('solr-core'), $cleanupThreshold, - $this->getIntOption('chunk-size'), + $this->input->getIntOption('chunk-size'), $paths ); $this->solrIndexerBuilder - ->resourceDir($this->getStringArgument('resource-dir')) + ->resourceDir($this->input->getStringArgument('resource-dir')) ->progressBar($this->progressBar) ->documentEnricherList($this->documentEnricherList) ->solrConnectionUrl( - $this->getStringArgument('solr-connection-url') + $this->input->getStringArgument('solr-connection-url') ); $indexer = $this->solrIndexerBuilder->build(); @@ -129,42 +130,6 @@ protected function execute( return Command::SUCCESS; } - private function getStringArgument(string $name): string - { - $value = $this->input->getArgument($name); - if (!is_string($value)) { - throw new InvalidArgumentException( - $name . ' must be a string' - ); - } - return $value; - } - - private function getIntOption(string $name): int - { - $value = $this->input->getOption($name); - if (!is_numeric($value)) { - throw new InvalidArgumentException( - $name . ' must be a integer: ' . $value - ); - } - return (int)$value; - } - - /** - * @return string[] - */ - private function getArrayArgument(string $name): array - { - $value = $this->input->getArgument($name); - if (!is_array($value)) { - throw new InvalidArgumentException( - $name . ' must be a array' - ); - } - return $value; - } - protected function errorReport(): void { foreach ($this->progressBar->getErrors() as $error) { diff --git a/src/Console/Command/Io/IndexerProgressProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php similarity index 97% rename from src/Console/Command/Io/IndexerProgressProgressBar.php rename to src/Console/Command/Io/IndexerProgressBar.php index f166fba..2ec81a3 100644 --- a/src/Console/Command/Io/IndexerProgressProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -12,7 +12,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Throwable; -class IndexerProgressProgressBar implements IndexerProgressHandler +class IndexerProgressBar implements IndexerProgressHandler { private OutputInterface $output; private ProgressBar $progressBar; diff --git a/src/Console/Command/Io/IndexerProgressBarFactory.php b/src/Console/Command/Io/IndexerProgressBarFactory.php new file mode 100644 index 0000000..cbcb2ef --- /dev/null +++ b/src/Console/Command/Io/IndexerProgressBarFactory.php @@ -0,0 +1,15 @@ +input->getOption($name); + if (!is_numeric($value)) { + throw new InvalidArgumentException( + 'option' . $name . ' must be a integer: ' . $value + ); + } + return (int)$value; + } + + public function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + 'argument' . $name . ' must be a string' + ); + } + return $value; + } + + /** + * @return string[] + */ + public function getArrayArgument(string $name): array + { + $value = $this->input->getArgument($name); + if (!is_array($value)) { + throw new InvalidArgumentException( + 'argument ' . $name . ' must be a array' + ); + } + return $value; + } +} diff --git a/src/Console/Command/SolrIndexerBuilder.php b/src/Console/Command/SolrIndexerBuilder.php index c9525b0..a88f16b 100644 --- a/src/Console/Command/SolrIndexerBuilder.php +++ b/src/Console/Command/SolrIndexerBuilder.php @@ -7,7 +7,7 @@ use Atoolo\Resource\Loader\ServerVarResourceBaseLocator; use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; -use Atoolo\Search\Console\Command\Io\IndexerProgressProgressBar; +use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexingAborter; @@ -25,7 +25,7 @@ class SolrIndexerBuilder * @var iterable> */ private iterable $documentEnricherList; - private IndexerProgressProgressBar $progressBar; + private IndexerProgressBar $progressBar; private string $solrConnectionUrl; public function resourceDir(string $resourceDir): SolrIndexerBuilder @@ -46,7 +46,7 @@ public function documentEnricherList( } public function progressBar( - IndexerProgressProgressBar $progressBar + IndexerProgressBar $progressBar ): SolrIndexerBuilder { $this->progressBar = $progressBar; return $this; diff --git a/test/Console/ApplicationTest.php b/test/Console/ApplicationTest.php index 804ef8d..816a968 100644 --- a/test/Console/ApplicationTest.php +++ b/test/Console/ApplicationTest.php @@ -6,10 +6,13 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; use Atoolo\Search\Console\Command\SolrIndexerBuilder; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; +#[CoversClass(Application::class)] class ApplicationTest extends TestCase { /** @@ -21,7 +24,7 @@ public function testConstruct(): void SolrIndexerBuilder::class ); $application = new Application([ - new Indexer([], $indexBuilder) + new Indexer([], $indexBuilder, new IndexerProgressBarFactory()) ]); $command = $application->get('atoolo:indexer'); $this->assertInstanceOf( diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 2189b32..f4ce781 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -6,30 +6,133 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\Io\IndexerProgressBar; +use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; use Atoolo\Search\Console\Command\SolrIndexerBuilder; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Tester\CommandTester; +#[CoversClass(Indexer::class)] class IndexerTest extends TestCase { + private CommandTester $commandTester; + /** * @throws Exception */ - public function testExecute(): void + public function setUp(): void { $indexBuilder = $this->createStub( SolrIndexerBuilder::class ); + + $indexer = new Indexer( + [], + $indexBuilder, + new IndexerProgressBarFactory() + ); + $application = new Application([ - new Indexer( - [], - $indexBuilder - ) + $indexer + ]); + + $command = $application->find('atoolo:indexer'); + $this->commandTester = new CommandTester($command); + } + + public function testExecuteIndexAll(): void + { + $this->commandTester->execute([ + // pass arguments to the helper + 'resource-dir' => 'abc', + 'solr-connection-url' => 'http://localhost:8080', + 'solr-core' => 'test' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<commandTester->execute([ + // pass arguments to the helper + 'resource-dir' => 'abc', + 'solr-connection-url' => 'http://localhost:8080', + 'solr-core' => 'test', + 'paths' => ['a.php', 'b.php'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + SolrIndexerBuilder::class + ); + + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $progressBarFactory = $this->createStub( + IndexerProgressBarFactory::class + ); + $progressBarFactory + ->method('create') + ->willReturn($progressBar); + + $indexer = new Indexer( + [], + $indexBuilder, + $progressBarFactory + ); + + $application = new Application([ + $indexer ]); $command = $application->find('atoolo:indexer'); $commandTester = new CommandTester($command); + $commandTester->execute([ // pass arguments to the helper 'resource-dir' => 'abc', @@ -41,15 +144,19 @@ public function testExecute(): void // the output of the command in the console $output = $commandTester->getDisplay(); + // phpcs:disable $this->assertEquals( <<createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 123, + $input->getIntOption('a'), + 'unexpected option value' + ); + } + + public function testGetIntOptWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getIntOption('a'); + } + + public function testGetStringArgument(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 'abc', + $input->getStringArgument('a'), + 'unexpected argument value' + ); + } + + public function testGetStringArgumentWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getStringArgument('a'); + } + + public function testGetArrayArgument(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn(['a', 'b', 'c']); + + $input = new TypifiedInput($symfonyInput); + $this->assertEquals( + ['a', 'b', 'c'], + $input->getArrayArgument('a'), + 'unexpected argument value' + ); + } + + public function testGetArrayArgumentWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getArrayArgument('a'); + } +} From 52deb6b528fa00b7a446ae8c978af77e8bc8e1e3 Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 31 Jan 2024 15:50:35 +0100 Subject: [PATCH 033/145] test: add more tests --- .gitignore | 1 + composer.json | 5 +- .../Command/Io/IndexerProgressBarFactory.php | 2 +- src/Dto/Indexer/IndexerStatus.php | 83 +------- src/Dto/Indexer/IndexerStatusState.php | 10 +- src/Service/Indexer/BackgroundIndexer.php | 29 +-- .../BackgroundIndexerProgressState.php | 20 +- src/Service/Indexer/IndexerStatusStore.php | 94 +++++++++ test/Dto/Indexer/IndexerStatusTest.php | 80 ++++++++ .../Indexer/IndexerStatusStoreTest.php | 181 ++++++++++++++++++ .../atoolo.search.index.test.status.json | 11 ++ 11 files changed, 399 insertions(+), 117 deletions(-) create mode 100644 src/Service/Indexer/IndexerStatusStore.php create mode 100644 test/Dto/Indexer/IndexerStatusTest.php create mode 100644 test/Service/Indexer/IndexerStatusStoreTest.php create mode 100644 test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json diff --git a/.gitignore b/.gitignore index f86b9c0..6fda8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !var/cache/.gitkeep /var/log/* !var/log/.gitkeep +/var/test/* /tools .phpactor.json composer.lock diff --git a/composer.json b/composer.json index 34bff57..0857c4b 100644 --- a/composer.json +++ b/composer.json @@ -29,6 +29,8 @@ "symfony/event-dispatcher": "^6.3 | ^7.0", "symfony/finder": "^6.3 | ^7.0", "symfony/lock": "^6.3 | ^7.0", + "symfony/property-access": "^6.3 | ^7.0", + "symfony/serializer": "^6.3 | ^7.0", "symfony/yaml": "^6.3 | ^7.0" }, "require-dev": { @@ -37,7 +39,8 @@ "phpcompatibility/php-compatibility": "^9.3", "phpunit/phpunit": "^10.4", "roave/security-advisories": "dev-latest", - "squizlabs/php_codesniffer": "^3.7" + "squizlabs/php_codesniffer": "^3.7", + "symfony/filesystem": "^6.3 | ^7.0" }, "scripts": { diff --git a/src/Console/Command/Io/IndexerProgressBarFactory.php b/src/Console/Command/Io/IndexerProgressBarFactory.php index cbcb2ef..207abe9 100644 --- a/src/Console/Command/Io/IndexerProgressBarFactory.php +++ b/src/Console/Command/Io/IndexerProgressBarFactory.php @@ -12,4 +12,4 @@ public function create(OutputInterface $output): IndexerProgressBar { return new IndexerProgressBar($output); } -} \ No newline at end of file +} diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php index 4b2ce3d..bba6d45 100644 --- a/src/Dto/Indexer/IndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -5,8 +5,6 @@ namespace Atoolo\Search\Dto\Indexer; use DateTime; -use InvalidArgumentException; -use JsonException; /** * @phpstan-type JsonStatus array{ @@ -60,88 +58,19 @@ public function getStatusLine(): string $endTime = new DateTime(); } $duration = $this->startTime->diff($endTime); + + $lastUpdate = $this->lastUpdate; + if ($lastUpdate->getTimestamp() === 0) { + $lastUpdate = $endTime; + } return '[' . $this->state->name . '] ' . 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . 'processed: ' . $this->processed . "/" . $this->total . ', ' . 'skipped: ' . $this->skipped . ', ' . - 'lastUpdate: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'lastUpdate: ' . $lastUpdate->format('d.m.Y H:i') . ', ' . 'updated: ' . $this->updated . ', ' . 'errors: ' . $this->errors; } - - /** - * @throws JsonException - */ - public static function load(string $file): IndexerStatus - { - if (!file_exists($file)) { - return self::empty(); - } - $content = file_get_contents($file); - if ($content === false) { - throw new InvalidArgumentException('Cannot read file ' . $file); - } - /** @var JsonStatus $data */ - $data = json_decode( - $content, - true, - 512, - JSON_THROW_ON_ERROR - ); - - $state = isset($data['state']) - ? IndexerStatusState::valueOf($data['state']) - : IndexerStatusState::UNKNOWN; - - $startTime = new DateTime(); - $startTime->setTimestamp($data['startTime']); - - $endTime = new DateTime(); - if ($data['endTime'] !== null) { - $endTime->setTimestamp($data['endTime']); - } else { - $endTime->setTimestamp(0); - } - - $lastUpdate = new DateTime(); - if ($data['lastUpdate'] !== null) { - $lastUpdate->setTimestamp($data['lastUpdate']); - } else { - $lastUpdate->setTimestamp(0); - } - - return new IndexerStatus( - $state, - $startTime, - $endTime, - $data['total'], - $data['processed'], - $data['skipped'] ?? 0, - $lastUpdate, - $data['updated'] ?? 0, - $data['errors'] ?? 0 - ); - } - - /** - * @throws JsonException - */ - public function store(string $file): void - { - $jsonString = json_encode([ - 'state' => $this->state->name, - 'statusLine' => $this->getStatusLine(), - 'startTime' => $this->startTime->getTimestamp(), - 'endTime' => $this->endTime?->getTimestamp(), - 'total' => $this->total, - 'processed' => $this->processed, - 'skipped' => $this->skipped, - 'lastUpdate' => $this->lastUpdate->getTimestamp(), - 'updated' => $this->updated, - 'errors' => $this->errors - ], JSON_THROW_ON_ERROR); - file_put_contents($file, $jsonString); - } } diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php index 48f1964..1de01a4 100644 --- a/src/Dto/Indexer/IndexerStatusState.php +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -4,12 +4,12 @@ namespace Atoolo\Search\Dto\Indexer; -enum IndexerStatusState +enum IndexerStatusState: string { - case UNKNOWN; - case RUNNING; - case INDEXED; - case ABORTED; + case UNKNOWN = 'UNKNOWN'; + case RUNNING = 'RUNNING'; + case INDEXED = 'INDEXED'; + case ABORTED = 'ABORTED'; public static function valueOf(string $name): IndexerStatusState { diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index a44f1cc..a408aae 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -9,12 +9,11 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\SolrClientFactory; -use JsonException; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; -use RuntimeException; use Symfony\Component\Lock\LockFactory; use Symfony\Component\Lock\Store\SemaphoreStore; +use Symfony\Component\Serializer\Exception\ExceptionInterface; class BackgroundIndexer implements Indexer { @@ -31,20 +30,10 @@ public function __construct( private readonly SolrClientFactory $clientFactory, private readonly IndexingAborter $aborter, private readonly string $source, - private readonly string $statusCacheDir, + private readonly IndexerStatusStore $statusStore, private readonly LoggerInterface $logger = new NullLogger() ) { $this->lockFactory = new LockFactory(new SemaphoreStore()); - if ( - !is_dir($concurrentDirectory = $this->statusCacheDir) && - !mkdir($concurrentDirectory) && - !is_dir($concurrentDirectory) - ) { - throw new RuntimeException(sprintf( - 'Directory "%s" was not created', - $concurrentDirectory - )); - } } /** @@ -74,18 +63,18 @@ public function index(IndexerParameter $parameter): IndexerStatus } /** - * @throws JsonException + * @throws ExceptionInterface */ public function getStatus(string $index): IndexerStatus { - $file = $this->getStatusFile($index); - return IndexerStatus::load($file); + return $this->statusStore->load($index); } private function getIndexer(string $index): SolrIndexer { $progressHandler = new BackgroundIndexerProgressState( - $this->getStatusFile($index), + $index, + $this->statusStore, $this->logger ); return new SolrIndexer( @@ -99,10 +88,4 @@ private function getIndexer(string $index): SolrIndexer $this->source ); } - - private function getStatusFile(string $index): string - { - return $this->statusCacheDir . - '/atoolo.search.index.' . $index . ".status.json"; - } } diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index 55fad3d..5fddc34 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -7,10 +7,11 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Dto\Indexer\IndexerStatusState; use DateTime; +use JsonException; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; +use Symfony\Component\Serializer\Exception\ExceptionInterface; use Throwable; -use JsonException; class BackgroundIndexerProgressState implements IndexerProgressHandler { @@ -19,7 +20,8 @@ class BackgroundIndexerProgressState implements IndexerProgressHandler private bool $isUpdate = false; public function __construct( - private readonly string $file, + private string $index, + private readonly IndexerStatusStore $statusStore, private readonly LoggerInterface $logger = new NullLogger() ) { } @@ -39,10 +41,13 @@ public function start(int $total): void ); } + /** + * @throws ExceptionInterface + */ public function startUpdate(int $total): void { $this->isUpdate = true; - $storedStatus = IndexerStatus::load($this->file); + $storedStatus = $this->statusStore->load($this->index); $this->status = new IndexerStatus( IndexerStatusState::RUNNING, $storedStatus->startTime, @@ -66,7 +71,7 @@ public function advance(int $step): void if ($this->isUpdate) { $this->status->updated += $step; } - $this->status->store($this->file); + $this->statusStore->store($this->index, $this->status); } @@ -97,7 +102,7 @@ public function finish(): void if ($this->status->state === IndexerStatusState::RUNNING) { $this->status->state = IndexerStatusState::INDEXED; } - $this->status->store($this->file); + $this->statusStore->store($this->index, $this->status); } public function abort(): void @@ -117,9 +122,4 @@ public function getStatus(): IndexerStatus { return $this->status; } - - public function getStatusFile(): string - { - return $this->file; - } } diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php new file mode 100644 index 0000000..3fabb1b --- /dev/null +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -0,0 +1,94 @@ +getStatusFile($index); + + if (!file_exists($file)) { + return IndexerStatus::empty(); + } + + $json = file_get_contents($file); + if ($json === false) { + throw new InvalidArgumentException('Cannot read file ' . $file); + } + + /** @var IndexerStatus $status */ + $status = $this + ->createSerializer() + ->deserialize($json, IndexerStatus::class, 'json'); + + return $status; + } + + public function store(string $index, IndexerStatus $status): void + { + $this->createBaseDirectory(); + + $file = $this->getStatusFile($index); + $json = $this + ->createSerializer() + ->serialize($status, 'json'); + $result = file_put_contents($file, $json); + if ($result === false) { + throw new RuntimeException( + 'Unable to write indexer-status file ' . $file + ); + } + } + + private function createBaseDirectory(): void + { + if ( + !is_dir($concurrentDirectory = $this->basedir) && + !mkdir($concurrentDirectory) && + !is_dir($concurrentDirectory) + ) { + throw new RuntimeException(sprintf( + 'Directory "%s" was not created', + $concurrentDirectory + )); + } + } + + private function createSerializer(): Serializer + { + $encoders = [new JsonEncoder()]; + $normalizers = [ + new BackedEnumNormalizer(), + new DateTimeNormalizer(), + new PropertyNormalizer() + ]; + + return new Serializer($normalizers, $encoders); + } + + private function getStatusFile(string $index): string + { + return $this->basedir . + '/atoolo.search.index.' . $index . ".status.json"; + } +} diff --git a/test/Dto/Indexer/IndexerStatusTest.php b/test/Dto/Indexer/IndexerStatusTest.php new file mode 100644 index 0000000..f4e0d67 --- /dev/null +++ b/test/Dto/Indexer/IndexerStatusTest.php @@ -0,0 +1,80 @@ +setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new \DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $lastUpdate = new \DateTime(); + $lastUpdate->setDate(2024, 1, 31); + $lastUpdate->setTime(13, 17, 12); + + $this->status = new IndexerStatus( + IndexerStatusState::INDEXED, + $startTime, + $endTime, + 10, + 5, + 4, + $lastUpdate, + 6, + 2 + ); + } + + public function testGetStatus(): void + { + $this->assertEquals( + '[INDEXED] ' . + 'start: 31.01.2024 11:15, ' . + 'time: 01h 01m 01s, ' . + 'processed: 5/10, ' . + 'skipped: 4, ' . + 'lastUpdate: 31.01.2024 13:17, ' . + 'updated: 6, ' . + 'errors: 2', + $this->status->getStatusLine(), + "unexpected status line" + ); + } + public function testEmpty(): void + { + $status = IndexerStatus::empty(); + + $dateTimePattern = '[0-9]{2}\.[0-9]{2}\.[0-9]{4} [0-9]{2}:[0-9]{2}'; + $patter = '/\[UNKNOWN] ' . + 'start: ' . $dateTimePattern . ', ' . + 'time: 00h 00m 00s, ' . + 'processed: 0\/0, ' . + 'skipped: 0, ' . + 'lastUpdate: ' . $dateTimePattern . ', ' . + 'updated: 0, ' . + 'errors: 0' . + '/'; + + $this->assertMatchesRegularExpression( + $patter, + $status->getStatusLine(), + "unexpected status line for empty status" + ); + } +} diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php new file mode 100644 index 0000000..06938af --- /dev/null +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -0,0 +1,181 @@ +exists(self::TEST_DIR)) { + $filesystem->chmod(self::TEST_DIR, 0777, 0000, true); + $filesystem->remove(self::TEST_DIR); + } + $filesystem->mkdir(self::TEST_DIR); + } + + public function testStore(): void + { + $status = $this->createIndexerStatus(); + + $store = new IndexerStatusStore(self::TEST_DIR); + $store->store('test', $status); + + $json = file_get_contents( + self::TEST_DIR . '/atoolo.search.index.test.status.json' + ); + + $expected = + '{' . + '"state":"INDEXED",' . + '"startTime":"2024-01-31T11:15:10+00:00",' . + '"endTime":"2024-01-31T12:16:11+00:00",' . + '"total":10,' . + '"processed":5,' . + '"skipped":4,' . + '"lastUpdate":"2024-01-31T13:17:12+00:00",' . + '"updated":6,' . + '"errors":2' . + '}'; + + $this->assertEquals($expected, $json, 'unexpected json string'); + } + + public function testStoreWithNonExistsBaseDir(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/not-exists'; + $store = new IndexerStatusStore($baseDir); + + $store->store('test', $status); + + $this->assertDirectoryExists( + $baseDir, + 'non exists basedir should be created' + ); + } + + public function testStoreWithNonWritableStatusFile(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/non-writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($baseDir); + $filesystem->chmod($baseDir, 0000); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + + public function testStoreWithNonCreatableBaseDir(): void + { + $status = $this->createIndexerStatus(); + $nonWritable = self::TEST_DIR . '/non-writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($nonWritable); + $filesystem->chmod($nonWritable, 0000); + + $store = new IndexerStatusStore($nonWritable . '/subdir'); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + + public function testLoad(): void + { + $baseDir = __DIR__ . '/../../resources/' . + 'Service/Indexer/IndexerStatusStore'; + + $store = new IndexerStatusStore($baseDir); + + $status = $store->load('test'); + + $expected = $this->createIndexerStatus(); + $this->assertEquals( + $expected, + $status, + 'unexpected status' + ); + } + + public function testLoadFileNotExists(): void + { + $baseDir = __DIR__ . '/../../resources/' . + 'Service/Indexer/IndexerStatusStore'; + + $store = new IndexerStatusStore($baseDir); + + $status = $store->load('test-not-exists'); + + $this->assertEquals( + 0, + $status->total, + 'empty status expected' + ); + } + + /** + * @throws ExceptionInterface + */ + public function testLoadFileNotReadable(): void + { + $file = self::TEST_DIR . '/' . 'atoolo.search.index.test-not-readable.status.json'; + + $filesystem = new Filesystem(); + $filesystem->touch($file); + $filesystem->chmod($file, 0000); + + $store = new IndexerStatusStore(self::TEST_DIR); + + $this->expectException(InvalidArgumentException::class); + $store->load('test-not-readable'); + } + + private function createIndexerStatus(): IndexerStatus + { + + $startTime = new \DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new \DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $lastUpdate = new \DateTime(); + $lastUpdate->setDate(2024, 1, 31); + $lastUpdate->setTime(13, 17, 12); + + return new IndexerStatus( + IndexerStatusState::INDEXED, + $startTime, + $endTime, + 10, + 5, + 4, + $lastUpdate, + 6, + 2 + ); + } +} diff --git a/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json new file mode 100644 index 0000000..bc347ed --- /dev/null +++ b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json @@ -0,0 +1,11 @@ +{ + "state": "INDEXED", + "startTime": "2024-01-31T11:15:10+00:00", + "endTime": "2024-01-31T12:16:11+00:00", + "total": 10, + "processed": 5, + "skipped": 4, + "lastUpdate": "2024-01-31T13:17:12+00:00", + "updated": 6, + "errors": 2 +} From 41806d3bde7113ae742d1359c12bd60569c8effe Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 31 Jan 2024 15:53:11 +0100 Subject: [PATCH 034/145] fix: phpsc error --- test/Service/Indexer/IndexerStatusStoreTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php index 06938af..ce6e0ef 100644 --- a/test/Service/Indexer/IndexerStatusStoreTest.php +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -139,7 +139,8 @@ public function testLoadFileNotExists(): void */ public function testLoadFileNotReadable(): void { - $file = self::TEST_DIR . '/' . 'atoolo.search.index.test-not-readable.status.json'; + $file = self::TEST_DIR . '/' . + 'atoolo.search.index.test-not-readable.status.json'; $filesystem = new Filesystem(); $filesystem->touch($file); From cf322918f59d9d68dc178c5cc1604ad808a01d8a Mon Sep 17 00:00:00 2001 From: veltrup Date: Wed, 31 Jan 2024 15:56:19 +0100 Subject: [PATCH 035/145] test: fix test for ci-server --- test/Console/Command/IndexerTest.php | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index f4ce781..8f17514 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -144,19 +144,10 @@ public function testExecuteIndexWithErrors(): void // the output of the command in the console $output = $commandTester->getDisplay(); - // phpcs:disable - $this->assertEquals( - <<assertStringContainsString( + 'errortest', + $output, + 'error message expected' ); - // phpcs:enable } } From 52da0f6c4473c88225dca31513f1ad437ad1a75b Mon Sep 17 00:00:00 2001 From: veltrup Date: Tue, 13 Feb 2024 11:17:31 +0100 Subject: [PATCH 036/145] feat: cli command atoolo:dump-index-document --- config/services.yml | 4 + src/Console/Command/DumpIndexDocument.php | 101 ++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/Console/Command/DumpIndexDocument.php diff --git a/config/services.yml b/config/services.yml index 3048dea..db4c495 100644 --- a/config/services.yml +++ b/config/services.yml @@ -13,6 +13,10 @@ services: arguments: - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } + Atoolo\Search\Console\Command\DumpIndexDocument: + arguments: + - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } + Atoolo\Search\Console\Application: public: true arguments: diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php new file mode 100644 index 0000000..e57b265 --- /dev/null +++ b/src/Console/Command/DumpIndexDocument.php @@ -0,0 +1,101 @@ +> $documentEnricherList + */ + public function __construct( + private readonly iterable $documentEnricherList + ) { + parent::__construct(); + } + + protected function configure(): void + { + $this + ->setHelp('Command to dump a index-document') + ->addArgument( + 'resource-dir', + InputArgument::REQUIRED, + 'Resource directory whose data is to be indexed.' + ) + ->addArgument( + 'paths', + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Resources paths or directories of resources to be indexed.' + ) + ; + } + + /** + * @throws JsonException + */ + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $typedInput = new TypifiedInput($input); + + $resourceDir = $typedInput->getStringArgument('resource-dir'); + + $subDirectory = null; + if (is_dir($resourceDir . '/objects')) { + $subDirectory = 'objects'; + } + + $_SERVER['RESOURCE_ROOT'] = $resourceDir; + $resourceBaseLocator = new ServerVarResourceBaseLocator( + 'RESOURCE_ROOT', + $subDirectory + ); + + $paths = $typedInput->getArrayArgument('paths'); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + + foreach ($paths as $path) { + $resource = $resourceLoader->load($path); + $doc = new IndexSchema2xDocument(); + $processId = 'process-id'; + + foreach ($this->documentEnricherList as $enricher) { + /** @var IndexSchema2xDocument $doc */ + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + + echo json_encode( + $doc->getFields(), + JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT + ); + } + + return Command::SUCCESS; + } +} From 7f1ccbef68d9c635f7e5c5bbed0ee1ec9e6cb934 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 14 Feb 2024 14:04:06 +0100 Subject: [PATCH 037/145] feat(indexer): use intro as description if description is not defined --- .../Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index d6436d6..7a92733 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -44,6 +44,11 @@ public function enrichDocument( $doc->description = $resource->getData()->getString( 'metadata.description' ); + if (empty($doc->description)) { + $doc->description = $resource->getData()->getString( + 'metadata.intro' + ); + } $doc->sp_objecttype = $resource->getObjectType(); $doc->sp_canonical = true; $doc->crawl_process_id = $processId; From abcd512077d75686690f1f2d378fde91b83d99d1 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 15 Feb 2024 09:15:25 +0100 Subject: [PATCH 038/145] feat: rename config/services.yml -> config/commands.yml --- bin/console | 4 ++-- config/{services.yml => commands.yml} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename config/{services.yml => commands.yml} (100%) diff --git a/bin/console b/bin/console index 089d452..9c84b77 100755 --- a/bin/console +++ b/bin/console @@ -4,15 +4,15 @@ require __DIR__.'/../vendor/autoload.php'; -use Symfony\Component\Config\FileLocator; use Atoolo\Search\Console\Application; +use Symfony\Component\Config\FileLocator; $container = new Symfony\Component\DependencyInjection\ContainerBuilder(); $loader = new Symfony\Component\DependencyInjection\Loader\YamlFileLoader( $container, new FileLocator(__DIR__ . '/../config')); -$loader->load('services.yml'); +$loader->load('commands.yml'); $container->compile(); $application = $container->get(Application::class); diff --git a/config/services.yml b/config/commands.yml similarity index 100% rename from config/services.yml rename to config/commands.yml From c40d02d2e835e7b46bbb2ac862435508859af802 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 15 Feb 2024 16:52:03 +0100 Subject: [PATCH 039/145] feat: content matcher to index full-text-content --- src/Console/Command/SolrIndexerBuilder.php | 15 ++- src/Service/Indexer/ContentCollector.php | 62 ++++++++++ .../Indexer/SiteKit/ContentMatcher.php | 14 +++ .../DefaultSchema2xDocumentEnricher.php | 109 +++++++++++++++++- .../Indexer/SiteKit/HeadlineMatcher.php | 29 +++++ .../Indexer/SiteKit/QuoteSectionMatcher.php | 51 ++++++++ .../Indexer/SiteKit/RichtTextMatcher.php | 22 ++++ test/Service/Indexer/ContentCollectorTest.php | 46 ++++++++ .../Indexer/SiteKit/HeadlineMatcherTest.php | 24 ++++ .../SiteKit/QuoteSectionMatcherTest.php | 32 +++++ .../Indexer/SiteKit/RichtTextMatcherTest.php | 26 +++++ 11 files changed, 424 insertions(+), 6 deletions(-) create mode 100644 src/Service/Indexer/ContentCollector.php create mode 100644 src/Service/Indexer/SiteKit/ContentMatcher.php create mode 100644 src/Service/Indexer/SiteKit/HeadlineMatcher.php create mode 100644 src/Service/Indexer/SiteKit/QuoteSectionMatcher.php create mode 100644 src/Service/Indexer/SiteKit/RichtTextMatcher.php create mode 100644 test/Service/Indexer/ContentCollectorTest.php create mode 100644 test/Service/Indexer/SiteKit/HeadlineMatcherTest.php create mode 100644 test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php create mode 100644 test/Service/Indexer/SiteKit/RichtTextMatcherTest.php diff --git a/src/Console/Command/SolrIndexerBuilder.php b/src/Console/Command/SolrIndexerBuilder.php index a88f16b..bd6920d 100644 --- a/src/Console/Command/SolrIndexerBuilder.php +++ b/src/Console/Command/SolrIndexerBuilder.php @@ -8,11 +8,15 @@ use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; +use Atoolo\Search\Service\Indexer\ContentCollector; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\LocationFinder; +use Atoolo\Search\Service\Indexer\SiteKit\ContentMatcher; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; +use Atoolo\Search\Service\Indexer\SiteKit\HeadlineMatcher; +use Atoolo\Search\Service\Indexer\SiteKit\RichtTextMatcher; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexer; use Atoolo\Search\Service\SolrParameterClientFactory; @@ -75,8 +79,17 @@ public function build(): SolrIndexer $navigationLoader = new SiteKitNavigationHierarchyLoader( $resourceLoader ); + + /** @var iterable $matcher */ + $matcher = [ + new HeadlineMatcher(), + new RichtTextMatcher(), + ]; + $contentCollector = new ContentCollector($matcher); + $schema21 = new DefaultSchema2xDocumentEnricher( - $navigationLoader + $navigationLoader, + $contentCollector ); /** @var array> $documentEnricherList */ diff --git a/src/Service/Indexer/ContentCollector.php b/src/Service/Indexer/ContentCollector.php new file mode 100644 index 0000000..372abef --- /dev/null +++ b/src/Service/Indexer/ContentCollector.php @@ -0,0 +1,62 @@ + $matchers + */ + public function __construct(private readonly iterable $matchers) + { + } + + /** + * @param array $data + */ + public function collect(array $data): string + { + $content = $this->walk([], $data); + return implode(' ', $content); + } + + /** + * @param string[] $path + * @param array $data + * @return string[] + */ + private function walk(array $path, array $data): array + { + $contentCollections = []; + foreach ($data as $key => $value) { + if (!is_array($value)) { + continue; + } + + if (is_string($key)) { + $path[] = $key; + } + + $matcherContent = []; + foreach ($this->matchers as $matcher) { + $content = $matcher->match($path, $value); + if (!is_string($content)) { + continue; + } + $matcherContent[] = $content; + } + $contentCollections[] = $matcherContent; + $contentCollections[] = $this->walk($path, $value); + + if (is_string($key)) { + array_pop($path); + } + } + + return array_merge([], ...$contentCollections); + } +} diff --git a/src/Service/Indexer/SiteKit/ContentMatcher.php b/src/Service/Indexer/SiteKit/ContentMatcher.php new file mode 100644 index 0000000..823ddb7 --- /dev/null +++ b/src/Service/Indexer/SiteKit/ContentMatcher.php @@ -0,0 +1,14 @@ + $value + */ + public function match(array $path, array $value): bool|string; +} diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 7a92733..72505cd 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -7,6 +7,7 @@ use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Resource; use Atoolo\Search\Exception\DocumentEnrichingException; +use Atoolo\Search\Service\Indexer\ContentCollector; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; @@ -14,12 +15,37 @@ use Exception; /** + * @phpstan-type Phone array{ + * countryCode?:string, + * areaCode?:string, + * localNumber?:string + * } + * @phpstan-type PhoneData array{phone:Phone} + * @phpstan-type PhoneList array + * @phpstan-type Email array{email:string} + * @phpstan-type EmailList array + * @phpstan-type ContactData array{ + * phoneList?:PhoneList, + * emailList:EmailList + * } + * @phpstan-type AddressData array{ + * buildingName?:string, + * street?:string, + * postOfficeBoxData?: array{ + * buildingName?:string + * } + * } + * @phpstan-type ContactPoint array{ + * contactData?:ContactData, + * addressData?:AddressData + * } * @implements DocumentEnricher */ class DefaultSchema2xDocumentEnricher implements DocumentEnricher { public function __construct( - private readonly SiteKitNavigationHierarchyLoader $navigationLoader + private readonly SiteKitNavigationHierarchyLoader $navigationLoader, + private readonly ContentCollector $contentCollector ) { } @@ -149,7 +175,7 @@ public function enrichDocument( if ($siteGroupId !== 0) { $sites[] = (string)$siteGroupId; } - $doc->sp_site = array_unique($sites, SORT_STRING); + $doc->sp_site = array_unique($sites); } catch (Exception $e) { throw new DocumentEnrichingException( $resource->getLocation(), @@ -226,9 +252,6 @@ public function enrichDocument( 'text/html; charset=UTF-8' ); $doc->meta_content_type = $contentType; - $doc->content = $resource->getData()->getString( - 'searchindexdata.content' - ); $accessType = $resource->getData()->getString('init.access.type'); @@ -252,9 +275,85 @@ public function enrichDocument( $doc->sp_source = ['internal']; + return $this->enrichContent($resource, $doc); + } + + /** + * @param IndexSchema2xDocument $doc + * @return IndexSchema2xDocument + */ + private function enrichContent( + Resource $resource, + IndexDocument $doc, + ): IndexDocument { + + $content = []; + $content[] = $resource->getData()->getString( + 'searchindexdata.content' + ); + + $content[] = $this->contentCollector->collect( + $resource->getData()->getArray('content') + ); + + /** @var ContactPoint $contactPoint */ + $contactPoint = $resource->getData()->getArray('metadata.contactPoint'); + $content[] = $this->contactPointToContent($contactPoint); + + $cleanContent = preg_replace( + '/\s+/', + ' ', + implode(' ', $content) + ); + + $doc->content = trim($cleanContent ?? ''); + return $doc; } + /** + * @param ContactPoint $contactPoint + * @return string + */ + private function contactPointToContent(array $contactPoint): string + { + if (empty($contactPoint)) { + return ''; + } + + $content = []; + foreach (($contactPoint['contactData']['phoneList'] ?? []) as $phone) { + $countryCode = $phone['phone']['countryCode'] ?? ''; + if ( + !empty($countryCode) && + !in_array($countryCode, $content, true) + ) { + $content[] = '+' . $countryCode; + } + $areaCode = $phone['phone']['areaCode'] ?? ''; + if (!empty($areaCode) && !in_array($areaCode, $content, true)) { + $content[] = $areaCode; + $content[] = '0' . $areaCode; + } + $content[] = ($phone['phone']['localNumber'] ?? ''); + } + foreach (($contactPoint['contactData']['emailList'] ?? []) as $email) { + $content[] = $email['email']; + } + + if (isset($contactPoint['addressData'])) { + $addressData = $contactPoint['addressData']; + $content[] = ($addressData['street'] ?? ''); + $content[] = ($addressData['buildingName'] ?? ''); + $content[] = ( + $addressData['postOfficeBoxData']['buildingName'] ?? + '' + ); + } + + return implode(' ', $content); + } + private function idWithoutSignature(string $id): int { $s = substr($id, -11); diff --git a/src/Service/Indexer/SiteKit/HeadlineMatcher.php b/src/Service/Indexer/SiteKit/HeadlineMatcher.php new file mode 100644 index 0000000..26e2466 --- /dev/null +++ b/src/Service/Indexer/SiteKit/HeadlineMatcher.php @@ -0,0 +1,29 @@ + [ + [ + "model" => [ + "richText" => [ + "normalized" => true, + "modelType" => "html.richText", + "text" => "

Ein Text

" + ] + ] + ] + ] + ]; + $content = $collector->collect($data); + + $this->assertEquals('

Ein Text

', $content, 'unexpected content'); + } +} diff --git a/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php new file mode 100644 index 0000000..6e1632c --- /dev/null +++ b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php @@ -0,0 +1,24 @@ + "Überschrift" + ]; + + $content = $matcher->match(['items', 'model'], $value); + + $this->assertEquals('Überschrift', $content, 'unexpected headline'); + } +} diff --git a/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php new file mode 100644 index 0000000..b6882cf --- /dev/null +++ b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php @@ -0,0 +1,32 @@ + "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEquals( + 'Quote-Text Citation', + $content, + 'unexpected quote text' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php new file mode 100644 index 0000000..5d98bf9 --- /dev/null +++ b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php @@ -0,0 +1,26 @@ + true, + "modelType" => "html.richText", + "text" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEquals('Ein Text', $content, 'unexpected content'); + } +} From 6222660794cd1166da99d9790921979f09a263cc Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 15 Feb 2024 17:23:00 +0100 Subject: [PATCH 040/145] feat: add category name to full-text content --- .../Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 72505cd..86a3f57 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -300,6 +300,12 @@ private function enrichContent( $contactPoint = $resource->getData()->getArray('metadata.contactPoint'); $content[] = $this->contactPointToContent($contactPoint); + /** @var array $categories */ + $categories = $resource->getData()->getArray('metadata.categories'); + foreach ($categories as $category) { + $content[] = $category['name'] ?? ''; + } + $cleanContent = preg_replace( '/\s+/', ' ', From aaa78a02b398fd6abb4113afc7a0c25af13788dc Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 20 Feb 2024 16:19:59 +0100 Subject: [PATCH 041/145] refactor: simplify dto's --- src/Console/Command/Search.php | 3 +- src/Dto/Indexer/IndexerStatusState.php | 12 --- src/Dto/Search/Query/Facet/Facet.php | 9 +- src/Dto/Search/Query/Facet/FacetField.php | 34 ++----- .../Search/Query/Facet/FacetMultiQuery.php | 30 ++---- src/Dto/Search/Query/Facet/FacetQuery.php | 21 +--- src/Dto/Search/Query/Facet/GroupFacet.php | 2 +- .../Search/Query/Facet/ObjectTypeFacet.php | 2 +- src/Dto/Search/Query/Facet/SiteFacet.php | 2 +- src/Dto/Search/Query/Filter/AndFilter.php | 2 - src/Dto/Search/Query/Filter/FieldFilter.php | 3 +- src/Dto/Search/Query/Filter/Filter.php | 17 +--- src/Dto/Search/Query/MoreLikeThisQuery.php | 53 +++------- src/Dto/Search/Query/SelectQuery.php | 96 +++---------------- src/Dto/Search/Query/SelectQueryBuilder.php | 94 ++++-------------- src/Dto/Search/Query/Sort/Criteria.php | 7 +- src/Dto/Search/Query/SuggestQuery.php | 40 ++------ src/Dto/Search/Result/Facet.php | 14 +-- src/Dto/Search/Result/FacetGroup.php | 17 +--- src/Dto/Search/Result/SearchResult.php | 48 ++-------- src/Dto/Search/Result/SuggestResult.php | 17 +--- src/Dto/Search/Result/Suggestion.php | 14 +-- src/Service/Search/SolrMoreLikeThis.php | 10 +- src/Service/Search/SolrSelect.php | 56 +++++------ src/Service/Search/SolrSuggest.php | 25 +++-- test/Dto/Indexer/IndexerParameterTest.php | 22 +++++ test/Dto/Indexer/IndexerStatusTest.php | 38 +++++++- 27 files changed, 207 insertions(+), 481 deletions(-) create mode 100644 test/Dto/Indexer/IndexerParameterTest.php diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index c06cd09..54302a5 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -7,6 +7,7 @@ use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Dto\Search\Query\SelectQuery; +use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; @@ -125,7 +126,7 @@ protected function createSearch(): SolrSelect protected function buildQuery(InputInterface $input): SelectQuery { - $builder = SelectQuery::builder(); + $builder = new SelectQueryBuilder(); $builder->index($this->index); $text = $input->getArgument('text'); diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php index 1de01a4..4985d19 100644 --- a/src/Dto/Indexer/IndexerStatusState.php +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -10,16 +10,4 @@ enum IndexerStatusState: string case RUNNING = 'RUNNING'; case INDEXED = 'INDEXED'; case ABORTED = 'ABORTED'; - - public static function valueOf(string $name): IndexerStatusState - { - foreach (self::cases() as $status) { - if ($name === $status->name) { - return $status; - } - } - throw new \ValueError( - "$name is not a valid backing value for enum " . self::class - ); - } } diff --git a/src/Dto/Search/Query/Facet/Facet.php b/src/Dto/Search/Query/Facet/Facet.php index e71f2ac..fd945d2 100644 --- a/src/Dto/Search/Query/Facet/Facet.php +++ b/src/Dto/Search/Query/Facet/Facet.php @@ -4,8 +4,11 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; -interface Facet +abstract class Facet { - public function getKey(): string; - public function getExcludeFilter(): ?string; + public function __construct( + public readonly string $key, + public readonly ?string $excludeFilter + ) { + } } diff --git a/src/Dto/Search/Query/Facet/FacetField.php b/src/Dto/Search/Query/Facet/FacetField.php index e4bab7e..b5b4223 100644 --- a/src/Dto/Search/Query/Facet/FacetField.php +++ b/src/Dto/Search/Query/Facet/FacetField.php @@ -4,39 +4,17 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; -class FacetField implements Facet +class FacetField extends Facet { /** * @param string[] $terms */ public function __construct( - private readonly string $key, - private readonly string $field, - private readonly array $terms, - private readonly ?string $excludeFilter + string $key, + public readonly string $field, + public readonly array $terms, + ?string $excludeFilter ) { - } - - public function getKey(): string - { - return $this->key; - } - - public function getField(): string - { - return $this->field; - } - - /** - * @return string[] - */ - public function getTerms(): array - { - return $this->terms; - } - - public function getExcludeFilter(): ?string - { - return $this->excludeFilter; + parent::__construct($key, $excludeFilter); } } diff --git a/src/Dto/Search/Query/Facet/FacetMultiQuery.php b/src/Dto/Search/Query/Facet/FacetMultiQuery.php index 4da4569..06b45a9 100644 --- a/src/Dto/Search/Query/Facet/FacetMultiQuery.php +++ b/src/Dto/Search/Query/Facet/FacetMultiQuery.php @@ -4,33 +4,17 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; -class FacetMultiQuery implements Facet +class FacetMultiQuery extends Facet { /** - * @param string $key - * @param FacetQuery[] $queryList + * @param FacetQuery[] $queries + * @param string|null $excludeFilter */ public function __construct( - private readonly string $key, - private readonly array $queryList, - private readonly ?string $excludeFilter + string $key, + public readonly array $queries, + ?string $excludeFilter ) { - } - - public function getKey(): string - { - return $this->key; - } - /** - * @return FacetQuery[] - */ - public function getQueryList(): array - { - return $this->queryList; - } - - public function getExcludeFilter(): ?string - { - return $this->excludeFilter; + parent::__construct($key, $excludeFilter); } } diff --git a/src/Dto/Search/Query/Facet/FacetQuery.php b/src/Dto/Search/Query/Facet/FacetQuery.php index a1b5deb..2841844 100644 --- a/src/Dto/Search/Query/Facet/FacetQuery.php +++ b/src/Dto/Search/Query/Facet/FacetQuery.php @@ -4,24 +4,13 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; -class FacetQuery implements Facet +class FacetQuery extends Facet { public function __construct( - private readonly string $key, - private readonly string $query, - private readonly ?string $excludeFilter + string $key, + public readonly string $query, + ?string $excludeFilter ) { - } - public function getKey(): string - { - return $this->key; - } - public function getQuery(): string - { - return $this->query; - } - public function getExcludeFilter(): ?string - { - return $this->excludeFilter; + parent::__construct($key, $excludeFilter); } } diff --git a/src/Dto/Search/Query/Facet/GroupFacet.php b/src/Dto/Search/Query/Facet/GroupFacet.php index 88606d9..0d260b7 100644 --- a/src/Dto/Search/Query/Facet/GroupFacet.php +++ b/src/Dto/Search/Query/Facet/GroupFacet.php @@ -11,7 +11,7 @@ class GroupFacet extends FacetField */ public function __construct( string $key, - array $groups, + public readonly array $groups, ?string $excludeFilter ) { parent::__construct( diff --git a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php index da3fffa..582cd48 100644 --- a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php @@ -11,7 +11,7 @@ class ObjectTypeFacet extends FacetField */ public function __construct( string $key, - array $objectTypes, + public readonly array $objectTypes, ?string $excludeFilter ) { parent::__construct( diff --git a/src/Dto/Search/Query/Facet/SiteFacet.php b/src/Dto/Search/Query/Facet/SiteFacet.php index 6ea341e..5527a55 100644 --- a/src/Dto/Search/Query/Facet/SiteFacet.php +++ b/src/Dto/Search/Query/Facet/SiteFacet.php @@ -11,7 +11,7 @@ class SiteFacet extends FacetField */ public function __construct( string $key, - array $sites, + public readonly array $sites, ?string $excludeFilter ) { parent::__construct( diff --git a/src/Dto/Search/Query/Filter/AndFilter.php b/src/Dto/Search/Query/Filter/AndFilter.php index de63492..5485cce 100644 --- a/src/Dto/Search/Query/Filter/AndFilter.php +++ b/src/Dto/Search/Query/Filter/AndFilter.php @@ -4,8 +4,6 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; -use Atoolo\Search\Dto\Search\Query\Filter\Filter; - class AndFilter extends Filter { /** diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index 524b591..6f60ee8 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -10,6 +10,7 @@ class FieldFilter extends Filter * @var string[] */ private readonly array $values; + public function __construct( ?string $key, private readonly string $field, @@ -42,7 +43,7 @@ public function exclude(): FieldFilter $field = '-' . $field; } return new FieldFilter( - $this->getKey(), + $this->key, $field, ...$this->values ); diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php index 73fcc0c..667dead 100644 --- a/src/Dto/Search/Query/Filter/Filter.php +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -10,23 +10,10 @@ abstract class Filter * @param string[] $tags */ public function __construct( - private readonly ?string $key, - private readonly array $tags = [] + public readonly ?string $key, + public readonly array $tags = [] ) { } - public function getKey(): ?string - { - return $this->key; - } - abstract public function getQuery(): string; - - /** - * @return string[] - */ - public function getTags(): array - { - return $this->tags; - } } diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index 2cc391a..ead04c7 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -6,49 +6,26 @@ use Atoolo\Search\Dto\Search\Query\Filter\Filter; +/** + * MoreLikeThis is a function in search technologies that finds similar + * documents or content to a given document or query. It analyzes the + * properties of the reference document, such as keywords or structure, + * to identify other documents with similar characteristics in the + * database or search index. + */ class MoreLikeThisQuery { /** - * @param string[] $fieldList - * @param Filter[] $filterList + * @param string $index name of the index to use + * @param Filter[] $filter + * @param string[] $fields */ public function __construct( - private readonly string $core, - private readonly string $location, - private readonly array $filterList = [], - private readonly int $limit = 5, - private readonly array $fieldList = ['description', 'content'] + public readonly string $index, + public readonly string $location, + public readonly array $filter = [], + public readonly int $limit = 5, + public readonly array $fields = ['description', 'content'] ) { } - - public function getCore(): string - { - return $this->core; - } - - public function getLocation(): string - { - return $this->location; - } - - /** - * @return Filter[] - */ - public function getFilterList(): array - { - return $this->filterList; - } - - public function getLimit(): int - { - return $this->limit; - } - - /** - * @return array - */ - public function getFieldList(): array - { - return $this->fieldList; - } } diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index a902eac..978cc44 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -10,88 +10,22 @@ class SelectQuery { - private readonly string $index; - private readonly string $text; - private readonly int $offset; - private readonly int $limit; /** - * @var Criteria[] + * @param Criteria[] $sort + * @param Filter[] $filter + * @param Facet[] $facets + * @internal Do not use the constructor directly, + * but the SelectQueryBuilder */ - private readonly array $sort; - /** - * @var Filter[] - */ - private readonly array $filterList; - /** - * @var Facet[] - */ - private readonly array $facetList; - private readonly QueryDefaultOperator $queryDefaultOperator; - - /** - * @internal - */ - public function __construct(SelectQueryBuilder $builder) - { - $this->index = $builder->getIndex(); - $this->text = $builder->getText(); - $this->offset = $builder->getOffset(); - $this->limit = $builder->getLimit(); - $this->sort = $builder->getSort(); - $this->filterList = $builder->getFilterList(); - $this->facetList = $builder->getFacetList(); - $this->queryDefaultOperator = $builder->getQueryDefaultOperator(); - } - - public static function builder(): SelectQueryBuilder - { - return new SelectQueryBuilder(); - } - - public function getIndex(): string - { - return $this->index; - } - - public function getText(): string - { - return $this->text; - } - - public function getOffset(): int - { - return $this->offset; - } - - public function getLimit(): int - { - return $this->limit; - } - - /** - * @return Criteria[] - */ - public function getSort(): array - { - return $this->sort; - } - - /** - * @return Filter[] - */ - public function getFilterList(): array - { - return $this->filterList; - } - /** - * @return Facet[] - */ - public function getFacetList(): array - { - return $this->facetList; - } - public function getQueryDefaultOperator(): QueryDefaultOperator - { - return $this->queryDefaultOperator; + public function __construct( + public readonly string $index, + public readonly string $text, + public readonly int $offset, + public readonly int $limit, + public readonly array $sort, + public readonly array $filter, + public readonly array $facets, + public readonly QueryDefaultOperator $queryDefaultOperator + ) { } } diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 60b759d..3bd8600 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -21,19 +21,16 @@ class SelectQueryBuilder /** * @var array */ - private array $filterList = []; + private array $filter = []; /** * @var array */ - private array $facetList = []; + private array $facets = []; private QueryDefaultOperator $queryDefaultOperator = QueryDefaultOperator::AND; - /** - * @internal - */ public function __construct() { } @@ -47,28 +44,12 @@ public function index(string $index): SelectQueryBuilder return $this; } - /** - * @internal - */ - public function getIndex(): string - { - return $this->index; - } - public function text(string $text): SelectQueryBuilder { $this->text = $text; return $this; } - /** - * @internal - */ - public function getText(): string - { - return $this->text; - } - public function offset(int $offset): SelectQueryBuilder { if ($offset < 0) { @@ -78,14 +59,6 @@ public function offset(int $offset): SelectQueryBuilder return $this; } - /** - * @internal - */ - public function getOffset(): int - { - return $this->offset; - } - public function limit(int $limit): SelectQueryBuilder { if ($limit < 0) { @@ -95,14 +68,6 @@ public function limit(int $limit): SelectQueryBuilder return $this; } - /** - * @internal - */ - public function getLimit(): int - { - return $this->limit; - } - public function sort(Criteria ...$criteriaList): SelectQueryBuilder { foreach ($criteriaList as $criteria) { @@ -111,61 +76,34 @@ public function sort(Criteria ...$criteriaList): SelectQueryBuilder return $this; } - /** - * @internal - * @return Criteria[] - */ - public function getSort(): array - { - return $this->sort; - } - public function filter(Filter ...$filterList): SelectQueryBuilder { foreach ($filterList as $filter) { - if (isset($this->filterList[$filter->getKey()])) { + if (isset($this->filter[$filter->key])) { throw new \InvalidArgumentException( - 'filter key "' . $filter->getKey() . + 'filter key "' . $filter->key . '" already exists' ); } - $this->filterList[$filter->getKey()] = $filter; + $this->filter[$filter->key] = $filter; } return $this; } - /** - * @internal - * @return Filter[] - */ - public function getFilterList(): array - { - return array_values($this->filterList); - } - public function facet(Facet ...$facetList): SelectQueryBuilder { foreach ($facetList as $facet) { - if (isset($this->facetList[$facet->getKey()])) { + if (isset($this->facets[$facet->key])) { throw new \InvalidArgumentException( - 'facet key "' . $facet->getKey() . + 'facet key "' . $facet->key . '" already exists' ); } - $this->facetList[$facet->getKey()] = $facet; + $this->facets[$facet->key] = $facet; } return $this; } - /** - * @internal - * @return Facet[] - */ - public function getFacetList(): array - { - return array_values($this->facetList); - } - public function queryDefaultOperator( QueryDefaultOperator $queryDefaultOperator ): SelectQueryBuilder { @@ -173,16 +111,20 @@ public function queryDefaultOperator( return $this; } - public function getQueryDefaultOperator(): QueryDefaultOperator - { - return $this->queryDefaultOperator; - } - public function build(): SelectQuery { if (empty($this->index)) { throw new \InvalidArgumentException('index is not set'); } - return new SelectQuery($this); + return new SelectQuery( + $this->index, + $this->text, + $this->offset, + $this->limit, + $this->sort, + $this->filter, + $this->facets, + $this->queryDefaultOperator + ); } } diff --git a/src/Dto/Search/Query/Sort/Criteria.php b/src/Dto/Search/Query/Sort/Criteria.php index 8daf3fa..8d309c5 100644 --- a/src/Dto/Search/Query/Sort/Criteria.php +++ b/src/Dto/Search/Query/Sort/Criteria.php @@ -7,12 +7,7 @@ abstract class Criteria { public function __construct( - private readonly Direction $direction + public readonly Direction $direction ) { } - - public function getDirection(): Direction - { - return $this->direction; - } } diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php index 231ed88..f1a9d8f 100644 --- a/src/Dto/Search/Query/SuggestQuery.php +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -6,43 +6,21 @@ use Atoolo\Search\Dto\Search\Query\Filter\Filter; +/** + * In the search context, "Suggest" refers to a feature that automatically + * makes suggestions as the user enters a search query to speed up and + * simplify the search process. + */ class SuggestQuery { /** * @param Filter[] $filter */ public function __construct( - private readonly string $index, - private readonly string $text, - private readonly array $filter = [], - private readonly int $limit = 10, - private readonly string $field = 'raw_content' + public readonly string $index, + public readonly string $text, + public readonly array $filter = [], + public readonly int $limit = 10 ) { } - - public function getIndex(): string - { - return $this->index; - } - public function getText(): string - { - return $this->text; - } - - public function getLimit(): int - { - return $this->limit; - } - - /** - * @return Filter[] - */ - public function getFilter(): array - { - return $this->filter; - } - public function getField(): string - { - return $this->field; - } } diff --git a/src/Dto/Search/Result/Facet.php b/src/Dto/Search/Result/Facet.php index ad31aa1..e0789ca 100644 --- a/src/Dto/Search/Result/Facet.php +++ b/src/Dto/Search/Result/Facet.php @@ -7,18 +7,8 @@ class Facet { public function __construct( - private readonly string $key, - private readonly int $hits + public readonly string $key, + public readonly int $hits ) { } - - public function getKey(): string - { - return $this->key; - } - - public function getHits(): int - { - return $this->hits; - } } diff --git a/src/Dto/Search/Result/FacetGroup.php b/src/Dto/Search/Result/FacetGroup.php index 1d6ae52..caf4de9 100644 --- a/src/Dto/Search/Result/FacetGroup.php +++ b/src/Dto/Search/Result/FacetGroup.php @@ -10,21 +10,8 @@ class FacetGroup * @param Facet[] $facets */ public function __construct( - private readonly string $key, - private readonly array $facets + public readonly string $key, + public readonly array $facets ) { } - - public function getKey(): string - { - return $this->key; - } - - /** - * @return Facet[] - */ - public function getFacets(): array - { - return $this->facets; - } } diff --git a/src/Dto/Search/Result/SearchResult.php b/src/Dto/Search/Result/SearchResult.php index 6da8229..904125f 100644 --- a/src/Dto/Search/Result/SearchResult.php +++ b/src/Dto/Search/Result/SearchResult.php @@ -19,12 +19,12 @@ class SearchResult implements IteratorAggregate * @param FacetGroup[] $facetGroups */ public function __construct( - private readonly int $total, - private readonly int $limit, - private readonly int $offset, - private readonly array $results, - private readonly array $facetGroups, - private readonly int $queryTime + public readonly int $total, + public readonly int $limit, + public readonly int $offset, + public readonly array $results, + public readonly array $facetGroups, + public readonly int $queryTime ) { } @@ -32,40 +32,4 @@ public function getIterator(): Traversable { return new ArrayIterator($this->results); } - - /** - * @return Resource[] - */ - public function getResults(): array - { - return $this->results; - } - - public function getTotal(): int - { - return $this->total; - } - - public function getLimit(): int - { - return $this->limit; - } - - public function getOffset(): int - { - return $this->offset; - } - - /** - * @return FacetGroup[] - */ - public function getFacetGroups(): array - { - return $this->facetGroups; - } - - public function getQueryTime(): int - { - return $this->queryTime; - } } diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php index 90ed1e4..964f11f 100644 --- a/src/Dto/Search/Result/SuggestResult.php +++ b/src/Dto/Search/Result/SuggestResult.php @@ -17,8 +17,8 @@ class SuggestResult implements IteratorAggregate * @param int $queryTime */ public function __construct( - private readonly array $suggestions, - private readonly int $queryTime + public readonly array $suggestions, + public readonly int $queryTime ) { } @@ -29,17 +29,4 @@ public function getIterator(): ArrayIterator { return new ArrayIterator($this->suggestions); } - - /** - * @return Suggestion[] - */ - public function getSuggestions(): array - { - return $this->suggestions; - } - - public function getQueryTime(): int - { - return $this->queryTime; - } } diff --git a/src/Dto/Search/Result/Suggestion.php b/src/Dto/Search/Result/Suggestion.php index bfe9293..0dd8426 100644 --- a/src/Dto/Search/Result/Suggestion.php +++ b/src/Dto/Search/Result/Suggestion.php @@ -7,18 +7,8 @@ class Suggestion { public function __construct( - private readonly string $term, - private readonly int $hits + public readonly string $term, + public readonly int $hits ) { } - - public function getTerm(): string - { - return $this->term; - } - - public function getHits(): int - { - return $this->hits; - } } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 5d2f2b9..de553e5 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -25,7 +25,7 @@ public function __construct( public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { - $client = $this->clientFactory->create($query->getCore()); + $client = $this->clientFactory->create($query->index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SelectResult $result */ $result = $client->execute($solrQuery); @@ -39,16 +39,16 @@ private function buildSolrQuery( $solrQuery = $client->createMoreLikeThis(); $solrQuery->setOmitHeader(false); - $solrQuery->setQuery('url:"' . $query->getLocation() . '"'); - $solrQuery->setMltFields($query->getFieldList()); - $solrQuery->setRows($query->getLimit()); + $solrQuery->setQuery('url:"' . $query->location . '"'); + $solrQuery->setMltFields($query->fields); + $solrQuery->setRows($query->limit); $solrQuery->setMinimumTermFrequency(2); $solrQuery->setMatchInclude(true); $solrQuery->createFilterQuery('nomedia') ->setQuery('-sp_objecttype:media'); // Filter - foreach ($query->getFilterList() as $filter) { + foreach ($query->filter as $filter) { $solrQuery->createFilterQuery($filter->getKey()) ->setQuery($filter->getQuery()) ->setTags($filter->getTags()); diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 4815811..6787a04 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -44,7 +44,7 @@ public function __construct( public function select(SelectQuery $query): SearchResult { - $client = $this->clientFactory->create($query->getIndex()); + $client = $this->clientFactory->create($query->index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SelectResult $result */ @@ -64,26 +64,26 @@ private function buildSolrQuery( $solrQuery = $solrQueryModifier->modify($solrQuery); } - $solrQuery->setStart($query->getOffset()); - $solrQuery->setRows($query->getLimit()); + $solrQuery->setStart($query->offset); + $solrQuery->setRows($query->limit); // to get query-time $solrQuery->setOmitHeader(false); - $this->addSortToSolrQuery($solrQuery, $query->getSort()); + $this->addSortToSolrQuery($solrQuery, $query->sort); $this->addRequiredFieldListToSolrQuery($solrQuery); - $this->addTextFilterToSolrQuery($solrQuery, $query->getText()); + $this->addTextFilterToSolrQuery($solrQuery, $query->text); $this->addQueryDefaultOperatorToSolrQuery( $solrQuery, - $query->getQueryDefaultOperator() + $query->queryDefaultOperator ); $this->addFilterQueriesToSolrQuery( $solrQuery, - $query->getFilterList() + $query->filter ); $this->addFacetListToSolrQuery( $solrQuery, - $query->getFacetList() + $query->facets ); return $solrQuery; @@ -101,7 +101,7 @@ private function addSortToSolrQuery( if ($criteria instanceof Name) { $field = 'sp_name'; } elseif ($criteria instanceof Headline) { - $field = 'sp_headline'; + $field = 'sp_title'; } elseif ($criteria instanceof Date) { $field = 'sp_date'; } elseif ($criteria instanceof Natural) { @@ -114,7 +114,7 @@ private function addSortToSolrQuery( ); } - $direction = strtolower($criteria->getDirection()->name); + $direction = strtolower($criteria->direction->name); $sorts[$field] = $direction; } @@ -170,10 +170,10 @@ private function addFilterQueriesToSolrQuery( ): void { foreach ($filterList as $filter) { - $key = $filter->getKey() ?? uniqid('', true); + $key = $filter->key ?? uniqid('', true); $solrQuery->createFilterQuery($key) ->setQuery($filter->getQuery()) - ->setTags($filter->getTags()); + ->setTags($filter->tags); } } @@ -207,16 +207,16 @@ private function addFacetFieldToSolrQuery( FacetField $facet ): void { $facetSet = $solrQuery->getFacetSet(); - $field = $facet->getField(); + $field = $facet->field; // https://solr.apache.org/guide/solr/latest/query-guide/faceting.html#tagging-and-excluding-filters - if ($facet->getExcludeFilter() !== null) { - $field = '{!ex=' . $facet->getExcludeFilter() . '}' . $field; + if ($facet->excludeFilter !== null) { + $field = '{!ex=' . $facet->excludeFilter . '}' . $field; } /** @var Field $solariumFacet */ - $solariumFacet = $facetSet->createFacetField($facet->getKey()); + $solariumFacet = $facetSet->createFacetField($facet->key); $solariumFacet ->setField($field) - ->setTerms($facet->getTerms()); + ->setTerms($facet->terms); } /** @@ -227,8 +227,8 @@ private function addFacetQueryToSolrQuery( FacetQuery $facet ): void { $facetSet = $solrQuery->getFacetSet(); - $facetSet->createFacetQuery($facet->getKey()) - ->setQuery($facet->getQuery()); + $facetSet->createFacetQuery($facet->key) + ->setQuery($facet->query); } /** @@ -239,11 +239,11 @@ private function addFacetMultiQueryToSolrQuery( FacetMultiQuery $facet ): void { $facetSet = $solrQuery->getFacetSet(); - $solrFacet = $facetSet->createFacetMultiQuery($facet->getKey()); - foreach ($facet->getQueryList() as $facetQuery) { + $solrFacet = $facetSet->createFacetMultiQuery($facet->key); + foreach ($facet->queries as $facetQuery) { $solrFacet->createQuery( - $facetQuery->getKey(), - $facetQuery->getQuery() + $facetQuery->key, + $facetQuery->query ); } } @@ -259,8 +259,8 @@ private function buildResult( return new SearchResult( $result->getNumFound() ?? 0, - $query->getLimit(), - $query->getOffset(), + $query->limit, + $query->offset, $resourceList, $facetGroupList, $result->getQueryTime() ?? 0 @@ -281,14 +281,14 @@ private function buildFacetGroupList( } $facetGroupList = []; - foreach ($query->getFacetList() as $facet) { + foreach ($query->facets as $facet) { /** @var ?\Solarium\Component\Result\Facet\Field $resultFacet */ - $resultFacet = $facetSet->getFacet($facet->getKey()); + $resultFacet = $facetSet->getFacet($facet->key); if ($resultFacet === null) { continue; } $facetGroupList[] = $this->buildFacetGroup( - $facet->getKey(), + $facet->key, $resultFacet ); } diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 33b0fcb..960ee49 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -26,6 +26,8 @@ */ class SolrSuggest implements SuggestSearcher { + private const INDEX_SUGGEST_FIELD = 'raw_content'; + public function __construct( private readonly SolrParameterClientFactory $clientFactory ) { @@ -36,11 +38,11 @@ public function __construct( */ public function suggest(SuggestQuery $query): SuggestResult { - $client = $this->clientFactory->create($query->getIndex()); + $client = $this->clientFactory->create($query->index); $solrQuery = $this->buildSolrQuery($client, $query); $solrResult = $client->select($solrQuery); - return $this->buildResult($solrResult, $query->getField()); + return $this->buildResult($solrResult); } private function buildSolrQuery( @@ -62,17 +64,17 @@ private function buildSolrQuery( $solrQuery->addParam("facet.method", "enum"); $solrQuery->addParam( "facet.prefix", - $query->getText() + $query->text ); - $solrQuery->addParam("facet.limit", $query->getLimit()); - $solrQuery->addParam("facet.field", $query->getField()); + $solrQuery->addParam("facet.limit", $query->limit); + $solrQuery->addParam("facet.field", self::INDEX_SUGGEST_FIELD); $solrQuery->setOmitHeader(false); $solrQuery->setStart(0); $solrQuery->setRows(0); // Filter - foreach ($query->getFilter() as $filter) { + foreach ($query->filter as $filter) { $solrQuery->createFilterQuery($filter->getKey()) ->setQuery($filter->getQuery()) ->setTags($filter->getTags()); @@ -82,12 +84,10 @@ private function buildSolrQuery( } private function buildResult( - SolrSelectResult $solrResult, - string $resultField + SolrSelectResult $solrResult ): SuggestResult { $suggestions = $this->parseSuggestion( - $solrResult->getResponse()->getBody(), - $resultField + $solrResult->getResponse()->getBody() ); return new SuggestResult( $suggestions, @@ -100,8 +100,7 @@ private function buildResult( * @return Suggestion[] */ private function parseSuggestion( - string $responseBody, - string $facetField + string $responseBody ): array { try { /** @var SolariumResponse $json */ @@ -112,7 +111,7 @@ private function parseSuggestion( JSON_THROW_ON_ERROR ); $facets = - $json['facet_counts']['facet_fields'][$facetField] + $json['facet_counts']['facet_fields'][self::INDEX_SUGGEST_FIELD] ?? []; $len = count($facets); diff --git a/test/Dto/Indexer/IndexerParameterTest.php b/test/Dto/Indexer/IndexerParameterTest.php new file mode 100644 index 0000000..de36053 --- /dev/null +++ b/test/Dto/Indexer/IndexerParameterTest.php @@ -0,0 +1,22 @@ +expectException(InvalidArgumentException::class); + new IndexerParameter( + 'test', + 0, + 9 + ); + } +} diff --git a/test/Dto/Indexer/IndexerStatusTest.php b/test/Dto/Indexer/IndexerStatusTest.php index f4e0d67..8d24020 100644 --- a/test/Dto/Indexer/IndexerStatusTest.php +++ b/test/Dto/Indexer/IndexerStatusTest.php @@ -6,6 +6,7 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Dto\Indexer\IndexerStatusState; +use DateTime; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -16,15 +17,15 @@ class IndexerStatusTest extends TestCase public function setUp(): void { - $startTime = new \DateTime(); + $startTime = new DateTime(); $startTime->setDate(2024, 1, 31); $startTime->setTime(11, 15, 10); - $endTime = new \DateTime(); + $endTime = new DateTime(); $endTime->setDate(2024, 1, 31); $endTime->setTime(12, 16, 11); - $lastUpdate = new \DateTime(); + $lastUpdate = new DateTime(); $lastUpdate->setDate(2024, 1, 31); $lastUpdate->setTime(13, 17, 12); @@ -77,4 +78,35 @@ public function testEmpty(): void "unexpected status line for empty status" ); } + + public function testStatusLineWithoutEndTime(): void + { + $startTime = new DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $lastUpdate = new DateTime(); + $lastUpdate->setTimestamp(0); + + $status = new IndexerStatus( + IndexerStatusState::UNKNOWN, + $startTime, + null, + 0, + 0, + 0, + $lastUpdate, + 0, + 0 + ); + + $dateTimePattern = '[0-9]{2}\.[0-9]{2}\.[0-9]{4} [0-9]{2}:[0-9]{2}'; + $patter = '/lastUpdate: ' . $dateTimePattern . ', /'; + + $this->assertMatchesRegularExpression( + $patter, + $status->getStatusLine(), + "unexpected status line without endTime" + ); + } } From 8856dac02bf6af945c089ad8f44fe2224a7f8e0a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 20 Feb 2024 16:23:51 +0100 Subject: [PATCH 042/145] refactor: QueryDefaultOperator as backed enum --- src/Dto/Search/Query/QueryDefaultOperator.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dto/Search/Query/QueryDefaultOperator.php b/src/Dto/Search/Query/QueryDefaultOperator.php index b485d5d..9d4904c 100644 --- a/src/Dto/Search/Query/QueryDefaultOperator.php +++ b/src/Dto/Search/Query/QueryDefaultOperator.php @@ -2,8 +2,8 @@ namespace Atoolo\Search\Dto\Search\Query; -enum QueryDefaultOperator +enum QueryDefaultOperator: string { - case AND; - case OR; + case AND = 'AND'; + case OR = 'OR'; } From e05ba48cfba514cd6ef4c4f4b76776896386c4d4 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 22 Feb 2024 17:35:38 +0100 Subject: [PATCH 043/145] test: add dto tests --- phpunit.xml | 3 +- src/Dto/Indexer/IndexerStatusState.php | 3 + src/Dto/Search/Query/Facet/CategoryFacet.php | 3 + .../Query/Facet/ContentSectionTypeFacet.php | 3 + src/Dto/Search/Query/Facet/Facet.php | 3 + src/Dto/Search/Query/Facet/FacetField.php | 3 + .../Search/Query/Facet/FacetMultiQuery.php | 3 + src/Dto/Search/Query/Facet/FacetQuery.php | 3 + src/Dto/Search/Query/Facet/GroupFacet.php | 3 + .../Search/Query/Facet/ObjectTypeFacet.php | 3 + src/Dto/Search/Query/Facet/SiteFacet.php | 3 + .../Search/Query/Filter/CategoryFilter.php | 3 + .../Query/Filter/ContentSectionTypeFilter.php | 3 + src/Dto/Search/Query/Filter/FieldFilter.php | 4 +- src/Dto/Search/Query/Filter/Filter.php | 3 + src/Dto/Search/Query/Filter/GroupFilter.php | 3 + .../Search/Query/Filter/ObjectTypeFilter.php | 3 + src/Dto/Search/Query/Filter/SiteFilter.php | 3 + src/Dto/Search/Query/MoreLikeThisQuery.php | 2 + src/Dto/Search/Query/QueryDefaultOperator.php | 3 + src/Dto/Search/Query/SelectQuery.php | 3 + src/Dto/Search/Query/SelectQueryBuilder.php | 4 +- src/Dto/Search/Query/Sort/Criteria.php | 3 + src/Dto/Search/Query/Sort/Date.php | 3 + src/Dto/Search/Query/Sort/Direction.php | 3 + src/Dto/Search/Query/Sort/Headline.php | 3 + src/Dto/Search/Query/Sort/Name.php | 5 +- src/Dto/Search/Query/Sort/Natural.php | 3 + src/Dto/Search/Query/Sort/Score.php | 3 + src/Dto/Search/Query/SuggestQuery.php | 2 + src/Dto/Search/Result/Facet.php | 3 + src/Dto/Search/Result/FacetGroup.php | 3 + src/Dto/Search/Result/SearchResult.php | 1 + src/Dto/Search/Result/SuggestResult.php | 1 + src/Dto/Search/Result/Suggestion.php | 3 + test/Dto/Indexer/IndexerParameterTest.php | 2 + .../Dto/Search/Query/Filter/AndFilterTest.php | 35 ++++ .../Search/Query/Filter/ArchiveFilterTest.php | 23 +++ .../Search/Query/Filter/FieldFilterTest.php | 51 ++++++ .../Dto/Search/Query/Filter/NotFilterTest.php | 28 ++++ test/Dto/Search/Query/Filter/OrFilterTest.php | 34 ++++ .../Search/Query/Filter/QueryFilterTest.php | 24 +++ .../Search/Query/SelectQueryBuilderTest.php | 150 ++++++++++++++++++ test/Service/Indexer/ContentCollectorTest.php | 2 + .../Indexer/IndexSchema2xDocumentTest.php | 2 + .../Indexer/SiteKit/HeadlineMatcherTest.php | 34 ++++ .../SiteKit/QuoteSectionMatcherTest.php | 81 ++++++++++ .../Indexer/SiteKit/RichtTextMatcherTest.php | 39 +++++ test/Service/Indexer/SolrIndexerTest.php | 61 ++++++- 49 files changed, 660 insertions(+), 9 deletions(-) create mode 100644 test/Dto/Search/Query/Filter/AndFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/ArchiveFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/FieldFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/NotFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/OrFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/QueryFilterTest.php create mode 100644 test/Dto/Search/Query/SelectQueryBuilderTest.php diff --git a/phpunit.xml b/phpunit.xml index 8544648..1120448 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,8 @@ cacheResultFile="var/cache/.phpunit.result.cache" cacheDirectory="var/cache/.phpunit.cache"> - + diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php index 4985d19..57bc6aa 100644 --- a/src/Dto/Indexer/IndexerStatusState.php +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Indexer; +/** + * @codeCoverageIgnore + */ enum IndexerStatusState: string { case UNKNOWN = 'UNKNOWN'; diff --git a/src/Dto/Search/Query/Facet/CategoryFacet.php b/src/Dto/Search/Query/Facet/CategoryFacet.php index 90d13c4..8996dd8 100644 --- a/src/Dto/Search/Query/Facet/CategoryFacet.php +++ b/src/Dto/Search/Query/Facet/CategoryFacet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class CategoryFacet extends FacetField { /** diff --git a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php index 54f2987..04b4e91 100644 --- a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class ContentSectionTypeFacet extends FacetField { /** diff --git a/src/Dto/Search/Query/Facet/Facet.php b/src/Dto/Search/Query/Facet/Facet.php index fd945d2..3e645b5 100644 --- a/src/Dto/Search/Query/Facet/Facet.php +++ b/src/Dto/Search/Query/Facet/Facet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ abstract class Facet { public function __construct( diff --git a/src/Dto/Search/Query/Facet/FacetField.php b/src/Dto/Search/Query/Facet/FacetField.php index b5b4223..3a1dce8 100644 --- a/src/Dto/Search/Query/Facet/FacetField.php +++ b/src/Dto/Search/Query/Facet/FacetField.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class FacetField extends Facet { /** diff --git a/src/Dto/Search/Query/Facet/FacetMultiQuery.php b/src/Dto/Search/Query/Facet/FacetMultiQuery.php index 06b45a9..df4e4a9 100644 --- a/src/Dto/Search/Query/Facet/FacetMultiQuery.php +++ b/src/Dto/Search/Query/Facet/FacetMultiQuery.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class FacetMultiQuery extends Facet { /** diff --git a/src/Dto/Search/Query/Facet/FacetQuery.php b/src/Dto/Search/Query/Facet/FacetQuery.php index 2841844..1f27aad 100644 --- a/src/Dto/Search/Query/Facet/FacetQuery.php +++ b/src/Dto/Search/Query/Facet/FacetQuery.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class FacetQuery extends Facet { public function __construct( diff --git a/src/Dto/Search/Query/Facet/GroupFacet.php b/src/Dto/Search/Query/Facet/GroupFacet.php index 0d260b7..2ec75cd 100644 --- a/src/Dto/Search/Query/Facet/GroupFacet.php +++ b/src/Dto/Search/Query/Facet/GroupFacet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class GroupFacet extends FacetField { /** diff --git a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php index 582cd48..4dede84 100644 --- a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class ObjectTypeFacet extends FacetField { /** diff --git a/src/Dto/Search/Query/Facet/SiteFacet.php b/src/Dto/Search/Query/Facet/SiteFacet.php index 5527a55..47290b1 100644 --- a/src/Dto/Search/Query/Facet/SiteFacet.php +++ b/src/Dto/Search/Query/Facet/SiteFacet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Facet; +/** + * @codeCoverageIgnore + */ class SiteFacet extends FacetField { /** diff --git a/src/Dto/Search/Query/Filter/CategoryFilter.php b/src/Dto/Search/Query/Filter/CategoryFilter.php index 9d6ad53..091c7d5 100644 --- a/src/Dto/Search/Query/Filter/CategoryFilter.php +++ b/src/Dto/Search/Query/Filter/CategoryFilter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ class CategoryFilter extends FieldFilter { public function __construct( diff --git a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php index 65b92d9..22183f9 100644 --- a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ class ContentSectionTypeFilter extends FieldFilter { public function __construct( diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index 6f60ee8..9b85e9a 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -4,6 +4,8 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +use InvalidArgumentException; + class FieldFilter extends Filter { /** @@ -17,7 +19,7 @@ public function __construct( string ...$values ) { if (count($values) === 0) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( 'values is an empty array' ); } diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php index 667dead..cb5edf1 100644 --- a/src/Dto/Search/Query/Filter/Filter.php +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ abstract class Filter { /** diff --git a/src/Dto/Search/Query/Filter/GroupFilter.php b/src/Dto/Search/Query/Filter/GroupFilter.php index 4c0b558..5c480f5 100644 --- a/src/Dto/Search/Query/Filter/GroupFilter.php +++ b/src/Dto/Search/Query/Filter/GroupFilter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ class GroupFilter extends FieldFilter { public function __construct( diff --git a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php index fbee75d..4c60c69 100644 --- a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ class ObjectTypeFilter extends FieldFilter { public function __construct( diff --git a/src/Dto/Search/Query/Filter/SiteFilter.php b/src/Dto/Search/Query/Filter/SiteFilter.php index 565a494..ffc3bc7 100644 --- a/src/Dto/Search/Query/Filter/SiteFilter.php +++ b/src/Dto/Search/Query/Filter/SiteFilter.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Filter; +/** + * @codeCoverageIgnore + */ class SiteFilter extends FieldFilter { public function __construct( diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index ead04c7..f0d8213 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -12,6 +12,8 @@ * properties of the reference document, such as keywords or structure, * to identify other documents with similar characteristics in the * database or search index. + * + * @codeCoverageIgnore */ class MoreLikeThisQuery { diff --git a/src/Dto/Search/Query/QueryDefaultOperator.php b/src/Dto/Search/Query/QueryDefaultOperator.php index 9d4904c..bada423 100644 --- a/src/Dto/Search/Query/QueryDefaultOperator.php +++ b/src/Dto/Search/Query/QueryDefaultOperator.php @@ -2,6 +2,9 @@ namespace Atoolo\Search\Dto\Search\Query; +/** + * @codeCoverageIgnore + */ enum QueryDefaultOperator: string { case AND = 'AND'; diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index 978cc44..c4d5c53 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -8,6 +8,9 @@ use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; +/** + * @codeCoverageIgnore + */ class SelectQuery { /** diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 3bd8600..56d7d18 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -122,8 +122,8 @@ public function build(): SelectQuery $this->offset, $this->limit, $this->sort, - $this->filter, - $this->facets, + array_values($this->filter), + array_values($this->facets), $this->queryDefaultOperator ); } diff --git a/src/Dto/Search/Query/Sort/Criteria.php b/src/Dto/Search/Query/Sort/Criteria.php index 8d309c5..e73e92d 100644 --- a/src/Dto/Search/Query/Sort/Criteria.php +++ b/src/Dto/Search/Query/Sort/Criteria.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ abstract class Criteria { public function __construct( diff --git a/src/Dto/Search/Query/Sort/Date.php b/src/Dto/Search/Query/Sort/Date.php index 467cf86..7f9d7cf 100644 --- a/src/Dto/Search/Query/Sort/Date.php +++ b/src/Dto/Search/Query/Sort/Date.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ class Date extends Criteria { } diff --git a/src/Dto/Search/Query/Sort/Direction.php b/src/Dto/Search/Query/Sort/Direction.php index b03e62e..a692f1a 100644 --- a/src/Dto/Search/Query/Sort/Direction.php +++ b/src/Dto/Search/Query/Sort/Direction.php @@ -2,6 +2,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ enum Direction { case ASC; diff --git a/src/Dto/Search/Query/Sort/Headline.php b/src/Dto/Search/Query/Sort/Headline.php index 90d25ec..03ed6d8 100644 --- a/src/Dto/Search/Query/Sort/Headline.php +++ b/src/Dto/Search/Query/Sort/Headline.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ class Headline extends Criteria { } diff --git a/src/Dto/Search/Query/Sort/Name.php b/src/Dto/Search/Query/Sort/Name.php index 0d67650..cbbfde6 100644 --- a/src/Dto/Search/Query/Sort/Name.php +++ b/src/Dto/Search/Query/Sort/Name.php @@ -4,8 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; -use Atoolo\Search\Dto\Search\Query\Sort\Criteria; - +/** + * @codeCoverageIgnore + */ class Name extends Criteria { } diff --git a/src/Dto/Search/Query/Sort/Natural.php b/src/Dto/Search/Query/Sort/Natural.php index 25b5de9..9f3b8c5 100644 --- a/src/Dto/Search/Query/Sort/Natural.php +++ b/src/Dto/Search/Query/Sort/Natural.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ class Natural extends Criteria { } diff --git a/src/Dto/Search/Query/Sort/Score.php b/src/Dto/Search/Query/Sort/Score.php index 07667cf..c99db51 100644 --- a/src/Dto/Search/Query/Sort/Score.php +++ b/src/Dto/Search/Query/Sort/Score.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Query\Sort; +/** + * @codeCoverageIgnore + */ class Score extends Criteria { } diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php index f1a9d8f..c4d76a6 100644 --- a/src/Dto/Search/Query/SuggestQuery.php +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -10,6 +10,8 @@ * In the search context, "Suggest" refers to a feature that automatically * makes suggestions as the user enters a search query to speed up and * simplify the search process. + * + * @codeCoverageIgnore */ class SuggestQuery { diff --git a/src/Dto/Search/Result/Facet.php b/src/Dto/Search/Result/Facet.php index e0789ca..f4ac8f2 100644 --- a/src/Dto/Search/Result/Facet.php +++ b/src/Dto/Search/Result/Facet.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Result; +/** + * @codeCoverageIgnore + */ class Facet { public function __construct( diff --git a/src/Dto/Search/Result/FacetGroup.php b/src/Dto/Search/Result/FacetGroup.php index caf4de9..9d8c350 100644 --- a/src/Dto/Search/Result/FacetGroup.php +++ b/src/Dto/Search/Result/FacetGroup.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Result; +/** + * @codeCoverageIgnore + */ class FacetGroup { /** diff --git a/src/Dto/Search/Result/SearchResult.php b/src/Dto/Search/Result/SearchResult.php index 904125f..c369422 100644 --- a/src/Dto/Search/Result/SearchResult.php +++ b/src/Dto/Search/Result/SearchResult.php @@ -11,6 +11,7 @@ /** * @implements IteratorAggregate + * @codeCoverageIgnore */ class SearchResult implements IteratorAggregate { diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php index 964f11f..015a868 100644 --- a/src/Dto/Search/Result/SuggestResult.php +++ b/src/Dto/Search/Result/SuggestResult.php @@ -9,6 +9,7 @@ /** * @implements IteratorAggregate +@codeCoverageIgnore */ class SuggestResult implements IteratorAggregate { diff --git a/src/Dto/Search/Result/Suggestion.php b/src/Dto/Search/Result/Suggestion.php index 0dd8426..b6b77e9 100644 --- a/src/Dto/Search/Result/Suggestion.php +++ b/src/Dto/Search/Result/Suggestion.php @@ -4,6 +4,9 @@ namespace Atoolo\Search\Dto\Search\Result; +/** + * @codeCoverageIgnore + */ class Suggestion { public function __construct( diff --git a/test/Dto/Indexer/IndexerParameterTest.php b/test/Dto/Indexer/IndexerParameterTest.php index de36053..91a0073 100644 --- a/test/Dto/Indexer/IndexerParameterTest.php +++ b/test/Dto/Indexer/IndexerParameterTest.php @@ -6,8 +6,10 @@ use Atoolo\Search\Dto\Indexer\IndexerParameter; use InvalidArgumentException; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(IndexerParameter::class)] class IndexerParameterTest extends TestCase { public function testToLowerChunkSize(): void diff --git a/test/Dto/Search/Query/Filter/AndFilterTest.php b/test/Dto/Search/Query/Filter/AndFilterTest.php new file mode 100644 index 0000000..10b8e9b --- /dev/null +++ b/test/Dto/Search/Query/Filter/AndFilterTest.php @@ -0,0 +1,35 @@ +createStub(Filter::class); + $a->method('getQuery') + ->willReturn('a'); + + $b = $this->createStub(Filter::class); + $b->method('getQuery') + ->willReturn('b'); + + $and = new AndFilter(null, [$a, $b]); + + assertEquals( + '(a AND b)', + $and->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/ArchiveFilterTest.php b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php new file mode 100644 index 0000000..48283eb --- /dev/null +++ b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php @@ -0,0 +1,23 @@ +assertEquals( + '-sp_archive:true', + $filter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/FieldFilterTest.php b/test/Dto/Search/Query/Filter/FieldFilterTest.php new file mode 100644 index 0000000..df44e46 --- /dev/null +++ b/test/Dto/Search/Query/Filter/FieldFilterTest.php @@ -0,0 +1,51 @@ +expectException(InvalidArgumentException::class); + new FieldFilter(null, 'test'); + } + + public function testGetQueryWithOneField(): void + { + $field = new FieldFilter(null, 'test', 'a'); + $this->assertEquals( + 'test:a', + $field->getQuery(), + 'unexpected query' + ); + } + + public function testGetQueryWithTwoFields(): void + { + $field = new FieldFilter(null, 'test', 'a', 'b'); + $this->assertEquals( + 'test:(a b)', + $field->getQuery(), + 'unexpected query' + ); + } + + public function testExclude(): void + { + $field = new FieldFilter(null, 'test', 'a'); + $exclude = $field->exclude(); + $this->assertEquals( + '-test:a', + $exclude->getQuery(), + 'unexpected exclude query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/NotFilterTest.php b/test/Dto/Search/Query/Filter/NotFilterTest.php new file mode 100644 index 0000000..575e7ac --- /dev/null +++ b/test/Dto/Search/Query/Filter/NotFilterTest.php @@ -0,0 +1,28 @@ +createStub(Filter::class); + $filter->method('getQuery') + ->willReturn('a:b'); + $notFilter = new NotFilter(null, $filter); + + $this->assertEquals( + 'NOT a:b', + $notFilter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/OrFilterTest.php b/test/Dto/Search/Query/Filter/OrFilterTest.php new file mode 100644 index 0000000..f072cfd --- /dev/null +++ b/test/Dto/Search/Query/Filter/OrFilterTest.php @@ -0,0 +1,34 @@ +createStub(Filter::class); + $a->method('getQuery') + ->willReturn('a'); + + $b = $this->createStub(Filter::class); + $b->method('getQuery') + ->willReturn('b'); + + $and = new OrFilter(null, [$a, $b]); + + assertEquals( + '(a OR b)', + $and->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/QueryFilterTest.php b/test/Dto/Search/Query/Filter/QueryFilterTest.php new file mode 100644 index 0000000..57f993f --- /dev/null +++ b/test/Dto/Search/Query/Filter/QueryFilterTest.php @@ -0,0 +1,24 @@ +assertEquals( + 'a:b', + $filter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SelectQueryBuilderTest.php new file mode 100644 index 0000000..6c5257a --- /dev/null +++ b/test/Dto/Search/Query/SelectQueryBuilderTest.php @@ -0,0 +1,150 @@ +builder = new SelectQueryBuilder(); + $this->builder->index('myindex'); + } + + public function testBuildWithoutIndex(): void + { + $this->expectException(InvalidArgumentException::class); + $builder = new SelectQueryBuilder(); + $builder->build(); + } + + public function testSetEmptyIndex(): void + { + $this->expectException(InvalidArgumentException::class); + $builder = new SelectQueryBuilder(); + $builder->index(''); + } + + public function testSetIndex(): void + { + $builder = new SelectQueryBuilder(); + $builder->index('myindex'); + $query = $builder->build(); + + $this->assertEquals( + 'myindex', + $query->index, + 'unexpected index' + ); + } + + public function testSetText(): void + { + $this->builder->text('abc'); + $query = $this->builder->build(); + $this->assertEquals('abc', $query->text, 'unexpected text'); + } + + public function testSetOffset(): void + { + $this->builder->offset(10); + $query = $this->builder->build(); + $this->assertEquals(10, $query->offset, 'unexpected offset'); + } + + public function testSetInvalidOffset(): void + { + $this->expectException(InvalidArgumentException::class); + $this->builder->offset(-1); + } + + public function testSetLimit(): void + { + $this->builder->limit(10); + $query = $this->builder->build(); + $this->assertEquals(10, $query->limit, 'unexpected limit'); + } + + public function testSetInvalidLimit(): void + { + $this->expectException(InvalidArgumentException::class); + $this->builder->limit(-1); + } + + public function testSetSort(): void + { + $criteria = $this->createStub(Criteria::class); + + $this->builder->sort($criteria); + $query = $this->builder->build(); + $this->assertEquals([$criteria], $query->sort, 'unexpected sort'); + } + + public function testSetFilter(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + $this->builder->filter($filter); + $query = $this->builder->build(); + $this->assertEquals([$filter], $query->filter, 'unexpected filter'); + } + + public function testSetTwoFilterWithSameKey(): void + { + $filterA = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + $filterB = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + + $this->expectException(InvalidArgumentException::class); + $this->builder->filter($filterA, $filterB); + } + + public function testSetFacet(): void + { + $facet = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + $this->builder->facet($facet); + $query = $this->builder->build(); + $this->assertEquals([$facet], $query->facets, 'unexpected facets'); + } + + public function testSetTwoFacetSWithSameKey(): void + { + $facetA = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + $facetB = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + + $this->expectException(InvalidArgumentException::class); + $this->builder->facet($facetA, $facetB); + } + + public function testSetQueryDefaultOperator(): void + { + $this->builder->queryDefaultOperator(QueryDefaultOperator::AND); + $query = $this->builder->build(); + $this->assertEquals( + QueryDefaultOperator::AND, + $query->queryDefaultOperator, + 'unexpected queryDefaultOperator' + ); + } +} diff --git a/test/Service/Indexer/ContentCollectorTest.php b/test/Service/Indexer/ContentCollectorTest.php index 9a47089..82a804d 100644 --- a/test/Service/Indexer/ContentCollectorTest.php +++ b/test/Service/Indexer/ContentCollectorTest.php @@ -6,8 +6,10 @@ use Atoolo\Search\Service\Indexer\ContentCollector; use Atoolo\Search\Service\Indexer\SiteKit\ContentMatcher; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(ContentCollector::class)] class ContentCollectorTest extends TestCase { public function testCollect(): void diff --git a/test/Service/Indexer/IndexSchema2xDocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php index e6f0f2a..a5a5282 100644 --- a/test/Service/Indexer/IndexSchema2xDocumentTest.php +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -5,8 +5,10 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(IndexSchema2xDocument::class)] class IndexSchema2xDocumentTest extends TestCase { public function testGetFields(): void diff --git a/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php index 6e1632c..1c23d99 100644 --- a/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php +++ b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php @@ -5,8 +5,10 @@ namespace Atoolo\Search\Test\Service\Indexer\SiteKit; use Atoolo\Search\Service\Indexer\SiteKit\HeadlineMatcher; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(HeadlineMatcher::class)] class HeadlineMatcherTest extends TestCase { public function testMatcher(): void @@ -21,4 +23,36 @@ public function testMatcher(): void $this->assertEquals('Überschrift', $content, 'unexpected headline'); } + + public function testMatcherNotMachedPathToShort(): void + { + $matcher = new HeadlineMatcher(); + + $value = [ + "headline" => "Überschrift" + ]; + + $content = $matcher->match(['model'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNotMachedNoModel(): void + { + $matcher = new HeadlineMatcher(); + + $value = [ + "headline" => "Überschrift" + ]; + + $content = $matcher->match(['items', 'modelX'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } } diff --git a/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php index b6882cf..a3d50a3 100644 --- a/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php +++ b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php @@ -5,8 +5,10 @@ namespace Atoolo\Search\Test\Service\Indexer\SiteKit; use Atoolo\Search\Service\Indexer\SiteKit\QuoteSectionMatcher; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(QuoteSectionMatcher::class)] class QuoteSectionMatcherTest extends TestCase { public function testMatcher(): void @@ -29,4 +31,83 @@ public function testMatcher(): void 'unexpected quote text' ); } + public function testMatcherNoMatchPathToShort(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchNoItems(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['itemsX'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchInvalidType(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quoteX", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchMissingModel(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "modelX" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } } diff --git a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php index 5d98bf9..16bbcc5 100644 --- a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php +++ b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php @@ -5,8 +5,10 @@ namespace Atoolo\Search\Test\Service\Indexer\SiteKit; use Atoolo\Search\Service\Indexer\SiteKit\RichtTextMatcher; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(RichtTextMatcher::class)] class RichtTextMatcherTest extends TestCase { public function testMatcher(): void @@ -23,4 +25,41 @@ public function testMatcher(): void $this->assertEquals('Ein Text', $content, 'unexpected content'); } + + public function testMatcherNotMatchedInvalidType(): void + { + $matcher = new RichtTextMatcher(); + + $value = [ + "normalized" => true, + "modelType" => "html.richTextX", + "text" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNotMatchedTextMissing(): void + { + $matcher = new RichtTextMatcher(); + + $value = [ + "normalized" => true, + "modelType" => "html.richText", + "textX" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + } diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/SolrIndexerTest.php index 36f3c30..80c9c3c 100644 --- a/test/Service/Indexer/SolrIndexerTest.php +++ b/test/Service/Indexer/SolrIndexerTest.php @@ -17,6 +17,7 @@ use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; +use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use Solarium\Client; use Solarium\QueryType\Server\CoreAdmin\Result\Result as CoreAdminResult; @@ -27,7 +28,7 @@ #[CoversClass(SolrIndexer::class)] class SolrIndexerTest extends TestCase { - private ResourceLoader $resourceLoader; + private ResourceLoader&Stub $resourceLoader; private IndexerProgressHandler $indexerProgressHandler; @@ -41,7 +42,7 @@ class SolrIndexerTest extends TestCase private UpdateResult $updateResult; - private IndexingAborter $aborter; + private IndexingAborter&Stub $aborter; private DocumentEnricher $documentEnricher; @@ -55,7 +56,7 @@ public function setUp(): void $this->indexerProgressHandler = $this->createMock( IndexerProgressHandler::class ); - $this->finder = $this->createStub(LocationFinder::class); + $this->finder = $this->createMock(LocationFinder::class); $this->documentEnricher = $this->createMock(DocumentEnricher::class); $this->translationSplitter = new SubDirTranslationSplitter(); $this->resourceLoader = $this->createStub(ResourceLoader::class); @@ -348,4 +349,58 @@ public function testIndexPaths(): void $this->indexer->index($parameter); } + + public function testIndexPathWithParameter(): void + { + $this->finder->expects($this->once()) + ->method('findPaths') + ->with($this->equalTo(['?a=b'])); + + $parameter = new IndexerParameter( + 'test', + 10, + 10, + [ + '?a=b' + ] + ); + + $this->indexer->index($parameter); + } + + public function testIndexPathWithParameterAndPath(): void + { + $this->finder->expects($this->once()) + ->method('findPaths') + ->with($this->equalTo(['/test.php'])); + + $parameter = new IndexerParameter( + 'test', + 10, + 10, + [ + '/test.php?a=b' + ] + ); + + $this->indexer->index($parameter); + } + + public function testIndexPathWithLocParameterAndPath(): void + { + $this->finder->expects($this->once()) + ->method('findPaths') + ->with($this->equalTo(['/test.php.translations/en.php'])); + + $parameter = new IndexerParameter( + 'test', + 10, + 10, + [ + '/test.php?loc=en' + ] + ); + + $this->indexer->index($parameter); + } } From 0b441b584b3c8e72dad4336ca69e5498a015638a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 23 Feb 2024 16:19:20 +0100 Subject: [PATCH 044/145] test: add more tests --- src/Dto/Search/Query/Sort/Criteria.php | 2 +- .../Search/InternalMediaResourceFactory.php | 14 +- .../Search/InternalResourceFactory.php | 11 +- src/Service/Search/SolrMoreLikeThis.php | 4 +- src/Service/Search/SolrSelect.php | 1 + src/Service/Search/SolrSuggest.php | 8 +- .../Search/Query/SelectQueryBuilderTest.php | 2 + test/Service/Indexer/SolrIndexerTest.php | 72 +++- .../Search/ExternalResourceFactoryTest.php | 93 +++++ .../InternalMediaResourceFactoryTest.php | 99 +++++ .../Search/InternalResourceFactoryTest.php | 91 +++++ .../SiteKit/DefaultBoostModifierTest.php | 22 ++ test/Service/Search/SolrMoreLikeThisTest.php | 80 ++++ .../SolrResultToResourceResolverTest.php | 88 +++++ test/Service/Search/SolrSelectTest.php | 362 ++++++++++++++++++ test/Service/Search/SolrSuggestTest.php | 131 +++++++ .../SolrParameterClientFactoryTest.php | 24 ++ 17 files changed, 1083 insertions(+), 21 deletions(-) create mode 100644 test/Service/Search/ExternalResourceFactoryTest.php create mode 100644 test/Service/Search/InternalMediaResourceFactoryTest.php create mode 100644 test/Service/Search/InternalResourceFactoryTest.php create mode 100644 test/Service/Search/SiteKit/DefaultBoostModifierTest.php create mode 100644 test/Service/Search/SolrMoreLikeThisTest.php create mode 100644 test/Service/Search/SolrResultToResourceResolverTest.php create mode 100644 test/Service/Search/SolrSelectTest.php create mode 100644 test/Service/Search/SolrSuggestTest.php create mode 100644 test/Service/SolrParameterClientFactoryTest.php diff --git a/src/Dto/Search/Query/Sort/Criteria.php b/src/Dto/Search/Query/Sort/Criteria.php index e73e92d..54ac7af 100644 --- a/src/Dto/Search/Query/Sort/Criteria.php +++ b/src/Dto/Search/Query/Sort/Criteria.php @@ -10,7 +10,7 @@ abstract class Criteria { public function __construct( - public readonly Direction $direction + public readonly Direction $direction = Direction::ASC ) { } } diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php index bdf7a2f..f5b62f4 100644 --- a/src/Service/Search/InternalMediaResourceFactory.php +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; +use LogicException; use Solarium\QueryType\Select\Result\Document; /** @@ -25,18 +26,27 @@ public function __construct( public function accept(Document $document): bool { $metaLocation = $this->getMetaLocation($document); + if ($metaLocation === null) { + return false; + } return $this->resourceLoader->exists($metaLocation); } public function create(Document $document): Resource { $metaLocation = $this->getMetaLocation($document); + if ($metaLocation === null) { + throw new LogicException('document should contains a url'); + } return $this->resourceLoader->load($metaLocation); } - private function getMetaLocation(Document $document): string + private function getMetaLocation(Document $document): ?string { - $location = $this->getField($document, 'url') ?? ''; + $location = $this->getField($document, 'url'); + if ($location === null) { + return null; + } return $location . '.meta.php'; } diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php index 0568e7e..30620f5 100644 --- a/src/Service/Search/InternalResourceFactory.php +++ b/src/Service/Search/InternalResourceFactory.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; +use LogicException; use Solarium\QueryType\Select\Result\Document; /** @@ -22,13 +23,19 @@ public function __construct( public function accept(Document $document): bool { - $location = $this->getField($document, 'url') ?? ''; + $location = $this->getField($document, 'url'); + if ($location === null) { + return false; + } return str_ends_with($location, '.php'); } public function create(Document $document): Resource { - $location = $this->getField($document, 'url') ?? ''; + $location = $this->getField($document, 'url'); + if ($location === null) { + throw new LogicException('document should contains a url'); + } return $this->resourceLoader->load($location); } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index de553e5..2036517 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -49,9 +49,9 @@ private function buildSolrQuery( // Filter foreach ($query->filter as $filter) { - $solrQuery->createFilterQuery($filter->getKey()) + $solrQuery->createFilterQuery($filter->key) ->setQuery($filter->getQuery()) - ->setTags($filter->getTags()); + ->setTags($filter->tags); } return $solrQuery; diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 6787a04..87d979e 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -171,6 +171,7 @@ private function addFilterQueriesToSolrQuery( foreach ($filterList as $filter) { $key = $filter->key ?? uniqid('', true); + $fq = $solrQuery->createFilterQuery($key); $solrQuery->createFilterQuery($key) ->setQuery($filter->getQuery()) ->setTags($filter->tags); diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 960ee49..9a829ef 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -8,7 +8,7 @@ use Atoolo\Search\Dto\Search\Result\Suggestion; use Atoolo\Search\Dto\Search\Result\SuggestResult; use Atoolo\Search\Exception\UnexpectedResultException; -use Atoolo\Search\Service\SolrParameterClientFactory; +use Atoolo\Search\Service\SolrClientFactory; use Atoolo\Search\SuggestSearcher; use JsonException; use Solarium\Core\Client\Client; @@ -29,7 +29,7 @@ class SolrSuggest implements SuggestSearcher private const INDEX_SUGGEST_FIELD = 'raw_content'; public function __construct( - private readonly SolrParameterClientFactory $clientFactory + private readonly SolrClientFactory $clientFactory ) { } @@ -75,9 +75,9 @@ private function buildSolrQuery( // Filter foreach ($query->filter as $filter) { - $solrQuery->createFilterQuery($filter->getKey()) + $solrQuery->createFilterQuery($filter->key) ->setQuery($filter->getQuery()) - ->setTags($filter->getTags()); + ->setTags($filter->tags); } return $solrQuery; diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SelectQueryBuilderTest.php index 6c5257a..65ab23f 100644 --- a/test/Dto/Search/Query/SelectQueryBuilderTest.php +++ b/test/Dto/Search/Query/SelectQueryBuilderTest.php @@ -10,8 +10,10 @@ use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use InvalidArgumentException; +use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; +#[CoversClass(SelectQueryBuilder::class)] class SelectQueryBuilderTest extends TestCase { private SelectQueryBuilder $builder; diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/SolrIndexerTest.php index 80c9c3c..b87e89c 100644 --- a/test/Service/Indexer/SolrIndexerTest.php +++ b/test/Service/Indexer/SolrIndexerTest.php @@ -17,6 +17,7 @@ use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; +use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use Solarium\Client; @@ -28,9 +29,11 @@ #[CoversClass(SolrIndexer::class)] class SolrIndexerTest extends TestCase { + private array $availableIndexes = ['test', 'test-en_US']; + private ResourceLoader&Stub $resourceLoader; - private IndexerProgressHandler $indexerProgressHandler; + private IndexerProgressHandler&MockObject $indexerProgressHandler; private SolrIndexer $indexer; @@ -44,7 +47,7 @@ class SolrIndexerTest extends TestCase private IndexingAborter&Stub $aborter; - private DocumentEnricher $documentEnricher; + private DocumentEnricher&MockObject $documentEnricher; private TranslationSplitter $translationSplitter; @@ -74,11 +77,17 @@ public function setUp(): void $this->updateQuery->method('createDocument') ->willReturn($this->createStub(IndexSchema2xDocument::class)); $coreAdminResult = $this->createStub(CoreAdminResult::class); - $coreAdminResultStatus = $this->createStub(StatusResult::class); - $coreAdminResultStatus->method('getCoreName') - ->willReturn('test'); $coreAdminResult->method('getStatusResults') - ->willReturn([$coreAdminResultStatus]); + ->willReturnCallback(function () { + $results = []; + foreach($this->availableIndexes as $index) { + $result = $this->createStub(StatusResult::class); + $result->method('getCoreName') + ->willReturn($index); + $results[] = $result; + } + return $results; + }); $this->solrClient->method('createUpdate') ->willReturn($this->updateQuery); $this->solrClient->method('update') @@ -131,7 +140,9 @@ public function testIndexAllWithChunks(): void $this->finder->method('findAll') ->willReturn([ '/a/b.php', + '/a/b.php.translations/en_US.php', '/a/c.php', + '/a/c.php.translations/fr_FR.php', '/a/d.php', '/a/e.php', '/a/f.php', @@ -140,7 +151,8 @@ public function testIndexAllWithChunks(): void '/a/i.php', '/a/j.php', '/a/k.php', - '/a/l.php' + '/a/l.php', + '/a/error.php' ]); $this->updateResult->method('getStatus') @@ -149,11 +161,17 @@ public function testIndexAllWithChunks(): void $this->documentEnricher->method('isIndexable') ->willReturn(true); - $this->documentEnricher->expects($this->exactly(11)) - ->method('enrichDocument'); + $this->documentEnricher + ->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + if ($resource->getLocation() === '/a/error.php') { + throw new \Exception('test'); + } + return $doc; + }); $addDocumentsCalls = 0; - $this->updateQuery->expects($this->exactly(2)) + $this->updateQuery->expects($this->exactly(3)) ->method('addDocuments') ->willReturnCallback( function ($documents) use (&$addDocumentsCalls) { @@ -181,6 +199,9 @@ function ($documents) use (&$addDocumentsCalls) { 10 ); + $this->indexerProgressHandler->expects($this->exactly(2)) + ->method('error'); + $this->indexer->index($parameter); } @@ -403,4 +424,35 @@ public function testIndexPathWithLocParameterAndPath(): void $this->indexer->index($parameter); } + + public function testWithoutAvailableIndexes(): void + { + + $this->availableIndexes = []; + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php', + '/a/d.php', + '/a/e.php', + '/a/f.php', + '/a/g.php', + '/a/h.php', + '/a/i.php', + '/a/j.php', + '/a/k.php', + '/a/l.php' + ]); + + $parameter = new IndexerParameter( + 'test', + 10, + 10 + ); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $this->indexer->index($parameter); + } } diff --git a/test/Service/Search/ExternalResourceFactoryTest.php b/test/Service/Search/ExternalResourceFactoryTest.php new file mode 100644 index 0000000..4f37a52 --- /dev/null +++ b/test/Service/Search/ExternalResourceFactoryTest.php @@ -0,0 +1,93 @@ +factory = new ExternalResourceFactory(); + } + + public function testAcceptHttps(): void + { + $document = $this->createDocument('https://www.sitepark.com'); + $this->assertTrue( + $this->factory->accept($document), + 'should be accepted' + ); + } + + public function testAcceptHttp(): void + { + $document = $this->createDocument('http://www.sitepark.com'); + $this->assertTrue( + $this->factory->accept($document), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document), + 'should be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('https://www.sitepark.com'); + $resource = $this->factory->create($document); + + $this->assertEquals( + 'https://www.sitepark.com', + $resource->getLocation(), + 'unexpected location' + ); + } + + public function testCreateWithName(): void + { + $document = $this->createDocument('https://www.sitepark.com', 'Test'); + $resource = $this->factory->create($document); + + $this->assertEquals( + 'Test', + $resource->getName(), + 'unexpected name' + ); + } + + public function testCreateWithMissingUrl(): void + { + $document = $this->createStub(Document::class); + + $this->expectException(\LogicException::class); + $this->factory->create($document); + } + + private function createDocument(string $url, string $title = ''): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url, + 'title' => $title + ]); + return $document; + } +} diff --git a/test/Service/Search/InternalMediaResourceFactoryTest.php b/test/Service/Search/InternalMediaResourceFactoryTest.php new file mode 100644 index 0000000..12d5f73 --- /dev/null +++ b/test/Service/Search/InternalMediaResourceFactoryTest.php @@ -0,0 +1,99 @@ +resourceLoader = $this->createStub( + ResourceLoader::class + ); + $this->factory = new InternalMediaResourceFactory( + $this->resourceLoader + ); + } + + public function testAccept(): void + { + $document = $this->createDocument('/image.jpg'); + $this->resourceLoader + ->method('exists') + ->willReturn(true); + + $this->assertTrue( + $this->factory->accept($document), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document), + 'should not be accepted' + ); + } + + public function testAcceptNotExists(): void + { + $document = $this->createDocument('/image.jpg'); + $this->resourceLoader + ->method('exists') + ->willReturn(false); + + $this->assertFalse( + $this->factory->accept($document), + 'should not be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('/image.jpg'); + $resource = $this->createStub(Resource::class); + $this->resourceLoader + ->method('load') + ->willReturn($resource); + + $this->assertEquals( + $resource, + $this->factory->create($document), + 'unexpected resource' + ); + } + + public function testCreateWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->expectException(LogicException::class); + $this->factory->create($document); + } + + private function createDocument(string $url): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url + ]); + return $document; + } +} diff --git a/test/Service/Search/InternalResourceFactoryTest.php b/test/Service/Search/InternalResourceFactoryTest.php new file mode 100644 index 0000000..0d83fc5 --- /dev/null +++ b/test/Service/Search/InternalResourceFactoryTest.php @@ -0,0 +1,91 @@ +resourceLoader = $this->createStub( + ResourceLoader::class + ); + $this->factory = new InternalResourceFactory( + $this->resourceLoader + ); + } + + public function testAccept(): void + { + $document = $this->createDocument('/test.php'); + $this->assertTrue( + $this->factory->accept($document), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document), + 'should not be accepted' + ); + } + + public function testAcceptWithWrongUrl(): void + { + $document = $this->createDocument('/test.txt'); + $this->assertFalse( + $this->factory->accept($document), + 'should not be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('/test.php'); + $resource = $this->createStub(Resource::class); + $this->resourceLoader + ->method('load') + ->willReturn($resource); + + $this->assertEquals( + $resource, + $this->factory->create($document), + 'unexpected resource' + ); + } + + public function testCreateWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->expectException(LogicException::class); + $this->factory->create($document); + } + + private function createDocument(string $url): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url + ]); + return $document; + } +} diff --git a/test/Service/Search/SiteKit/DefaultBoostModifierTest.php b/test/Service/Search/SiteKit/DefaultBoostModifierTest.php new file mode 100644 index 0000000..0a08579 --- /dev/null +++ b/test/Service/Search/SiteKit/DefaultBoostModifierTest.php @@ -0,0 +1,22 @@ +modify($query); + + $this->assertNotNull($modifiedQuery->getDisMax()->getBoostQueries()); + } +} diff --git a/test/Service/Search/SolrMoreLikeThisTest.php b/test/Service/Search/SolrMoreLikeThisTest.php new file mode 100644 index 0000000..01a27f3 --- /dev/null +++ b/test/Service/Search/SolrMoreLikeThisTest.php @@ -0,0 +1,80 @@ +createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrMoreLikeThisQuery::class); + $filterQuery = new FilterQuery(); + $query->method('createFilterQuery')->willReturn($filterQuery); + + $client->method('createMoreLikeThis')->willReturn($query); + + $result = $this->createStub(SelectResult::class); + $client->method('execute')->willReturn($result); + + $this->resource = $this->createStub(Resource::class); + + $resultToResourceResolver = $this->createStub( + SolrResultToResourceResolver::class + ); + $resultToResourceResolver + ->method('loadResourceList') + ->willReturn([$this->resource]); + + $this->searcher = new SolrMoreLikeThis( + $clientFactory, + $resultToResourceResolver + ); + } + + public function testMoreLikeThis(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new MoreLikeThisQuery( + 'myindex', + '/test.php', + [$filter] + ); + + $searchResult = $this->searcher->moreLikeThis($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } +} diff --git a/test/Service/Search/SolrResultToResourceResolverTest.php b/test/Service/Search/SolrResultToResourceResolverTest.php new file mode 100644 index 0000000..efe1a31 --- /dev/null +++ b/test/Service/Search/SolrResultToResourceResolverTest.php @@ -0,0 +1,88 @@ +createStub(Document::class); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resource = $this->createStub(Resource::class); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(true); + $resourceFactory->method('create')->willReturn($resource); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList($result); + + $this->assertEquals( + [$resource], + $resourceList, + 'unexpected resourceList' + ); + } + + public function testLoadResourceListWithoutAcceptedFactory(): void + { + $document = $this->createStub(Document::class); + $document->method('getFields')->willReturn(['url' => 'test']); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(false); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList($result); + + $this->assertEmpty( + $resourceList, + 'resourceList should be empty' + ); + } + + public function testLoadResourceListWithoutAcceptedFactoryNoUrl(): void + { + $document = $this->createStub(Document::class); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(false); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList($result); + + $this->assertEmpty( + $resourceList, + 'resourceList should be empty' + ); + } + +} diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSelectTest.php new file mode 100644 index 0000000..b8e3e96 --- /dev/null +++ b/test/Service/Search/SolrSelectTest.php @@ -0,0 +1,362 @@ +createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrSelectQuery::class); + + $query->method('createFilterQuery') + ->willReturn(new FilterQuery()); + $query->method('getFacetSet') + ->willReturn(new FacetSet()); + + $client->method('createSelect')->willReturn($query); + + $this->result = $this->createStub(SelectResult::class); + $client->method('execute')->willReturn($this->result); + + $this->resource = $this->createStub(Resource::class); + + $resultToResourceResolver = $this->createStub( + SolrResultToResourceResolver::class + ); + + $solrQueryModifier = $this->createStub(SolrQueryModifier::class); + $solrQueryModifier->method('modify')->willReturn($query); + + $resultToResourceResolver + ->method('loadResourceList') + ->willReturn([$this->resource]); + + $this->searcher = new SolrSelect( + $clientFactory, + [$solrQueryModifier], + $resultToResourceResolver + ); + } + + public function testSelectEmpty(): void + { + $query = new SelectQuery( + 'myindex', + '', + 0, + 1, + [ + ], + [], + [], + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithText(): void + { + $query = new SelectQuery( + 'myindex', + 'cat dog', + 0, + 10, + [ + ], + [], + [], + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithSort(): void + { + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [ + new Name(), + new Headline(), + new Date(), + new Natural(), + new Score(), + ], + [], + [], + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithInvalidSort(): void + { + $sort = $this->createStub(Criteria::class); + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [$sort], + [], + [], + QueryDefaultOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->select($query); + } + + public function testSelectWithAndDefaultOperator(): void + { + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [], + [], + QueryDefaultOperator::AND + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithFilter(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [$filter], + [], + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithFacets(): void + { + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [], + $facets, + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithInvalidFacets(): void + { + + $facets = [ + $this->createStub(Facet::class) + ]; + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [], + $facets, + QueryDefaultOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->select($query); + } + + public function testResultFacets(): void + { + + $facet = new Field([ + 'content' => 10, + 'media' => 5 + ]); + $facetSet = new \Solarium\Component\Result\FacetSet([ + 'objectType' => $facet + ]); + + $this->result->method('getFacetSet') + ->willReturn($facetSet); + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [], + $facets, + QueryDefaultOperator::OR + ); + + $searchResult = $this->searcher->select($query); + + $this->assertEquals( + 'objectType', + $searchResult->facetGroups[0]->key, + 'unexpected results' + ); + } + + public function testInvalidResultFacets(): void + { + + $facet = new Field([ + 'content' => 'nonint', + ]); + $facetSet = new \Solarium\Component\Result\FacetSet([ + 'objectType' => $facet + ]); + + $this->result->method('getFacetSet') + ->willReturn($facetSet); + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SelectQuery( + 'myindex', + '', + 0, + 10, + [], + [], + $facets, + QueryDefaultOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->select($query); + } +} diff --git a/test/Service/Search/SolrSuggestTest.php b/test/Service/Search/SolrSuggestTest.php new file mode 100644 index 0000000..3eef755 --- /dev/null +++ b/test/Service/Search/SolrSuggestTest.php @@ -0,0 +1,131 @@ +createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrSelectQuery::class); + + $query->method('createFilterQuery') + ->willReturn(new FilterQuery()); + + $client->method('createSelect')->willReturn($query); + + $this->result = $this->createStub(SelectResult::class); + $client->method('select')->willReturn($this->result); + + $this->searcher = new SolrSuggest($clientFactory); + } + + public function testSuggest(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new SuggestQuery( + 'myindex', + 'cat', + [$filter] + ); + + $response = new Response(<<result->method('getResponse')->willReturn($response); + + $suggestResult = $this->searcher->suggest($query); + + $expected = [ + new Suggestion('category', 10), + new Suggestion('catalog', 5), + ]; + + $this->assertEquals( + $expected, + $suggestResult->suggestions, + 'unexpected suggestion' + ); + } + + public function testEmptySuggest(): void + { + $query = new SuggestQuery( + 'myindex', + 'cat', + ); + + $response = new Response(<<result->method('getResponse')->willReturn($response); + + $suggestResult = $this->searcher->suggest($query); + + $this->assertEmpty( + $suggestResult->suggestions, + 'suggestion should be empty' + ); + } + + public function testInvalidSuggestResponse(): void + { + $query = new SuggestQuery( + 'myindex', + 'cat', + ); + + $response = new Response("none json"); + + $this->result->method('getResponse')->willReturn($response); + + $this->expectException(UnexpectedResultException::class); + $this->searcher->suggest($query); + } +} diff --git a/test/Service/SolrParameterClientFactoryTest.php b/test/Service/SolrParameterClientFactoryTest.php new file mode 100644 index 0000000..49cd0af --- /dev/null +++ b/test/Service/SolrParameterClientFactoryTest.php @@ -0,0 +1,24 @@ +create('myindex'); + $this->assertNotNull($client, 'client instance expected'); + } +} From 586301a96ea6ce0d08b559572ff166a6a8f860a2 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 26 Feb 2024 17:45:53 +0100 Subject: [PATCH 045/145] test: add more tests --- src/Console/Command/DumpIndexDocument.php | 43 +- .../Command/IndexDocumentDumperBuilder.php | 51 ++ src/Console/Command/Io/IndexerProgressBar.php | 1 + src/Console/Command/MoreLikeThis.php | 69 +- src/Console/Command/Search.php | 89 +- .../Command/SolrMoreLikeThisBuilder.php | 64 ++ src/Console/Command/SolrSelectBuilder.php | 69 ++ src/Console/Command/SolrSuggestBuilder.php | 35 + src/Console/Command/Suggest.php | 66 +- src/Service/Indexer/BackgroundIndexer.php | 33 +- .../BackgroundIndexerProgressState.php | 8 - src/Service/Indexer/IndexDocumentDumper.php | 46 + src/Service/Indexer/IndexingAborter.php | 4 +- .../DefaultSchema2xDocumentEnricher.php | 7 +- src/Service/Indexer/SolrIndexerFactory.php | 40 + .../Console/Command/DumpIndexDocumentTest.php | 72 ++ .../IndexDocumentDumperBuilderTest.php | 23 + .../Io/IndexerProgressBarFactoryTest.php | 21 + .../Command/Io/IndexerProgressBarTest.php | 155 ++++ test/Console/Command/MoreLikeThisTest.php | 84 ++ test/Console/Command/SearchTest.php | 103 +++ .../Command/SolrIndexerBuilderTest.php | 34 + .../Command/SolrMoreLikeThisBuilderTest.php | 24 + .../Console/Command/SolrSelectBuilderTest.php | 23 + .../Command/SolrSuggestBuilderTest.php | 22 + test/Console/Command/SuggestTest.php | 78 ++ .../Dto/Search/Query/Filter/AndFilterTest.php | 2 +- .../Search/Query/Filter/ArchiveFilterTest.php | 2 +- test/Dto/Search/Query/Filter/OrFilterTest.php | 1 + .../Search/Query/Filter/QueryFilterTest.php | 1 - .../DocumentEnrichingExceptionTest.php | 21 + ...ssMatchingResourceFactoryExceptionTest.php | 21 + .../UnexpectedResultExceptionTest.php | 21 + .../BackgroundIndexerProgressStateTest.php | 150 ++++ .../Service/Indexer/BackgroundIndexerTest.php | 91 ++ test/Service/Indexer/DocumentEnrichter.php | 9 + .../Indexer/IndexDocumentDumperTest.php | 38 + test/Service/Indexer/IndexingAborterTest.php | 61 ++ .../DefaultSchema2xDocumentEnricherTest.php | 794 ++++++++++++++++++ .../Indexer/SiteKit/RichtTextMatcherTest.php | 1 - .../Indexer/SolrIndexerFactoryTest.php | 35 + test/Service/Indexer/SolrIndexerTest.php | 2 +- .../Search/ExternalResourceFactoryTest.php | 1 - .../SolrResultToResourceResolverTest.php | 2 - 44 files changed, 2280 insertions(+), 237 deletions(-) create mode 100644 src/Console/Command/IndexDocumentDumperBuilder.php create mode 100644 src/Console/Command/SolrMoreLikeThisBuilder.php create mode 100644 src/Console/Command/SolrSelectBuilder.php create mode 100644 src/Console/Command/SolrSuggestBuilder.php create mode 100644 src/Service/Indexer/IndexDocumentDumper.php create mode 100644 src/Service/Indexer/SolrIndexerFactory.php create mode 100644 test/Console/Command/DumpIndexDocumentTest.php create mode 100644 test/Console/Command/IndexDocumentDumperBuilderTest.php create mode 100644 test/Console/Command/Io/IndexerProgressBarFactoryTest.php create mode 100644 test/Console/Command/Io/IndexerProgressBarTest.php create mode 100644 test/Console/Command/MoreLikeThisTest.php create mode 100644 test/Console/Command/SearchTest.php create mode 100644 test/Console/Command/SolrIndexerBuilderTest.php create mode 100644 test/Console/Command/SolrMoreLikeThisBuilderTest.php create mode 100644 test/Console/Command/SolrSelectBuilderTest.php create mode 100644 test/Console/Command/SolrSuggestBuilderTest.php create mode 100644 test/Console/Command/SuggestTest.php create mode 100644 test/Exception/DocumentEnrichingExceptionTest.php create mode 100644 test/Exception/MissMatchingResourceFactoryExceptionTest.php create mode 100644 test/Exception/UnexpectedResultExceptionTest.php create mode 100644 test/Service/Indexer/BackgroundIndexerProgressStateTest.php create mode 100644 test/Service/Indexer/BackgroundIndexerTest.php create mode 100644 test/Service/Indexer/DocumentEnrichter.php create mode 100644 test/Service/Indexer/IndexDocumentDumperTest.php create mode 100644 test/Service/Indexer/IndexingAborterTest.php create mode 100644 test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php create mode 100644 test/Service/Indexer/SolrIndexerFactoryTest.php diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php index e57b265..4a28774 100644 --- a/src/Console/Command/DumpIndexDocument.php +++ b/src/Console/Command/DumpIndexDocument.php @@ -4,12 +4,9 @@ namespace Atoolo\Search\Console\Command; -use Atoolo\Resource\Loader\ServerVarResourceBaseLocator; -use Atoolo\Resource\Loader\SiteKitLoader; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; -use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use JsonException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -28,7 +25,8 @@ class DumpIndexDocument extends Command * @param iterable> $documentEnricherList */ public function __construct( - private readonly iterable $documentEnricherList + private readonly iterable $documentEnricherList, + private readonly IndexDocumentDumperBuilder $indexDocumentDumperBuilder ) { parent::__construct(); } @@ -62,38 +60,19 @@ protected function execute( $resourceDir = $typedInput->getStringArgument('resource-dir'); - $subDirectory = null; - if (is_dir($resourceDir . '/objects')) { - $subDirectory = 'objects'; - } - - $_SERVER['RESOURCE_ROOT'] = $resourceDir; - $resourceBaseLocator = new ServerVarResourceBaseLocator( - 'RESOURCE_ROOT', - $subDirectory - ); + $dumper = $this->indexDocumentDumperBuilder + ->resourceDir($resourceDir) + ->documentEnricherList($this->documentEnricherList) + ->build(); $paths = $typedInput->getArrayArgument('paths'); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - - foreach ($paths as $path) { - $resource = $resourceLoader->load($path); - $doc = new IndexSchema2xDocument(); - $processId = 'process-id'; - - foreach ($this->documentEnricherList as $enricher) { - /** @var IndexSchema2xDocument $doc */ - $doc = $enricher->enrichDocument( - $resource, - $doc, - $processId - ); - } + $dump = $dumper->dump($paths); - echo json_encode( - $doc->getFields(), + foreach ($dump as $fields) { + $output->writeln(json_encode( + $fields, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT - ); + )); } return Command::SUCCESS; diff --git a/src/Console/Command/IndexDocumentDumperBuilder.php b/src/Console/Command/IndexDocumentDumperBuilder.php new file mode 100644 index 0000000..36edd70 --- /dev/null +++ b/src/Console/Command/IndexDocumentDumperBuilder.php @@ -0,0 +1,51 @@ +> + */ + private iterable $documentEnricherList; + + public function resourceDir(string $resourceDir): IndexDocumentDumperBuilder + { + $this->resourceDir = $resourceDir; + return $this; + } + + /** + * phpcs:ignore + * @param iterable> $documentEnricherList + */ + public function documentEnricherList( + iterable $documentEnricherList + ): IndexDocumentDumperBuilder { + $this->documentEnricherList = $documentEnricherList; + return $this; + } + + public function build(): IndexDocumentDumper + { + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + + return new IndexDocumentDumper( + $resourceLoader, + $this->documentEnricherList + ); + } +} diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index 2ec81a3..dcc5664 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -58,6 +58,7 @@ public function advance(int $step): void public function skip(int $step): void { + $this->status->skipped++; } private function formatProgressBar(string $color): void diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 7e07ca2..bed1aa9 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -4,17 +4,10 @@ namespace Atoolo\Search\Console\Command; -use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; +use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\Service\Search\ExternalResourceFactory; -use Atoolo\Search\Service\Search\InternalMediaResourceFactory; -use Atoolo\Search\Service\Search\InternalResourceFactory; use Atoolo\Search\Service\Search\SolrMoreLikeThis; -use Atoolo\Search\Service\Search\SolrResultToResourceResolver; -use Atoolo\Search\Service\SolrParameterClientFactory; -use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -29,9 +22,14 @@ class MoreLikeThis extends Command { private SymfonyStyle $io; - private InputInterface $input; + private TypifiedInput $input; private string $solrCore; - private string $resourceDir; + + public function __construct( + private readonly SolrMoreLikeThisBuilder $solrMoreLikeThisBuilder + ) { + parent::__construct(); + } protected function configure(): void { @@ -66,12 +64,11 @@ protected function execute( OutputInterface $output ): int { - $this->input = $input; + $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $this->getStringArgument('solr-core'); - $this->resourceDir = $this->getStringArgument('resource-dir'); - $location = $this->getStringArgument('location'); + $this->solrCore = $this->input->getStringArgument('solr-core'); + $location = $this->input->getStringArgument('location'); $searcher = $this->createSearcher(); $query = $this->buildQuery($location); @@ -81,46 +78,16 @@ protected function execute( return Command::SUCCESS; } - private function getStringArgument(string $name): string - { - $value = $this->input->getArgument($name); - if (!is_string($value)) { - throw new InvalidArgumentException( - $name . ' must be a string' - ); - } - return $value; - } - protected function createSearcher(): SolrMoreLikeThis { - $resourceBaseLocator = new StaticResourceBaseLocator( - $this->resourceDir + $this->solrMoreLikeThisBuilder->solrConnectionUrl( + $this->input->getStringArgument('solr-connection-url') ); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - /** @var string[] */ - $url = parse_url($this->getStringArgument('solr-connection-url')); - $clientFactory = new SolrParameterClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 - ); - $resourceFactoryList = [ - new ExternalResourceFactory(), - new InternalResourceFactory($resourceLoader), - new InternalMediaResourceFactory($resourceLoader) - ]; - $solrResultToResourceResolver = new SolrResultToResourceResolver( - $resourceFactoryList + $this->solrMoreLikeThisBuilder->resourceDir( + $this->input->getStringArgument('resource-dir') ); - return new SolrMoreLikeThis( - $clientFactory, - $solrResultToResourceResolver - ); + return $this->solrMoreLikeThisBuilder->build(); } protected function buildQuery(string $location): MoreLikeThisQuery @@ -137,10 +104,10 @@ protected function buildQuery(string $location): MoreLikeThisQuery protected function outputResult(SearchResult $result): void { - $this->io->text($result->getTotal() . " Results:"); + $this->io->text($result->total . " Results:"); foreach ($result as $resource) { $this->io->text($resource->getLocation()); } - $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); } } diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 54302a5..ba4aa71 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -4,19 +4,11 @@ namespace Atoolo\Search\Console\Command; -use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; +use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\Service\Search\ExternalResourceFactory; -use Atoolo\Search\Service\Search\InternalMediaResourceFactory; -use Atoolo\Search\Service\Search\InternalResourceFactory; -use Atoolo\Search\Service\Search\SiteKit\DefaultBoostModifier; -use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\Search\SolrSelect; -use Atoolo\Search\Service\SolrParameterClientFactory; -use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -31,15 +23,24 @@ class Search extends Command { private SymfonyStyle $io; - private InputInterface $input; + private TypifiedInput $input; private string $index; - private string $resourceDir; + public function __construct( + private readonly SolrSelectBuilder $solrSelectBuilder + ) { + parent::__construct(); + } protected function configure(): void { $this ->setHelp('Command to performs a search') + ->addArgument( + 'solr-connection-url', + InputArgument::REQUIRED, + 'Solr connection url.' + ) ->addArgument( 'index', InputArgument::REQUIRED, @@ -63,10 +64,9 @@ protected function execute( OutputInterface $output ): int { - $this->input = $input; + $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->resourceDir = $this->getStringArgument('resource-dir'); - $this->index = $this->getStringArgument('index'); + $this->index = $this->input->getStringArgument('index'); $searcher = $this->createSearch(); $query = $this->buildQuery($input); @@ -78,50 +78,15 @@ protected function execute( return Command::SUCCESS; } - private function getStringArgument(string $name): string - { - $value = $this->input->getArgument($name); - if (!is_string($value)) { - throw new InvalidArgumentException( - $name . ' must be a string' - ); - } - return $value; - } - protected function createSearch(): SolrSelect { - $resourceBaseLocator = new StaticResourceBaseLocator( - $this->resourceDir - ); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - /** @var string[] */ - $url = parse_url($this->getStringArgument('solr-connection-url')); - $clientFactory = new SolrParameterClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 - ); - $defaultBoosting = new DefaultBoostModifier(); - - $resourceFactoryList = [ - new ExternalResourceFactory(), - new InternalResourceFactory($resourceLoader), - new InternalMediaResourceFactory($resourceLoader) - ]; - - $solrResultToResourceResolver = new SolrResultToResourceResolver( - $resourceFactoryList + $this->solrSelectBuilder->resourceDir( + $this->input->getStringArgument('resource-dir') ); - - return new SolrSelect( - $clientFactory, - [$defaultBoosting], - $solrResultToResourceResolver + $this->solrSelectBuilder->solrConnectionUrl( + $this->input->getStringArgument('solr-connection-url') ); + return $this->solrSelectBuilder->build(); } protected function buildQuery(InputInterface $input): SelectQuery @@ -144,25 +109,25 @@ protected function buildQuery(InputInterface $input): SelectQuery protected function outputResult( SearchResult $result ): void { - $this->io->title('Results (' . $result->getTotal() . ')'); + $this->io->title('Results (' . $result->total . ')'); foreach ($result as $resource) { $this->io->text($resource->getLocation()); } - if (count($result->getFacetGroups()) > 0) { + if (count($result->facetGroups) > 0) { $this->io->title('Facets'); - foreach ($result->getFacetGroups() as $facetGroup) { - $this->io->section($facetGroup->getKey()); + foreach ($result->facetGroups as $facetGroup) { + $this->io->section($facetGroup->key); $listing = []; - foreach ($facetGroup->getFacets() as $facet) { + foreach ($facetGroup->facets as $facet) { $listing[] = - $facet->getKey() . - ' (' . $facet->getHits() . ')'; + $facet->key . + ' (' . $facet->hits . ')'; } $this->io->listing($listing); } } - $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); } } diff --git a/src/Console/Command/SolrMoreLikeThisBuilder.php b/src/Console/Command/SolrMoreLikeThisBuilder.php new file mode 100644 index 0000000..8f72a7d --- /dev/null +++ b/src/Console/Command/SolrMoreLikeThisBuilder.php @@ -0,0 +1,64 @@ +resourceDir = $resourceDir; + return $this; + } + + public function solrConnectionUrl( + string $solrConnectionUrl + ): SolrMoreLikeThisBuilder { + $this->solrConnectionUrl = $solrConnectionUrl; + return $this; + } + + public function build(): SolrMoreLikeThis + { + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + /** @var string[] */ + $url = parse_url($this->solrConnectionUrl); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), + $url['path'] ?? '', + null, + 0 + ); + $resourceFactoryList = [ + new ExternalResourceFactory(), + new InternalResourceFactory($resourceLoader), + new InternalMediaResourceFactory($resourceLoader) + ]; + $solrResultToResourceResolver = new SolrResultToResourceResolver( + $resourceFactoryList + ); + + return new SolrMoreLikeThis( + $clientFactory, + $solrResultToResourceResolver + ); + } +} diff --git a/src/Console/Command/SolrSelectBuilder.php b/src/Console/Command/SolrSelectBuilder.php new file mode 100644 index 0000000..7e6b3e0 --- /dev/null +++ b/src/Console/Command/SolrSelectBuilder.php @@ -0,0 +1,69 @@ +resourceDir = $resourceDir; + return $this; + } + + public function solrConnectionUrl( + string $solrConnectionUrl + ): SolrSelectBuilder { + $this->solrConnectionUrl = $solrConnectionUrl; + return $this; + } + + public function build(): SolrSelect + { + $resourceBaseLocator = new StaticResourceBaseLocator( + $this->resourceDir + ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); + /** @var string[] */ + $url = parse_url($this->solrConnectionUrl); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), + $url['path'] ?? '', + null, + 0 + ); + $defaultBoosting = new DefaultBoostModifier(); + + $resourceFactoryList = [ + new ExternalResourceFactory(), + new InternalResourceFactory($resourceLoader), + new InternalMediaResourceFactory($resourceLoader) + ]; + + $solrResultToResourceResolver = new SolrResultToResourceResolver( + $resourceFactoryList + ); + + return new SolrSelect( + $clientFactory, + [$defaultBoosting], + $solrResultToResourceResolver + ); + } +} diff --git a/src/Console/Command/SolrSuggestBuilder.php b/src/Console/Command/SolrSuggestBuilder.php new file mode 100644 index 0000000..77aca7a --- /dev/null +++ b/src/Console/Command/SolrSuggestBuilder.php @@ -0,0 +1,35 @@ +solrConnectionUrl = $solrConnectionUrl; + return $this; + } + + public function build(): SolrSuggest + { + /** @var string[] $url */ + $url = parse_url($this->solrConnectionUrl); + $clientFactory = new SolrParameterClientFactory( + $url['scheme'], + $url['host'], + (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), + $url['path'] ?? '', + null, + 0 + ); + return new SolrSuggest($clientFactory); + } +} diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index edc8ff0..b273f89 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -4,13 +4,12 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\Filter\ArchiveFilter; use Atoolo\Search\Dto\Search\Query\Filter\ObjectTypeFilter; use Atoolo\Search\Dto\Search\Query\SuggestQuery; use Atoolo\Search\Dto\Search\Result\SuggestResult; use Atoolo\Search\Service\Search\SolrSuggest; -use Atoolo\Search\Service\SolrParameterClientFactory; -use InvalidArgumentException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -24,14 +23,25 @@ )] class Suggest extends Command { - private InputInterface $input; + private TypifiedInput $input; private SymfonyStyle $io; private string $solrCore; + public function __construct( + private readonly SolrSuggestBuilder $solrSuggestBuilder + ) { + parent::__construct(); + } + protected function configure(): void { $this ->setHelp('Command to performs a suggest search') + ->addArgument( + 'solr-connection-url', + InputArgument::REQUIRED, + 'Solr connection url.' + ) ->addArgument( 'solr-core', InputArgument::REQUIRED, @@ -49,10 +59,10 @@ protected function execute( InputInterface $input, OutputInterface $output ): int { - $this->input = $input; + $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $this->getStringArgument('solr-core'); - $terms = $this->getArrayArgument('terms'); + $this->solrCore = $this->input->getStringArgument('solr-core'); + $terms = $this->input->getArrayArgument('terms'); $search = $this->createSearcher(); $query = $this->buildQuery($terms); @@ -66,42 +76,10 @@ protected function execute( protected function createSearcher(): SolrSuggest { - /** @var string[] $url */ - $url = parse_url($this->getStringArgument('solr-connection-url')); - $clientFactory = new SolrParameterClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 + $this->solrSuggestBuilder->solrConnectionUrl( + $this->input->getStringArgument('solr-connection-url') ); - return new SolrSuggest($clientFactory); - } - - private function getStringArgument(string $name): string - { - $value = $this->input->getArgument($name); - if (!is_string($value)) { - throw new InvalidArgumentException( - $name . ' must be a string' - ); - } - return $value; - } - - /** - * @return string[] - */ - private function getArrayArgument(string $name): array - { - $value = $this->input->getArgument($name); - if (!is_array($value)) { - throw new InvalidArgumentException( - $name . ' must be a array' - ); - } - return $value; + return $this->solrSuggestBuilder->build(); } /** @@ -125,10 +103,10 @@ protected function outputResult(SuggestResult $result): void { foreach ($result as $suggest) { $this->io->text( - $suggest->getTerm() . - ' (' . $suggest->getHits() . ')' + $suggest->term . + ' (' . $suggest->hits . ')' ); } - $this->io->text('Query-Time: ' . $result->getQueryTime() . 'ms'); + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); } } diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index a408aae..7de602d 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -4,11 +4,9 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; -use Atoolo\Search\Service\SolrClientFactory; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Symfony\Component\Lock\LockFactory; @@ -17,23 +15,14 @@ class BackgroundIndexer implements Indexer { - private LockFactory $lockFactory; - - /** - * @param iterable> $documentEnricherList - */ public function __construct( - private readonly iterable $documentEnricherList, - private readonly LocationFinder $finder, - private readonly ResourceLoader $resourceLoader, - private readonly TranslationSplitter $translationSplitter, - private readonly SolrClientFactory $clientFactory, - private readonly IndexingAborter $aborter, - private readonly string $source, + private readonly SolrIndexerFactory $indexerFactory, private readonly IndexerStatusStore $statusStore, - private readonly LoggerInterface $logger = new NullLogger() + private readonly LoggerInterface $logger = new NullLogger(), + private readonly LockFactory $lockFactory = new LockFactory( + new SemaphoreStore() + ) ) { - $this->lockFactory = new LockFactory(new SemaphoreStore()); } /** @@ -77,15 +66,7 @@ private function getIndexer(string $index): SolrIndexer $this->statusStore, $this->logger ); - return new SolrIndexer( - $this->documentEnricherList, - $progressHandler, - $this->finder, - $this->resourceLoader, - $this->translationSplitter, - $this->clientFactory, - $this->aborter, - $this->source - ); + + return $this->indexerFactory->create($progressHandler); } } diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index 5fddc34..3b9d167 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -110,14 +110,6 @@ public function abort(): void $this->status->state = IndexerStatusState::ABORTED; } - /** - * @return array - */ - public function getErrors(): array - { - return []; - } - public function getStatus(): IndexerStatus { return $this->status; diff --git a/src/Service/Indexer/IndexDocumentDumper.php b/src/Service/Indexer/IndexDocumentDumper.php new file mode 100644 index 0000000..ff695c1 --- /dev/null +++ b/src/Service/Indexer/IndexDocumentDumper.php @@ -0,0 +1,46 @@ +> $documentEnricherList + */ + public function __construct( + private readonly ResourceLoader $resourceLoader, + private readonly iterable $documentEnricherList + ) { + } + + /** + * @param string[] $paths + * @return array>. + */ + public function dump(array $paths): array + { + $documents = []; + foreach ($paths as $path) { + $resource = $this->resourceLoader->load($path); + $doc = new IndexSchema2xDocument(); + $processId = 'process-id'; + + foreach ($this->documentEnricherList as $enricher) { + /** @var IndexSchema2xDocument $doc */ + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + + $documents[] = $doc->getFields(); + } + + return $documents; + } +} diff --git a/src/Service/Indexer/IndexingAborter.php b/src/Service/Indexer/IndexingAborter.php index 31e7cba..b5e2ce4 100644 --- a/src/Service/Indexer/IndexingAborter.php +++ b/src/Service/Indexer/IndexingAborter.php @@ -4,8 +4,6 @@ namespace Atoolo\Search\Service\Indexer; -use phpDocumentor\Reflection\Types\Boolean; - class IndexingAborter { public function __construct( @@ -30,6 +28,6 @@ public function aborted(string $index): void private function getAbortMarkerFile(string $index): string { - return $this->workdir . '/background-indexer-' . $index . ".abort"; + return $this->workdir . '/background-indexer-' . $index . '.abort'; } } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 86a3f57..e200240 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -222,7 +222,9 @@ public function enrichDocument( $groupPathAsIdList[] = $group['id']; } - $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; + if (count($groupPathAsIdList) > 2) { + $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; + } $doc->sp_group_path = $groupPathAsIdList; /** @var array $schedulingList */ @@ -352,8 +354,7 @@ private function contactPointToContent(array $contactPoint): string $content[] = ($addressData['street'] ?? ''); $content[] = ($addressData['buildingName'] ?? ''); $content[] = ( - $addressData['postOfficeBoxData']['buildingName'] ?? - '' + $addressData['postOfficeBoxData']['buildingName'] ?? '' ); } diff --git a/src/Service/Indexer/SolrIndexerFactory.php b/src/Service/Indexer/SolrIndexerFactory.php new file mode 100644 index 0000000..2a7caf6 --- /dev/null +++ b/src/Service/Indexer/SolrIndexerFactory.php @@ -0,0 +1,40 @@ +> $documentEnricherList + */ + public function __construct( + private readonly iterable $documentEnricherList, + private readonly LocationFinder $finder, + private readonly ResourceLoader $resourceLoader, + private readonly TranslationSplitter $translationSplitter, + private readonly SolrClientFactory $clientFactory, + private readonly IndexingAborter $aborter, + private readonly string $source + ) { + } + + public function create( + IndexerProgressHandler $progressHandler + ): SolrIndexer { + return new SolrIndexer( + $this->documentEnricherList, + $progressHandler, + $this->finder, + $this->resourceLoader, + $this->translationSplitter, + $this->clientFactory, + $this->aborter, + $this->source + ); + } +} diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php new file mode 100644 index 0000000..ad9487a --- /dev/null +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -0,0 +1,72 @@ +createStub(IndexDocumentDumper::class); + $dumper->method('dump') + ->willReturn([ + ['sp_id' => '123'] + ]); + $dumperBuilder = $this->createStub(IndexDocumentDumperBuilder::class); + $dumperBuilder->method('resourceDir') + ->willReturn($dumperBuilder); + $dumperBuilder->method('documentEnricherList') + ->willReturn($dumperBuilder); + $dumperBuilder->method('build') + ->willReturn($dumper); + + $dumperCommand = new DumpIndexDocument( + [], + $dumperBuilder + ); + + $application = new Application([ + $dumperCommand + ]); + + $command = $application->find('atoolo:dump-index-document'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $this->commandTester->execute([ + 'resource-dir' => 'abc', + 'paths' => ['test.php'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<resourceDir('test'); + $builder->documentEnricherList([]); + + $this->expectNotToPerformAssertions(); + $builder->build(); + } +} diff --git a/test/Console/Command/Io/IndexerProgressBarFactoryTest.php b/test/Console/Command/Io/IndexerProgressBarFactoryTest.php new file mode 100644 index 0000000..5d659d4 --- /dev/null +++ b/test/Console/Command/Io/IndexerProgressBarFactoryTest.php @@ -0,0 +1,21 @@ +expectNotToPerformAssertions(); + $factory->create($this->createStub(OutputInterface::class)); + } +} diff --git a/test/Console/Command/Io/IndexerProgressBarTest.php b/test/Console/Command/Io/IndexerProgressBarTest.php new file mode 100644 index 0000000..a8c3426 --- /dev/null +++ b/test/Console/Command/Io/IndexerProgressBarTest.php @@ -0,0 +1,155 @@ +createStub(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $progressBar->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testStartUpdate(): void + { + $output = $this->createStub(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + + $progressBar->startUpdate(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $progressBar->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + /** + * @phpcs:disable Generic.Files.LineLength + */ + public function testAdvance(): void + { + $output = $this->createMock(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + $output->expects($this->once()) + ->method('write') + ->with($this->stringStartsWith(<<••➤⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬] 10% + < 1 sec +END + )); + + $progressBar->advance(1); + } + + public function testSkip(): void + { + $output = $this->createStub(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + + $progressBar->start(10); + + $progressBar->skip(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*skipped: 1,/', + $progressBar->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testError(): void + { + $output = $this->createMock(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + + //phpcs:ignore Generic.Files.LineLength.TooLong + $output->expects($this->once()) + ->method('write') + ->with($this->stringStartsWith(<<➤⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬] 0% + < 1 sec +END + )); + + $progressBar->error(new Exception('test')); + $progressBar->advance(0); + } + + public function testGetError(): void + { + $output = $this->createMock(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + $progressBar->error(new Exception('test')); + + $this->assertCount( + 1, + $progressBar->getErrors(), + 'unexpected error count' + ); + } + + public function testFinish(): void + { + $output = $this->createMock(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + + $output->expects($this->once()) + ->method('write') + ->with($this->stringStartsWith(<<••••••••••••••••••••••••••••] 100% + < 1 sec +END + )); + + $progressBar->finish(); + } + + public function testAbort(): void + { + $output = $this->createMock(OutputInterface::class); + $progressBar = new IndexerProgressBar($output); + $progressBar->start(10); + + + $output->expects($this->once()) + ->method('write') + ->with($this->stringStartsWith(<<••••••••••••••••••••••••••••] 100% + < 1 sec +END + )); + + $progressBar->abort(); + } +} diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php new file mode 100644 index 0000000..dfc85a9 --- /dev/null +++ b/test/Console/Command/MoreLikeThisTest.php @@ -0,0 +1,84 @@ +createStub(Resource::class); + $resultResource->method('getLocation') + ->willReturn('/test2.php'); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [], + 10 + ); + $solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); + $solrMoreLikeThis->method('moreLikeThis') + ->willReturn($result); + $moreLikeThisBuilder = $this->createStub( + SolrMoreLikeThisBuilder::class + ); + $moreLikeThisBuilder->method('build') + ->willReturn($solrMoreLikeThis); + + $moreLinkeThis = new MoreLikeThis( + $moreLikeThisBuilder + ); + + $application = new Application([ + $moreLinkeThis + ]); + + $command = $application->find('atoolo:mlt'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $this->commandTester->execute([ + // pass arguments to the helper + 'resource-dir' => 'abc', + 'solr-connection-url' => 'http://localhost:8080', + 'solr-core' => 'test', + 'location' => '/test.php' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub(Resource::class); + $resultResource->method('getLocation') + ->willReturn('/test.php'); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [new FacetGroup( + 'objectType', + [new Facet( + 'content', + 1 + )] + )], + 10 + ); + $solrSelect = $this->createStub(SolrSelect::class); + $solrSelect->method('select') + ->willReturn($result); + + $solrSelectBuilder = $this->createStub( + SolrSelectBuilder::class + ); + $solrSelectBuilder->method('build') + ->willReturn($solrSelect); + + $search = new Search( + $solrSelectBuilder + ); + + $application = new Application([ + $search + ]); + + $command = $application->find('atoolo:search'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $this->commandTester->execute([ + 'solr-connection-url' => 'http://localhost:8382', + 'index' => 'test', + 'resource-dir' => 'abc', + 'text' => ['test', 'abc'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<resourceDir($resourceDir) + ->documentEnricherList([$this->createStub(DocumentEnricher::class)]) + ->progressBar($this->createStub(IndexerProgressBar::class)) + ->solrConnectionUrl('http://localhost:8382'); + + $this->expectNotToPerformAssertions(); + $builder->build(); + } +} diff --git a/test/Console/Command/SolrMoreLikeThisBuilderTest.php b/test/Console/Command/SolrMoreLikeThisBuilderTest.php new file mode 100644 index 0000000..8836bf4 --- /dev/null +++ b/test/Console/Command/SolrMoreLikeThisBuilderTest.php @@ -0,0 +1,24 @@ +resourceDir('test.php') + ->solrConnectionUrl('http://localhost:8382'); + + $this->expectNotToPerformAssertions(); + $builder->build(); + } +} diff --git a/test/Console/Command/SolrSelectBuilderTest.php b/test/Console/Command/SolrSelectBuilderTest.php new file mode 100644 index 0000000..82c9208 --- /dev/null +++ b/test/Console/Command/SolrSelectBuilderTest.php @@ -0,0 +1,23 @@ +resourceDir('test') + ->solrConnectionUrl('http://localhost:8382'); + + $this->expectNotToPerformAssertions(); + $builder->build(); + } +} diff --git a/test/Console/Command/SolrSuggestBuilderTest.php b/test/Console/Command/SolrSuggestBuilderTest.php new file mode 100644 index 0000000..621c7cc --- /dev/null +++ b/test/Console/Command/SolrSuggestBuilderTest.php @@ -0,0 +1,22 @@ +solrConnectionUrl('http://localhost:8283'); + + $this->expectNotToPerformAssertions(); + $builder->build(); + } +} diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php new file mode 100644 index 0000000..b090a15 --- /dev/null +++ b/test/Console/Command/SuggestTest.php @@ -0,0 +1,78 @@ +createStub(SolrSuggest::class); + $solrSuggest->method('suggest') + ->willReturn($result); + $solrSuggestBuilder = $this->createStub( + SolrSuggestBuilder::class + ); + $solrSuggestBuilder->method('build') + ->willReturn($solrSuggest); + + $suggest = new Suggest( + $solrSuggestBuilder + ); + + $application = new Application([ + $suggest + ]); + + $command = $application->find('atoolo:suggest'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $this->commandTester->execute([ + 'solr-connection-url' => 'http://localhost:8382', + 'solr-core' => 'test', + 'terms' => ['sec'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub(Filter::class); diff --git a/test/Dto/Search/Query/Filter/ArchiveFilterTest.php b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php index 48283eb..022d96d 100644 --- a/test/Dto/Search/Query/Filter/ArchiveFilterTest.php +++ b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php @@ -17,7 +17,7 @@ public function testAndQuery(): void $this->assertEquals( '-sp_archive:true', $filter->getQuery(), - 'unexpected query' + 'unexpected query' ); } } diff --git a/test/Dto/Search/Query/Filter/OrFilterTest.php b/test/Dto/Search/Query/Filter/OrFilterTest.php index f072cfd..becc11f 100644 --- a/test/Dto/Search/Query/Filter/OrFilterTest.php +++ b/test/Dto/Search/Query/Filter/OrFilterTest.php @@ -8,6 +8,7 @@ use Atoolo\Search\Dto\Search\Query\Filter\OrFilter; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; + use function PHPUnit\Framework\assertEquals; #[CoversClass(OrFilter::class)] diff --git a/test/Dto/Search/Query/Filter/QueryFilterTest.php b/test/Dto/Search/Query/Filter/QueryFilterTest.php index 57f993f..d629422 100644 --- a/test/Dto/Search/Query/Filter/QueryFilterTest.php +++ b/test/Dto/Search/Query/Filter/QueryFilterTest.php @@ -11,7 +11,6 @@ #[CoversClass(QueryFilter::class)] class QueryFilterTest extends TestCase { - public function testGetQuery(): void { $filter = new QueryFilter(null, 'a:b'); diff --git a/test/Exception/DocumentEnrichingExceptionTest.php b/test/Exception/DocumentEnrichingExceptionTest.php new file mode 100644 index 0000000..949ac52 --- /dev/null +++ b/test/Exception/DocumentEnrichingExceptionTest.php @@ -0,0 +1,21 @@ +assertEquals( + '/test.php', + $e->getLocation(), + 'unexpected location' + ); + } +} diff --git a/test/Exception/MissMatchingResourceFactoryExceptionTest.php b/test/Exception/MissMatchingResourceFactoryExceptionTest.php new file mode 100644 index 0000000..7c1022e --- /dev/null +++ b/test/Exception/MissMatchingResourceFactoryExceptionTest.php @@ -0,0 +1,21 @@ +assertEquals( + '/test.php', + $e->getLocation(), + 'unexpected location' + ); + } +} diff --git a/test/Exception/UnexpectedResultExceptionTest.php b/test/Exception/UnexpectedResultExceptionTest.php new file mode 100644 index 0000000..60cb767 --- /dev/null +++ b/test/Exception/UnexpectedResultExceptionTest.php @@ -0,0 +1,21 @@ +assertEquals( + 'test', + $e->getResult(), + 'unexpected result' + ); + } +} diff --git a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php new file mode 100644 index 0000000..a5e3cf8 --- /dev/null +++ b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php @@ -0,0 +1,150 @@ +statusStore = $this->createMock(IndexerStatusStore::class); + $this->state = new BackgroundIndexerProgressState( + 'test', + $this->statusStore + ); + } + + public function testStart(): void + { + $this->state->start(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testUpdate(): void + { + + $this->statusStore + ->method('load') + ->willReturn(IndexerStatus::empty()); + + $this->state->startUpdate(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testAdvance(): void + { + + $this->state->start(10); + + $this->statusStore->expects($this->once()) + ->method('store'); + + $this->state->advance(1); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 1\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testAdvanceForUpdate(): void + { + + $this->statusStore + ->method('load') + ->willReturn(IndexerStatus::empty()); + + $this->state->startUpdate(10); + + $this->statusStore->expects($this->once()) + ->method('store'); + + $this->state->advance(1); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 1\/10,.*updated: 1,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testSkip(): void + { + + $this->state->start(10); + + $this->state->skip(1); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*skipped: 1,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testError(): void + { + $this->state->start(10); + + $this->state->error(new Exception('test')); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*errors: 1$/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testFinish(): void + { + $this->state->start(10); + + $this->statusStore->expects($this->once()) + ->method('store'); + + $this->state->finish(); + + $this->assertMatchesRegularExpression( + '/\[INDEXED]/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testAbort(): void + { + $this->state->start(10); + + $this->state->abort(); + + $this->assertMatchesRegularExpression( + '/\[ABORTED]/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } +} diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php new file mode 100644 index 0000000..31edd8a --- /dev/null +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -0,0 +1,91 @@ +solrIndexer = $this->createMock(SolrIndexer::class); + $solrIndexerFactory = $this->createStub(SolrIndexerFactory::class); + $solrIndexerFactory->method('create') + ->willReturn($this->solrIndexer); + $this->statusStore = $this->createMock(IndexerStatusStore::class); + $this->indexer = new BackgroundIndexer( + $solrIndexerFactory, + $this->statusStore + ); + } + + public function testRemove(): void + { + $this->solrIndexer->expects($this->once()) + ->method('remove'); + $this->indexer->remove('test', ['123']); + } + + public function testAbort(): void + { + $this->solrIndexer->expects($this->once()) + ->method('abort'); + $this->indexer->abort('test'); + } + + public function testIndex(): void + { + $params = new IndexerParameter('test'); + $this->solrIndexer->expects($this->once()) + ->method('index'); + $this->indexer->index($params); + } + + public function testIndexIfLocked(): void + { + $lockFactory = $this->createStub(LockFactory::class); + $indexer = new BackgroundIndexer( + $this->createStub(SolrIndexerFactory::class), + $this->statusStore, + new NullLogger(), + $lockFactory + ); + + $lock = $this->createStub(SharedLockInterface::class); + $lock->method('acquire') + ->willReturn(false); + $lockFactory->method('createLock') + ->willReturn($lock); + + $this->solrIndexer->expects($this->exactly(0)) + ->method('index'); + + $params = new IndexerParameter('test'); + $indexer->index($params); + } + + public function testGetStatus(): void + { + $params = new IndexerParameter('test'); + $this->statusStore->expects($this->once()) + ->method('load'); + $this->indexer->getStatus('test'); + } +} diff --git a/test/Service/Indexer/DocumentEnrichter.php b/test/Service/Indexer/DocumentEnrichter.php new file mode 100644 index 0000000..c5fed38 --- /dev/null +++ b/test/Service/Indexer/DocumentEnrichter.php @@ -0,0 +1,9 @@ +createStub(ResourceLoader::class); + $documentEnricher = $this->createStub(DocumentEnricher::class); + $documentEnricher->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + $doc->sp_id = '123'; + return $doc; + }); + $dumper = new IndexDocumentDumper( + $resourceLoader, + [$documentEnricher] + ); + + $dump = $dumper->dump(['/test.php']); + + $this->assertEquals( + [['sp_id' => '123']], + $dump, + 'unexpected dump' + ); + } +} diff --git a/test/Service/Indexer/IndexingAborterTest.php b/test/Service/Indexer/IndexingAborterTest.php new file mode 100644 index 0000000..9b46c08 --- /dev/null +++ b/test/Service/Indexer/IndexingAborterTest.php @@ -0,0 +1,61 @@ +file = $workdir . '/background-indexer-test.abort'; + unlink($this->file); + $this->aborter = new IndexingAborter($workdir); + } + + public function testShouldNotAborted(): void + { + $this->assertFalse( + $this->aborter->shouldAborted('test'), + 'should not aborted' + ); + } + public function testShouldAborted(): void + { + touch($this->file); + $this->assertTrue( + $this->aborter->shouldAborted('test'), + 'should not aborted' + ); + } + + public function testAbort(): void + { + $this->aborter->abort('test'); + $this->assertFileExists( + $this->file, + 'abort call should create file' + ); + } + + public function testAborted(): void + { + touch($this->file); + $this->aborter->aborted('test'); + $this->assertFileDoesNotExist( + $this->file, + 'aborted call should remove file' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php new file mode 100644 index 0000000..522e71a --- /dev/null +++ b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php @@ -0,0 +1,794 @@ +createStub( + SiteKitNavigationHierarchyLoader::class + ); + $navigationLoader + ->method('loadRoot') + ->willReturnCallback(function ($location) { + if ($location === 'throwException') { + throw new InvalidResourceException($location); + } + return $this->createResource(['init' => [ + 'siteGroup' => ['id' => 999] + ]]); + }); + $contentCollector = $this->createStub(ContentCollector::class); + $contentCollector + ->method('collect') + ->willReturn('collected content'); + + $this->enricher = new DefaultSchema2xDocumentEnricher( + $navigationLoader, + $contentCollector + ); + } + + public function testIndexable(): void + { + $resource = $this->createResource([ + 'init' => [ + ] + ]); + + $this->assertTrue( + $this->enricher->isIndexable($resource), + 'should be indexalbe' + ); + } + + public function testNotIndexable(): void + { + $resource = $this->createResource([ + 'init' => [ + 'noIndex' => true + ] + ]); + + $this->assertFalse( + $this->enricher->isIndexable($resource), + 'should be indexalbe' + ); + } + + public function testEnrichSpId(): void + { + $resource = $this->createStub(Resource::class); + $resource->method('getId')->willReturn('123'); + $doc = $this->enrichWithResource($resource); + $this->assertEquals('123', $doc->sp_id, 'unexpected id'); + } + + public function testEnrichName(): void + { + $resource = $this->createStub(Resource::class); + $resource->method('getName')->willReturn('test'); + $doc = $this->enrichWithResource($resource); + $this->assertEquals('test', $doc->sp_name, 'unexpected name'); + } + + public function testEnrichObjectType(): void + { + $resource = $this->createStub(Resource::class); + $resource->method('getObjectType')->willReturn('test'); + $doc = $this->enrichWithResource($resource); + $this->assertEquals( + 'test', + $doc->sp_objecttype, + 'unexpected objectType' + ); + } + + public function testEnrichAnchor(): void + { + $doc = $this->enrichWithData(['init' => ['anchor' => 'abc']]); + $this->assertEquals('abc', $doc->sp_anchor, 'unexpected ancohr'); + } + + public function testEnrichTitle(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'abc']]); + $this->assertEquals('abc', $doc->title, 'unexpected title'); + } + + public function testEnrichDescriptionWithInto(): void + { + $doc = $this->enrichWithData(['metadata' => ['intro' => 'abc']]); + $this->assertEquals( + 'abc', + $doc->description, + 'unexpected description' + ); + } + + public function testEnrichCanonical(): void + { + $doc = $this->enrichWithData([]); + $this->assertTrue( + $doc->sp_canonical, + 'unexpected canonical' + ); + } + + public function testEnrichCrawlProcessId(): void + { + $resource = $this->createStub(Resource::class); + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + $this->assertEquals( + $doc->crawl_process_id, + 'progress-id', + 'unexpected progress id' + ); + } + + public function testEnrichId(): void + { + $doc = $this->enrichWithData(['init' => ['url' => '/test.php']]); + $this->assertEquals( + '/test.php', + $doc->id, + 'unexpected id' + ); + } + + public function testEnrichUrl(): void + { + $doc = $this->enrichWithData(['init' => ['url' => '/test.php']]); + $this->assertEquals( + '/test.php', + $doc->url, + 'unexpected url' + ); + } + + public function testEnrichMediaId(): void + { + $doc = $this->enrichWithData(['init' => ['mediaUrl' => '/test.php']]); + $this->assertEquals( + '/test.php', + $doc->id, + 'unexpected id' + ); + } + + public function testEnrichMediaUrl(): void + { + $doc = $this->enrichWithData(['init' => ['mediaUrl' => '/test.php']]); + $this->assertEquals( + '/test.php', + $doc->url, + 'unexpected url' + ); + } + + public function testEnrichSpContentType(): void + { + $doc = $this->enrichWithData([ + 'init' => [ + 'objectType' => 'content', + 'contentSectionTypes' => ['text', 'linkList'] + ], + 'base' => [ + 'teaser' => [ + 'headline' => 'test', + 'image' => [ + 'copyright' => 'test' + ], + 'text' => 'test' + ] + ] + ]); + $this->assertEquals( + [ + 'content', + 'article', + 'text', + 'linkList', + 'teaserImage', + 'teaserImageCopyright', + 'teaserHeadline', + 'teaserText' + ], + $doc->sp_contenttype, + 'unexpected sp_contenttype' + ); + } + + public function testEnrichDefaultLanguage(): void + { + $doc = $this->enrichWithData([]); + $this->assertEquals( + 'de', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguage(): void + { + $doc = $this->enrichWithData(['init' => ['locale' => 'en_US']]); + $this->assertEquals( + 'en', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguageWithShortLocale(): void + { + $doc = $this->enrichWithData(['init' => ['locale' => 'en']]); + $this->assertEquals( + 'en', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguageOverGroupPath(): void + { + $doc = $this->enrichWithData(['init' => [ + 'groupPath' => [ + ['locale' => 'fr_FR'], + ['locale' => 'it_IT'] + ] + ]]); + $this->assertEquals( + 'it', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichDefaultMetaContentLanguage(): void + { + $doc = $this->enrichWithData([]); + $this->assertEquals( + 'de', + $doc->meta_content_language, + 'unexpected language' + ); + } + + public function testEnrichChanged(): void + { + $doc = $this->enrichWithData(['init' => ['changed' => 1708932236]]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_changed, + 'unexpected sp_changed' + ); + } + + public function testEnrichGenerated(): void + { + $doc = $this->enrichWithData(['init' => ['generated' => 1708932236]]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_generated, + 'unexpected sp_generated' + ); + } + + public function testEnrichDate(): void + { + $doc = $this->enrichWithData(['base' => ['date' => 1708932236]]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_date, + 'unexpected sp_generated' + ); + } + + public function testEnrichArchive(): void + { + $doc = $this->enrichWithData(['base' => ['archive' => true]]); + $this->assertTrue( + $doc->sp_archive, + 'unexpected language' + ); + } + + public function testEnrichSpTitle(): void + { + $doc = $this->enrichWithData(['metadata' => ['headline' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSpTitleWithTeaserHeadlineFallback(): void + { + $doc = $this->enrichWithData(['base' => [ + 'title' => 'test', + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSpTitleWithTeaserTitleFallback(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSortValue(): void + { + $doc = $this->enrichWithData(['base' => [ + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichSortValueWithHeadlineFallback(): void + { + $doc = $this->enrichWithData(['base' => [ + 'title' => 'test', + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichSortValueWithTitleFallback(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichKeywords(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'keywords' => ['abc', 'cde'] + ]]); + $this->assertEquals( + ['abc', 'cde'], + $doc->keywords, + 'unexpected keywords' + ); + } + + public function testEnrichBoostKeywords(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'boostKeywords' => ['abc', 'cde'] + ]]); + $this->assertEquals( + 'abc cde', + $doc->sp_boost_keywords, + 'unexpected keywords' + ); + } + + public function testEnrichSpSites(): void + { + $doc = $this->enrichWithData(['base' => [ + 'trees' => [ + 'navigation' => [ + 'parents' => [ + ['siteGroup' => ['id' => '123']], + ['siteGroup' => ['id' => '456']] + ] + ] + ] + ]]); + $this->assertEquals( + ['123', '456', '999'], + $doc->sp_site, + 'unexpected keywords' + ); + } + + public function testEnrichSpSitesWithInvalidRootResource(): void + { + $resource = $this->createResource([]); + $resource->method('getLocation')->willReturn('throwException'); + + $this->expectException(DocumentEnrichingException::class); + $this->enrichWithResource($resource); + } + + public function testEnrichGeoPoints(): void + { + $doc = $this->enrichWithData(['base' => [ + 'geo' => [ + 'wkt' => [ + 'primary' => [ + 'value' => 'test' + ] + ] + ] + ]]); + $this->assertEquals( + ['value' => 'test'], + $doc->sp_geo_points, + 'unexpected sp_geo_points' + ); + } + + public function testEnrichCategories(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'categories' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + ['1', '2', '3'], + $doc->sp_category, + 'unexpected sp_category' + ); + } + + public function testEnrichCategoryPath(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'categoriesPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + ['1', '2', '3'], + $doc->sp_category_path, + 'unexpected sp_category_path' + ); + } + + public function testEnrichSpGroup(): void + { + $doc = $this->enrichWithData(['init' => [ + 'groupPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + 2, + $doc->sp_group, + 'unexpected sp_group' + ); + } + + public function testEnrichSpGroupPath(): void + { + $doc = $this->enrichWithData(['init' => [ + 'groupPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + [1, 2, 3], + $doc->sp_group_path, + 'unexpected sp_group_path' + ); + } + + public function testEnrichDateViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'base' => ['date' => 1707549836], + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test'], + ['from' => 1709105036, 'contentType' => 'test'], + ] + ] + ]); + + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_date, + 'unexpected sp_date' + ); + } + + public function testEnrichContentTypeViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'init' => ['objectType' => 'content'], + 'base' => ['date' => 1707549836], + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test1'], + ['from' => 1709105036, 'contentType' => 'test2'], + ['from' => 1707981836, 'contentType' => 'test1'], + ] + ] + ]); + + $this->assertEquals( + ['content', 'article', 'test1', 'test2'], + $doc->sp_contenttype, + 'unexpected sp_contenttype' + ); + } + + public function testEnrichDateListViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test'], + ['from' => 1709105036, 'contentType' => 'test'], + ] + ] + ]); + + $dateA = new DateTime(); + $dateA->setTimestamp(1708932236); + $dateB = new DateTime(); + $dateB->setTimestamp(1709105036); + + $this->assertEquals( + [$dateA, $dateB], + $doc->sp_date_list, + 'unexpected sp_date' + ); + } + + public function testEnrichDefaultMetaContentType(): void + { + $doc = $this->enrichWithData([ + ]); + + $this->assertEquals( + 'text/html; charset=UTF-8', + $doc->meta_content_type, + 'unexpected meta_content_type' + ); + } + + public function testEnrichIncludeGroups(): void + { + $doc = $this->enrichWithData(['init' => [ + 'access' => [ + 'type' => 'allow', + 'groups' => ['100010100000001028'] + ] + ]]); + + $this->assertEquals( + ['1028'], + $doc->include_groups, + 'unexpected include_groups' + ); + } + + public function testEnrichIncludeAllGroups(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + ['all'], + $doc->include_groups, + 'unexpected include_groups' + ); + } + + public function testEnrichExcludeGroups(): void + { + $doc = $this->enrichWithData(['init' => [ + 'access' => [ + 'type' => 'deny', + 'groups' => ['100010100000001028'] + ] + ]]); + + $this->assertEquals( + ['1028'], + $doc->exclude_groups, + 'unexpected exclude_groups' + ); + } + + public function testEnrichNonExcludeGroups(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + ['none'], + $doc->exclude_groups, + 'unexpected exclude_groups' + ); + } + + public function testEnrichMetaContentType(): void + { + $doc = $this->enrichWithData([ + 'base' => ['mime' => 'application/pdf'] + ]); + + $this->assertEquals( + 'application/pdf', + $doc->meta_content_type, + 'unexpected meta_content_type' + ); + } + + public function testEnrichInternal(): void + { + $doc = $this->enrichWithData([ + ]); + + $this->assertEquals( + ['internal'], + $doc->sp_source, + 'unexpected sp_source' + ); + } + + public function testEnrichContent(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'categories' => [ + ['name' => 'CategoryA'], + ['name' => 'CategoryB'] + ] + ], + 'searchindexdata' => ['content' => 'abc'] + ]); + + $this->assertEquals( + 'abc collected content CategoryA CategoryB', + $doc->content, + 'unexpected content' + ); + } + + public function testEnrichContactPointContent(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'contactPoint' => [ + 'contactData' => [ + 'phoneList' => [ + ['phone' => [ + 'countryCode' => '49', + 'areaCode' => '251', + 'localNumber' => '123' + ]], + ['phone' => [ + 'countryCode' => '49', + 'areaCode' => '2571', + 'localNumber' => '456' + ]] + ], + 'emailList' => [ + ['email' => 'test1@sitepark.com'], + ['email' => 'test2@sitepark.com'] + ] + ], + 'addressData' => [ + 'street' => 'Neubrückenstr', + 'buildingName' => 'Pressehaus', + 'postOfficeBoxData' => [ + 'buildingName' => 'Sitepark' + ] + ] + ], + ] + ]); + + $this->assertEquals( + 'collected content +49 251 0251 123 +49 2571 02571 456 ' . + 'test1@sitepark.com test2@sitepark.com ' . + 'Neubrückenstr Pressehaus Sitepark', + $doc->content, + 'unexpected content' + ); + } + + private function enrichWithResource( + Resource $resource + ): IndexSchema2xDocument { + /** @var IndexSchema2xDocument $doc */ + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + return $doc; + } + + /** + * @param array> $data + */ + private function enrichWithData( + array $data + ): IndexSchema2xDocument { + $resource = $this->createResource($data); + /** @var IndexSchema2xDocument $doc */ + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + return $doc; + } + + /** + * @param array> $data + */ + private function createResource(array $data): Resource&Stub + { + $dataBag = new DataBag($data); + $resource = $this->createStub(Resource::class); + $resource->method('getData')->willReturn($dataBag); + $resource->method('getObjectType')->willReturn( + $data['init']['objectType'] ?? '' + ); + return $resource; + } +} diff --git a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php index 16bbcc5..07a0443 100644 --- a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php +++ b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php @@ -61,5 +61,4 @@ public function testMatcherNotMatchedTextMissing(): void 'should not find any content' ); } - } diff --git a/test/Service/Indexer/SolrIndexerFactoryTest.php b/test/Service/Indexer/SolrIndexerFactoryTest.php new file mode 100644 index 0000000..c7b2ea9 --- /dev/null +++ b/test/Service/Indexer/SolrIndexerFactoryTest.php @@ -0,0 +1,35 @@ +createStub(LocationFinder::class), + $this->createStub(ResourceLoader::class), + $this->createStub(TranslationSplitter::class), + $this->createStub(SolrClientFactory::class), + $this->createStub(IndexingAborter::class), + '' + ); + + $this->expectNotToPerformAssertions(); + $factory->create( + $this->createStub(IndexerProgressHandler::class) + ); + } +} diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/SolrIndexerTest.php index b87e89c..a7a6bd3 100644 --- a/test/Service/Indexer/SolrIndexerTest.php +++ b/test/Service/Indexer/SolrIndexerTest.php @@ -80,7 +80,7 @@ public function setUp(): void $coreAdminResult->method('getStatusResults') ->willReturnCallback(function () { $results = []; - foreach($this->availableIndexes as $index) { + foreach ($this->availableIndexes as $index) { $result = $this->createStub(StatusResult::class); $result->method('getCoreName') ->willReturn($index); diff --git a/test/Service/Search/ExternalResourceFactoryTest.php b/test/Service/Search/ExternalResourceFactoryTest.php index 4f37a52..9b8ffb1 100644 --- a/test/Service/Search/ExternalResourceFactoryTest.php +++ b/test/Service/Search/ExternalResourceFactoryTest.php @@ -12,7 +12,6 @@ #[CoversClass(ExternalResourceFactory::class)] class ExternalResourceFactoryTest extends TestCase { - private ExternalResourceFactory $factory; protected function setUp(): void diff --git a/test/Service/Search/SolrResultToResourceResolverTest.php b/test/Service/Search/SolrResultToResourceResolverTest.php index efe1a31..5c4dd05 100644 --- a/test/Service/Search/SolrResultToResourceResolverTest.php +++ b/test/Service/Search/SolrResultToResourceResolverTest.php @@ -16,7 +16,6 @@ #[CoversClass(SolrResultToResourceResolver::class)] class SolrResultToResourceResolverTest extends TestCase { - public function testLoadResourceList(): void { $document = $this->createStub(Document::class); @@ -84,5 +83,4 @@ public function testLoadResourceListWithoutAcceptedFactoryNoUrl(): void 'resourceList should be empty' ); } - } From 3d1062b223f2552b7549cc3ab287e95de3778257 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 27 Feb 2024 09:26:32 +0100 Subject: [PATCH 046/145] Resource base locator unified --- src/Console/Command/Indexer.php | 23 ++++++++------ .../Command/ResourceBaseLocatorBuilder.php | 24 ++++++++++++++ src/Console/Command/SolrIndexerBuilder.php | 15 ++++----- .../Command/SolrMoreLikeThisBuilder.php | 14 +++++++-- src/Console/Command/SolrSelectBuilder.php | 8 +++-- src/Service/Search/SolrMoreLikeThis.php | 4 +-- .../ResourceBaseLocatorBuilderTest.php | 31 +++++++++++++++++++ .../Command/SolrIndexerBuilderTest.php | 12 +++---- .../Command/SolrMoreLikeThisBuilderTest.php | 5 ++- .../Console/Command/SolrSelectBuilderTest.php | 5 ++- test/Service/Search/SolrMoreLikeThisTest.php | 4 +-- 11 files changed, 109 insertions(+), 36 deletions(-) create mode 100644 src/Console/Command/ResourceBaseLocatorBuilder.php create mode 100644 test/Console/Command/ResourceBaseLocatorBuilderTest.php diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index f7cc21e..40d93a3 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -25,7 +25,7 @@ class Indexer extends Command { private IndexerProgressBar $progressBar; private SymfonyStyle $io; - private TypifiedInput $input; + private OutputInterface $output; /** * phpcs:ignore @@ -90,14 +90,15 @@ protected function execute( OutputInterface $output ): int { - $this->input = new TypifiedInput($input); + $typedInput = new TypifiedInput($input); + $this->output = $output; $this->io = new SymfonyStyle($input, $output); $this->progressBar = $this->progressBarFactory->create($output); - $paths = $this->input->getArrayArgument('paths'); + $paths = $typedInput->getArrayArgument('paths'); $cleanupThreshold = empty($paths) - ? $this->input->getIntOption('cleanup-threshold') + ? $typedInput->getIntOption('cleanup-threshold') : 0; if (empty($paths)) { @@ -108,18 +109,18 @@ protected function execute( } $parameter = new IndexerParameter( - $this->input->getStringArgument('solr-core'), + $typedInput->getStringArgument('solr-core'), $cleanupThreshold, - $this->input->getIntOption('chunk-size'), + $typedInput->getIntOption('chunk-size'), $paths ); $this->solrIndexerBuilder - ->resourceDir($this->input->getStringArgument('resource-dir')) + ->resourceDir($typedInput->getStringArgument('resource-dir')) ->progressBar($this->progressBar) ->documentEnricherList($this->documentEnricherList) ->solrConnectionUrl( - $this->input->getStringArgument('solr-connection-url') + $typedInput->getStringArgument('solr-connection-url') ); $indexer = $this->solrIndexerBuilder->build(); @@ -133,7 +134,11 @@ protected function execute( protected function errorReport(): void { foreach ($this->progressBar->getErrors() as $error) { - $this->io->error($error->getMessage()); + if ($this->io->isVerbose() && $this->getApplication() !== null) { + $this->getApplication()->renderThrowable($error, $this->output); + } else { + $this->io->error($error->getMessage()); + } } } } diff --git a/src/Console/Command/ResourceBaseLocatorBuilder.php b/src/Console/Command/ResourceBaseLocatorBuilder.php new file mode 100644 index 0000000..f6c3c7b --- /dev/null +++ b/src/Console/Command/ResourceBaseLocatorBuilder.php @@ -0,0 +1,24 @@ +resourceDir = $resourceDir; @@ -65,14 +68,8 @@ public function solrConnectionUrl( public function build(): SolrIndexer { - $subDirectory = null; - if (is_dir($this->resourceDir . '/objects')) { - $subDirectory = 'objects'; - } - $_SERVER['RESOURCE_ROOT'] = $this->resourceDir; - $resourceBaseLocator = new ServerVarResourceBaseLocator( - 'RESOURCE_ROOT', - $subDirectory + $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( + $this->resourceDir ); $finder = new LocationFinder($resourceBaseLocator); $resourceLoader = new SiteKitLoader($resourceBaseLocator); diff --git a/src/Console/Command/SolrMoreLikeThisBuilder.php b/src/Console/Command/SolrMoreLikeThisBuilder.php index 8f72a7d..68bd995 100644 --- a/src/Console/Command/SolrMoreLikeThisBuilder.php +++ b/src/Console/Command/SolrMoreLikeThisBuilder.php @@ -5,19 +5,26 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; use Atoolo\Search\Service\Search\SolrMoreLikeThis; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\SolrParameterClientFactory; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; class SolrMoreLikeThisBuilder { private string $resourceDir; private string $solrConnectionUrl; + public function __construct( + private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder, + private readonly LoggerInterface $logger = new NullLogger() + ) { + } + public function resourceDir(string $resourceDir): SolrMoreLikeThisBuilder { $this->resourceDir = $resourceDir; @@ -33,7 +40,7 @@ public function solrConnectionUrl( public function build(): SolrMoreLikeThis { - $resourceBaseLocator = new StaticResourceBaseLocator( + $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( $this->resourceDir ); $resourceLoader = new SiteKitLoader($resourceBaseLocator); @@ -53,7 +60,8 @@ public function build(): SolrMoreLikeThis new InternalMediaResourceFactory($resourceLoader) ]; $solrResultToResourceResolver = new SolrResultToResourceResolver( - $resourceFactoryList + $resourceFactoryList, + $this->logger ); return new SolrMoreLikeThis( diff --git a/src/Console/Command/SolrSelectBuilder.php b/src/Console/Command/SolrSelectBuilder.php index 7e6b3e0..090470f 100644 --- a/src/Console/Command/SolrSelectBuilder.php +++ b/src/Console/Command/SolrSelectBuilder.php @@ -5,7 +5,6 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; @@ -19,6 +18,11 @@ class SolrSelectBuilder private string $resourceDir; private string $solrConnectionUrl; + public function __construct( + private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder + ) { + } + public function resourceDir(string $resourceDir): SolrSelectBuilder { $this->resourceDir = $resourceDir; @@ -34,7 +38,7 @@ public function solrConnectionUrl( public function build(): SolrSelect { - $resourceBaseLocator = new StaticResourceBaseLocator( + $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( $this->resourceDir ); $resourceLoader = new SiteKitLoader($resourceBaseLocator); diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 2036517..fd694c0 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -10,7 +10,7 @@ use Atoolo\Search\Service\SolrClientFactory; use Solarium\Core\Client\Client; use Solarium\QueryType\MoreLikeThis\Query as SolrMoreLikeThisQuery; -use Solarium\QueryType\Select\Result\Result as SelectResult; +use Solarium\QueryType\MoreLikeThis\Result as SolrMoreLikeThisResult; /** * Implementation of the "More-Like-This" on the basis of a Solr index. @@ -58,7 +58,7 @@ private function buildSolrQuery( } private function buildResult( - SelectResult $result + SolrMoreLikeThisResult $result ): SearchResult { $resourceList = $this->resultToResourceResolver diff --git a/test/Console/Command/ResourceBaseLocatorBuilderTest.php b/test/Console/Command/ResourceBaseLocatorBuilderTest.php new file mode 100644 index 0000000..7c3337b --- /dev/null +++ b/test/Console/Command/ResourceBaseLocatorBuilderTest.php @@ -0,0 +1,31 @@ +build($resourceDir); + + $this->assertEquals( + $objectsDir, + $locator->locate(), + 'unexpected resource dir' + ); + } +} diff --git a/test/Console/Command/SolrIndexerBuilderTest.php b/test/Console/Command/SolrIndexerBuilderTest.php index 14f56bf..d0f74db 100644 --- a/test/Console/Command/SolrIndexerBuilderTest.php +++ b/test/Console/Command/SolrIndexerBuilderTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Console\Command; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; +use Atoolo\Search\Console\Command\ResourceBaseLocatorBuilder; use Atoolo\Search\Console\Command\SolrIndexerBuilder; use Atoolo\Search\Service\Indexer\DocumentEnricher; use PHPUnit\Framework\Attributes\CoversClass; @@ -16,14 +17,11 @@ class SolrIndexerBuilderTest extends TestCase public function testBuild(): void { - $resourceDir = __DIR__ . - '/../../../var/test/SolrIndexerBuilderTest'; - $objectsDir = $resourceDir . '/objects'; - mkdir($objectsDir, 0777, true); - - $builder = new SolrIndexerBuilder(); + $builder = new SolrIndexerBuilder( + $this->createStub(ResourceBaseLocatorBuilder::class) + ); $builder - ->resourceDir($resourceDir) + ->resourceDir('test') ->documentEnricherList([$this->createStub(DocumentEnricher::class)]) ->progressBar($this->createStub(IndexerProgressBar::class)) ->solrConnectionUrl('http://localhost:8382'); diff --git a/test/Console/Command/SolrMoreLikeThisBuilderTest.php b/test/Console/Command/SolrMoreLikeThisBuilderTest.php index 8836bf4..00aabcf 100644 --- a/test/Console/Command/SolrMoreLikeThisBuilderTest.php +++ b/test/Console/Command/SolrMoreLikeThisBuilderTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Search\Console\Command\ResourceBaseLocatorBuilder; use Atoolo\Search\Console\Command\SolrMoreLikeThisBuilder; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -13,7 +14,9 @@ class SolrMoreLikeThisBuilderTest extends TestCase { public function testBuild(): void { - $builder = new SolrMoreLikeThisBuilder(); + $builder = new SolrMoreLikeThisBuilder( + $this->createStub(ResourceBaseLocatorBuilder::class) + ); $builder ->resourceDir('test.php') ->solrConnectionUrl('http://localhost:8382'); diff --git a/test/Console/Command/SolrSelectBuilderTest.php b/test/Console/Command/SolrSelectBuilderTest.php index 82c9208..c81d423 100644 --- a/test/Console/Command/SolrSelectBuilderTest.php +++ b/test/Console/Command/SolrSelectBuilderTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Search\Console\Command\ResourceBaseLocatorBuilder; use Atoolo\Search\Console\Command\SolrSelectBuilder; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -13,7 +14,9 @@ class SolrSelectBuilderTest extends TestCase { public function testBuild(): void { - $builder = new SolrSelectBuilder(); + $builder = new SolrSelectBuilder( + $this->createStub(ResourceBaseLocatorBuilder::class) + ); $builder->resourceDir('test') ->solrConnectionUrl('http://localhost:8382'); diff --git a/test/Service/Search/SolrMoreLikeThisTest.php b/test/Service/Search/SolrMoreLikeThisTest.php index 01a27f3..882b48a 100644 --- a/test/Service/Search/SolrMoreLikeThisTest.php +++ b/test/Service/Search/SolrMoreLikeThisTest.php @@ -15,8 +15,8 @@ use PHPUnit\Framework\TestCase; use Solarium\Client; use Solarium\QueryType\MoreLikeThis\Query as SolrMoreLikeThisQuery; +use Solarium\QueryType\MoreLikeThis\Result as SolrMoreLikeThisResult; use Solarium\QueryType\Select\Query\FilterQuery; -use Solarium\QueryType\Select\Result\Result as SelectResult; #[CoversClass(SolrMoreLikeThis::class)] class SolrMoreLikeThisTest extends TestCase @@ -39,7 +39,7 @@ protected function setUp(): void $client->method('createMoreLikeThis')->willReturn($query); - $result = $this->createStub(SelectResult::class); + $result = $this->createStub(SolrMoreLikeThisResult::class); $client->method('execute')->willReturn($result); $this->resource = $this->createStub(Resource::class); From f1000ea2a2fc824b545e27b7d8651b747a39338e Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 27 Feb 2024 11:55:37 +0100 Subject: [PATCH 047/145] docs: update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c91cb6..c7e0153 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,6 @@ # Atoolo search -Provides services with which a Solr index can be filled and searched for [resources](https://github.com/sitepark/atoolo-resource) via this index. +Provides services with which a Solr index can be filled and searched for [resources](https://github.com/sitepark/atoolo-resource) via a index. + +[Documentation](https://sitepark.github.io/atoolo-docs/develop/components/search/) From d8530b6847b9009546ab40711778c2165dcccfb5 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 27 Feb 2024 11:56:01 +0100 Subject: [PATCH 048/145] docs: update class documentation for SolrIndexer --- src/Service/Indexer/SolrIndexer.php | 76 +++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/SolrIndexer.php index 72e25ad..84af6c1 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/SolrIndexer.php @@ -15,7 +15,25 @@ use Throwable; /** - * Implementation of the indexer on the basis of a Solr index. + * Implementation of the indexer on the basis of a Solr index. + * + * Resources are loaded via the indexer, mapped to an IndexDocument and + * then transferred to Solr in order to index it. + * + * This is done in several stages: + * + * 1. first, the PHP files containing the resource data are determined via + * the file system. This can be an entire directory tree or just + * individual files. + * 2. resources may have been translated into several languages and are also + * available in translated form as PHP files in the file system. A separate + * Solr index is used for each language. The PHP files are therefore + * assigned to the respective language. + * 3. the resources are loaded separately for each language, mapped to the + * index documents and indexed for the corresponding index. + * 4. for performance reasons, the documents are not indexed individually, + * but always a list of documents. The entire list is divided into chunks + * and indexed chunk-wise. */ class SolrIndexer implements Indexer { @@ -52,6 +70,10 @@ public function abort(string $index): void $this->aborter->abort($index); } + /** + * Indexes an entire directory structure or only selected files + * if `paths` was specified in `$parameter`. + */ public function index(IndexerParameter $parameter): IndexerStatus { if (empty($parameter->paths)) { @@ -65,6 +87,12 @@ public function index(IndexerParameter $parameter): IndexerStatus } /** + * If a path is to be indexed in a translated language, this can also + * be specified via the URL parameter `loc`. For example, + * `/dir/file.php?loc=it_IT` defines that the path + * `/dir/file.php.translations/it_IT.php` is to be used. + * This method translates the URL parameter into the correct path. + * * @param string[] $pathList * @return string[] */ @@ -89,6 +117,8 @@ private function mapTranslationPaths(array $pathList): array } /** + * Indexes the resources of all passed paths. + * * @param array $pathList */ private function indexResources( @@ -107,12 +137,36 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } - $availableIndexes = $this->getAvailableIndexes(); + $splitterResult = $this->translationSplitter->split($pathList); - $processId = uniqid('', true); + $this->indexTranslationSplittedResources( + $parameter, + $availableIndexes, + $splitterResult + ); - $splitterResult = $this->translationSplitter->split($pathList); + $this->indexerProgressHandler->finish(); + + return $this->indexerProgressHandler->getStatus(); + } + + /** + * There is a separate Solr index for each language. This allows + * language-specific tokenizers and other language-relevant configurations + * to be used. Via the `$splitterResult` all paths are separated according + * to their languages and can be indexed separately. Each language is + * indexed separately here. + * + * @param string[] $availableIndexes + */ + private function indexTranslationSplittedResources( + IndexerParameter $parameter, + array $availableIndexes, + TranslationSplitterResult $splitterResult + ): void { + + $processId = uniqid('', true); if (in_array($parameter->index, $availableIndexes)) { $this->indexResourcesPerLanguageIndex( @@ -142,13 +196,11 @@ private function indexResources( )); } } - - $this->indexerProgressHandler->finish(); - - return $this->indexerProgressHandler->getStatus(); } /** + * The resources for a language are indexed here. + * * @param string[] $pathList */ private function indexResourcesPerLanguageIndex( @@ -185,7 +237,13 @@ private function indexResourcesPerLanguageIndex( $this->commit($index); } - /** + /** + * For performance reasons, not every resource is indexed individually, + * but the index documents are first generated from several resources. + * These are then passed to Solr for indexing via a request. These + * methods accept a chunk with all paths that are to be indexed via a + * request. + * * @param string[] $pathList */ private function indexChunks( From 0e927a88d50072dba21808be95ac1b4dd1d85efc Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 27 Feb 2024 14:26:25 +0100 Subject: [PATCH 049/145] fix: remove debug code --- src/Service/Search/SolrSelect.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 87d979e..6787a04 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -171,7 +171,6 @@ private function addFilterQueriesToSolrQuery( foreach ($filterList as $filter) { $key = $filter->key ?? uniqid('', true); - $fq = $solrQuery->createFilterQuery($key); $solrQuery->createFilterQuery($key) ->setQuery($filter->getQuery()) ->setTags($filter->tags); From 6f71e7c9e9e513a19e2858ce6288ee52fb6c9ab6 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 1 Mar 2024 14:07:31 +0100 Subject: [PATCH 050/145] docs: update phpdoc --- src/Service/Indexer/LocationFinder.php | 10 ++++++++++ src/Service/Indexer/SiteKit/ContentMatcher.php | 5 +++++ src/Service/Indexer/TranslationSplitter.php | 4 ++++ src/Service/Search/SiteKit/DefaultBoostModifier.php | 7 +++++++ 4 files changed, 26 insertions(+) diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index 1560216..c03b3de 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -7,6 +7,16 @@ use Atoolo\Resource\ResourceBaseLocator; use Symfony\Component\Finder\Finder; +/** + * The LocationFinder searches recursively for files that represent a + * resource. The corresponding files are + * recognized and returned according to certain rules. + * + * The rules are: + * - files must have the extension `.php + * - files must not be located in the `WEB-IES` directory + * - files must not have the name `.*-1015t.php.*` + */ class LocationFinder { public function __construct( diff --git a/src/Service/Indexer/SiteKit/ContentMatcher.php b/src/Service/Indexer/SiteKit/ContentMatcher.php index 823ddb7..7865aea 100644 --- a/src/Service/Indexer/SiteKit/ContentMatcher.php +++ b/src/Service/Indexer/SiteKit/ContentMatcher.php @@ -4,6 +4,11 @@ namespace Atoolo\Search\Service\Indexer\SiteKit; +/** + * The `ContentMatcher` interface is implemented in order to extract from the + * content structure of resources to extract the content that is relevant for the + * relevant for the `content` field of the search index. + */ interface ContentMatcher { /** diff --git a/src/Service/Indexer/TranslationSplitter.php b/src/Service/Indexer/TranslationSplitter.php index c9b757e..ab88453 100644 --- a/src/Service/Indexer/TranslationSplitter.php +++ b/src/Service/Indexer/TranslationSplitter.php @@ -4,6 +4,10 @@ namespace Atoolo\Search\Service\Indexer; +/** + * The `TranslationSplitter` interface is implemented in order to + * group a list of paths in the respective languages. + */ interface TranslationSplitter { /** diff --git a/src/Service/Search/SiteKit/DefaultBoostModifier.php b/src/Service/Search/SiteKit/DefaultBoostModifier.php index e209bb0..25be65a 100644 --- a/src/Service/Search/SiteKit/DefaultBoostModifier.php +++ b/src/Service/Search/SiteKit/DefaultBoostModifier.php @@ -7,6 +7,13 @@ use Atoolo\Search\Service\Search\SolrQueryModifier; use Solarium\QueryType\Select\Query\Query as SelectQuery; +/** + * Set solr boost definitions for certain fields. + * This influences the relevance of the fields in the search. + * + * The values defined here have been developed through experience in + * many projects. + */ class DefaultBoostModifier implements SolrQueryModifier { public function modify(SelectQuery $query): SelectQuery From 358acfdf8a95c09330d99e9684ad8cc25b2779d6 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 1 Mar 2024 14:09:50 +0100 Subject: [PATCH 051/145] style: fix max line length --- src/Service/Indexer/SiteKit/ContentMatcher.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Service/Indexer/SiteKit/ContentMatcher.php b/src/Service/Indexer/SiteKit/ContentMatcher.php index 7865aea..1d23e68 100644 --- a/src/Service/Indexer/SiteKit/ContentMatcher.php +++ b/src/Service/Indexer/SiteKit/ContentMatcher.php @@ -6,8 +6,8 @@ /** * The `ContentMatcher` interface is implemented in order to extract from the - * content structure of resources to extract the content that is relevant for the - * relevant for the `content` field of the search index. + * content structure of resources to extract the content that is relevant + * for the relevant for the `content` field of the search index. */ interface ContentMatcher { From 441bea305e50316a875e8ca0cec50bfc1eb0bd39 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 1 Mar 2024 15:56:03 +0100 Subject: [PATCH 052/145] refactor: change filter constructor --- src/Console/Command/Suggest.php | 2 +- src/Dto/Search/Query/Filter/AndFilter.php | 2 +- src/Dto/Search/Query/Filter/CategoryFilter.php | 11 +++++++---- .../Search/Query/Filter/ContentSectionTypeFilter.php | 11 +++++++---- src/Dto/Search/Query/Filter/FieldFilter.php | 11 +++++++---- src/Dto/Search/Query/Filter/GroupFilter.php | 11 +++++++---- src/Dto/Search/Query/Filter/NotFilter.php | 2 +- src/Dto/Search/Query/Filter/ObjectTypeFilter.php | 11 +++++++---- src/Dto/Search/Query/Filter/OrFilter.php | 3 ++- src/Dto/Search/Query/Filter/QueryFilter.php | 4 ++-- src/Dto/Search/Query/Filter/SiteFilter.php | 11 +++++++---- src/Service/Search/SolrMoreLikeThis.php | 2 +- test/Dto/Search/Query/Filter/AndFilterTest.php | 2 +- test/Dto/Search/Query/Filter/FieldFilterTest.php | 8 ++++---- test/Dto/Search/Query/Filter/NotFilterTest.php | 2 +- test/Dto/Search/Query/Filter/OrFilterTest.php | 2 +- test/Dto/Search/Query/Filter/QueryFilterTest.php | 2 +- 17 files changed, 58 insertions(+), 39 deletions(-) diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index b273f89..b4c59b6 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -87,7 +87,7 @@ protected function createSearcher(): SolrSuggest */ protected function buildQuery(array $terms): SuggestQuery { - $excludeMedia = new ObjectTypeFilter('media', 'media'); + $excludeMedia = new ObjectTypeFilter(['media'], 'media'); $excludeMedia = $excludeMedia->exclude(); return new SuggestQuery( $this->solrCore, diff --git a/src/Dto/Search/Query/Filter/AndFilter.php b/src/Dto/Search/Query/Filter/AndFilter.php index 5485cce..253a492 100644 --- a/src/Dto/Search/Query/Filter/AndFilter.php +++ b/src/Dto/Search/Query/Filter/AndFilter.php @@ -10,8 +10,8 @@ class AndFilter extends Filter * @param Filter[] $filter */ public function __construct( - ?string $key, private readonly array $filter, + ?string $key = null, array $tags = [] ) { parent::__construct($key, $tags); diff --git a/src/Dto/Search/Query/Filter/CategoryFilter.php b/src/Dto/Search/Query/Filter/CategoryFilter.php index 091c7d5..f49ebb3 100644 --- a/src/Dto/Search/Query/Filter/CategoryFilter.php +++ b/src/Dto/Search/Query/Filter/CategoryFilter.php @@ -9,14 +9,17 @@ */ class CategoryFilter extends FieldFilter { + /** + * @param string[] $category + */ public function __construct( - ?string $key, - string ...$category + array $category, + ?string $key = null ) { parent::__construct( - $key, 'sp_category_path', - ...$category + $category, + $key ); } } diff --git a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php index 22183f9..90b2ae0 100644 --- a/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ContentSectionTypeFilter.php @@ -9,14 +9,17 @@ */ class ContentSectionTypeFilter extends FieldFilter { + /** + * @param string[] $contentTypes + */ public function __construct( - ?string $key, - string ...$contentTypes + array $contentTypes, + ?string $key = null, ) { parent::__construct( - $key, 'sp_contenttype', - ...$contentTypes + $contentTypes, + $key ); } } diff --git a/src/Dto/Search/Query/Filter/FieldFilter.php b/src/Dto/Search/Query/Filter/FieldFilter.php index 9b85e9a..3caa308 100644 --- a/src/Dto/Search/Query/Filter/FieldFilter.php +++ b/src/Dto/Search/Query/Filter/FieldFilter.php @@ -13,10 +13,13 @@ class FieldFilter extends Filter */ private readonly array $values; + /** + * @param string[] $values + */ public function __construct( - ?string $key, private readonly string $field, - string ...$values + array $values, + ?string $key = null ) { if (count($values) === 0) { throw new InvalidArgumentException( @@ -45,9 +48,9 @@ public function exclude(): FieldFilter $field = '-' . $field; } return new FieldFilter( - $this->key, $field, - ...$this->values + $this->values, + $this->key ); } } diff --git a/src/Dto/Search/Query/Filter/GroupFilter.php b/src/Dto/Search/Query/Filter/GroupFilter.php index 5c480f5..5ec4333 100644 --- a/src/Dto/Search/Query/Filter/GroupFilter.php +++ b/src/Dto/Search/Query/Filter/GroupFilter.php @@ -9,14 +9,17 @@ */ class GroupFilter extends FieldFilter { + /** + * @param string[] $group + */ public function __construct( - ?string $key, - string ...$group + array $group, + ?string $key = null, ) { parent::__construct( - $key, 'sp_group_path', - ...$group + $group, + $key ); } } diff --git a/src/Dto/Search/Query/Filter/NotFilter.php b/src/Dto/Search/Query/Filter/NotFilter.php index 381ecfe..22aef3e 100644 --- a/src/Dto/Search/Query/Filter/NotFilter.php +++ b/src/Dto/Search/Query/Filter/NotFilter.php @@ -7,8 +7,8 @@ class NotFilter extends Filter { public function __construct( - ?string $key, private readonly Filter $filter, + ?string $key = null, array $tags = [] ) { parent::__construct($key, $tags); diff --git a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php index 4c60c69..f854b0e 100644 --- a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php +++ b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php @@ -9,14 +9,17 @@ */ class ObjectTypeFilter extends FieldFilter { + /** + * @param string[] $objectTypes + */ public function __construct( - ?string $key, - string ...$objectTypes + array $objectTypes, + ?string $key = null, ) { parent::__construct( - $key, 'sp_objecttype', - ...$objectTypes + $objectTypes, + $key ); } } diff --git a/src/Dto/Search/Query/Filter/OrFilter.php b/src/Dto/Search/Query/Filter/OrFilter.php index 3282fb9..0781f07 100644 --- a/src/Dto/Search/Query/Filter/OrFilter.php +++ b/src/Dto/Search/Query/Filter/OrFilter.php @@ -8,10 +8,11 @@ class OrFilter extends Filter { /** * @param Filter[] $filter + * @param string[] $tags */ public function __construct( - ?string $key, private readonly array $filter, + ?string $key = null, array $tags = [] ) { parent::__construct($key, $tags); diff --git a/src/Dto/Search/Query/Filter/QueryFilter.php b/src/Dto/Search/Query/Filter/QueryFilter.php index 2ff8133..6a25853 100644 --- a/src/Dto/Search/Query/Filter/QueryFilter.php +++ b/src/Dto/Search/Query/Filter/QueryFilter.php @@ -7,8 +7,8 @@ class QueryFilter extends Filter { public function __construct( - ?string $key, - private readonly string $query + private readonly string $query, + ?string $key = null, ) { parent::__construct( $key diff --git a/src/Dto/Search/Query/Filter/SiteFilter.php b/src/Dto/Search/Query/Filter/SiteFilter.php index ffc3bc7..37b786d 100644 --- a/src/Dto/Search/Query/Filter/SiteFilter.php +++ b/src/Dto/Search/Query/Filter/SiteFilter.php @@ -9,14 +9,17 @@ */ class SiteFilter extends FieldFilter { + /** + * @param string[] $site + */ public function __construct( - ?string $key, - string ...$site + array $site, + ?string $key = null, ) { parent::__construct( - $key, 'sp_site', - ...$site + $site, + $key ); } } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index fd694c0..1a14494 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -27,7 +27,7 @@ public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { $client = $this->clientFactory->create($query->index); $solrQuery = $this->buildSolrQuery($client, $query); - /** @var SelectResult $result */ + /** @var SolrMoreLikeThisResult $result */ $result = $client->execute($solrQuery); return $this->buildResult($result); } diff --git a/test/Dto/Search/Query/Filter/AndFilterTest.php b/test/Dto/Search/Query/Filter/AndFilterTest.php index b84e913..dea1659 100644 --- a/test/Dto/Search/Query/Filter/AndFilterTest.php +++ b/test/Dto/Search/Query/Filter/AndFilterTest.php @@ -24,7 +24,7 @@ public function testAndQuery(): void $b->method('getQuery') ->willReturn('b'); - $and = new AndFilter(null, [$a, $b]); + $and = new AndFilter([$a, $b]); assertEquals( '(a AND b)', diff --git a/test/Dto/Search/Query/Filter/FieldFilterTest.php b/test/Dto/Search/Query/Filter/FieldFilterTest.php index df44e46..13d45ae 100644 --- a/test/Dto/Search/Query/Filter/FieldFilterTest.php +++ b/test/Dto/Search/Query/Filter/FieldFilterTest.php @@ -15,12 +15,12 @@ class FieldFilterTest extends TestCase public function testEmptyValues(): void { $this->expectException(InvalidArgumentException::class); - new FieldFilter(null, 'test'); + new FieldFilter('test', []); } public function testGetQueryWithOneField(): void { - $field = new FieldFilter(null, 'test', 'a'); + $field = new FieldFilter('test', ['a']); $this->assertEquals( 'test:a', $field->getQuery(), @@ -30,7 +30,7 @@ public function testGetQueryWithOneField(): void public function testGetQueryWithTwoFields(): void { - $field = new FieldFilter(null, 'test', 'a', 'b'); + $field = new FieldFilter('test', ['a', 'b']); $this->assertEquals( 'test:(a b)', $field->getQuery(), @@ -40,7 +40,7 @@ public function testGetQueryWithTwoFields(): void public function testExclude(): void { - $field = new FieldFilter(null, 'test', 'a'); + $field = new FieldFilter('test', ['a']); $exclude = $field->exclude(); $this->assertEquals( '-test:a', diff --git a/test/Dto/Search/Query/Filter/NotFilterTest.php b/test/Dto/Search/Query/Filter/NotFilterTest.php index 575e7ac..a4bfecb 100644 --- a/test/Dto/Search/Query/Filter/NotFilterTest.php +++ b/test/Dto/Search/Query/Filter/NotFilterTest.php @@ -17,7 +17,7 @@ public function testGetQuery(): void $filter = $this->createStub(Filter::class); $filter->method('getQuery') ->willReturn('a:b'); - $notFilter = new NotFilter(null, $filter); + $notFilter = new NotFilter($filter); $this->assertEquals( 'NOT a:b', diff --git a/test/Dto/Search/Query/Filter/OrFilterTest.php b/test/Dto/Search/Query/Filter/OrFilterTest.php index becc11f..5d29ae5 100644 --- a/test/Dto/Search/Query/Filter/OrFilterTest.php +++ b/test/Dto/Search/Query/Filter/OrFilterTest.php @@ -24,7 +24,7 @@ public function testOrQuery(): void $b->method('getQuery') ->willReturn('b'); - $and = new OrFilter(null, [$a, $b]); + $and = new OrFilter([$a, $b]); assertEquals( '(a OR b)', diff --git a/test/Dto/Search/Query/Filter/QueryFilterTest.php b/test/Dto/Search/Query/Filter/QueryFilterTest.php index d629422..d0a1315 100644 --- a/test/Dto/Search/Query/Filter/QueryFilterTest.php +++ b/test/Dto/Search/Query/Filter/QueryFilterTest.php @@ -13,7 +13,7 @@ class QueryFilterTest extends TestCase { public function testGetQuery(): void { - $filter = new QueryFilter(null, 'a:b'); + $filter = new QueryFilter('a:b'); $this->assertEquals( 'a:b', $filter->getQuery(), From 2920d51336527ea7f3d2bc4ac1e3058553e0d190 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 4 Mar 2024 07:43:52 +0100 Subject: [PATCH 053/145] feat: excludeFilter for facet dto is optional --- src/Dto/Search/Query/Facet/CategoryFacet.php | 2 +- src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php | 2 +- src/Dto/Search/Query/Facet/Facet.php | 2 +- src/Dto/Search/Query/Facet/FacetField.php | 2 +- src/Dto/Search/Query/Facet/FacetMultiQuery.php | 2 +- src/Dto/Search/Query/Facet/FacetQuery.php | 2 +- src/Dto/Search/Query/Facet/GroupFacet.php | 2 +- src/Dto/Search/Query/Facet/ObjectTypeFacet.php | 2 +- src/Dto/Search/Query/Facet/SiteFacet.php | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Dto/Search/Query/Facet/CategoryFacet.php b/src/Dto/Search/Query/Facet/CategoryFacet.php index 8996dd8..cdf1c42 100644 --- a/src/Dto/Search/Query/Facet/CategoryFacet.php +++ b/src/Dto/Search/Query/Facet/CategoryFacet.php @@ -15,7 +15,7 @@ class CategoryFacet extends FacetField public function __construct( string $key, array $categories, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct( $key, diff --git a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php index 04b4e91..a7e37d0 100644 --- a/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ContentSectionTypeFacet.php @@ -15,7 +15,7 @@ class ContentSectionTypeFacet extends FacetField public function __construct( string $key, array $contentSectionTypes, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct( $key, diff --git a/src/Dto/Search/Query/Facet/Facet.php b/src/Dto/Search/Query/Facet/Facet.php index 3e645b5..8a32533 100644 --- a/src/Dto/Search/Query/Facet/Facet.php +++ b/src/Dto/Search/Query/Facet/Facet.php @@ -11,7 +11,7 @@ abstract class Facet { public function __construct( public readonly string $key, - public readonly ?string $excludeFilter + public readonly ?string $excludeFilter = null ) { } } diff --git a/src/Dto/Search/Query/Facet/FacetField.php b/src/Dto/Search/Query/Facet/FacetField.php index 3a1dce8..c1f1725 100644 --- a/src/Dto/Search/Query/Facet/FacetField.php +++ b/src/Dto/Search/Query/Facet/FacetField.php @@ -16,7 +16,7 @@ public function __construct( string $key, public readonly string $field, public readonly array $terms, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct($key, $excludeFilter); } diff --git a/src/Dto/Search/Query/Facet/FacetMultiQuery.php b/src/Dto/Search/Query/Facet/FacetMultiQuery.php index df4e4a9..bd3b693 100644 --- a/src/Dto/Search/Query/Facet/FacetMultiQuery.php +++ b/src/Dto/Search/Query/Facet/FacetMultiQuery.php @@ -16,7 +16,7 @@ class FacetMultiQuery extends Facet public function __construct( string $key, public readonly array $queries, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct($key, $excludeFilter); } diff --git a/src/Dto/Search/Query/Facet/FacetQuery.php b/src/Dto/Search/Query/Facet/FacetQuery.php index 1f27aad..c34ac60 100644 --- a/src/Dto/Search/Query/Facet/FacetQuery.php +++ b/src/Dto/Search/Query/Facet/FacetQuery.php @@ -12,7 +12,7 @@ class FacetQuery extends Facet public function __construct( string $key, public readonly string $query, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct($key, $excludeFilter); } diff --git a/src/Dto/Search/Query/Facet/GroupFacet.php b/src/Dto/Search/Query/Facet/GroupFacet.php index 2ec75cd..1632637 100644 --- a/src/Dto/Search/Query/Facet/GroupFacet.php +++ b/src/Dto/Search/Query/Facet/GroupFacet.php @@ -15,7 +15,7 @@ class GroupFacet extends FacetField public function __construct( string $key, public readonly array $groups, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct( $key, diff --git a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php index 4dede84..869067a 100644 --- a/src/Dto/Search/Query/Facet/ObjectTypeFacet.php +++ b/src/Dto/Search/Query/Facet/ObjectTypeFacet.php @@ -15,7 +15,7 @@ class ObjectTypeFacet extends FacetField public function __construct( string $key, public readonly array $objectTypes, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct( $key, diff --git a/src/Dto/Search/Query/Facet/SiteFacet.php b/src/Dto/Search/Query/Facet/SiteFacet.php index 47290b1..5b57124 100644 --- a/src/Dto/Search/Query/Facet/SiteFacet.php +++ b/src/Dto/Search/Query/Facet/SiteFacet.php @@ -15,7 +15,7 @@ class SiteFacet extends FacetField public function __construct( string $key, public readonly array $sites, - ?string $excludeFilter + ?string $excludeFilter = null ) { parent::__construct( $key, From 2b5aae2acb094983d7a1b9dbd7554dfaffe95d71 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 5 Mar 2024 09:04:35 +0100 Subject: [PATCH 054/145] fix: php warnings --- phpunit.xml | 6 +- src/Service/Indexer/IndexerStatusStore.php | 21 ++++++- test/Console/Command/IndexerTest.php | 59 +++++++++++++++++++ .../ResourceBaseLocatorBuilderTest.php | 4 +- .../Indexer/IndexerStatusStoreTest.php | 32 +++++++++- test/Service/Indexer/IndexingAborterTest.php | 8 ++- .../DefaultSchema2xDocumentEnricherTest.php | 8 +-- 7 files changed, 128 insertions(+), 10 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 1120448..f2d23b0 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -5,7 +5,11 @@ bootstrap="vendor/autoload.php" executionOrder="random" cacheResultFile="var/cache/.phpunit.result.cache" - cacheDirectory="var/cache/.phpunit.cache"> + cacheDirectory="var/cache/.phpunit.cache" + displayDetailsOnTestsThatTriggerDeprecations="true" + displayDetailsOnTestsThatTriggerErrors="true" + displayDetailsOnTestsThatTriggerNotices="true" + displayDetailsOnTestsThatTriggerWarnings="true"> diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index 3fabb1b..dbbbea1 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -31,9 +31,15 @@ public function load(string $index): IndexerStatus return IndexerStatus::empty(); } + if (!is_readable($file)) { + throw new InvalidArgumentException('Cannot read file ' . $file); + } + $json = file_get_contents($file); if ($json === false) { + // @codeCoverageIgnoreStart throw new InvalidArgumentException('Cannot read file ' . $file); + // @codeCoverageIgnoreEnd } /** @var IndexerStatus $status */ @@ -49,14 +55,21 @@ public function store(string $index, IndexerStatus $status): void $this->createBaseDirectory(); $file = $this->getStatusFile($index); + if (file_exists($file) && !is_writable($file)) { + throw new RuntimeException( + 'File ' . $file . ' is not writable' + ); + } $json = $this ->createSerializer() ->serialize($status, 'json'); $result = file_put_contents($file, $json); if ($result === false) { + // @codeCoverageIgnoreStart throw new RuntimeException( 'Unable to write indexer-status file ' . $file ); + // @codeCoverageIgnoreEnd } } @@ -64,7 +77,7 @@ private function createBaseDirectory(): void { if ( !is_dir($concurrentDirectory = $this->basedir) && - !mkdir($concurrentDirectory) && + !@mkdir($concurrentDirectory) && !is_dir($concurrentDirectory) ) { throw new RuntimeException(sprintf( @@ -72,6 +85,12 @@ private function createBaseDirectory(): void $concurrentDirectory )); } + + if (!is_writable($this->basedir)) { + throw new RuntimeException( + 'Directory ' . $this->basedir . ' is not writable' + ); + } } private function createSerializer(): Serializer diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 8f17514..695d7c2 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -12,6 +12,7 @@ use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; +use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; #[CoversClass(Indexer::class)] @@ -150,4 +151,62 @@ public function testExecuteIndexWithErrors(): void 'error message expected' ); } + + + /** + * @throws Exception + */ + public function testExecuteIndexWithErrorsAndStackTrace(): void + { + + $indexBuilder = $this->createStub( + SolrIndexerBuilder::class + ); + + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $progressBarFactory = $this->createStub( + IndexerProgressBarFactory::class + ); + $progressBarFactory + ->method('create') + ->willReturn($progressBar); + + $indexer = new Indexer( + [], + $indexBuilder, + $progressBarFactory + ); + + $application = new Application([ + $indexer + ]); + + $command = $application->find('atoolo:indexer'); + $commandTester = new CommandTester($command); + + $commandTester->execute([ + // pass arguments to the helper + 'resource-dir' => 'abc', + 'solr-connection-url' => 'http://localhost:8080', + 'solr-core' => 'test' + ], [ + 'verbosity' => OutputInterface::VERBOSITY_VERBOSE + ]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'Exception trace', + $output, + 'error message should contains stack trace' + ); + } } diff --git a/test/Console/Command/ResourceBaseLocatorBuilderTest.php b/test/Console/Command/ResourceBaseLocatorBuilderTest.php index 7c3337b..1950f32 100644 --- a/test/Console/Command/ResourceBaseLocatorBuilderTest.php +++ b/test/Console/Command/ResourceBaseLocatorBuilderTest.php @@ -17,7 +17,9 @@ public function testBuild(): void $resourceDir = __DIR__ . '/../../../var/test/ResourceBaseLocatorBuilderTest'; $objectsDir = $resourceDir . '/objects'; - mkdir($objectsDir, 0777, true); + if (!is_dir($objectsDir)) { + mkdir($objectsDir, 0777, true); + } $builder = new ResourceBaseLocatorBuilder(); $locator = $builder->build($resourceDir); diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php index ce6e0ef..7fd2bba 100644 --- a/test/Service/Indexer/IndexerStatusStoreTest.php +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -71,7 +71,7 @@ public function testStoreWithNonExistsBaseDir(): void ); } - public function testStoreWithNonWritableStatusFile(): void + public function testStoreWithNonWritableBaseDir(): void { $status = $this->createIndexerStatus(); $baseDir = self::TEST_DIR . '/non-writable'; @@ -86,6 +86,36 @@ public function testStoreWithNonWritableStatusFile(): void $store->store('test', $status); } + public function testStoreWithNonWritableStatusFile(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($baseDir); + + $file = $baseDir . '/atoolo.search.index.test-not-writable.status.json'; + touch($file); + $filesystem->chmod($file, 0000); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test-not-writable', $status); + } + + public function testStoreWithBaseDirNoADirectory(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/non-dir'; + touch($baseDir); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + public function testStoreWithNonCreatableBaseDir(): void { $status = $this->createIndexerStatus(); diff --git a/test/Service/Indexer/IndexingAborterTest.php b/test/Service/Indexer/IndexingAborterTest.php index 9b46c08..3bad523 100644 --- a/test/Service/Indexer/IndexingAborterTest.php +++ b/test/Service/Indexer/IndexingAborterTest.php @@ -18,9 +18,13 @@ public function setUp(): void { $workdir = __DIR__ . '/../../../var/test/IndexingAborterTest'; - mkdir($workdir, 0777, true); + if (!is_dir($workdir)) { + mkdir($workdir, 0777, true); + } $this->file = $workdir . '/background-indexer-test.abort'; - unlink($this->file); + if (file_exists($this->file)) { + unlink($this->file); + } $this->aborter = new IndexingAborter($workdir); } diff --git a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php index 522e71a..8bfddc7 100644 --- a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php +++ b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php @@ -254,8 +254,8 @@ public function testEnrichLanguageOverGroupPath(): void { $doc = $this->enrichWithData(['init' => [ 'groupPath' => [ - ['locale' => 'fr_FR'], - ['locale' => 'it_IT'] + ['id' => 1, 'locale' => 'fr_FR'], + ['id' => 2, 'locale' => 'it_IT'] ] ]]); $this->assertEquals( @@ -693,8 +693,8 @@ public function testEnrichContent(): void $doc = $this->enrichWithData([ 'metadata' => [ 'categories' => [ - ['name' => 'CategoryA'], - ['name' => 'CategoryB'] + ['id' => 1, 'name' => 'CategoryA'], + ['id' => 2, 'name' => 'CategoryB'] ] ], 'searchindexdata' => ['content' => 'abc'] From b32ddec9b2ed66585141c8edcaae1b89d3964b7d Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 7 Mar 2024 14:06:36 +0100 Subject: [PATCH 055/145] fix : IndexDocumentDumperBuilder must use ResourceBaseLocatorBuilder --- src/Console/Command/IndexDocumentDumperBuilder.php | 9 +++++++-- test/Console/Command/IndexDocumentDumperBuilderTest.php | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Console/Command/IndexDocumentDumperBuilder.php b/src/Console/Command/IndexDocumentDumperBuilder.php index 36edd70..bb8e3b2 100644 --- a/src/Console/Command/IndexDocumentDumperBuilder.php +++ b/src/Console/Command/IndexDocumentDumperBuilder.php @@ -5,7 +5,6 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; -use Atoolo\Resource\Loader\StaticResourceBaseLocator; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexDocumentDumper; @@ -19,6 +18,11 @@ class IndexDocumentDumperBuilder */ private iterable $documentEnricherList; + public function __construct( + private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder + ) { + } + public function resourceDir(string $resourceDir): IndexDocumentDumperBuilder { $this->resourceDir = $resourceDir; @@ -38,9 +42,10 @@ public function documentEnricherList( public function build(): IndexDocumentDumper { - $resourceBaseLocator = new StaticResourceBaseLocator( + $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( $this->resourceDir ); + $resourceLoader = new SiteKitLoader($resourceBaseLocator); return new IndexDocumentDumper( diff --git a/test/Console/Command/IndexDocumentDumperBuilderTest.php b/test/Console/Command/IndexDocumentDumperBuilderTest.php index 1636f4f..c15fb84 100644 --- a/test/Console/Command/IndexDocumentDumperBuilderTest.php +++ b/test/Console/Command/IndexDocumentDumperBuilderTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Console\Command; use Atoolo\Search\Console\Command\IndexDocumentDumperBuilder; +use Atoolo\Search\Console\Command\ResourceBaseLocatorBuilder; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -13,7 +14,9 @@ class IndexDocumentDumperBuilderTest extends TestCase { public function testBuild(): void { - $builder = new IndexDocumentDumperBuilder(); + $builder = new IndexDocumentDumperBuilder( + $this->createStub(ResourceBaseLocatorBuilder::class) + ); $builder->resourceDir('test'); $builder->documentEnricherList([]); From 5f30c8b6633e33fcab3a034f9ce2b6c5908df0eb Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 7 Mar 2024 14:08:04 +0100 Subject: [PATCH 056/145] feat: IndexSchema2xDocument::setMetaString must also accept arrays --- src/Service/Indexer/IndexSchema2xDocument.php | 7 ++++++- .../Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index e2a152d..bb86837 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -125,7 +125,12 @@ class IndexSchema2xDocument extends Document implements IndexDocument */ private array $metaString = []; - public function setMetaString(string $name, string $value): void + /** + * @param string $name + * @param string|string[] $value + * @return void + */ + public function setMetaString(string $name, string|array $value): void { $this->metaString[$name] = $value; } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index e200240..f4d2a71 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -377,8 +377,8 @@ private function idWithoutSignature(string $id): int * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/Expose.php#L38 * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 - * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Controller/Environment.php#L76 * - https://gitlab.sitepark.com/ies-modules/sitekit-real-estate/blob/develop/src/publish/php/SP/RealEstate/Component/Expose.php#L47 + * - https://gitlab.sitepark.com/sitekit/xzufi-php/-/blob/develop/php/SP/Xzufi/Renderer/SolrData.php?ref_type=heads#L78 */ private function getLocaleFromResource(Resource $resource): string From c1b2473f5612070cd8cae648e6b41e7d4a14280a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 7 Mar 2024 16:44:54 +0100 Subject: [PATCH 057/145] feat: add IndexSchema2xDocument::setMetaBool() --- src/Service/Indexer/IndexSchema2xDocument.php | 21 +++++++++++++++++-- .../Indexer/IndexSchema2xDocumentTest.php | 12 +++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index bb86837..95ec62a 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -121,10 +121,15 @@ class IndexSchema2xDocument extends Document implements IndexDocument public ?string $sp_citygov_function; /** - * @var array + * @var array */ private array $metaString = []; + /** + * @var array + */ + private array $metaBool = []; + /** * @param string $name * @param string|string[] $value @@ -135,6 +140,11 @@ public function setMetaString(string $name, string|array $value): void $this->metaString[$name] = $value; } + public function setMetaBool(string $name, bool $value): void + { + $this->metaBool[$name] = $value; + } + /** * @return array */ @@ -161,7 +171,10 @@ static function ($value, $key) { return false; } - if ($key === 'metaString') { + if ( + $key === 'metaString' || + $key === 'metaBool' + ) { return false; } return true; @@ -172,6 +185,10 @@ static function ($value, $key) { $fields['sp_meta_string_' . $key] = $value; } + foreach ($this->metaBool as $key => $value) { + $fields['sp_meta_bool_' . $key] = $value; + } + return $fields; } } diff --git a/test/Service/Indexer/IndexSchema2xDocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php index a5a5282..dbbfbf0 100644 --- a/test/Service/Indexer/IndexSchema2xDocumentTest.php +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -34,4 +34,16 @@ public function testSetMetaString(): void 'unexpected meta fields' ); } + + public function testSetMetaBool(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaBool('myname', true); + + $this->assertEquals( + ['sp_meta_bool_myname' => true], + $doc->getFields(), + 'unexpected meta fields' + ); + } } From 299485a70bf6fae6dcb14671bb2e89dea3cbdc23 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 12 Mar 2024 16:45:20 +0100 Subject: [PATCH 058/145] refactor: extract SolrIndexService --- src/Console/Command/Indexer.php | 2 +- ...php => InternalResourceIndexerBuilder.php} | 24 ++-- src/Service/Indexer/BackgroundIndexer.php | 4 +- src/Service/Indexer/IndexDocumentDumper.php | 4 +- ...ndexer.php => InternalResourceIndexer.php} | 92 +++--------- ...php => InternalResourceIndexerFactory.php} | 11 +- .../DefaultSchema2xDocumentEnricher.php | 1 - src/Service/Indexer/SolrIndexService.php | 88 ++++++++++++ src/Service/Indexer/SolrIndexUpdater.php | 43 ++++++ test/Console/ApplicationTest.php | 4 +- test/Console/Command/IndexerTest.php | 8 +- .../Command/SolrIndexerBuilderTest.php | 6 +- .../Service/Indexer/BackgroundIndexerTest.php | 12 +- ...=> InternalResourceIndexerFactoryTest.php} | 10 +- ...st.php => InternalResourceIndexerTest.php} | 136 +++++++----------- test/Service/Indexer/SolrIndexServiceTest.php | 72 ++++++++++ test/Service/Indexer/SolrIndexUpdaterTest.php | 52 +++++++ 17 files changed, 364 insertions(+), 205 deletions(-) rename src/Console/Command/{SolrIndexerBuilder.php => InternalResourceIndexerBuilder.php} (86%) rename src/Service/Indexer/{SolrIndexer.php => InternalResourceIndexer.php} (81%) rename src/Service/Indexer/{SolrIndexerFactory.php => InternalResourceIndexerFactory.php} (80%) create mode 100644 src/Service/Indexer/SolrIndexService.php create mode 100644 src/Service/Indexer/SolrIndexUpdater.php rename test/Service/Indexer/{SolrIndexerFactoryTest.php => InternalResourceIndexerFactoryTest.php} (74%) rename test/Service/Indexer/{SolrIndexerTest.php => InternalResourceIndexerTest.php} (72%) create mode 100644 test/Service/Indexer/SolrIndexServiceTest.php create mode 100644 test/Service/Indexer/SolrIndexUpdaterTest.php diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 40d93a3..e25dedc 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -33,7 +33,7 @@ class Indexer extends Command */ public function __construct( private readonly iterable $documentEnricherList, - private readonly SolrIndexerBuilder $solrIndexerBuilder, + private readonly InternalResourceIndexerBuilder $solrIndexerBuilder, private readonly IndexerProgressBarFactory $progressBarFactory ) { parent::__construct(); diff --git a/src/Console/Command/SolrIndexerBuilder.php b/src/Console/Command/InternalResourceIndexerBuilder.php similarity index 86% rename from src/Console/Command/SolrIndexerBuilder.php rename to src/Console/Command/InternalResourceIndexerBuilder.php index c09a238..302adc1 100644 --- a/src/Console/Command/SolrIndexerBuilder.php +++ b/src/Console/Command/InternalResourceIndexerBuilder.php @@ -11,16 +11,17 @@ use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexDocument; use Atoolo\Search\Service\Indexer\IndexingAborter; +use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Atoolo\Search\Service\Indexer\LocationFinder; use Atoolo\Search\Service\Indexer\SiteKit\ContentMatcher; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; use Atoolo\Search\Service\Indexer\SiteKit\HeadlineMatcher; use Atoolo\Search\Service\Indexer\SiteKit\RichtTextMatcher; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; -use Atoolo\Search\Service\Indexer\SolrIndexer; +use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\SolrParameterClientFactory; -class SolrIndexerBuilder +class InternalResourceIndexerBuilder { private string $resourceDir; /** @@ -35,8 +36,9 @@ public function __construct( private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder ) { } - public function resourceDir(string $resourceDir): SolrIndexerBuilder - { + public function resourceDir( + string $resourceDir + ): InternalResourceIndexerBuilder { $this->resourceDir = $resourceDir; return $this; } @@ -47,26 +49,26 @@ public function resourceDir(string $resourceDir): SolrIndexerBuilder */ public function documentEnricherList( iterable $documentEnricherList - ): SolrIndexerBuilder { + ): InternalResourceIndexerBuilder { $this->documentEnricherList = $documentEnricherList; return $this; } public function progressBar( IndexerProgressBar $progressBar - ): SolrIndexerBuilder { + ): InternalResourceIndexerBuilder { $this->progressBar = $progressBar; return $this; } public function solrConnectionUrl( string $solrConnectionUrl - ): SolrIndexerBuilder { + ): InternalResourceIndexerBuilder { $this->solrConnectionUrl = $solrConnectionUrl; return $this; } - public function build(): SolrIndexer + public function build(): InternalResourceIndexer { $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( $this->resourceDir @@ -106,17 +108,19 @@ public function build(): SolrIndexer 0 ); + $solrIndexService = new SolrIndexService($clientFactory); + $translationSplitter = new SubDirTranslationSplitter(); $aborter = new IndexingAborter('.'); - return new SolrIndexer( + return new InternalResourceIndexer( $documentEnricherList, $this->progressBar, $finder, $resourceLoader, $translationSplitter, - $clientFactory, + $solrIndexService, $aborter, 'internal' ); diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 7de602d..b148103 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -16,7 +16,7 @@ class BackgroundIndexer implements Indexer { public function __construct( - private readonly SolrIndexerFactory $indexerFactory, + private readonly InternalResourceIndexerFactory $indexerFactory, private readonly IndexerStatusStore $statusStore, private readonly LoggerInterface $logger = new NullLogger(), private readonly LockFactory $lockFactory = new LockFactory( @@ -59,7 +59,7 @@ public function getStatus(string $index): IndexerStatus return $this->statusStore->load($index); } - private function getIndexer(string $index): SolrIndexer + private function getIndexer(string $index): InternalResourceIndexer { $progressHandler = new BackgroundIndexerProgressState( $index, diff --git a/src/Service/Indexer/IndexDocumentDumper.php b/src/Service/Indexer/IndexDocumentDumper.php index ff695c1..dd51daf 100644 --- a/src/Service/Indexer/IndexDocumentDumper.php +++ b/src/Service/Indexer/IndexDocumentDumper.php @@ -19,7 +19,9 @@ public function __construct( /** * @param string[] $paths - * @return array>. + * @return array> + * Returns the raw array data of the documents to be able to + * output them as JSON, for example. */ public function dump(array $paths): array { diff --git a/src/Service/Indexer/SolrIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php similarity index 81% rename from src/Service/Indexer/SolrIndexer.php rename to src/Service/Indexer/InternalResourceIndexer.php index 84af6c1..025a165 100644 --- a/src/Service/Indexer/SolrIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -9,7 +9,6 @@ use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; -use Atoolo\Search\Service\SolrClientFactory; use Exception; use Solarium\QueryType\Update\Result as UpdateResult; use Throwable; @@ -35,7 +34,7 @@ * but always a list of documents. The entire list is divided into chunks * and indexed chunk-wise. */ -class SolrIndexer implements Indexer +class InternalResourceIndexer implements Indexer { /** * @param iterable> $documentEnricherList @@ -46,7 +45,7 @@ public function __construct( private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, private readonly TranslationSplitter $translationSplitter, - private readonly SolrClientFactory $clientFactory, + private readonly SolrIndexService $indexService, private readonly IndexingAborter $aborter, private readonly string $source ) { @@ -61,8 +60,8 @@ public function remove(string $index, array $idList): void return; } - $this->deleteByIdList($index, $idList); - $this->commit($index); + $this->indexService->deleteByIdList($index, $this->source, $idList); + $this->indexService->commit($index); } public function abort(string $index): void @@ -137,7 +136,7 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } - $availableIndexes = $this->getAvailableIndexes(); + $availableIndexes = $this->indexService->getAvailableIndexes(); $splitterResult = $this->translationSplitter->split($pathList); $this->indexTranslationSplittedResources( @@ -232,9 +231,13 @@ private function indexResourcesPerLanguageIndex( $parameter->cleanupThreshold > 0 && $successCount >= $parameter->cleanupThreshold ) { - $this->deleteByProcessId($index, $processId); + $this->indexService->deleteExcludingProcessId( + $index, + $this->source, + $processId + ); } - $this->commit($index); + $this->indexService->commit($index); } /** @@ -322,12 +325,9 @@ private function add( string $processId, array $resources ): UpdateResult { - $client = $this->clientFactory->create($solrCore); - $update = $client->createUpdate(); - $update->setDocumentClass(IndexSchema2xDocument::class); + $updater = $this->indexService->updater($solrCore); - $documents = []; foreach ($resources as $resource) { foreach ($this->documentEnricherList as $enricher) { if (!$enricher->isIndexable($resource)) { @@ -337,7 +337,7 @@ private function add( } try { /** @var IndexSchema2xDocument $doc */ - $doc = $update->createDocument(); + $doc = $updater->createDocument(); foreach ($this->documentEnricherList as $enricher) { $doc = $enricher->enrichDocument( $resource, @@ -345,81 +345,21 @@ private function add( $processId ); } - $documents[] = $doc; + $updater->addDocument($doc); } catch (Throwable $e) { $this->indexerProgressHandler->error($e); } } - // add the documents and a commit command to the update query - $update->addDocuments($documents); // this executes the query and returns the result - return $client->update($update); - } - - private function deleteByProcessId(string $core, string $processId): void - { - $this->deleteByQuery( - $core, - '-crawl_process_id:' . $processId . ' AND ' . - ' sp_source:' . $this->source - ); - } - - /** - * @param string[] $idList - */ - private function deleteByIdList(string $core, array $idList): void - { - $this->deleteByQuery( - $core, - 'sp_id:(' . implode(' ', $idList) . ') AND ' . - 'sp_source:' . $this->source - ); + return $updater->update(); } private function deleteErrorProtocol(string $core): void { - $this->deleteByQuery( + $this->indexService->deleteByQuery( $core, 'crawl_status:error OR crawl_status:warning' ); } - - private function deleteByQuery(string $core, string $query): void - { - $client = $this->clientFactory->create($core); - $update = $client->createUpdate(); - $update->addDeleteQuery($query); - $client->update($update); - } - - private function commit(string $core): void - { - $client = $this->clientFactory->create($core); - $update = $client->createUpdate(); - $update->addCommit(); - $update->addOptimize(); - $client->update($update); - } - - /** - * @return string[] - */ - private function getAvailableIndexes(): array - { - $client = $this->clientFactory->create(''); - $coreAdminQuery = $client->createCoreAdmin(); - $statusAction = $coreAdminQuery->createStatus(); - $coreAdminQuery->setAction($statusAction); - - $availableIndexes = []; - $response = $client->coreAdmin($coreAdminQuery); - $statusResults = $response->getStatusResults() ?? []; - foreach ($statusResults as $statusResult) { - $availableIndexes[] = $statusResult->getCoreName(); - } - - return $availableIndexes; - } } diff --git a/src/Service/Indexer/SolrIndexerFactory.php b/src/Service/Indexer/InternalResourceIndexerFactory.php similarity index 80% rename from src/Service/Indexer/SolrIndexerFactory.php rename to src/Service/Indexer/InternalResourceIndexerFactory.php index 2a7caf6..708d1e1 100644 --- a/src/Service/Indexer/SolrIndexerFactory.php +++ b/src/Service/Indexer/InternalResourceIndexerFactory.php @@ -5,9 +5,8 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\ResourceLoader; -use Atoolo\Search\Service\SolrClientFactory; -class SolrIndexerFactory +class InternalResourceIndexerFactory { /** * @param iterable> $documentEnricherList @@ -17,7 +16,7 @@ public function __construct( private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, private readonly TranslationSplitter $translationSplitter, - private readonly SolrClientFactory $clientFactory, + private readonly SolrIndexService $solrService, private readonly IndexingAborter $aborter, private readonly string $source ) { @@ -25,14 +24,14 @@ public function __construct( public function create( IndexerProgressHandler $progressHandler - ): SolrIndexer { - return new SolrIndexer( + ): InternalResourceIndexer { + return new InternalResourceIndexer( $this->documentEnricherList, $progressHandler, $this->finder, $this->resourceLoader, $this->translationSplitter, - $this->clientFactory, + $this->solrService, $this->aborter, $this->source ); diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index f4d2a71..84c89d7 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -378,7 +378,6 @@ private function idWithoutSignature(string $id): int * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 * - https://gitlab.sitepark.com/ies-modules/sitekit-real-estate/blob/develop/src/publish/php/SP/RealEstate/Component/Expose.php#L47 - * - https://gitlab.sitepark.com/sitekit/xzufi-php/-/blob/develop/php/SP/Xzufi/Renderer/SolrData.php?ref_type=heads#L78 */ private function getLocaleFromResource(Resource $resource): string diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php new file mode 100644 index 0000000..30fe789 --- /dev/null +++ b/src/Service/Indexer/SolrIndexService.php @@ -0,0 +1,88 @@ +clientFactory->create($core); + $update = $client->createUpdate(); + $update->setDocumentClass(IndexSchema2xDocument::class); + + return new SolrIndexUpdater($client, $update); + } + + public function deleteExcludingProcessId( + string $core, + string $source, + string $processId + ): void { + $this->deleteByQuery( + $core, + '-crawl_process_id:' . $processId . ' AND ' . + ' sp_source:' . $source + ); + } + + /** + * @param string[] $idList + */ + public function deleteByIdList( + string $core, + string $source, + array $idList + ): void { + $this->deleteByQuery( + $core, + 'sp_id:(' . implode(' ', $idList) . ') AND ' . + 'sp_source:' . $source + ); + } + + public function deleteByQuery(string $core, string $query): void + { + $client = $this->clientFactory->create($core); + $update = $client->createUpdate(); + $update->addDeleteQuery($query); + $client->update($update); + } + + public function commit(string $core): void + { + $client = $this->clientFactory->create($core); + $update = $client->createUpdate(); + $update->addCommit(); + $update->addOptimize(); + $client->update($update); + } + + /** + * @return string[] + */ + public function getAvailableIndexes(): array + { + $client = $this->clientFactory->create(''); + $coreAdminQuery = $client->createCoreAdmin(); + $statusAction = $coreAdminQuery->createStatus(); + $coreAdminQuery->setAction($statusAction); + + $availableIndexes = []; + $response = $client->coreAdmin($coreAdminQuery); + $statusResults = $response->getStatusResults() ?? []; + foreach ($statusResults as $statusResult) { + $availableIndexes[] = $statusResult->getCoreName(); + } + + return $availableIndexes; + } +} diff --git a/src/Service/Indexer/SolrIndexUpdater.php b/src/Service/Indexer/SolrIndexUpdater.php new file mode 100644 index 0000000..1a1e0fb --- /dev/null +++ b/src/Service/Indexer/SolrIndexUpdater.php @@ -0,0 +1,43 @@ +update->createDocument(); + return $doc; + } + + public function addDocument(Document $document): void + { + $this->documents[] = $document; + } + + public function update(): UpdateResult + { + $this->update->addDocuments($this->documents); + $this->documents = []; + return $this->client->update($this->update); + } +} diff --git a/test/Console/ApplicationTest.php b/test/Console/ApplicationTest.php index 816a968..ab5efb1 100644 --- a/test/Console/ApplicationTest.php +++ b/test/Console/ApplicationTest.php @@ -6,8 +6,8 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; -use Atoolo\Search\Console\Command\SolrIndexerBuilder; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; @@ -21,7 +21,7 @@ class ApplicationTest extends TestCase public function testConstruct(): void { $indexBuilder = $this->createStub( - SolrIndexerBuilder::class + InternalResourceIndexerBuilder::class ); $application = new Application([ new Indexer([], $indexBuilder, new IndexerProgressBarFactory()) diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 695d7c2..502fc82 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -6,9 +6,9 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; +use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; -use Atoolo\Search\Console\Command\SolrIndexerBuilder; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; @@ -26,7 +26,7 @@ class IndexerTest extends TestCase public function setUp(): void { $indexBuilder = $this->createStub( - SolrIndexerBuilder::class + InternalResourceIndexerBuilder::class ); $indexer = new Indexer( @@ -104,7 +104,7 @@ public function testExecuteIndexWithErrors(): void { $indexBuilder = $this->createStub( - SolrIndexerBuilder::class + InternalResourceIndexerBuilder::class ); $progressBar = $this->createStub( @@ -160,7 +160,7 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void { $indexBuilder = $this->createStub( - SolrIndexerBuilder::class + InternalResourceIndexerBuilder::class ); $progressBar = $this->createStub( diff --git a/test/Console/Command/SolrIndexerBuilderTest.php b/test/Console/Command/SolrIndexerBuilderTest.php index d0f74db..2c6b122 100644 --- a/test/Console/Command/SolrIndexerBuilderTest.php +++ b/test/Console/Command/SolrIndexerBuilderTest.php @@ -4,20 +4,20 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Console\Command\ResourceBaseLocatorBuilder; -use Atoolo\Search\Console\Command\SolrIndexerBuilder; use Atoolo\Search\Service\Indexer\DocumentEnricher; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; -#[CoversClass(SolrIndexerBuilder::class)] +#[CoversClass(InternalResourceIndexerBuilder::class)] class SolrIndexerBuilderTest extends TestCase { public function testBuild(): void { - $builder = new SolrIndexerBuilder( + $builder = new InternalResourceIndexerBuilder( $this->createStub(ResourceBaseLocatorBuilder::class) ); $builder diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index 31edd8a..27d3517 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -7,8 +7,8 @@ use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\BackgroundIndexer; use Atoolo\Search\Service\Indexer\IndexerStatusStore; -use Atoolo\Search\Service\Indexer\SolrIndexer; -use Atoolo\Search\Service\Indexer\SolrIndexerFactory; +use Atoolo\Search\Service\Indexer\InternalResourceIndexer; +use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -19,14 +19,14 @@ #[CoversClass(BackgroundIndexer::class)] class BackgroundIndexerTest extends TestCase { - private SolrIndexer&MockObject $solrIndexer; + private InternalResourceIndexer&MockObject $solrIndexer; private IndexerStatusStore&MockObject $statusStore; private BackgroundIndexer $indexer; public function setUp(): void { - $this->solrIndexer = $this->createMock(SolrIndexer::class); - $solrIndexerFactory = $this->createStub(SolrIndexerFactory::class); + $this->solrIndexer = $this->createMock(InternalResourceIndexer::class); + $solrIndexerFactory = $this->createStub(InternalResourceIndexerFactory::class); $solrIndexerFactory->method('create') ->willReturn($this->solrIndexer); $this->statusStore = $this->createMock(IndexerStatusStore::class); @@ -62,7 +62,7 @@ public function testIndexIfLocked(): void { $lockFactory = $this->createStub(LockFactory::class); $indexer = new BackgroundIndexer( - $this->createStub(SolrIndexerFactory::class), + $this->createStub(InternalResourceIndexerFactory::class), $this->statusStore, new NullLogger(), $lockFactory diff --git a/test/Service/Indexer/SolrIndexerFactoryTest.php b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php similarity index 74% rename from test/Service/Indexer/SolrIndexerFactoryTest.php rename to test/Service/Indexer/InternalResourceIndexerFactoryTest.php index c7b2ea9..95e9561 100644 --- a/test/Service/Indexer/SolrIndexerFactoryTest.php +++ b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php @@ -7,22 +7,22 @@ use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; +use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; use Atoolo\Search\Service\Indexer\LocationFinder; -use Atoolo\Search\Service\Indexer\SolrIndexerFactory; +use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\Indexer\TranslationSplitter; -use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\TestCase; -class SolrIndexerFactoryTest extends TestCase +class InternalResourceIndexerFactoryTest extends TestCase { public function testCreate(): void { - $factory = new SolrIndexerFactory( + $factory = new InternalResourceIndexerFactory( [], $this->createStub(LocationFinder::class), $this->createStub(ResourceLoader::class), $this->createStub(TranslationSplitter::class), - $this->createStub(SolrClientFactory::class), + $this->createStub(SolrIndexService::class), $this->createStub(IndexingAborter::class), '' ); diff --git a/test/Service/Indexer/SolrIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php similarity index 72% rename from test/Service/Indexer/SolrIndexerTest.php rename to test/Service/Indexer/InternalResourceIndexerTest.php index a7a6bd3..c143fa6 100644 --- a/test/Service/Indexer/SolrIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -10,24 +10,21 @@ use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; +use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Atoolo\Search\Service\Indexer\LocationFinder; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; -use Atoolo\Search\Service\Indexer\SolrIndexer; +use Atoolo\Search\Service\Indexer\SolrIndexService; +use Atoolo\Search\Service\Indexer\SolrIndexUpdater; use Atoolo\Search\Service\Indexer\TranslationSplitter; -use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; -use Solarium\Client; -use Solarium\QueryType\Server\CoreAdmin\Result\Result as CoreAdminResult; -use Solarium\QueryType\Server\CoreAdmin\Result\StatusResult; -use Solarium\QueryType\Update\Query\Query as UpdateQuery; use Solarium\QueryType\Update\Result as UpdateResult; -#[CoversClass(SolrIndexer::class)] -class SolrIndexerTest extends TestCase +#[CoversClass(InternalResourceIndexer::class)] +class InternalResourceIndexerTest extends TestCase { private array $availableIndexes = ['test', 'test-en_US']; @@ -35,17 +32,17 @@ class SolrIndexerTest extends TestCase private IndexerProgressHandler&MockObject $indexerProgressHandler; - private SolrIndexer $indexer; + private InternalResourceIndexer $indexer; - private Client $solrClient; + private SolrIndexService $solrIndexService; - private LocationFinder $finder; + private LocationFinder&MockObject $finder; - private UpdateQuery $updateQuery; + private SolrIndexUpdater&MockObject $updater; - private UpdateResult $updateResult; + private UpdateResult&Stub $updateResult; - private IndexingAborter&Stub $aborter; + private IndexingAborter&MockObject $aborter; private DocumentEnricher&MockObject $documentEnricher; @@ -61,6 +58,11 @@ public function setUp(): void ); $this->finder = $this->createMock(LocationFinder::class); $this->documentEnricher = $this->createMock(DocumentEnricher::class); + $this->documentEnricher + ->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + return $doc; + }); $this->translationSplitter = new SubDirTranslationSplitter(); $this->resourceLoader = $this->createStub(ResourceLoader::class); $this->resourceLoader->method('load') @@ -70,41 +72,28 @@ public function setUp(): void ->willReturn($path); return $resource; }); - $solrClientFactory = $this->createStub(SolrClientFactory::class); + $this->solrIndexService = $this->createMock(SolrIndexService::class); $this->updateResult = $this->createStub(UpdateResult::class); - $this->solrClient = $this->createMock(Client::class); - $this->updateQuery = $this->createMock(UpdateQuery::class); - $this->updateQuery->method('createDocument') - ->willReturn($this->createStub(IndexSchema2xDocument::class)); - $coreAdminResult = $this->createStub(CoreAdminResult::class); - $coreAdminResult->method('getStatusResults') + $this->updater = $this->createMock(SolrIndexUpdater::class); + $this->updater->method('update')->willReturn($this->updateResult); + $this->updater->method('createDocument')->willReturn( + new IndexSchema2xDocument() + ); + $this->solrIndexService->method('getAvailableIndexes') ->willReturnCallback(function () { - $results = []; - foreach ($this->availableIndexes as $index) { - $result = $this->createStub(StatusResult::class); - $result->method('getCoreName') - ->willReturn($index); - $results[] = $result; - } - return $results; + return $this->availableIndexes; }); - $this->solrClient->method('createUpdate') - ->willReturn($this->updateQuery); - $this->solrClient->method('update') - ->willReturn($this->updateResult); - $this->solrClient->method('coreAdmin') - ->willReturn($coreAdminResult); - $solrClientFactory->method('create') - ->willReturn($this->solrClient); + $this->solrIndexService->method('updater') + ->willReturn($this->updater); $this->aborter = $this->createMock(IndexingAborter::class); - $this->indexer = new SolrIndexer( + $this->indexer = new InternalResourceIndexer( [ $this->documentEnricher ], $this->indexerProgressHandler, $this->finder, $this->resourceLoader, $this->translationSplitter, - $solrClientFactory, + $this->solrIndexService, $this->aborter, 'test' ); @@ -121,16 +110,16 @@ public function testAbort(): void public function testRemove(): void { - $this->solrClient->expects($this->exactly(2)) - ->method('update'); + $this->solrIndexService->expects($this->once()) + ->method('deleteByIdList'); $this->indexer->remove('test', ['123']); } public function testRemoveEmpty(): void { - $this->solrClient->expects($this->exactly(0)) - ->method('update'); + $this->solrIndexService->expects($this->exactly(0)) + ->method('deleteByIdList'); $this->indexer->remove('test', []); } @@ -170,28 +159,11 @@ public function testIndexAllWithChunks(): void return $doc; }); - $addDocumentsCalls = 0; - $this->updateQuery->expects($this->exactly(3)) - ->method('addDocuments') - ->willReturnCallback( - function ($documents) use (&$addDocumentsCalls) { - if ($addDocumentsCalls === 0) { - $this->assertCount( - 10, - $documents, - "10 documents expected" - ); - } elseif ($addDocumentsCalls === 1) { - $this->assertCount( - 1, - $documents, - "1 document expected" - ); - } - $addDocumentsCalls++; - return $this->updateQuery; - } - ); + $this->updater->expects($this->exactly(12)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(3)) + ->method('update'); $parameter = new IndexerParameter( 'test', @@ -222,19 +194,11 @@ public function testIndexSkipResource(): void return ($location !== '/a/b.php'); }); - $this->documentEnricher->expects($this->exactly(1)) - ->method('enrichDocument'); - - $this->updateQuery->expects($this->exactly(1)) - ->method('addDocuments') - ->willReturnCallback(function ($documents) { - $this->assertCount( - 1, - $documents, - "one document exprected" - ); - return $this->updateQuery; - }); + $this->updater->expects($this->exactly(1)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); $parameter = new IndexerParameter( 'test', @@ -244,6 +208,7 @@ public function testIndexSkipResource(): void $this->indexer->index($parameter); } + public function testAborted(): void { $this->finder->method('findAll') @@ -347,16 +312,11 @@ public function testIndexPaths(): void $this->documentEnricher->method('isIndexable') ->willReturn(true); - $this->updateQuery->expects($this->exactly(1)) - ->method('addDocuments') - ->willReturnCallback(function ($documents) { - $this->assertCount( - 2, - $documents, - "two documents exprected" - ); - return $this->updateQuery; - }); + $this->updater->expects($this->exactly(2)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); $parameter = new IndexerParameter( 'test', diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php new file mode 100644 index 0000000..b69854b --- /dev/null +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -0,0 +1,72 @@ +client = $this->createMock(Client::class); + $this->factory = $this->createMock(SolrClientFactory::class); + $this->factory->method('create')->willReturn($this->client); + $this->indexService = new SolrIndexService($this->factory); + } + public function testUpdater(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->updater('test'); + } + + public function testDeleteExcludingProcessId(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->deleteExcludingProcessId('test', 'test', 'test'); + } + + public function testDeleteByIdList(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->deleteByIdList('test', 'test', ['test']); + } + + public function testByQuery(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->deleteByQuery('test', 'test'); + } + + public function testCommit(): void + { + $this->client->expects($this->once())->method('update'); + $this->indexService->commit('test'); + } + + public function testGetAvailableCores(): void + { + $statusResult = $this->createStub(StatusResult::class); + $statusResult->method('getCoreName')->willReturn('test'); + $response = $this->createStub(CoreAdminResult::class); + $response->method('getStatusResults')->willReturn([$statusResult]); + $this->client->method('coreAdmin')->willReturn($response); + + $cores = $this->indexService->getAvailableIndexes(); + + $this->assertEquals(['test'], $cores, 'Cores should be returned'); + } +} diff --git a/test/Service/Indexer/SolrIndexUpdaterTest.php b/test/Service/Indexer/SolrIndexUpdaterTest.php new file mode 100644 index 0000000..66c6db5 --- /dev/null +++ b/test/Service/Indexer/SolrIndexUpdaterTest.php @@ -0,0 +1,52 @@ +updateQuery = $this->createMock(UpdateQuery::class); + $this->updateQuery->method('createDocument')->willReturn( + new IndexSchema2xDocument() + ); + $this->client = $this->createMock(Client::class); + $this->updater = new SolrIndexUpdater( + $this->client, + $this->updateQuery + ); + } + + public function testCreateDocument(): void + { + $this->updateQuery->expects($this->once())->method('createDocument'); + $this->updater->createDocument(); + } + + public function testAddAndUpdate(): void + { + $doc = $this->updater->createDocument(); + $this->updater->addDocument($doc); + $this->updateQuery->expects($this->once()) + ->method('addDocuments') + ->with([$doc]); + $this->updater->update(); + } +} From a644145345895f729393fef1c7759aa21ac9ada9 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 12 Mar 2024 17:05:31 +0100 Subject: [PATCH 059/145] fix: analyser tests --- src/Service/Indexer/InternalResourceIndexer.php | 1 + test/Service/Indexer/BackgroundIndexerTest.php | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 025a165..0fa5529 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -339,6 +339,7 @@ private function add( /** @var IndexSchema2xDocument $doc */ $doc = $updater->createDocument(); foreach ($this->documentEnricherList as $enricher) { + /** @var IndexSchema2xDocument $doc */ $doc = $enricher->enrichDocument( $resource, $doc, diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index 27d3517..5039b12 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -26,7 +26,9 @@ class BackgroundIndexerTest extends TestCase public function setUp(): void { $this->solrIndexer = $this->createMock(InternalResourceIndexer::class); - $solrIndexerFactory = $this->createStub(InternalResourceIndexerFactory::class); + $solrIndexerFactory = $this->createStub( + InternalResourceIndexerFactory::class + ); $solrIndexerFactory->method('create') ->willReturn($this->solrIndexer); $this->statusStore = $this->createMock(IndexerStatusStore::class); From 6c2a7759374ac9992e77a000e9f30c7fcee7cce9 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 12 Mar 2024 17:34:25 +0100 Subject: [PATCH 060/145] feat: date_from and date_to for DefaultSchema2xRceEventDocumentEnricher --- src/Service/Indexer/IndexSchema2xDocument.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 95ec62a..dcf489d 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -29,6 +29,8 @@ class IndexSchema2xDocument extends Document implements IndexDocument public ?DateTime $sp_changed; public ?DateTime $sp_generated; public ?DateTime $sp_date; + public ?DateTime $sp_date_from; + public ?DateTime $sp_date_to; /** * @var DateTime[] */ From cc386baca5b90bacee52d00feaa3df2a396c7132 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 12 Mar 2024 17:43:05 +0100 Subject: [PATCH 061/145] feat: add IndexSchema2xDocument::setMetaText() --- src/Service/Indexer/IndexSchema2xDocument.php | 20 +++++++++++++++++-- .../Indexer/IndexSchema2xDocumentTest.php | 12 +++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index dcf489d..86e6893 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -127,21 +127,32 @@ class IndexSchema2xDocument extends Document implements IndexDocument */ private array $metaString = []; + /** + * @var array + */ + private array $metaText = []; + /** * @var array */ private array $metaBool = []; /** - * @param string $name * @param string|string[] $value - * @return void */ public function setMetaString(string $name, string|array $value): void { $this->metaString[$name] = $value; } + /** + * @param string|string[] $value + */ + public function setMetaText(string $name, string|array $value): void + { + $this->metaText[$name] = $value; + } + public function setMetaBool(string $name, bool $value): void { $this->metaBool[$name] = $value; @@ -175,6 +186,7 @@ static function ($value, $key) { if ( $key === 'metaString' || + $key === 'metaText' || $key === 'metaBool' ) { return false; @@ -187,6 +199,10 @@ static function ($value, $key) { $fields['sp_meta_string_' . $key] = $value; } + foreach ($this->metaText as $key => $value) { + $fields['sp_meta_text_' . $key] = $value; + } + foreach ($this->metaBool as $key => $value) { $fields['sp_meta_bool_' . $key] = $value; } diff --git a/test/Service/Indexer/IndexSchema2xDocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php index dbbfbf0..a0e9a36 100644 --- a/test/Service/Indexer/IndexSchema2xDocumentTest.php +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -35,6 +35,18 @@ public function testSetMetaString(): void ); } + public function testSetMetaText(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaText('myname', 'myvalue'); + + $this->assertEquals( + ['sp_meta_text_myname' => 'myvalue'], + $doc->getFields(), + 'unexpected meta fields' + ); + } + public function testSetMetaBool(): void { $doc = new IndexSchema2xDocument(); From 8f95a63e8cc37c7816305a766a7df3e357df8688 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 13 Mar 2024 15:09:05 +0100 Subject: [PATCH 062/145] feat: add contenttype field for index-document --- src/Service/Indexer/IndexSchema2xDocument.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 86e6893..476d2e5 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -19,6 +19,7 @@ class IndexSchema2xDocument extends Document implements IndexDocument public ?string $crawl_process_id; public ?string $id; public ?string $url; + public ?string $contenttype; /** * @var string[] */ From 7f628bbe0e3ac2b1139ecdd6b828cbf6aaaf09f5 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 15 Mar 2024 09:54:25 +0100 Subject: [PATCH 063/145] chore: work in process --- .../InternalResourceIndexerBuilder.php | 4 +- src/Console/Command/Io/IndexerProgressBar.php | 9 +- .../Command/Io/IndexerProgressBarFactory.php | 4 +- .../Command/SolrMoreLikeThisBuilder.php | 4 +- src/Console/Command/SolrSelectBuilder.php | 4 +- src/Console/Command/SolrSuggestBuilder.php | 4 +- src/Service/Indexer/IndexingAborter.php | 5 +- src/Service/Indexer/SolrIndexService.php | 31 ++++-- ...ory.php => ParameterSolrClientFactory.php} | 8 +- src/Service/ResourceChannelBasedCoreName.php | 24 +++++ src/Service/ServerVarSolrClientFactory.php | 95 +++++++++++++++++++ src/Service/SolrClientFactory.php | 2 +- src/Service/SolrCoreName.php | 10 ++ .../SolrParameterClientFactoryTest.php | 6 +- 14 files changed, 176 insertions(+), 34 deletions(-) rename src/Service/{SolrParameterClientFactory.php => ParameterSolrClientFactory.php} (83%) create mode 100644 src/Service/ResourceChannelBasedCoreName.php create mode 100644 src/Service/ServerVarSolrClientFactory.php create mode 100644 src/Service/SolrCoreName.php diff --git a/src/Console/Command/InternalResourceIndexerBuilder.php b/src/Console/Command/InternalResourceIndexerBuilder.php index 302adc1..05f5c3d 100644 --- a/src/Console/Command/InternalResourceIndexerBuilder.php +++ b/src/Console/Command/InternalResourceIndexerBuilder.php @@ -19,7 +19,7 @@ use Atoolo\Search\Service\Indexer\SiteKit\RichtTextMatcher; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexService; -use Atoolo\Search\Service\SolrParameterClientFactory; +use Atoolo\Search\Service\ParameterSolrClientFactory; class InternalResourceIndexerBuilder { @@ -99,7 +99,7 @@ public function build(): InternalResourceIndexer /** @var string[] $url */ $url = parse_url($this->solrConnectionUrl); - $clientFactory = new SolrParameterClientFactory( + $clientFactory = new ParameterSolrClientFactory( $url['scheme'], $url['host'], (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382)), diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index dcc5664..4a59006 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -9,6 +9,7 @@ use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use DateTime; use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\OutputInterface; use Throwable; @@ -23,14 +24,10 @@ class IndexerProgressBar implements IndexerProgressHandler */ private array $errors = []; - public function __construct(OutputInterface $output) - { - $this->output = $output; - } - public function start(int $total): void { - $this->progressBar = new ProgressBar($this->output, $total); + $output = new ConsoleOutput(); + $this->progressBar = new ProgressBar($output, $total); $this->formatProgressBar('green'); $this->status = new IndexerStatus( IndexerStatusState::RUNNING, diff --git a/src/Console/Command/Io/IndexerProgressBarFactory.php b/src/Console/Command/Io/IndexerProgressBarFactory.php index 207abe9..5662fab 100644 --- a/src/Console/Command/Io/IndexerProgressBarFactory.php +++ b/src/Console/Command/Io/IndexerProgressBarFactory.php @@ -10,6 +10,8 @@ class IndexerProgressBarFactory { public function create(OutputInterface $output): IndexerProgressBar { - return new IndexerProgressBar($output); + $progressBar = new IndexerProgressBar(); + $progressBar->init($output); + return $progressBar; } } diff --git a/src/Console/Command/SolrMoreLikeThisBuilder.php b/src/Console/Command/SolrMoreLikeThisBuilder.php index 68bd995..c6a6969 100644 --- a/src/Console/Command/SolrMoreLikeThisBuilder.php +++ b/src/Console/Command/SolrMoreLikeThisBuilder.php @@ -5,12 +5,12 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; +use Atoolo\Search\Service\ParameterSolrClientFactory; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; use Atoolo\Search\Service\Search\SolrMoreLikeThis; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; -use Atoolo\Search\Service\SolrParameterClientFactory; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -46,7 +46,7 @@ public function build(): SolrMoreLikeThis $resourceLoader = new SiteKitLoader($resourceBaseLocator); /** @var string[] */ $url = parse_url($this->solrConnectionUrl); - $clientFactory = new SolrParameterClientFactory( + $clientFactory = new ParameterSolrClientFactory( $url['scheme'], $url['host'], (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), diff --git a/src/Console/Command/SolrSelectBuilder.php b/src/Console/Command/SolrSelectBuilder.php index 090470f..e582136 100644 --- a/src/Console/Command/SolrSelectBuilder.php +++ b/src/Console/Command/SolrSelectBuilder.php @@ -5,13 +5,13 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\Loader\SiteKitLoader; +use Atoolo\Search\Service\ParameterSolrClientFactory; use Atoolo\Search\Service\Search\ExternalResourceFactory; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use Atoolo\Search\Service\Search\InternalResourceFactory; use Atoolo\Search\Service\Search\SiteKit\DefaultBoostModifier; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\Search\SolrSelect; -use Atoolo\Search\Service\SolrParameterClientFactory; class SolrSelectBuilder { @@ -44,7 +44,7 @@ public function build(): SolrSelect $resourceLoader = new SiteKitLoader($resourceBaseLocator); /** @var string[] */ $url = parse_url($this->solrConnectionUrl); - $clientFactory = new SolrParameterClientFactory( + $clientFactory = new ParameterSolrClientFactory( $url['scheme'], $url['host'], (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), diff --git a/src/Console/Command/SolrSuggestBuilder.php b/src/Console/Command/SolrSuggestBuilder.php index 77aca7a..7b7f2b2 100644 --- a/src/Console/Command/SolrSuggestBuilder.php +++ b/src/Console/Command/SolrSuggestBuilder.php @@ -4,8 +4,8 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Search\Service\ParameterSolrClientFactory; use Atoolo\Search\Service\Search\SolrSuggest; -use Atoolo\Search\Service\SolrParameterClientFactory; class SolrSuggestBuilder { @@ -22,7 +22,7 @@ public function build(): SolrSuggest { /** @var string[] $url */ $url = parse_url($this->solrConnectionUrl); - $clientFactory = new SolrParameterClientFactory( + $clientFactory = new ParameterSolrClientFactory( $url['scheme'], $url['host'], (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), diff --git a/src/Service/Indexer/IndexingAborter.php b/src/Service/Indexer/IndexingAborter.php index b5e2ce4..a59acd1 100644 --- a/src/Service/Indexer/IndexingAborter.php +++ b/src/Service/Indexer/IndexingAborter.php @@ -7,7 +7,8 @@ class IndexingAborter { public function __construct( - private readonly string $workdir + private readonly string $workdir, + private readonly string $type ) { } @@ -28,6 +29,6 @@ public function aborted(string $index): void private function getAbortMarkerFile(string $index): string { - return $this->workdir . '/background-indexer-' . $index . '.abort'; + return $this->workdir . '/' . $this->type . '-' . $index . '.abort'; } } diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php index 30fe789..51c051d 100644 --- a/src/Service/Indexer/SolrIndexService.php +++ b/src/Service/Indexer/SolrIndexService.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Search\Service\SolrClientFactory; +use LogicException; class SolrIndexService { @@ -13,9 +14,19 @@ public function __construct( ) { } - public function updater(string $core): SolrIndexUpdater + public function getIndex(?string $locale = null): string { - $client = $this->clientFactory->create($core); + $client = $this->clientFactory->create($locale); + $core = $client->getEndpoint()->getCore(); + if ($core === null) { + throw new LogicException('Core is not set in Solr client'); + } + return $core; + } + + public function updater(?string $locale = null): SolrIndexUpdater + { + $client = $this->clientFactory->create($locale); $update = $client->createUpdate(); $update->setDocumentClass(IndexSchema2xDocument::class); @@ -23,12 +34,12 @@ public function updater(string $core): SolrIndexUpdater } public function deleteExcludingProcessId( - string $core, + ?string $locale, string $source, string $processId ): void { $this->deleteByQuery( - $core, + $locale, '-crawl_process_id:' . $processId . ' AND ' . ' sp_source:' . $source ); @@ -38,28 +49,28 @@ public function deleteExcludingProcessId( * @param string[] $idList */ public function deleteByIdList( - string $core, + ?string $locale, string $source, array $idList ): void { $this->deleteByQuery( - $core, + $locale, 'sp_id:(' . implode(' ', $idList) . ') AND ' . 'sp_source:' . $source ); } - public function deleteByQuery(string $core, string $query): void + public function deleteByQuery(?string $locale, string $query): void { - $client = $this->clientFactory->create($core); + $client = $this->clientFactory->create($locale); $update = $client->createUpdate(); $update->addDeleteQuery($query); $client->update($update); } - public function commit(string $core): void + public function commit(?string $locale): void { - $client = $this->clientFactory->create($core); + $client = $this->clientFactory->create($locale); $update = $client->createUpdate(); $update->addCommit(); $update->addOptimize(); diff --git a/src/Service/SolrParameterClientFactory.php b/src/Service/ParameterSolrClientFactory.php similarity index 83% rename from src/Service/SolrParameterClientFactory.php rename to src/Service/ParameterSolrClientFactory.php index 1ea2e19..2778121 100644 --- a/src/Service/SolrParameterClientFactory.php +++ b/src/Service/ParameterSolrClientFactory.php @@ -11,12 +11,13 @@ /** * With this SolrClientFactory implementation, the necessary connection data * for the Solr client is transferred via the client constructor argument. - * The SolrParameterClientFactory can be registered as a DependencyInjection + * The ParameterSolrClientFactory can be registered as a DependencyInjection * service by passing the necessary transfer arguments as parameters. */ -class SolrParameterClientFactory implements SolrClientFactory +class ParameterSolrClientFactory implements SolrClientFactory { public function __construct( + private readonly SolrCoreName $coreName, private readonly string $scheme, private readonly string $host, private readonly int $port, @@ -26,8 +27,9 @@ public function __construct( ) { } - public function create(string $core): Client + public function create(?string $locale = null): Client { + $core = $this->coreName->name($locale); $adapter = new Curl(); $adapter->setTimeout($this->timeout); $adapter->setProxy($this->proxy); diff --git a/src/Service/ResourceChannelBasedCoreName.php b/src/Service/ResourceChannelBasedCoreName.php new file mode 100644 index 0000000..65f4720 --- /dev/null +++ b/src/Service/ResourceChannelBasedCoreName.php @@ -0,0 +1,24 @@ +resourceChannelFactory->create(); + if ($locale === null) { + return $resourceChannel->searchIndex; + } + return $resourceChannel->searchIndex . '-' . $locale; + } +} diff --git a/src/Service/ServerVarSolrClientFactory.php b/src/Service/ServerVarSolrClientFactory.php new file mode 100644 index 0000000..e8c9b88 --- /dev/null +++ b/src/Service/ServerVarSolrClientFactory.php @@ -0,0 +1,95 @@ +coreName->name($locale); + $adapter = new Curl(); + /* + $adapter->setTimeout($this->timeout); + $adapter->setProxy($this->proxy); + */ + $eventDispatcher = new EventDispatcher(); + $config = [ + 'endpoint' => $this->getEndpointConfig($core) + ]; + + // create a client instance + return new Client( + $adapter, + $eventDispatcher, + $config + ); + } + + /** + * @return array + */ + public function getEndpointConfig(string $core): array + { + $url = $_SERVER['SOLR_URL'] ?? ''; + + if (!empty($url)) { + $url = parse_url($url); + $scheme = $url['scheme'] ?? 'http'; + $host = $url['host'] ?? 'localhost'; + $port = (string)( + $url['port'] ?? + ( + $scheme === 'https' ? + '443' : + self::IES_WEBNODE_SOLR_PORT + ) + ); + $path = $url['path'] ?? ''; + } else { + $scheme = $_SERVER['SOLR_SCHEME'] ?? 'http'; + $host = (string)($_SERVER['SOLR_HOST'] ?? 'localhost'); + $port = $_SERVER['SOLR_PORT'] ?? self::IES_WEBNODE_SOLR_PORT; + $path = ''; + } + + return [ + $host => [ + 'scheme' => $scheme, + 'host' => $host, + 'port' => $port, + 'path' => $path, + 'core' => $core + ] + ]; + } +} diff --git a/src/Service/SolrClientFactory.php b/src/Service/SolrClientFactory.php index ce26af6..e2aebb0 100644 --- a/src/Service/SolrClientFactory.php +++ b/src/Service/SolrClientFactory.php @@ -12,5 +12,5 @@ */ interface SolrClientFactory { - public function create(string $core): Client; + public function create(?string $locale = null): Client; } diff --git a/src/Service/SolrCoreName.php b/src/Service/SolrCoreName.php new file mode 100644 index 0000000..6853c93 --- /dev/null +++ b/src/Service/SolrCoreName.php @@ -0,0 +1,10 @@ + Date: Fri, 15 Mar 2024 09:55:07 +0100 Subject: [PATCH 064/145] ci: trigger rebuild From 0a376383f3e583603fa8359258f4c33171e1a5b3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 18 Mar 2024 17:04:26 +0100 Subject: [PATCH 065/145] refactor: renmae QueryDefaultOperator to DefaultQueryOperator --- .../Command/ResourceBaseLocatorBuilder.php | 7 +----- ...tOperator.php => DefaultQueryOperator.php} | 2 +- src/Dto/Search/Query/SelectQuery.php | 2 +- src/Dto/Search/Query/SelectQueryBuilder.php | 12 +++++----- src/Service/Search/SolrSelect.php | 8 +++---- .../ResourceBaseLocatorBuilderTest.php | 2 +- .../Search/Query/SelectQueryBuilderTest.php | 8 +++---- test/Service/Search/SolrSelectTest.php | 22 +++++++++---------- 8 files changed, 29 insertions(+), 34 deletions(-) rename src/Dto/Search/Query/{QueryDefaultOperator.php => DefaultQueryOperator.php} (78%) diff --git a/src/Console/Command/ResourceBaseLocatorBuilder.php b/src/Console/Command/ResourceBaseLocatorBuilder.php index f6c3c7b..3205f64 100644 --- a/src/Console/Command/ResourceBaseLocatorBuilder.php +++ b/src/Console/Command/ResourceBaseLocatorBuilder.php @@ -11,14 +11,9 @@ class ResourceBaseLocatorBuilder { public function build(string $resourceDir): ResourceBaseLocator { - $subDirectory = null; - if (is_dir($resourceDir . '/objects')) { - $subDirectory = 'objects'; - } $_SERVER['RESOURCE_ROOT'] = $resourceDir; return new ServerVarResourceBaseLocator( - 'RESOURCE_ROOT', - $subDirectory + 'RESOURCE_ROOT' ); } } diff --git a/src/Dto/Search/Query/QueryDefaultOperator.php b/src/Dto/Search/Query/DefaultQueryOperator.php similarity index 78% rename from src/Dto/Search/Query/QueryDefaultOperator.php rename to src/Dto/Search/Query/DefaultQueryOperator.php index bada423..68dbc2b 100644 --- a/src/Dto/Search/Query/QueryDefaultOperator.php +++ b/src/Dto/Search/Query/DefaultQueryOperator.php @@ -5,7 +5,7 @@ /** * @codeCoverageIgnore */ -enum QueryDefaultOperator: string +enum DefaultQueryOperator: string { case AND = 'AND'; case OR = 'OR'; diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index c4d5c53..c1e39c6 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -28,7 +28,7 @@ public function __construct( public readonly array $sort, public readonly array $filter, public readonly array $facets, - public readonly QueryDefaultOperator $queryDefaultOperator + public readonly DefaultQueryOperator $defaultQueryOperator ) { } } diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 56d7d18..552168f 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -28,8 +28,8 @@ class SelectQueryBuilder */ private array $facets = []; - private QueryDefaultOperator $queryDefaultOperator = - QueryDefaultOperator::AND; + private DefaultQueryOperator $defaultQueryOperator = + DefaultQueryOperator::AND; public function __construct() { @@ -104,10 +104,10 @@ public function facet(Facet ...$facetList): SelectQueryBuilder return $this; } - public function queryDefaultOperator( - QueryDefaultOperator $queryDefaultOperator + public function defaultQueryOperator( + DefaultQueryOperator $defaultQueryOperator ): SelectQueryBuilder { - $this->queryDefaultOperator = $queryDefaultOperator; + $this->defaultQueryOperator = $defaultQueryOperator; return $this; } @@ -124,7 +124,7 @@ public function build(): SelectQuery $this->sort, array_values($this->filter), array_values($this->facets), - $this->queryDefaultOperator + $this->defaultQueryOperator ); } } diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 6787a04..0ad8a37 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -4,11 +4,11 @@ namespace Atoolo\Search\Service\Search; +use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\FacetField; use Atoolo\Search\Dto\Search\Query\Facet\FacetMultiQuery; use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; use Atoolo\Search\Dto\Search\Query\Filter\Filter; -use Atoolo\Search\Dto\Search\Query\QueryDefaultOperator; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; @@ -75,7 +75,7 @@ private function buildSolrQuery( $this->addTextFilterToSolrQuery($solrQuery, $query->text); $this->addQueryDefaultOperatorToSolrQuery( $solrQuery, - $query->queryDefaultOperator + $query->defaultQueryOperator ); $this->addFilterQueriesToSolrQuery( $solrQuery, @@ -148,9 +148,9 @@ static function ($term) use ($solrQuery) { private function addQueryDefaultOperatorToSolrQuery( SolrSelectQuery $solrQuery, - QueryDefaultOperator $operator + DefaultQueryOperator $operator ): void { - if ($operator === QueryDefaultOperator::OR) { + if ($operator === DefaultQueryOperator::OR) { $solrQuery->setQueryDefaultOperator( SolrSelectQuery::QUERY_OPERATOR_OR ); diff --git a/test/Console/Command/ResourceBaseLocatorBuilderTest.php b/test/Console/Command/ResourceBaseLocatorBuilderTest.php index 1950f32..8a1a918 100644 --- a/test/Console/Command/ResourceBaseLocatorBuilderTest.php +++ b/test/Console/Command/ResourceBaseLocatorBuilderTest.php @@ -25,7 +25,7 @@ public function testBuild(): void $locator = $builder->build($resourceDir); $this->assertEquals( - $objectsDir, + $resourceDir, $locator->locate(), 'unexpected resource dir' ); diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SelectQueryBuilderTest.php index 65ab23f..1e0eef2 100644 --- a/test/Dto/Search/Query/SelectQueryBuilderTest.php +++ b/test/Dto/Search/Query/SelectQueryBuilderTest.php @@ -4,9 +4,9 @@ namespace Atoolo\Search\Test\Dto\Search\Query; +use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; -use Atoolo\Search\Dto\Search\Query\QueryDefaultOperator; use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use InvalidArgumentException; @@ -141,11 +141,11 @@ public function testSetTwoFacetSWithSameKey(): void public function testSetQueryDefaultOperator(): void { - $this->builder->queryDefaultOperator(QueryDefaultOperator::AND); + $this->builder->defaultQueryOperator(DefaultQueryOperator::AND); $query = $this->builder->build(); $this->assertEquals( - QueryDefaultOperator::AND, - $query->queryDefaultOperator, + DefaultQueryOperator::AND, + $query->defaultQueryOperator, 'unexpected queryDefaultOperator' ); } diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSelectTest.php index b8e3e96..e1e410e 100644 --- a/test/Service/Search/SolrSelectTest.php +++ b/test/Service/Search/SolrSelectTest.php @@ -5,12 +5,12 @@ namespace Atoolo\Search\Test\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Facet\FacetMultiQuery; use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; use Atoolo\Search\Dto\Search\Query\Facet\ObjectTypeFacet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; -use Atoolo\Search\Dto\Search\Query\QueryDefaultOperator; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; @@ -93,7 +93,7 @@ public function testSelectEmpty(): void ], [], [], - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -116,7 +116,7 @@ public function testSelectWithText(): void ], [], [], - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -144,7 +144,7 @@ public function testSelectWithSort(): void ], [], [], - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -168,7 +168,7 @@ public function testSelectWithInvalidSort(): void [$sort], [], [], - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $this->expectException(InvalidArgumentException::class); @@ -185,7 +185,7 @@ public function testSelectWithAndDefaultOperator(): void [], [], [], - QueryDefaultOperator::AND + DefaultQueryOperator::AND ); $searchResult = $this->searcher->select($query); @@ -211,7 +211,7 @@ public function testSelectWithFilter(): void [], [$filter], [], - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -244,7 +244,7 @@ public function testSelectWithFacets(): void [], [], $facets, - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -271,7 +271,7 @@ public function testSelectWithInvalidFacets(): void [], [], $facets, - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $this->expectException(InvalidArgumentException::class); @@ -310,7 +310,7 @@ public function testResultFacets(): void [], [], $facets, - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -353,7 +353,7 @@ public function testInvalidResultFacets(): void [], [], $facets, - QueryDefaultOperator::OR + DefaultQueryOperator::OR ); $this->expectException(InvalidArgumentException::class); From 212ae442cd5d93c02a6e500fe7b6ac44200303d7 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 19 Mar 2024 13:44:32 +0100 Subject: [PATCH 066/145] chore: add branch-alias for feature/core-name --- composer.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/composer.json b/composer.json index 0857c4b..095b476 100644 --- a/composer.json +++ b/composer.json @@ -42,6 +42,12 @@ "squizlabs/php_codesniffer": "^3.7", "symfony/filesystem": "^6.3 | ^7.0" }, + "extra": { + "branch-alias": { + "feature/core-name": "dev-feature/initial-implementation" + } + }, + "scripts": { "post-install-cmd": "phive --no-progress install --force-accept-unsigned --trust-gpg-keys C00543248C87FB13,4AA394086372C20A,CF1A108D0E7AE720,51C67305FFC2E5C0", From ab87006d19dc3327e2eb7b24eee1e3cca02cf7ec Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 20 Mar 2024 08:26:28 +0100 Subject: [PATCH 067/145] refactor: renme DefaultQueryOperator to QueryOperator --- ...ultQueryOperator.php => QueryOperator.php} | 2 +- src/Dto/Search/Query/SelectQuery.php | 2 +- src/Dto/Search/Query/SelectQueryBuilder.php | 6 ++--- src/Service/Search/SolrSelect.php | 6 ++--- .../Search/Query/SelectQueryBuilderTest.php | 6 ++--- test/Service/Search/SolrSelectTest.php | 22 +++++++++---------- 6 files changed, 22 insertions(+), 22 deletions(-) rename src/Dto/Search/Query/{DefaultQueryOperator.php => QueryOperator.php} (78%) diff --git a/src/Dto/Search/Query/DefaultQueryOperator.php b/src/Dto/Search/Query/QueryOperator.php similarity index 78% rename from src/Dto/Search/Query/DefaultQueryOperator.php rename to src/Dto/Search/Query/QueryOperator.php index 68dbc2b..bc478d6 100644 --- a/src/Dto/Search/Query/DefaultQueryOperator.php +++ b/src/Dto/Search/Query/QueryOperator.php @@ -5,7 +5,7 @@ /** * @codeCoverageIgnore */ -enum DefaultQueryOperator: string +enum QueryOperator: string { case AND = 'AND'; case OR = 'OR'; diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index c1e39c6..661f992 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -28,7 +28,7 @@ public function __construct( public readonly array $sort, public readonly array $filter, public readonly array $facets, - public readonly DefaultQueryOperator $defaultQueryOperator + public readonly QueryOperator $defaultQueryOperator ) { } } diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index 552168f..fd9c573 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -28,8 +28,8 @@ class SelectQueryBuilder */ private array $facets = []; - private DefaultQueryOperator $defaultQueryOperator = - DefaultQueryOperator::AND; + private QueryOperator $defaultQueryOperator = + QueryOperator::AND; public function __construct() { @@ -105,7 +105,7 @@ public function facet(Facet ...$facetList): SelectQueryBuilder } public function defaultQueryOperator( - DefaultQueryOperator $defaultQueryOperator + QueryOperator $defaultQueryOperator ): SelectQueryBuilder { $this->defaultQueryOperator = $defaultQueryOperator; return $this; diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 0ad8a37..d936886 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -4,11 +4,11 @@ namespace Atoolo\Search\Service\Search; -use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\FacetField; use Atoolo\Search\Dto\Search\Query\Facet\FacetMultiQuery; use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; use Atoolo\Search\Dto\Search\Query\Filter\Filter; +use Atoolo\Search\Dto\Search\Query\QueryOperator; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; @@ -148,9 +148,9 @@ static function ($term) use ($solrQuery) { private function addQueryDefaultOperatorToSolrQuery( SolrSelectQuery $solrQuery, - DefaultQueryOperator $operator + QueryOperator $operator ): void { - if ($operator === DefaultQueryOperator::OR) { + if ($operator === QueryOperator::OR) { $solrQuery->setQueryDefaultOperator( SolrSelectQuery::QUERY_OPERATOR_OR ); diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SelectQueryBuilderTest.php index 1e0eef2..e02c4ce 100644 --- a/test/Dto/Search/Query/SelectQueryBuilderTest.php +++ b/test/Dto/Search/Query/SelectQueryBuilderTest.php @@ -4,9 +4,9 @@ namespace Atoolo\Search\Test\Dto\Search\Query; -use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; +use Atoolo\Search\Dto\Search\Query\QueryOperator; use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use InvalidArgumentException; @@ -141,10 +141,10 @@ public function testSetTwoFacetSWithSameKey(): void public function testSetQueryDefaultOperator(): void { - $this->builder->defaultQueryOperator(DefaultQueryOperator::AND); + $this->builder->defaultQueryOperator(QueryOperator::AND); $query = $this->builder->build(); $this->assertEquals( - DefaultQueryOperator::AND, + QueryOperator::AND, $query->defaultQueryOperator, 'unexpected queryDefaultOperator' ); diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSelectTest.php index e1e410e..f71d27a 100644 --- a/test/Service/Search/SolrSelectTest.php +++ b/test/Service/Search/SolrSelectTest.php @@ -5,12 +5,12 @@ namespace Atoolo\Search\Test\Service\Search; use Atoolo\Resource\Resource; -use Atoolo\Search\Dto\Search\Query\DefaultQueryOperator; use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Facet\FacetMultiQuery; use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; use Atoolo\Search\Dto\Search\Query\Facet\ObjectTypeFacet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; +use Atoolo\Search\Dto\Search\Query\QueryOperator; use Atoolo\Search\Dto\Search\Query\SelectQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; @@ -93,7 +93,7 @@ public function testSelectEmpty(): void ], [], [], - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -116,7 +116,7 @@ public function testSelectWithText(): void ], [], [], - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -144,7 +144,7 @@ public function testSelectWithSort(): void ], [], [], - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -168,7 +168,7 @@ public function testSelectWithInvalidSort(): void [$sort], [], [], - DefaultQueryOperator::OR + QueryOperator::OR ); $this->expectException(InvalidArgumentException::class); @@ -185,7 +185,7 @@ public function testSelectWithAndDefaultOperator(): void [], [], [], - DefaultQueryOperator::AND + QueryOperator::AND ); $searchResult = $this->searcher->select($query); @@ -211,7 +211,7 @@ public function testSelectWithFilter(): void [], [$filter], [], - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -244,7 +244,7 @@ public function testSelectWithFacets(): void [], [], $facets, - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -271,7 +271,7 @@ public function testSelectWithInvalidFacets(): void [], [], $facets, - DefaultQueryOperator::OR + QueryOperator::OR ); $this->expectException(InvalidArgumentException::class); @@ -310,7 +310,7 @@ public function testResultFacets(): void [], [], $facets, - DefaultQueryOperator::OR + QueryOperator::OR ); $searchResult = $this->searcher->select($query); @@ -353,7 +353,7 @@ public function testInvalidResultFacets(): void [], [], $facets, - DefaultQueryOperator::OR + QueryOperator::OR ); $this->expectException(InvalidArgumentException::class); From e10800dc0a4ef39fcd45764c6d971ba4940849ea Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 11:24:54 +0100 Subject: [PATCH 068/145] refactore: automatic index determination including multi-lang indexes --- composer.json | 5 +- config/commands.yml | 59 +++++++- src/Console/Command/DumpIndexDocument.php | 25 +--- .../Command/IndexDocumentDumperBuilder.php | 56 ------- src/Console/Command/Indexer.php | 42 +----- .../InternalResourceIndexerBuilder.php | 128 ---------------- src/Console/Command/Io/IndexerProgressBar.php | 8 +- .../Command/Io/IndexerProgressBarFactory.php | 17 --- src/Console/Command/Io/TypifiedInput.php | 11 ++ src/Console/Command/MoreLikeThis.php | 52 ++----- .../Command/ResourceBaseLocatorBuilder.php | 19 --- src/Console/Command/Search.php | 46 ++---- .../Command/SolrMoreLikeThisBuilder.php | 72 --------- src/Console/Command/SolrSelectBuilder.php | 73 ---------- src/Console/Command/SolrSuggestBuilder.php | 35 ----- src/Console/Command/Suggest.php | 39 ++--- src/Dto/Indexer/IndexerParameter.php | 1 - src/Dto/Search/Query/MoreLikeThisQuery.php | 3 +- src/Dto/Search/Query/SelectQuery.php | 2 +- src/Dto/Search/Query/SelectQueryBuilder.php | 18 +-- src/Dto/Search/Query/SuggestQuery.php | 2 +- src/Indexer.php | 4 +- src/Service/IndexName.php | 18 +++ src/Service/Indexer/BackgroundIndexer.php | 31 ++-- .../Indexer/InternalResourceIndexer.php | 81 +++++++---- src/Service/Indexer/SolrIndexService.php | 76 ++++++---- src/Service/ParameterSolrClientFactory.php | 4 +- src/Service/ResourceChannelBasedCoreName.php | 24 --- src/Service/ResourceChannelBasedIndexName.php | 64 ++++++++ .../Search/ExternalResourceFactory.php | 5 +- .../Search/InternalMediaResourceFactory.php | 4 +- .../Search/InternalResourceFactory.php | 4 +- src/Service/Search/ResourceFactory.php | 2 +- src/Service/Search/SolrMoreLikeThis.php | 12 +- .../Search/SolrResultToResourceResolver.php | 8 +- src/Service/Search/SolrSelect.php | 12 +- src/Service/Search/SolrSuggest.php | 5 +- src/Service/ServerVarSolrClientFactory.php | 19 +-- src/Service/SolrClientFactory.php | 2 +- src/Service/SolrCoreName.php | 10 -- test/Console/ApplicationTest.php | 9 +- .../Console/Command/DumpIndexDocumentTest.php | 11 +- .../IndexDocumentDumperBuilderTest.php | 26 ---- test/Console/Command/IndexerTest.php | 87 +++-------- .../Io/IndexerProgressBarFactoryTest.php | 21 --- test/Console/Command/Io/TypifiedInputTest.php | 29 ++++ test/Console/Command/MoreLikeThisTest.php | 18 +-- .../ResourceBaseLocatorBuilderTest.php | 33 ----- test/Console/Command/SearchTest.php | 18 +-- .../Command/SolrIndexerBuilderTest.php | 32 ---- .../Command/SolrMoreLikeThisBuilderTest.php | 27 ---- .../Console/Command/SolrSelectBuilderTest.php | 26 ---- .../Command/SolrSuggestBuilderTest.php | 22 --- test/Console/Command/SuggestTest.php | 15 +- test/Dto/Indexer/IndexerParameterTest.php | 1 - .../Search/Query/SelectQueryBuilderTest.php | 39 ++--- .../Service/Indexer/BackgroundIndexerTest.php | 17 ++- test/Service/Indexer/IndexingAborterTest.php | 2 +- .../Indexer/InternalResourceIndexerTest.php | 35 +++-- test/Service/Indexer/SolrIndexServiceTest.php | 61 ++++++-- .../ResourceChannelBasedIndexNameTest.php | 82 +++++++++++ .../Search/ExternalResourceFactoryTest.php | 6 +- .../InternalMediaResourceFactoryTest.php | 4 +- .../Search/InternalResourceFactoryTest.php | 4 +- test/Service/Search/SolrMoreLikeThisTest.php | 6 +- .../SolrResultToResourceResolverTest.php | 6 +- test/Service/Search/SolrSelectTest.php | 23 +-- test/Service/Search/SolrSuggestTest.php | 4 +- .../ServerVarSolrClientFactoryTest.php | 137 ++++++++++++++++++ 69 files changed, 782 insertions(+), 1117 deletions(-) delete mode 100644 src/Console/Command/IndexDocumentDumperBuilder.php delete mode 100644 src/Console/Command/InternalResourceIndexerBuilder.php delete mode 100644 src/Console/Command/Io/IndexerProgressBarFactory.php delete mode 100644 src/Console/Command/ResourceBaseLocatorBuilder.php delete mode 100644 src/Console/Command/SolrMoreLikeThisBuilder.php delete mode 100644 src/Console/Command/SolrSelectBuilder.php delete mode 100644 src/Console/Command/SolrSuggestBuilder.php create mode 100644 src/Service/IndexName.php delete mode 100644 src/Service/ResourceChannelBasedCoreName.php create mode 100644 src/Service/ResourceChannelBasedIndexName.php delete mode 100644 src/Service/SolrCoreName.php delete mode 100644 test/Console/Command/IndexDocumentDumperBuilderTest.php delete mode 100644 test/Console/Command/Io/IndexerProgressBarFactoryTest.php delete mode 100644 test/Console/Command/ResourceBaseLocatorBuilderTest.php delete mode 100644 test/Console/Command/SolrIndexerBuilderTest.php delete mode 100644 test/Console/Command/SolrMoreLikeThisBuilderTest.php delete mode 100644 test/Console/Command/SolrSelectBuilderTest.php delete mode 100644 test/Console/Command/SolrSuggestBuilderTest.php create mode 100644 test/Service/ResourceChannelBasedIndexNameTest.php create mode 100644 test/Service/ServerVarSolrClientFactoryTest.php diff --git a/composer.json b/composer.json index 095b476..36e54c9 100644 --- a/composer.json +++ b/composer.json @@ -31,8 +31,9 @@ "symfony/lock": "^6.3 | ^7.0", "symfony/property-access": "^6.3 | ^7.0", "symfony/serializer": "^6.3 | ^7.0", - "symfony/yaml": "^6.3 | ^7.0" - }, + "symfony/yaml": "^6.3 | ^7.0", + "ext-intl": "*" + }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^1.0", "infection/infection": "^0.27.6", diff --git a/config/commands.yml b/config/commands.yml index db4c495..48b5be2 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -9,13 +9,68 @@ services: Atoolo\Search\Console\: resource: '../src/Console' - Atoolo\Search\Console\Command\Indexer: + Atoolo\Search\Service\Indexer\IndexDocumentDumper: arguments: + - '@atoolo.resource.resourceLoader' - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } Atoolo\Search\Console\Command\DumpIndexDocument: arguments: - - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } + - '@Atoolo\Search\Service\Indexer\IndexDocumentDumper' + + atoolo.search.indexer.progressBar: + class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' + + atoolo.search.indexer.internalResourceIndexer: + class: Atoolo\Search\Service\Indexer\InternalResourceIndexer + arguments: + - !tagged_iterator atoolo.search.indexer.documentEnricher.schema2x + - '@atoolo.search.indexer.progressBar' + - '@atoolo.search.indexer.locationFinder' + - '@atoolo.resource.resourceLoader' + - '@atoolo.search.indexer.translationSplitter' + - '@atoolo.search.indexer.solrIndexService' + - '@atoolo.search.indexer.aborter' + - 'internal' + + atoolo.search.selectSearcher: + class: Atoolo\Search\Service\Search\SolrSelect + arguments: + - '@atoolo.search.indexName' + - '@atoolo.search.solariumClientFactory' + - !tagged_iterator atoolo.search.queryModifier + - '@atoolo.search.resultToResourceResolver' + + atoolo.search.suggestSearcher: + class: Atoolo\Search\Service\Search\SolrSuggest + arguments: + - '@atoolo.search.indexName' + - '@atoolo.search.solariumClientFactory' + + atoolo.search.moreLikeThisSearcher: + class: Atoolo\Search\Service\Search\SolrMoreLikeThis + arguments: + - '@atoolo.search.indexName' + - '@atoolo.search.solariumClientFactory' + - '@atoolo.search.resultToResourceResolver' + + Atoolo\Search\Console\Command\Indexer: + arguments: + - '@atoolo.search.indexer.progressBar' + - '@atoolo.search.indexer.internalResourceIndexer' + + Atoolo\Search\Console\Command\Search: + arguments: + - '@atoolo.search.selectSearcher' + + Atoolo\Search\Console\Command\Suggest: + arguments: + - '@atoolo.search.suggestSearcher' + + Atoolo\Search\Console\Command\MoreLikeThis: + arguments: + - '@atoolo.search.moreLikeThisSearcher' + Atoolo\Search\Console\Application: public: true diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php index 4a28774..76515e5 100644 --- a/src/Console/Command/DumpIndexDocument.php +++ b/src/Console/Command/DumpIndexDocument.php @@ -5,8 +5,7 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Search\Console\Command\Io\TypifiedInput; -use Atoolo\Search\Service\Indexer\DocumentEnricher; -use Atoolo\Search\Service\Indexer\IndexDocument; +use Atoolo\Search\Service\Indexer\IndexDocumentDumper; use JsonException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -20,13 +19,8 @@ )] class DumpIndexDocument extends Command { - /** - * phpcs:ignore - * @param iterable> $documentEnricherList - */ public function __construct( - private readonly iterable $documentEnricherList, - private readonly IndexDocumentDumperBuilder $indexDocumentDumperBuilder + private readonly IndexDocumentDumper $dumper ) { parent::__construct(); } @@ -35,11 +29,6 @@ protected function configure(): void { $this ->setHelp('Command to dump a index-document') - ->addArgument( - 'resource-dir', - InputArgument::REQUIRED, - 'Resource directory whose data is to be indexed.' - ) ->addArgument( 'paths', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, @@ -58,15 +47,9 @@ protected function execute( $typedInput = new TypifiedInput($input); - $resourceDir = $typedInput->getStringArgument('resource-dir'); - - $dumper = $this->indexDocumentDumperBuilder - ->resourceDir($resourceDir) - ->documentEnricherList($this->documentEnricherList) - ->build(); - $paths = $typedInput->getArrayArgument('paths'); - $dump = $dumper->dump($paths); + + $dump = $this->dumper->dump($paths); foreach ($dump as $fields) { $output->writeln(json_encode( diff --git a/src/Console/Command/IndexDocumentDumperBuilder.php b/src/Console/Command/IndexDocumentDumperBuilder.php deleted file mode 100644 index bb8e3b2..0000000 --- a/src/Console/Command/IndexDocumentDumperBuilder.php +++ /dev/null @@ -1,56 +0,0 @@ -> - */ - private iterable $documentEnricherList; - - public function __construct( - private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder - ) { - } - - public function resourceDir(string $resourceDir): IndexDocumentDumperBuilder - { - $this->resourceDir = $resourceDir; - return $this; - } - - /** - * phpcs:ignore - * @param iterable> $documentEnricherList - */ - public function documentEnricherList( - iterable $documentEnricherList - ): IndexDocumentDumperBuilder { - $this->documentEnricherList = $documentEnricherList; - return $this; - } - - public function build(): IndexDocumentDumper - { - $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( - $this->resourceDir - ); - - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - - return new IndexDocumentDumper( - $resourceLoader, - $this->documentEnricherList - ); - } -} diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index e25dedc..5adf2c4 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -5,11 +5,9 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; -use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Indexer\IndexerParameter; -use Atoolo\Search\Service\Indexer\DocumentEnricher; -use Atoolo\Search\Service\Indexer\IndexDocument; +use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -23,18 +21,12 @@ )] class Indexer extends Command { - private IndexerProgressBar $progressBar; private SymfonyStyle $io; private OutputInterface $output; - /** - * phpcs:ignore - * @param iterable> $documentEnricherList - */ public function __construct( - private readonly iterable $documentEnricherList, - private readonly InternalResourceIndexerBuilder $solrIndexerBuilder, - private readonly IndexerProgressBarFactory $progressBarFactory + private readonly IndexerProgressBar $progressBar, + private readonly InternalResourceIndexer $indexer, ) { parent::__construct(); } @@ -43,21 +35,6 @@ protected function configure(): void { $this ->setHelp('Command to fill a search index') - ->addArgument( - 'solr-connection-url', - InputArgument::REQUIRED, - 'Solr connection url.' - ) - ->addArgument( - 'solr-core', - InputArgument::REQUIRED, - 'Solr core to be used.' - ) - ->addArgument( - 'resource-dir', - InputArgument::REQUIRED, - 'Resource directory whose data is to be indexed.' - ) ->addArgument( 'paths', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, @@ -93,7 +70,6 @@ protected function execute( $typedInput = new TypifiedInput($input); $this->output = $output; $this->io = new SymfonyStyle($input, $output); - $this->progressBar = $this->progressBarFactory->create($output); $paths = $typedInput->getArrayArgument('paths'); @@ -109,22 +85,12 @@ protected function execute( } $parameter = new IndexerParameter( - $typedInput->getStringArgument('solr-core'), $cleanupThreshold, $typedInput->getIntOption('chunk-size'), $paths ); - $this->solrIndexerBuilder - ->resourceDir($typedInput->getStringArgument('resource-dir')) - ->progressBar($this->progressBar) - ->documentEnricherList($this->documentEnricherList) - ->solrConnectionUrl( - $typedInput->getStringArgument('solr-connection-url') - ); - - $indexer = $this->solrIndexerBuilder->build(); - $indexer->index($parameter); + $this->indexer->index($parameter); $this->errorReport(); diff --git a/src/Console/Command/InternalResourceIndexerBuilder.php b/src/Console/Command/InternalResourceIndexerBuilder.php deleted file mode 100644 index 05f5c3d..0000000 --- a/src/Console/Command/InternalResourceIndexerBuilder.php +++ /dev/null @@ -1,128 +0,0 @@ -> - */ - private iterable $documentEnricherList; - private IndexerProgressBar $progressBar; - private string $solrConnectionUrl; - - public function __construct( - private readonly ResourceBaseLocatorBuilder $resourceBaseLocatorBuilder - ) { - } - public function resourceDir( - string $resourceDir - ): InternalResourceIndexerBuilder { - $this->resourceDir = $resourceDir; - return $this; - } - - /** - * phpcs:ignore - * @param iterable> $documentEnricherList - */ - public function documentEnricherList( - iterable $documentEnricherList - ): InternalResourceIndexerBuilder { - $this->documentEnricherList = $documentEnricherList; - return $this; - } - - public function progressBar( - IndexerProgressBar $progressBar - ): InternalResourceIndexerBuilder { - $this->progressBar = $progressBar; - return $this; - } - - public function solrConnectionUrl( - string $solrConnectionUrl - ): InternalResourceIndexerBuilder { - $this->solrConnectionUrl = $solrConnectionUrl; - return $this; - } - - public function build(): InternalResourceIndexer - { - $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( - $this->resourceDir - ); - $finder = new LocationFinder($resourceBaseLocator); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - $navigationLoader = new SiteKitNavigationHierarchyLoader( - $resourceLoader - ); - - /** @var iterable $matcher */ - $matcher = [ - new HeadlineMatcher(), - new RichtTextMatcher(), - ]; - $contentCollector = new ContentCollector($matcher); - - $schema21 = new DefaultSchema2xDocumentEnricher( - $navigationLoader, - $contentCollector - ); - - /** @var array> $documentEnricherList */ - $documentEnricherList = [$schema21]; - foreach ($this->documentEnricherList as $enricher) { - $documentEnricherList[] = $enricher; - } - /** @var string[] $url */ - $url = parse_url($this->solrConnectionUrl); - - $clientFactory = new ParameterSolrClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8382)), - $url['path'] ?? '', - null, - 0 - ); - - $solrIndexService = new SolrIndexService($clientFactory); - - $translationSplitter = new SubDirTranslationSplitter(); - - $aborter = new IndexingAborter('.'); - - return new InternalResourceIndexer( - $documentEnricherList, - $this->progressBar, - $finder, - $resourceLoader, - $translationSplitter, - $solrIndexService, - $aborter, - 'internal' - ); - } -} diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index 4a59006..0fc84c7 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -24,10 +24,14 @@ class IndexerProgressBar implements IndexerProgressHandler */ private array $errors = []; + public function __construct(OutputInterface $output = new ConsoleOutput()) + { + $this->output = $output; + } + public function start(int $total): void { - $output = new ConsoleOutput(); - $this->progressBar = new ProgressBar($output, $total); + $this->progressBar = new ProgressBar($this->output, $total); $this->formatProgressBar('green'); $this->status = new IndexerStatus( IndexerStatusState::RUNNING, diff --git a/src/Console/Command/Io/IndexerProgressBarFactory.php b/src/Console/Command/Io/IndexerProgressBarFactory.php deleted file mode 100644 index 5662fab..0000000 --- a/src/Console/Command/Io/IndexerProgressBarFactory.php +++ /dev/null @@ -1,17 +0,0 @@ -init($output); - return $progressBar; - } -} diff --git a/src/Console/Command/Io/TypifiedInput.php b/src/Console/Command/Io/TypifiedInput.php index 3532b32..12a4d74 100644 --- a/src/Console/Command/Io/TypifiedInput.php +++ b/src/Console/Command/Io/TypifiedInput.php @@ -17,6 +17,17 @@ public function __construct(private readonly InputInterface $input) { } + public function getStringOption(string $name): string + { + $value = $this->input->getOption($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + 'option' . $name . ' must be a string: ' . $value + ); + } + return $value; + } + public function getIntOption(string $name): int { $value = $this->input->getOption($name); diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index bed1aa9..bee3443 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -23,10 +23,9 @@ class MoreLikeThis extends Command { private SymfonyStyle $io; private TypifiedInput $input; - private string $solrCore; public function __construct( - private readonly SolrMoreLikeThisBuilder $solrMoreLikeThisBuilder + private readonly SolrMoreLikeThis $searcher ) { parent::__construct(); } @@ -35,27 +34,19 @@ protected function configure(): void { $this ->setHelp('Command to performs a more-like-this search') - ->addArgument( - 'solr-connection-url', - InputArgument::REQUIRED, - 'Solr connection url.' - ) - ->addArgument( - 'solr-core', - InputArgument::REQUIRED, - 'Solr core to be used.' - ) - ->addArgument( - 'resource-dir', - InputArgument::REQUIRED, - 'Resource directory where the resources can be found.' - ) ->addArgument( 'location', InputArgument::REQUIRED, 'Location of the resource to which the MoreLikeThis ' . 'search is to be applied..' ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) ; } @@ -67,35 +58,24 @@ protected function execute( $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $this->input->getStringArgument('solr-core'); $location = $this->input->getStringArgument('location'); + $lang = $this->input->getStringOption('lang'); - $searcher = $this->createSearcher(); - $query = $this->buildQuery($location); - $result = $searcher->moreLikeThis($query); + $query = $this->buildQuery($location, $lang); + $result = $this->searcher->moreLikeThis($query); $this->outputResult($result); return Command::SUCCESS; } - protected function createSearcher(): SolrMoreLikeThis - { - $this->solrMoreLikeThisBuilder->solrConnectionUrl( - $this->input->getStringArgument('solr-connection-url') - ); - $this->solrMoreLikeThisBuilder->resourceDir( - $this->input->getStringArgument('resource-dir') - ); - - return $this->solrMoreLikeThisBuilder->build(); - } - - protected function buildQuery(string $location): MoreLikeThisQuery - { + protected function buildQuery( + string $location, + string $lang + ): MoreLikeThisQuery { $filterList = []; return new MoreLikeThisQuery( - $this->solrCore, $location, + $lang, $filterList, 5, ['content'] diff --git a/src/Console/Command/ResourceBaseLocatorBuilder.php b/src/Console/Command/ResourceBaseLocatorBuilder.php deleted file mode 100644 index 3205f64..0000000 --- a/src/Console/Command/ResourceBaseLocatorBuilder.php +++ /dev/null @@ -1,19 +0,0 @@ -setHelp('Command to performs a search') - ->addArgument( - 'solr-connection-url', - InputArgument::REQUIRED, - 'Solr connection url.' - ) - ->addArgument( - 'index', - InputArgument::REQUIRED, - 'Solr core to be used.' - ) - ->addArgument( - 'resource-dir', - InputArgument::REQUIRED, - 'Resource directory whose data is to be indexed.' - ) ->addArgument( 'text', InputArgument::IS_ARRAY, 'Text with which to search.' ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) ; } @@ -66,38 +57,27 @@ protected function execute( $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->index = $this->input->getStringArgument('index'); - $searcher = $this->createSearch(); $query = $this->buildQuery($input); - $result = $searcher->select($query); + $result = $this->searcher->select($query); $this->outputResult($result); return Command::SUCCESS; } - protected function createSearch(): SolrSelect - { - $this->solrSelectBuilder->resourceDir( - $this->input->getStringArgument('resource-dir') - ); - $this->solrSelectBuilder->solrConnectionUrl( - $this->input->getStringArgument('solr-connection-url') - ); - return $this->solrSelectBuilder->build(); - } - protected function buildQuery(InputInterface $input): SelectQuery { $builder = new SelectQueryBuilder(); - $builder->index($this->index); - $text = $input->getArgument('text'); + $text = $this->input->getArrayArgument('text'); if (is_array($text)) { $builder->text(implode(' ', $text)); } + $lang = $this->input->getStringOption('lang'); + $builder->lang($lang); + // TODO: filter diff --git a/src/Console/Command/SolrMoreLikeThisBuilder.php b/src/Console/Command/SolrMoreLikeThisBuilder.php deleted file mode 100644 index c6a6969..0000000 --- a/src/Console/Command/SolrMoreLikeThisBuilder.php +++ /dev/null @@ -1,72 +0,0 @@ -resourceDir = $resourceDir; - return $this; - } - - public function solrConnectionUrl( - string $solrConnectionUrl - ): SolrMoreLikeThisBuilder { - $this->solrConnectionUrl = $solrConnectionUrl; - return $this; - } - - public function build(): SolrMoreLikeThis - { - $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( - $this->resourceDir - ); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - /** @var string[] */ - $url = parse_url($this->solrConnectionUrl); - $clientFactory = new ParameterSolrClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 - ); - $resourceFactoryList = [ - new ExternalResourceFactory(), - new InternalResourceFactory($resourceLoader), - new InternalMediaResourceFactory($resourceLoader) - ]; - $solrResultToResourceResolver = new SolrResultToResourceResolver( - $resourceFactoryList, - $this->logger - ); - - return new SolrMoreLikeThis( - $clientFactory, - $solrResultToResourceResolver - ); - } -} diff --git a/src/Console/Command/SolrSelectBuilder.php b/src/Console/Command/SolrSelectBuilder.php deleted file mode 100644 index e582136..0000000 --- a/src/Console/Command/SolrSelectBuilder.php +++ /dev/null @@ -1,73 +0,0 @@ -resourceDir = $resourceDir; - return $this; - } - - public function solrConnectionUrl( - string $solrConnectionUrl - ): SolrSelectBuilder { - $this->solrConnectionUrl = $solrConnectionUrl; - return $this; - } - - public function build(): SolrSelect - { - $resourceBaseLocator = $this->resourceBaseLocatorBuilder->build( - $this->resourceDir - ); - $resourceLoader = new SiteKitLoader($resourceBaseLocator); - /** @var string[] */ - $url = parse_url($this->solrConnectionUrl); - $clientFactory = new ParameterSolrClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 - ); - $defaultBoosting = new DefaultBoostModifier(); - - $resourceFactoryList = [ - new ExternalResourceFactory(), - new InternalResourceFactory($resourceLoader), - new InternalMediaResourceFactory($resourceLoader) - ]; - - $solrResultToResourceResolver = new SolrResultToResourceResolver( - $resourceFactoryList - ); - - return new SolrSelect( - $clientFactory, - [$defaultBoosting], - $solrResultToResourceResolver - ); - } -} diff --git a/src/Console/Command/SolrSuggestBuilder.php b/src/Console/Command/SolrSuggestBuilder.php deleted file mode 100644 index 7b7f2b2..0000000 --- a/src/Console/Command/SolrSuggestBuilder.php +++ /dev/null @@ -1,35 +0,0 @@ -solrConnectionUrl = $solrConnectionUrl; - return $this; - } - - public function build(): SolrSuggest - { - /** @var string[] $url */ - $url = parse_url($this->solrConnectionUrl); - $clientFactory = new ParameterSolrClientFactory( - $url['scheme'], - $url['host'], - (int)($url['port'] ?? ($url['scheme'] === 'https' ? 443 : 8983)), - $url['path'] ?? '', - null, - 0 - ); - return new SolrSuggest($clientFactory); - } -} diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index b4c59b6..2362b72 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -25,10 +25,9 @@ class Suggest extends Command { private TypifiedInput $input; private SymfonyStyle $io; - private string $solrCore; public function __construct( - private readonly SolrSuggestBuilder $solrSuggestBuilder + private readonly SolrSuggest $search ) { parent::__construct(); } @@ -37,21 +36,18 @@ protected function configure(): void { $this ->setHelp('Command to performs a suggest search') - ->addArgument( - 'solr-connection-url', - InputArgument::REQUIRED, - 'Solr connection url.' - ) - ->addArgument( - 'solr-core', - InputArgument::REQUIRED, - 'Solr core to be used.' - ) ->addArgument( 'terms', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'Suggest terms.' ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) ; } @@ -61,37 +57,28 @@ protected function execute( ): int { $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $this->solrCore = $this->input->getStringArgument('solr-core'); $terms = $this->input->getArrayArgument('terms'); + $lang = $this->input->getStringOption('lang'); - $search = $this->createSearcher(); - $query = $this->buildQuery($terms); + $query = $this->buildQuery($terms, $lang); - $result = $search->suggest($query); + $result = $this->search->suggest($query); $this->outputResult($result); return Command::SUCCESS; } - protected function createSearcher(): SolrSuggest - { - $this->solrSuggestBuilder->solrConnectionUrl( - $this->input->getStringArgument('solr-connection-url') - ); - return $this->solrSuggestBuilder->build(); - } - /** * @param string[] $terms */ - protected function buildQuery(array $terms): SuggestQuery + protected function buildQuery(array $terms, string $lang): SuggestQuery { $excludeMedia = new ObjectTypeFilter(['media'], 'media'); $excludeMedia = $excludeMedia->exclude(); return new SuggestQuery( - $this->solrCore, implode(' ', $terms), + $lang, [ new ArchiveFilter(), $excludeMedia diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index 456926d..10fd6e6 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -10,7 +10,6 @@ class IndexerParameter * @param string[] $paths */ public function __construct( - public readonly string $index, public readonly int $cleanupThreshold = 0, public readonly int $chunkSize = 500, public readonly array $paths = [] diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index f0d8213..47bf55f 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -18,13 +18,12 @@ class MoreLikeThisQuery { /** - * @param string $index name of the index to use * @param Filter[] $filter * @param string[] $fields */ public function __construct( - public readonly string $index, public readonly string $location, + public readonly string $lang = '', public readonly array $filter = [], public readonly int $limit = 5, public readonly array $fields = ['description', 'content'] diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SelectQuery.php index 661f992..2fe5b4a 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SelectQuery.php @@ -21,8 +21,8 @@ class SelectQuery * but the SelectQueryBuilder */ public function __construct( - public readonly string $index, public readonly string $text, + public readonly string $lang, public readonly int $offset, public readonly int $limit, public readonly array $sort, diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index fd9c573..f91bc34 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -10,8 +10,8 @@ class SelectQueryBuilder { - private string $index = ''; private string $text = ''; + private string $lang = ''; private int $offset = 0; private int $limit = 10; /** @@ -35,18 +35,15 @@ public function __construct() { } - public function index(string $index): SelectQueryBuilder + public function text(string $text): SelectQueryBuilder { - if (empty($index)) { - throw new \InvalidArgumentException('index is empty'); - } - $this->index = $index; + $this->text = $text; return $this; } - public function text(string $text): SelectQueryBuilder + public function lang(string $lang): SelectQueryBuilder { - $this->text = $text; + $this->lang = $lang; return $this; } @@ -113,12 +110,9 @@ public function defaultQueryOperator( public function build(): SelectQuery { - if (empty($this->index)) { - throw new \InvalidArgumentException('index is not set'); - } return new SelectQuery( - $this->index, $this->text, + $this->lang, $this->offset, $this->limit, $this->sort, diff --git a/src/Dto/Search/Query/SuggestQuery.php b/src/Dto/Search/Query/SuggestQuery.php index c4d76a6..771e0d9 100644 --- a/src/Dto/Search/Query/SuggestQuery.php +++ b/src/Dto/Search/Query/SuggestQuery.php @@ -19,8 +19,8 @@ class SuggestQuery * @param Filter[] $filter */ public function __construct( - public readonly string $index, public readonly string $text, + public readonly string $lang = '', public readonly array $filter = [], public readonly int $limit = 10 ) { diff --git a/src/Indexer.php b/src/Indexer.php index 7760eed..dcbe958 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -20,10 +20,10 @@ interface Indexer { public function index(IndexerParameter $parameter): IndexerStatus; - public function abort(string $index): void; + public function abort(): void; /** * @param string[] $idList */ - public function remove(string $index, array $idList): void; + public function remove(array $idList): void; } diff --git a/src/Service/IndexName.php b/src/Service/IndexName.php new file mode 100644 index 0000000..fd6a7f3 --- /dev/null +++ b/src/Service/IndexName.php @@ -0,0 +1,18 @@ +getIndexer($index)->remove($index, $idList); + $this->getIndexer()->remove($idList); } - public function abort(string $index): void + public function abort(): void { - $this->getIndexer($index)->abort($index); + $this->getIndexer()->abort(); } public function index(IndexerParameter $parameter): IndexerStatus { - $lock = $this->lockFactory->createLock($parameter->index); + $lock = $this->lockFactory->createLock($this->getIndex()); if (!$lock->acquire()) { return IndexerStatus::empty(); } try { - return $this->getIndexer($parameter->index)->index($parameter); + return $this->getIndexer()->index($parameter); } finally { $lock->release(); } @@ -54,19 +56,28 @@ public function index(IndexerParameter $parameter): IndexerStatus /** * @throws ExceptionInterface */ - public function getStatus(string $index): IndexerStatus + public function getStatus(): IndexerStatus { - return $this->statusStore->load($index); + return $this->statusStore->load($this->getIndex()); } - private function getIndexer(string $index): InternalResourceIndexer + private function getIndexer(): InternalResourceIndexer { $progressHandler = new BackgroundIndexerProgressState( - $index, + $this->index->name(''), $this->statusStore, $this->logger ); return $this->indexerFactory->create($progressHandler); } + + private function getIndex(): string + { + /* + * The indexer always requires the default index, as the language is + * determined via the resources to be indexed. + */ + return $this->index->name(''); + } } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 0fa5529..770bdbf 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -54,19 +54,22 @@ public function __construct( /** * @param string[] $idList */ - public function remove(string $index, array $idList): void + public function remove(array $idList): void { if (empty($idList)) { return; } - $this->indexService->deleteByIdList($index, $this->source, $idList); - $this->indexService->commit($index); + $this->indexService->deleteByIdListForAllLanguages( + $this->source, + $idList + ); + $this->indexService->commitForAllLanguages(); } - public function abort(string $index): void + public function abort(): void { - $this->aborter->abort($index); + $this->aborter->abort($this->getBaseIndex()); } /** @@ -85,6 +88,11 @@ public function index(IndexerParameter $parameter): IndexerStatus return $this->indexResources($parameter, $pathList); } + private function getBaseIndex(): string + { + return $this->indexService->getIndex(''); + } + /** * If a path is to be indexed in a translated language, this can also * be specified via the URL parameter `loc`. For example, @@ -130,13 +138,13 @@ private function indexResources( $total = count($pathList); if (empty($parameter->paths)) { - $this->deleteErrorProtocol($parameter->index); + $this->deleteErrorProtocol($this->getBaseIndex()); $this->indexerProgressHandler->start($total); } else { $this->indexerProgressHandler->startUpdate($total); } - $availableIndexes = $this->indexService->getAvailableIndexes(); + $availableIndexes = $this->indexService->getManagedIndexes(); $splitterResult = $this->translationSplitter->split($pathList); $this->indexTranslationSplittedResources( @@ -167,31 +175,39 @@ private function indexTranslationSplittedResources( $processId = uniqid('', true); - if (in_array($parameter->index, $availableIndexes)) { - $this->indexResourcesPerLanguageIndex( - $processId, - $parameter, - $parameter->index, - $splitterResult->getBases() - ); - } else { - $this->indexerProgressHandler->error(new Exception( - 'Index "' . $parameter->index . '" not found' - )); + $index = $this->indexService->getIndex(''); + + if (count($splitterResult->getBases()) > 0) { + if (in_array($index, $availableIndexes)) { + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + '', + $splitterResult->getBases() + ); + } else { + $this->indexerProgressHandler->error(new Exception( + 'Index "' . $index . '" not found' + )); + } } foreach ($splitterResult->getLocales() as $locale) { - $localeIndex = $parameter->index . '-' . $locale; - if (in_array($localeIndex, $availableIndexes)) { + $lang = substr($locale, 0, 2); + $langIndex = $this->indexService->getIndex($lang); + if ( + $index !== $langIndex && + in_array($langIndex, $availableIndexes) + ) { $this->indexResourcesPerLanguageIndex( $processId, $parameter, - $localeIndex, + $lang, $splitterResult->getTranslations($locale) ); } else { $this->indexerProgressHandler->error(new Exception( - 'Index "' . $localeIndex . '" not found' + 'Index "' . $langIndex . '" not found' )); } } @@ -205,7 +221,7 @@ private function indexTranslationSplittedResources( private function indexResourcesPerLanguageIndex( string $processId, IndexerParameter $parameter, - string $index, + string $lang, array $pathList ): void { $offset = 0; @@ -214,7 +230,7 @@ private function indexResourcesPerLanguageIndex( while (true) { $indexedCount = $this->indexChunks( $processId, - $index, + $lang, $pathList, $offset, $parameter->chunkSize @@ -232,12 +248,12 @@ private function indexResourcesPerLanguageIndex( $successCount >= $parameter->cleanupThreshold ) { $this->indexService->deleteExcludingProcessId( - $index, + $lang, $this->source, $processId ); } - $this->indexService->commit($index); + $this->indexService->commit($lang); } /** @@ -251,7 +267,7 @@ private function indexResourcesPerLanguageIndex( */ private function indexChunks( string $processId, - string $solrCore, + string $lang, array $pathList, int $offset, int $length @@ -264,8 +280,9 @@ private function indexChunks( if ($resourceList === false) { return false; } - if ($this->aborter->shouldAborted($solrCore)) { - $this->aborter->aborted($solrCore); + $index = $this->indexService->getIndex($lang); + if ($this->aborter->shouldAborted($index)) { + $this->aborter->aborted($index); $this->indexerProgressHandler->abort(); return false; } @@ -273,7 +290,7 @@ private function indexChunks( return 0; } $this->indexerProgressHandler->advance(count($resourceList)); - $result = $this->add($solrCore, $processId, $resourceList); + $result = $this->add($lang, $processId, $resourceList); if ($result->getStatus() !== 0) { $this->indexerProgressHandler->error(new Exception( @@ -321,12 +338,12 @@ private function loadResources( * @param array $resources */ private function add( - string $solrCore, + string $lang, string $processId, array $resources ): UpdateResult { - $updater = $this->indexService->updater($solrCore); + $updater = $this->indexService->updater($lang); foreach ($resources as $resource) { foreach ($this->documentEnricherList as $enricher) { diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php index 51c051d..da2eefd 100644 --- a/src/Service/Indexer/SolrIndexService.php +++ b/src/Service/Indexer/SolrIndexService.php @@ -4,29 +4,26 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; -use LogicException; +use Solarium\Client; class SolrIndexService { public function __construct( + private readonly IndexName $index, private readonly SolrClientFactory $clientFactory ) { } - public function getIndex(?string $locale = null): string + public function getIndex(string $lang): string { - $client = $this->clientFactory->create($locale); - $core = $client->getEndpoint()->getCore(); - if ($core === null) { - throw new LogicException('Core is not set in Solr client'); - } - return $core; + return $this->index->name($lang); } - public function updater(?string $locale = null): SolrIndexUpdater + public function updater(string $lang): SolrIndexUpdater { - $client = $this->clientFactory->create($locale); + $client = $this->createClient($lang); $update = $client->createUpdate(); $update->setDocumentClass(IndexSchema2xDocument::class); @@ -34,12 +31,12 @@ public function updater(?string $locale = null): SolrIndexUpdater } public function deleteExcludingProcessId( - ?string $locale, + string $lang, string $source, string $processId ): void { $this->deleteByQuery( - $locale, + $lang, '-crawl_process_id:' . $processId . ' AND ' . ' sp_source:' . $source ); @@ -48,52 +45,81 @@ public function deleteExcludingProcessId( /** * @param string[] $idList */ - public function deleteByIdList( - ?string $locale, + public function deleteByIdListForAllLanguages( string $source, array $idList ): void { - $this->deleteByQuery( - $locale, + $this->deleteByQueryForAllLanguages( 'sp_id:(' . implode(' ', $idList) . ') AND ' . 'sp_source:' . $source ); } - public function deleteByQuery(?string $locale, string $query): void + public function deleteByQueryForAllLanguages(string $query): void { - $client = $this->clientFactory->create($locale); + foreach ($this->getManagedIndexes() as $index) { + $client = $this->clientFactory->create($index); + $update = $client->createUpdate(); + $update->addDeleteQuery($query); + $client->update($update); + } + } + + public function deleteByQuery(string $lang, string $query): void + { + $client = $this->createClient($lang); $update = $client->createUpdate(); $update->addDeleteQuery($query); $client->update($update); } - public function commit(?string $locale): void + public function commit(string $lang): void { - $client = $this->clientFactory->create($locale); + $client = $this->createClient($lang); $update = $client->createUpdate(); $update->addCommit(); $update->addOptimize(); $client->update($update); } + public function commitForAllLanguages(): void + { + foreach ($this->getManagedIndexes() as $index) { + $client = $this->clientFactory->create($index); + $update = $client->createUpdate(); + $update->addCommit(); + $update->addOptimize(); + $client->update($update); + } + } + /** * @return string[] */ - public function getAvailableIndexes(): array + public function getManagedIndexes(): array { - $client = $this->clientFactory->create(''); + $client = $this->createClient(''); $coreAdminQuery = $client->createCoreAdmin(); $statusAction = $coreAdminQuery->createStatus(); $coreAdminQuery->setAction($statusAction); - $availableIndexes = []; + $requiredIndexes = $this->index->names(); + + $managedIndexes = []; $response = $client->coreAdmin($coreAdminQuery); $statusResults = $response->getStatusResults() ?? []; foreach ($statusResults as $statusResult) { - $availableIndexes[] = $statusResult->getCoreName(); + $index = $statusResult->getCoreName(); + if (in_array($index, $requiredIndexes, true)) { + $managedIndexes[] = $index; + } } - return $availableIndexes; + return $managedIndexes; + } + + private function createClient(string $lang): Client + { + return $this->clientFactory->create($this->index->name($lang)); } } diff --git a/src/Service/ParameterSolrClientFactory.php b/src/Service/ParameterSolrClientFactory.php index 2778121..58a190f 100644 --- a/src/Service/ParameterSolrClientFactory.php +++ b/src/Service/ParameterSolrClientFactory.php @@ -17,7 +17,6 @@ class ParameterSolrClientFactory implements SolrClientFactory { public function __construct( - private readonly SolrCoreName $coreName, private readonly string $scheme, private readonly string $host, private readonly int $port, @@ -27,9 +26,8 @@ public function __construct( ) { } - public function create(?string $locale = null): Client + public function create(string $core): Client { - $core = $this->coreName->name($locale); $adapter = new Curl(); $adapter->setTimeout($this->timeout); $adapter->setProxy($this->proxy); diff --git a/src/Service/ResourceChannelBasedCoreName.php b/src/Service/ResourceChannelBasedCoreName.php deleted file mode 100644 index 65f4720..0000000 --- a/src/Service/ResourceChannelBasedCoreName.php +++ /dev/null @@ -1,24 +0,0 @@ -resourceChannelFactory->create(); - if ($locale === null) { - return $resourceChannel->searchIndex; - } - return $resourceChannel->searchIndex . '-' . $locale; - } -} diff --git a/src/Service/ResourceChannelBasedIndexName.php b/src/Service/ResourceChannelBasedIndexName.php new file mode 100644 index 0000000..d99a4de --- /dev/null +++ b/src/Service/ResourceChannelBasedIndexName.php @@ -0,0 +1,64 @@ +resourceChannelFactory->create(); + + $locale = $this->langToAvailableLocale($resourceChannel, $lang); + if (empty($locale)) { + return $resourceChannel->searchIndex; + } + return $resourceChannel->searchIndex . '-' . $locale; + } + + /** + * The returned list contains the default index name and the index + * name of all language-specific indexes. + * + * @return string[] + */ + public function names(): array + { + $resourceChannel = $this->resourceChannelFactory->create(); + $names = [$resourceChannel->searchIndex]; + foreach ( + $resourceChannel->translationLocales as $locale + ) { + $names[] = $resourceChannel->searchIndex . '-' . $locale; + } + return $names; + } + + private function langToAvailableLocale( + ResourceChannel $resourceChannel, + string $lang + ): string { + + if (empty($lang)) { + return $lang; + } + + foreach ( + $resourceChannel->translationLocales as $availableLocale + ) { + if (str_starts_with($availableLocale, $lang)) { + return $availableLocale; + } + } + return ''; + } +} diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index 20c25ea..b0d8199 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -29,7 +29,7 @@ public function accept(Document $document): bool ); } - public function create(Document $document): Resource + public function create(Document $document, string $lang): Resource { $location = $this->getField($document, 'url'); if ($location === null) { @@ -41,7 +41,8 @@ public function create(Document $document): Resource '', $this->getField($document, 'title') ?? '', 'external', - [] + $this->getField($document, 'meta_content_language') ?? '', + [], ); } diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php index f5b62f4..59841b2 100644 --- a/src/Service/Search/InternalMediaResourceFactory.php +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -32,13 +32,13 @@ public function accept(Document $document): bool return $this->resourceLoader->exists($metaLocation); } - public function create(Document $document): Resource + public function create(Document $document, string $lang): Resource { $metaLocation = $this->getMetaLocation($document); if ($metaLocation === null) { throw new LogicException('document should contains a url'); } - return $this->resourceLoader->load($metaLocation); + return $this->resourceLoader->load($metaLocation, $lang); } private function getMetaLocation(Document $document): ?string diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php index 30620f5..4aa344f 100644 --- a/src/Service/Search/InternalResourceFactory.php +++ b/src/Service/Search/InternalResourceFactory.php @@ -30,13 +30,13 @@ public function accept(Document $document): bool return str_ends_with($location, '.php'); } - public function create(Document $document): Resource + public function create(Document $document, string $lang): Resource { $location = $this->getField($document, 'url'); if ($location === null) { throw new LogicException('document should contains a url'); } - return $this->resourceLoader->load($location); + return $this->resourceLoader->load($location, $lang); } private function getField(Document $document, string $name): ?string diff --git a/src/Service/Search/ResourceFactory.php b/src/Service/Search/ResourceFactory.php index 6fbaea0..e1e7dc8 100644 --- a/src/Service/Search/ResourceFactory.php +++ b/src/Service/Search/ResourceFactory.php @@ -23,5 +23,5 @@ interface ResourceFactory { public function accept(Document $document): bool; - public function create(Document $document): Resource; + public function create(Document $document, string $lang): Resource; } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 1a14494..cd8c6ca 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -7,6 +7,7 @@ use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\MoreLikeThisSearcher; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Core\Client\Client; use Solarium\QueryType\MoreLikeThis\Query as SolrMoreLikeThisQuery; @@ -18,6 +19,7 @@ class SolrMoreLikeThis implements MoreLikeThisSearcher { public function __construct( + private readonly IndexName $index, private readonly SolrClientFactory $clientFactory, private readonly SolrResultToResourceResolver $resultToResourceResolver ) { @@ -25,11 +27,12 @@ public function __construct( public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { - $client = $this->clientFactory->create($query->index); + $index = $this->index->name($query->lang); + $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SolrMoreLikeThisResult $result */ $result = $client->execute($solrQuery); - return $this->buildResult($result); + return $this->buildResult($result, $query->lang); } private function buildSolrQuery( @@ -58,11 +61,12 @@ private function buildSolrQuery( } private function buildResult( - SolrMoreLikeThisResult $result + SolrMoreLikeThisResult $result, + string $lang ): SearchResult { $resourceList = $this->resultToResourceResolver - ->loadResourceList($result); + ->loadResourceList($result, $lang); return new SearchResult( $result->getNumFound() ?? -1, diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index 68da172..eaccdda 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -30,13 +30,13 @@ public function __construct( /** * @return array */ - public function loadResourceList(SelectResult $result): array + public function loadResourceList(SelectResult $result, string $lang): array { $resourceList = []; /** @var Document $document */ foreach ($result as $document) { try { - $resourceList[] = $this->loadResource($document); + $resourceList[] = $this->loadResource($document, $lang); } catch (\Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); } @@ -44,12 +44,12 @@ public function loadResourceList(SelectResult $result): array return $resourceList; } - private function loadResource(Document $document): Resource + private function loadResource(Document $document, string $lang): Resource { foreach ($this->resourceFactoryList as $resourceFactory) { if ($resourceFactory->accept($document)) { - return $resourceFactory->create($document); + return $resourceFactory->create($document, $lang); } } diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index d936886..8c4c3eb 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -20,6 +20,7 @@ use Atoolo\Search\Dto\Search\Result\FacetGroup; use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\SelectSearcher; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use InvalidArgumentException; use Solarium\Component\Facet\Field; @@ -36,6 +37,7 @@ class SolrSelect implements SelectSearcher * @param iterable $solrQueryModifierList */ public function __construct( + private readonly IndexName $index, private readonly SolrClientFactory $clientFactory, private readonly iterable $solrQueryModifierList, private readonly SolrResultToResourceResolver $resultToResourceResolver @@ -44,12 +46,13 @@ public function __construct( public function select(SelectQuery $query): SearchResult { - $client = $this->clientFactory->create($query->index); + $index = $this->index->name($query->lang); + $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SelectResult $result */ $result = $client->execute($solrQuery); - return $this->buildResult($query, $result); + return $this->buildResult($query, $result, $query->lang); } private function buildSolrQuery( @@ -250,11 +253,12 @@ private function addFacetMultiQueryToSolrQuery( private function buildResult( SelectQuery $query, - SelectResult $result + SelectResult $result, + string $lang ): SearchResult { $resourceList = $this->resultToResourceResolver - ->loadResourceList($result); + ->loadResourceList($result, $lang); $facetGroupList = $this->buildFacetGroupList($query, $result); return new SearchResult( diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 9a829ef..45fc6d5 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -8,6 +8,7 @@ use Atoolo\Search\Dto\Search\Result\Suggestion; use Atoolo\Search\Dto\Search\Result\SuggestResult; use Atoolo\Search\Exception\UnexpectedResultException; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use Atoolo\Search\SuggestSearcher; use JsonException; @@ -29,6 +30,7 @@ class SolrSuggest implements SuggestSearcher private const INDEX_SUGGEST_FIELD = 'raw_content'; public function __construct( + private readonly IndexName $index, private readonly SolrClientFactory $clientFactory ) { } @@ -38,7 +40,8 @@ public function __construct( */ public function suggest(SuggestQuery $query): SuggestResult { - $client = $this->clientFactory->create($query->index); + $index = $this->index->name($query->lang); + $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); $solrResult = $client->select($solrQuery); diff --git a/src/Service/ServerVarSolrClientFactory.php b/src/Service/ServerVarSolrClientFactory.php index e8c9b88..a63bb39 100644 --- a/src/Service/ServerVarSolrClientFactory.php +++ b/src/Service/ServerVarSolrClientFactory.php @@ -9,28 +9,13 @@ use Symfony\Component\EventDispatcher\EventDispatcher; /** - * Determines the Solr connection information via server variables. - * The following server variables are used: - * - SOLR_SCHEME (default: http) - * - SOLR_HOST (default: localhost) - * - SOLR_PORT (default: 8382) - * - SOLR_URL (default: '') - * - * The SOLR_URL variable is used to specify the full URL to the Solr server. - * If the SOLR_URL variable is set, the other variables are ignored. */ class ServerVarSolrClientFactory implements SolrClientFactory { private const IES_WEBNODE_SOLR_PORT = '8382'; - public function __construct( - private readonly SolrCoreName $coreName - ) { - } - - public function create(?string $locale = null): Client + public function create(string $core): Client { - $core = $this->coreName->name($locale); $adapter = new Curl(); /* $adapter->setTimeout($this->timeout); @@ -58,7 +43,7 @@ public function create(?string $locale = null): Client * core: string * }> */ - public function getEndpointConfig(string $core): array + private function getEndpointConfig(string $core): array { $url = $_SERVER['SOLR_URL'] ?? ''; diff --git a/src/Service/SolrClientFactory.php b/src/Service/SolrClientFactory.php index e2aebb0..ce26af6 100644 --- a/src/Service/SolrClientFactory.php +++ b/src/Service/SolrClientFactory.php @@ -12,5 +12,5 @@ */ interface SolrClientFactory { - public function create(?string $locale = null): Client; + public function create(string $core): Client; } diff --git a/src/Service/SolrCoreName.php b/src/Service/SolrCoreName.php deleted file mode 100644 index 6853c93..0000000 --- a/src/Service/SolrCoreName.php +++ /dev/null @@ -1,10 +0,0 @@ -createStub( - InternalResourceIndexerBuilder::class + $indexer = $this->createStub( + InternalResourceIndexer::class ); + $progressBar = $this->createStub(IndexerProgressBar::class); $application = new Application([ - new Indexer([], $indexBuilder, new IndexerProgressBarFactory()) + new Indexer($progressBar, $indexer) ]); $command = $application->get('atoolo:indexer'); $this->assertInstanceOf( diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php index ad9487a..55ef55c 100644 --- a/test/Console/Command/DumpIndexDocumentTest.php +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -28,17 +28,9 @@ public function setUp(): void ->willReturn([ ['sp_id' => '123'] ]); - $dumperBuilder = $this->createStub(IndexDocumentDumperBuilder::class); - $dumperBuilder->method('resourceDir') - ->willReturn($dumperBuilder); - $dumperBuilder->method('documentEnricherList') - ->willReturn($dumperBuilder); - $dumperBuilder->method('build') - ->willReturn($dumper); $dumperCommand = new DumpIndexDocument( - [], - $dumperBuilder + $dumper ); $application = new Application([ @@ -52,7 +44,6 @@ public function setUp(): void public function testExecute(): void { $this->commandTester->execute([ - 'resource-dir' => 'abc', 'paths' => ['test.php'] ]); diff --git a/test/Console/Command/IndexDocumentDumperBuilderTest.php b/test/Console/Command/IndexDocumentDumperBuilderTest.php deleted file mode 100644 index c15fb84..0000000 --- a/test/Console/Command/IndexDocumentDumperBuilderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -createStub(ResourceBaseLocatorBuilder::class) - ); - $builder->resourceDir('test'); - $builder->documentEnricherList([]); - - $this->expectNotToPerformAssertions(); - $builder->build(); - } -} diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 502fc82..4ec616a 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -6,9 +6,8 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; -use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; -use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; +use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; @@ -25,19 +24,17 @@ class IndexerTest extends TestCase */ public function setUp(): void { - $indexBuilder = $this->createStub( - InternalResourceIndexerBuilder::class + $indexer = $this->createStub( + InternalResourceIndexer::class ); + $progressBar = $this->createStub(IndexerProgressBar::class); - $indexer = new Indexer( - [], - $indexBuilder, - new IndexerProgressBarFactory() + $command = new Indexer( + $progressBar, + $indexer, ); - $application = new Application([ - $indexer - ]); + $application = new Application([$command]); $command = $application->find('atoolo:indexer'); $this->commandTester = new CommandTester($command); @@ -45,12 +42,7 @@ public function setUp(): void public function testExecuteIndexAll(): void { - $this->commandTester->execute([ - // pass arguments to the helper - 'resource-dir' => 'abc', - 'solr-connection-url' => 'http://localhost:8080', - 'solr-core' => 'test' - ]); + $this->commandTester->execute([]); $this->commandTester->assertCommandIsSuccessful(); @@ -72,9 +64,6 @@ public function testExecuteIndexPath(): void { $this->commandTester->execute([ // pass arguments to the helper - 'resource-dir' => 'abc', - 'solr-connection-url' => 'http://localhost:8080', - 'solr-core' => 'test', 'paths' => ['a.php', 'b.php'] ]); @@ -103,8 +92,8 @@ public function testExecuteIndexPath(): void public function testExecuteIndexWithErrors(): void { - $indexBuilder = $this->createStub( - InternalResourceIndexerBuilder::class + $indexer = $this->createStub( + InternalResourceIndexer::class ); $progressBar = $this->createStub( @@ -114,32 +103,17 @@ public function testExecuteIndexWithErrors(): void ->method('getErrors') ->willReturn([new \Exception('errortest')]); - $progressBarFactory = $this->createStub( - IndexerProgressBarFactory::class - ); - $progressBarFactory - ->method('create') - ->willReturn($progressBar); - - $indexer = new Indexer( - [], - $indexBuilder, - $progressBarFactory + $command = new Indexer( + $progressBar, + $indexer, ); - $application = new Application([ - $indexer - ]); + $application = new Application([$command]); $command = $application->find('atoolo:indexer'); $commandTester = new CommandTester($command); - $commandTester->execute([ - // pass arguments to the helper - 'resource-dir' => 'abc', - 'solr-connection-url' => 'http://localhost:8080', - 'solr-core' => 'test' - ]); + $commandTester->execute([]); $commandTester->assertCommandIsSuccessful(); @@ -159,8 +133,8 @@ public function testExecuteIndexWithErrors(): void public function testExecuteIndexWithErrorsAndStackTrace(): void { - $indexBuilder = $this->createStub( - InternalResourceIndexerBuilder::class + $indexer = $this->createStub( + InternalResourceIndexer::class ); $progressBar = $this->createStub( @@ -170,32 +144,17 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void ->method('getErrors') ->willReturn([new \Exception('errortest')]); - $progressBarFactory = $this->createStub( - IndexerProgressBarFactory::class - ); - $progressBarFactory - ->method('create') - ->willReturn($progressBar); - - $indexer = new Indexer( - [], - $indexBuilder, - $progressBarFactory + $command = new Indexer( + $progressBar, + $indexer, ); - $application = new Application([ - $indexer - ]); + $application = new Application([$command]); $command = $application->find('atoolo:indexer'); $commandTester = new CommandTester($command); - $commandTester->execute([ - // pass arguments to the helper - 'resource-dir' => 'abc', - 'solr-connection-url' => 'http://localhost:8080', - 'solr-core' => 'test' - ], [ + $commandTester->execute([], [ 'verbosity' => OutputInterface::VERBOSITY_VERBOSE ]); diff --git a/test/Console/Command/Io/IndexerProgressBarFactoryTest.php b/test/Console/Command/Io/IndexerProgressBarFactoryTest.php deleted file mode 100644 index 5d659d4..0000000 --- a/test/Console/Command/Io/IndexerProgressBarFactoryTest.php +++ /dev/null @@ -1,21 +0,0 @@ -expectNotToPerformAssertions(); - $factory->create($this->createStub(OutputInterface::class)); - } -} diff --git a/test/Console/Command/Io/TypifiedInputTest.php b/test/Console/Command/Io/TypifiedInputTest.php index 320d4b1..d6ae49f 100644 --- a/test/Console/Command/Io/TypifiedInputTest.php +++ b/test/Console/Command/Io/TypifiedInputTest.php @@ -42,6 +42,35 @@ public function testGetIntOptWithInvalidValue(): void $input->getIntOption('a'); } + public function testGetStringOption(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 'abc', + $input->getStringOption('a'), + 'unexpected option value' + ); + } + + public function testGetStringOptWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getStringOption('a'); + } + public function testGetStringArgument(): void { $symfonyInput = $this->createStub(InputInterface::class); diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index dfc85a9..05139b5 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -7,7 +7,6 @@ use Atoolo\Resource\Resource; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\MoreLikeThis; -use Atoolo\Search\Console\Command\SolrMoreLikeThisBuilder; use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\Service\Search\SolrMoreLikeThis; use PHPUnit\Framework\Attributes\CoversClass; @@ -39,19 +38,10 @@ public function setUp(): void $solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); $solrMoreLikeThis->method('moreLikeThis') ->willReturn($result); - $moreLikeThisBuilder = $this->createStub( - SolrMoreLikeThisBuilder::class - ); - $moreLikeThisBuilder->method('build') - ->willReturn($solrMoreLikeThis); - $moreLinkeThis = new MoreLikeThis( - $moreLikeThisBuilder - ); + $command = new MoreLikeThis($solrMoreLikeThis); - $application = new Application([ - $moreLinkeThis - ]); + $application = new Application([$command]); $command = $application->find('atoolo:mlt'); $this->commandTester = new CommandTester($command); @@ -60,10 +50,6 @@ public function setUp(): void public function testExecute(): void { $this->commandTester->execute([ - // pass arguments to the helper - 'resource-dir' => 'abc', - 'solr-connection-url' => 'http://localhost:8080', - 'solr-core' => 'test', 'location' => '/test.php' ]); diff --git a/test/Console/Command/ResourceBaseLocatorBuilderTest.php b/test/Console/Command/ResourceBaseLocatorBuilderTest.php deleted file mode 100644 index 8a1a918..0000000 --- a/test/Console/Command/ResourceBaseLocatorBuilderTest.php +++ /dev/null @@ -1,33 +0,0 @@ -build($resourceDir); - - $this->assertEquals( - $resourceDir, - $locator->locate(), - 'unexpected resource dir' - ); - } -} diff --git a/test/Console/Command/SearchTest.php b/test/Console/Command/SearchTest.php index ac7831f..e5db978 100644 --- a/test/Console/Command/SearchTest.php +++ b/test/Console/Command/SearchTest.php @@ -7,7 +7,6 @@ use Atoolo\Resource\Resource; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Search; -use Atoolo\Search\Console\Command\SolrSelectBuilder; use Atoolo\Search\Dto\Search\Result\Facet; use Atoolo\Search\Dto\Search\Result\FacetGroup; use Atoolo\Search\Dto\Search\Result\SearchResult; @@ -48,19 +47,9 @@ public function setUp(): void $solrSelect->method('select') ->willReturn($result); - $solrSelectBuilder = $this->createStub( - SolrSelectBuilder::class - ); - $solrSelectBuilder->method('build') - ->willReturn($solrSelect); - - $search = new Search( - $solrSelectBuilder - ); + $command = new Search($solrSelect); - $application = new Application([ - $search - ]); + $application = new Application([$command]); $command = $application->find('atoolo:search'); $this->commandTester = new CommandTester($command); @@ -69,9 +58,6 @@ public function setUp(): void public function testExecute(): void { $this->commandTester->execute([ - 'solr-connection-url' => 'http://localhost:8382', - 'index' => 'test', - 'resource-dir' => 'abc', 'text' => ['test', 'abc'] ]); diff --git a/test/Console/Command/SolrIndexerBuilderTest.php b/test/Console/Command/SolrIndexerBuilderTest.php deleted file mode 100644 index 2c6b122..0000000 --- a/test/Console/Command/SolrIndexerBuilderTest.php +++ /dev/null @@ -1,32 +0,0 @@ -createStub(ResourceBaseLocatorBuilder::class) - ); - $builder - ->resourceDir('test') - ->documentEnricherList([$this->createStub(DocumentEnricher::class)]) - ->progressBar($this->createStub(IndexerProgressBar::class)) - ->solrConnectionUrl('http://localhost:8382'); - - $this->expectNotToPerformAssertions(); - $builder->build(); - } -} diff --git a/test/Console/Command/SolrMoreLikeThisBuilderTest.php b/test/Console/Command/SolrMoreLikeThisBuilderTest.php deleted file mode 100644 index 00aabcf..0000000 --- a/test/Console/Command/SolrMoreLikeThisBuilderTest.php +++ /dev/null @@ -1,27 +0,0 @@ -createStub(ResourceBaseLocatorBuilder::class) - ); - $builder - ->resourceDir('test.php') - ->solrConnectionUrl('http://localhost:8382'); - - $this->expectNotToPerformAssertions(); - $builder->build(); - } -} diff --git a/test/Console/Command/SolrSelectBuilderTest.php b/test/Console/Command/SolrSelectBuilderTest.php deleted file mode 100644 index c81d423..0000000 --- a/test/Console/Command/SolrSelectBuilderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -createStub(ResourceBaseLocatorBuilder::class) - ); - $builder->resourceDir('test') - ->solrConnectionUrl('http://localhost:8382'); - - $this->expectNotToPerformAssertions(); - $builder->build(); - } -} diff --git a/test/Console/Command/SolrSuggestBuilderTest.php b/test/Console/Command/SolrSuggestBuilderTest.php deleted file mode 100644 index 621c7cc..0000000 --- a/test/Console/Command/SolrSuggestBuilderTest.php +++ /dev/null @@ -1,22 +0,0 @@ -solrConnectionUrl('http://localhost:8283'); - - $this->expectNotToPerformAssertions(); - $builder->build(); - } -} diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index b090a15..11d2758 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -35,19 +35,10 @@ public function setUp(): void $solrSuggest = $this->createStub(SolrSuggest::class); $solrSuggest->method('suggest') ->willReturn($result); - $solrSuggestBuilder = $this->createStub( - SolrSuggestBuilder::class - ); - $solrSuggestBuilder->method('build') - ->willReturn($solrSuggest); - $suggest = new Suggest( - $solrSuggestBuilder - ); + $command = new Suggest($solrSuggest); - $application = new Application([ - $suggest - ]); + $application = new Application([$command]); $command = $application->find('atoolo:suggest'); $this->commandTester = new CommandTester($command); @@ -56,8 +47,6 @@ public function setUp(): void public function testExecute(): void { $this->commandTester->execute([ - 'solr-connection-url' => 'http://localhost:8382', - 'solr-core' => 'test', 'terms' => ['sec'] ]); diff --git a/test/Dto/Indexer/IndexerParameterTest.php b/test/Dto/Indexer/IndexerParameterTest.php index 91a0073..5894985 100644 --- a/test/Dto/Indexer/IndexerParameterTest.php +++ b/test/Dto/Indexer/IndexerParameterTest.php @@ -16,7 +16,6 @@ public function testToLowerChunkSize(): void { $this->expectException(InvalidArgumentException::class); new IndexerParameter( - 'test', 0, 9 ); diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SelectQueryBuilderTest.php index e02c4ce..fbe92b4 100644 --- a/test/Dto/Search/Query/SelectQueryBuilderTest.php +++ b/test/Dto/Search/Query/SelectQueryBuilderTest.php @@ -11,6 +11,7 @@ use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; #[CoversClass(SelectQueryBuilder::class)] @@ -21,34 +22,6 @@ class SelectQueryBuilderTest extends TestCase protected function setUp(): void { $this->builder = new SelectQueryBuilder(); - $this->builder->index('myindex'); - } - - public function testBuildWithoutIndex(): void - { - $this->expectException(InvalidArgumentException::class); - $builder = new SelectQueryBuilder(); - $builder->build(); - } - - public function testSetEmptyIndex(): void - { - $this->expectException(InvalidArgumentException::class); - $builder = new SelectQueryBuilder(); - $builder->index(''); - } - - public function testSetIndex(): void - { - $builder = new SelectQueryBuilder(); - $builder->index('myindex'); - $query = $builder->build(); - - $this->assertEquals( - 'myindex', - $query->index, - 'unexpected index' - ); } public function testSetText(): void @@ -58,6 +31,13 @@ public function testSetText(): void $this->assertEquals('abc', $query->text, 'unexpected text'); } + public function testSetLang(): void + { + $this->builder->lang('en'); + $query = $this->builder->build(); + $this->assertEquals('en', $query->lang, 'unexpected lang'); + } + public function testSetOffset(): void { $this->builder->offset(10); @@ -84,6 +64,9 @@ public function testSetInvalidLimit(): void $this->builder->limit(-1); } + /** + * @throws Exception + */ public function testSetSort(): void { $criteria = $this->createStub(Criteria::class); diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index 5039b12..b49b2f0 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -9,6 +9,7 @@ use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; +use Atoolo\Search\Service\IndexName; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -19,12 +20,15 @@ #[CoversClass(BackgroundIndexer::class)] class BackgroundIndexerTest extends TestCase { + private IndexName $indexName; private InternalResourceIndexer&MockObject $solrIndexer; private IndexerStatusStore&MockObject $statusStore; private BackgroundIndexer $indexer; public function setUp(): void { + + $this->indexName = $this->createMock(IndexName::class); $this->solrIndexer = $this->createMock(InternalResourceIndexer::class); $solrIndexerFactory = $this->createStub( InternalResourceIndexerFactory::class @@ -34,6 +38,7 @@ public function setUp(): void $this->statusStore = $this->createMock(IndexerStatusStore::class); $this->indexer = new BackgroundIndexer( $solrIndexerFactory, + $this->indexName, $this->statusStore ); } @@ -42,19 +47,19 @@ public function testRemove(): void { $this->solrIndexer->expects($this->once()) ->method('remove'); - $this->indexer->remove('test', ['123']); + $this->indexer->remove(['123']); } public function testAbort(): void { $this->solrIndexer->expects($this->once()) ->method('abort'); - $this->indexer->abort('test'); + $this->indexer->abort(); } public function testIndex(): void { - $params = new IndexerParameter('test'); + $params = new IndexerParameter(); $this->solrIndexer->expects($this->once()) ->method('index'); $this->indexer->index($params); @@ -65,6 +70,7 @@ public function testIndexIfLocked(): void $lockFactory = $this->createStub(LockFactory::class); $indexer = new BackgroundIndexer( $this->createStub(InternalResourceIndexerFactory::class), + $this->createStub(IndexName::class), $this->statusStore, new NullLogger(), $lockFactory @@ -79,15 +85,14 @@ public function testIndexIfLocked(): void $this->solrIndexer->expects($this->exactly(0)) ->method('index'); - $params = new IndexerParameter('test'); + $params = new IndexerParameter(); $indexer->index($params); } public function testGetStatus(): void { - $params = new IndexerParameter('test'); $this->statusStore->expects($this->once()) ->method('load'); - $this->indexer->getStatus('test'); + $this->indexer->getStatus(); } } diff --git a/test/Service/Indexer/IndexingAborterTest.php b/test/Service/Indexer/IndexingAborterTest.php index 3bad523..2e63ddc 100644 --- a/test/Service/Indexer/IndexingAborterTest.php +++ b/test/Service/Indexer/IndexingAborterTest.php @@ -25,7 +25,7 @@ public function setUp(): void if (file_exists($this->file)) { unlink($this->file); } - $this->aborter = new IndexingAborter($workdir); + $this->aborter = new IndexingAborter($workdir, 'background-indexer'); } public function testShouldNotAborted(): void diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index c143fa6..836ec94 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -26,6 +26,9 @@ #[CoversClass(InternalResourceIndexer::class)] class InternalResourceIndexerTest extends TestCase { + /** + * @var string[] + */ private array $availableIndexes = ['test', 'test-en_US']; private ResourceLoader&Stub $resourceLoader; @@ -79,10 +82,17 @@ public function setUp(): void $this->updater->method('createDocument')->willReturn( new IndexSchema2xDocument() ); - $this->solrIndexService->method('getAvailableIndexes') + $this->solrIndexService->method('getManagedIndexes') ->willReturnCallback(function () { return $this->availableIndexes; }); + $this->solrIndexService->method('getIndex') + ->willReturnCallback(function ($lang) { + if ($lang === 'en') { + return 'test-en_US'; + } + return 'test'; + }); $this->solrIndexService->method('updater') ->willReturn($this->updater); $this->aborter = $this->createMock(IndexingAborter::class); @@ -95,7 +105,7 @@ public function setUp(): void $this->translationSplitter, $this->solrIndexService, $this->aborter, - 'test' + 'test-source' ); } @@ -105,23 +115,23 @@ public function testAbort(): void ->method('abort') ->with('test'); - $this->indexer->abort('test'); + $this->indexer->abort(); } public function testRemove(): void { $this->solrIndexService->expects($this->once()) - ->method('deleteByIdList'); + ->method('deleteByIdListForAllLanguages'); - $this->indexer->remove('test', ['123']); + $this->indexer->remove(['123']); } public function testRemoveEmpty(): void { $this->solrIndexService->expects($this->exactly(0)) - ->method('deleteByIdList'); + ->method('deleteByIdListForAllLanguages'); - $this->indexer->remove('test', []); + $this->indexer->remove([]); } public function testIndexAllWithChunks(): void @@ -166,7 +176,6 @@ public function testIndexAllWithChunks(): void ->method('update'); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -201,7 +210,6 @@ public function testIndexSkipResource(): void ->method('update'); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -227,7 +235,6 @@ public function testAborted(): void ->method('abort'); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -253,7 +260,6 @@ public function testWithUnsuccessfulStatus(): void ->method('error'); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -275,7 +281,6 @@ public function testWithInvalidResource(): void ->method('error'); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -289,7 +294,6 @@ public function testEmptyStatus(): void ->willReturn([]); $parameter = new IndexerParameter( - 'test', 10, 10 ); @@ -319,7 +323,6 @@ public function testIndexPaths(): void ->method('update'); $parameter = new IndexerParameter( - 'test', 10, 10, [ @@ -338,7 +341,6 @@ public function testIndexPathWithParameter(): void ->with($this->equalTo(['?a=b'])); $parameter = new IndexerParameter( - 'test', 10, 10, [ @@ -356,7 +358,6 @@ public function testIndexPathWithParameterAndPath(): void ->with($this->equalTo(['/test.php'])); $parameter = new IndexerParameter( - 'test', 10, 10, [ @@ -374,7 +375,6 @@ public function testIndexPathWithLocParameterAndPath(): void ->with($this->equalTo(['/test.php.translations/en.php'])); $parameter = new IndexerParameter( - 'test', 10, 10, [ @@ -405,7 +405,6 @@ public function testWithoutAvailableIndexes(): void ]); $parameter = new IndexerParameter( - 'test', 10, 10 ); diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php index b69854b..cfb523b 100644 --- a/test/Service/Indexer/SolrIndexServiceTest.php +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Search\Service\Indexer\SolrIndexService; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\MockObject; @@ -16,48 +17,82 @@ #[CoversClass(SolrIndexService::class)] class SolrIndexServiceTest extends TestCase { + private IndexName $indexName; private SolrIndexService $indexService; private SolrClientFactory&MockObject $factory; private Client&MockObject $client; public function setUp(): void { + $statusResult = $this->createStub(StatusResult::class); + $statusResult->method('getCoreName')->willReturn('test'); + $statusResultEn = $this->createStub(StatusResult::class); + $statusResultEn->method('getCoreName')->willReturn('test-en_US'); + $response = $this->createStub(CoreAdminResult::class); + $response->method('getStatusResults')->willReturn([ + $statusResult, + $statusResultEn + ]); + $this->client = $this->createMock(Client::class); + $this->client->method('coreAdmin')->willReturn($response); + $this->indexName = $this->createMock(IndexName::class); + $this->indexName->method('name')->willReturn('test', 'test-en_US'); + $this->indexName->method('names')->willReturn(['test', 'test-en_US']); $this->factory = $this->createMock(SolrClientFactory::class); $this->factory->method('create')->willReturn($this->client); - $this->indexService = new SolrIndexService($this->factory); + $this->indexService = new SolrIndexService( + $this->indexName, + $this->factory + ); } public function testUpdater(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->updater('test'); + $this->indexService->updater(''); + } + + public function testGetIndex(): void + { + $index = $this->indexService->getIndex(''); + $this->assertEquals( + 'test', + $index, + 'Index name should be returned' + ); } public function testDeleteExcludingProcessId(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->deleteExcludingProcessId('test', 'test', 'test'); + $this->indexService->deleteExcludingProcessId('', 'test', 'test'); } - public function testDeleteByIdList(): void + public function testDeleteByIdListForAllLanguages(): void { - $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->deleteByIdList('test', 'test', ['test']); + $this->client->expects($this->exactly(2))->method('createUpdate'); + $this->indexService->deleteByIdListForAllLanguages('test', ['test']); } public function testByQuery(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->deleteByQuery('test', 'test'); + $this->indexService->deleteByQuery('', 'test'); } public function testCommit(): void { $this->client->expects($this->once())->method('update'); - $this->indexService->commit('test'); + $this->indexService->commit(''); + } + + public function testCommitForAllLanguages(): void + { + $this->client->expects($this->exactly(2))->method('update'); + $this->indexService->commitForAllLanguages(); } - public function testGetAvailableCores(): void + public function testGetManagedIndexes(): void { $statusResult = $this->createStub(StatusResult::class); $statusResult->method('getCoreName')->willReturn('test'); @@ -65,8 +100,12 @@ public function testGetAvailableCores(): void $response->method('getStatusResults')->willReturn([$statusResult]); $this->client->method('coreAdmin')->willReturn($response); - $cores = $this->indexService->getAvailableIndexes(); + $cores = $this->indexService->getManagedIndexes(); - $this->assertEquals(['test'], $cores, 'Cores should be returned'); + $this->assertEquals( + ['test', 'test-en_US'], + $cores, + 'Cores should be returned' + ); } } diff --git a/test/Service/ResourceChannelBasedIndexNameTest.php b/test/Service/ResourceChannelBasedIndexNameTest.php new file mode 100644 index 0000000..25cee94 --- /dev/null +++ b/test/Service/ResourceChannelBasedIndexNameTest.php @@ -0,0 +1,82 @@ +createStub( + ResourceChannelFactory::class + ); + $resourceChannel = new ResourceChannel( + '', + '', + '', + '', + false, + '', + '', + '', + 'test', + ['en_US'] + ); + + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + + $this->indexName = new ResourceChannelBasedIndexName( + $resourceChannelFactory + ); + } + + public function testName(): void + { + $this->assertEquals( + 'test', + $this->indexName->name(''), + 'The default index name should be returned ' . + 'if no language is given' + ); + } + + public function testNameWithLang(): void + { + $this->assertEquals( + 'test-en_US', + $this->indexName->name('en'), + 'The language-specific index name should be returned ' . + 'if a language is given' + ); + } + + public function testNameWithUnsupportedLang(): void + { + $this->assertEquals( + 'test', + $this->indexName->name('it'), + 'The default index name should be returned ' . + 'if the language is not supported' + ); + } + + public function testNames(): void + { + $this->assertEquals( + ['test', 'test-en_US'], + $this->indexName->names(), + 'All index names should be returned' + ); + } +} diff --git a/test/Service/Search/ExternalResourceFactoryTest.php b/test/Service/Search/ExternalResourceFactoryTest.php index 9b8ffb1..b4f3c9f 100644 --- a/test/Service/Search/ExternalResourceFactoryTest.php +++ b/test/Service/Search/ExternalResourceFactoryTest.php @@ -49,7 +49,7 @@ public function testAcceptWithoutUrl(): void public function testCreate(): void { $document = $this->createDocument('https://www.sitepark.com'); - $resource = $this->factory->create($document); + $resource = $this->factory->create($document, 'de'); $this->assertEquals( 'https://www.sitepark.com', @@ -61,7 +61,7 @@ public function testCreate(): void public function testCreateWithName(): void { $document = $this->createDocument('https://www.sitepark.com', 'Test'); - $resource = $this->factory->create($document); + $resource = $this->factory->create($document, 'de'); $this->assertEquals( 'Test', @@ -75,7 +75,7 @@ public function testCreateWithMissingUrl(): void $document = $this->createStub(Document::class); $this->expectException(\LogicException::class); - $this->factory->create($document); + $this->factory->create($document, 'de'); } private function createDocument(string $url, string $title = ''): Document diff --git a/test/Service/Search/InternalMediaResourceFactoryTest.php b/test/Service/Search/InternalMediaResourceFactoryTest.php index 12d5f73..925ed79 100644 --- a/test/Service/Search/InternalMediaResourceFactoryTest.php +++ b/test/Service/Search/InternalMediaResourceFactoryTest.php @@ -74,7 +74,7 @@ public function testCreate(): void $this->assertEquals( $resource, - $this->factory->create($document), + $this->factory->create($document, 'de'), 'unexpected resource' ); } @@ -83,7 +83,7 @@ public function testCreateWithoutUrl(): void { $document = $this->createStub(Document::class); $this->expectException(LogicException::class); - $this->factory->create($document); + $this->factory->create($document, 'de'); } private function createDocument(string $url): Document diff --git a/test/Service/Search/InternalResourceFactoryTest.php b/test/Service/Search/InternalResourceFactoryTest.php index 0d83fc5..a457d05 100644 --- a/test/Service/Search/InternalResourceFactoryTest.php +++ b/test/Service/Search/InternalResourceFactoryTest.php @@ -66,7 +66,7 @@ public function testCreate(): void $this->assertEquals( $resource, - $this->factory->create($document), + $this->factory->create($document, 'de'), 'unexpected resource' ); } @@ -75,7 +75,7 @@ public function testCreateWithoutUrl(): void { $document = $this->createStub(Document::class); $this->expectException(LogicException::class); - $this->factory->create($document); + $this->factory->create($document, 'de'); } private function createDocument(string $url): Document diff --git a/test/Service/Search/SolrMoreLikeThisTest.php b/test/Service/Search/SolrMoreLikeThisTest.php index 882b48a..56c868a 100644 --- a/test/Service/Search/SolrMoreLikeThisTest.php +++ b/test/Service/Search/SolrMoreLikeThisTest.php @@ -7,6 +7,7 @@ use Atoolo\Resource\Resource; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\Search\SolrMoreLikeThis; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\SolrClientFactory; @@ -27,6 +28,8 @@ class SolrMoreLikeThisTest extends TestCase protected function setUp(): void { + + $indexName = $this->createStub(IndexName::class); $clientFactory = $this->createStub( SolrClientFactory::class ); @@ -52,6 +55,7 @@ protected function setUp(): void ->willReturn([$this->resource]); $this->searcher = new SolrMoreLikeThis( + $indexName, $clientFactory, $resultToResourceResolver ); @@ -64,8 +68,8 @@ public function testMoreLikeThis(): void ->getMock(); $query = new MoreLikeThisQuery( - 'myindex', '/test.php', + '', [$filter] ); diff --git a/test/Service/Search/SolrResultToResourceResolverTest.php b/test/Service/Search/SolrResultToResourceResolverTest.php index 5c4dd05..3589d37 100644 --- a/test/Service/Search/SolrResultToResourceResolverTest.php +++ b/test/Service/Search/SolrResultToResourceResolverTest.php @@ -32,7 +32,7 @@ public function testLoadResourceList(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result); + $resourceList = $resolver->loadResourceList($result, ''); $this->assertEquals( [$resource], @@ -55,7 +55,7 @@ public function testLoadResourceListWithoutAcceptedFactory(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result); + $resourceList = $resolver->loadResourceList($result, ''); $this->assertEmpty( $resourceList, @@ -76,7 +76,7 @@ public function testLoadResourceListWithoutAcceptedFactoryNoUrl(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result); + $resourceList = $resolver->loadResourceList($result, ''); $this->assertEmpty( $resourceList, diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSelectTest.php index f71d27a..4d843de 100644 --- a/test/Service/Search/SolrSelectTest.php +++ b/test/Service/Search/SolrSelectTest.php @@ -18,6 +18,7 @@ use Atoolo\Search\Dto\Search\Query\Sort\Name; use Atoolo\Search\Dto\Search\Query\Sort\Natural; use Atoolo\Search\Dto\Search\Query\Sort\Score; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\Search\SolrQueryModifier; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use Atoolo\Search\Service\Search\SolrSelect; @@ -44,6 +45,7 @@ class SolrSelectTest extends TestCase protected function setUp(): void { + $indexName = $this->createStub(IndexName::class); $clientFactory = $this->createStub( SolrClientFactory::class ); @@ -76,6 +78,7 @@ protected function setUp(): void ->willReturn([$this->resource]); $this->searcher = new SolrSelect( + $indexName, $clientFactory, [$solrQueryModifier], $resultToResourceResolver @@ -85,7 +88,7 @@ protected function setUp(): void public function testSelectEmpty(): void { $query = new SelectQuery( - 'myindex', + '', '', 0, 1, @@ -108,8 +111,8 @@ public function testSelectEmpty(): void public function testSelectWithText(): void { $query = new SelectQuery( - 'myindex', 'cat dog', + '', 0, 10, [ @@ -131,7 +134,7 @@ public function testSelectWithText(): void public function testSelectWithSort(): void { $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -161,7 +164,7 @@ public function testSelectWithInvalidSort(): void $sort = $this->createStub(Criteria::class); $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -178,7 +181,7 @@ public function testSelectWithInvalidSort(): void public function testSelectWithAndDefaultOperator(): void { $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -204,7 +207,7 @@ public function testSelectWithFilter(): void ->getMock(); $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -237,7 +240,7 @@ public function testSelectWithFacets(): void ]; $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -264,7 +267,7 @@ public function testSelectWithInvalidFacets(): void ]; $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -303,7 +306,7 @@ public function testResultFacets(): void ]; $query = new SelectQuery( - 'myindex', + '', '', 0, 10, @@ -346,7 +349,7 @@ public function testInvalidResultFacets(): void ]; $query = new SelectQuery( - 'myindex', + '', '', 0, 10, diff --git a/test/Service/Search/SolrSuggestTest.php b/test/Service/Search/SolrSuggestTest.php index 3eef755..734420d 100644 --- a/test/Service/Search/SolrSuggestTest.php +++ b/test/Service/Search/SolrSuggestTest.php @@ -8,6 +8,7 @@ use Atoolo\Search\Dto\Search\Query\SuggestQuery; use Atoolo\Search\Dto\Search\Result\Suggestion; use Atoolo\Search\Exception\UnexpectedResultException; +use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\Search\SolrSuggest; use Atoolo\Search\Service\SolrClientFactory; use PHPUnit\Framework\Attributes\CoversClass; @@ -28,6 +29,7 @@ class SolrSuggestTest extends TestCase protected function setUp(): void { + $indexName = $this->createStub(IndexName::class); $clientFactory = $this->createStub( SolrClientFactory::class ); @@ -44,7 +46,7 @@ protected function setUp(): void $this->result = $this->createStub(SelectResult::class); $client->method('select')->willReturn($this->result); - $this->searcher = new SolrSuggest($clientFactory); + $this->searcher = new SolrSuggest($indexName, $clientFactory); } public function testSuggest(): void diff --git a/test/Service/ServerVarSolrClientFactoryTest.php b/test/Service/ServerVarSolrClientFactoryTest.php new file mode 100644 index 0000000..d47425c --- /dev/null +++ b/test/Service/ServerVarSolrClientFactoryTest.php @@ -0,0 +1,137 @@ +create('core'); + + $expectedEndPoint = new Endpoint([ + 'host' => 'localhost', + 'port' => '8382', + 'path' => '', + 'core' => 'core', + 'scheme' => 'http', + 'key' => 'localhost' + ]); + + $this->assertEquals( + $expectedEndPoint, + $client->getEndpoint(), + 'unexpected endpoint configuration' + ); + } + + public function testCreateWithUrlServerVar(): void + { + $_SERVER['SOLR_URL'] = 'https://solr.example.com:8983/path'; + + $factory = new ServerVarSolrClientFactory(); + $client = $factory->create('core'); + + $expectedEndPoint = new Endpoint([ + 'host' => 'solr.example.com', + 'port' => '8983', + 'path' => '/path', + 'core' => 'core', + 'scheme' => 'https', + 'key' => 'solr.example.com' + ]); + + $this->assertEquals( + $expectedEndPoint, + $client->getEndpoint(), + 'unexpected endpoint configuration' + ); + } + + public function testCreateWithHttpUrlWithoutPortServerVar(): void + { + $_SERVER['SOLR_URL'] = 'http://solr'; + + $factory = new ServerVarSolrClientFactory(); + $client = $factory->create('core'); + + $expectedEndPoint = new Endpoint([ + 'host' => 'solr', + 'port' => '8382', + 'path' => '', + 'core' => 'core', + 'scheme' => 'http', + 'key' => 'solr' + ]); + + $this->assertEquals( + $expectedEndPoint, + $client->getEndpoint(), + 'unexpected endpoint configuration' + ); + } + + public function testCreateWithHttpsUrlWithoutPortServerVar(): void + { + $_SERVER['SOLR_URL'] = 'https://solr'; + + $factory = new ServerVarSolrClientFactory(); + $client = $factory->create('core'); + + $expectedEndPoint = new Endpoint([ + 'host' => 'solr', + 'port' => '443', + 'path' => '', + 'core' => 'core', + 'scheme' => 'https', + 'key' => 'solr' + ]); + + $this->assertEquals( + $expectedEndPoint, + $client->getEndpoint(), + 'unexpected endpoint configuration' + ); + } + + public function testCreateWithSchemaHostAndPortServerVar(): void + { + $_SERVER['SOLR_SCHEME'] = 'https'; + $_SERVER['SOLR_HOST'] = 'solr.example.com'; + $_SERVER['SOLR_PORT'] = '8983'; + + $factory = new ServerVarSolrClientFactory(); + $client = $factory->create('core'); + + $expectedEndPoint = new Endpoint([ + 'host' => 'solr.example.com', + 'port' => '8983', + 'path' => '', + 'core' => 'core', + 'scheme' => 'https', + 'key' => 'solr.example.com' + ]); + + $this->assertEquals( + $expectedEndPoint, + $client->getEndpoint(), + 'unexpected endpoint configuration' + ); + } +} From 067f65a51d116f6027beb39a1ead6148bf2bef4a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 11:27:05 +0100 Subject: [PATCH 069/145] chore: composer update --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 36e54c9..0d47240 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,7 @@ "prefer-stable": true, "require": { "php": ">=8.1 <8.4.0", - "atoolo/resource": "dev-main", + "atoolo/resource": "dev-feature/resource-channel", "solarium/solarium": "^6.3", "symfony/config": "^6.3 | ^7.0", "symfony/console": "^6.3 | ^7.0", From 56aa128c2346d6ec63cdadee85825d4745af7d65 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 11:28:05 +0100 Subject: [PATCH 070/145] chore: composer update --- composer.json | 88 ++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/composer.json b/composer.json index 0d47240..0e2b408 100644 --- a/composer.json +++ b/composer.json @@ -3,37 +3,27 @@ "description": "Indexing und searching", "license": "MIT", "type": "library", - "authors": [{ - "name": "veltrup", - "email": "veltrup@sitepark.com" - }], - "autoload": { - "psr-4": { - "Atoolo\\Search\\": "src" - } - }, - "autoload-dev": { - "psr-4": { - "Atoolo\\Search\\Test\\": "test" + "authors": [ + { + "name": "veltrup", + "email": "veltrup@sitepark.com" } - }, - "minimum-stability": "dev", - "prefer-stable": true, + ], "require": { "php": ">=8.1 <8.4.0", + "ext-intl": "*", "atoolo/resource": "dev-feature/resource-channel", "solarium/solarium": "^6.3", - "symfony/config": "^6.3 | ^7.0", - "symfony/console": "^6.3 | ^7.0", - "symfony/dependency-injection": "^6.3 | ^7.0", - "symfony/event-dispatcher": "^6.3 | ^7.0", - "symfony/finder": "^6.3 | ^7.0", - "symfony/lock": "^6.3 | ^7.0", - "symfony/property-access": "^6.3 | ^7.0", - "symfony/serializer": "^6.3 | ^7.0", - "symfony/yaml": "^6.3 | ^7.0", - "ext-intl": "*" - }, + "symfony/config": "^6.3 || ^7.0", + "symfony/console": "^6.3 || ^7.0", + "symfony/dependency-injection": "^6.3 || ^7.0", + "symfony/event-dispatcher": "^6.3 || ^7.0", + "symfony/finder": "^6.3 || ^7.0", + "symfony/lock": "^6.3 || ^7.0", + "symfony/property-access": "^6.3 || ^7.0", + "symfony/serializer": "^6.3 || ^7.0", + "symfony/yaml": "^6.3 || ^7.0" + }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^1.0", "infection/infection": "^0.27.6", @@ -41,16 +31,33 @@ "phpunit/phpunit": "^10.4", "roave/security-advisories": "dev-latest", "squizlabs/php_codesniffer": "^3.7", - "symfony/filesystem": "^6.3 | ^7.0" + "symfony/filesystem": "^6.3 || ^7.0" }, - "extra": { - "branch-alias": { - "feature/core-name": "dev-feature/initial-implementation" + "repositories": {}, + "minimum-stability": "dev", + "prefer-stable": true, + "autoload": { + "psr-4": { + "Atoolo\\Search\\": "src" } }, - + "autoload-dev": { + "psr-4": { + "Atoolo\\Search\\Test\\": "test" + } + }, + "config": { + "allow-plugins": { + "dealerdirect/phpcodesniffer-composer-installer": true, + "infection/extension-installer": true + }, + "optimize-autoloader": true, + "preferred-install": { + "*": "dist" + }, + "sort-packages": true + }, "scripts": { - "post-install-cmd": "phive --no-progress install --force-accept-unsigned --trust-gpg-keys C00543248C87FB13,4AA394086372C20A,CF1A108D0E7AE720,51C67305FFC2E5C0", "analyse": [ "@analyse:phplint", @@ -75,20 +82,7 @@ "test": [ "@test:phpunit" ], - "test:phpunit": "./tools/phpunit.phar -c phpunit.xml --coverage-text", - "test:infection": "vendor/bin/infection --threads=8 --no-progress --only-covered -s || exit 0" - }, - "config": { - "allow-plugins": { - "dealerdirect/phpcodesniffer-composer-installer": true, - "infection/extension-installer": true - }, - "optimize-autoloader": true, - "preferred-install": { - "*": "dist" - }, - "sort-packages": true - }, - "repositories": { + "test:infection": "vendor/bin/infection --threads=8 --no-progress --only-covered -s || exit 0", + "test:phpunit": "./tools/phpunit.phar -c phpunit.xml --coverage-text" } } From 7513a47ca699ac21a670d19440d81327934051f3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:27:30 +0100 Subject: [PATCH 071/145] docs: fix incorrect grammar --- src/Console/Command/Indexer.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index e25dedc..9e4ee66 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -67,9 +67,10 @@ protected function configure(): void 'cleanup-threshold', null, InputArgument::OPTIONAL, - 'Specifies the number of indexed documents from ' . - 'which indexing is considered successful and old entries ' . - 'can be deleted. Is only used for full indexing.', + 'Specifies the number of documents required to be indexed ' . + 'successfully for the entire process to be considered successfull. ' . + 'Old entries will only ever be removed if this threshold is reached. ' . + 'Only relevant for full-indexing.', 0 ) ->addOption( From a0cbf2dd4c064684835d08cfcd586fd85f3659e8 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:29:41 +0100 Subject: [PATCH 072/145] style: fix code style --- src/Console/Command/Indexer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 9e4ee66..b8a62d4 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -68,9 +68,9 @@ protected function configure(): void null, InputArgument::OPTIONAL, 'Specifies the number of documents required to be indexed ' . - 'successfully for the entire process to be considered successfull. ' . - 'Old entries will only ever be removed if this threshold is reached. ' . - 'Only relevant for full-indexing.', + 'successfully for the entire process to be considered ' . + 'successfull. Old entries will only ever be removed if this ' . + 'threshold is reached. Only relevant for full-indexing.', 0 ) ->addOption( From acab5c304533829aa5c360704e1d1df4926da70b Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:31:53 +0100 Subject: [PATCH 073/145] style: fix indent --- src/Console/Command/Io/IndexerProgressBar.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index dcc5664..ce29560 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -68,7 +68,7 @@ private function formatProgressBar(string $color): void $this->progressBar->setProgressCharacter('➤'); $this->progressBar->setFormat( "%current%/%max% [%bar%] %percent:3s%%\n" . - " %estimated:-20s% %memory:20s%" + " %estimated:-20s% %memory:20s%" ); } From 5325c82700f29971a0af83b4bd5f3d7744c5676a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:33:02 +0100 Subject: [PATCH 074/145] style: string concatenation --- src/Console/Command/Io/TypifiedInput.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Console/Command/Io/TypifiedInput.php b/src/Console/Command/Io/TypifiedInput.php index 3532b32..bdc3b61 100644 --- a/src/Console/Command/Io/TypifiedInput.php +++ b/src/Console/Command/Io/TypifiedInput.php @@ -22,7 +22,7 @@ public function getIntOption(string $name): int $value = $this->input->getOption($name); if (!is_numeric($value)) { throw new InvalidArgumentException( - 'option' . $name . ' must be a integer: ' . $value + 'option ' . $name . ' must be a integer: ' . $value ); } return (int)$value; @@ -33,7 +33,7 @@ public function getStringArgument(string $name): string $value = $this->input->getArgument($name); if (!is_string($value)) { throw new InvalidArgumentException( - 'argument' . $name . ' must be a string' + 'argument ' . $name . ' must be a string' ); } return $value; From cd9c8fd8bc6b2a64a72607f5000ed9c8dba27f79 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:46:00 +0100 Subject: [PATCH 075/145] refactore: text argument as string --- src/Console/Command/Search.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index ba4aa71..a13c308 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -53,7 +53,7 @@ protected function configure(): void ) ->addArgument( 'text', - InputArgument::IS_ARRAY, + InputArgument::REQUIRED, 'Text with which to search.' ) ; @@ -94,10 +94,8 @@ protected function buildQuery(InputInterface $input): SelectQuery $builder = new SelectQueryBuilder(); $builder->index($this->index); - $text = $input->getArgument('text'); - if (is_array($text)) { - $builder->text(implode(' ', $text)); - } + $text = $this->input->getStringArgument('text'); + $builder->text($text); // TODO: filter From ca7e76b8ea5a30b11c4ac62cd45084573e961ec3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:53:26 +0100 Subject: [PATCH 076/145] refactore: terms argument as string --- src/Console/Command/Suggest.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index b4c59b6..69ed654 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -36,7 +36,7 @@ public function __construct( protected function configure(): void { $this - ->setHelp('Command to performs a suggest search') + ->setHelp('Command that performs a suggest search') ->addArgument( 'solr-connection-url', InputArgument::REQUIRED, @@ -49,7 +49,7 @@ protected function configure(): void ) ->addArgument( 'terms', - InputArgument::REQUIRED | InputArgument::IS_ARRAY, + InputArgument::REQUIRED, 'Suggest terms.' ) ; @@ -62,7 +62,7 @@ protected function execute( $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); $this->solrCore = $this->input->getStringArgument('solr-core'); - $terms = $this->input->getArrayArgument('terms'); + $terms = $this->input->getStringArgument('terms'); $search = $this->createSearcher(); $query = $this->buildQuery($terms); @@ -82,16 +82,13 @@ protected function createSearcher(): SolrSuggest return $this->solrSuggestBuilder->build(); } - /** - * @param string[] $terms - */ - protected function buildQuery(array $terms): SuggestQuery + protected function buildQuery(string $terms): SuggestQuery { $excludeMedia = new ObjectTypeFilter(['media'], 'media'); $excludeMedia = $excludeMedia->exclude(); return new SuggestQuery( $this->solrCore, - implode(' ', $terms), + $terms, [ new ArchiveFilter(), $excludeMedia From 913148baf19716c36052aab784e5c086b2d7bfd1 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 13:59:23 +0100 Subject: [PATCH 077/145] refactor IndexerStatusState::INDEXED to IndexerStatusState::FINISHED --- src/Console/Command/Io/IndexerProgressBar.php | 2 +- src/Dto/Indexer/IndexerStatusState.php | 2 +- src/Service/Indexer/BackgroundIndexerProgressState.php | 2 +- test/Console/Command/SearchTest.php | 2 +- test/Console/Command/SuggestTest.php | 2 +- test/Dto/Indexer/IndexerStatusTest.php | 4 ++-- test/Service/Indexer/BackgroundIndexerProgressStateTest.php | 2 +- test/Service/Indexer/IndexerStatusStoreTest.php | 4 ++-- .../IndexerStatusStore/atoolo.search.index.test.status.json | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index ce29560..9908627 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -82,7 +82,7 @@ public function error(Throwable $throwable): void public function finish(): void { $this->progressBar->finish(); - $this->status->state = IndexerStatusState::INDEXED; + $this->status->state = IndexerStatusState::FINISHED; $this->status->endTime = new DateTime(); } diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php index 57bc6aa..2a7c19d 100644 --- a/src/Dto/Indexer/IndexerStatusState.php +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -11,6 +11,6 @@ enum IndexerStatusState: string { case UNKNOWN = 'UNKNOWN'; case RUNNING = 'RUNNING'; - case INDEXED = 'INDEXED'; + case FINISHED = 'FINISHED'; case ABORTED = 'ABORTED'; } diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index 3b9d167..9e9fa53 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -100,7 +100,7 @@ public function finish(): void $this->status->endTime = new DateTime(); } if ($this->status->state === IndexerStatusState::RUNNING) { - $this->status->state = IndexerStatusState::INDEXED; + $this->status->state = IndexerStatusState::FINISHED; } $this->statusStore->store($this->index, $this->status); } diff --git a/test/Console/Command/SearchTest.php b/test/Console/Command/SearchTest.php index ac7831f..0e9bc3e 100644 --- a/test/Console/Command/SearchTest.php +++ b/test/Console/Command/SearchTest.php @@ -72,7 +72,7 @@ public function testExecute(): void 'solr-connection-url' => 'http://localhost:8382', 'index' => 'test', 'resource-dir' => 'abc', - 'text' => ['test', 'abc'] + 'text' => 'test abc' ]); $this->commandTester->assertCommandIsSuccessful(); diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index b090a15..72c6f50 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -58,7 +58,7 @@ public function testExecute(): void $this->commandTester->execute([ 'solr-connection-url' => 'http://localhost:8382', 'solr-core' => 'test', - 'terms' => ['sec'] + 'terms' => 'sec' ]); $this->commandTester->assertCommandIsSuccessful(); diff --git a/test/Dto/Indexer/IndexerStatusTest.php b/test/Dto/Indexer/IndexerStatusTest.php index 8d24020..0346b1f 100644 --- a/test/Dto/Indexer/IndexerStatusTest.php +++ b/test/Dto/Indexer/IndexerStatusTest.php @@ -30,7 +30,7 @@ public function setUp(): void $lastUpdate->setTime(13, 17, 12); $this->status = new IndexerStatus( - IndexerStatusState::INDEXED, + IndexerStatusState::FINISHED, $startTime, $endTime, 10, @@ -45,7 +45,7 @@ public function setUp(): void public function testGetStatus(): void { $this->assertEquals( - '[INDEXED] ' . + '[FINISHED] ' . 'start: 31.01.2024 11:15, ' . 'time: 01h 01m 01s, ' . 'processed: 5/10, ' . diff --git a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php index a5e3cf8..4283bcb 100644 --- a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php +++ b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php @@ -129,7 +129,7 @@ public function testFinish(): void $this->state->finish(); $this->assertMatchesRegularExpression( - '/\[INDEXED]/', + '/\[FINISHED]/', $this->state->getStatus()->getStatusLine(), "unexpected status line" ); diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php index 7fd2bba..d4cf26d 100644 --- a/test/Service/Indexer/IndexerStatusStoreTest.php +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -43,7 +43,7 @@ public function testStore(): void $expected = '{' . - '"state":"INDEXED",' . + '"state":"FINISHED",' . '"startTime":"2024-01-31T11:15:10+00:00",' . '"endTime":"2024-01-31T12:16:11+00:00",' . '"total":10,' . @@ -198,7 +198,7 @@ private function createIndexerStatus(): IndexerStatus $lastUpdate->setTime(13, 17, 12); return new IndexerStatus( - IndexerStatusState::INDEXED, + IndexerStatusState::FINISHED, $startTime, $endTime, 10, diff --git a/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json index bc347ed..212e4b2 100644 --- a/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json +++ b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json @@ -1,5 +1,5 @@ { - "state": "INDEXED", + "state": "FINISHED", "startTime": "2024-01-31T11:15:10+00:00", "endTime": "2024-01-31T12:16:11+00:00", "total": 10, From 52bd92139cdfee8e1b9ab3876146a6e7221eb5ef Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 14:18:54 +0100 Subject: [PATCH 078/145] refactor: builder methods could return static --- src/Dto/Search/Query/SelectQueryBuilder.php | 40 ++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index fd9c573..e7ff039 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -35,7 +35,10 @@ public function __construct() { } - public function index(string $index): SelectQueryBuilder + /** + * @return $this + */ + public function index(string $index): static { if (empty($index)) { throw new \InvalidArgumentException('index is empty'); @@ -44,13 +47,19 @@ public function index(string $index): SelectQueryBuilder return $this; } - public function text(string $text): SelectQueryBuilder + /** + * @return $this + */ + public function text(string $text): static { $this->text = $text; return $this; } - public function offset(int $offset): SelectQueryBuilder + /** + * @return $this + */ + public function offset(int $offset): static { if ($offset < 0) { throw new \InvalidArgumentException('offset is lower then 0'); @@ -59,7 +68,10 @@ public function offset(int $offset): SelectQueryBuilder return $this; } - public function limit(int $limit): SelectQueryBuilder + /** + * @return $this + */ + public function limit(int $limit): static { if ($limit < 0) { throw new \InvalidArgumentException('offset is lower then 0'); @@ -68,7 +80,10 @@ public function limit(int $limit): SelectQueryBuilder return $this; } - public function sort(Criteria ...$criteriaList): SelectQueryBuilder + /** + * @return $this + */ + public function sort(Criteria ...$criteriaList): static { foreach ($criteriaList as $criteria) { $this->sort[] = $criteria; @@ -76,7 +91,10 @@ public function sort(Criteria ...$criteriaList): SelectQueryBuilder return $this; } - public function filter(Filter ...$filterList): SelectQueryBuilder + /** + * @return $this + */ + public function filter(Filter ...$filterList): static { foreach ($filterList as $filter) { if (isset($this->filter[$filter->key])) { @@ -90,7 +108,10 @@ public function filter(Filter ...$filterList): SelectQueryBuilder return $this; } - public function facet(Facet ...$facetList): SelectQueryBuilder + /** + * @return $this + */ + public function facet(Facet ...$facetList): static { foreach ($facetList as $facet) { if (isset($this->facets[$facet->key])) { @@ -104,9 +125,12 @@ public function facet(Facet ...$facetList): SelectQueryBuilder return $this; } + /** + * @return $this + */ public function defaultQueryOperator( QueryOperator $defaultQueryOperator - ): SelectQueryBuilder { + ): static { $this->defaultQueryOperator = $defaultQueryOperator; return $this; } From 26a5d7c2eb4c06ad3a0dc4db2f55b285fd4e4111 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 14:21:00 +0100 Subject: [PATCH 079/145] fix: exception message --- src/Dto/Search/Query/SelectQueryBuilder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SelectQueryBuilder.php index e7ff039..2328e6b 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SelectQueryBuilder.php @@ -74,7 +74,7 @@ public function offset(int $offset): static public function limit(int $limit): static { if ($limit < 0) { - throw new \InvalidArgumentException('offset is lower then 0'); + throw new \InvalidArgumentException('limit is lower then 0'); } $this->limit = $limit; return $this; From e1ea53a256cebe576593b621352ee7a3285b1a7a Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 14:42:33 +0100 Subject: [PATCH 080/145] refactor: default values for all members --- src/Service/Indexer/IndexSchema2xDocument.php | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 476d2e5..f840965 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -9,120 +9,120 @@ class IndexSchema2xDocument extends Document implements IndexDocument { - public string $sp_id; - public ?string $sp_name; - public ?string $sp_anchor; - public ?string $title; - public ?string $description; - public ?string $sp_objecttype; - public bool $sp_canonical; - public ?string $crawl_process_id; - public ?string $id; - public ?string $url; - public ?string $contenttype; + public ?string $sp_id = null; + public ?string $sp_name = null; + public ?string $sp_anchor = null; + public ?string $title = null; + public ?string $description = null; + public ?string $sp_objecttype = null; + public ?bool $sp_canonical = null; + public ?string $crawl_process_id = null; + public ?string $id = null; + public ?string $url = null; + public ?string $contenttype = null; /** * @var string[] */ - public array $sp_contenttype; - public ?string $sp_language; - public ?string $meta_content_language; - public ?string $meta_content_type; - public ?DateTime $sp_changed; - public ?DateTime $sp_generated; - public ?DateTime $sp_date; - public ?DateTime $sp_date_from; - public ?DateTime $sp_date_to; + public ?array $sp_contenttype = null; + public ?string $sp_language = null; + public ?string $meta_content_language = null; + public ?string $meta_content_type = null; + public ?DateTime $sp_changed = null; + public ?DateTime $sp_generated = null; + public ?DateTime $sp_date = null; + public ?DateTime $sp_date_from = null; + public ?DateTime $sp_date_to = null; /** * @var DateTime[] */ - public array $sp_date_list; - public bool $sp_archive; - public ?string $sp_title; - public ?string $sp_sortvalue; + public ?array $sp_date_list = null; + public ?bool $sp_archive = null; + public ?string $sp_title = null; + public ?string $sp_sortvalue = null; /** * @var string[] */ - public array $keywords; - public string $sp_boost_keywords; + public ?array $keywords = null; + public ?string $sp_boost_keywords = null; /** * @var string[] */ - public array $sp_site; + public ?array $sp_site = null; /** * @var string[] */ - public array $sp_geo_points; + public ?array $sp_geo_points = null; /** * @var string[] */ - public array $sp_category; + public ?array $sp_category = null; /** * @var string[] */ - public array $sp_category_path; - public int $sp_group; + public ?array $sp_category_path = null; + public ?int $sp_group = null; /** * @var int[] */ - public array $sp_group_path; - public ?string $content; + public ?array $sp_group_path = null; + public ?string $content = null; /** * @var string[] */ - public array $include_groups; + public ?array $include_groups = null; /** * @var string[] */ - public array $exclude_groups; + public ?array $exclude_groups = null; /** * @var string[] */ - public array $sp_source; + public ?array $sp_source = null; /** * @var string[] */ - public array $sp_citygov_phone; + public ?array $sp_citygov_phone = null; /** * @var string[] */ - public array $sp_citygov_email; + public ?array $sp_citygov_email = null; - public ?string $sp_citygov_address; + public ?string $sp_citygov_address = null; - public ?string $sp_citygov_startletter; + public ?string $sp_citygov_startletter = null; /** * @var string[] */ - public array $sp_citygov_organisationtoken; + public ?array $sp_citygov_organisationtoken = null; - public int $sp_organisation; + public ?int $sp_organisation = null; - public ?string $sp_citygov_firstname; + public ?string $sp_citygov_firstname = null; - public ?string $sp_citygov_lastname; + public ?string $sp_citygov_lastname = null; /** * List of Organisation Ids * @var int[] */ - public array $sp_organisation_path; + public ?array $sp_organisation_path = null; /** * List of Organisationnames * @var string[] */ - public array $sp_citygov_organisation; + public ?array $sp_citygov_organisation = null; /** * List of Productnames * @var string[] */ - public array $sp_citygov_product; + public ?array $sp_citygov_product = null; - public ?string $sp_citygov_function; + public ?string $sp_citygov_function = null; /** * @var array */ From e0bc2933212026d00bcf47ba942d9e4b9f27c244 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 14:55:24 +0100 Subject: [PATCH 081/145] refactor: inherited and meta fields --- src/Service/Indexer/IndexSchema2xDocument.php | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index f840965..12e9c21 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -9,6 +9,19 @@ class IndexSchema2xDocument extends Document implements IndexDocument { + + private const INHERITED_FIELDS = [ + 'fields', + 'modifiers', + 'fieldBoosts' + ]; + + private const META_FIELDS = [ + 'metaString', + 'metaText', + 'metaBool' + ]; + public ?string $sp_id = null; public ?string $sp_name = null; public ?string $sp_anchor = null; @@ -143,7 +156,7 @@ class IndexSchema2xDocument extends Document implements IndexDocument */ public function setMetaString(string $name, string|array $value): void { - $this->metaString[$name] = $value; + $this->metaString['sp_meta_string_' . $name] = $value; } /** @@ -151,12 +164,12 @@ public function setMetaString(string $name, string|array $value): void */ public function setMetaText(string $name, string|array $value): void { - $this->metaText[$name] = $value; + $this->metaText['sp_meta_text_' . $name] = $value; } public function setMetaBool(string $name, bool $value): void { - $this->metaBool[$name] = $value; + $this->metaBool['sp_meta_bool_' . $name] = $value; } /** @@ -166,7 +179,7 @@ public function getFields(): array { $fields = get_object_vars($this); - // filter out inherited fields + // filter out inherited and meta fields $fields = array_filter( $fields, @@ -176,38 +189,24 @@ static function ($value, $key) { return false; } - $inheritedFields = [ - 'fields', - 'modifiers', - 'fieldBoosts' - ]; - if (in_array($key, $inheritedFields, true)) { + if (in_array($key, self::INHERITED_FIELDS, true)) { return false; } - if ( - $key === 'metaString' || - $key === 'metaText' || - $key === 'metaBool' - ) { + if (in_array($key, self::META_FIELDS, true)) { return false; } + return true; }, ARRAY_FILTER_USE_BOTH ); - foreach ($this->metaString as $key => $value) { - $fields['sp_meta_string_' . $key] = $value; - } - foreach ($this->metaText as $key => $value) { - $fields['sp_meta_text_' . $key] = $value; - } - - foreach ($this->metaBool as $key => $value) { - $fields['sp_meta_bool_' . $key] = $value; - } - - return $fields; + return array_merge( + $fields, + $this->metaString, + $this->metaText, + $this->metaBool + ); } } From da8eef35a41eea6cc25e8473cf151e09bb09d437 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 14:56:47 +0100 Subject: [PATCH 082/145] refactor: inherited and meta fields --- src/Service/Indexer/IndexSchema2xDocument.php | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 12e9c21..853b17f 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -177,36 +177,17 @@ public function setMetaBool(string $name, bool $value): void */ public function getFields(): array { - $fields = get_object_vars($this); - - // filter out inherited and meta fields - - $fields = array_filter( - $fields, - static function ($value, $key) { - - if (is_null($value)) { - return false; - } - - if (in_array($key, self::INHERITED_FIELDS, true)) { - return false; - } - - if (in_array($key, self::META_FIELDS, true)) { - return false; - } - - return true; - }, - ARRAY_FILTER_USE_BOTH - ); - - return array_merge( - $fields, - $this->metaString, - $this->metaText, - $this->metaBool - ); + return [ + ...array_filter( + get_object_vars($this), + fn($value, $key) => !is_null($value) + && !in_array($key, self::INHERITED_FIELDS, true) + && !in_array($key, self::META_FIELDS, true), + ARRAY_FILTER_USE_BOTH + ), + ... $this->metaString, + ... $this->metaText, + ... $this->metaBool, + ]; } } From b07662d74c82fadfd67a39d1488c05126a880e3e Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:03:03 +0100 Subject: [PATCH 083/145] style: fix empty line --- src/Service/Indexer/IndexSchema2xDocument.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php index 853b17f..d3fa585 100644 --- a/src/Service/Indexer/IndexSchema2xDocument.php +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -9,7 +9,6 @@ class IndexSchema2xDocument extends Document implements IndexDocument { - private const INHERITED_FIELDS = [ 'fields', 'modifiers', From 401ce2a4d416b87470955efba3dbcfb75c236cfe Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:03:26 +0100 Subject: [PATCH 084/145] refactore: create dir recursive --- src/Service/Indexer/IndexerStatusStore.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index dbbbea1..8f41e12 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -77,8 +77,14 @@ private function createBaseDirectory(): void { if ( !is_dir($concurrentDirectory = $this->basedir) && - !@mkdir($concurrentDirectory) && - !is_dir($concurrentDirectory) + ( + !@mkdir( + $concurrentDirectory, + 0777, + true + ) && + !is_dir($concurrentDirectory) + ) ) { throw new RuntimeException(sprintf( 'Directory "%s" was not created', From 1c40c70547fd562a821d477fc433145a20615538 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:06:42 +0100 Subject: [PATCH 085/145] refactore: create the serializer in the constructor --- src/Service/Indexer/IndexerStatusStore.php | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index 8f41e12..334f747 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -16,8 +16,18 @@ class IndexerStatusStore { + private readonly Serializer $serializer; + public function __construct(private readonly string $basedir) { + $encoders = [new JsonEncoder()]; + $normalizers = [ + new BackedEnumNormalizer(), + new DateTimeNormalizer(), + new PropertyNormalizer() + ]; + + $this->serializer = new Serializer($normalizers, $encoders); } /** @@ -44,7 +54,7 @@ public function load(string $index): IndexerStatus /** @var IndexerStatus $status */ $status = $this - ->createSerializer() + ->serializer ->deserialize($json, IndexerStatus::class, 'json'); return $status; @@ -61,7 +71,7 @@ public function store(string $index, IndexerStatus $status): void ); } $json = $this - ->createSerializer() + ->serializer ->serialize($status, 'json'); $result = file_put_contents($file, $json); if ($result === false) { @@ -99,18 +109,6 @@ private function createBaseDirectory(): void } } - private function createSerializer(): Serializer - { - $encoders = [new JsonEncoder()]; - $normalizers = [ - new BackedEnumNormalizer(), - new DateTimeNormalizer(), - new PropertyNormalizer() - ]; - - return new Serializer($normalizers, $encoders); - } - private function getStatusFile(string $index): string { return $this->basedir . From 78541dc309861d93d579ef3a073f158fec373c97 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:17:13 +0100 Subject: [PATCH 086/145] refactore: aborter method names --- src/Service/Indexer/IndexingAborter.php | 6 +++--- .../Indexer/InternalResourceIndexer.php | 6 +++--- test/Service/Indexer/IndexingAborterTest.php | 20 +++++++++---------- .../Indexer/InternalResourceIndexerTest.php | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Service/Indexer/IndexingAborter.php b/src/Service/Indexer/IndexingAborter.php index b5e2ce4..a02e547 100644 --- a/src/Service/Indexer/IndexingAborter.php +++ b/src/Service/Indexer/IndexingAborter.php @@ -11,17 +11,17 @@ public function __construct( ) { } - public function shouldAborted(string $index): bool + public function isAbortionRequested(string $index): bool { return file_exists($this->getAbortMarkerFile($index)); } - public function abort(string $index): void + public function requestAbortion(string $index): void { touch($this->getAbortMarkerFile($index)); } - public function aborted(string $index): void + public function resetAbortionRequest(string $index): void { unlink($this->getAbortMarkerFile($index)); } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 0fa5529..2673a96 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -66,7 +66,7 @@ public function remove(string $index, array $idList): void public function abort(string $index): void { - $this->aborter->abort($index); + $this->aborter->requestAbortion($index); } /** @@ -264,8 +264,8 @@ private function indexChunks( if ($resourceList === false) { return false; } - if ($this->aborter->shouldAborted($solrCore)) { - $this->aborter->aborted($solrCore); + if ($this->aborter->isAbortionRequested($solrCore)) { + $this->aborter->resetAbortionRequest($solrCore); $this->indexerProgressHandler->abort(); return false; } diff --git a/test/Service/Indexer/IndexingAborterTest.php b/test/Service/Indexer/IndexingAborterTest.php index 3bad523..8c736d6 100644 --- a/test/Service/Indexer/IndexingAborterTest.php +++ b/test/Service/Indexer/IndexingAborterTest.php @@ -28,38 +28,38 @@ public function setUp(): void $this->aborter = new IndexingAborter($workdir); } - public function testShouldNotAborted(): void + public function testIsAbortionRequested(): void { $this->assertFalse( - $this->aborter->shouldAborted('test'), + $this->aborter->isAbortionRequested('test'), 'should not aborted' ); } - public function testShouldAborted(): void + public function testIsAbortionRequestedWithExistsMarkerFile(): void { touch($this->file); $this->assertTrue( - $this->aborter->shouldAborted('test'), + $this->aborter->isAbortionRequested('test'), 'should not aborted' ); } - public function testAbort(): void + public function testRequestAbortion(): void { - $this->aborter->abort('test'); + $this->aborter->requestAbortion('test'); $this->assertFileExists( $this->file, - 'abort call should create file' + 'requestAbortion call should create file' ); } - public function testAborted(): void + public function testResetAbortionRequest(): void { touch($this->file); - $this->aborter->aborted('test'); + $this->aborter->resetAbortionRequest('test'); $this->assertFileDoesNotExist( $this->file, - 'aborted call should remove file' + 'resetAbortionRequest call should remove file' ); } } diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index c143fa6..8c37ff3 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -102,7 +102,7 @@ public function setUp(): void public function testAbort(): void { $this->aborter->expects($this->once()) - ->method('abort') + ->method('requestAbortion') ->with('test'); $this->indexer->abort('test'); @@ -217,11 +217,11 @@ public function testAborted(): void '/a/c.php' ]); - $this->aborter->method('shouldAborted') + $this->aborter->method('isAbortionRequested') ->willReturn(true); $this->aborter->expects($this->once()) - ->method('aborted'); + ->method('resetAbortionRequest'); $this->indexerProgressHandler->expects($this->once()) ->method('abort'); From 76f2980b8c281202d11cb4646fa77eadbe8ec63b Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:21:16 +0100 Subject: [PATCH 087/145] refactore: rename method mapTranslationPaths() to normalizePaths() --- src/Service/Indexer/InternalResourceIndexer.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 2673a96..e8e7414 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -78,7 +78,7 @@ public function index(IndexerParameter $parameter): IndexerStatus if (empty($parameter->paths)) { $pathList = $this->finder->findAll(); } else { - $mappedPaths = $this->mapTranslationPaths($parameter->paths); + $mappedPaths = $this->normalizePaths($parameter->paths); $pathList = $this->finder->findPaths($mappedPaths); } @@ -86,8 +86,8 @@ public function index(IndexerParameter $parameter): IndexerStatus } /** - * If a path is to be indexed in a translated language, this can also - * be specified via the URL parameter `loc`. For example, + * A path can signal to be translated into another language via + * the URL parameter loc. For example, * `/dir/file.php?loc=it_IT` defines that the path * `/dir/file.php.translations/it_IT.php` is to be used. * This method translates the URL parameter into the correct path. @@ -95,7 +95,7 @@ public function index(IndexerParameter $parameter): IndexerStatus * @param string[] $pathList * @return string[] */ - private function mapTranslationPaths(array $pathList): array + private function normalizePaths(array $pathList): array { return array_map(static function ($path) { $queryString = parse_url($path, PHP_URL_QUERY); From 2b1ec13432fef0c342d54d71bc71f16d10df9ce2 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:44:19 +0100 Subject: [PATCH 088/145] refactore: ContentMatcher::match() return type and phpdoc added --- .../Indexer/InternalResourceIndexer.php | 2 +- .../Indexer/SiteKit/ContentMatcher.php | 22 ++++++++++++++++--- .../Indexer/SiteKit/HeadlineMatcher.php | 2 +- .../Indexer/SiteKit/QuoteSectionMatcher.php | 2 +- .../Indexer/SiteKit/RichtTextMatcher.php | 2 +- test/Service/Indexer/ContentCollectorTest.php | 2 +- 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index e8e7414..587924d 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -219,12 +219,12 @@ private function indexResourcesPerLanguageIndex( $offset, $parameter->chunkSize ); + gc_collect_cycles(); if ($indexedCount === false) { break; } $successCount += $indexedCount; $offset += $parameter->chunkSize; - gc_collect_cycles(); } if ( diff --git a/src/Service/Indexer/SiteKit/ContentMatcher.php b/src/Service/Indexer/SiteKit/ContentMatcher.php index 1d23e68..d1a75fa 100644 --- a/src/Service/Indexer/SiteKit/ContentMatcher.php +++ b/src/Service/Indexer/SiteKit/ContentMatcher.php @@ -12,8 +12,24 @@ interface ContentMatcher { /** - * @param string[] $path - * @param array $value + * @param string[] $path Contains all array-keys of the nested data + * structure that lead to the transferred value. E.g. ['a', 'b', 'c'] + * for the following structure. + * ``` + * [ + * 'a' => [ + * 'b' => [ + * 'c' => [ + * ... + * ] + * ] + * ] + * ] + * ``` + * @param array $value Value within a data structure + * that is to be checked. + * @return string|false The extracted content or `false` if the + * content is not relevant for the search index. */ - public function match(array $path, array $value): bool|string; + public function match(array $path, array $value): string|false; } diff --git a/src/Service/Indexer/SiteKit/HeadlineMatcher.php b/src/Service/Indexer/SiteKit/HeadlineMatcher.php index 26e2466..4b18549 100644 --- a/src/Service/Indexer/SiteKit/HeadlineMatcher.php +++ b/src/Service/Indexer/SiteKit/HeadlineMatcher.php @@ -9,7 +9,7 @@ class HeadlineMatcher implements ContentMatcher /** * @inheritDoc */ - public function match(array $path, array $value): bool|string + public function match(array $path, array $value): string|false { $len = count($path); if ($len < 2) { diff --git a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php index 43a51f0..9a3f548 100644 --- a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php +++ b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php @@ -12,7 +12,7 @@ class QuoteSectionMatcher implements ContentMatcher /** * @inheritDoc */ - public function match(array $path, array $value): bool|string + public function match(array $path, array $value): string|false { $len = count($path); if ($len < 1) { diff --git a/src/Service/Indexer/SiteKit/RichtTextMatcher.php b/src/Service/Indexer/SiteKit/RichtTextMatcher.php index 7ac3b27..614c962 100644 --- a/src/Service/Indexer/SiteKit/RichtTextMatcher.php +++ b/src/Service/Indexer/SiteKit/RichtTextMatcher.php @@ -9,7 +9,7 @@ class RichtTextMatcher implements ContentMatcher /** * @inheritDoc */ - public function match(array $path, array $value): bool|string + public function match(array $path, array $value): string|false { $modelType = $value['modelType'] ?? false; if ($modelType !== 'html.richText') { diff --git a/test/Service/Indexer/ContentCollectorTest.php b/test/Service/Indexer/ContentCollectorTest.php index 82a804d..9e73e41 100644 --- a/test/Service/Indexer/ContentCollectorTest.php +++ b/test/Service/Indexer/ContentCollectorTest.php @@ -15,7 +15,7 @@ class ContentCollectorTest extends TestCase public function testCollect(): void { $matcher = (new class implements ContentMatcher { - public function match(array $path, array $value): bool|string + public function match(array $path, array $value): string|false { $modelType = $value['modelType'] ?? false; if ($modelType !== 'html.richText') { From 2637fbfc9aca2a1af5e106823ab4186c2e73c792 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:45:59 +0100 Subject: [PATCH 089/145] refactor: upate phpstan-type for QuoteSectionMatcher --- src/Service/Indexer/SiteKit/QuoteSectionMatcher.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php index 9a3f548..5107991 100644 --- a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php +++ b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php @@ -5,7 +5,7 @@ namespace Atoolo\Search\Service\Indexer\SiteKit; /** - * @phpstan-type Model array{quote?:string, citation?:string} + * @phpstan-type Model array{quote?: ?string, citation?: ?string} */ class QuoteSectionMatcher implements ContentMatcher { From 24f32ab2e75b9d82a19b2f03dfd26ccec6c39f93 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:47:46 +0100 Subject: [PATCH 090/145] refactor: rename SolrIndexService::getAvailableIndexes to SolrIndexService::getAvailableIndices --- src/Service/Indexer/InternalResourceIndexer.php | 2 +- src/Service/Indexer/SolrIndexService.php | 2 +- test/Service/Indexer/InternalResourceIndexerTest.php | 2 +- test/Service/Indexer/SolrIndexServiceTest.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 587924d..fcefd74 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -136,7 +136,7 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } - $availableIndexes = $this->indexService->getAvailableIndexes(); + $availableIndexes = $this->indexService->getAvailableIndices(); $splitterResult = $this->translationSplitter->split($pathList); $this->indexTranslationSplittedResources( diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php index 30fe789..240a705 100644 --- a/src/Service/Indexer/SolrIndexService.php +++ b/src/Service/Indexer/SolrIndexService.php @@ -69,7 +69,7 @@ public function commit(string $core): void /** * @return string[] */ - public function getAvailableIndexes(): array + public function getAvailableIndices(): array { $client = $this->clientFactory->create(''); $coreAdminQuery = $client->createCoreAdmin(); diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 8c37ff3..eba0b7e 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -79,7 +79,7 @@ public function setUp(): void $this->updater->method('createDocument')->willReturn( new IndexSchema2xDocument() ); - $this->solrIndexService->method('getAvailableIndexes') + $this->solrIndexService->method('getAvailableIndices') ->willReturnCallback(function () { return $this->availableIndexes; }); diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php index b69854b..a865242 100644 --- a/test/Service/Indexer/SolrIndexServiceTest.php +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -65,7 +65,7 @@ public function testGetAvailableCores(): void $response->method('getStatusResults')->willReturn([$statusResult]); $this->client->method('coreAdmin')->willReturn($response); - $cores = $this->indexService->getAvailableIndexes(); + $cores = $this->indexService->getAvailableIndices(); $this->assertEquals(['test'], $cores, 'Cores should be returned'); } From 6bee895f6ee17468da1eee7e7ab4662f34a20d79 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 15:55:21 +0100 Subject: [PATCH 091/145] feat: add sp_id to external resources --- src/Service/Search/ExternalResourceFactory.php | 2 +- src/Service/Search/SolrSelect.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index 20c25ea..3271e35 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -38,7 +38,7 @@ public function create(Document $document): Resource return new Resource( $location, - '', + $this->getField($document, 'sp_id') ?? '', $this->getField($document, 'title') ?? '', 'external', [] diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index d936886..317e508 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -124,7 +124,7 @@ private function addSortToSolrQuery( private function addRequiredFieldListToSolrQuery( SolrSelectQuery $solrQuery ): void { - $solrQuery->setFields(['url']); + $solrQuery->setFields(['url', 'title', 'sp_id']); } private function addTextFilterToSolrQuery( From 0ebb6991b4100a531418c7af645a3194a47ef6da Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 16:02:43 +0100 Subject: [PATCH 092/145] refactor: if $len is uneven we run into an error --- src/Service/Search/SolrSuggest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 9a829ef..31331cf 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -117,7 +117,7 @@ private function parseSuggestion( $len = count($facets); $suggestions = []; - for ($i = 0; $i < $len; $i += 2) { + for ($i = 0; $i < $len - 1; $i += 2) { $term = $facets[$i]; $hits = (int)$facets[$i + 1]; $suggestions[] = new Suggestion($term, $hits); From da65a682b9db525377af0adf0a60d736680cf256 Mon Sep 17 00:00:00 2001 From: Holger Veltrup <92872893+sitepark-veltrup@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:28:59 +0100 Subject: [PATCH 093/145] refactor: Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mario Schäper <95750382+sitepark-schaeper@users.noreply.github.com> --- src/Console/Command/MoreLikeThis.php | 2 +- src/Dto/Search/Result/SuggestResult.php | 2 +- src/Exception/UnexpectedResultException.php | 2 +- src/Indexer.php | 2 +- src/Service/Indexer/ContentCollector.php | 2 +- src/Service/Indexer/DocumentEnricher.php | 5 ++-- src/Service/Indexer/IndexerStatusStore.php | 16 +++++++--- .../Indexer/InternalResourceIndexer.php | 8 ++--- .../Indexer/SiteKit/ContentMatcher.php | 4 +-- .../Search/ExternalResourceFactory.php | 10 ++----- .../Search/InternalMediaResourceFactory.php | 8 ++--- .../Search/InternalResourceFactory.php | 8 ++--- src/Service/Search/SolrMoreLikeThis.php | 6 ++-- .../Search/SolrResultToResourceResolver.php | 13 +-------- src/Service/Search/SolrSelect.php | 29 +++++++------------ src/Service/Search/SolrSuggest.php | 12 ++++---- 16 files changed, 53 insertions(+), 76 deletions(-) diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index bed1aa9..1946af0 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -54,7 +54,7 @@ protected function configure(): void 'location', InputArgument::REQUIRED, 'Location of the resource to which the MoreLikeThis ' . - 'search is to be applied..' + 'search is to be applied.' ) ; } diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php index 015a868..9d5a27a 100644 --- a/src/Dto/Search/Result/SuggestResult.php +++ b/src/Dto/Search/Result/SuggestResult.php @@ -9,7 +9,7 @@ /** * @implements IteratorAggregate -@codeCoverageIgnore + * @codeCoverageIgnore */ class SuggestResult implements IteratorAggregate { diff --git a/src/Exception/UnexpectedResultException.php b/src/Exception/UnexpectedResultException.php index 0a5ffda..9d4bf21 100644 --- a/src/Exception/UnexpectedResultException.php +++ b/src/Exception/UnexpectedResultException.php @@ -15,7 +15,7 @@ public function __construct( ?\Throwable $previous = null ) { parent::__construct( - $message . "\n" . $this->result, + $message . ": " . $this->result, $code, $previous ); diff --git a/src/Indexer.php b/src/Indexer.php index 7760eed..ea4a3f9 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -13,7 +13,7 @@ * The main task of an indexer is to systematically analyze documents or * content in order to extract relevant information from them. This information * is structured and stored in a search index to enable efficient search - * queries. The indexer organizes the data and extract hierarchical structure + * queries. The indexer organizes the data and extracts hierarchical structures * that search engines use to deliver fast and accurate search results. */ interface Indexer diff --git a/src/Service/Indexer/ContentCollector.php b/src/Service/Indexer/ContentCollector.php index 372abef..0ff5287 100644 --- a/src/Service/Indexer/ContentCollector.php +++ b/src/Service/Indexer/ContentCollector.php @@ -57,6 +57,6 @@ private function walk(array $path, array $data): array } } - return array_merge([], ...$contentCollections); + return array_merge(...$contentCollections); } } diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php index 6b3b06f..57869e2 100644 --- a/src/Service/Indexer/DocumentEnricher.php +++ b/src/Service/Indexer/DocumentEnricher.php @@ -18,8 +18,9 @@ interface DocumentEnricher public function isIndexable(Resource $resource): bool; /** - * @param T $doc - * @return T + * @template E of T + * @param E $doc + * @return E * @throws DocumentEnrichingException */ public function enrichDocument( diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index 334f747..3cb4cfd 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -48,7 +48,12 @@ public function load(string $index): IndexerStatus $json = file_get_contents($file); if ($json === false) { // @codeCoverageIgnoreStart - throw new InvalidArgumentException('Cannot read file ' . $file); + $message = 'Failed to read file ' . $file; + $error = error_get_last(); + if ($error !== null) { + $message .= ': ' . $error['message']; + } + throw new RuntimeException($message); // @codeCoverageIgnoreEnd } @@ -76,9 +81,12 @@ public function store(string $index, IndexerStatus $status): void $result = file_put_contents($file, $json); if ($result === false) { // @codeCoverageIgnoreStart - throw new RuntimeException( - 'Unable to write indexer-status file ' . $file - ); + $message = 'Unable to write indexer-status file ' . $file; + $error = error_get_last(); + if ($error !== null) { + $message .= ': ' . $error['message']; + } + throw new RuntimeException($message); // @codeCoverageIgnoreEnd } } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index fcefd74..61fc939 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -295,17 +295,15 @@ private function loadResources( int $length ): array|false { - $maxLength = (count($pathList) - $offset); + $maxLength = count($pathList) - $offset; if ($maxLength <= 0) { return false; } - if ($length > $maxLength) { - $length = $maxLength; - } + $end = min($length, $maxLength) + $offset; $resourceList = []; - for ($i = $offset; $i < ($length + $offset); $i++) { + for ($i = $offset; $i < $end; $i++) { $path = $pathList[$i]; try { $resource = $this->resourceLoader->load($path); diff --git a/src/Service/Indexer/SiteKit/ContentMatcher.php b/src/Service/Indexer/SiteKit/ContentMatcher.php index d1a75fa..6e60acd 100644 --- a/src/Service/Indexer/SiteKit/ContentMatcher.php +++ b/src/Service/Indexer/SiteKit/ContentMatcher.php @@ -6,8 +6,8 @@ /** * The `ContentMatcher` interface is implemented in order to extract from the - * content structure of resources to extract the content that is relevant - * for the relevant for the `content` field of the search index. + * content structure of resources the content that is relevant for the `content` + * field of the search index. */ interface ContentMatcher { diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index 3271e35..0312504 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -9,7 +9,7 @@ use Solarium\QueryType\Select\Result\Document; /** - * External resources are data that are not provided by the CMS, but are + * External resources are data that is not provided by the CMS, but are * transferred to the Solr index via special indexers, for example. * In these cases, the search result leads to an external page. * This factory recognizes external content using the Solr document field url. @@ -33,7 +33,7 @@ public function create(Document $document): Resource { $location = $this->getField($document, 'url'); if ($location === null) { - throw new LogicException('document should contains a url'); + throw new LogicException('document should contain an url'); } return new Resource( @@ -47,10 +47,6 @@ public function create(Document $document): Resource private function getField(Document $document, string $name): ?string { - $fields = $document->getFields(); - if (!isset($fields[$name])) { - return null; - } - return $fields[$name]; + return $document->getFields()[$name] ?? null; } } diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php index f5b62f4..5dd2fbf 100644 --- a/src/Service/Search/InternalMediaResourceFactory.php +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -36,7 +36,7 @@ public function create(Document $document): Resource { $metaLocation = $this->getMetaLocation($document); if ($metaLocation === null) { - throw new LogicException('document should contains a url'); + throw new LogicException('document should contain an url'); } return $this->resourceLoader->load($metaLocation); } @@ -52,10 +52,6 @@ private function getMetaLocation(Document $document): ?string private function getField(Document $document, string $name): ?string { - $fields = $document->getFields(); - if (!isset($fields[$name])) { - return null; - } - return $fields[$name]; + return $document->getFields()[$name] ?? null; } } diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php index 30620f5..3d701aa 100644 --- a/src/Service/Search/InternalResourceFactory.php +++ b/src/Service/Search/InternalResourceFactory.php @@ -34,17 +34,13 @@ public function create(Document $document): Resource { $location = $this->getField($document, 'url'); if ($location === null) { - throw new LogicException('document should contains a url'); + throw new LogicException('document should contain an url'); } return $this->resourceLoader->load($location); } private function getField(Document $document, string $name): ?string { - $fields = $document->getFields(); - if (!isset($fields[$name])) { - return null; - } - return $fields[$name]; + return $document->getFields()[$name] ?? null; } } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 1a14494..d00c664 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -49,9 +49,9 @@ private function buildSolrQuery( // Filter foreach ($query->filter as $filter) { - $solrQuery->createFilterQuery($filter->key) - ->setQuery($filter->getQuery()) - ->setTags($filter->tags); + $filterQuery = $solrQuery->createFilterQuery($filter->key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); } return $solrQuery; diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index 68da172..df69a1f 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -53,17 +53,6 @@ private function loadResource(Document $document): Resource } } - $location = $this->getField($document, 'url') ?? ''; - - throw new MissMatchingResourceFactoryException($location); - } - - private function getField(Document $document, string $name): ?string - { - $fields = $document->getFields(); - if (!isset($fields[$name])) { - return null; - } - return $fields[$name]; + throw new MissMatchingResourceFactoryException($document->getFields()['url'] ?? ''); } } diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSelect.php index 317e508..1d5e9c4 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSelect.php @@ -37,8 +37,8 @@ class SolrSelect implements SelectSearcher */ public function __construct( private readonly SolrClientFactory $clientFactory, - private readonly iterable $solrQueryModifierList, - private readonly SolrResultToResourceResolver $resultToResourceResolver + private readonly SolrResultToResourceResolver $resultToResourceResolver, + private readonly iterable $solrQueryModifierList = [] ) { } @@ -136,10 +136,7 @@ private function addTextFilterToSolrQuery( } $terms = explode(' ', $text); $terms = array_map( - static function ($term) use ($solrQuery) { - $term = trim($term); - return $solrQuery->getHelper()->escapeTerm($term); - }, + fn ($term) => $solrQuery->getHelper()->escapeTerm(trim($term)), $terms ); $text = implode(' ', $terms); @@ -150,15 +147,11 @@ private function addQueryDefaultOperatorToSolrQuery( SolrSelectQuery $solrQuery, QueryOperator $operator ): void { - if ($operator === QueryOperator::OR) { - $solrQuery->setQueryDefaultOperator( - SolrSelectQuery::QUERY_OPERATOR_OR - ); - } else { - $solrQuery->setQueryDefaultOperator( - SolrSelectQuery::QUERY_OPERATOR_AND - ); - } + $solrQuery->setQueryDefaultOperator( + $operator === QueryOperator::OR + ? SolrSelectQuery::QUERY_OPERATOR_OR + : SolrSelectQuery::QUERY_OPERATOR_AND + ); } /** @@ -171,9 +164,9 @@ private function addFilterQueriesToSolrQuery( foreach ($filterList as $filter) { $key = $filter->key ?? uniqid('', true); - $solrQuery->createFilterQuery($key) - ->setQuery($filter->getQuery()) - ->setTags($filter->tags); + $filterQuery = $solrQuery->createFilterQuery($key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); } } diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 31331cf..01812f0 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -16,13 +16,13 @@ use Solarium\QueryType\Select\Result\Result as SolrSelectResult; /** + * Implementation of the "suggest search" based on a Solr index. + * * @phpstan-type SolariumResponse array{ * facet_counts: array{ - * facet_fields:array> + * facet_fields: array> * } * } - * - * Implementation of the "suggest search" based on a Solr index. */ class SolrSuggest implements SuggestSearcher { @@ -75,9 +75,9 @@ private function buildSolrQuery( // Filter foreach ($query->filter as $filter) { - $solrQuery->createFilterQuery($filter->key) - ->setQuery($filter->getQuery()) - ->setTags($filter->tags); + $filterQuery = $solrQuery->createFilterQuery($filter->key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); } return $solrQuery; From 7f0911da6d509f4431ab1d710f00cd8885237f15 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 16:49:18 +0100 Subject: [PATCH 094/145] test: fix tests --- src/Console/Command/SolrSelectBuilder.php | 2 +- .../Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php | 5 +++-- src/Service/Search/SolrResultToResourceResolver.php | 4 +++- test/Service/Search/SolrSelectTest.php | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Console/Command/SolrSelectBuilder.php b/src/Console/Command/SolrSelectBuilder.php index 090470f..bdce652 100644 --- a/src/Console/Command/SolrSelectBuilder.php +++ b/src/Console/Command/SolrSelectBuilder.php @@ -66,8 +66,8 @@ public function build(): SolrSelect return new SolrSelect( $clientFactory, + $solrResultToResourceResolver, [$defaultBoosting], - $solrResultToResourceResolver ); } } diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 84c89d7..8c44cb1 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -281,8 +281,9 @@ public function enrichDocument( } /** - * @param IndexSchema2xDocument $doc - * @return IndexSchema2xDocument + * @template E of IndexSchema2xDocument + * @param E $doc + * @return E */ private function enrichContent( Resource $resource, diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index df69a1f..5d9da16 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -53,6 +53,8 @@ private function loadResource(Document $document): Resource } } - throw new MissMatchingResourceFactoryException($document->getFields()['url'] ?? ''); + throw new MissMatchingResourceFactoryException( + $document->getFields()['url'] ?? '' + ); } } diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSelectTest.php index f71d27a..a2b5005 100644 --- a/test/Service/Search/SolrSelectTest.php +++ b/test/Service/Search/SolrSelectTest.php @@ -77,8 +77,8 @@ protected function setUp(): void $this->searcher = new SolrSelect( $clientFactory, + $resultToResourceResolver, [$solrQueryModifier], - $resultToResourceResolver ); } From f4bdbd2d839105759bbfcacdb65536d0c27cd927 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 17:11:53 +0100 Subject: [PATCH 095/145] refactor: reduce code --- .../DefaultSchema2xDocumentEnricher.php | 128 +++++++----------- 1 file changed, 48 insertions(+), 80 deletions(-) diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 8c44cb1..79714de 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Indexer\SiteKit; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Resource; use Atoolo\Search\Exception\DocumentEnrichingException; @@ -63,13 +64,18 @@ public function enrichDocument( IndexDocument $doc, string $processId ): IndexDocument { + + $data = $resource->getData(); + $init = new DataBag($data->getAssociativeArray('init')); + $base = new DataBag($data->getAssociativeArray('base')); + $metadata = new DataBag($data->getAssociativeArray('metadata')); + $doc->sp_id = $resource->getId(); $doc->sp_name = $resource->getName(); - $doc->sp_anchor = $resource->getData()->getString('init.anchor'); - $doc->title = $resource->getData()->getString('base.title'); - $doc->description = $resource->getData()->getString( - 'metadata.description' - ); + $doc->sp_anchor = $init->getString('anchor'); + $doc->title = $base->getString('title'); + $doc->description = $metadata->getString('description'); + if (empty($doc->description)) { $doc->description = $resource->getData()->getString( 'metadata.intro' @@ -79,36 +85,29 @@ public function enrichDocument( $doc->sp_canonical = true; $doc->crawl_process_id = $processId; - $mediaUrl = $resource->getData()->getString('init.mediaUrl'); - if (!empty($mediaUrl)) { - $doc->id = $mediaUrl; - $doc->url = $mediaUrl; - } else { - $url = $resource->getData()->getString('init.url'); - $doc->id = $url; - $doc->url = $url; - } + $url = $init->getString('mediaUrl') + ?: $init->getString('url'); + $doc->id = $url; + $doc->url = $url; /** @var string[] $spContentType */ $spContentType = [$resource->getObjectType()]; - if ($resource->getData()->getBool('init.media') !== true) { + if ($init->getBool('media') !== true) { $spContentType[] = 'article'; } - $contentSectionTypes = $resource->getData()->getArray( - 'init.contentSectionTypes' - ); + $contentSectionTypes = $init->getArray('contentSectionTypes'); $spContentType = array_merge($spContentType, $contentSectionTypes); - if ($resource->getData()->has('base.teaser.image')) { + if ($base->has('teaser.image')) { $spContentType[] = 'teaserImage'; } - if ($resource->getData()->has('base.teaser.image.copyright')) { + if ($base->has('teaser.image.copyright')) { $spContentType[] = 'teaserImageCopyright'; } - if ($resource->getData()->has('base.teaser.headline')) { + if ($base->has('teaser.headline')) { $spContentType[] = 'teaserHeadline'; } - if ($resource->getData()->has('base.teaser.text')) { + if ($base->has('teaser.text')) { $spContentType[] = 'teaserText'; } $doc->sp_contenttype = $spContentType; @@ -119,47 +118,34 @@ public function enrichDocument( $doc->meta_content_language = $lang; $doc->sp_changed = $this->toDateTime( - $resource->getData()->getInt('init.changed') + $init->getInt('changed') ); $doc->sp_generated = $this->toDateTime( - $resource->getData()->getInt('init.generated') + $init->getInt('generated') ); $doc->sp_date = $this->toDateTime( - $resource->getData()->getInt('base.date') + $base->getInt('date') ); - $doc->sp_archive = $resource->getData()->getBool('base.archive'); + $doc->sp_archive = $base->getBool('archive'); - $headline = $resource->getData()->getString('metadata.headline'); - if (empty($headline)) { - $headline = $resource->getData()->getString('base.teaser.headline'); - } - if (empty($headline)) { - $headline = $resource->getData()->getString('base.title'); - } + $headline = $metadata->getString('headline') + ?: $base->getString('teaser.headline') + ?: $base->getString('title'); $doc->sp_title = $headline; - // However, the teaser heading, if specified, must be used for sorting - $sortHeadline = $resource->getData()->getString('base.teaser.headline'); - if (empty($sortHeadline)) { - $sortHeadline = $resource->getData()->getString( - 'metadata.headline' - ); - } - if (empty($sortHeadline)) { - $sortHeadline = $resource->getData()->getString('base.title'); - } + $sortHeadline = $base->getString('teaser.headline') + ?: $metadata->getString('headline') + ?: $base->getString('title'); $doc->sp_sortvalue = $sortHeadline; /** @var string[] $keyword */ - $keyword = $resource->getData()->getArray('metadata.keywords'); + $keyword = $metadata->getArray('keywords'); $doc->keywords = $keyword; $doc->sp_boost_keywords = implode( ' ', - $resource->getData()->getArray( - 'metadata.boostKeywords' - ) + $metadata->getArray('boostKeywords') ); try { @@ -186,15 +172,13 @@ public function enrichDocument( } /** @var string[] $wktPrimaryList */ - $wktPrimaryList = $resource->getData()->getArray( - 'base.geo.wkt.primary' - ); + $wktPrimaryList = $base->getArray('geo.wkt.primary'); if (!empty($wktPrimaryList)) { $doc->sp_geo_points = $wktPrimaryList; } /** @var array $categoryList */ - $categoryList = $resource->getData()->getArray('metadata.categories'); + $categoryList = $metadata->getArray('categories'); if (!empty($categoryList)) { $categoryIdList = []; foreach ($categoryList as $category) { @@ -204,9 +188,7 @@ public function enrichDocument( } /** @var array $categoryPath */ - $categoryPath = $resource->getData()->getArray( - 'metadata.categoriesPath' - ); + $categoryPath = $metadata->getArray('categoriesPath'); if (!empty($categoryPath)) { $categoryIdPath = []; foreach ($categoryPath as $category) { @@ -216,7 +198,7 @@ public function enrichDocument( } /** @var array $groupPath */ - $groupPath = $resource->getData()->getArray('init.groupPath'); + $groupPath = $init->getArray('groupPath'); $groupPathAsIdList = []; foreach ($groupPath as $group) { $groupPathAsIdList[] = $group['id']; @@ -228,7 +210,7 @@ public function enrichDocument( $doc->sp_group_path = $groupPathAsIdList; /** @var array $schedulingList */ - $schedulingList = $resource->getData()->getArray('metadata.scheduling'); + $schedulingList = $metadata->getArray('scheduling'); if (!empty($schedulingList)) { $doc->sp_date = $this->toDateTime($schedulingList[0]['from']); $dateList = []; @@ -249,16 +231,16 @@ public function enrichDocument( $doc->sp_date_list = $dateList; } - $contentType = $resource->getData()->getString( - 'base.mime', + $contentType = $base->getString( + 'mime', 'text/html; charset=UTF-8' ); $doc->meta_content_type = $contentType; - $accessType = $resource->getData()->getString('init.access.type'); + $accessType = $init->getString('access.type'); /** @var string[] $groups */ - $groups = $resource->getData()->getArray('init.access.groups'); + $groups = $init->getArray('access.groups'); if ($accessType === 'allow' && !empty($groups)) { $doc->include_groups = array_map( @@ -344,19 +326,18 @@ private function contactPointToContent(array $contactPoint): string $content[] = $areaCode; $content[] = '0' . $areaCode; } - $content[] = ($phone['phone']['localNumber'] ?? ''); + $content[] = $phone['phone']['localNumber'] ?? ''; } - foreach (($contactPoint['contactData']['emailList'] ?? []) as $email) { + foreach ($contactPoint['contactData']['emailList'] ?? [] as $email) { $content[] = $email['email']; } if (isset($contactPoint['addressData'])) { $addressData = $contactPoint['addressData']; - $content[] = ($addressData['street'] ?? ''); - $content[] = ($addressData['buildingName'] ?? ''); - $content[] = ( - $addressData['postOfficeBoxData']['buildingName'] ?? '' - ); + $content[] = $addressData['street'] ?? ''; + $content[] = $addressData['buildingName'] ?? ''; + $content[] = $addressData['postOfficeBoxData']['buildingName'] + ?? ''; } return implode(' ', $content); @@ -368,19 +349,6 @@ private function idWithoutSignature(string $id): int return (int)$s; } - /* Customization - * - https://gitlab.sitepark.com/customer-projects/fhdo/blob/develop/fhdo-module/src/publish/php/SP/Fhdo/Component/Content/DetailPage/StartletterIndexSupplier.php#L31 - * - https://gitlab.sitepark.com/apis/sitekit-php/blob/develop/php/SP/SiteKit/Component/Content/NewsdeskRss.php#L235 - * - https://gitlab.sitepark.com/customer-projects/fhdo/blob/develop/fhdo-module/src/publish/php/SP/Fhdo/Component/SearchMetadataExtension.php#L41 - * - https://gitlab.sitepark.com/customer-projects/paderborn/blob/develop/paderborn-module/src/publish/php/SP/Paderborn/Component/FscEntity.php#L67 - * - https://gitlab.sitepark.com/customer-projects/paderborn/blob/develop/paderborn-module/src/publish/php/SP/Paderborn/Component/FscContactPerson.php#L24 - * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/ParkingSpaceExpose.php#L38 - * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/Expose.php#L38 - * - https://gitlab.sitepark.com/customer-projects/stadtundland/blob/develop/stadtundland-module/src/publish/php/SP/Stadtundland/Component/PurchaseExpose.php#L38 - * - https://gitlab.sitepark.com/ies-modules/citycall/blob/develop/citycall-module/src/main/php/src/SP/CityCall/Component/Intro.php#L51 - * - https://gitlab.sitepark.com/ies-modules/sitekit-real-estate/blob/develop/src/publish/php/SP/RealEstate/Component/Expose.php#L47 - */ - private function getLocaleFromResource(Resource $resource): string { From 47e5287fd615e67815ec5366f3bab6626f7a9d21 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 17:31:36 +0100 Subject: [PATCH 096/145] refactor: rename SolrIndexService::getManagedIndexes() to SolrIndexService::getManagedIndices() --- src/Service/Indexer/InternalResourceIndexer.php | 12 ++++++------ src/Service/Indexer/SolrIndexService.php | 6 +++--- test/Service/Indexer/InternalResourceIndexerTest.php | 2 +- test/Service/Indexer/SolrIndexServiceTest.php | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 094777f..728e4a6 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -144,12 +144,12 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } - $availableIndexes = $this->indexService->getManagedIndexes(); + $managedIndices = $this->indexService->getManagedIndices(); $splitterResult = $this->translationSplitter->split($pathList); $this->indexTranslationSplittedResources( $parameter, - $availableIndexes, + $managedIndices, $splitterResult ); @@ -165,11 +165,11 @@ private function indexResources( * to their languages and can be indexed separately. Each language is * indexed separately here. * - * @param string[] $availableIndexes + * @param string[] $managedIndices */ private function indexTranslationSplittedResources( IndexerParameter $parameter, - array $availableIndexes, + array $managedIndices, TranslationSplitterResult $splitterResult ): void { @@ -178,7 +178,7 @@ private function indexTranslationSplittedResources( $index = $this->indexService->getIndex(''); if (count($splitterResult->getBases()) > 0) { - if (in_array($index, $availableIndexes)) { + if (in_array($index, $managedIndices)) { $this->indexResourcesPerLanguageIndex( $processId, $parameter, @@ -197,7 +197,7 @@ private function indexTranslationSplittedResources( $langIndex = $this->indexService->getIndex($lang); if ( $index !== $langIndex && - in_array($langIndex, $availableIndexes) + in_array($langIndex, $managedIndices) ) { $this->indexResourcesPerLanguageIndex( $processId, diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php index da2eefd..c6fae66 100644 --- a/src/Service/Indexer/SolrIndexService.php +++ b/src/Service/Indexer/SolrIndexService.php @@ -57,7 +57,7 @@ public function deleteByIdListForAllLanguages( public function deleteByQueryForAllLanguages(string $query): void { - foreach ($this->getManagedIndexes() as $index) { + foreach ($this->getManagedIndices() as $index) { $client = $this->clientFactory->create($index); $update = $client->createUpdate(); $update->addDeleteQuery($query); @@ -84,7 +84,7 @@ public function commit(string $lang): void public function commitForAllLanguages(): void { - foreach ($this->getManagedIndexes() as $index) { + foreach ($this->getManagedIndices() as $index) { $client = $this->clientFactory->create($index); $update = $client->createUpdate(); $update->addCommit(); @@ -96,7 +96,7 @@ public function commitForAllLanguages(): void /** * @return string[] */ - public function getManagedIndexes(): array + public function getManagedIndices(): array { $client = $this->createClient(''); $coreAdminQuery = $client->createCoreAdmin(); diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index b93fadf..479cb5e 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -82,7 +82,7 @@ public function setUp(): void $this->updater->method('createDocument')->willReturn( new IndexSchema2xDocument() ); - $this->solrIndexService->method('getManagedIndexes') + $this->solrIndexService->method('getManagedIndices') ->willReturnCallback(function () { return $this->availableIndexes; }); diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php index cfb523b..8f09505 100644 --- a/test/Service/Indexer/SolrIndexServiceTest.php +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -100,12 +100,12 @@ public function testGetManagedIndexes(): void $response->method('getStatusResults')->willReturn([$statusResult]); $this->client->method('coreAdmin')->willReturn($response); - $cores = $this->indexService->getManagedIndexes(); + $indices = $this->indexService->getManagedIndices(); $this->assertEquals( ['test', 'test-en_US'], - $cores, - 'Cores should be returned' + $indices, + 'Indices should be returned' ); } } From b3925fd813d27e3e393eb06a08e2b97afd19823f Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 17:45:23 +0100 Subject: [PATCH 097/145] refactor: reduce code duplications --- .../Indexer/InternalResourceIndexer.php | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 728e4a6..0e15f55 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -144,12 +144,10 @@ private function indexResources( $this->indexerProgressHandler->startUpdate($total); } - $managedIndices = $this->indexService->getManagedIndices(); $splitterResult = $this->translationSplitter->split($pathList); $this->indexTranslationSplittedResources( $parameter, - $managedIndices, $splitterResult ); @@ -164,52 +162,47 @@ private function indexResources( * to be used. Via the `$splitterResult` all paths are separated according * to their languages and can be indexed separately. Each language is * indexed separately here. - * - * @param string[] $managedIndices */ private function indexTranslationSplittedResources( IndexerParameter $parameter, - array $managedIndices, TranslationSplitterResult $splitterResult ): void { + $managedIndices = $this->indexService->getManagedIndices(); + $processId = uniqid('', true); $index = $this->indexService->getIndex(''); if (count($splitterResult->getBases()) > 0) { - if (in_array($index, $managedIndices)) { - $this->indexResourcesPerLanguageIndex( - $processId, - $parameter, - '', - $splitterResult->getBases() - ); - } else { - $this->indexerProgressHandler->error(new Exception( - 'Index "' . $index . '" not found' - )); - } + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + '', + $index, + $splitterResult->getBases() + ); } foreach ($splitterResult->getLocales() as $locale) { $lang = substr($locale, 0, 2); $langIndex = $this->indexService->getIndex($lang); - if ( - $index !== $langIndex && - in_array($langIndex, $managedIndices) - ) { - $this->indexResourcesPerLanguageIndex( - $processId, - $parameter, - $lang, - $splitterResult->getTranslations($locale) - ); - } else { + + if ($index === $langIndex) { $this->indexerProgressHandler->error(new Exception( - 'Index "' . $langIndex . '" not found' + 'No Index for language "' . $lang . '" ' . + 'found (base index: "' . $index . '")' )); + continue; } + + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + $lang, + $langIndex, + $splitterResult->getTranslations($locale) + ); } } @@ -222,11 +215,20 @@ private function indexResourcesPerLanguageIndex( string $processId, IndexerParameter $parameter, string $lang, + string $index, array $pathList ): void { $offset = 0; $successCount = 0; + $managedIndices = $this->indexService->getManagedIndices(); + if (!in_array($index, $managedIndices)) { + $this->indexerProgressHandler->error(new Exception( + 'Index "' . $index . '" not found' + )); + return; + } + while (true) { $indexedCount = $this->indexChunks( $processId, From 7764bd660c39c3af94e4dcd6d6b77442d365b6f4 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 17:48:03 +0100 Subject: [PATCH 098/145] refactor: cleanup code --- .../Indexer/InternalResourceIndexer.php | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 0e15f55..02c6fd4 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -168,21 +168,17 @@ private function indexTranslationSplittedResources( TranslationSplitterResult $splitterResult ): void { - $managedIndices = $this->indexService->getManagedIndices(); - $processId = uniqid('', true); $index = $this->indexService->getIndex(''); - if (count($splitterResult->getBases()) > 0) { - $this->indexResourcesPerLanguageIndex( - $processId, - $parameter, - '', - $index, - $splitterResult->getBases() - ); - } + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + '', + $index, + $splitterResult->getBases() + ); foreach ($splitterResult->getLocales() as $locale) { $lang = substr($locale, 0, 2); @@ -218,6 +214,11 @@ private function indexResourcesPerLanguageIndex( string $index, array $pathList ): void { + + if (empty($pathList)) { + return; + } + $offset = 0; $successCount = 0; From 054c64d36b67e153060d2f35774a62fdac098d84 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 21 Mar 2024 17:49:37 +0100 Subject: [PATCH 099/145] refactor: cleanup code --- src/Service/Indexer/InternalResourceIndexer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 02c6fd4..c223158 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -234,6 +234,7 @@ private function indexResourcesPerLanguageIndex( $indexedCount = $this->indexChunks( $processId, $lang, + $index, $pathList, $offset, $parameter->chunkSize @@ -271,6 +272,7 @@ private function indexResourcesPerLanguageIndex( private function indexChunks( string $processId, string $lang, + string $index, array $pathList, int $offset, int $length @@ -283,7 +285,6 @@ private function indexChunks( if ($resourceList === false) { return false; } - $index = $this->indexService->getIndex($lang); if ($this->aborter->isAbortionRequested($index)) { $this->aborter->resetAbortionRequest($index); $this->indexerProgressHandler->abort(); From afbf8d90404bd059986594d8a18c5e38dbcd0873 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 07:56:19 +0100 Subject: [PATCH 100/145] feat: add indexer-filter --- src/Service/Indexer/DocumentEnricher.php | 2 -- src/Service/Indexer/IndexerFilter.php | 12 ++++++++++++ src/Service/Indexer/InternalResourceIndexer.php | 9 ++++----- .../Indexer/InternalResourceIndexerFactory.php | 2 ++ .../SiteKit/DefaultSchema2xDocumentEnricher.php | 6 ------ src/Service/Indexer/SiteKit/NoIndexFilter.php | 17 +++++++++++++++++ .../InternalResourceIndexerFactoryTest.php | 2 ++ .../Indexer/InternalResourceIndexerTest.php | 10 ++++++++++ 8 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 src/Service/Indexer/IndexerFilter.php create mode 100644 src/Service/Indexer/SiteKit/NoIndexFilter.php diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php index 57869e2..95c0e39 100644 --- a/src/Service/Indexer/DocumentEnricher.php +++ b/src/Service/Indexer/DocumentEnricher.php @@ -15,8 +15,6 @@ */ interface DocumentEnricher { - public function isIndexable(Resource $resource): bool; - /** * @template E of T * @param E $doc diff --git a/src/Service/Indexer/IndexerFilter.php b/src/Service/Indexer/IndexerFilter.php new file mode 100644 index 0000000..eb06a61 --- /dev/null +++ b/src/Service/Indexer/IndexerFilter.php @@ -0,0 +1,12 @@ +indexService->updater($lang); foreach ($resources as $resource) { - foreach ($this->documentEnricherList as $enricher) { - if (!$enricher->isIndexable($resource)) { - $this->indexerProgressHandler->skip(1); - continue 2; - } + if ($this->indexerFilter->accept($resource) === false) { + $this->indexerProgressHandler->skip(1); + continue; } try { /** @var IndexSchema2xDocument $doc */ diff --git a/src/Service/Indexer/InternalResourceIndexerFactory.php b/src/Service/Indexer/InternalResourceIndexerFactory.php index 708d1e1..5f134f4 100644 --- a/src/Service/Indexer/InternalResourceIndexerFactory.php +++ b/src/Service/Indexer/InternalResourceIndexerFactory.php @@ -13,6 +13,7 @@ class InternalResourceIndexerFactory */ public function __construct( private readonly iterable $documentEnricherList, + private readonly IndexerFilter $indexerFilter, private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, private readonly TranslationSplitter $translationSplitter, @@ -27,6 +28,7 @@ public function create( ): InternalResourceIndexer { return new InternalResourceIndexer( $this->documentEnricherList, + $this->indexerFilter, $progressHandler, $this->finder, $this->resourceLoader, diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 79714de..0f6f775 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -50,12 +50,6 @@ public function __construct( ) { } - public function isIndexable(Resource $resource): bool - { - $noIndex = $resource->getData()->getBool('init.noIndex'); - return $noIndex !== true; - } - /** * @throws DocumentEnrichingException */ diff --git a/src/Service/Indexer/SiteKit/NoIndexFilter.php b/src/Service/Indexer/SiteKit/NoIndexFilter.php new file mode 100644 index 0000000..97859ce --- /dev/null +++ b/src/Service/Indexer/SiteKit/NoIndexFilter.php @@ -0,0 +1,17 @@ +getData()->getBool('init.noIndex'); + return $noIndex !== true; + } +} diff --git a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php index 95e9561..66bf3bc 100644 --- a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php +++ b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Resource\ResourceLoader; +use Atoolo\Search\Service\Indexer\IndexerFilter; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; @@ -19,6 +20,7 @@ public function testCreate(): void { $factory = new InternalResourceIndexerFactory( [], + $this->createStub(IndexerFilter::class), $this->createStub(LocationFinder::class), $this->createStub(ResourceLoader::class), $this->createStub(TranslationSplitter::class), diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 479cb5e..dcf749d 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -7,6 +7,7 @@ use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\DocumentEnricher; +use Atoolo\Search\Service\Indexer\IndexerFilter; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; @@ -31,6 +32,8 @@ class InternalResourceIndexerTest extends TestCase */ private array $availableIndexes = ['test', 'test-en_US']; + public IndexerFilter&MockObject $indexerFilter; + private ResourceLoader&Stub $resourceLoader; private IndexerProgressHandler&MockObject $indexerProgressHandler; @@ -56,6 +59,12 @@ class InternalResourceIndexerTest extends TestCase */ public function setUp(): void { + $this->indexerFilter = $this->createMock( + IndexerFilter::class + ); + $this->indexerFilter->method('accept') + ->willReturn(true); + $this->indexerProgressHandler = $this->createMock( IndexerProgressHandler::class ); @@ -99,6 +108,7 @@ public function setUp(): void $this->indexer = new InternalResourceIndexer( [ $this->documentEnricher ], + $this->indexerFilter, $this->indexerProgressHandler, $this->finder, $this->resourceLoader, From 784cb4e36db1e74e865f7b6d5076beb0fee04837 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 09:00:04 +0100 Subject: [PATCH 101/145] test: fix test for intexer-filter --- .../Indexer/InternalResourceIndexerTest.php | 12 +++------ .../DefaultSchema2xDocumentEnricherTest.php | 27 ------------------- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index dcf749d..ad5a2ef 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -62,8 +62,6 @@ public function setUp(): void $this->indexerFilter = $this->createMock( IndexerFilter::class ); - $this->indexerFilter->method('accept') - ->willReturn(true); $this->indexerProgressHandler = $this->createMock( IndexerProgressHandler::class @@ -166,8 +164,7 @@ public function testIndexAllWithChunks(): void $this->updateResult->method('getStatus') ->willReturn(0); - - $this->documentEnricher->method('isIndexable') + $this->indexerFilter->method('accept') ->willReturn(true); $this->documentEnricher @@ -207,7 +204,7 @@ public function testIndexSkipResource(): void $this->updateResult->method('getStatus') ->willReturn(0); - $this->documentEnricher->method('isIndexable') + $this->indexerFilter->method('accept') ->willReturnCallback(function (Resource $resource) { $location = $resource->getLocation(); return ($location !== '/a/b.php'); @@ -263,9 +260,6 @@ public function testWithUnsuccessfulStatus(): void $this->updateResult->method('getStatus') ->willReturn(500); - $this->documentEnricher->method('isIndexable') - ->willReturn(true); - $this->indexerProgressHandler->expects($this->once()) ->method('error'); @@ -323,7 +317,7 @@ public function testIndexPaths(): void $this->updateResult->method('getStatus') ->willReturn(0); - $this->documentEnricher->method('isIndexable') + $this->indexerFilter->method('accept') ->willReturn(true); $this->updater->expects($this->exactly(2)) diff --git a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php index 8bfddc7..de1cee6 100644 --- a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php +++ b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php @@ -46,33 +46,6 @@ public function setUp(): void ); } - public function testIndexable(): void - { - $resource = $this->createResource([ - 'init' => [ - ] - ]); - - $this->assertTrue( - $this->enricher->isIndexable($resource), - 'should be indexalbe' - ); - } - - public function testNotIndexable(): void - { - $resource = $this->createResource([ - 'init' => [ - 'noIndex' => true - ] - ]); - - $this->assertFalse( - $this->enricher->isIndexable($resource), - 'should be indexalbe' - ); - } - public function testEnrichSpId(): void { $resource = $this->createStub(Resource::class); From 5c956faffcd8524466aabe50ff6af309e328ce18 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 09:14:28 +0100 Subject: [PATCH 102/145] refactor: rename Select to Search --- src/Console/Command/Search.php | 14 ++--- src/Console/Command/Suggest.php | 2 +- .../{SelectQuery.php => SearchQuery.php} | 4 +- ...ueryBuilder.php => SearchQueryBuilder.php} | 6 +-- ...eLikeThisSearcher.php => MoreLikeThis.php} | 2 +- src/{SelectSearcher.php => Searcher.php} | 6 +-- src/Service/Search/SolrMoreLikeThis.php | 4 +- .../Search/{SolrSelect.php => SolrSearch.php} | 14 ++--- src/Service/Search/SolrSuggest.php | 6 +-- src/{SuggestSearcher.php => Suggest.php} | 4 +- test/Console/Command/SearchTest.php | 6 +-- test/Console/Command/SuggestTest.php | 2 +- ...derTest.php => SearchQueryBuilderTest.php} | 10 ++-- ...{SolrSelectTest.php => SolrSearchTest.php} | 52 +++++++++---------- test/Service/Search/SolrSuggestTest.php | 6 +-- 15 files changed, 69 insertions(+), 69 deletions(-) rename src/Dto/Search/Query/{SelectQuery.php => SearchQuery.php} (93%) rename src/Dto/Search/Query/{SelectQueryBuilder.php => SearchQueryBuilder.php} (97%) rename src/{MoreLikeThisSearcher.php => MoreLikeThis.php} (95%) rename src/{SelectSearcher.php => Searcher.php} (59%) rename src/Service/Search/{SolrSelect.php => SolrSearch.php} (97%) rename src/{SuggestSearcher.php => Suggest.php} (80%) rename test/Dto/Search/Query/{SelectQueryBuilderTest.php => SearchQueryBuilderTest.php} (94%) rename test/Service/Search/{SolrSelectTest.php => SolrSearchTest.php} (87%) diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 633b8c3..b0c47a5 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -5,10 +5,10 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Search\Console\Command\Io\TypifiedInput; -use Atoolo\Search\Dto\Search\Query\SelectQuery; -use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; +use Atoolo\Search\Dto\Search\Query\SearchQuery; +use Atoolo\Search\Dto\Search\Query\SearchQueryBuilder; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\Service\Search\SolrSelect; +use Atoolo\Search\Service\Search\SolrSearch; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -26,7 +26,7 @@ class Search extends Command private TypifiedInput $input; public function __construct( - private readonly SolrSelect $searcher + private readonly SolrSearch $searcher ) { parent::__construct(); } @@ -60,16 +60,16 @@ protected function execute( $query = $this->buildQuery($input); - $result = $this->searcher->select($query); + $result = $this->searcher->search($query); $this->outputResult($result); return Command::SUCCESS; } - protected function buildQuery(InputInterface $input): SelectQuery + protected function buildQuery(InputInterface $input): SearchQuery { - $builder = new SelectQueryBuilder(); + $builder = new SearchQueryBuilder(); $text = $this->input->getStringArgument('text'); $builder->text($text); diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index 393e5c7..272bbd4 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -62,7 +62,7 @@ protected function execute( $query = $this->buildQuery($terms, $lang); - $result = $this->search->suggest($query); + $result = $this->search->search($query); $this->outputResult($result); diff --git a/src/Dto/Search/Query/SelectQuery.php b/src/Dto/Search/Query/SearchQuery.php similarity index 93% rename from src/Dto/Search/Query/SelectQuery.php rename to src/Dto/Search/Query/SearchQuery.php index 2fe5b4a..7ba985c 100644 --- a/src/Dto/Search/Query/SelectQuery.php +++ b/src/Dto/Search/Query/SearchQuery.php @@ -11,14 +11,14 @@ /** * @codeCoverageIgnore */ -class SelectQuery +class SearchQuery { /** * @param Criteria[] $sort * @param Filter[] $filter * @param Facet[] $facets * @internal Do not use the constructor directly, - * but the SelectQueryBuilder + * but the SearchQueryBuilder */ public function __construct( public readonly string $text, diff --git a/src/Dto/Search/Query/SelectQueryBuilder.php b/src/Dto/Search/Query/SearchQueryBuilder.php similarity index 97% rename from src/Dto/Search/Query/SelectQueryBuilder.php rename to src/Dto/Search/Query/SearchQueryBuilder.php index 1ad7d15..ea7b465 100644 --- a/src/Dto/Search/Query/SelectQueryBuilder.php +++ b/src/Dto/Search/Query/SearchQueryBuilder.php @@ -8,7 +8,7 @@ use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; -class SelectQueryBuilder +class SearchQueryBuilder { private string $text = ''; private string $lang = ''; @@ -132,9 +132,9 @@ public function defaultQueryOperator( return $this; } - public function build(): SelectQuery + public function build(): SearchQuery { - return new SelectQuery( + return new SearchQuery( $this->text, $this->lang, $this->offset, diff --git a/src/MoreLikeThisSearcher.php b/src/MoreLikeThis.php similarity index 95% rename from src/MoreLikeThisSearcher.php rename to src/MoreLikeThis.php index cbe0d22..3b06970 100644 --- a/src/MoreLikeThisSearcher.php +++ b/src/MoreLikeThis.php @@ -18,7 +18,7 @@ * * The reference point is specified via the location of a resource. */ -interface MoreLikeThisSearcher +interface MoreLikeThis { public function moreLikeThis( MoreLikeThisQuery $query diff --git a/src/SelectSearcher.php b/src/Searcher.php similarity index 59% rename from src/SelectSearcher.php rename to src/Searcher.php index f2a91c6..b3ac640 100644 --- a/src/SelectSearcher.php +++ b/src/Searcher.php @@ -4,13 +4,13 @@ namespace Atoolo\Search; -use Atoolo\Search\Dto\Search\Query\SelectQuery; +use Atoolo\Search\Dto\Search\Query\SearchQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; /** * The service interface for a search with full-text, filter and facet support. */ -interface SelectSearcher +interface Searcher { - public function select(SelectQuery $query): SearchResult; + public function search(SearchQuery $query): SearchResult; } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index b265242..4d0d370 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -6,7 +6,7 @@ use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\MoreLikeThisSearcher; +use Atoolo\Search\MoreLikeThis; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Core\Client\Client; @@ -16,7 +16,7 @@ /** * Implementation of the "More-Like-This" on the basis of a Solr index. */ -class SolrMoreLikeThis implements MoreLikeThisSearcher +class SolrMoreLikeThis implements MoreLikeThis { public function __construct( private readonly IndexName $index, diff --git a/src/Service/Search/SolrSelect.php b/src/Service/Search/SolrSearch.php similarity index 97% rename from src/Service/Search/SolrSelect.php rename to src/Service/Search/SolrSearch.php index 04ee965..0ae309c 100644 --- a/src/Service/Search/SolrSelect.php +++ b/src/Service/Search/SolrSearch.php @@ -9,7 +9,7 @@ use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\QueryOperator; -use Atoolo\Search\Dto\Search\Query\SelectQuery; +use Atoolo\Search\Dto\Search\Query\SearchQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; use Atoolo\Search\Dto\Search\Query\Sort\Headline; @@ -19,7 +19,7 @@ use Atoolo\Search\Dto\Search\Result\Facet; use Atoolo\Search\Dto\Search\Result\FacetGroup; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\SelectSearcher; +use Atoolo\Search\Searcher; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use InvalidArgumentException; @@ -31,7 +31,7 @@ /** * Implementation of the searcher on the basis of a Solr index. */ -class SolrSelect implements SelectSearcher +class SolrSearch implements Searcher { /** * @param iterable $solrQueryModifierList @@ -44,7 +44,7 @@ public function __construct( ) { } - public function select(SelectQuery $query): SearchResult + public function search(SearchQuery $query): SearchResult { $index = $this->index->name($query->lang); $client = $this->clientFactory->create($index); @@ -57,7 +57,7 @@ public function select(SelectQuery $query): SearchResult private function buildSolrQuery( Client $client, - SelectQuery $query + SearchQuery $query ): SolrSelectQuery { $solrQuery = $client->createSelect(); @@ -245,7 +245,7 @@ private function addFacetMultiQueryToSolrQuery( } private function buildResult( - SelectQuery $query, + SearchQuery $query, SelectResult $result, string $lang ): SearchResult { @@ -268,7 +268,7 @@ private function buildResult( * @return FacetGroup[] */ private function buildFacetGroupList( - SelectQuery $query, + SearchQuery $query, SelectResult $result ): array { diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 81fc1ec..7b5ee53 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -10,7 +10,7 @@ use Atoolo\Search\Exception\UnexpectedResultException; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; -use Atoolo\Search\SuggestSearcher; +use Atoolo\Search\Suggest; use JsonException; use Solarium\Core\Client\Client; use Solarium\QueryType\Select\Query\Query as SolrSelectQuery; @@ -25,7 +25,7 @@ * } * } */ -class SolrSuggest implements SuggestSearcher +class SolrSuggest implements Suggest { private const INDEX_SUGGEST_FIELD = 'raw_content'; @@ -38,7 +38,7 @@ public function __construct( /** * @throws UnexpectedResultException */ - public function suggest(SuggestQuery $query): SuggestResult + public function search(SuggestQuery $query): SuggestResult { $index = $this->index->name($query->lang); $client = $this->clientFactory->create($index); diff --git a/src/SuggestSearcher.php b/src/Suggest.php similarity index 80% rename from src/SuggestSearcher.php rename to src/Suggest.php index e5ff274..6cc743d 100644 --- a/src/SuggestSearcher.php +++ b/src/Suggest.php @@ -13,7 +13,7 @@ * A "suggest search" is a search function that automatically displays * suggestions or auto-completions to users as they enter search queries. */ -interface SuggestSearcher +interface Suggest { - public function suggest(SuggestQuery $query): SuggestResult; + public function search(SuggestQuery $query): SuggestResult; } diff --git a/test/Console/Command/SearchTest.php b/test/Console/Command/SearchTest.php index 5b992ba..012a01b 100644 --- a/test/Console/Command/SearchTest.php +++ b/test/Console/Command/SearchTest.php @@ -10,7 +10,7 @@ use Atoolo\Search\Dto\Search\Result\Facet; use Atoolo\Search\Dto\Search\Result\FacetGroup; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\Service\Search\SolrSelect; +use Atoolo\Search\Service\Search\SolrSearch; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; @@ -43,8 +43,8 @@ public function setUp(): void )], 10 ); - $solrSelect = $this->createStub(SolrSelect::class); - $solrSelect->method('select') + $solrSelect = $this->createStub(SolrSearch::class); + $solrSelect->method('search') ->willReturn($result); $command = new Search($solrSelect); diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index 21cc40b..3b3ec56 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -33,7 +33,7 @@ public function setUp(): void 10 ); $solrSuggest = $this->createStub(SolrSuggest::class); - $solrSuggest->method('suggest') + $solrSuggest->method('search') ->willReturn($result); $command = new Suggest($solrSuggest); diff --git a/test/Dto/Search/Query/SelectQueryBuilderTest.php b/test/Dto/Search/Query/SearchQueryBuilderTest.php similarity index 94% rename from test/Dto/Search/Query/SelectQueryBuilderTest.php rename to test/Dto/Search/Query/SearchQueryBuilderTest.php index fbe92b4..40f2e5e 100644 --- a/test/Dto/Search/Query/SelectQueryBuilderTest.php +++ b/test/Dto/Search/Query/SearchQueryBuilderTest.php @@ -7,21 +7,21 @@ use Atoolo\Search\Dto\Search\Query\Facet\Facet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\QueryOperator; -use Atoolo\Search\Dto\Search\Query\SelectQueryBuilder; +use Atoolo\Search\Dto\Search\Query\SearchQueryBuilder; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; -#[CoversClass(SelectQueryBuilder::class)] -class SelectQueryBuilderTest extends TestCase +#[CoversClass(SearchQueryBuilder::class)] +class SearchQueryBuilderTest extends TestCase { - private SelectQueryBuilder $builder; + private SearchQueryBuilder $builder; protected function setUp(): void { - $this->builder = new SelectQueryBuilder(); + $this->builder = new SearchQueryBuilder(); } public function testSetText(): void diff --git a/test/Service/Search/SolrSelectTest.php b/test/Service/Search/SolrSearchTest.php similarity index 87% rename from test/Service/Search/SolrSelectTest.php rename to test/Service/Search/SolrSearchTest.php index b5c3f4f..94d6ac1 100644 --- a/test/Service/Search/SolrSelectTest.php +++ b/test/Service/Search/SolrSearchTest.php @@ -11,7 +11,7 @@ use Atoolo\Search\Dto\Search\Query\Facet\ObjectTypeFacet; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\QueryOperator; -use Atoolo\Search\Dto\Search\Query\SelectQuery; +use Atoolo\Search\Dto\Search\Query\SearchQuery; use Atoolo\Search\Dto\Search\Query\Sort\Criteria; use Atoolo\Search\Dto\Search\Query\Sort\Date; use Atoolo\Search\Dto\Search\Query\Sort\Headline; @@ -21,7 +21,7 @@ use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\Search\SolrQueryModifier; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; -use Atoolo\Search\Service\Search\SolrSelect; +use Atoolo\Search\Service\Search\SolrSearch; use Atoolo\Search\Service\SolrClientFactory; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; @@ -34,14 +34,14 @@ use Solarium\QueryType\Select\Query\Query as SolrSelectQuery; use Solarium\QueryType\Select\Result\Result as SelectResult; -#[CoversClass(SolrSelect::class)] -class SolrSelectTest extends TestCase +#[CoversClass(SolrSearch::class)] +class SolrSearchTest extends TestCase { private Resource|Stub $resource; private SelectResult|Stub $result; - private SolrSelect $searcher; + private SolrSearch $searcher; protected function setUp(): void { @@ -77,7 +77,7 @@ protected function setUp(): void ->method('loadResourceList') ->willReturn([$this->resource]); - $this->searcher = new SolrSelect( + $this->searcher = new SolrSearch( $indexName, $clientFactory, $resultToResourceResolver, @@ -87,7 +87,7 @@ protected function setUp(): void public function testSelectEmpty(): void { - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -99,7 +99,7 @@ public function testSelectEmpty(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -110,7 +110,7 @@ public function testSelectEmpty(): void public function testSelectWithText(): void { - $query = new SelectQuery( + $query = new SearchQuery( 'cat dog', '', 0, @@ -122,7 +122,7 @@ public function testSelectWithText(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -133,7 +133,7 @@ public function testSelectWithText(): void public function testSelectWithSort(): void { - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -150,7 +150,7 @@ public function testSelectWithSort(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -163,7 +163,7 @@ public function testSelectWithInvalidSort(): void { $sort = $this->createStub(Criteria::class); - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -175,12 +175,12 @@ public function testSelectWithInvalidSort(): void ); $this->expectException(InvalidArgumentException::class); - $this->searcher->select($query); + $this->searcher->search($query); } public function testSelectWithAndDefaultOperator(): void { - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -191,7 +191,7 @@ public function testSelectWithAndDefaultOperator(): void QueryOperator::AND ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -206,7 +206,7 @@ public function testSelectWithFilter(): void ->setConstructorArgs(['test', []]) ->getMock(); - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -217,7 +217,7 @@ public function testSelectWithFilter(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -239,7 +239,7 @@ public function testSelectWithFacets(): void ) ]; - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -250,7 +250,7 @@ public function testSelectWithFacets(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( [$this->resource], @@ -266,7 +266,7 @@ public function testSelectWithInvalidFacets(): void $this->createStub(Facet::class) ]; - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -278,7 +278,7 @@ public function testSelectWithInvalidFacets(): void ); $this->expectException(InvalidArgumentException::class); - $this->searcher->select($query); + $this->searcher->search($query); } public function testResultFacets(): void @@ -305,7 +305,7 @@ public function testResultFacets(): void ) ]; - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -316,7 +316,7 @@ public function testResultFacets(): void QueryOperator::OR ); - $searchResult = $this->searcher->select($query); + $searchResult = $this->searcher->search($query); $this->assertEquals( 'objectType', @@ -348,7 +348,7 @@ public function testInvalidResultFacets(): void ) ]; - $query = new SelectQuery( + $query = new SearchQuery( '', '', 0, @@ -360,6 +360,6 @@ public function testInvalidResultFacets(): void ); $this->expectException(InvalidArgumentException::class); - $this->searcher->select($query); + $this->searcher->search($query); } } diff --git a/test/Service/Search/SolrSuggestTest.php b/test/Service/Search/SolrSuggestTest.php index 734420d..6541b55 100644 --- a/test/Service/Search/SolrSuggestTest.php +++ b/test/Service/Search/SolrSuggestTest.php @@ -78,7 +78,7 @@ public function testSuggest(): void $this->result->method('getResponse')->willReturn($response); - $suggestResult = $this->searcher->suggest($query); + $suggestResult = $this->searcher->search($query); $expected = [ new Suggestion('category', 10), @@ -108,7 +108,7 @@ public function testEmptySuggest(): void $this->result->method('getResponse')->willReturn($response); - $suggestResult = $this->searcher->suggest($query); + $suggestResult = $this->searcher->search($query); $this->assertEmpty( $suggestResult->suggestions, @@ -128,6 +128,6 @@ public function testInvalidSuggestResponse(): void $this->result->method('getResponse')->willReturn($response); $this->expectException(UnexpectedResultException::class); - $this->searcher->suggest($query); + $this->searcher->search($query); } } From 1a2081d2a419dee418e05167cf07daf6652f2d70 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 09:33:29 +0100 Subject: [PATCH 103/145] refactor Suggest::search to Suggest::suggest --- src/Console/Command/Suggest.php | 2 +- src/Service/Search/SolrSuggest.php | 2 +- src/Suggest.php | 2 +- test/Console/Command/SuggestTest.php | 2 +- test/Service/Search/SolrSuggestTest.php | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index 272bbd4..393e5c7 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -62,7 +62,7 @@ protected function execute( $query = $this->buildQuery($terms, $lang); - $result = $this->search->search($query); + $result = $this->search->suggest($query); $this->outputResult($result); diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index 7b5ee53..c86328b 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -38,7 +38,7 @@ public function __construct( /** * @throws UnexpectedResultException */ - public function search(SuggestQuery $query): SuggestResult + public function suggest(SuggestQuery $query): SuggestResult { $index = $this->index->name($query->lang); $client = $this->clientFactory->create($index); diff --git a/src/Suggest.php b/src/Suggest.php index 6cc743d..f8e53e3 100644 --- a/src/Suggest.php +++ b/src/Suggest.php @@ -15,5 +15,5 @@ */ interface Suggest { - public function search(SuggestQuery $query): SuggestResult; + public function suggest(SuggestQuery $query): SuggestResult; } diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index 3b3ec56..21cc40b 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -33,7 +33,7 @@ public function setUp(): void 10 ); $solrSuggest = $this->createStub(SolrSuggest::class); - $solrSuggest->method('search') + $solrSuggest->method('suggest') ->willReturn($result); $command = new Suggest($solrSuggest); diff --git a/test/Service/Search/SolrSuggestTest.php b/test/Service/Search/SolrSuggestTest.php index 6541b55..734420d 100644 --- a/test/Service/Search/SolrSuggestTest.php +++ b/test/Service/Search/SolrSuggestTest.php @@ -78,7 +78,7 @@ public function testSuggest(): void $this->result->method('getResponse')->willReturn($response); - $suggestResult = $this->searcher->search($query); + $suggestResult = $this->searcher->suggest($query); $expected = [ new Suggestion('category', 10), @@ -108,7 +108,7 @@ public function testEmptySuggest(): void $this->result->method('getResponse')->willReturn($response); - $suggestResult = $this->searcher->search($query); + $suggestResult = $this->searcher->suggest($query); $this->assertEmpty( $suggestResult->suggestions, @@ -128,6 +128,6 @@ public function testInvalidSuggestResponse(): void $this->result->method('getResponse')->willReturn($response); $this->expectException(UnexpectedResultException::class); - $this->searcher->search($query); + $this->searcher->suggest($query); } } From 3d819f0073533b0a3fc549f0f9dac29df5d86dda Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 09:39:44 +0100 Subject: [PATCH 104/145] refactor Searcher to Search --- src/{Searcher.php => Search.php} | 2 +- src/Service/Search/SolrSearch.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/{Searcher.php => Search.php} (94%) diff --git a/src/Searcher.php b/src/Search.php similarity index 94% rename from src/Searcher.php rename to src/Search.php index b3ac640..164fa15 100644 --- a/src/Searcher.php +++ b/src/Search.php @@ -10,7 +10,7 @@ /** * The service interface for a search with full-text, filter and facet support. */ -interface Searcher +interface Search { public function search(SearchQuery $query): SearchResult; } diff --git a/src/Service/Search/SolrSearch.php b/src/Service/Search/SolrSearch.php index 0ae309c..01954f1 100644 --- a/src/Service/Search/SolrSearch.php +++ b/src/Service/Search/SolrSearch.php @@ -19,7 +19,7 @@ use Atoolo\Search\Dto\Search\Result\Facet; use Atoolo\Search\Dto\Search\Result\FacetGroup; use Atoolo\Search\Dto\Search\Result\SearchResult; -use Atoolo\Search\Searcher; +use Atoolo\Search\Search; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use InvalidArgumentException; @@ -31,7 +31,7 @@ /** * Implementation of the searcher on the basis of a Solr index. */ -class SolrSearch implements Searcher +class SolrSearch implements Search { /** * @param iterable $solrQueryModifierList From 795241f340895881e19df99dbd7d806c5eac2630 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 10:06:00 +0100 Subject: [PATCH 105/145] refactor: rename IndexerFilter to ResourceFilter --- src/Service/Indexer/InternalResourceIndexer.php | 4 ++-- src/Service/Indexer/InternalResourceIndexerFactory.php | 2 +- .../Indexer/{IndexerFilter.php => ResourceFilter.php} | 2 +- src/Service/Indexer/SiteKit/NoIndexFilter.php | 4 ++-- test/Service/Indexer/InternalResourceIndexerFactoryTest.php | 4 ++-- test/Service/Indexer/InternalResourceIndexerTest.php | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) rename src/Service/Indexer/{IndexerFilter.php => ResourceFilter.php} (86%) diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 0b639c9..1ecc9b3 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -41,7 +41,7 @@ class InternalResourceIndexer implements Indexer */ public function __construct( private readonly iterable $documentEnricherList, - private readonly IndexerFilter $indexerFilter, + private readonly ResourceFilter $resourceFilter, private readonly IndexerProgressHandler $indexerProgressHandler, private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, @@ -349,7 +349,7 @@ private function add( $updater = $this->indexService->updater($lang); foreach ($resources as $resource) { - if ($this->indexerFilter->accept($resource) === false) { + if ($this->resourceFilter->accept($resource) === false) { $this->indexerProgressHandler->skip(1); continue; } diff --git a/src/Service/Indexer/InternalResourceIndexerFactory.php b/src/Service/Indexer/InternalResourceIndexerFactory.php index 5f134f4..3c1c996 100644 --- a/src/Service/Indexer/InternalResourceIndexerFactory.php +++ b/src/Service/Indexer/InternalResourceIndexerFactory.php @@ -13,7 +13,7 @@ class InternalResourceIndexerFactory */ public function __construct( private readonly iterable $documentEnricherList, - private readonly IndexerFilter $indexerFilter, + private readonly ResourceFilter $indexerFilter, private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, private readonly TranslationSplitter $translationSplitter, diff --git a/src/Service/Indexer/IndexerFilter.php b/src/Service/Indexer/ResourceFilter.php similarity index 86% rename from src/Service/Indexer/IndexerFilter.php rename to src/Service/Indexer/ResourceFilter.php index eb06a61..b104ae7 100644 --- a/src/Service/Indexer/IndexerFilter.php +++ b/src/Service/Indexer/ResourceFilter.php @@ -6,7 +6,7 @@ use Atoolo\Resource\Resource; -interface IndexerFilter +interface ResourceFilter { public function accept(Resource $resource): bool; } diff --git a/src/Service/Indexer/SiteKit/NoIndexFilter.php b/src/Service/Indexer/SiteKit/NoIndexFilter.php index 97859ce..da5e62d 100644 --- a/src/Service/Indexer/SiteKit/NoIndexFilter.php +++ b/src/Service/Indexer/SiteKit/NoIndexFilter.php @@ -5,9 +5,9 @@ namespace Atoolo\Search\Service\Indexer\SiteKit; use Atoolo\Resource\Resource; -use Atoolo\Search\Service\Indexer\IndexerFilter; +use Atoolo\Search\Service\Indexer\ResourceFilter; -class NoIndexFilter implements IndexerFilter +class NoIndexFilter implements ResourceFilter { public function accept(Resource $resource): bool { diff --git a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php index 66bf3bc..cb4e170 100644 --- a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php +++ b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php @@ -5,11 +5,11 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Resource\ResourceLoader; -use Atoolo\Search\Service\Indexer\IndexerFilter; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; use Atoolo\Search\Service\Indexer\LocationFinder; +use Atoolo\Search\Service\Indexer\ResourceFilter; use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\Indexer\TranslationSplitter; use PHPUnit\Framework\TestCase; @@ -20,7 +20,7 @@ public function testCreate(): void { $factory = new InternalResourceIndexerFactory( [], - $this->createStub(IndexerFilter::class), + $this->createStub(ResourceFilter::class), $this->createStub(LocationFinder::class), $this->createStub(ResourceLoader::class), $this->createStub(TranslationSplitter::class), diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index ad5a2ef..e55b899 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -7,12 +7,12 @@ use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\DocumentEnricher; -use Atoolo\Search\Service\Indexer\IndexerFilter; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Atoolo\Search\Service\Indexer\LocationFinder; +use Atoolo\Search\Service\Indexer\ResourceFilter; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\Indexer\SolrIndexUpdater; @@ -32,7 +32,7 @@ class InternalResourceIndexerTest extends TestCase */ private array $availableIndexes = ['test', 'test-en_US']; - public IndexerFilter&MockObject $indexerFilter; + public ResourceFilter&MockObject $indexerFilter; private ResourceLoader&Stub $resourceLoader; @@ -60,7 +60,7 @@ class InternalResourceIndexerTest extends TestCase public function setUp(): void { $this->indexerFilter = $this->createMock( - IndexerFilter::class + ResourceFilter::class ); $this->indexerProgressHandler = $this->createMock( From be69fb501ddc6e27a24fdf3b0e22434e9e30ca85 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 10:47:12 +0100 Subject: [PATCH 106/145] fix: service configuration --- config/commands.yml | 20 +++++++++++-------- src/Console/Command/MoreLikeThis.php | 4 ++++ src/Console/Command/Search.php | 6 ++++++ src/Console/Command/Suggest.php | 5 +++++ .../BackgroundIndexerProgressState.php | 14 +++++++++---- src/Service/Search/SolrMoreLikeThis.php | 2 +- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/config/commands.yml b/config/commands.yml index 48b5be2..f03a2ee 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -21,10 +21,14 @@ services: atoolo.search.indexer.progressBar: class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' + atoolo.search.indexer.resourceFilter: + class: Atoolo\Search\Service\Indexer\SiteKit\NoIndexFilter + atoolo.search.indexer.internalResourceIndexer: class: Atoolo\Search\Service\Indexer\InternalResourceIndexer arguments: - !tagged_iterator atoolo.search.indexer.documentEnricher.schema2x + - '@atoolo.search.indexer.resourceFilter' - '@atoolo.search.indexer.progressBar' - '@atoolo.search.indexer.locationFinder' - '@atoolo.resource.resourceLoader' @@ -33,21 +37,21 @@ services: - '@atoolo.search.indexer.aborter' - 'internal' - atoolo.search.selectSearcher: - class: Atoolo\Search\Service\Search\SolrSelect + atoolo.search.search: + class: Atoolo\Search\Service\Search\SolrSearch arguments: - '@atoolo.search.indexName' - '@atoolo.search.solariumClientFactory' - - !tagged_iterator atoolo.search.queryModifier - '@atoolo.search.resultToResourceResolver' + - !tagged_iterator atoolo.search.queryModifier - atoolo.search.suggestSearcher: + atoolo.search.suggest: class: Atoolo\Search\Service\Search\SolrSuggest arguments: - '@atoolo.search.indexName' - '@atoolo.search.solariumClientFactory' - atoolo.search.moreLikeThisSearcher: + atoolo.search.moreLikeThis: class: Atoolo\Search\Service\Search\SolrMoreLikeThis arguments: - '@atoolo.search.indexName' @@ -61,15 +65,15 @@ services: Atoolo\Search\Console\Command\Search: arguments: - - '@atoolo.search.selectSearcher' + - '@atoolo.search.search' Atoolo\Search\Console\Command\Suggest: arguments: - - '@atoolo.search.suggestSearcher' + - '@atoolo.search.suggest' Atoolo\Search\Console\Command\MoreLikeThis: arguments: - - '@atoolo.search.moreLikeThisSearcher' + - '@atoolo.search.moreLikeThis' Atoolo\Search\Console\Application: diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index d72bc3e..92f86e4 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -84,6 +84,10 @@ protected function buildQuery( protected function outputResult(SearchResult $result): void { + if ($result->total === 0) { + $this->io->text('No results found.'); + return; + } $this->io->text($result->total . " Results:"); foreach ($result as $resource) { $this->io->text($resource->getLocation()); diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index b0c47a5..c319469 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -84,6 +84,12 @@ protected function buildQuery(InputInterface $input): SearchQuery protected function outputResult( SearchResult $result ): void { + + if ($result->total === 0) { + $this->io->text('No results found'); + return; + } + $this->io->title('Results (' . $result->total . ')'); foreach ($result as $resource) { $this->io->text($resource->getLocation()); diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index 393e5c7..a36bf12 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -85,6 +85,11 @@ protected function buildQuery(string $terms, string $lang): SuggestQuery protected function outputResult(SuggestResult $result): void { + if (empty($result->suggestions)) { + $this->io->text('No suggestions found'); + return; + } + foreach ($result as $suggest) { $this->io->text( $suggest->term . diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/BackgroundIndexerProgressState.php index 9e9fa53..3a69c07 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/BackgroundIndexerProgressState.php @@ -6,6 +6,7 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Dto\Indexer\IndexerStatusState; +use Atoolo\Search\Service\IndexName; use DateTime; use JsonException; use Psr\Log\LoggerInterface; @@ -20,7 +21,7 @@ class BackgroundIndexerProgressState implements IndexerProgressHandler private bool $isUpdate = false; public function __construct( - private string $index, + private readonly IndexName $index, private readonly IndexerStatusStore $statusStore, private readonly LoggerInterface $logger = new NullLogger() ) { @@ -47,7 +48,7 @@ public function start(int $total): void public function startUpdate(int $total): void { $this->isUpdate = true; - $storedStatus = $this->statusStore->load($this->index); + $storedStatus = $this->statusStore->load($this->getIndex()); $this->status = new IndexerStatus( IndexerStatusState::RUNNING, $storedStatus->startTime, @@ -71,7 +72,7 @@ public function advance(int $step): void if ($this->isUpdate) { $this->status->updated += $step; } - $this->statusStore->store($this->index, $this->status); + $this->statusStore->store($this->getIndex(), $this->status); } @@ -102,7 +103,7 @@ public function finish(): void if ($this->status->state === IndexerStatusState::RUNNING) { $this->status->state = IndexerStatusState::FINISHED; } - $this->statusStore->store($this->index, $this->status); + $this->statusStore->store($this->getIndex(), $this->status); } public function abort(): void @@ -114,4 +115,9 @@ public function getStatus(): IndexerStatus { return $this->status; } + + private function getIndex(): string + { + return $this->index->name(''); + } } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 4d0d370..f098d2d 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -69,7 +69,7 @@ private function buildResult( ->loadResourceList($result, $lang); return new SearchResult( - $result->getNumFound() ?? -1, + $result->getNumFound() ?? 0, 0, 0, $resourceList, From 9f986eb9a83264d23d71d4f839d53ac2ac2242f3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 11:00:00 +0100 Subject: [PATCH 107/145] feat: outputs the channel for the cli commandos --- config/commands.yml | 4 ++++ src/Console/Command/DumpIndexDocument.php | 7 +++++++ src/Console/Command/Indexer.php | 9 +++++++-- src/Console/Command/MoreLikeThis.php | 5 +++++ src/Console/Command/Search.php | 9 +++++++-- src/Console/Command/Suggest.php | 5 +++++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/config/commands.yml b/config/commands.yml index f03a2ee..27715ef 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -60,19 +60,23 @@ services: Atoolo\Search\Console\Command\Indexer: arguments: + - '@atoolo.resource.resourceChannelFactory' - '@atoolo.search.indexer.progressBar' - '@atoolo.search.indexer.internalResourceIndexer' Atoolo\Search\Console\Command\Search: arguments: + - '@atoolo.resource.resourceChannelFactory' - '@atoolo.search.search' Atoolo\Search\Console\Command\Suggest: arguments: + - '@atoolo.resource.resourceChannelFactory' - '@atoolo.search.suggest' Atoolo\Search\Console\Command\MoreLikeThis: arguments: + - '@atoolo.resource.resourceChannelFactory' - '@atoolo.search.moreLikeThis' diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php index 76515e5..8ab9682 100644 --- a/src/Console/Command/DumpIndexDocument.php +++ b/src/Console/Command/DumpIndexDocument.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Service\Indexer\IndexDocumentDumper; use JsonException; @@ -12,6 +13,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( name: 'atoolo:dump-index-document', @@ -20,6 +22,7 @@ class DumpIndexDocument extends Command { public function __construct( + private readonly ResourceChannelFactory $channelFactory, private readonly IndexDocumentDumper $dumper ) { parent::__construct(); @@ -49,6 +52,10 @@ protected function execute( $paths = $typedInput->getArrayArgument('paths'); + $resourceChannel = $this->channelFactory->create(); + $io = new SymfonyStyle($input, $output); + $io->title('Channel: ' . $resourceChannel->name); + $dump = $this->dumper->dump($paths); foreach ($dump as $fields) { diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 783eb20..2ef14ba 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Indexer\IndexerParameter; @@ -25,6 +26,7 @@ class Indexer extends Command private OutputInterface $output; public function __construct( + private readonly ResourceChannelFactory $channelFactory, private readonly IndexerProgressBar $progressBar, private readonly InternalResourceIndexer $indexer, ) { @@ -78,10 +80,13 @@ protected function execute( ? $typedInput->getIntOption('cleanup-threshold') : 0; + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + if (empty($paths)) { - $this->io->title('Index all resources'); + $this->io->section('Index all resources'); } else { - $this->io->title('Index resource paths'); + $this->io->section('Index resource paths'); $this->io->listing($paths); } diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 92f86e4..72a17cc 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; @@ -25,6 +26,7 @@ class MoreLikeThis extends Command private TypifiedInput $input; public function __construct( + private readonly ResourceChannelFactory $channelFactory, private readonly SolrMoreLikeThis $searcher ) { parent::__construct(); @@ -61,6 +63,9 @@ protected function execute( $location = $this->input->getStringArgument('location'); $lang = $this->input->getStringOption('lang'); + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + $query = $this->buildQuery($location, $lang); $result = $this->searcher->moreLikeThis($query); $this->outputResult($result); diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index c319469..ed8f0ae 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\SearchQuery; use Atoolo\Search\Dto\Search\Query\SearchQueryBuilder; @@ -26,6 +27,7 @@ class Search extends Command private TypifiedInput $input; public function __construct( + private readonly ResourceChannelFactory $channelFactory, private readonly SolrSearch $searcher ) { parent::__construct(); @@ -58,6 +60,9 @@ protected function execute( $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + $query = $this->buildQuery($input); $result = $this->searcher->search($query); @@ -90,13 +95,13 @@ protected function outputResult( return; } - $this->io->title('Results (' . $result->total . ')'); + $this->io->section('Results (' . $result->total . ')'); foreach ($result as $resource) { $this->io->text($resource->getLocation()); } if (count($result->facetGroups) > 0) { - $this->io->title('Facets'); + $this->io->section('Facets'); foreach ($result->facetGroups as $facetGroup) { $this->io->section($facetGroup->key); $listing = []; diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index a36bf12..75b426e 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Console\Command; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\Filter\ArchiveFilter; use Atoolo\Search\Dto\Search\Query\Filter\ObjectTypeFilter; @@ -27,6 +28,7 @@ class Suggest extends Command private SymfonyStyle $io; public function __construct( + private readonly ResourceChannelFactory $channelFactory, private readonly SolrSuggest $search ) { parent::__construct(); @@ -60,6 +62,9 @@ protected function execute( $terms = $this->input->getStringArgument('terms'); $lang = $this->input->getStringOption('lang'); + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + $query = $this->buildQuery($terms, $lang); $result = $this->search->suggest($query); From ac00376c56f45a5d0a63ded2b4a25a0a5f337d8e Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 22 Mar 2024 11:29:55 +0100 Subject: [PATCH 108/145] test: fix tests --- src/Service/Indexer/BackgroundIndexer.php | 2 +- test/Console/ApplicationTest.php | 6 +++- .../Console/Command/DumpIndexDocumentTest.php | 26 ++++++++++++++ test/Console/Command/IndexerTest.php | 36 ++++++++++++++++--- test/Console/Command/MoreLikeThisTest.php | 26 +++++++++++++- test/Console/Command/SearchTest.php | 29 +++++++++++++-- test/Console/Command/SuggestTest.php | 26 +++++++++++++- .../BackgroundIndexerProgressStateTest.php | 5 ++- 8 files changed, 144 insertions(+), 12 deletions(-) diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 4c5b8c2..6a1337c 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -64,7 +64,7 @@ public function getStatus(): IndexerStatus private function getIndexer(): InternalResourceIndexer { $progressHandler = new BackgroundIndexerProgressState( - $this->index->name(''), + $this->index, $this->statusStore, $this->logger ); diff --git a/test/Console/ApplicationTest.php b/test/Console/ApplicationTest.php index 7ac7003..54d8558 100644 --- a/test/Console/ApplicationTest.php +++ b/test/Console/ApplicationTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Console; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; @@ -22,12 +23,15 @@ class ApplicationTest extends TestCase */ public function testConstruct(): void { + $resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); $indexer = $this->createStub( InternalResourceIndexer::class ); $progressBar = $this->createStub(IndexerProgressBar::class); $application = new Application([ - new Indexer($progressBar, $indexer) + new Indexer($resourceChannelFactory, $progressBar, $indexer) ]); $command = $application->get('atoolo:indexer'); $this->assertInstanceOf( diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php index 55ef55c..ad381ab 100644 --- a/test/Console/Command/DumpIndexDocumentTest.php +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -4,6 +4,8 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Resource\ResourceChannel; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\DumpIndexDocument; use Atoolo\Search\Console\Command\IndexDocumentDumperBuilder; @@ -23,6 +25,25 @@ class DumpIndexDocumentTest extends TestCase */ public function setUp(): void { + $resourceChannel = new ResourceChannel( + '', + 'WWW', + '', + '', + false, + '', + '', + '', + 'test', + [] + ); + + $resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $dumper = $this->createStub(IndexDocumentDumper::class); $dumper->method('dump') ->willReturn([ @@ -30,6 +51,7 @@ public function setUp(): void ]); $dumperCommand = new DumpIndexDocument( + $resourceChannelFactory, $dumper ); @@ -52,6 +74,10 @@ public function testExecute(): void $output = $this->commandTester->getDisplay(); $this->assertEquals( <<resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $this->resourceChannelFactory->method('create') + ->willReturn($resourceChannel); $indexer = $this->createStub( InternalResourceIndexer::class ); $progressBar = $this->createStub(IndexerProgressBar::class); $command = new Indexer( + $this->resourceChannelFactory, $progressBar, $indexer, ); @@ -51,8 +73,11 @@ public function testExecuteIndexAll(): void $this->assertEquals( <<assertEquals( <<createStub( InternalResourceIndexer::class ); - $progressBar = $this->createStub( IndexerProgressBar::class ); @@ -104,6 +131,7 @@ public function testExecuteIndexWithErrors(): void ->willReturn([new \Exception('errortest')]); $command = new Indexer( + $this->resourceChannelFactory, $progressBar, $indexer, ); @@ -136,7 +164,6 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void $indexer = $this->createStub( InternalResourceIndexer::class ); - $progressBar = $this->createStub( IndexerProgressBar::class ); @@ -145,6 +172,7 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void ->willReturn([new \Exception('errortest')]); $command = new Indexer( + $this->resourceChannelFactory, $progressBar, $indexer, ); diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index 05139b5..4c3faca 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -5,6 +5,8 @@ namespace Atoolo\Search\Test\Console\Command; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceChannel; +use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\MoreLikeThis; use Atoolo\Search\Dto\Search\Result\SearchResult; @@ -24,6 +26,24 @@ class MoreLikeThisTest extends TestCase */ public function setUp(): void { + $resourceChannel = new ResourceChannel( + '', + 'WWW', + '', + '', + false, + '', + '', + '', + 'test', + [] + ); + + $resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); $resultResource = $this->createStub(Resource::class); $resultResource->method('getLocation') ->willReturn('/test2.php'); @@ -39,7 +59,7 @@ public function setUp(): void $solrMoreLikeThis->method('moreLikeThis') ->willReturn($result); - $command = new MoreLikeThis($solrMoreLikeThis); + $command = new MoreLikeThis($resourceChannelFactory, $solrMoreLikeThis); $application = new Application([$command]); @@ -59,6 +79,10 @@ public function testExecute(): void $output = $this->commandTester->getDisplay(); $this->assertEquals( <<createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); $resultResource = $this->createStub(Resource::class); $resultResource->method('getLocation') ->willReturn('/test.php'); @@ -47,7 +67,7 @@ public function setUp(): void $solrSelect->method('search') ->willReturn($result); - $command = new Search($solrSelect); + $command = new Search($resourceChannelFactory, $solrSelect); $application = new Application([$command]); @@ -67,13 +87,16 @@ public function testExecute(): void $this->assertEquals( <<createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); $result = new SuggestResult( [ new Suggestion('security', 10), @@ -36,7 +56,7 @@ public function setUp(): void $solrSuggest->method('suggest') ->willReturn($result); - $command = new Suggest($solrSuggest); + $command = new Suggest($resourceChannelFactory, $solrSuggest); $application = new Application([$command]); @@ -56,6 +76,10 @@ public function testExecute(): void $output = $this->commandTester->getDisplay(); $this->assertEquals( <<createMock(IndexName::class); + $indexName->method('name')->willReturn('test'); $this->statusStore = $this->createMock(IndexerStatusStore::class); $this->state = new BackgroundIndexerProgressState( - 'test', + $indexName, $this->statusStore ); } From deb8d2399a8a3d975e0c4ac955a534f8fc9a69db Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 25 Mar 2024 07:26:14 +0100 Subject: [PATCH 109/145] refactor: path for dump-index-document is required --- config/commands.yml | 8 ++++---- src/Console/Command/DumpIndexDocument.php | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/commands.yml b/config/commands.yml index 27715ef..cebce08 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -14,10 +14,6 @@ services: - '@atoolo.resource.resourceLoader' - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } - Atoolo\Search\Console\Command\DumpIndexDocument: - arguments: - - '@Atoolo\Search\Service\Indexer\IndexDocumentDumper' - atoolo.search.indexer.progressBar: class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' @@ -79,6 +75,10 @@ services: - '@atoolo.resource.resourceChannelFactory' - '@atoolo.search.moreLikeThis' + Atoolo\Search\Console\Command\DumpIndexDocument: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@Atoolo\Search\Service\Indexer\IndexDocumentDumper' Atoolo\Search\Console\Application: public: true diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php index 8ab9682..85a4d5e 100644 --- a/src/Console/Command/DumpIndexDocument.php +++ b/src/Console/Command/DumpIndexDocument.php @@ -34,7 +34,7 @@ protected function configure(): void ->setHelp('Command to dump a index-document') ->addArgument( 'paths', - InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'Resources paths or directories of resources to be indexed.' ) ; From 9abecb7a6b3e0e56a3aa7ea398e7845136769a03 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 25 Mar 2024 07:27:51 +0100 Subject: [PATCH 110/145] docs: add param doc for MoreLikeThisQuery::fields --- src/Dto/Search/Query/MoreLikeThisQuery.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index f0d8213..1037f9d 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -20,7 +20,9 @@ class MoreLikeThisQuery /** * @param string $index name of the index to use * @param Filter[] $filter - * @param string[] $fields + * @param string[] $fields The fields specified here must be part of the + * index schema and determine which fields are relevant for determining + * which entries are similar. */ public function __construct( public readonly string $index, From 083230c4afdaa042906232c2b116ef4d594575ee Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 25 Mar 2024 07:35:58 +0100 Subject: [PATCH 111/145] refactor: reduce code --- src/Service/Indexer/SiteKit/QuoteSectionMatcher.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php index 5107991..c8e64b2 100644 --- a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php +++ b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php @@ -37,14 +37,8 @@ public function match(array $path, array $value): string|false /** @var Model $model */ $content = []; - $quote = $model['quote'] ?? ''; - if (is_string($quote)) { - $content[] = $quote; - } - $citation = $model['citation'] ?? ''; - if (is_string($citation)) { - $content[] = $citation; - } + $content[] = $model['quote'] ?? ''; + $content[] = $model['citation'] ?? ''; return implode(' ', $content); } From 6be96d6f86717164715b6a487cc155409819fdfe Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 28 Mar 2024 17:55:04 +0100 Subject: [PATCH 112/145] feat: IndexCollection --- config/commands.yml | 52 +----- src/Console/Command/DumpIndexDocument.php | 2 +- src/Console/Command/Indexer.php | 69 ++++---- src/Console/Command/Io/IndexerProgressBar.php | 63 ++++--- src/Console/Command/MoreLikeThis.php | 2 +- src/Console/Command/Search.php | 2 +- src/Console/Command/Suggest.php | 2 +- src/Dto/Indexer/IndexerConfiguration.php | 17 ++ src/Dto/Indexer/IndexerParameter.php | 3 +- src/Indexer.php | 16 +- src/Service/AbstractIndexer.php | 82 +++++++++ src/Service/Indexer/BackgroundIndexer.php | 61 +++---- src/Service/Indexer/IndexerCollection.php | 41 +++++ .../Indexer/IndexerConfigurationLoader.php | 101 +++++++++++ ...ressState.php => IndexerProgressState.php} | 40 ++--- src/Service/Indexer/IndexerStatusStore.php | 12 +- .../Indexer/InternalResourceIndexer.php | 162 ++++++++++++------ .../InternalResourceIndexerFactory.php | 41 ----- .../SiteKit/SubDirTranslationSplitter.php | 30 +++- .../BackgroundIndexerProgressStateTest.php | 8 +- .../InternalResourceIndexerFactoryTest.php | 37 ---- 21 files changed, 521 insertions(+), 322 deletions(-) create mode 100644 src/Dto/Indexer/IndexerConfiguration.php create mode 100644 src/Service/AbstractIndexer.php create mode 100644 src/Service/Indexer/IndexerCollection.php create mode 100644 src/Service/Indexer/IndexerConfigurationLoader.php rename src/Service/Indexer/{BackgroundIndexerProgressState.php => IndexerProgressState.php} (74%) delete mode 100644 src/Service/Indexer/InternalResourceIndexerFactory.php delete mode 100644 test/Service/Indexer/InternalResourceIndexerFactoryTest.php diff --git a/config/commands.yml b/config/commands.yml index cebce08..e555bbc 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -9,56 +9,14 @@ services: Atoolo\Search\Console\: resource: '../src/Console' - Atoolo\Search\Service\Indexer\IndexDocumentDumper: - arguments: - - '@atoolo.resource.resourceLoader' - - !tagged_iterator { tag: 'atoolo.search.indexer.documentEnricher.schema2x' } - - atoolo.search.indexer.progressBar: - class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' - - atoolo.search.indexer.resourceFilter: - class: Atoolo\Search\Service\Indexer\SiteKit\NoIndexFilter - - atoolo.search.indexer.internalResourceIndexer: - class: Atoolo\Search\Service\Indexer\InternalResourceIndexer - arguments: - - !tagged_iterator atoolo.search.indexer.documentEnricher.schema2x - - '@atoolo.search.indexer.resourceFilter' - - '@atoolo.search.indexer.progressBar' - - '@atoolo.search.indexer.locationFinder' - - '@atoolo.resource.resourceLoader' - - '@atoolo.search.indexer.translationSplitter' - - '@atoolo.search.indexer.solrIndexService' - - '@atoolo.search.indexer.aborter' - - 'internal' - - atoolo.search.search: - class: Atoolo\Search\Service\Search\SolrSearch - arguments: - - '@atoolo.search.indexName' - - '@atoolo.search.solariumClientFactory' - - '@atoolo.search.resultToResourceResolver' - - !tagged_iterator atoolo.search.queryModifier - - atoolo.search.suggest: - class: Atoolo\Search\Service\Search\SolrSuggest - arguments: - - '@atoolo.search.indexName' - - '@atoolo.search.solariumClientFactory' - - atoolo.search.moreLikeThis: - class: Atoolo\Search\Service\Search\SolrMoreLikeThis - arguments: - - '@atoolo.search.indexName' - - '@atoolo.search.solariumClientFactory' - - '@atoolo.search.resultToResourceResolver' + atoolo.search.indexer.console.progressBar: + class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' Atoolo\Search\Console\Command\Indexer: arguments: - '@atoolo.resource.resourceChannelFactory' - - '@atoolo.search.indexer.progressBar' - - '@atoolo.search.indexer.internalResourceIndexer' + - '@atoolo.search.indexer.console.progressBar' + - '@atoolo.search.indexer.indexerCollection' Atoolo\Search\Console\Command\Search: arguments: @@ -78,7 +36,7 @@ services: Atoolo\Search\Console\Command\DumpIndexDocument: arguments: - '@atoolo.resource.resourceChannelFactory' - - '@Atoolo\Search\Service\Indexer\IndexDocumentDumper' + - '@atoolo.search.indexer.indexDocumentDumper' Atoolo\Search\Console\Application: public: true diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php index 85a4d5e..4355329 100644 --- a/src/Console/Command/DumpIndexDocument.php +++ b/src/Console/Command/DumpIndexDocument.php @@ -16,7 +16,7 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'atoolo:dump-index-document', + name: 'search:dump-index-document', description: 'Dump a index document' )] class DumpIndexDocument extends Command diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 2ef14ba..0fe8fd2 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -7,8 +7,7 @@ use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Console\Command\Io\TypifiedInput; -use Atoolo\Search\Dto\Indexer\IndexerParameter; -use Atoolo\Search\Service\Indexer\InternalResourceIndexer; +use Atoolo\Search\Service\Indexer\IndexerCollection; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -17,7 +16,7 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'atoolo:indexer', + name: 'search:indexer', description: 'Fill a search index' )] class Indexer extends Command @@ -28,7 +27,7 @@ class Indexer extends Command public function __construct( private readonly ResourceChannelFactory $channelFactory, private readonly IndexerProgressBar $progressBar, - private readonly InternalResourceIndexer $indexer, + private readonly IndexerCollection $indexers, ) { parent::__construct(); } @@ -42,26 +41,6 @@ protected function configure(): void InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'Resources paths or directories of resources to be indexed.' ) - ->addOption( - 'cleanup-threshold', - null, - InputArgument::OPTIONAL, - 'Specifies the number of documents required to be indexed ' . - 'successfully for the entire process to be considered ' . - 'successfull. Old entries will only ever be removed if this ' . - 'threshold is reached. Only relevant for full-indexing.', - 0 - ) - ->addOption( - 'chunk-size', - null, - InputArgument::OPTIONAL, - 'The chunk size determines how many documents are ' . - 'indexed in an update request. The default value is 500. ' . - 'Higher values no longer have a positive effect. Smaller ' . - 'values can be selected if the memory limit is reached.', - 500 - ) ; } @@ -76,35 +55,51 @@ protected function execute( $paths = $typedInput->getArrayArgument('paths'); - $cleanupThreshold = empty($paths) - ? $typedInput->getIntOption('cleanup-threshold') - : 0; - $resourceChannel = $this->channelFactory->create(); $this->io->title('Channel: ' . $resourceChannel->name); + /* if (empty($paths)) { $this->io->section('Index all resources'); } else { $this->io->section('Index resource paths'); $this->io->listing($paths); } + */ + + foreach ($this->indexers->getIndexers() as $indexer) { + if ($indexer->enabled()) { + $this->io->newLine(); + $this->io->section( + 'Index with Indexer "' . $indexer->getName() . '"' + ); + $progressHandler = $indexer->getProgressHandler(); + $this->progressBar->init($progressHandler); + $indexer->setProgressHandler($this->progressBar); + try { + $status = $indexer->index(); + } finally { + $indexer->setProgressHandler($progressHandler); + } + $this->io->newLine(2); + $this->io->section("Status"); + $this->io->text($status->getStatusLine()); + $this->io->newLine(); + $this->errorReport(); + } + } - $parameter = new IndexerParameter( - $cleanupThreshold, - $typedInput->getIntOption('chunk-size'), - $paths - ); - - $this->indexer->index($parameter); - - $this->errorReport(); return Command::SUCCESS; } protected function errorReport(): void { + if (empty($this->progressBar->getErrors())) { + return; + } + $this->io->section("Error Report"); + foreach ($this->progressBar->getErrors() as $error) { if ($this->io->isVerbose() && $this->getApplication() !== null) { $this->getApplication()->renderThrowable($error, $this->output); diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index e4ce283..e2b366e 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -5,61 +5,67 @@ namespace Atoolo\Search\Console\Command\Io; use Atoolo\Search\Dto\Indexer\IndexerStatus; -use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; -use DateTime; +use JsonException; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Serializer\Exception\ExceptionInterface; use Throwable; class IndexerProgressBar implements IndexerProgressHandler { - private OutputInterface $output; - private ProgressBar $progressBar; - private IndexerStatus $status; + private ?ProgressBar $progressBar; + + private IndexerProgressHandler $currentProgressHandler; /** * @var array */ private array $errors = []; - public function __construct(OutputInterface $output = new ConsoleOutput()) - { - $this->output = $output; + public function __construct( + private readonly OutputInterface $output = new ConsoleOutput() + ) { + } + + public function init( + IndexerProgressHandler $progressHandler + ): void { + $this->currentProgressHandler = $progressHandler; + $this->errors = []; + $this->progressBar = null; } public function start(int $total): void { + $this->currentProgressHandler->start($total); $this->progressBar = new ProgressBar($this->output, $total); $this->formatProgressBar('green'); - $this->status = new IndexerStatus( - IndexerStatusState::RUNNING, - new DateTime(), - null, - $total, - 0, - 0, - new DateTime(), - 0, - 0 - ); } + /** + * @throws ExceptionInterface + */ public function startUpdate(int $total): void { - $this->start($total); + $this->currentProgressHandler->startUpdate($total); + $this->progressBar = new ProgressBar($this->output, $total); + $this->formatProgressBar('green'); } + /** + * @throws JsonException + */ public function advance(int $step): void { + $this->currentProgressHandler->advance($step); $this->progressBar->advance($step); - $this->status->processed += $step; } public function skip(int $step): void { - $this->status->skipped++; + $this->currentProgressHandler->skip($step); } private function formatProgressBar(string $color): void @@ -75,16 +81,18 @@ private function formatProgressBar(string $color): void public function error(Throwable $throwable): void { + $this->currentProgressHandler->error($throwable); $this->formatProgressBar('red'); $this->errors[] = $throwable; - $this->status->errors++; } + /** + * @throws JsonException + */ public function finish(): void { + $this->currentProgressHandler->finish(); $this->progressBar->finish(); - $this->status->state = IndexerStatusState::FINISHED; - $this->status->endTime = new DateTime(); } /** @@ -97,13 +105,12 @@ public function getErrors(): array public function getStatus(): IndexerStatus { - return $this->status; + return $this->currentProgressHandler->getStatus(); } public function abort(): void { + $this->currentProgressHandler->abort(); $this->progressBar->finish(); - $this->status->state = IndexerStatusState::ABORTED; - $this->status->endTime = new DateTime(); } } diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 72a17cc..e16f0c4 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -17,7 +17,7 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'atoolo:mlt', + name: 'search:mlt', description: 'Performs a more-like-this search' )] class MoreLikeThis extends Command diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index ed8f0ae..93782f7 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -18,7 +18,7 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'atoolo:search', + name: 'search:search', description: 'Performs a search' )] class Search extends Command diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php index 75b426e..4d81c34 100644 --- a/src/Console/Command/Suggest.php +++ b/src/Console/Command/Suggest.php @@ -19,7 +19,7 @@ use Symfony\Component\Console\Style\SymfonyStyle; #[AsCommand( - name: 'atoolo:suggest', + name: 'search:suggest', description: 'Performs a suggest search' )] class Suggest extends Command diff --git a/src/Dto/Indexer/IndexerConfiguration.php b/src/Dto/Indexer/IndexerConfiguration.php new file mode 100644 index 0000000..6bdf453 --- /dev/null +++ b/src/Dto/Indexer/IndexerConfiguration.php @@ -0,0 +1,17 @@ +chunkSize < 10) { throw new \InvalidArgumentException( - 'chunk Size must be greater than 9' + 'chunk size must be greater than 9' ); } } diff --git a/src/Indexer.php b/src/Indexer.php index 16c2448..f18fbd7 100644 --- a/src/Indexer.php +++ b/src/Indexer.php @@ -4,8 +4,8 @@ namespace Atoolo\Search; -use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Atoolo\Search\Service\Indexer\IndexerProgressHandler; /** * The service interface for indexing a search index. @@ -18,10 +18,22 @@ */ interface Indexer { - public function index(IndexerParameter $parameter): IndexerStatus; + public function getName(): string; + + public function getSource(): string; + + public function getProgressHandler(): IndexerProgressHandler; + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void; + + public function index(): IndexerStatus; public function abort(): void; + public function enabled(): bool; + /** * @param string[] $idList */ diff --git a/src/Service/AbstractIndexer.php b/src/Service/AbstractIndexer.php new file mode 100644 index 0000000..097f5ae --- /dev/null +++ b/src/Service/AbstractIndexer.php @@ -0,0 +1,82 @@ +indexName->name('') . '-' . $this->source; + } + + protected function getConfig(): IndexerConfiguration + { + return $this->config ??= $this->configLoader->load($this->source); + } + + public function getName(): string + { + return $this->getConfig()->name; + } + + public function getSource(): string + { + return $this->source; + } + + public function getProgressHandler(): IndexerProgressHandler + { + return $this->progressHandler; + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + $this->progressHandler = $progressHandler; + } + + public function abort(): void + { + $this->aborter->requestAbortion($this->getKey()); + } + + protected function isAbortionRequested(): bool + { + return $this->aborter->isAbortionRequested($this->getKey()); + } + + public function enabled(): bool + { + return $this->configLoader->exists($this->source); + } + + /** + * @inheritDoc + */ + abstract public function index(): IndexerStatus; + + /** + * @inheritDoc + */ + abstract public function remove(array $idList): void; +} diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 6a1337c..997090d 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -4,53 +4,59 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\IndexName; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Symfony\Component\Lock\LockFactory; -use Symfony\Component\Lock\Store\SemaphoreStore; use Symfony\Component\Serializer\Exception\ExceptionInterface; class BackgroundIndexer implements Indexer { public function __construct( - private readonly InternalResourceIndexerFactory $indexerFactory, + private readonly InternalResourceIndexer $indexer, private readonly IndexName $index, private readonly IndexerStatusStore $statusStore, - private readonly LoggerInterface $logger = new NullLogger(), - private readonly LockFactory $lockFactory = new LockFactory( - new SemaphoreStore() - ) ) { } + public function enabled(): bool + { + return true; + } + public function getSource(): string + { + return $this->indexer->getSource(); + } + + public function getProgressHandler(): IndexerProgressHandler + { + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + } + + public function getName(): string + { + return "Background Indexer"; + } + /** * @param string[] $idList */ public function remove(array $idList): void { - $this->getIndexer()->remove($idList); + $this->indexer->remove($idList); } public function abort(): void { - $this->getIndexer()->abort(); + $this->indexer->abort(); } - public function index(IndexerParameter $parameter): IndexerStatus + public function index(): IndexerStatus { - $lock = $this->lockFactory->createLock($this->getIndex()); - if (!$lock->acquire()) { - return IndexerStatus::empty(); - } - try { - return $this->getIndexer()->index($parameter); - } finally { - $lock->release(); - } + return $this->indexer->index(); } /** @@ -61,17 +67,6 @@ public function getStatus(): IndexerStatus return $this->statusStore->load($this->getIndex()); } - private function getIndexer(): InternalResourceIndexer - { - $progressHandler = new BackgroundIndexerProgressState( - $this->index, - $this->statusStore, - $this->logger - ); - - return $this->indexerFactory->create($progressHandler); - } - private function getIndex(): string { /* diff --git a/src/Service/Indexer/IndexerCollection.php b/src/Service/Indexer/IndexerCollection.php new file mode 100644 index 0000000..6a9514f --- /dev/null +++ b/src/Service/Indexer/IndexerCollection.php @@ -0,0 +1,41 @@ + $indexers + */ + public function __construct( + private readonly iterable $indexers, + ) { + } + + public function getIndexer(string $source): Indexer + { + foreach ($this->indexers as $indexer) { + if ($indexer->getSource() === $source) { + return $indexer; + } + } + throw new InvalidArgumentException( + 'Indexer not found for source: ' . $source + ); + } + + /** + * @return array + */ + public function getIndexers(): array + { + return $this->indexers instanceof \Traversable + ? iterator_to_array($this->indexers) + : $this->indexers; + } +} diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php new file mode 100644 index 0000000..6811529 --- /dev/null +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -0,0 +1,101 @@ + + */ + public function loadAll(): array + { + $dir = $this->resourceBaseLocator->locate() . '/indexer'; + if (!is_dir($dir)) { + return []; + } + + $files = glob($dir . '/*.php'); + if ($files === false) { + return []; + } + + $configurations = []; + foreach ($files as $file) { + $configurations[] = $this->load($file); + } + return $configurations; + } + + private function getFile(string $source): string + { + return $this->resourceBaseLocator->locate() . + '/indexer/' . $source . '.php'; + } + + public function exists(string $source): bool + { + $file = $this->getFile($source); + return file_exists($file); + } + + public function load(string $source): IndexerConfiguration + { + $file = $this->getFile($source); + + if (!file_exists($file)) { + return new IndexerConfiguration( + $source, + $source, + new DataBag([]), + ); + } + + $saveErrorReporting = error_reporting(); + + try { + error_reporting(E_ERROR | E_PARSE); + ob_start(); + $data = require $file; + if (!is_array($data)) { + throw new RuntimeException( + 'The indexer configuration ' . + $file . ' should return an array' + ); + } + + if (isset($data['source']) === false) { + throw new RuntimeException( + 'The indexer configuration ' . + $file . ' should have a source key' + ); + } + + if ($data['source'] !== $source) { + throw new RuntimeException( + 'source key in ' . $file . ' should match the filename' + ); + } + + return new IndexerConfiguration( + $data['source'], + $data['name'] ?? $data['source'], + new DataBag($data['data'] ?? []), + ); + } finally { + ob_end_clean(); + error_reporting($saveErrorReporting); + } + } +} diff --git a/src/Service/Indexer/BackgroundIndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php similarity index 74% rename from src/Service/Indexer/BackgroundIndexerProgressState.php rename to src/Service/Indexer/IndexerProgressState.php index 3a69c07..c18db26 100644 --- a/src/Service/Indexer/BackgroundIndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -8,22 +8,19 @@ use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\IndexName; use DateTime; -use JsonException; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; use Symfony\Component\Serializer\Exception\ExceptionInterface; use Throwable; -class BackgroundIndexerProgressState implements IndexerProgressHandler +class IndexerProgressState implements IndexerProgressHandler { - private IndexerStatus $status; + private ?IndexerStatus $status = null; private bool $isUpdate = false; public function __construct( private readonly IndexName $index, private readonly IndexerStatusStore $statusStore, - private readonly LoggerInterface $logger = new NullLogger() + private readonly string $source, ) { } @@ -48,7 +45,7 @@ public function start(int $total): void public function startUpdate(int $total): void { $this->isUpdate = true; - $storedStatus = $this->statusStore->load($this->getIndex()); + $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); $this->status = new IndexerStatus( IndexerStatusState::RUNNING, $storedStatus->startTime, @@ -62,9 +59,6 @@ public function startUpdate(int $total): void ); } - /** - * @throws JsonException - */ public function advance(int $step): void { $this->status->processed += $step; @@ -72,10 +66,12 @@ public function advance(int $step): void if ($this->isUpdate) { $this->status->updated += $step; } - $this->statusStore->store($this->getIndex(), $this->status); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); } - public function skip(int $step): void { $this->status->skipped += $step; @@ -84,17 +80,8 @@ public function skip(int $step): void public function error(Throwable $throwable): void { $this->status->errors++; - $this->logger->error( - $throwable->getMessage(), - [ - 'exception' => $throwable, - ] - ); } - /** - * @throws JsonException - */ public function finish(): void { if (!$this->isUpdate) { @@ -103,7 +90,7 @@ public function finish(): void if ($this->status->state === IndexerStatusState::RUNNING) { $this->status->state = IndexerStatusState::FINISHED; } - $this->statusStore->store($this->getIndex(), $this->status); + $this->statusStore->store($this->getStatusStoreKey(), $this->status); } public function abort(): void @@ -113,11 +100,14 @@ public function abort(): void public function getStatus(): IndexerStatus { - return $this->status; + if ($this->status !== null) { + return $this->status; + } + return $this->statusStore->load($this->getStatusStoreKey()); } - private function getIndex(): string + private function getStatusStoreKey(): string { - return $this->index->name(''); + return $this->index->name('') . '-' . $this->source; } } diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index 3cb4cfd..52ba416 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -33,9 +33,9 @@ public function __construct(private readonly string $basedir) /** * @throws ExceptionInterface */ - public function load(string $index): IndexerStatus + public function load(string $key): IndexerStatus { - $file = $this->getStatusFile($index); + $file = $this->getStatusFile($key); if (!file_exists($file)) { return IndexerStatus::empty(); @@ -65,11 +65,11 @@ public function load(string $index): IndexerStatus return $status; } - public function store(string $index, IndexerStatus $status): void + public function store(string $key, IndexerStatus $status): void { $this->createBaseDirectory(); - $file = $this->getStatusFile($index); + $file = $this->getStatusFile($key); if (file_exists($file) && !is_writable($file)) { throw new RuntimeException( 'File ' . $file . ' is not writable' @@ -117,9 +117,9 @@ private function createBaseDirectory(): void } } - private function getStatusFile(string $index): string + private function getStatusFile(string $key): string { return $this->basedir . - '/atoolo.search.index.' . $index . ".status.json"; + '/atoolo.search.index.' . $key . ".status.json"; } } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 1ecc9b3..5f3eed9 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -10,7 +10,11 @@ use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Exception; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use Solarium\QueryType\Update\Result as UpdateResult; +use Symfony\Component\Lock\LockFactory; +use Symfony\Component\Lock\Store\SemaphoreStore; use Throwable; /** @@ -36,22 +40,55 @@ */ class InternalResourceIndexer implements Indexer { + private IndexerParameter $parameter; + /** * @param iterable> $documentEnricherList */ public function __construct( private readonly iterable $documentEnricherList, private readonly ResourceFilter $resourceFilter, - private readonly IndexerProgressHandler $indexerProgressHandler, + private IndexerProgressHandler $progressHandler, private readonly LocationFinder $finder, private readonly ResourceLoader $resourceLoader, private readonly TranslationSplitter $translationSplitter, private readonly SolrIndexService $indexService, private readonly IndexingAborter $aborter, - private readonly string $source + private readonly IndexerConfigurationLoader $configLoader, + private readonly string $source, + private readonly LockFactory $lockFactory = new LockFactory( + new SemaphoreStore() + ), + private readonly LoggerInterface $logger = new NullLogger() ) { } + public function enabled(): bool + { + return true; + } + + public function getName(): string + { + return $this->getIndexerParameter()->name; + } + + public function getProgressHandler(): IndexerProgressHandler + { + return $this->progressHandler; + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + $this->progressHandler = $progressHandler; + } + + public function getSource(): string + { + return $this->source; + } + /** * @param string[] $idList */ @@ -77,51 +114,56 @@ public function abort(): void * Indexes an entire directory structure or only selected files * if `paths` was specified in `$parameter`. */ - public function index(IndexerParameter $parameter): IndexerStatus + public function index(): IndexerStatus { - if (empty($parameter->paths)) { - $pathList = $this->finder->findAll(); - } else { - $mappedPaths = $this->normalizePaths($parameter->paths); - $pathList = $this->finder->findPaths($mappedPaths); + $lock = $this->lockFactory->createLock('indexer.' . $this->source); + if (!$lock->acquire()) { + return $this->progressHandler->getStatus(); } + try { + $param = $this->getIndexerParameter(); + return $this->indexResources($param, $this->finder->findAll()); + } finally { + $lock->release(); + gc_collect_cycles(); + } + } - return $this->indexResources($parameter, $pathList); + /** + * @param string[] $paths + */ + public function update(array $paths): IndexerStatus + { + $param = $this->loadIndexerParameter(); + return $this->indexResources( + $param, + $this->finder->findPaths($paths) + ); } - private function getBaseIndex(): string + private function getIndexerParameter(): IndexerParameter { - return $this->indexService->getIndex(''); + return $this->parameter ??= ($this->loadIndexerParameter()); } - /** - * A path can signal to be translated into another language via - * the URL parameter loc. For example, - * `/dir/file.php?loc=it_IT` defines that the path - * `/dir/file.php.translations/it_IT.php` is to be used. - * This method translates the URL parameter into the correct path. - * - * @param string[] $pathList - * @return string[] - */ - private function normalizePaths(array $pathList): array + private function loadIndexerParameter(): IndexerParameter { - return array_map(static function ($path) { - $queryString = parse_url($path, PHP_URL_QUERY); - if (!is_string($queryString)) { - return $path; - } - $urlPath = parse_url($path, PHP_URL_PATH); - if (!is_string($urlPath)) { - return $path; - } - parse_str($queryString, $params); - if (!isset($params['loc']) || !is_string($params['loc'])) { - return $urlPath; - } - $loc = $params['loc']; - return $urlPath . '.translations/' . $loc . ".php"; - }, $pathList); + $config = $this->configLoader->load($this->source); + return new IndexerParameter( + $config->name, + $config->data->getInt( + 'cleanupThreshold' + ), + $config->data->getInt( + 'chunkSize', + 500 + ) + ); + } + + private function getBaseIndex(): string + { + return $this->indexService->getIndex(''); } /** @@ -140,9 +182,9 @@ private function indexResources( $total = count($pathList); if (empty($parameter->paths)) { $this->deleteErrorProtocol($this->getBaseIndex()); - $this->indexerProgressHandler->start($total); + $this->progressHandler->start($total); } else { - $this->indexerProgressHandler->startUpdate($total); + $this->progressHandler->startUpdate($total); } $splitterResult = $this->translationSplitter->split($pathList); @@ -152,9 +194,9 @@ private function indexResources( $splitterResult ); - $this->indexerProgressHandler->finish(); + $this->progressHandler->finish(); - return $this->indexerProgressHandler->getStatus(); + return $this->progressHandler->getStatus(); } /** @@ -186,10 +228,10 @@ private function indexTranslationSplittedResources( $langIndex = $this->indexService->getIndex($lang); if ($index === $langIndex) { - $this->indexerProgressHandler->error(new Exception( + $this->handleError( 'No Index for language "' . $lang . '" ' . 'found (base index: "' . $index . '")' - )); + ); continue; } @@ -225,9 +267,7 @@ private function indexResourcesPerLanguageIndex( $managedIndices = $this->indexService->getManagedIndices(); if (!in_array($index, $managedIndices)) { - $this->indexerProgressHandler->error(new Exception( - 'Index "' . $index . '" not found' - )); + $this->handleError('Index "' . $index . '" not found'); return; } @@ -288,19 +328,17 @@ private function indexChunks( } if ($this->aborter->isAbortionRequested($index)) { $this->aborter->resetAbortionRequest($index); - $this->indexerProgressHandler->abort(); + $this->progressHandler->abort(); return false; } if (empty($resourceList)) { return 0; } - $this->indexerProgressHandler->advance(count($resourceList)); + $this->progressHandler->advance(count($resourceList)); $result = $this->add($lang, $processId, $resourceList); if ($result->getStatus() !== 0) { - $this->indexerProgressHandler->error(new Exception( - $result->getResponse()->getStatusMessage() - )); + $this->handleError($result->getResponse()->getStatusMessage()); return 0; } @@ -331,7 +369,7 @@ private function loadResources( $resource = $this->resourceLoader->load($path); $resourceList[] = $resource; } catch (Throwable $e) { - $this->indexerProgressHandler->error($e); + $this->handleError($e); } } return $resourceList; @@ -350,7 +388,7 @@ private function add( foreach ($resources as $resource) { if ($this->resourceFilter->accept($resource) === false) { - $this->indexerProgressHandler->skip(1); + $this->progressHandler->skip(1); continue; } try { @@ -366,7 +404,7 @@ private function add( } $updater->addDocument($doc); } catch (Throwable $e) { - $this->indexerProgressHandler->error($e); + $this->handleError($e); } } @@ -374,6 +412,20 @@ private function add( return $updater->update(); } + private function handleError(Throwable|string $error): void + { + if (is_string($error)) { + $error = new Exception($error); + } + $this->progressHandler->error($error); + $this->logger->error( + $error->getMessage(), + [ + 'exception' => $error, + ] + ); + } + private function deleteErrorProtocol(string $core): void { $this->indexService->deleteByQuery( diff --git a/src/Service/Indexer/InternalResourceIndexerFactory.php b/src/Service/Indexer/InternalResourceIndexerFactory.php deleted file mode 100644 index 3c1c996..0000000 --- a/src/Service/Indexer/InternalResourceIndexerFactory.php +++ /dev/null @@ -1,41 +0,0 @@ -> $documentEnricherList - */ - public function __construct( - private readonly iterable $documentEnricherList, - private readonly ResourceFilter $indexerFilter, - private readonly LocationFinder $finder, - private readonly ResourceLoader $resourceLoader, - private readonly TranslationSplitter $translationSplitter, - private readonly SolrIndexService $solrService, - private readonly IndexingAborter $aborter, - private readonly string $source - ) { - } - - public function create( - IndexerProgressHandler $progressHandler - ): InternalResourceIndexer { - return new InternalResourceIndexer( - $this->documentEnricherList, - $this->indexerFilter, - $progressHandler, - $this->finder, - $this->resourceLoader, - $this->translationSplitter, - $this->solrService, - $this->aborter, - $this->source - ); - } -} diff --git a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php index 1dbceef..e01e86d 100644 --- a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php +++ b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php @@ -38,8 +38,9 @@ public function split(array $pathList): TranslationSplitterResult private function extractLocaleFromPath(string $path): string { - $filename = basename($path); - $parentDirName = basename(dirname($path)); + $normalizedPath = $this->normalizePath($path); + $filename = basename($normalizedPath); + $parentDirName = basename(dirname($normalizedPath)); if (!str_ends_with($parentDirName, '.php.translations')) { return 'default'; @@ -47,4 +48,29 @@ private function extractLocaleFromPath(string $path): string return basename($filename, '.php'); } + + /** + * A path can signal to be translated into another language via + * the URL parameter loc. For example, + * `/dir/file.php?loc=it_IT` defines that the path + * `/dir/file.php.translations/it_IT.php` is to be used. + * This method translates the URL parameter into the correct path. + */ + private function normalizePath(string $path): string + { + $queryString = parse_url($path, PHP_URL_QUERY); + if (!is_string($queryString)) { + return $path; + } + $urlPath = parse_url($path, PHP_URL_PATH); + if (!is_string($urlPath)) { + return $path; + } + parse_str($queryString, $params); + if (!isset($params['loc']) || !is_string($params['loc'])) { + return $urlPath; + } + $loc = $params['loc']; + return $urlPath . '.translations/' . $loc . ".php"; + } } diff --git a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php index b783905..4462b84 100644 --- a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php +++ b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php @@ -5,7 +5,7 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Search\Dto\Indexer\IndexerStatus; -use Atoolo\Search\Service\Indexer\BackgroundIndexerProgressState; +use Atoolo\Search\Service\Indexer\IndexerProgressState; use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\IndexName; use Exception; @@ -13,18 +13,18 @@ use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; -#[CoversClass(BackgroundIndexerProgressState::class)] +#[CoversClass(IndexerProgressState::class)] class BackgroundIndexerProgressStateTest extends TestCase { private IndexerStatusStore&MockObject $statusStore; - private BackgroundIndexerProgressState $state; + private IndexerProgressState $state; public function setUp(): void { $indexName = $this->createMock(IndexName::class); $indexName->method('name')->willReturn('test'); $this->statusStore = $this->createMock(IndexerStatusStore::class); - $this->state = new BackgroundIndexerProgressState( + $this->state = new IndexerProgressState( $indexName, $this->statusStore ); diff --git a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php b/test/Service/Indexer/InternalResourceIndexerFactoryTest.php deleted file mode 100644 index cb4e170..0000000 --- a/test/Service/Indexer/InternalResourceIndexerFactoryTest.php +++ /dev/null @@ -1,37 +0,0 @@ -createStub(ResourceFilter::class), - $this->createStub(LocationFinder::class), - $this->createStub(ResourceLoader::class), - $this->createStub(TranslationSplitter::class), - $this->createStub(SolrIndexService::class), - $this->createStub(IndexingAborter::class), - '' - ); - - $this->expectNotToPerformAssertions(); - $factory->create( - $this->createStub(IndexerProgressHandler::class) - ); - } -} From 6d736d881f3e2ab8136d89eded85e660c26950b5 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 07:47:38 +0200 Subject: [PATCH 113/145] fix: phpstan errors --- src/Service/Indexer/BackgroundIndexer.php | 1 + src/Service/Indexer/IndexerProgressState.php | 26 ++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 997090d..8ab883a 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -29,6 +29,7 @@ public function getSource(): string public function getProgressHandler(): IndexerProgressHandler { + return $this->indexer->getProgressHandler(); } public function setProgressHandler( diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index c18db26..7fec343 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -8,6 +8,7 @@ use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\IndexName; use DateTime; +use LogicException; use Symfony\Component\Serializer\Exception\ExceptionInterface; use Throwable; @@ -61,6 +62,11 @@ public function startUpdate(int $total): void public function advance(int $step): void { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } $this->status->processed += $step; $this->status->lastUpdate = new DateTime(); if ($this->isUpdate) { @@ -74,16 +80,31 @@ public function advance(int $step): void public function skip(int $step): void { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } $this->status->skipped += $step; } public function error(Throwable $throwable): void { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } $this->status->errors++; } public function finish(): void { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } if (!$this->isUpdate) { $this->status->endTime = new DateTime(); } @@ -95,6 +116,11 @@ public function finish(): void public function abort(): void { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } $this->status->state = IndexerStatusState::ABORTED; } From 5b74bac45e9b796604e156d59a5550c141cc4f92 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 07:48:04 +0200 Subject: [PATCH 114/145] refactor: use named arguments --- src/Console/Command/Io/IndexerProgressBar.php | 14 +++++++------- src/Console/Command/MoreLikeThis.php | 10 +++++----- src/Dto/Search/Query/SearchQueryBuilder.php | 16 ++++++++-------- src/Service/Search/ExternalResourceFactory.php | 12 ++++++------ src/Service/Search/SolrMoreLikeThis.php | 12 ++++++------ src/Service/Search/SolrSearch.php | 12 ++++++------ 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index e2b366e..3041dc0 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -60,7 +60,7 @@ public function startUpdate(int $total): void public function advance(int $step): void { $this->currentProgressHandler->advance($step); - $this->progressBar->advance($step); + $this->progressBar?->advance($step); } public function skip(int $step): void @@ -70,10 +70,10 @@ public function skip(int $step): void private function formatProgressBar(string $color): void { - $this->progressBar->setBarCharacter('•'); - $this->progressBar->setEmptyBarCharacter('⚬'); - $this->progressBar->setProgressCharacter('➤'); - $this->progressBar->setFormat( + $this->progressBar?->setBarCharacter('•'); + $this->progressBar?->setEmptyBarCharacter('⚬'); + $this->progressBar?->setProgressCharacter('➤'); + $this->progressBar?->setFormat( "%current%/%max% [%bar%] %percent:3s%%\n" . " %estimated:-20s% %memory:20s%" ); @@ -92,7 +92,7 @@ public function error(Throwable $throwable): void public function finish(): void { $this->currentProgressHandler->finish(); - $this->progressBar->finish(); + $this->progressBar?->finish(); } /** @@ -111,6 +111,6 @@ public function getStatus(): IndexerStatus public function abort(): void { $this->currentProgressHandler->abort(); - $this->progressBar->finish(); + $this->progressBar?->finish(); } } diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index e16f0c4..3afbeeb 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -79,11 +79,11 @@ protected function buildQuery( ): MoreLikeThisQuery { $filterList = []; return new MoreLikeThisQuery( - $location, - $lang, - $filterList, - 5, - ['content'] + location: $location, + lang: $lang, + filter: $filterList, + limit: 5, + fields: ['content'] ); } diff --git a/src/Dto/Search/Query/SearchQueryBuilder.php b/src/Dto/Search/Query/SearchQueryBuilder.php index ea7b465..fd6b6c5 100644 --- a/src/Dto/Search/Query/SearchQueryBuilder.php +++ b/src/Dto/Search/Query/SearchQueryBuilder.php @@ -135,14 +135,14 @@ public function defaultQueryOperator( public function build(): SearchQuery { return new SearchQuery( - $this->text, - $this->lang, - $this->offset, - $this->limit, - $this->sort, - array_values($this->filter), - array_values($this->facets), - $this->defaultQueryOperator + text: $this->text, + lang: $this->lang, + offset: $this->offset, + limit: $this->limit, + sort: $this->sort, + filter: array_values($this->filter), + facets: array_values($this->facets), + defaultQueryOperator: $this->defaultQueryOperator ); } } diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index ef82403..0b6450d 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -37,12 +37,12 @@ public function create(Document $document, string $lang): Resource } return new Resource( - $location, - $this->getField($document, 'sp_id') ?? '', - $this->getField($document, 'title') ?? '', - 'external', - $this->getField($document, 'meta_content_language') ?? '', - [], + location: $location, + id: $this->getField($document, 'sp_id') ?? '', + name: $this->getField($document, 'title') ?? '', + objectType: 'external', + lang: $this->getField($document, 'meta_content_language') ?? '', + data: [], ); } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index f098d2d..7b48f2f 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -69,12 +69,12 @@ private function buildResult( ->loadResourceList($result, $lang); return new SearchResult( - $result->getNumFound() ?? 0, - 0, - 0, - $resourceList, - [], - $result->getQueryTime() ?? -1 + total: $result->getNumFound() ?? 0, + limit: 0, + offset: 0, + results: $resourceList, + facetGroups: [], + queryTime: $result->getQueryTime() ?? -1 ); } } diff --git a/src/Service/Search/SolrSearch.php b/src/Service/Search/SolrSearch.php index 01954f1..7b2d13d 100644 --- a/src/Service/Search/SolrSearch.php +++ b/src/Service/Search/SolrSearch.php @@ -255,12 +255,12 @@ private function buildResult( $facetGroupList = $this->buildFacetGroupList($query, $result); return new SearchResult( - $result->getNumFound() ?? 0, - $query->limit, - $query->offset, - $resourceList, - $facetGroupList, - $result->getQueryTime() ?? 0 + total:$result->getNumFound() ?? 0, + limit: $query->limit, + offset: $query->offset, + results: $resourceList, + facetGroups: $facetGroupList, + queryTime: $result->getQueryTime() ?? 0 ); } From 080645c17831848aa9e9f87fb58ab6d2d03ee322 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 09:20:46 +0200 Subject: [PATCH 115/145] test: tests adjusted after refactoring --- config/commands.yml | 6 + src/Console/Command/Indexer.php | 10 - .../Command/IndexerInternalResourceUpdate.php | 99 +++++++++ test/Console/ApplicationTest.php | 6 +- .../Console/Command/DumpIndexDocumentTest.php | 3 +- .../IndexerInternalResourceUpdateTest.php | 195 ++++++++++++++++++ test/Console/Command/IndexerTest.php | 60 +++--- .../Command/Io/IndexerProgressBarTest.php | 153 ++++++-------- test/Console/Command/MoreLikeThisTest.php | 2 +- test/Console/Command/SearchTest.php | 2 +- test/Console/Command/SuggestTest.php | 2 +- test/Dto/Indexer/IndexerParameterTest.php | 1 + .../BackgroundIndexerProgressStateTest.php | 3 +- .../Service/Indexer/BackgroundIndexerTest.php | 19 +- .../Indexer/InternalResourceIndexerTest.php | 122 ++++------- 15 files changed, 441 insertions(+), 242 deletions(-) create mode 100644 src/Console/Command/IndexerInternalResourceUpdate.php create mode 100644 test/Console/Command/IndexerInternalResourceUpdateTest.php diff --git a/config/commands.yml b/config/commands.yml index e555bbc..faa017c 100644 --- a/config/commands.yml +++ b/config/commands.yml @@ -18,6 +18,12 @@ services: - '@atoolo.search.indexer.console.progressBar' - '@atoolo.search.indexer.indexerCollection' + Atoolo\Search\Console\Command\IndexerInternalResourceUpdate: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.indexer.console.progressBar' + - '@atoolo.search.indexer.internalResourceIndexer' + Atoolo\Search\Console\Command\Search: arguments: - '@atoolo.resource.resourceChannelFactory' diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 0fe8fd2..4cc1d82 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -58,15 +58,6 @@ protected function execute( $resourceChannel = $this->channelFactory->create(); $this->io->title('Channel: ' . $resourceChannel->name); - /* - if (empty($paths)) { - $this->io->section('Index all resources'); - } else { - $this->io->section('Index resource paths'); - $this->io->listing($paths); - } - */ - foreach ($this->indexers->getIndexers() as $indexer) { if ($indexer->enabled()) { $this->io->newLine(); @@ -89,7 +80,6 @@ protected function execute( } } - return Command::SUCCESS; } diff --git a/src/Console/Command/IndexerInternalResourceUpdate.php b/src/Console/Command/IndexerInternalResourceUpdate.php new file mode 100644 index 0000000..c710fdc --- /dev/null +++ b/src/Console/Command/IndexerInternalResourceUpdate.php @@ -0,0 +1,99 @@ +setHelp('Command to update internal resources in search index') + ->addArgument( + 'paths', + InputArgument::REQUIRED | InputArgument::IS_ARRAY, + 'Resources paths or directories of resources to be updated.' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $typedInput = new TypifiedInput($input); + $this->output = $output; + $this->io = new SymfonyStyle($input, $output); + + $paths = $typedInput->getArrayArgument('paths'); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + $this->io->section( + 'Index resource paths with Indexer "' . + $this->indexer->getName() . '"' + ); + $this->io->listing($paths); + $progressHandler = $this->indexer->getProgressHandler(); + $this->progressBar->init($progressHandler); + $this->indexer->setProgressHandler($this->progressBar); + try { + $status = $this->indexer->update($paths); + } finally { + $this->indexer->setProgressHandler($progressHandler); + } + $this->io->newLine(2); + $this->io->section("Status"); + $this->io->text($status->getStatusLine()); + $this->io->newLine(); + $this->errorReport(); + + + return Command::SUCCESS; + } + + protected function errorReport(): void + { + if (empty($this->progressBar->getErrors())) { + return; + } + $this->io->section("Error Report"); + + foreach ($this->progressBar->getErrors() as $error) { + if ($this->io->isVerbose() && $this->getApplication() !== null) { + $this->getApplication()->renderThrowable($error, $this->output); + } else { + $this->io->error($error->getMessage()); + } + } + } +} diff --git a/test/Console/ApplicationTest.php b/test/Console/ApplicationTest.php index 54d8558..b543926 100644 --- a/test/Console/ApplicationTest.php +++ b/test/Console/ApplicationTest.php @@ -10,6 +10,7 @@ use Atoolo\Search\Console\Command\InternalResourceIndexerBuilder; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Console\Command\Io\IndexerProgressBarFactory; +use Atoolo\Search\Service\Indexer\IndexerCollection; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; @@ -29,11 +30,12 @@ public function testConstruct(): void $indexer = $this->createStub( InternalResourceIndexer::class ); + $indexers = new IndexerCollection([$indexer]); $progressBar = $this->createStub(IndexerProgressBar::class); $application = new Application([ - new Indexer($resourceChannelFactory, $progressBar, $indexer) + new Indexer($resourceChannelFactory, $progressBar, $indexers) ]); - $command = $application->get('atoolo:indexer'); + $command = $application->get('search:indexer'); $this->assertInstanceOf( Indexer::class, $command, diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php index ad381ab..63d54fc 100644 --- a/test/Console/Command/DumpIndexDocumentTest.php +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -8,7 +8,6 @@ use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\DumpIndexDocument; -use Atoolo\Search\Console\Command\IndexDocumentDumperBuilder; use Atoolo\Search\Service\Indexer\IndexDocumentDumper; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; @@ -59,7 +58,7 @@ public function setUp(): void $dumperCommand ]); - $command = $application->find('atoolo:dump-index-document'); + $command = $application->find('search:dump-index-document'); $this->commandTester = new CommandTester($command); } diff --git a/test/Console/Command/IndexerInternalResourceUpdateTest.php b/test/Console/Command/IndexerInternalResourceUpdateTest.php new file mode 100644 index 0000000..9da80cc --- /dev/null +++ b/test/Console/Command/IndexerInternalResourceUpdateTest.php @@ -0,0 +1,195 @@ +resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $this->resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $indexer = $this->createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub(IndexerProgressBar::class); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $this->commandTester = new CommandTester($command); + } + + public function testExecuteIndexPath(): void + { + $this->commandTester->execute([ + // pass arguments to the helper + 'paths' => ['a.php', 'b.php'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $commandTester = new CommandTester($command); + + $commandTester->execute([ 'paths' => ['a.php']]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'errortest', + $output, + 'error message expected' + ); + } + + + /** + * @throws Exception + */ + public function testExecuteIndexWithErrorsAndStackTrace(): void + { + + $indexer = $this->createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $commandTester = new CommandTester($command); + + $commandTester->execute( + [ + 'paths' => ['a.php'] + ], + [ + 'verbosity' => OutputInterface::VERBOSITY_VERBOSE + ] + ); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'Exception trace', + $output, + 'error message should contains stack trace' + ); + } +} diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 533a362..43c7441 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -9,7 +9,7 @@ use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Indexer; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; -use Atoolo\Search\Service\Indexer\InternalResourceIndexer; +use Atoolo\Search\Service\Indexer\IndexerCollection; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\TestCase; @@ -46,19 +46,22 @@ public function setUp(): void $this->resourceChannelFactory->method('create') ->willReturn($resourceChannel); $indexer = $this->createStub( - InternalResourceIndexer::class + \Atoolo\Search\Indexer::class ); + $indexer->method('enabled') + ->willReturn(true); + $indexers = new IndexerCollection([$indexer]); $progressBar = $this->createStub(IndexerProgressBar::class); $command = new Indexer( $this->resourceChannelFactory, $progressBar, - $indexer, + $indexers, ); $application = new Application([$command]); - $command = $application->find('atoolo:indexer'); + $command = $application->find('search:indexer'); $this->commandTester = new CommandTester($command); } @@ -76,37 +79,16 @@ public function testExecuteIndexAll(): void Channel: WWW ============ -Index all resources -------------------- +Index with Indexer "" +--------------------- -EOF, - $output - ); - } - - public function testExecuteIndexPath(): void - { - $this->commandTester->execute([ - // pass arguments to the helper - 'paths' => ['a.php', 'b.php'] - ]); - - $this->commandTester->assertCommandIsSuccessful(); - - // the output of the command in the console - $output = $this->commandTester->getDisplay(); - $this->assertEquals( - <<createStub( - InternalResourceIndexer::class + \Atoolo\Search\Indexer::class ); + $indexer->method('enabled') + ->willReturn(true); + $indexers = new IndexerCollection([$indexer]); $progressBar = $this->createStub( IndexerProgressBar::class ); @@ -133,12 +118,12 @@ public function testExecuteIndexWithErrors(): void $command = new Indexer( $this->resourceChannelFactory, $progressBar, - $indexer, + $indexers, ); $application = new Application([$command]); - $command = $application->find('atoolo:indexer'); + $command = $application->find('search:indexer'); $commandTester = new CommandTester($command); $commandTester->execute([]); @@ -162,8 +147,11 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void { $indexer = $this->createStub( - InternalResourceIndexer::class + \Atoolo\Search\Indexer::class ); + $indexer->method('enabled') + ->willReturn(true); + $indexers = new IndexerCollection([$indexer]); $progressBar = $this->createStub( IndexerProgressBar::class ); @@ -174,12 +162,12 @@ public function testExecuteIndexWithErrorsAndStackTrace(): void $command = new Indexer( $this->resourceChannelFactory, $progressBar, - $indexer, + $indexers, ); $application = new Application([$command]); - $command = $application->find('atoolo:indexer'); + $command = $application->find('search:indexer'); $commandTester = new CommandTester($command); $commandTester->execute([], [ diff --git a/test/Console/Command/Io/IndexerProgressBarTest.php b/test/Console/Command/Io/IndexerProgressBarTest.php index a8c3426..83d11c7 100644 --- a/test/Console/Command/Io/IndexerProgressBarTest.php +++ b/test/Console/Command/Io/IndexerProgressBarTest.php @@ -5,8 +5,10 @@ namespace Atoolo\Search\Test\Console\Command\Io; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; +use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Exception; use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Output\OutputInterface; @@ -16,31 +18,36 @@ #[CoversClass(IndexerProgressBar::class)] class IndexerProgressBarTest extends TestCase { + private IndexerProgressBar $progressBar; + + private IndexerProgressHandler&MockObject $progressHandler; + + private OutputInterface&MockObject $output; + + public function setUp(): void + { + $this->output = $this->createMock(OutputInterface::class); + $this->progressHandler = $this->createMock(IndexerProgressHandler::class); + $this->progressBar = new IndexerProgressBar($this->output); + $this->progressBar->init($this->progressHandler); + } + public function testStart(): void { - $output = $this->createStub(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); - - $this->assertMatchesRegularExpression( - '/\[RUNNING].*processed: 0\/10,/', - $progressBar->getStatus()->getStatusLine(), - "unexpected status line" - ); + $this->progressHandler + ->expects($this->once()) + ->method('start') + ->with(10); + $this->progressBar->start(10); } public function testStartUpdate(): void { - $output = $this->createStub(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - - $progressBar->startUpdate(10); - - $this->assertMatchesRegularExpression( - '/\[RUNNING].*processed: 0\/10,/', - $progressBar->getStatus()->getStatusLine(), - "unexpected status line" - ); + $this->progressHandler + ->expects($this->once()) + ->method('startUpdate') + ->with(10); + $this->progressBar->startUpdate(10); } /** @@ -48,108 +55,76 @@ public function testStartUpdate(): void */ public function testAdvance(): void { - $output = $this->createMock(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); - $output->expects($this->once()) - ->method('write') - ->with($this->stringStartsWith(<<progressBar->start(10); - 1/10 [••➤⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬] 10% - < 1 sec -END - )); + $this->progressHandler + ->expects($this->once()) + ->method('advance') + ->with(1); - $progressBar->advance(1); + $this->progressHandler->advance(1); } public function testSkip(): void { - $output = $this->createStub(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - - $progressBar->start(10); + $this->progressBar->start(10); - $progressBar->skip(10); + $this->progressHandler + ->expects($this->once()) + ->method('skip') + ->with(10); - $this->assertMatchesRegularExpression( - '/\[RUNNING].*skipped: 1,/', - $progressBar->getStatus()->getStatusLine(), - "unexpected status line" - ); + $this->progressBar->skip(10); } public function testError(): void { - $output = $this->createMock(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); - - - //phpcs:ignore Generic.Files.LineLength.TooLong - $output->expects($this->once()) - ->method('write') - ->with($this->stringStartsWith(<<➤⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬⚬] 0% - < 1 sec -END - )); - - $progressBar->error(new Exception('test')); - $progressBar->advance(0); + + $this->progressBar->start(10); + + $e = new Exception('test'); + + $this->progressHandler + ->expects($this->once()) + ->method('error') + ->with($e); + + $this->progressBar->error($e); } public function testGetError(): void { - $output = $this->createMock(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); + $this->progressBar->start(10); - $progressBar->error(new Exception('test')); + $e = new Exception('test'); + $this->progressBar->error($e); $this->assertCount( 1, - $progressBar->getErrors(), + $this->progressBar->getErrors(), 'unexpected error count' ); } public function testFinish(): void { - $output = $this->createMock(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); - - $output->expects($this->once()) - ->method('write') - ->with($this->stringStartsWith(<<progressBar->start(10); -10/10 [••••••••••••••••••••••••••••] 100% - < 1 sec -END - )); - - $progressBar->finish(); + $this->progressHandler + ->expects($this->once()) + ->method('finish'); + $this->progressBar->finish(); } public function testAbort(): void { - $output = $this->createMock(OutputInterface::class); - $progressBar = new IndexerProgressBar($output); - $progressBar->start(10); - - - $output->expects($this->once()) - ->method('write') - ->with($this->stringStartsWith(<<progressBar->start(10); -10/10 [••••••••••••••••••••••••••••] 100% - < 1 sec -END - )); - - $progressBar->abort(); - } + $this->progressHandler + ->expects($this->once()) + ->method('abort'); + $this->progressBar->abort(); + } } diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index 4c3faca..c20b0b1 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -63,7 +63,7 @@ public function setUp(): void $application = new Application([$command]); - $command = $application->find('atoolo:mlt'); + $command = $application->find('search:mlt'); $this->commandTester = new CommandTester($command); } diff --git a/test/Console/Command/SearchTest.php b/test/Console/Command/SearchTest.php index 6763f87..4f0fbc8 100644 --- a/test/Console/Command/SearchTest.php +++ b/test/Console/Command/SearchTest.php @@ -71,7 +71,7 @@ public function setUp(): void $application = new Application([$command]); - $command = $application->find('atoolo:search'); + $command = $application->find('search:search'); $this->commandTester = new CommandTester($command); } diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index 3f8bef3..bd734ae 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -60,7 +60,7 @@ public function setUp(): void $application = new Application([$command]); - $command = $application->find('atoolo:suggest'); + $command = $application->find('search:suggest'); $this->commandTester = new CommandTester($command); } diff --git a/test/Dto/Indexer/IndexerParameterTest.php b/test/Dto/Indexer/IndexerParameterTest.php index 5894985..eb5f61e 100644 --- a/test/Dto/Indexer/IndexerParameterTest.php +++ b/test/Dto/Indexer/IndexerParameterTest.php @@ -16,6 +16,7 @@ public function testToLowerChunkSize(): void { $this->expectException(InvalidArgumentException::class); new IndexerParameter( + '', 0, 9 ); diff --git a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php index 4462b84..72b3123 100644 --- a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php +++ b/test/Service/Indexer/BackgroundIndexerProgressStateTest.php @@ -26,7 +26,8 @@ public function setUp(): void $this->statusStore = $this->createMock(IndexerStatusStore::class); $this->state = new IndexerProgressState( $indexName, - $this->statusStore + $this->statusStore, + 'source' ); } diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index b49b2f0..c44db16 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -8,12 +8,10 @@ use Atoolo\Search\Service\Indexer\BackgroundIndexer; use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; -use Atoolo\Search\Service\Indexer\InternalResourceIndexerFactory; use Atoolo\Search\Service\IndexName; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; use Symfony\Component\Lock\LockFactory; use Symfony\Component\Lock\SharedLockInterface; @@ -30,14 +28,9 @@ public function setUp(): void $this->indexName = $this->createMock(IndexName::class); $this->solrIndexer = $this->createMock(InternalResourceIndexer::class); - $solrIndexerFactory = $this->createStub( - InternalResourceIndexerFactory::class - ); - $solrIndexerFactory->method('create') - ->willReturn($this->solrIndexer); $this->statusStore = $this->createMock(IndexerStatusStore::class); $this->indexer = new BackgroundIndexer( - $solrIndexerFactory, + $this->solrIndexer, $this->indexName, $this->statusStore ); @@ -59,7 +52,7 @@ public function testAbort(): void public function testIndex(): void { - $params = new IndexerParameter(); + $params = new IndexerParameter(''); $this->solrIndexer->expects($this->once()) ->method('index'); $this->indexer->index($params); @@ -69,11 +62,9 @@ public function testIndexIfLocked(): void { $lockFactory = $this->createStub(LockFactory::class); $indexer = new BackgroundIndexer( - $this->createStub(InternalResourceIndexerFactory::class), + $this->createStub(InternalResourceIndexer::class), $this->createStub(IndexName::class), - $this->statusStore, - new NullLogger(), - $lockFactory + $this->statusStore ); $lock = $this->createStub(SharedLockInterface::class); @@ -85,7 +76,7 @@ public function testIndexIfLocked(): void $this->solrIndexer->expects($this->exactly(0)) ->method('index'); - $params = new IndexerParameter(); + $params = new IndexerParameter(''); $indexer->index($params); } diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index e55b899..053726e 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -2,11 +2,14 @@ namespace Atoolo\Search\Test\Service\Indexer; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; +use Atoolo\Search\Dto\Indexer\IndexerConfiguration; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\DocumentEnricher; +use Atoolo\Search\Service\Indexer\IndexerConfigurationLoader; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexingAborter; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; @@ -54,6 +57,10 @@ class InternalResourceIndexerTest extends TestCase private TranslationSplitter $translationSplitter; + private IndexerConfigurationLoader&MockObject $indexerConfigurationLoader; + + private IndexerConfiguration $indexerConfiguration; + /** * @throws Exception */ @@ -103,6 +110,19 @@ public function setUp(): void $this->solrIndexService->method('updater') ->willReturn($this->updater); $this->aborter = $this->createMock(IndexingAborter::class); + $this->indexerConfiguration = new IndexerConfiguration( + '', + '', + new DataBag([ + 'cleanupThreshold' => 10, + 'chunkSize' => 10 + ]) + ); + $this->indexerConfigurationLoader = $this->createMock( + IndexerConfigurationLoader::class + ); + $this->indexerConfigurationLoader->method('load') + ->willReturn($this->indexerConfiguration); $this->indexer = new InternalResourceIndexer( [ $this->documentEnricher ], @@ -113,6 +133,7 @@ public function setUp(): void $this->translationSplitter, $this->solrIndexService, $this->aborter, + $this->indexerConfigurationLoader, 'test-source' ); } @@ -182,15 +203,10 @@ public function testIndexAllWithChunks(): void $this->updater->expects($this->exactly(3)) ->method('update'); - $parameter = new IndexerParameter( - 10, - 10 - ); - $this->indexerProgressHandler->expects($this->exactly(2)) ->method('error'); - $this->indexer->index($parameter); + $this->indexer->index(); } public function testIndexSkipResource(): void @@ -216,12 +232,7 @@ public function testIndexSkipResource(): void $this->updater->expects($this->exactly(1)) ->method('update'); - $parameter = new IndexerParameter( - 10, - 10 - ); - - $this->indexer->index($parameter); + $this->indexer->index(); } public function testAborted(): void @@ -241,12 +252,7 @@ public function testAborted(): void $this->indexerProgressHandler->expects($this->once()) ->method('abort'); - $parameter = new IndexerParameter( - 10, - 10 - ); - - $this->indexer->index($parameter); + $this->indexer->index(); } public function testWithUnsuccessfulStatus(): void @@ -263,12 +269,7 @@ public function testWithUnsuccessfulStatus(): void $this->indexerProgressHandler->expects($this->once()) ->method('error'); - $parameter = new IndexerParameter( - 10, - 10 - ); - - $this->indexer->index($parameter); + $this->indexer->index(); } public function testWithInvalidResource(): void @@ -285,6 +286,7 @@ public function testWithInvalidResource(): void ->method('error'); $parameter = new IndexerParameter( + '', 10, 10 ); @@ -297,16 +299,11 @@ public function testEmptyStatus(): void $this->finder->method('findAll') ->willReturn([]); - $parameter = new IndexerParameter( - 10, - 10 - ); - - $status = $this->indexer->index($parameter); + $status = $this->indexer->index(); $this->assertEquals(0, $status->total, 'total should be 0'); } - public function testIndexPaths(): void + public function testUpdate(): void { $this->finder->method('findPaths') ->willReturn([ @@ -326,67 +323,21 @@ public function testIndexPaths(): void $this->updater->expects($this->exactly(1)) ->method('update'); - $parameter = new IndexerParameter( - 10, - 10, - [ - '/a/b.php', - '/a/c.php' - ] - ); - - $this->indexer->index($parameter); + $this->indexer->update([ + '/a/b.php', + '/a/c.php' + ]); } - public function testIndexPathWithParameter(): void + public function testUpdateWithParameter(): void { $this->finder->expects($this->once()) ->method('findPaths') ->with($this->equalTo(['?a=b'])); - $parameter = new IndexerParameter( - 10, - 10, - [ - '?a=b' - ] - ); - - $this->indexer->index($parameter); - } - - public function testIndexPathWithParameterAndPath(): void - { - $this->finder->expects($this->once()) - ->method('findPaths') - ->with($this->equalTo(['/test.php'])); - - $parameter = new IndexerParameter( - 10, - 10, - [ - '/test.php?a=b' - ] - ); - - $this->indexer->index($parameter); - } - - public function testIndexPathWithLocParameterAndPath(): void - { - $this->finder->expects($this->once()) - ->method('findPaths') - ->with($this->equalTo(['/test.php.translations/en.php'])); - - $parameter = new IndexerParameter( - 10, - 10, - [ - '/test.php?loc=en' - ] - ); - - $this->indexer->index($parameter); + $this->indexer->update([ + '?a=b' + ]); } public function testWithoutAvailableIndexes(): void @@ -409,6 +360,7 @@ public function testWithoutAvailableIndexes(): void ]); $parameter = new IndexerParameter( + '', 10, 10 ); From c3fff86332c52628fb045e3fb278565638a217c8 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 09:25:52 +0200 Subject: [PATCH 116/145] feat: add source option for indexer cli --- src/Console/Command/Indexer.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php index 4cc1d82..7c55c62 100644 --- a/src/Console/Command/Indexer.php +++ b/src/Console/Command/Indexer.php @@ -41,6 +41,13 @@ protected function configure(): void InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'Resources paths or directories of resources to be indexed.' ) + ->addOption( + 'source', + null, + InputArgument::OPTIONAL, + 'Uses only the indexer of a specific source', + '' + ) ; } @@ -53,12 +60,15 @@ protected function execute( $this->output = $output; $this->io = new SymfonyStyle($input, $output); - $paths = $typedInput->getArrayArgument('paths'); + $source = $typedInput->getStringOption('source'); $resourceChannel = $this->channelFactory->create(); $this->io->title('Channel: ' . $resourceChannel->name); foreach ($this->indexers->getIndexers() as $indexer) { + if (!empty($source) && $indexer->getSource() !== $source) { + continue; + } if ($indexer->enabled()) { $this->io->newLine(); $this->io->section( From 9653151b65a4bae35ffd6df2d45b184362f2af14 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 13:43:34 +0200 Subject: [PATCH 117/145] test: add more tests --- src/Dto/Indexer/IndexerConfiguration.php | 3 + src/Dto/Indexer/IndexerParameter.php | 6 +- src/Service/Indexer/BackgroundIndexer.php | 1 + .../Indexer/IndexerConfigurationLoader.php | 25 +--- .../Indexer/InternalResourceIndexer.php | 48 ++++--- .../DefaultSchema2xDocumentEnricher.php | 9 +- .../SiteKit/SubDirTranslationSplitter.php | 17 ++- .../IndexerInternalResourceUpdateTest.php | 3 +- test/Console/Command/IndexerTest.php | 62 ++++++++- .../Command/Io/IndexerProgressBarTest.php | 37 ++++- test/Console/Command/MoreLikeThisTest.php | 60 +++++++- test/Console/Command/SearchTest.php | 64 +++++++-- test/Console/Command/SuggestTest.php | 50 +++++-- test/Service/AbstractIndexerTest.php | 121 ++++++++++++++++ .../Service/Indexer/BackgroundIndexerTest.php | 72 ++++++++-- .../Service/Indexer/IndexerCollectionTest.php | 52 +++++++ .../IndexerConfigurationLoaderTest.php | 130 ++++++++++++++++++ ...eTest.php => IndexerProgressStateTest.php} | 64 +++++++-- .../Indexer/InternalResourceIndexerTest.php | 130 ++++++++++++++---- .../Indexer/SiteKit/NoIndexFilterTest.php | 50 +++++++ .../SiteKit/SubDirTranslationSplitterTest.php | 48 +++++++ test/Service/TextIndexer.php | 34 +++++ .../not-a-directory/indexer | 1 + .../return-string/indexer/return-string.php | 3 + .../with-internal/indexer/internal.php | 9 ++ .../without-source/indexer/without-source.php | 5 + 26 files changed, 952 insertions(+), 152 deletions(-) create mode 100644 test/Service/AbstractIndexerTest.php create mode 100644 test/Service/Indexer/IndexerCollectionTest.php create mode 100644 test/Service/Indexer/IndexerConfigurationLoaderTest.php rename test/Service/Indexer/{BackgroundIndexerProgressStateTest.php => IndexerProgressStateTest.php} (71%) create mode 100644 test/Service/Indexer/SiteKit/NoIndexFilterTest.php create mode 100644 test/Service/TextIndexer.php create mode 100644 test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer create mode 100644 test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php create mode 100644 test/resources/Service/Indexer/IndexerConfigurationLoader/with-internal/indexer/internal.php create mode 100644 test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php diff --git a/src/Dto/Indexer/IndexerConfiguration.php b/src/Dto/Indexer/IndexerConfiguration.php index 6bdf453..d03df38 100644 --- a/src/Dto/Indexer/IndexerConfiguration.php +++ b/src/Dto/Indexer/IndexerConfiguration.php @@ -6,6 +6,9 @@ use Atoolo\Resource\DataBag; +/** + * @codeCoverageIgnore + */ class IndexerConfiguration { public function __construct( diff --git a/src/Dto/Indexer/IndexerParameter.php b/src/Dto/Indexer/IndexerParameter.php index 50f4c73..2bceb82 100644 --- a/src/Dto/Indexer/IndexerParameter.php +++ b/src/Dto/Indexer/IndexerParameter.php @@ -6,14 +6,10 @@ class IndexerParameter { - /** - * @param string[] $paths - */ public function __construct( public readonly string $name, public readonly int $cleanupThreshold = 0, - public readonly int $chunkSize = 500, - public readonly array $paths = [] + public readonly int $chunkSize = 500 ) { if ($this->chunkSize < 10) { throw new \InvalidArgumentException( diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 8ab883a..1b72147 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -35,6 +35,7 @@ public function getProgressHandler(): IndexerProgressHandler public function setProgressHandler( IndexerProgressHandler $progressHandler ): void { + $this->indexer->setProgressHandler($progressHandler); } public function getName(): string diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php index 6811529..12af65d 100644 --- a/src/Service/Indexer/IndexerConfigurationLoader.php +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -26,14 +26,12 @@ public function loadAll(): array return []; } - $files = glob($dir . '/*.php'); - if ($files === false) { - return []; - } + $files = glob($dir . '/*.php') ?: []; $configurations = []; foreach ($files as $file) { - $configurations[] = $this->load($file); + $source = pathinfo($file, PATHINFO_FILENAME); + $configurations[] = $this->load($source); } return $configurations; } @@ -75,22 +73,9 @@ public function load(string $source): IndexerConfiguration ); } - if (isset($data['source']) === false) { - throw new RuntimeException( - 'The indexer configuration ' . - $file . ' should have a source key' - ); - } - - if ($data['source'] !== $source) { - throw new RuntimeException( - 'source key in ' . $file . ' should match the filename' - ); - } - return new IndexerConfiguration( - $data['source'], - $data['name'] ?? $data['source'], + $source, + $data['name'] ?? $source, new DataBag($data['data'] ?? []), ); } finally { diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 5f3eed9..a242e97 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -118,15 +118,26 @@ public function index(): IndexerStatus { $lock = $this->lockFactory->createLock('indexer.' . $this->source); if (!$lock->acquire()) { + $this->logger->notice( + 'Indexer with source "' . $this->source . '" is already running' + ); return $this->progressHandler->getStatus(); } try { + $paths = $this->finder->findAll(); + $this->deleteErrorProtocol($this->getBaseIndex()); + $total = count($paths); + $this->progressHandler->start($total); + $param = $this->getIndexerParameter(); - return $this->indexResources($param, $this->finder->findAll()); + $this->indexResources($param, $this->finder->findAll()); } finally { $lock->release(); + $this->progressHandler->finish(); gc_collect_cycles(); } + + return $this->progressHandler->getStatus(); } /** @@ -134,11 +145,19 @@ public function index(): IndexerStatus */ public function update(array $paths): IndexerStatus { - $param = $this->loadIndexerParameter(); - return $this->indexResources( - $param, - $this->finder->findPaths($paths) - ); + $collectedPaths = $this->finder->findPaths($paths); + $total = count($collectedPaths); + $this->progressHandler->startUpdate($total); + + try { + $param = $this->loadIndexerParameter(); + $this->indexResources($param, $collectedPaths); + } finally { + $this->progressHandler->finish(); + gc_collect_cycles(); + } + + return $this->progressHandler->getStatus(); } private function getIndexerParameter(): IndexerParameter @@ -174,29 +193,16 @@ private function getBaseIndex(): string private function indexResources( IndexerParameter $parameter, array $pathList - ): IndexerStatus { + ): void { if (count($pathList) === 0) { - return IndexerStatus::empty(); - } - - $total = count($pathList); - if (empty($parameter->paths)) { - $this->deleteErrorProtocol($this->getBaseIndex()); - $this->progressHandler->start($total); - } else { - $this->progressHandler->startUpdate($total); + return; } $splitterResult = $this->translationSplitter->split($pathList); - $this->indexTranslationSplittedResources( $parameter, $splitterResult ); - - $this->progressHandler->finish(); - - return $this->progressHandler->getStatus(); } /** diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 0f6f775..0d130de 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -327,11 +327,10 @@ private function contactPointToContent(array $contactPoint): string } if (isset($contactPoint['addressData'])) { - $addressData = $contactPoint['addressData']; - $content[] = $addressData['street'] ?? ''; - $content[] = $addressData['buildingName'] ?? ''; - $content[] = $addressData['postOfficeBoxData']['buildingName'] - ?? ''; + $data = $contactPoint['addressData']; + $content[] = $data['street'] ?? ''; + $content[] = $data['buildingName'] ?? ''; + $content[] = $data['postOfficeBoxData']['buildingName'] ?? ''; } return implode(' ', $content); diff --git a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php index e01e86d..ca3b6d9 100644 --- a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php +++ b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php @@ -22,15 +22,19 @@ public function split(array $pathList): TranslationSplitterResult $bases = []; $translations = []; foreach ($pathList as $path) { - $locale = $this->extractLocaleFromPath($path); + $normalizedPath = $this->normalizePath($path); + if (empty($normalizedPath)) { + continue; + } + $locale = $this->extractLocaleFromPath($normalizedPath); if ($locale === 'default') { - $bases[] = $path; + $bases[] = $normalizedPath; continue; } if (!isset($translations[$locale])) { $translations[$locale] = []; } - $translations[$locale][] = $path; + $translations[$locale][] = $normalizedPath; } return new TranslationSplitterResult($bases, $translations); @@ -38,9 +42,8 @@ public function split(array $pathList): TranslationSplitterResult private function extractLocaleFromPath(string $path): string { - $normalizedPath = $this->normalizePath($path); - $filename = basename($normalizedPath); - $parentDirName = basename(dirname($normalizedPath)); + $filename = basename($path); + $parentDirName = basename(dirname($path)); if (!str_ends_with($parentDirName, '.php.translations')) { return 'default'; @@ -64,7 +67,7 @@ private function normalizePath(string $path): string } $urlPath = parse_url($path, PHP_URL_PATH); if (!is_string($urlPath)) { - return $path; + return ''; } parse_str($queryString, $params); if (!isset($params['loc']) || !is_string($params['loc'])) { diff --git a/test/Console/Command/IndexerInternalResourceUpdateTest.php b/test/Console/Command/IndexerInternalResourceUpdateTest.php index 9da80cc..71f0bf7 100644 --- a/test/Console/Command/IndexerInternalResourceUpdateTest.php +++ b/test/Console/Command/IndexerInternalResourceUpdateTest.php @@ -7,7 +7,6 @@ use Atoolo\Resource\ResourceChannel; use Atoolo\Resource\ResourceChannelFactory; use Atoolo\Search\Console\Application; -use Atoolo\Search\Console\Command\Indexer; use Atoolo\Search\Console\Command\IndexerInternalResourceUpdate; use Atoolo\Search\Console\Command\Io\IndexerProgressBar; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; @@ -17,7 +16,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; -#[CoversClass(Indexer::class)] +#[CoversClass(IndexerInternalResourceUpdate::class)] class IndexerInternalResourceUpdateTest extends TestCase { private ResourceChannelFactory $resourceChannelFactory; diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 43c7441..756ebd8 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -45,12 +45,28 @@ public function setUp(): void ); $this->resourceChannelFactory->method('create') ->willReturn($resourceChannel); - $indexer = $this->createStub( + $indexerA = $this->createStub( \Atoolo\Search\Indexer::class ); - $indexer->method('enabled') + $indexerA->method('enabled') ->willReturn(true); - $indexers = new IndexerCollection([$indexer]); + $indexerA->method('getSource') + ->willReturn('indexer_a'); + $indexerA->method('getName') + ->willReturn('Indexer A'); + $indexerB = $this->createStub( + \Atoolo\Search\Indexer::class + ); + $indexerB->method('enabled') + ->willReturn(false); + $indexerB->method('getSource') + ->willReturn('indexer_b'); + $indexerB->method('getName') + ->willReturn('Indexer B'); + $indexers = new IndexerCollection([ + $indexerA, + $indexerB + ]); $progressBar = $this->createStub(IndexerProgressBar::class); $command = new Indexer( @@ -65,7 +81,7 @@ public function setUp(): void $this->commandTester = new CommandTester($command); } - public function testExecuteIndexAll(): void + public function testExecuteAllEnabledIndexer(): void { $this->commandTester->execute([]); @@ -80,8 +96,8 @@ public function testExecuteIndexAll(): void ============ -Index with Indexer "" ---------------------- +Index with Indexer "Indexer A" +------------------------------ @@ -96,6 +112,40 @@ public function testExecuteIndexAll(): void ); } + public function testExecuteIndexerA(): void + { + $this->commandTester->execute( + [ + '--source' => 'indexer_a' + ] + ); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<output = $this->createMock(OutputInterface::class); + $output = $this->createMock(OutputInterface::class); $this->progressHandler = $this->createMock(IndexerProgressHandler::class); - $this->progressBar = new IndexerProgressBar($this->output); + $this->progressBar = new IndexerProgressBar($output); $this->progressBar->init($this->progressHandler); } @@ -41,6 +45,9 @@ public function testStart(): void $this->progressBar->start(10); } + /** + * @throws ExceptionInterface + */ public function testStartUpdate(): void { $this->progressHandler @@ -52,6 +59,7 @@ public function testStartUpdate(): void /** * @phpcs:disable Generic.Files.LineLength + * @throws JsonException */ public function testAdvance(): void { @@ -63,7 +71,7 @@ public function testAdvance(): void ->method('advance') ->with(1); - $this->progressHandler->advance(1); + $this->progressBar->advance(1); } public function testSkip(): void @@ -107,6 +115,9 @@ public function testGetError(): void ); } + /** + * @throws JsonException + */ public function testFinish(): void { @@ -126,5 +137,19 @@ public function testAbort(): void ->expects($this->once()) ->method('abort'); $this->progressBar->abort(); - } + } + + /** + * @throws \PHPUnit\Framework\MockObject\Exception + */ + public function testGetStatus(): void + { + $status = $this->createStub(IndexerStatus::class); + $this->progressHandler->method('getStatus')->willReturn($status); + $this->progressHandler + ->expects($this->once()) + ->method('getStatus') + ->willReturn($status); + $this->progressBar->getStatus(); + } } diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index c20b0b1..2066dfd 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -13,6 +13,7 @@ use Atoolo\Search\Service\Search\SolrMoreLikeThis; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\Exception; +use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use Symfony\Component\Console\Tester\CommandTester; @@ -21,6 +22,8 @@ class MoreLikeThisTest extends TestCase { private CommandTester $commandTester; + private SolrMoreLikeThis&Stub $solrMoreLikeThis; + /** * @throws Exception */ @@ -55,11 +58,9 @@ public function setUp(): void [], 10 ); - $solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); - $solrMoreLikeThis->method('moreLikeThis') - ->willReturn($result); + $this->solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); - $command = new MoreLikeThis($resourceChannelFactory, $solrMoreLikeThis); + $command = new MoreLikeThis($resourceChannelFactory, $this->solrMoreLikeThis); $application = new Application([$command]); @@ -69,6 +70,21 @@ public function setUp(): void public function testExecute(): void { + + $resultResource = $this->createStub(Resource::class); + $resultResource->method('getLocation') + ->willReturn('/test2.php'); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [], + 10 + ); + $this->solrMoreLikeThis->method('moreLikeThis') + ->willReturn($result); + $this->commandTester->execute([ 'location' => '/test.php' ]); @@ -91,4 +107,40 @@ public function testExecute(): void $output ); } + + public function testExecuteNoResult(): void + { + + $result = new SearchResult( + 0, + 1, + 0, + [], + [], + 10 + ); + $this->solrMoreLikeThis->method('moreLikeThis') + ->willReturn($result); + + $this->commandTester->execute([ + 'location' => '/test.php' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<method('create') ->willReturn($resourceChannel); + + $this->solrSearch = $this->createStub(SolrSearch::class); + $command = new Search($resourceChannelFactory, $this->solrSearch); + + $application = new Application([$command]); + + $command = $application->find('search:search'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $resultResource = $this->createStub(Resource::class); $resultResource->method('getLocation') ->willReturn('/test.php'); @@ -63,20 +78,10 @@ public function setUp(): void )], 10 ); - $solrSelect = $this->createStub(SolrSearch::class); - $solrSelect->method('search') - ->willReturn($result); - $command = new Search($resourceChannelFactory, $solrSelect); - - $application = new Application([$command]); - - $command = $application->find('search:search'); - $this->commandTester = new CommandTester($command); - } + $this->solrSearch->method('search') + ->willReturn($result); - public function testExecute(): void - { $this->commandTester->execute([ 'text' => 'test abc' ]); @@ -105,6 +110,41 @@ public function testExecute(): void Query-Time: 10ms +EOF, + $output + ); + } + + public function testExecuteWithEmtpyResult(): void + { + + $result = new SearchResult( + 0, + 1, + 0, + [], + [], + 10 + ); + + $this->solrSearch->method('search') + ->willReturn($result); + + $this->commandTester->execute([ + 'text' => 'test abc' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<method('create') ->willReturn($resourceChannel); + $this->solrSuggest = $this->createStub(SolrSuggest::class); + + $command = new Suggest($resourceChannelFactory, $this->solrSuggest); + + $application = new Application([$command]); + + $command = $application->find('search:suggest'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { $result = new SuggestResult( [ new Suggestion('security', 10), @@ -52,20 +66,38 @@ public function setUp(): void ], 10 ); - $solrSuggest = $this->createStub(SolrSuggest::class); - $solrSuggest->method('suggest') + $this->solrSuggest->method('suggest') ->willReturn($result); - $command = new Suggest($resourceChannelFactory, $solrSuggest); + $this->commandTester->execute([ + 'terms' => 'sec' + ]); - $application = new Application([$command]); + $this->commandTester->assertCommandIsSuccessful(); - $command = $application->find('search:suggest'); - $this->commandTester = new CommandTester($command); + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<solrSuggest->method('suggest') + ->willReturn($result); + $this->commandTester->execute([ 'terms' => 'sec' ]); @@ -80,9 +112,7 @@ public function testExecute(): void Channel: WWW ============ - security (10) - section (5) - Query-Time: 10ms + No suggestions found EOF, $output diff --git a/test/Service/AbstractIndexerTest.php b/test/Service/AbstractIndexerTest.php new file mode 100644 index 0000000..d9a9de4 --- /dev/null +++ b/test/Service/AbstractIndexerTest.php @@ -0,0 +1,121 @@ +createMock(IndexName::class); + $indexName->method('name') + ->willReturn('www'); + $this->progressHandler = $this->createMock(IndexerProgressHandler::class); + $this->aborter = $this->createMock(IndexingAborter::class); + $config = new IndexerConfiguration( + 'test', + 'Test', + new DataBag([]) + ); + $this->configLoader = $this->createMock(IndexerConfigurationLoader::class); + $this->configLoader->method('load') + ->willReturn($config); + $this->indexer = new TextIndexer( + $indexName, + $this->progressHandler, + $this->aborter, + $this->configLoader, + 'test' + ); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Test', + $this->indexer->getName(), + 'The name of the indexer should be "Test"' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'test', + $this->indexer->getSource(), + 'The source of the indexer should be "test"' + ); + } + + public function testGetProgressHandler(): void + { + $this->assertEquals( + $this->progressHandler, + $this->indexer->getProgressHandler(), + 'The progress handler should be the ' . + 'same as the one passed to the constructor' + ); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->indexer->setProgressHandler($progressHandler); + + $this->assertEquals( + $progressHandler, + $this->indexer->getProgressHandler(), + 'The progress handler should be the ' . + 'same as the one passed to the setProgressHandler method' + ); + } + + public function testAbort(): void + { + + $this->aborter->expects($this->once()) + ->method('requestAbortion') + ->with('www-test'); + $this->indexer->abort(); + } + + public function testEnabled(): void + { + + $this->configLoader->expects($this->once()) + ->method('exists') + ->with('test'); + $this->indexer->enabled(); + } + + public function testIsAbortionRequested(): void + { + + $this->aborter->expects($this->once()) + ->method('isAbortionRequested') + ->with('www-test'); + $this->indexer->isAbortionRequested(); + } +} diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index c44db16..0c2a2d6 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -4,8 +4,8 @@ namespace Atoolo\Search\Test\Service\Indexer; -use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\BackgroundIndexer; +use Atoolo\Search\Service\Indexer\IndexerProgressHandler; use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\Indexer\InternalResourceIndexer; use Atoolo\Search\Service\IndexName; @@ -19,18 +19,26 @@ class BackgroundIndexerTest extends TestCase { private IndexName $indexName; - private InternalResourceIndexer&MockObject $solrIndexer; + private InternalResourceIndexer&MockObject $internalResourceIndexer; private IndexerStatusStore&MockObject $statusStore; + private IndexerProgressHandler $progressHandler; private BackgroundIndexer $indexer; public function setUp(): void { + $this->progressHandler = $this->createStub(IndexerProgressHandler::class); $this->indexName = $this->createMock(IndexName::class); - $this->solrIndexer = $this->createMock(InternalResourceIndexer::class); + $this->internalResourceIndexer = $this->createMock( + InternalResourceIndexer::class + ); + $this->internalResourceIndexer->method('getSource') + ->willReturn('internal'); + $this->internalResourceIndexer->method('getProgressHandler') + ->willReturn($this->progressHandler); $this->statusStore = $this->createMock(IndexerStatusStore::class); $this->indexer = new BackgroundIndexer( - $this->solrIndexer, + $this->internalResourceIndexer, $this->indexName, $this->statusStore ); @@ -38,24 +46,23 @@ public function setUp(): void public function testRemove(): void { - $this->solrIndexer->expects($this->once()) + $this->internalResourceIndexer->expects($this->once()) ->method('remove'); $this->indexer->remove(['123']); } public function testAbort(): void { - $this->solrIndexer->expects($this->once()) + $this->internalResourceIndexer->expects($this->once()) ->method('abort'); $this->indexer->abort(); } public function testIndex(): void { - $params = new IndexerParameter(''); - $this->solrIndexer->expects($this->once()) + $this->internalResourceIndexer->expects($this->once()) ->method('index'); - $this->indexer->index($params); + $this->indexer->index(); } public function testIndexIfLocked(): void @@ -73,11 +80,10 @@ public function testIndexIfLocked(): void $lockFactory->method('createLock') ->willReturn($lock); - $this->solrIndexer->expects($this->exactly(0)) + $this->internalResourceIndexer->expects($this->exactly(0)) ->method('index'); - $params = new IndexerParameter(''); - $indexer->index($params); + $indexer->index(); } public function testGetStatus(): void @@ -86,4 +92,46 @@ public function testGetStatus(): void ->method('load'); $this->indexer->getStatus(); } + + public function testEnable(): void + { + $this->assertTrue( + $this->indexer->enabled(), + 'indexer should always be enabled' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'internal', + $this->indexer->getSource(), + 'Unexpected source' + ); + } + + public function testGetProgressHandler(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('getProgressHandler'); + $this->indexer->getProgressHandler(); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->internalResourceIndexer->expects($this->once()) + ->method('setProgressHandler'); + $this->indexer->setProgressHandler($progressHandler); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Background Indexer', + $this->indexer->getName(), + 'Unexpected name' + ); + } + } diff --git a/test/Service/Indexer/IndexerCollectionTest.php b/test/Service/Indexer/IndexerCollectionTest.php new file mode 100644 index 0000000..88be22c --- /dev/null +++ b/test/Service/Indexer/IndexerCollectionTest.php @@ -0,0 +1,52 @@ +createStub(Indexer::class); + $indexer->method('getSource')->willReturn('test'); + $indexers = new IndexerCollection([$indexer]); + $this->assertNotNull($indexers->getIndexer('test')); + } + + public function testGetMissingIndexer(): void + { + $indexers = new IndexerCollection([]); + $this->expectException(\InvalidArgumentException::class); + $indexers->getIndexer('test'); + } + + public function testGetIndexers(): void + { + $indexer = $this->createStub(Indexer::class); + $indexers = new IndexerCollection([$indexer]); + $this->assertCount( + 1, + $indexers->getIndexers(), + 'unexpected number of indexers' + ); + } + + public function testGetIndexersWithIterable(): void + { + $indexer = $this->createStub(Indexer::class); + $indexers = new IndexerCollection(new ArrayIterator([$indexer])); + $this->assertCount( + 1, + $indexers->getIndexers(), + 'unexpected number of indexers' + ); + } +} diff --git a/test/Service/Indexer/IndexerConfigurationLoaderTest.php b/test/Service/Indexer/IndexerConfigurationLoaderTest.php new file mode 100644 index 0000000..5b7cea8 --- /dev/null +++ b/test/Service/Indexer/IndexerConfigurationLoaderTest.php @@ -0,0 +1,130 @@ +resourceBaseLocator = $this->createStub( + ResourceBaseLocator::class + ); + $this->loader = new IndexerConfigurationLoader( + $this->resourceBaseLocator + ); + } + + public function testExists(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/with-internal'); + $this->assertTrue( + $this->loader->exists('internal'), + 'Internal config should exist' + ); + } + + public function testLoad(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/with-internal'); + + $config = $this->loader->load('internal'); + + $expected = new IndexerConfiguration( + 'internal', + 'Internal Indexer', + new DataBag([ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ]), + ); + + $this->assertEquals( + $expected, + $config, + 'unexpected config' + ); + } + + public function testLoadNotExists(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/with-internal'); + + $config = $this->loader->load('not-exists'); + + $expected = new IndexerConfiguration( + 'not-exists', + 'not-exists', + new DataBag([ + ]), + ); + + $this->assertEquals( + $expected, + $config, + 'unexpected config' + ); + } + + public function testLoadAll(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/with-internal'); + + $expected = new IndexerConfiguration( + 'internal', + 'Internal Indexer', + new DataBag([ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ]), + ); + + $this->assertEquals( + [$expected], + $this->loader->loadAll(), + 'unexpected config' + ); + } + + public function testLoadAllNotADirectory(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/not-a-directory'); + + $this->assertEquals( + [], + $this->loader->loadAll(), + 'should return empty array' + ); + } + + public function testLoadAllWithConfigReturnString(): void + { + $this->resourceBaseLocator->method('locate') + ->willReturn(self::RESOURCE_BASE . '/return-string'); + + $this->expectException(RuntimeException::class); + $this->loader->loadAll(); + } +} diff --git a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php b/test/Service/Indexer/IndexerProgressStateTest.php similarity index 71% rename from test/Service/Indexer/BackgroundIndexerProgressStateTest.php rename to test/Service/Indexer/IndexerProgressStateTest.php index 72b3123..08f067a 100644 --- a/test/Service/Indexer/BackgroundIndexerProgressStateTest.php +++ b/test/Service/Indexer/IndexerProgressStateTest.php @@ -9,12 +9,13 @@ use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\IndexName; use Exception; +use LogicException; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; #[CoversClass(IndexerProgressState::class)] -class BackgroundIndexerProgressStateTest extends TestCase +class IndexerProgressStateTest extends TestCase { private IndexerStatusStore&MockObject $statusStore; private IndexerProgressState $state; @@ -58,21 +59,10 @@ public function testUpdate(): void ); } - public function testAdvance(): void + public function testAdvanceWithoutStart(): void { - - $this->state->start(10); - - $this->statusStore->expects($this->once()) - ->method('store'); - + $this->expectException(LogicException::class); $this->state->advance(1); - - $this->assertMatchesRegularExpression( - '/\[RUNNING].*processed: 1\/10,/', - $this->state->getStatus()->getStatusLine(), - "unexpected status line" - ); } public function testAdvanceForUpdate(): void @@ -110,6 +100,12 @@ public function testSkip(): void ); } + public function testSkipWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->skip(1); + } + public function testError(): void { $this->state->start(10); @@ -123,6 +119,12 @@ public function testError(): void ); } + public function testErrorWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->error(new Exception('test')); + } + public function testFinish(): void { $this->state->start(10); @@ -139,6 +141,12 @@ public function testFinish(): void ); } + public function testFinishWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->finish(); + } + public function testAbort(): void { $this->state->start(10); @@ -151,4 +159,32 @@ public function testAbort(): void "unexpected status line" ); } + + public function testAbortWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->abort(); + } + + public function testGetStatus(): void + { + $indexName = $this->createMock(IndexName::class); + $indexName->method('name')->willReturn('test'); + $status = $this->createMock(IndexerStatus::class); + $statusStore = $this->createMock(IndexerStatusStore::class); + $statusStore->method('load') + ->willReturn($status); + + $state = new IndexerProgressState( + $indexName, + $statusStore, + 'source' + ); + + $this->assertEquals( + $status, + $state->getStatus(), + 'unexpected status' + ); + } } diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 053726e..c10b0a5 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -7,7 +7,6 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; -use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexerConfigurationLoader; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; @@ -25,7 +24,10 @@ use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; use Solarium\QueryType\Update\Result as UpdateResult; +use Symfony\Component\Lock\LockFactory; +use Symfony\Component\Lock\Store\SemaphoreStore; #[CoversClass(InternalResourceIndexer::class)] class InternalResourceIndexerTest extends TestCase @@ -43,7 +45,7 @@ class InternalResourceIndexerTest extends TestCase private InternalResourceIndexer $indexer; - private SolrIndexService $solrIndexService; + private SolrIndexService&MockObject $solrIndexService; private LocationFinder&MockObject $finder; @@ -61,6 +63,10 @@ class InternalResourceIndexerTest extends TestCase private IndexerConfiguration $indexerConfiguration; + private LockFactory $lockFactory; + + private LoggerInterface&MockObject $logger; + /** * @throws Exception */ @@ -111,8 +117,8 @@ public function setUp(): void ->willReturn($this->updater); $this->aborter = $this->createMock(IndexingAborter::class); $this->indexerConfiguration = new IndexerConfiguration( - '', - '', + 'test-source', + 'Indexer-Name', new DataBag([ 'cleanupThreshold' => 10, 'chunkSize' => 10 @@ -124,6 +130,9 @@ public function setUp(): void $this->indexerConfigurationLoader->method('load') ->willReturn($this->indexerConfiguration); + $this->lockFactory = new LockFactory(new SemaphoreStore()); + $this->logger = $this->createMock(LoggerInterface::class); + $this->indexer = new InternalResourceIndexer( [ $this->documentEnricher ], $this->indexerFilter, @@ -134,7 +143,9 @@ public function setUp(): void $this->solrIndexService, $this->aborter, $this->indexerConfigurationLoader, - 'test-source' + 'test-source', + $this->lockFactory, + $this->logger ); } @@ -285,22 +296,7 @@ public function testWithInvalidResource(): void $this->indexerProgressHandler->expects($this->once()) ->method('error'); - $parameter = new IndexerParameter( - '', - 10, - 10 - ); - - $this->indexer->index($parameter); - } - - public function testEmptyStatus(): void - { - $this->finder->method('findAll') - ->willReturn([]); - - $status = $this->indexer->index(); - $this->assertEquals(0, $status->total, 'total should be 0'); + $this->indexer->index(); } public function testUpdate(): void @@ -329,6 +325,30 @@ public function testUpdate(): void ]); } + public function testUpdateOtherLang(): void + { + $this->finder->method('findPaths') + ->willReturn([ + '/a/b.php.translations/en_US.php', + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->indexerFilter->method('accept') + ->willReturn(true); + + $this->updater->expects($this->exactly(1)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); + + $this->indexer->update([ + '/a/b.php.translations/en_US.php', + ]); + } + public function testUpdateWithParameter(): void { $this->finder->expects($this->once()) @@ -359,15 +379,69 @@ public function testWithoutAvailableIndexes(): void '/a/l.php' ]); - $parameter = new IndexerParameter( - '', - 10, - 10 - ); - $this->indexerProgressHandler->expects($this->once()) ->method('error'); - $this->indexer->index($parameter); + $this->indexer->index(); + } + + public function testEnabled(): void + { + $this->assertTrue( + $this->indexer->enabled(), + 'indexer should be always enabled' + ); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Indexer-Name', + $this->indexer->getName(), + 'unexpected Indexer Name' + ); + } + + public function testGetProgressHandler(): void + { + $this->assertEquals( + $this->indexerProgressHandler, + $this->indexer->getProgressHandler(), + 'unexpected progress handler' + ); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->indexer->setProgressHandler($progressHandler); + $this->assertEquals( + $progressHandler, + $this->indexer->getProgressHandler(), + 'unexpected progress handler' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'test-source', + $this->indexer->getSource(), + 'unexpected source' + ); + } + + public function testLock(): void + { + $this->logger->expects($this->once()) + ->method('notice') + ->with('Indexer with source "test-source" is already running'); + $lock = $this->lockFactory->createLock('indexer.test-source'); + try { + $lock->acquire(); + $this->indexer->index(); + } finally { + $lock->release(); + } } } diff --git a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php new file mode 100644 index 0000000..efa5f24 --- /dev/null +++ b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php @@ -0,0 +1,50 @@ +assertTrue( + $filter->accept($resource), + "resource should be accepted" + ); + } + + public function testAcceptWithNoIndex(): void + { + $resource = new Resource( + 'test', + 'test', + 'test', + 'test', + '', + ['init' => ['noIndex' => true]] + ); + $filter = new NoIndexFilter(); + $this->assertFalse( + $filter->accept($resource), + "resource should not be accepted" + ); + } + +} + diff --git a/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php index bd7f780..3765878 100644 --- a/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php +++ b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php @@ -62,4 +62,52 @@ public function testGetTranlsations(): void 'unexpected translations' ); } + + public function testSplitWithLocParameter(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php?loc=en_US', + ]); + + $translations = $result->getTranslations('en_US'); + + $expected = [ + '/a/b.php.translations/en_US.php', + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } + + public function testSplitWithOutPath(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '?a=b', + ]); + + $this->assertEquals( + [], + $result->getBases(), + 'bases should be empty' + ); + } + + public function testSplitWithUnsupportedParameter(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/test.php?a=b', + ]); + + $this->assertEquals( + ['/test.php'], + $result->getBases(), + 'unexpected bases' + ); + } } diff --git a/test/Service/TextIndexer.php b/test/Service/TextIndexer.php new file mode 100644 index 0000000..7612695 --- /dev/null +++ b/test/Service/TextIndexer.php @@ -0,0 +1,34 @@ +progressHandler->getStatus(); + } + + /** + * @inheritDoc + */ + public function remove(array $idList): void + { + } + + /** + * make this method public for testing + */ + public function isAbortionRequested(): bool + { + return parent::isAbortionRequested(); + } +} diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer b/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php b/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php new file mode 100644 index 0000000..05d0e7d --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php @@ -0,0 +1,3 @@ + 'Internal Indexer', + 'data' => [ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ], +]; diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php b/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php new file mode 100644 index 0000000..196f10a --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php @@ -0,0 +1,5 @@ + 'Without Source' +]; From ddc1f74e60e4386f5ef2cf6777238dbe9dbb3638 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 2 Apr 2024 13:52:39 +0200 Subject: [PATCH 118/145] style: fix codestyles --- test/Console/Command/MoreLikeThisTest.php | 6 ++++-- test/Console/Command/SuggestTest.php | 2 +- test/Service/AbstractIndexerTest.php | 8 ++++++-- test/Service/Indexer/BackgroundIndexerTest.php | 5 +++-- test/Service/Indexer/SiteKit/NoIndexFilterTest.php | 2 -- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index 2066dfd..d4cf095 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -60,7 +60,10 @@ public function setUp(): void ); $this->solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); - $command = new MoreLikeThis($resourceChannelFactory, $this->solrMoreLikeThis); + $command = new MoreLikeThis( + $resourceChannelFactory, + $this->solrMoreLikeThis + ); $application = new Application([$command]); @@ -142,5 +145,4 @@ public function testExecuteNoResult(): void $output ); } - } diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index 38ee8e3..2f615e4 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -94,7 +94,7 @@ public function testExecute(): void public function testExecuteNoResult(): void { - $result = new SuggestResult([],10); + $result = new SuggestResult([], 10); $this->solrSuggest->method('suggest') ->willReturn($result); diff --git a/test/Service/AbstractIndexerTest.php b/test/Service/AbstractIndexerTest.php index d9a9de4..434721e 100644 --- a/test/Service/AbstractIndexerTest.php +++ b/test/Service/AbstractIndexerTest.php @@ -32,14 +32,18 @@ public function setUp(): void $indexName = $this->createMock(IndexName::class); $indexName->method('name') ->willReturn('www'); - $this->progressHandler = $this->createMock(IndexerProgressHandler::class); + $this->progressHandler = $this->createMock( + IndexerProgressHandler::class + ); $this->aborter = $this->createMock(IndexingAborter::class); $config = new IndexerConfiguration( 'test', 'Test', new DataBag([]) ); - $this->configLoader = $this->createMock(IndexerConfigurationLoader::class); + $this->configLoader = $this->createMock( + IndexerConfigurationLoader::class + ); $this->configLoader->method('load') ->willReturn($config); $this->indexer = new TextIndexer( diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index 0c2a2d6..cc6a2b5 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -27,7 +27,9 @@ class BackgroundIndexerTest extends TestCase public function setUp(): void { - $this->progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->progressHandler = $this->createStub( + IndexerProgressHandler::class + ); $this->indexName = $this->createMock(IndexName::class); $this->internalResourceIndexer = $this->createMock( InternalResourceIndexer::class @@ -133,5 +135,4 @@ public function testGetName(): void 'Unexpected name' ); } - } diff --git a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php index efa5f24..9e3927f 100644 --- a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php +++ b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php @@ -45,6 +45,4 @@ public function testAcceptWithNoIndex(): void "resource should not be accepted" ); } - } - From 73c230ca89ff931232523e3dd2314ddc6e074ed9 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 9 Apr 2024 14:23:47 +0200 Subject: [PATCH 119/145] fix: get indexer status --- src/Service/Indexer/BackgroundIndexer.php | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 1b72147..0052cea 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -66,15 +66,11 @@ public function index(): IndexerStatus */ public function getStatus(): IndexerStatus { - return $this->statusStore->load($this->getIndex()); + return $this->statusStore->load($this->getStatusStoreKey()); } - private function getIndex(): string + private function getStatusStoreKey(): string { - /* - * The indexer always requires the default index, as the language is - * determined via the resources to be indexed. - */ - return $this->index->name(''); + return $this->index->name('') . '-' . $this->indexer->getSource(); } } From 958ca86b719f3e3948d6bfb17fee745e284d676c Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 9 Apr 2024 15:46:34 +0200 Subject: [PATCH 120/145] feat: timeout for solr requests increased to 30 seconds. --- src/Service/ServerVarSolrClientFactory.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Service/ServerVarSolrClientFactory.php b/src/Service/ServerVarSolrClientFactory.php index a63bb39..44ec30b 100644 --- a/src/Service/ServerVarSolrClientFactory.php +++ b/src/Service/ServerVarSolrClientFactory.php @@ -14,11 +14,14 @@ class ServerVarSolrClientFactory implements SolrClientFactory { private const IES_WEBNODE_SOLR_PORT = '8382'; + private int $timeoutInSeconds = 30; + + public function create(string $core): Client { $adapter = new Curl(); + $adapter->setTimeout($this->timeoutInSeconds); /* - $adapter->setTimeout($this->timeout); $adapter->setProxy($this->proxy); */ $eventDispatcher = new EventDispatcher(); From ac95e5e469dd745dca3dcc161a87b29861383072 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 9 Apr 2024 16:36:18 +0200 Subject: [PATCH 121/145] fix: add filter without key --- src/Dto/Search/Query/SearchQueryBuilder.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Dto/Search/Query/SearchQueryBuilder.php b/src/Dto/Search/Query/SearchQueryBuilder.php index fd6b6c5..17f2d94 100644 --- a/src/Dto/Search/Query/SearchQueryBuilder.php +++ b/src/Dto/Search/Query/SearchQueryBuilder.php @@ -19,7 +19,7 @@ class SearchQueryBuilder */ private array $sort = []; /** - * @var array + * @var array */ private array $filter = []; @@ -94,13 +94,17 @@ public function sort(Criteria ...$criteriaList): static public function filter(Filter ...$filterList): static { foreach ($filterList as $filter) { - if (isset($this->filter[$filter->key])) { - throw new \InvalidArgumentException( - 'filter key "' . $filter->key . + if ($filter->key !== null) { + foreach ($this->filter as $existingFilter) { + if ($existingFilter->key === $filter->key) { + throw new \InvalidArgumentException( + 'filter key "' . $filter->key . '" already exists' - ); + ); + } + } } - $this->filter[$filter->key] = $filter; + $this->filter[] = $filter; } return $this; } @@ -140,7 +144,7 @@ public function build(): SearchQuery offset: $this->offset, limit: $this->limit, sort: $this->sort, - filter: array_values($this->filter), + filter: $this->filter, facets: array_values($this->facets), defaultQueryOperator: $this->defaultQueryOperator ); From 98400b567801a5526c49f1e81ff4c94564ec93cb Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Tue, 9 Apr 2024 16:39:18 +0200 Subject: [PATCH 122/145] feat: DefaultQueryOperator is now 'OR' if not set --- src/Dto/Search/Query/SearchQueryBuilder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dto/Search/Query/SearchQueryBuilder.php b/src/Dto/Search/Query/SearchQueryBuilder.php index 17f2d94..0f63f7c 100644 --- a/src/Dto/Search/Query/SearchQueryBuilder.php +++ b/src/Dto/Search/Query/SearchQueryBuilder.php @@ -29,7 +29,7 @@ class SearchQueryBuilder private array $facets = []; private QueryOperator $defaultQueryOperator = - QueryOperator::AND; + QueryOperator::OR; public function __construct() { From 085b3b0d791e18fd183133cdb65e021f7658cb51 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 10 Apr 2024 15:39:29 +0200 Subject: [PATCH 123/145] refactor: indexUpdate --- src/Service/Indexer/BackgroundIndexer.php | 8 ++++++++ .../Indexer/InternalResourceIndexer.php | 20 +++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 0052cea..417acec 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -61,6 +61,14 @@ public function index(): IndexerStatus return $this->indexer->index(); } + /** + * @param array $paths + */ + public function update(array $paths): IndexerStatus + { + return $this->indexer->update($paths); + } + /** * @throws ExceptionInterface */ diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index a242e97..24b2b2f 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -56,10 +56,10 @@ public function __construct( private readonly IndexingAborter $aborter, private readonly IndexerConfigurationLoader $configLoader, private readonly string $source, + private readonly LoggerInterface $logger = new NullLogger(), private readonly LockFactory $lockFactory = new LockFactory( new SemaphoreStore() ), - private readonly LoggerInterface $logger = new NullLogger() ) { } @@ -116,20 +116,28 @@ public function abort(): void */ public function index(): IndexerStatus { - $lock = $this->lockFactory->createLock('indexer.' . $this->source); + $lock = $this->lockFactory->createLock( + 'indexer.' . $this->getBaseIndex() + ); if (!$lock->acquire()) { - $this->logger->notice( - 'Indexer with source "' . $this->source . '" is already running' - ); + $this->logger->notice('Indexer is already running', [ + 'index' => $this->getBaseIndex() + ]); return $this->progressHandler->getStatus(); } + $param = $this->getIndexerParameter(); + + $this->logger->info('Start indexing', [ + 'index' => $this->getBaseIndex(), + 'chunkSize' => $param->chunkSize, + 'cleanupThreshold' => $param->cleanupThreshold, + ]); try { $paths = $this->finder->findAll(); $this->deleteErrorProtocol($this->getBaseIndex()); $total = count($paths); $this->progressHandler->start($total); - $param = $this->getIndexerParameter(); $this->indexResources($param, $this->finder->findAll()); } finally { $lock->release(); From 23a8796dafa4107d70183a8c454d432c47e9f6dd Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 10 Apr 2024 15:58:03 +0200 Subject: [PATCH 124/145] test: fix tests --- test/Service/Indexer/InternalResourceIndexerTest.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index c10b0a5..214342b 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -144,8 +144,8 @@ public function setUp(): void $this->aborter, $this->indexerConfigurationLoader, 'test-source', + $this->logger, $this->lockFactory, - $this->logger ); } @@ -435,8 +435,10 @@ public function testLock(): void { $this->logger->expects($this->once()) ->method('notice') - ->with('Indexer with source "test-source" is already running'); - $lock = $this->lockFactory->createLock('indexer.test-source'); + ->with('Indexer is already running', [ + 'index' => 'test' + ]); + $lock = $this->lockFactory->createLock('indexer.test'); try { $lock->acquire(); $this->indexer->index(); From 57de5c5eb8ed341e1cfcf2e3932ebaa8e0223411 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Wed, 10 Apr 2024 17:54:54 +0200 Subject: [PATCH 125/145] feat: add indexer preparing state --- src/Console/Command/Io/IndexerProgressBar.php | 12 ++++++ src/Dto/Indexer/IndexerStatus.php | 13 ++++++- src/Dto/Indexer/IndexerStatusState.php | 1 + .../Indexer/IndexerProgressHandler.php | 1 + src/Service/Indexer/IndexerProgressState.php | 37 ++++++++++++++++++- .../Indexer/InternalResourceIndexer.php | 3 ++ 6 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php index 3041dc0..bf96aac 100644 --- a/src/Console/Command/Io/IndexerProgressBar.php +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -19,6 +19,8 @@ class IndexerProgressBar implements IndexerProgressHandler private IndexerProgressHandler $currentProgressHandler; + private int $prepareLines = 0; + /** * @var array */ @@ -37,8 +39,18 @@ public function init( $this->progressBar = null; } + public function prepare(string $message): void + { + $this->currentProgressHandler->prepare($message); + $this->output->writeln($message); + $this->prepareLines++; + } + public function start(int $total): void { + if ($this->prepareLines > 0) { + $this->output->write("\x1b[" . $this->prepareLines . "A"); + } $this->currentProgressHandler->start($total); $this->progressBar = new ProgressBar($this->output, $total); $this->formatProgressBar('green'); diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php index bba6d45..7ea77aa 100644 --- a/src/Dto/Indexer/IndexerStatus.php +++ b/src/Dto/Indexer/IndexerStatus.php @@ -31,7 +31,8 @@ public function __construct( public int $skipped, public DateTime $lastUpdate, public int $updated, - public int $errors + public int $errors, + public string $prepareMessage = '' ) { } @@ -53,6 +54,7 @@ public static function empty(): IndexerStatus public function getStatusLine(): string { + $endTime = $this->endTime; if ($endTime === null || $endTime->getTimestamp() === 0) { $endTime = new DateTime(); @@ -63,6 +65,15 @@ public function getStatusLine(): string if ($lastUpdate->getTimestamp() === 0) { $lastUpdate = $endTime; } + + if ($this->state === IndexerStatusState::PREPARING) { + return + '[' . $this->state->name . '] ' . + 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . + 'message: ' . $this->prepareMessage; + } + return '[' . $this->state->name . '] ' . 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php index 2a7c19d..8e22865 100644 --- a/src/Dto/Indexer/IndexerStatusState.php +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -10,6 +10,7 @@ enum IndexerStatusState: string { case UNKNOWN = 'UNKNOWN'; + case PREPARING = 'PREPARING'; case RUNNING = 'RUNNING'; case FINISHED = 'FINISHED'; case ABORTED = 'ABORTED'; diff --git a/src/Service/Indexer/IndexerProgressHandler.php b/src/Service/Indexer/IndexerProgressHandler.php index 49449e8..58c00b4 100644 --- a/src/Service/Indexer/IndexerProgressHandler.php +++ b/src/Service/Indexer/IndexerProgressHandler.php @@ -9,6 +9,7 @@ interface IndexerProgressHandler { + public function prepare(string $message): void; public function start(int $total): void; public function startUpdate(int $total): void; public function advance(int $step): void; diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index 7fec343..56df104 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -25,11 +25,38 @@ public function __construct( ) { } + public function prepare(string $message): void + { + $this->status = new IndexerStatus( + IndexerStatusState::PREPARING, + new DateTime(), + null, + 0, + 0, + 0, + new DateTime(), + 0, + 0, + $message + ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); + } + public function start(int $total): void { + $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); + + $startTime = new DateTime(); + if ($storedStatus->state === IndexerStatusState::PREPARING) { + $startTime = $storedStatus->startTime; + } + $this->status = new IndexerStatus( IndexerStatusState::RUNNING, - new DateTime(), + $startTime, null, $total, 0, @@ -38,6 +65,10 @@ public function start(int $total): void 0, 0 ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); } /** @@ -58,6 +89,10 @@ public function startUpdate(int $total): void $storedStatus->updated, $storedStatus->errors, ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); } public function advance(int $step): void diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 24b2b2f..a372961 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -132,6 +132,9 @@ public function index(): IndexerStatus 'chunkSize' => $param->chunkSize, 'cleanupThreshold' => $param->cleanupThreshold, ]); + + $this->progressHandler->prepare('Collect resource locations'); + try { $paths = $this->finder->findAll(); $this->deleteErrorProtocol($this->getBaseIndex()); From 956eb4a6168530e4d6865ef9cbb14c2e6fb2ab09 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 15 Apr 2024 13:54:41 +0200 Subject: [PATCH 126/145] feat: resource-layout indexer config location --- src/Service/Indexer/IndexerConfigurationLoader.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php index 12af65d..3977036 100644 --- a/src/Service/Indexer/IndexerConfigurationLoader.php +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -23,6 +23,10 @@ public function loadAll(): array { $dir = $this->resourceBaseLocator->locate() . '/indexer'; if (!is_dir($dir)) { + $dir = $this->resourceBaseLocator->locate() . '/../configs/indexer'; + } + if (!is_dir($dir)) { + echo $dir . " not found \n"; return []; } @@ -38,8 +42,13 @@ public function loadAll(): array private function getFile(string $source): string { - return $this->resourceBaseLocator->locate() . + $file = $this->resourceBaseLocator->locate() . '/indexer/' . $source . '.php'; + if (file_exists($file)) { + return $file; + } + return $this->resourceBaseLocator->locate() . + '/../configs/indexer/' . $source . '.php'; } public function exists(string $source): bool From a7e6f3f49f758583a6289b63c791b5768dbf9164 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 15 Apr 2024 16:04:15 +0200 Subject: [PATCH 127/145] chore: remove debug --- src/Service/Indexer/IndexerConfigurationLoader.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php index 3977036..009238a 100644 --- a/src/Service/Indexer/IndexerConfigurationLoader.php +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -26,7 +26,6 @@ public function loadAll(): array $dir = $this->resourceBaseLocator->locate() . '/../configs/indexer'; } if (!is_dir($dir)) { - echo $dir . " not found \n"; return []; } From 04b863937766741d7768281668ea1416fd5cf552 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 15 Apr 2024 16:04:28 +0200 Subject: [PATCH 128/145] test: fix tests --- .../Command/Io/IndexerProgressBarTest.php | 20 +++++++ test/Dto/Indexer/IndexerStatusTest.php | 37 ++++++++++++- .../Service/Indexer/BackgroundIndexerTest.php | 7 +++ .../Indexer/IndexerProgressStateTest.php | 55 +++++++++++++++++++ .../Indexer/IndexerStatusStoreTest.php | 3 +- 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/test/Console/Command/Io/IndexerProgressBarTest.php b/test/Console/Command/Io/IndexerProgressBarTest.php index 712a9b2..e338f14 100644 --- a/test/Console/Command/Io/IndexerProgressBarTest.php +++ b/test/Console/Command/Io/IndexerProgressBarTest.php @@ -36,6 +36,15 @@ public function setUp(): void $this->progressBar->init($this->progressHandler); } + public function testPrepare(): void + { + $this->progressHandler + ->expects($this->once()) + ->method('prepare') + ->with('test'); + $this->progressBar->prepare('test'); + } + public function testStart(): void { $this->progressHandler @@ -45,6 +54,17 @@ public function testStart(): void $this->progressBar->start(10); } + public function testStartAfterPrepare(): void + { + $this->progressBar->prepare('test'); + + $this->progressHandler + ->expects($this->once()) + ->method('start') + ->with(10); + $this->progressBar->start(10); + } + /** * @throws ExceptionInterface */ diff --git a/test/Dto/Indexer/IndexerStatusTest.php b/test/Dto/Indexer/IndexerStatusTest.php index 0346b1f..30458f6 100644 --- a/test/Dto/Indexer/IndexerStatusTest.php +++ b/test/Dto/Indexer/IndexerStatusTest.php @@ -42,7 +42,7 @@ public function setUp(): void ); } - public function testGetStatus(): void + public function testGetStatusLine(): void { $this->assertEquals( '[FINISHED] ' . @@ -57,6 +57,41 @@ public function testGetStatus(): void "unexpected status line" ); } + + public function testGetStatusLineForPreparing(): void + { + + $startTime = new DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $status = new IndexerStatus( + IndexerStatusState::PREPARING, + $startTime, + $endTime, + 10, + 5, + 4, + $startTime, + 6, + 2, + 'prepare message' + ); + + $this->assertEquals( + '[PREPARING] ' . + 'start: 31.01.2024 11:15, ' . + 'time: 01h 01m 01s, ' . + 'message: prepare message', + $status->getStatusLine(), + "unexpected status line" + ); + } + public function testEmpty(): void { $status = IndexerStatus::empty(); diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php index cc6a2b5..f69dd79 100644 --- a/test/Service/Indexer/BackgroundIndexerTest.php +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -67,6 +67,13 @@ public function testIndex(): void $this->indexer->index(); } + public function testUpdate(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('update'); + $this->indexer->update(['/index.php']); + } + public function testIndexIfLocked(): void { $lockFactory = $this->createStub(LockFactory::class); diff --git a/test/Service/Indexer/IndexerProgressStateTest.php b/test/Service/Indexer/IndexerProgressStateTest.php index 08f067a..00ae371 100644 --- a/test/Service/Indexer/IndexerProgressStateTest.php +++ b/test/Service/Indexer/IndexerProgressStateTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\Indexer\IndexerProgressState; use Atoolo\Search\Service\Indexer\IndexerStatusStore; use Atoolo\Search\Service\IndexName; @@ -20,11 +21,21 @@ class IndexerProgressStateTest extends TestCase private IndexerStatusStore&MockObject $statusStore; private IndexerProgressState $state; + private ?IndexerStatus $status = null; + public function setUp(): void { $indexName = $this->createMock(IndexName::class); $indexName->method('name')->willReturn('test'); + + $this->status = IndexerStatus::empty(); + $this->statusStore = $this->createMock(IndexerStatusStore::class); + $that = $this; + $this->statusStore->method('load') + ->willReturnCallback(function () use ($that) { + return $that->status; + }); $this->state = new IndexerProgressState( $indexName, $this->statusStore, @@ -32,6 +43,17 @@ public function setUp(): void ); } + public function testPrepare(): void + { + $this->state->prepare('prepare message'); + + $this->assertMatchesRegularExpression( + '/\[PREPARING].*prepare message.*/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + public function testStart(): void { $this->state->start(10); @@ -43,6 +65,39 @@ public function testStart(): void ); } + public function testStartAfterPrepare(): void + { + + $startTime = new \DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new \DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $this->status = new IndexerStatus( + IndexerStatusState::PREPARING, + $startTime, + $endTime, + 0, + 0, + 0, + $endTime, + 0, + 0, + 'prepare message' + ); + + $this->state->start(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*start: 31\.01\.2024 11:15,.*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + public function testUpdate(): void { diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php index d4cf26d..18badf8 100644 --- a/test/Service/Indexer/IndexerStatusStoreTest.php +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -51,7 +51,8 @@ public function testStore(): void '"skipped":4,' . '"lastUpdate":"2024-01-31T13:17:12+00:00",' . '"updated":6,' . - '"errors":2' . + '"errors":2,' . + '"prepareMessage":""' . '}'; $this->assertEquals($expected, $json, 'unexpected json string'); From dcd934330a72a6c9c4de011c06e591250ca5ad64 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 18 Apr 2024 12:39:44 +0200 Subject: [PATCH 129/145] refactor: use ResourceLocation and ResourceLanguage --- src/Console/Command/MoreLikeThis.php | 18 ++- src/Console/Command/Search.php | 2 +- src/Dto/Search/Query/MoreLikeThisQuery.php | 4 +- src/Service/AbstractIndexer.php | 7 +- src/Service/IndexName.php | 4 +- src/Service/Indexer/BackgroundIndexer.php | 4 +- src/Service/Indexer/IndexDocumentDumper.php | 4 +- .../Indexer/IndexerConfigurationLoader.php | 16 +- src/Service/Indexer/IndexerProgressState.php | 16 +- .../Indexer/InternalResourceIndexer.php | 51 +++---- src/Service/Indexer/LocationFinder.php | 6 +- .../DefaultSchema2xDocumentEnricher.php | 55 ++++--- src/Service/Indexer/SiteKit/NoIndexFilter.php | 2 +- .../SiteKit/SubDirTranslationSplitter.php | 38 +++-- src/Service/Indexer/SolrIndexService.php | 15 +- .../Indexer/TranslationSplitterResult.php | 31 ++-- src/Service/ResourceChannelBasedIndexName.php | 29 ++-- .../Search/ExternalResourceFactory.php | 12 +- .../Search/InternalMediaResourceFactory.php | 27 ++-- .../Search/InternalResourceFactory.php | 19 ++- src/Service/Search/ResourceFactory.php | 8 +- src/Service/Search/SolrMoreLikeThis.php | 7 +- .../Search/SolrResultToResourceResolver.php | 18 ++- src/Service/Search/SolrSearch.php | 8 +- src/Service/Search/SolrSuggest.php | 4 +- src/Service/ServerVarSolrClientFactory.php | 83 ----------- .../Console/Command/DumpIndexDocumentTest.php | 1 + .../IndexerInternalResourceUpdateTest.php | 1 + test/Console/Command/IndexerTest.php | 1 + test/Console/Command/MoreLikeThisTest.php | 25 +++- test/Console/Command/SearchTest.php | 14 +- test/Console/Command/SuggestTest.php | 1 + .../IndexerConfigurationLoaderTest.php | 76 ++++++---- .../Indexer/InternalResourceIndexerTest.php | 33 +++-- test/Service/Indexer/LocationFinderTest.php | 20 ++- .../DefaultSchema2xDocumentEnricherTest.php | 102 +++++++------ .../Indexer/SiteKit/NoIndexFilterTest.php | 10 +- .../SiteKit/SubDirTranslationSplitterTest.php | 23 ++- test/Service/Indexer/SolrIndexServiceTest.php | 18 ++- .../Indexer/TranslationSplitterResultTest.php | 51 ++++--- .../ResourceChannelBasedIndexNameTest.php | 17 +-- .../Search/ExternalResourceFactoryTest.php | 23 ++- .../InternalMediaResourceFactoryTest.php | 11 +- .../Search/InternalResourceFactoryTest.php | 11 +- test/Service/Search/SolrMoreLikeThisTest.php | 4 +- .../SolrResultToResourceResolverTest.php | 16 +- .../ServerVarSolrClientFactoryTest.php | 137 ------------------ 47 files changed, 512 insertions(+), 571 deletions(-) delete mode 100644 src/Service/ServerVarSolrClientFactory.php delete mode 100644 test/Service/ServerVarSolrClientFactoryTest.php diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 3afbeeb..79ddf37 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -5,6 +5,8 @@ namespace Atoolo\Search\Console\Command; use Atoolo\Resource\ResourceChannelFactory; +use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Console\Command\Io\TypifiedInput; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; @@ -60,13 +62,17 @@ protected function execute( $this->input = new TypifiedInput($input); $this->io = new SymfonyStyle($input, $output); - $location = $this->input->getStringArgument('location'); - $lang = $this->input->getStringOption('lang'); + $location = ResourceLocation::of( + $this->input->getStringArgument('location'), + ResourceLanguage::of( + $this->input->getStringOption('lang') + ) + ); $resourceChannel = $this->channelFactory->create(); $this->io->title('Channel: ' . $resourceChannel->name); - $query = $this->buildQuery($location, $lang); + $query = $this->buildQuery($location); $result = $this->searcher->moreLikeThis($query); $this->outputResult($result); @@ -74,13 +80,11 @@ protected function execute( } protected function buildQuery( - string $location, - string $lang + ResourceLocation $location, ): MoreLikeThisQuery { $filterList = []; return new MoreLikeThisQuery( location: $location, - lang: $lang, filter: $filterList, limit: 5, fields: ['content'] @@ -95,7 +99,7 @@ protected function outputResult(SearchResult $result): void } $this->io->text($result->total . " Results:"); foreach ($result as $resource) { - $this->io->text($resource->getLocation()); + $this->io->text($resource->location); } $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); } diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php index 93782f7..47b3322 100644 --- a/src/Console/Command/Search.php +++ b/src/Console/Command/Search.php @@ -97,7 +97,7 @@ protected function outputResult( $this->io->section('Results (' . $result->total . ')'); foreach ($result as $resource) { - $this->io->text($resource->getLocation()); + $this->io->text($resource->location); } if (count($result->facetGroups) > 0) { diff --git a/src/Dto/Search/Query/MoreLikeThisQuery.php b/src/Dto/Search/Query/MoreLikeThisQuery.php index 1607bea..3b55d9e 100644 --- a/src/Dto/Search/Query/MoreLikeThisQuery.php +++ b/src/Dto/Search/Query/MoreLikeThisQuery.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Dto\Search\Query; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Search\Query\Filter\Filter; /** @@ -24,8 +25,7 @@ class MoreLikeThisQuery * which entries are similar. */ public function __construct( - public readonly string $location, - public readonly string $lang = '', + public readonly ResourceLocation $location, public readonly array $filter = [], public readonly int $limit = 5, public readonly array $fields = ['description', 'content'] diff --git a/src/Service/AbstractIndexer.php b/src/Service/AbstractIndexer.php index 097f5ae..7a28243 100644 --- a/src/Service/AbstractIndexer.php +++ b/src/Service/AbstractIndexer.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; @@ -26,7 +27,8 @@ public function __construct( protected function getKey(): string { - return $this->indexName->name('') . '-' . $this->source; + return $this->indexName->name(ResourceLanguage::default()) . + '-' . $this->source; } protected function getConfig(): IndexerConfiguration @@ -70,9 +72,6 @@ public function enabled(): bool return $this->configLoader->exists($this->source); } - /** - * @inheritDoc - */ abstract public function index(): IndexerStatus; /** diff --git a/src/Service/IndexName.php b/src/Service/IndexName.php index fd6a7f3..2fe9163 100644 --- a/src/Service/IndexName.php +++ b/src/Service/IndexName.php @@ -4,9 +4,11 @@ namespace Atoolo\Search\Service; +use Atoolo\Resource\ResourceLanguage; + interface IndexName { - public function name(string $lang): string; + public function name(ResourceLanguage $lang): string; /** * The returned list contains the default index name and the index diff --git a/src/Service/Indexer/BackgroundIndexer.php b/src/Service/Indexer/BackgroundIndexer.php index 417acec..7feb834 100644 --- a/src/Service/Indexer/BackgroundIndexer.php +++ b/src/Service/Indexer/BackgroundIndexer.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; use Atoolo\Search\Service\IndexName; @@ -79,6 +80,7 @@ public function getStatus(): IndexerStatus private function getStatusStoreKey(): string { - return $this->index->name('') . '-' . $this->indexer->getSource(); + return $this->index->name(ResourceLanguage::default()) . + '-' . $this->indexer->getSource(); } } diff --git a/src/Service/Indexer/IndexDocumentDumper.php b/src/Service/Indexer/IndexDocumentDumper.php index dd51daf..d98860c 100644 --- a/src/Service/Indexer/IndexDocumentDumper.php +++ b/src/Service/Indexer/IndexDocumentDumper.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\ResourceLoader; +use Atoolo\Resource\ResourceLocation; class IndexDocumentDumper { @@ -27,7 +28,8 @@ public function dump(array $paths): array { $documents = []; foreach ($paths as $path) { - $resource = $this->resourceLoader->load($path); + $location = ResourceLocation::of($path); + $resource = $this->resourceLoader->load($location); $doc = new IndexSchema2xDocument(); $processId = 'process-id'; diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php index 009238a..fe9dfb9 100644 --- a/src/Service/Indexer/IndexerConfigurationLoader.php +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -5,14 +5,14 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\DataBag; -use Atoolo\Resource\ResourceBaseLocator; +use Atoolo\Resource\ResourceChannel; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; use RuntimeException; class IndexerConfigurationLoader { public function __construct( - private readonly ResourceBaseLocator $resourceBaseLocator + private readonly ResourceChannel $resourceChannel ) { } @@ -21,10 +21,7 @@ public function __construct( */ public function loadAll(): array { - $dir = $this->resourceBaseLocator->locate() . '/indexer'; - if (!is_dir($dir)) { - $dir = $this->resourceBaseLocator->locate() . '/../configs/indexer'; - } + $dir = $this->resourceChannel->resourceDir . '/indexer'; if (!is_dir($dir)) { return []; } @@ -41,13 +38,8 @@ public function loadAll(): array private function getFile(string $source): string { - $file = $this->resourceBaseLocator->locate() . + return $this->resourceChannel->resourceDir . '/indexer/' . $source . '.php'; - if (file_exists($file)) { - return $file; - } - return $this->resourceBaseLocator->locate() . - '/../configs/indexer/' . $source . '.php'; } public function exists(string $source): bool diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index 56df104..f531f13 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Dto\Indexer\IndexerStatusState; use Atoolo\Search\Service\IndexName; @@ -45,6 +46,9 @@ public function prepare(string $message): void ); } + /** + * @throws ExceptionInterface + */ public function start(int $total): void { $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); @@ -159,16 +163,18 @@ public function abort(): void $this->status->state = IndexerStatusState::ABORTED; } + /** + * @throws ExceptionInterface + */ public function getStatus(): IndexerStatus { - if ($this->status !== null) { - return $this->status; - } - return $this->statusStore->load($this->getStatusStoreKey()); + return $this->status + ?? $this->statusStore->load($this->getStatusStoreKey()); } private function getStatusStoreKey(): string { - return $this->index->name('') . '-' . $this->source; + return $this->index->name(ResourceLanguage::default()) . + '-' . $this->source; } } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index a372961..9868707 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -5,7 +5,9 @@ namespace Atoolo\Search\Service\Indexer; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Resource\ResourceLoader; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; use Atoolo\Search\Indexer; @@ -137,7 +139,7 @@ public function index(): IndexerStatus try { $paths = $this->finder->findAll(); - $this->deleteErrorProtocol($this->getBaseIndex()); + $this->deleteErrorProtocol(); $total = count($paths); $this->progressHandler->start($total); @@ -193,7 +195,7 @@ private function loadIndexerParameter(): IndexerParameter private function getBaseIndex(): string { - return $this->indexService->getIndex(''); + return $this->indexService->getIndex(ResourceLanguage::default()); } /** @@ -230,23 +232,22 @@ private function indexTranslationSplittedResources( $processId = uniqid('', true); - $index = $this->indexService->getIndex(''); + $index = $this->indexService->getIndex(ResourceLanguage::default()); $this->indexResourcesPerLanguageIndex( $processId, $parameter, - '', + ResourceLanguage::default(), $index, $splitterResult->getBases() ); - foreach ($splitterResult->getLocales() as $locale) { - $lang = substr($locale, 0, 2); + foreach ($splitterResult->getLanguages() as $lang) { $langIndex = $this->indexService->getIndex($lang); if ($index === $langIndex) { $this->handleError( - 'No Index for language "' . $lang . '" ' . + 'No Index for language "' . $lang->code . '" ' . 'found (base index: "' . $index . '")' ); continue; @@ -257,7 +258,7 @@ private function indexTranslationSplittedResources( $parameter, $lang, $langIndex, - $splitterResult->getTranslations($locale) + $splitterResult->getTranslations($lang) ); } } @@ -265,17 +266,17 @@ private function indexTranslationSplittedResources( /** * The resources for a language are indexed here. * - * @param string[] $pathList + * @param ResourceLocation[] $locations */ private function indexResourcesPerLanguageIndex( string $processId, IndexerParameter $parameter, - string $lang, + ResourceLanguage $lang, string $index, - array $pathList + array $locations ): void { - if (empty($pathList)) { + if (empty($locations)) { return; } @@ -293,7 +294,7 @@ private function indexResourcesPerLanguageIndex( $processId, $lang, $index, - $pathList, + $locations, $offset, $parameter->chunkSize ); @@ -325,18 +326,18 @@ private function indexResourcesPerLanguageIndex( * methods accept a chunk with all paths that are to be indexed via a * request. * - * @param string[] $pathList + * @param ResourceLocation[] $locations */ private function indexChunks( string $processId, - string $lang, + ResourceLanguage $lang, string $index, - array $pathList, + array $locations, int $offset, int $length ): int|false { $resourceList = $this->loadResources( - $pathList, + $locations, $offset, $length ); @@ -363,16 +364,16 @@ private function indexChunks( } /** - * @param string[] $pathList + * @param ResourceLocation[] $locations * @return Resource[]|false */ private function loadResources( - array $pathList, + array $locations, int $offset, int $length ): array|false { - $maxLength = count($pathList) - $offset; + $maxLength = count($locations) - $offset; if ($maxLength <= 0) { return false; } @@ -381,9 +382,9 @@ private function loadResources( $resourceList = []; for ($i = $offset; $i < $end; $i++) { - $path = $pathList[$i]; + $location = $locations[$i]; try { - $resource = $this->resourceLoader->load($path); + $resource = $this->resourceLoader->load($location); $resourceList[] = $resource; } catch (Throwable $e) { $this->handleError($e); @@ -396,7 +397,7 @@ private function loadResources( * @param array $resources */ private function add( - string $lang, + ResourceLanguage $lang, string $processId, array $resources ): UpdateResult { @@ -443,10 +444,10 @@ private function handleError(Throwable|string $error): void ); } - private function deleteErrorProtocol(string $core): void + private function deleteErrorProtocol(): void { $this->indexService->deleteByQuery( - $core, + ResourceLanguage::default(), 'crawl_status:error OR crawl_status:warning' ); } diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php index c03b3de..7adae22 100644 --- a/src/Service/Indexer/LocationFinder.php +++ b/src/Service/Indexer/LocationFinder.php @@ -4,7 +4,7 @@ namespace Atoolo\Search\Service\Indexer; -use Atoolo\Resource\ResourceBaseLocator; +use Atoolo\Resource\ResourceChannel; use Symfony\Component\Finder\Finder; /** @@ -20,7 +20,7 @@ class LocationFinder { public function __construct( - private readonly ResourceBaseLocator $baseLocator + private readonly ResourceChannel $resourceChannel ) { } @@ -93,7 +93,7 @@ public function findPaths(array $paths): array private function getBasePath(): string { - return rtrim($this->baseLocator->locate(), '/'); + return rtrim($this->resourceChannel->resourceDir, '/'); } private function toRelativePath(string $path): string diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 0d130de..08af41e 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -59,37 +59,36 @@ public function enrichDocument( string $processId ): IndexDocument { - $data = $resource->getData(); - $init = new DataBag($data->getAssociativeArray('init')); + $data = $resource->data; $base = new DataBag($data->getAssociativeArray('base')); $metadata = new DataBag($data->getAssociativeArray('metadata')); - $doc->sp_id = $resource->getId(); - $doc->sp_name = $resource->getName(); - $doc->sp_anchor = $init->getString('anchor'); + $doc->sp_id = $resource->id; + $doc->sp_name = $resource->name; + $doc->sp_anchor = $data->getString('anchor'); $doc->title = $base->getString('title'); $doc->description = $metadata->getString('description'); if (empty($doc->description)) { - $doc->description = $resource->getData()->getString( + $doc->description = $resource->data->getString( 'metadata.intro' ); } - $doc->sp_objecttype = $resource->getObjectType(); + $doc->sp_objecttype = $resource->objectType; $doc->sp_canonical = true; $doc->crawl_process_id = $processId; - $url = $init->getString('mediaUrl') - ?: $init->getString('url'); + $url = $data->getString('mediaUrl') + ?: $data->getString('url'); $doc->id = $url; $doc->url = $url; /** @var string[] $spContentType */ - $spContentType = [$resource->getObjectType()]; - if ($init->getBool('media') !== true) { + $spContentType = [$resource->objectType]; + if ($data->getBool('media') !== true) { $spContentType[] = 'article'; } - $contentSectionTypes = $init->getArray('contentSectionTypes'); + $contentSectionTypes = $data->getArray('contentSectionTypes'); $spContentType = array_merge($spContentType, $contentSectionTypes); if ($base->has('teaser.image')) { @@ -112,10 +111,10 @@ public function enrichDocument( $doc->meta_content_language = $lang; $doc->sp_changed = $this->toDateTime( - $init->getInt('changed') + $data->getInt('changed') ); $doc->sp_generated = $this->toDateTime( - $init->getInt('generated') + $data->getInt('generated') ); $doc->sp_date = $this->toDateTime( $base->getInt('date') @@ -146,11 +145,11 @@ public function enrichDocument( $sites = $this->getParentSiteGroupIdList($resource); $navigationRoot = $this->navigationLoader->loadRoot( - $resource->getLocation() + $resource->toLocation() ); - $siteGroupId = $navigationRoot->getData()->getInt( - 'init.siteGroup.id' + $siteGroupId = $navigationRoot->data->getInt( + 'siteGroup.id' ); if ($siteGroupId !== 0) { $sites[] = (string)$siteGroupId; @@ -158,7 +157,7 @@ public function enrichDocument( $doc->sp_site = array_unique($sites); } catch (Exception $e) { throw new DocumentEnrichingException( - $resource->getLocation(), + $resource->location, 'Unable to set sp_site: ' . $e->getMessage(), 0, $e @@ -192,7 +191,7 @@ public function enrichDocument( } /** @var array $groupPath */ - $groupPath = $init->getArray('groupPath'); + $groupPath = $data->getArray('groupPath'); $groupPathAsIdList = []; foreach ($groupPath as $group) { $groupPathAsIdList[] = $group['id']; @@ -231,10 +230,10 @@ public function enrichDocument( ); $doc->meta_content_type = $contentType; - $accessType = $init->getString('access.type'); + $accessType = $data->getString('access.type'); /** @var string[] $groups */ - $groups = $init->getArray('access.groups'); + $groups = $data->getArray('access.groups'); if ($accessType === 'allow' && !empty($groups)) { $doc->include_groups = array_map( @@ -267,20 +266,20 @@ private function enrichContent( ): IndexDocument { $content = []; - $content[] = $resource->getData()->getString( + $content[] = $resource->data->getString( 'searchindexdata.content' ); $content[] = $this->contentCollector->collect( - $resource->getData()->getArray('content') + $resource->data->getArray('content') ); /** @var ContactPoint $contactPoint */ - $contactPoint = $resource->getData()->getArray('metadata.contactPoint'); + $contactPoint = $resource->data->getArray('metadata.contactPoint'); $content[] = $this->contactPointToContent($contactPoint); /** @var array $categories */ - $categories = $resource->getData()->getArray('metadata.categories'); + $categories = $resource->data->getArray('metadata.categories'); foreach ($categories as $category) { $content[] = $category['name'] ?? ''; } @@ -345,13 +344,13 @@ private function idWithoutSignature(string $id): int private function getLocaleFromResource(Resource $resource): string { - $locale = $resource->getData()->getString('init.locale'); + $locale = $resource->data->getString('locale'); if ($locale !== '') { return $locale; } /** @var array $groupPath */ - $groupPath = $resource->getData()->getArray('init.groupPath'); + $groupPath = $resource->data->getArray('groupPath'); if (!empty($groupPath)) { $len = count($groupPath); for ($i = $len - 1; $i >= 0; $i--) { @@ -412,7 +411,7 @@ private function getParentSiteGroupIdList(Resource $resource): array */ private function getNavigationParents(Resource $resource): array { - return $resource->getData()->getAssociativeArray( + return $resource->data->getAssociativeArray( 'base.trees.navigation.parents' ); } diff --git a/src/Service/Indexer/SiteKit/NoIndexFilter.php b/src/Service/Indexer/SiteKit/NoIndexFilter.php index da5e62d..e6672fe 100644 --- a/src/Service/Indexer/SiteKit/NoIndexFilter.php +++ b/src/Service/Indexer/SiteKit/NoIndexFilter.php @@ -11,7 +11,7 @@ class NoIndexFilter implements ResourceFilter { public function accept(Resource $resource): bool { - $noIndex = $resource->getData()->getBool('init.noIndex'); + $noIndex = $resource->data->getBool('noIndex'); return $noIndex !== true; } } diff --git a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php index ca3b6d9..7aa2205 100644 --- a/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php +++ b/src/Service/Indexer/SiteKit/SubDirTranslationSplitter.php @@ -4,6 +4,8 @@ namespace Atoolo\Search\Service\Indexer\SiteKit; +use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Service\Indexer\TranslationSplitter; use Atoolo\Search\Service\Indexer\TranslationSplitterResult; @@ -22,34 +24,42 @@ public function split(array $pathList): TranslationSplitterResult $bases = []; $translations = []; foreach ($pathList as $path) { - $normalizedPath = $this->normalizePath($path); - if (empty($normalizedPath)) { + $location = $this->toResourceLocation($path); + if ($location === null) { continue; } - $locale = $this->extractLocaleFromPath($normalizedPath); - if ($locale === 'default') { - $bases[] = $normalizedPath; + if ($location->lang === ResourceLanguage::default()) { + $bases[] = $location; continue; } - if (!isset($translations[$locale])) { - $translations[$locale] = []; + if (!isset($translations[$location->lang->code])) { + $translations[$location->lang->code] = []; } - $translations[$locale][] = $normalizedPath; + $translations[$location->lang->code][] = $location; } return new TranslationSplitterResult($bases, $translations); } - private function extractLocaleFromPath(string $path): string + private function toResourceLocation(string $path): ?ResourceLocation { - $filename = basename($path); - $parentDirName = basename(dirname($path)); + $normalizedPath = $this->normalizePath($path); + if (empty($normalizedPath)) { + return null; + } - if (!str_ends_with($parentDirName, '.php.translations')) { - return 'default'; + $pos = strrpos($normalizedPath, '.php.translations'); + if ($pos === false) { + return ResourceLocation::of($normalizedPath); } - return basename($filename, '.php'); + $localeFilename = basename($normalizedPath); + $locale = basename($localeFilename, '.php'); + + return ResourceLocation::of( + substr($normalizedPath, 0, $pos + 4), + ResourceLanguage::of($locale) + ); } /** diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php index c6fae66..0f18227 100644 --- a/src/Service/Indexer/SolrIndexService.php +++ b/src/Service/Indexer/SolrIndexService.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; use Solarium\Client; @@ -16,12 +17,12 @@ public function __construct( ) { } - public function getIndex(string $lang): string + public function getIndex(ResourceLanguage $lang): string { return $this->index->name($lang); } - public function updater(string $lang): SolrIndexUpdater + public function updater(ResourceLanguage $lang): SolrIndexUpdater { $client = $this->createClient($lang); $update = $client->createUpdate(); @@ -31,7 +32,7 @@ public function updater(string $lang): SolrIndexUpdater } public function deleteExcludingProcessId( - string $lang, + ResourceLanguage $lang, string $source, string $processId ): void { @@ -65,7 +66,7 @@ public function deleteByQueryForAllLanguages(string $query): void } } - public function deleteByQuery(string $lang, string $query): void + public function deleteByQuery(ResourceLanguage $lang, string $query): void { $client = $this->createClient($lang); $update = $client->createUpdate(); @@ -73,7 +74,7 @@ public function deleteByQuery(string $lang, string $query): void $client->update($update); } - public function commit(string $lang): void + public function commit(ResourceLanguage $lang): void { $client = $this->createClient($lang); $update = $client->createUpdate(); @@ -98,7 +99,7 @@ public function commitForAllLanguages(): void */ public function getManagedIndices(): array { - $client = $this->createClient(''); + $client = $this->createClient(ResourceLanguage::default()); $coreAdminQuery = $client->createCoreAdmin(); $statusAction = $coreAdminQuery->createStatus(); $coreAdminQuery->setAction($statusAction); @@ -118,7 +119,7 @@ public function getManagedIndices(): array return $managedIndexes; } - private function createClient(string $lang): Client + private function createClient(ResourceLanguage $lang): Client { return $this->clientFactory->create($this->index->name($lang)); } diff --git a/src/Service/Indexer/TranslationSplitterResult.php b/src/Service/Indexer/TranslationSplitterResult.php index 8fe13a2..7d784a2 100644 --- a/src/Service/Indexer/TranslationSplitterResult.php +++ b/src/Service/Indexer/TranslationSplitterResult.php @@ -4,11 +4,14 @@ namespace Atoolo\Search\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; + class TranslationSplitterResult { /** - * @param string[] $bases - * @param array> $translations + * @param ResourceLocation[] $bases + * @param array> $translations */ public function __construct( private readonly array $bases, @@ -17,28 +20,32 @@ public function __construct( } /** - * @return string[] + * @return ResourceLanguage[] */ - public function getLocales(): array + public function getLanguages(): array { - $locales = array_keys($this->translations); - sort($locales); - return $locales; + $languages = array_keys($this->translations); + sort($languages); + return array_map( + static fn($lang) + => ResourceLanguage::of($lang), + $languages + ); } /** - * @return string[] + * @return ResourceLocation[] */ public function getBases(): array { return $this->bases; } - /** - * @return string[] + /** + * @return ResourceLocation[] */ - public function getTranslations(string $locale): array + public function getTranslations(ResourceLanguage $lang): array { - return $this->translations[$locale] ?? []; + return $this->translations[$lang->code] ?? []; } } diff --git a/src/Service/ResourceChannelBasedIndexName.php b/src/Service/ResourceChannelBasedIndexName.php index d99a4de..f481dcc 100644 --- a/src/Service/ResourceChannelBasedIndexName.php +++ b/src/Service/ResourceChannelBasedIndexName.php @@ -5,24 +5,22 @@ namespace Atoolo\Search\Service; use Atoolo\Resource\ResourceChannel; -use Atoolo\Resource\ResourceChannelFactory; +use Atoolo\Resource\ResourceLanguage; class ResourceChannelBasedIndexName implements IndexName { public function __construct( - private readonly ResourceChannelFactory $resourceChannelFactory + private readonly ResourceChannel $resourceChannel ) { } - public function name(string $lang): string + public function name(ResourceLanguage $lang): string { - $resourceChannel = $this->resourceChannelFactory->create(); - - $locale = $this->langToAvailableLocale($resourceChannel, $lang); + $locale = $this->langToAvailableLocale($this->resourceChannel, $lang); if (empty($locale)) { - return $resourceChannel->searchIndex; + return $this->resourceChannel->searchIndex; } - return $resourceChannel->searchIndex . '-' . $locale; + return $this->resourceChannel->searchIndex . '-' . $locale; } /** @@ -33,29 +31,28 @@ public function name(string $lang): string */ public function names(): array { - $resourceChannel = $this->resourceChannelFactory->create(); - $names = [$resourceChannel->searchIndex]; + $names = [$this->resourceChannel->searchIndex]; foreach ( - $resourceChannel->translationLocales as $locale + $this->resourceChannel->translationLocales as $locale ) { - $names[] = $resourceChannel->searchIndex . '-' . $locale; + $names[] = $this->resourceChannel->searchIndex . '-' . $locale; } return $names; } private function langToAvailableLocale( ResourceChannel $resourceChannel, - string $lang + ResourceLanguage $lang ): string { - if (empty($lang)) { - return $lang; + if ($lang === ResourceLanguage::default()) { + return ''; } foreach ( $resourceChannel->translationLocales as $availableLocale ) { - if (str_starts_with($availableLocale, $lang)) { + if (str_starts_with($availableLocale, $lang->code)) { return $availableLocale; } } diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php index 0b6450d..892b6cb 100644 --- a/src/Service/Search/ExternalResourceFactory.php +++ b/src/Service/Search/ExternalResourceFactory.php @@ -4,7 +4,9 @@ namespace Atoolo\Search\Service\Search; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use LogicException; use Solarium\QueryType\Select\Result\Document; @@ -17,7 +19,7 @@ */ class ExternalResourceFactory implements ResourceFactory { - public function accept(Document $document): bool + public function accept(Document $document, ResourceLanguage $lang): bool { $location = $this->getField($document, 'url'); if ($location === null) { @@ -29,7 +31,7 @@ public function accept(Document $document): bool ); } - public function create(Document $document, string $lang): Resource + public function create(Document $document, ResourceLanguage $lang): Resource { $location = $this->getField($document, 'url'); if ($location === null) { @@ -41,8 +43,10 @@ public function create(Document $document, string $lang): Resource id: $this->getField($document, 'sp_id') ?? '', name: $this->getField($document, 'title') ?? '', objectType: 'external', - lang: $this->getField($document, 'meta_content_language') ?? '', - data: [], + lang: ResourceLanguage::of( + $this->getField($document, 'meta_content_language') + ), + data: new DataBag([]), ); } diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php index 4e4e01a..ee0e3a5 100644 --- a/src/Service/Search/InternalMediaResourceFactory.php +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -5,7 +5,9 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Resource\ResourceLoader; +use Atoolo\Resource\ResourceLocation; use LogicException; use Solarium\QueryType\Select\Result\Document; @@ -23,31 +25,36 @@ public function __construct( ) { } - public function accept(Document $document): bool + public function accept(Document $document, ResourceLanguage $lang): bool { - $metaLocation = $this->getMetaLocation($document); + $metaLocation = $this->getMetaLocation($document, $lang); if ($metaLocation === null) { return false; } return $this->resourceLoader->exists($metaLocation); } - public function create(Document $document, string $lang): Resource + public function create(Document $document, ResourceLanguage $lang): Resource { - $metaLocation = $this->getMetaLocation($document); + $metaLocation = $this->getMetaLocation($document, $lang); if ($metaLocation === null) { throw new LogicException('document should contain an url'); } - return $this->resourceLoader->load($metaLocation, $lang); + return $this->resourceLoader->load($metaLocation); } - private function getMetaLocation(Document $document): ?string - { - $location = $this->getField($document, 'url'); - if ($location === null) { + private function getMetaLocation( + Document $document, + ResourceLanguage $lang + ): ?ResourceLocation { + $url = $this->getField($document, 'url'); + if ($url === null) { return null; } - return $location . '.meta.php'; + return ResourceLocation::of( + $url . '.meta.php', + $lang + ); } private function getField(Document $document, string $name): ?string diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php index 746647d..a02b6e9 100644 --- a/src/Service/Search/InternalResourceFactory.php +++ b/src/Service/Search/InternalResourceFactory.php @@ -5,7 +5,9 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Resource\ResourceLoader; +use Atoolo\Resource\ResourceLocation; use LogicException; use Solarium\QueryType\Select\Result\Document; @@ -21,26 +23,27 @@ public function __construct( ) { } - public function accept(Document $document): bool + public function accept(Document $document, ResourceLanguage $lang): bool { - $location = $this->getField($document, 'url'); + $location = $this->getUrl($document); if ($location === null) { return false; } return str_ends_with($location, '.php'); } - public function create(Document $document, string $lang): Resource + public function create(Document $document, ResourceLanguage $lang): Resource { - $location = $this->getField($document, 'url'); - if ($location === null) { + $url = $this->getUrl($document); + if ($url === null) { throw new LogicException('document should contain an url'); } - return $this->resourceLoader->load($location, $lang); + $location = ResourceLocation::of($url, $lang); + return $this->resourceLoader->load($location); } - private function getField(Document $document, string $name): ?string + private function getUrl(Document $document): ?string { - return $document->getFields()[$name] ?? null; + return $document->getFields()['url'] ?? null; } } diff --git a/src/Service/Search/ResourceFactory.php b/src/Service/Search/ResourceFactory.php index e1e7dc8..3738025 100644 --- a/src/Service/Search/ResourceFactory.php +++ b/src/Service/Search/ResourceFactory.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Solarium\QueryType\Select\Result\Document; /** @@ -22,6 +23,9 @@ */ interface ResourceFactory { - public function accept(Document $document): bool; - public function create(Document $document, string $lang): Resource; + public function accept(Document $document, ResourceLanguage $lang): bool; + public function create( + Document $document, + ResourceLanguage $lang + ): Resource; } diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 7b48f2f..6987b84 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Search; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Dto\Search\Result\SearchResult; use Atoolo\Search\MoreLikeThis; @@ -27,12 +28,12 @@ public function __construct( public function moreLikeThis(MoreLikeThisQuery $query): SearchResult { - $index = $this->index->name($query->lang); + $index = $this->index->name($query->location->lang); $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SolrMoreLikeThisResult $result */ $result = $client->execute($solrQuery); - return $this->buildResult($result, $query->lang); + return $this->buildResult($result, $query->location->lang); } private function buildSolrQuery( @@ -62,7 +63,7 @@ private function buildSolrQuery( private function buildResult( SolrMoreLikeThisResult $result, - string $lang + ResourceLanguage $lang ): SearchResult { $resourceList = $this->resultToResourceResolver diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index c7f1b33..cec0978 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -5,7 +5,9 @@ namespace Atoolo\Search\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Exception\MissMatchingResourceFactoryException; +use Exception; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Solarium\QueryType\Select\Result\Document; @@ -30,25 +32,29 @@ public function __construct( /** * @return array */ - public function loadResourceList(SelectResult $result, string $lang): array - { + public function loadResourceList( + SelectResult $result, + ResourceLanguage $lang + ): array { $resourceList = []; /** @var Document $document */ foreach ($result as $document) { try { $resourceList[] = $this->loadResource($document, $lang); - } catch (\Exception $e) { + } catch (Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); } } return $resourceList; } - private function loadResource(Document $document, string $lang): Resource - { + private function loadResource( + Document $document, + ResourceLanguage $lang + ): Resource { foreach ($this->resourceFactoryList as $resourceFactory) { - if ($resourceFactory->accept($document)) { + if ($resourceFactory->accept($document, $lang)) { return $resourceFactory->create($document, $lang); } } diff --git a/src/Service/Search/SolrSearch.php b/src/Service/Search/SolrSearch.php index 7b2d13d..07a06e8 100644 --- a/src/Service/Search/SolrSearch.php +++ b/src/Service/Search/SolrSearch.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Search; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Search\Query\Facet\FacetField; use Atoolo\Search\Dto\Search\Query\Facet\FacetMultiQuery; use Atoolo\Search\Dto\Search\Query\Facet\FacetQuery; @@ -46,13 +47,14 @@ public function __construct( public function search(SearchQuery $query): SearchResult { - $index = $this->index->name($query->lang); + $lang = ResourceLanguage::of($query->lang); + $index = $this->index->name($lang); $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); /** @var SelectResult $result */ $result = $client->execute($solrQuery); - return $this->buildResult($query, $result, $query->lang); + return $this->buildResult($query, $result, $lang); } private function buildSolrQuery( @@ -247,7 +249,7 @@ private function addFacetMultiQueryToSolrQuery( private function buildResult( SearchQuery $query, SelectResult $result, - string $lang + ResourceLanguage $lang ): SearchResult { $resourceList = $this->resultToResourceResolver diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php index c86328b..1557590 100644 --- a/src/Service/Search/SolrSuggest.php +++ b/src/Service/Search/SolrSuggest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Service\Search; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Dto\Search\Query\SuggestQuery; use Atoolo\Search\Dto\Search\Result\Suggestion; use Atoolo\Search\Dto\Search\Result\SuggestResult; @@ -40,7 +41,8 @@ public function __construct( */ public function suggest(SuggestQuery $query): SuggestResult { - $index = $this->index->name($query->lang); + $lang = ResourceLanguage::of($query->lang); + $index = $this->index->name($lang); $client = $this->clientFactory->create($index); $solrQuery = $this->buildSolrQuery($client, $query); diff --git a/src/Service/ServerVarSolrClientFactory.php b/src/Service/ServerVarSolrClientFactory.php deleted file mode 100644 index 44ec30b..0000000 --- a/src/Service/ServerVarSolrClientFactory.php +++ /dev/null @@ -1,83 +0,0 @@ -setTimeout($this->timeoutInSeconds); - /* - $adapter->setProxy($this->proxy); - */ - $eventDispatcher = new EventDispatcher(); - $config = [ - 'endpoint' => $this->getEndpointConfig($core) - ]; - - // create a client instance - return new Client( - $adapter, - $eventDispatcher, - $config - ); - } - - /** - * @return array - */ - private function getEndpointConfig(string $core): array - { - $url = $_SERVER['SOLR_URL'] ?? ''; - - if (!empty($url)) { - $url = parse_url($url); - $scheme = $url['scheme'] ?? 'http'; - $host = $url['host'] ?? 'localhost'; - $port = (string)( - $url['port'] ?? - ( - $scheme === 'https' ? - '443' : - self::IES_WEBNODE_SOLR_PORT - ) - ); - $path = $url['path'] ?? ''; - } else { - $scheme = $_SERVER['SOLR_SCHEME'] ?? 'http'; - $host = (string)($_SERVER['SOLR_HOST'] ?? 'localhost'); - $port = $_SERVER['SOLR_PORT'] ?? self::IES_WEBNODE_SOLR_PORT; - $path = ''; - } - - return [ - $host => [ - 'scheme' => $scheme, - 'host' => $host, - 'port' => $port, - 'path' => $path, - 'core' => $core - ] - ]; - } -} diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php index 63d54fc..95de790 100644 --- a/test/Console/Command/DumpIndexDocumentTest.php +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -33,6 +33,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); diff --git a/test/Console/Command/IndexerInternalResourceUpdateTest.php b/test/Console/Command/IndexerInternalResourceUpdateTest.php index 71f0bf7..1777c14 100644 --- a/test/Console/Command/IndexerInternalResourceUpdateTest.php +++ b/test/Console/Command/IndexerInternalResourceUpdateTest.php @@ -36,6 +36,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php index 756ebd8..8f596e1 100644 --- a/test/Console/Command/IndexerTest.php +++ b/test/Console/Command/IndexerTest.php @@ -36,6 +36,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php index d4cf095..4c2517d 100644 --- a/test/Console/Command/MoreLikeThisTest.php +++ b/test/Console/Command/MoreLikeThisTest.php @@ -4,9 +4,11 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceChannel; use Atoolo\Resource\ResourceChannelFactory; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\MoreLikeThis; use Atoolo\Search\Dto\Search\Result\SearchResult; @@ -38,6 +40,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); @@ -47,9 +50,14 @@ public function setUp(): void ); $resourceChannelFactory->method('create') ->willReturn($resourceChannel); - $resultResource = $this->createStub(Resource::class); - $resultResource->method('getLocation') - ->willReturn('/test2.php'); + $resultResource = new Resource( + '/test2.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); $result = new SearchResult( 1, 1, @@ -74,9 +82,14 @@ public function setUp(): void public function testExecute(): void { - $resultResource = $this->createStub(Resource::class); - $resultResource->method('getLocation') - ->willReturn('/test2.php'); + $resultResource = new Resource( + '/test2.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); $result = new SearchResult( 1, 1, diff --git a/test/Console/Command/SearchTest.php b/test/Console/Command/SearchTest.php index 99ceb7f..88110b3 100644 --- a/test/Console/Command/SearchTest.php +++ b/test/Console/Command/SearchTest.php @@ -4,9 +4,11 @@ namespace Atoolo\Search\Test\Console\Command; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceChannel; use Atoolo\Resource\ResourceChannelFactory; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Console\Application; use Atoolo\Search\Console\Command\Search; use Atoolo\Search\Dto\Search\Result\Facet; @@ -39,6 +41,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); @@ -61,9 +64,14 @@ public function setUp(): void public function testExecute(): void { - $resultResource = $this->createStub(Resource::class); - $resultResource->method('getLocation') - ->willReturn('/test.php'); + $resultResource = new Resource( + '/test.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); $result = new SearchResult( 1, 1, diff --git a/test/Console/Command/SuggestTest.php b/test/Console/Command/SuggestTest.php index 2f615e4..16ed18d 100644 --- a/test/Console/Command/SuggestTest.php +++ b/test/Console/Command/SuggestTest.php @@ -38,6 +38,7 @@ public function setUp(): void '', '', '', + '', 'test', [] ); diff --git a/test/Service/Indexer/IndexerConfigurationLoaderTest.php b/test/Service/Indexer/IndexerConfigurationLoaderTest.php index 5b7cea8..c61f907 100644 --- a/test/Service/Indexer/IndexerConfigurationLoaderTest.php +++ b/test/Service/Indexer/IndexerConfigurationLoaderTest.php @@ -5,11 +5,10 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Resource\DataBag; -use Atoolo\Resource\ResourceBaseLocator; +use Atoolo\Resource\ResourceChannel; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; use Atoolo\Search\Service\Indexer\IndexerConfigurationLoader; use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use RuntimeException; @@ -18,36 +17,26 @@ class IndexerConfigurationLoaderTest extends TestCase { private const RESOURCE_BASE = __DIR__ . '/../../resources/' . 'Service/Indexer/IndexerConfigurationLoader'; - private IndexerConfigurationLoader $loader; - private ResourceBaseLocator&Stub $resourceBaseLocator; - - public function setUp(): void - { - $this->resourceBaseLocator = $this->createStub( - ResourceBaseLocator::class - ); - $this->loader = new IndexerConfigurationLoader( - $this->resourceBaseLocator - ); - } public function testExists(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/with-internal'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); $this->assertTrue( - $this->loader->exists('internal'), + $loader->exists('internal'), 'Internal config should exist' ); } public function testLoad(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/with-internal'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); - $config = $this->loader->load('internal'); + $config = $loader->load('internal'); $expected = new IndexerConfiguration( 'internal', @@ -67,10 +56,11 @@ public function testLoad(): void public function testLoadNotExists(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/with-internal'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); - $config = $this->loader->load('not-exists'); + $config = $loader->load('not-exists'); $expected = new IndexerConfiguration( 'not-exists', @@ -88,8 +78,9 @@ public function testLoadNotExists(): void public function testLoadAll(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/with-internal'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); $expected = new IndexerConfiguration( 'internal', @@ -102,29 +93,50 @@ public function testLoadAll(): void $this->assertEquals( [$expected], - $this->loader->loadAll(), + $loader->loadAll(), 'unexpected config' ); } public function testLoadAllNotADirectory(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/not-a-directory'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/not-a-directory' + ); $this->assertEquals( [], - $this->loader->loadAll(), + $loader->loadAll(), 'should return empty array' ); } public function testLoadAllWithConfigReturnString(): void { - $this->resourceBaseLocator->method('locate') - ->willReturn(self::RESOURCE_BASE . '/return-string'); + $loader = $this->createLoader( + self::RESOURCE_BASE . '/return-string' + ); $this->expectException(RuntimeException::class); - $this->loader->loadAll(); + $loader->loadAll(); + } + + private function createLoader( + string $resourceDir + ): IndexerConfigurationLoader { + $resourceChannel = new ResourceChannel( + '', + '', + '', + '', + false, + '', + '', + '', + $resourceDir, + '', + [] + ); + return new IndexerConfigurationLoader($resourceChannel); } } diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 214342b..7a435ab 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLoader; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexerConfigurationLoader; @@ -19,12 +20,13 @@ use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\Indexer\SolrIndexUpdater; use Atoolo\Search\Service\Indexer\TranslationSplitter; +use Exception; use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\MockObject\Exception; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use Psr\Log\LoggerInterface; +use RuntimeException; use Solarium\QueryType\Update\Result as UpdateResult; use Symfony\Component\Lock\LockFactory; use Symfony\Component\Lock\Store\SemaphoreStore; @@ -89,11 +91,15 @@ public function setUp(): void $this->translationSplitter = new SubDirTranslationSplitter(); $this->resourceLoader = $this->createStub(ResourceLoader::class); $this->resourceLoader->method('load') - ->willReturnCallback(function ($path) { - $resource = $this->createStub(Resource::class); - $resource->method('getLocation') - ->willReturn($path); - return $resource; + ->willReturnCallback(function ($location) { + return new Resource( + $location->location, + '', + '', + '', + $location->lang, + new DataBag([]) + ); }); $this->solrIndexService = $this->createMock(SolrIndexService::class); $this->updateResult = $this->createStub(UpdateResult::class); @@ -108,7 +114,7 @@ public function setUp(): void }); $this->solrIndexService->method('getIndex') ->willReturnCallback(function ($lang) { - if ($lang === 'en') { + if ($lang->code === 'en') { return 'test-en_US'; } return 'test'; @@ -202,8 +208,8 @@ public function testIndexAllWithChunks(): void $this->documentEnricher ->method('enrichDocument') ->willReturnCallback(function ($resource, $doc) { - if ($resource->getLocation() === '/a/error.php') { - throw new \Exception('test'); + if ($resource->location === '/a/error.php') { + throw new RuntimeException('test'); } return $doc; }); @@ -233,8 +239,7 @@ public function testIndexSkipResource(): void $this->indexerFilter->method('accept') ->willReturnCallback(function (Resource $resource) { - $location = $resource->getLocation(); - return ($location !== '/a/b.php'); + return ($resource->location !== '/a/b.php'); }); $this->updater->expects($this->exactly(1)) @@ -291,7 +296,11 @@ public function testWithInvalidResource(): void ]); $this->resourceLoader->method('load') - ->willThrowException(new InvalidResourceException('/a/b.php')); + ->willThrowException( + new InvalidResourceException( + ResourceLocation::of('/a/b.php') + ) + ); $this->indexerProgressHandler->expects($this->once()) ->method('error'); diff --git a/test/Service/Indexer/LocationFinderTest.php b/test/Service/Indexer/LocationFinderTest.php index 7acceca..10cc69b 100644 --- a/test/Service/Indexer/LocationFinderTest.php +++ b/test/Service/Indexer/LocationFinderTest.php @@ -3,6 +3,7 @@ namespace Atoolo\Search\Test\Service\Indexer; use Atoolo\Resource\Loader\StaticResourceBaseLocator; +use Atoolo\Resource\ResourceChannel; use Atoolo\Search\Service\Indexer\LocationFinder; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; @@ -14,15 +15,26 @@ class LocationFinderTest extends TestCase private LocationFinder $locationFinder; protected function setUp(): void { - $base = realpath( + $resourceDir = realpath( __DIR__ . '/../../resources/Service/Indexer/LocationFinder' ); - if ($base === false) { + if ($resourceDir === false) { throw new InvalidArgumentException('basepath not found'); } - $this->locationFinder = new LocationFinder( - new StaticResourceBaseLocator($base) + $resourceChannel = new ResourceChannel( + '', + '', + '', + '', + false, + '', + '', + '', + $resourceDir, + '', + [] ); + $this->locationFinder = new LocationFinder($resourceChannel); } public function testFindAll(): void diff --git a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php index de1cee6..214448f 100644 --- a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php +++ b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php @@ -8,12 +8,12 @@ use Atoolo\Resource\Exception\InvalidResourceException; use Atoolo\Resource\Loader\SiteKitNavigationHierarchyLoader; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Exception\DocumentEnrichingException; use Atoolo\Search\Service\Indexer\ContentCollector; use Atoolo\Search\Service\Indexer\IndexSchema2xDocument; use Atoolo\Search\Service\Indexer\SiteKit\DefaultSchema2xDocumentEnricher; use DateTime; -use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; class DefaultSchema2xDocumentEnricherTest extends TestCase @@ -28,12 +28,12 @@ public function setUp(): void $navigationLoader ->method('loadRoot') ->willReturnCallback(function ($location) { - if ($location === 'throwException') { + if ($location->location === 'throwException') { throw new InvalidResourceException($location); } - return $this->createResource(['init' => [ + return $this->createResource([ 'siteGroup' => ['id' => 999] - ]]); + ]); }); $contentCollector = $this->createStub(ContentCollector::class); $contentCollector @@ -48,24 +48,32 @@ public function setUp(): void public function testEnrichSpId(): void { - $resource = $this->createStub(Resource::class); - $resource->method('getId')->willReturn('123'); + $resource = new Resource( + '', + '123', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); $doc = $this->enrichWithResource($resource); $this->assertEquals('123', $doc->sp_id, 'unexpected id'); } public function testEnrichName(): void { - $resource = $this->createStub(Resource::class); - $resource->method('getName')->willReturn('test'); + $resource = $this->createResource([ + 'name' => 'test' + ]); $doc = $this->enrichWithResource($resource); $this->assertEquals('test', $doc->sp_name, 'unexpected name'); } public function testEnrichObjectType(): void { - $resource = $this->createStub(Resource::class); - $resource->method('getObjectType')->willReturn('test'); + $resource = $this->createResource([ + 'objectType' => 'test' + ]); $doc = $this->enrichWithResource($resource); $this->assertEquals( 'test', @@ -76,7 +84,7 @@ public function testEnrichObjectType(): void public function testEnrichAnchor(): void { - $doc = $this->enrichWithData(['init' => ['anchor' => 'abc']]); + $doc = $this->enrichWithData(['anchor' => 'abc']); $this->assertEquals('abc', $doc->sp_anchor, 'unexpected ancohr'); } @@ -107,7 +115,7 @@ public function testEnrichCanonical(): void public function testEnrichCrawlProcessId(): void { - $resource = $this->createStub(Resource::class); + $resource = $this->createResource([]); $doc = $this->enricher->enrichDocument( $resource, new IndexSchema2xDocument(), @@ -122,7 +130,7 @@ public function testEnrichCrawlProcessId(): void public function testEnrichId(): void { - $doc = $this->enrichWithData(['init' => ['url' => '/test.php']]); + $doc = $this->enrichWithData(['url' => '/test.php']); $this->assertEquals( '/test.php', $doc->id, @@ -132,7 +140,7 @@ public function testEnrichId(): void public function testEnrichUrl(): void { - $doc = $this->enrichWithData(['init' => ['url' => '/test.php']]); + $doc = $this->enrichWithData(['url' => '/test.php']); $this->assertEquals( '/test.php', $doc->url, @@ -142,7 +150,7 @@ public function testEnrichUrl(): void public function testEnrichMediaId(): void { - $doc = $this->enrichWithData(['init' => ['mediaUrl' => '/test.php']]); + $doc = $this->enrichWithData(['mediaUrl' => '/test.php']); $this->assertEquals( '/test.php', $doc->id, @@ -152,7 +160,7 @@ public function testEnrichMediaId(): void public function testEnrichMediaUrl(): void { - $doc = $this->enrichWithData(['init' => ['mediaUrl' => '/test.php']]); + $doc = $this->enrichWithData(['mediaUrl' => '/test.php']); $this->assertEquals( '/test.php', $doc->url, @@ -163,10 +171,8 @@ public function testEnrichMediaUrl(): void public function testEnrichSpContentType(): void { $doc = $this->enrichWithData([ - 'init' => [ - 'objectType' => 'content', - 'contentSectionTypes' => ['text', 'linkList'] - ], + 'objectType' => 'content', + 'contentSectionTypes' => ['text', 'linkList'], 'base' => [ 'teaser' => [ 'headline' => 'test', @@ -205,7 +211,7 @@ public function testEnrichDefaultLanguage(): void public function testEnrichLanguage(): void { - $doc = $this->enrichWithData(['init' => ['locale' => 'en_US']]); + $doc = $this->enrichWithData(['locale' => 'en_US']); $this->assertEquals( 'en', $doc->sp_language, @@ -215,7 +221,7 @@ public function testEnrichLanguage(): void public function testEnrichLanguageWithShortLocale(): void { - $doc = $this->enrichWithData(['init' => ['locale' => 'en']]); + $doc = $this->enrichWithData(['locale' => 'en']); $this->assertEquals( 'en', $doc->sp_language, @@ -225,12 +231,12 @@ public function testEnrichLanguageWithShortLocale(): void public function testEnrichLanguageOverGroupPath(): void { - $doc = $this->enrichWithData(['init' => [ + $doc = $this->enrichWithData([ 'groupPath' => [ ['id' => 1, 'locale' => 'fr_FR'], ['id' => 2, 'locale' => 'it_IT'] ] - ]]); + ]); $this->assertEquals( 'it', $doc->sp_language, @@ -250,7 +256,7 @@ public function testEnrichDefaultMetaContentLanguage(): void public function testEnrichChanged(): void { - $doc = $this->enrichWithData(['init' => ['changed' => 1708932236]]); + $doc = $this->enrichWithData(['changed' => 1708932236]); $expected = new DateTime(); $expected->setTimestamp(1708932236); @@ -263,7 +269,7 @@ public function testEnrichChanged(): void public function testEnrichGenerated(): void { - $doc = $this->enrichWithData(['init' => ['generated' => 1708932236]]); + $doc = $this->enrichWithData(['generated' => 1708932236]); $expected = new DateTime(); $expected->setTimestamp(1708932236); @@ -415,8 +421,9 @@ public function testEnrichSpSites(): void public function testEnrichSpSitesWithInvalidRootResource(): void { - $resource = $this->createResource([]); - $resource->method('getLocation')->willReturn('throwException'); + $resource = $this->createResource([ + 'url' => 'throwException' + ]); $this->expectException(DocumentEnrichingException::class); $this->enrichWithResource($resource); @@ -474,13 +481,13 @@ public function testEnrichCategoryPath(): void public function testEnrichSpGroup(): void { - $doc = $this->enrichWithData(['init' => [ + $doc = $this->enrichWithData([ 'groupPath' => [ ['id' => 1], ['id' => 2], ['id' => 3], ] - ]]); + ]); $this->assertEquals( 2, $doc->sp_group, @@ -490,13 +497,13 @@ public function testEnrichSpGroup(): void public function testEnrichSpGroupPath(): void { - $doc = $this->enrichWithData(['init' => [ + $doc = $this->enrichWithData([ 'groupPath' => [ ['id' => 1], ['id' => 2], ['id' => 3], ] - ]]); + ]); $this->assertEquals( [1, 2, 3], $doc->sp_group_path, @@ -529,7 +536,7 @@ public function testEnrichDateViaScheduling(): void public function testEnrichContentTypeViaScheduling(): void { $doc = $this->enrichWithData([ - 'init' => ['objectType' => 'content'], + 'objectType' => 'content', 'base' => ['date' => 1707549836], 'metadata' => [ 'scheduling' => [ @@ -572,8 +579,7 @@ public function testEnrichDateListViaScheduling(): void public function testEnrichDefaultMetaContentType(): void { - $doc = $this->enrichWithData([ - ]); + $doc = $this->enrichWithData([]); $this->assertEquals( 'text/html; charset=UTF-8', @@ -584,12 +590,12 @@ public function testEnrichDefaultMetaContentType(): void public function testEnrichIncludeGroups(): void { - $doc = $this->enrichWithData(['init' => [ + $doc = $this->enrichWithData([ 'access' => [ 'type' => 'allow', 'groups' => ['100010100000001028'] ] - ]]); + ]); $this->assertEquals( ['1028'], @@ -611,12 +617,12 @@ public function testEnrichIncludeAllGroups(): void public function testEnrichExcludeGroups(): void { - $doc = $this->enrichWithData(['init' => [ + $doc = $this->enrichWithData([ 'access' => [ 'type' => 'deny', 'groups' => ['100010100000001028'] ] - ]]); + ]); $this->assertEquals( ['1028'], @@ -651,8 +657,7 @@ public function testEnrichMetaContentType(): void public function testEnrichInternal(): void { - $doc = $this->enrichWithData([ - ]); + $doc = $this->enrichWithData([]); $this->assertEquals( ['internal'], @@ -754,14 +759,15 @@ private function enrichWithData( /** * @param array> $data */ - private function createResource(array $data): Resource&Stub + private function createResource(array $data): Resource { - $dataBag = new DataBag($data); - $resource = $this->createStub(Resource::class); - $resource->method('getData')->willReturn($dataBag); - $resource->method('getObjectType')->willReturn( - $data['init']['objectType'] ?? '' + return new Resource( + $data['url'] ?? '', + $data['id'] ?? '123', + $data['name'] ?? '', + $data['objectType'] ?? '', + ResourceLanguage::of($data['locale'] ?? ''), + new DataBag($data) ); - return $resource; } } diff --git a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php index 9e3927f..5dc31a9 100644 --- a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php +++ b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php @@ -4,7 +4,9 @@ namespace Atoolo\Search\Test\Service\Indexer\SiteKit; +use Atoolo\Resource\DataBag; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\Indexer\SiteKit\NoIndexFilter; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -19,8 +21,8 @@ public function testAccept(): void 'test', 'test', 'test', - '', - [] + ResourceLanguage::default(), + new DataBag([]) ); $filter = new NoIndexFilter(); $this->assertTrue( @@ -36,8 +38,8 @@ public function testAcceptWithNoIndex(): void 'test', 'test', 'test', - '', - ['init' => ['noIndex' => true]] + ResourceLanguage::default(), + new DataBag(['noIndex' => true]) ); $filter = new NoIndexFilter(); $this->assertFalse( diff --git a/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php index 3765878..a50a3d6 100644 --- a/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php +++ b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php @@ -2,6 +2,8 @@ namespace Atoolo\Search\Test\Service\Indexer\SiteKit; +use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Service\Indexer\SiteKit\SubDirTranslationSplitter; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -21,7 +23,7 @@ public function testBases(): void ); } - public function testLocales(): void + public function testLanguages(): void { $splitter = new SubDirTranslationSplitter(); $result = $splitter->split([ @@ -31,8 +33,11 @@ public function testLocales(): void ]); $this->assertEquals( - ['en_US', 'it_IT'], - $result->getLocales(), + [ + ResourceLanguage::of('en_US'), + ResourceLanguage::of('it_IT'), + ], + $result->getLanguages(), 'unexpected locales' ); } @@ -49,11 +54,12 @@ public function testGetTranlsations(): void '/c/d.php.translations/en_US.php' ]); - $translations = $result->getTranslations('it_IT'); + $lang = ResourceLanguage::of('it_IT'); + $translations = $result->getTranslations($lang); $expected = [ - '/a/b.php.translations/it_IT.php', - '/c/d.php.translations/it_IT.php' + ResourceLocation::of('/a/b.php', $lang), + ResourceLocation::of('/c/d.php', $lang), ]; $this->assertEquals( @@ -70,10 +76,11 @@ public function testSplitWithLocParameter(): void '/a/b.php?loc=en_US', ]); - $translations = $result->getTranslations('en_US'); + $lang = ResourceLanguage::of('en_US'); + $translations = $result->getTranslations($lang); $expected = [ - '/a/b.php.translations/en_US.php', + ResourceLocation::of('/a/b.php', $lang) ]; $this->assertEquals( diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php index 8f09505..e4c2bf2 100644 --- a/test/Service/Indexer/SolrIndexServiceTest.php +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\Indexer\SolrIndexService; use Atoolo\Search\Service\IndexName; use Atoolo\Search\Service\SolrClientFactory; @@ -49,12 +50,12 @@ public function setUp(): void public function testUpdater(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->updater(''); + $this->indexService->updater(ResourceLanguage::default()); } public function testGetIndex(): void { - $index = $this->indexService->getIndex(''); + $index = $this->indexService->getIndex(ResourceLanguage::default()); $this->assertEquals( 'test', $index, @@ -65,7 +66,11 @@ public function testGetIndex(): void public function testDeleteExcludingProcessId(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->deleteExcludingProcessId('', 'test', 'test'); + $this->indexService->deleteExcludingProcessId( + ResourceLanguage::default(), + 'test', + 'test' + ); } public function testDeleteByIdListForAllLanguages(): void @@ -77,13 +82,16 @@ public function testDeleteByIdListForAllLanguages(): void public function testByQuery(): void { $this->client->expects($this->once())->method('createUpdate'); - $this->indexService->deleteByQuery('', 'test'); + $this->indexService->deleteByQuery( + ResourceLanguage::default(), + 'test' + ); } public function testCommit(): void { $this->client->expects($this->once())->method('update'); - $this->indexService->commit(''); + $this->indexService->commit(ResourceLanguage::default()); } public function testCommitForAllLanguages(): void diff --git a/test/Service/Indexer/TranslationSplitterResultTest.php b/test/Service/Indexer/TranslationSplitterResultTest.php index 69790bc..ffe279c 100644 --- a/test/Service/Indexer/TranslationSplitterResultTest.php +++ b/test/Service/Indexer/TranslationSplitterResultTest.php @@ -2,6 +2,8 @@ namespace Atoolo\Search\Test\Service\Indexer; +use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Service\Indexer\TranslationSplitterResult; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -11,52 +13,64 @@ class TranslationSplitterResultTest extends TestCase { public function testGetBases(): void { - $result = new TranslationSplitterResult(['/a/b.php'], []); + $result = new TranslationSplitterResult( + [ + ResourceLocation::of('/a/b.php') + ], + [] + ); $this->assertEquals( - ['/a/b.php'], + [ + ResourceLocation::of('/a/b.php') + ], $result->getBases(), 'unexpected bases' ); } - public function testLocales(): void + public function testLanguages(): void { + $it = ResourceLanguage::of('it_IT'); + $en = ResourceLanguage::of('en_US'); + $result = new TranslationSplitterResult( [], [ - 'it_IT' => ['/a/b.php.translations/it_IT.php'], - 'en_US' => ['/a/b.php.translations/en_US.php'] + $it->code => [ResourceLocation::of('/a/b.php', $it)], + $en->code => [ResourceLocation::of('/a/b.php', $en)], ] ); $this->assertEquals( - ['en_US', 'it_IT'], - $result->getLocales(), - 'unexpected locales' + [$en, $it], + $result->getLanguages(), + 'unexpected languages' ); } public function testGetTranslations(): void { + $it = ResourceLanguage::of('it_IT'); + $en = ResourceLanguage::of('en_US'); $result = new TranslationSplitterResult( [], [ - 'it_IT' => [ - '/a/b.php.translations/it_IT.php', - '/c/d.php.translations/it_IT.php', + $it->code => [ + ResourceLocation::of('/a/b.php', $it), + ResourceLocation::of('/c/d.php', $it), ], - 'en_US' => [ - '/a/b.php.translations/en_US.php', - '/a/b.php.translations/en_US.php', + $en->code => [ + ResourceLocation::of('/a/b.php', $en), + ResourceLocation::of('/a/b.php', $en), ] ] ); - $translations = $result->getTranslations('it_IT'); + $translations = $result->getTranslations($it); $expected = [ - '/a/b.php.translations/it_IT.php', - '/c/d.php.translations/it_IT.php' + ResourceLocation::of('/a/b.php', $it), + ResourceLocation::of('/c/d.php', $it), ]; $this->assertEquals( @@ -68,10 +82,11 @@ public function testGetTranslations(): void public function testGetMissingTranslations(): void { $result = new TranslationSplitterResult([], []); + $lang = ResourceLanguage::of('en_US'); $this->assertEquals( [], - $result->getTranslations('en_US'), + $result->getTranslations($lang), 'empty array expected' ); } diff --git a/test/Service/ResourceChannelBasedIndexNameTest.php b/test/Service/ResourceChannelBasedIndexNameTest.php index 25cee94..84a501b 100644 --- a/test/Service/ResourceChannelBasedIndexNameTest.php +++ b/test/Service/ResourceChannelBasedIndexNameTest.php @@ -5,7 +5,7 @@ namespace Atoolo\Search\Test\Service; use Atoolo\Resource\ResourceChannel; -use Atoolo\Resource\ResourceChannelFactory; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\ResourceChannelBasedIndexName; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -17,9 +17,6 @@ class ResourceChannelBasedIndexNameTest extends TestCase public function setUp(): void { - $resourceChannelFactory = $this->createStub( - ResourceChannelFactory::class - ); $resourceChannel = new ResourceChannel( '', '', @@ -29,15 +26,13 @@ public function setUp(): void '', '', '', + '', 'test', ['en_US'] ); - $resourceChannelFactory->method('create') - ->willReturn($resourceChannel); - $this->indexName = new ResourceChannelBasedIndexName( - $resourceChannelFactory + $resourceChannel ); } @@ -45,7 +40,7 @@ public function testName(): void { $this->assertEquals( 'test', - $this->indexName->name(''), + $this->indexName->name(ResourceLanguage::default()), 'The default index name should be returned ' . 'if no language is given' ); @@ -55,7 +50,7 @@ public function testNameWithLang(): void { $this->assertEquals( 'test-en_US', - $this->indexName->name('en'), + $this->indexName->name(ResourceLanguage::of('en')), 'The language-specific index name should be returned ' . 'if a language is given' ); @@ -65,7 +60,7 @@ public function testNameWithUnsupportedLang(): void { $this->assertEquals( 'test', - $this->indexName->name('it'), + $this->indexName->name(ResourceLanguage::of('it')), 'The default index name should be returned ' . 'if the language is not supported' ); diff --git a/test/Service/Search/ExternalResourceFactoryTest.php b/test/Service/Search/ExternalResourceFactoryTest.php index b4f3c9f..8d2e218 100644 --- a/test/Service/Search/ExternalResourceFactoryTest.php +++ b/test/Service/Search/ExternalResourceFactoryTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Service\Search; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\Search\ExternalResourceFactory; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -23,7 +24,7 @@ public function testAcceptHttps(): void { $document = $this->createDocument('https://www.sitepark.com'); $this->assertTrue( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should be accepted' ); } @@ -32,7 +33,7 @@ public function testAcceptHttp(): void { $document = $this->createDocument('http://www.sitepark.com'); $this->assertTrue( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should be accepted' ); } @@ -41,7 +42,7 @@ public function testAcceptWithoutUrl(): void { $document = $this->createStub(Document::class); $this->assertFalse( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should be accepted' ); } @@ -49,11 +50,14 @@ public function testAcceptWithoutUrl(): void public function testCreate(): void { $document = $this->createDocument('https://www.sitepark.com'); - $resource = $this->factory->create($document, 'de'); + $resource = $this->factory->create( + $document, + ResourceLanguage::of('en') + ); $this->assertEquals( 'https://www.sitepark.com', - $resource->getLocation(), + $resource->location, 'unexpected location' ); } @@ -61,11 +65,14 @@ public function testCreate(): void public function testCreateWithName(): void { $document = $this->createDocument('https://www.sitepark.com', 'Test'); - $resource = $this->factory->create($document, 'de'); + $resource = $this->factory->create( + $document, + ResourceLanguage::of('en') + ); $this->assertEquals( 'Test', - $resource->getName(), + $resource->name, 'unexpected name' ); } @@ -75,7 +82,7 @@ public function testCreateWithMissingUrl(): void $document = $this->createStub(Document::class); $this->expectException(\LogicException::class); - $this->factory->create($document, 'de'); + $this->factory->create($document, ResourceLanguage::of('en')); } private function createDocument(string $url, string $title = ''): Document diff --git a/test/Service/Search/InternalMediaResourceFactoryTest.php b/test/Service/Search/InternalMediaResourceFactoryTest.php index 925ed79..0528021 100644 --- a/test/Service/Search/InternalMediaResourceFactoryTest.php +++ b/test/Service/Search/InternalMediaResourceFactoryTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Service\Search\InternalMediaResourceFactory; use LogicException; @@ -37,7 +38,7 @@ public function testAccept(): void ->willReturn(true); $this->assertTrue( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should be accepted' ); } @@ -46,7 +47,7 @@ public function testAcceptWithoutUrl(): void { $document = $this->createStub(Document::class); $this->assertFalse( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should not be accepted' ); } @@ -59,7 +60,7 @@ public function testAcceptNotExists(): void ->willReturn(false); $this->assertFalse( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should not be accepted' ); } @@ -74,7 +75,7 @@ public function testCreate(): void $this->assertEquals( $resource, - $this->factory->create($document, 'de'), + $this->factory->create($document, ResourceLanguage::of('en')), 'unexpected resource' ); } @@ -83,7 +84,7 @@ public function testCreateWithoutUrl(): void { $document = $this->createStub(Document::class); $this->expectException(LogicException::class); - $this->factory->create($document, 'de'); + $this->factory->create($document, ResourceLanguage::of('de')); } private function createDocument(string $url): Document diff --git a/test/Service/Search/InternalResourceFactoryTest.php b/test/Service/Search/InternalResourceFactoryTest.php index a457d05..f78a8e7 100644 --- a/test/Service/Search/InternalResourceFactoryTest.php +++ b/test/Service/Search/InternalResourceFactoryTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Resource\ResourceLoader; use Atoolo\Search\Service\Search\InternalResourceFactory; use LogicException; @@ -33,7 +34,7 @@ public function testAccept(): void { $document = $this->createDocument('/test.php'); $this->assertTrue( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should be accepted' ); } @@ -42,7 +43,7 @@ public function testAcceptWithoutUrl(): void { $document = $this->createStub(Document::class); $this->assertFalse( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should not be accepted' ); } @@ -51,7 +52,7 @@ public function testAcceptWithWrongUrl(): void { $document = $this->createDocument('/test.txt'); $this->assertFalse( - $this->factory->accept($document), + $this->factory->accept($document, ResourceLanguage::default()), 'should not be accepted' ); } @@ -66,7 +67,7 @@ public function testCreate(): void $this->assertEquals( $resource, - $this->factory->create($document, 'de'), + $this->factory->create($document, ResourceLanguage::of('de')), 'unexpected resource' ); } @@ -75,7 +76,7 @@ public function testCreateWithoutUrl(): void { $document = $this->createStub(Document::class); $this->expectException(LogicException::class); - $this->factory->create($document, 'de'); + $this->factory->create($document, ResourceLanguage::of('de')); } private function createDocument(string $url): Document diff --git a/test/Service/Search/SolrMoreLikeThisTest.php b/test/Service/Search/SolrMoreLikeThisTest.php index 56c868a..4195f9d 100644 --- a/test/Service/Search/SolrMoreLikeThisTest.php +++ b/test/Service/Search/SolrMoreLikeThisTest.php @@ -5,6 +5,7 @@ namespace Atoolo\Search\Test\Service\Search; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Search\Query\Filter\Filter; use Atoolo\Search\Dto\Search\Query\MoreLikeThisQuery; use Atoolo\Search\Service\IndexName; @@ -68,8 +69,7 @@ public function testMoreLikeThis(): void ->getMock(); $query = new MoreLikeThisQuery( - '/test.php', - '', + ResourceLocation::of('/test.php'), [$filter] ); diff --git a/test/Service/Search/SolrResultToResourceResolverTest.php b/test/Service/Search/SolrResultToResourceResolverTest.php index 3589d37..93cf6c6 100644 --- a/test/Service/Search/SolrResultToResourceResolverTest.php +++ b/test/Service/Search/SolrResultToResourceResolverTest.php @@ -6,6 +6,7 @@ use ArrayIterator; use Atoolo\Resource\Resource; +use Atoolo\Resource\ResourceLanguage; use Atoolo\Search\Service\Search\ResourceFactory; use Atoolo\Search\Service\Search\SolrResultToResourceResolver; use PHPUnit\Framework\Attributes\CoversClass; @@ -32,7 +33,10 @@ public function testLoadResourceList(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result, ''); + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); $this->assertEquals( [$resource], @@ -55,7 +59,10 @@ public function testLoadResourceListWithoutAcceptedFactory(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result, ''); + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); $this->assertEmpty( $resourceList, @@ -76,7 +83,10 @@ public function testLoadResourceListWithoutAcceptedFactoryNoUrl(): void $resolver = new SolrResultToResourceResolver([$resourceFactory]); - $resourceList = $resolver->loadResourceList($result, ''); + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); $this->assertEmpty( $resourceList, diff --git a/test/Service/ServerVarSolrClientFactoryTest.php b/test/Service/ServerVarSolrClientFactoryTest.php deleted file mode 100644 index d47425c..0000000 --- a/test/Service/ServerVarSolrClientFactoryTest.php +++ /dev/null @@ -1,137 +0,0 @@ -create('core'); - - $expectedEndPoint = new Endpoint([ - 'host' => 'localhost', - 'port' => '8382', - 'path' => '', - 'core' => 'core', - 'scheme' => 'http', - 'key' => 'localhost' - ]); - - $this->assertEquals( - $expectedEndPoint, - $client->getEndpoint(), - 'unexpected endpoint configuration' - ); - } - - public function testCreateWithUrlServerVar(): void - { - $_SERVER['SOLR_URL'] = 'https://solr.example.com:8983/path'; - - $factory = new ServerVarSolrClientFactory(); - $client = $factory->create('core'); - - $expectedEndPoint = new Endpoint([ - 'host' => 'solr.example.com', - 'port' => '8983', - 'path' => '/path', - 'core' => 'core', - 'scheme' => 'https', - 'key' => 'solr.example.com' - ]); - - $this->assertEquals( - $expectedEndPoint, - $client->getEndpoint(), - 'unexpected endpoint configuration' - ); - } - - public function testCreateWithHttpUrlWithoutPortServerVar(): void - { - $_SERVER['SOLR_URL'] = 'http://solr'; - - $factory = new ServerVarSolrClientFactory(); - $client = $factory->create('core'); - - $expectedEndPoint = new Endpoint([ - 'host' => 'solr', - 'port' => '8382', - 'path' => '', - 'core' => 'core', - 'scheme' => 'http', - 'key' => 'solr' - ]); - - $this->assertEquals( - $expectedEndPoint, - $client->getEndpoint(), - 'unexpected endpoint configuration' - ); - } - - public function testCreateWithHttpsUrlWithoutPortServerVar(): void - { - $_SERVER['SOLR_URL'] = 'https://solr'; - - $factory = new ServerVarSolrClientFactory(); - $client = $factory->create('core'); - - $expectedEndPoint = new Endpoint([ - 'host' => 'solr', - 'port' => '443', - 'path' => '', - 'core' => 'core', - 'scheme' => 'https', - 'key' => 'solr' - ]); - - $this->assertEquals( - $expectedEndPoint, - $client->getEndpoint(), - 'unexpected endpoint configuration' - ); - } - - public function testCreateWithSchemaHostAndPortServerVar(): void - { - $_SERVER['SOLR_SCHEME'] = 'https'; - $_SERVER['SOLR_HOST'] = 'solr.example.com'; - $_SERVER['SOLR_PORT'] = '8983'; - - $factory = new ServerVarSolrClientFactory(); - $client = $factory->create('core'); - - $expectedEndPoint = new Endpoint([ - 'host' => 'solr.example.com', - 'port' => '8983', - 'path' => '', - 'core' => 'core', - 'scheme' => 'https', - 'key' => 'solr.example.com' - ]); - - $this->assertEquals( - $expectedEndPoint, - $client->getEndpoint(), - 'unexpected endpoint configuration' - ); - } -} From 861e0ff082623111139252e9800105c65c044fd2 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 18 Apr 2024 15:49:44 +0200 Subject: [PATCH 130/145] refactore: use ResourceLocation --- src/Exception/DocumentEnrichingException.php | 7 ++++--- src/Exception/MissMatchingResourceFactoryException.php | 7 ++++--- src/Service/Indexer/InternalResourceIndexer.php | 7 ++++++- .../Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php | 2 +- src/Service/Search/SolrResultToResourceResolver.php | 6 +++++- test/Exception/DocumentEnrichingExceptionTest.php | 5 +++-- .../Exception/MissMatchingResourceFactoryExceptionTest.php | 7 +++++-- 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/Exception/DocumentEnrichingException.php b/src/Exception/DocumentEnrichingException.php index 8e8392b..98bedec 100644 --- a/src/Exception/DocumentEnrichingException.php +++ b/src/Exception/DocumentEnrichingException.php @@ -4,24 +4,25 @@ namespace Atoolo\Search\Exception; +use Atoolo\Resource\ResourceLocation; use RuntimeException; class DocumentEnrichingException extends RuntimeException { public function __construct( - private readonly string $location, + private readonly ResourceLocation $location, string $message = "", int $code = 0, ?\Throwable $previous = null ) { parent::__construct( - $location . ': ' . $message, + $location->__toString() . ': ' . $message, $code, $previous ); } - public function getLocation(): string + public function getLocation(): ResourceLocation { return $this->location; } diff --git a/src/Exception/MissMatchingResourceFactoryException.php b/src/Exception/MissMatchingResourceFactoryException.php index e01833a..f381727 100644 --- a/src/Exception/MissMatchingResourceFactoryException.php +++ b/src/Exception/MissMatchingResourceFactoryException.php @@ -4,24 +4,25 @@ namespace Atoolo\Search\Exception; +use Atoolo\Resource\ResourceLocation; use RuntimeException; class MissMatchingResourceFactoryException extends RuntimeException { public function __construct( - private readonly string $location, + private readonly ResourceLocation $location, string $message = "", int $code = 0, ?\Throwable $previous = null ) { parent::__construct( - $location . ': ' . $message, + $location->__toString() . ': ' . $message, $code, $previous ); } - public function getLocation(): string + public function getLocation(): ResourceLocation { return $this->location; } diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 9868707..3547f60 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -158,7 +158,12 @@ public function index(): IndexerStatus */ public function update(array $paths): IndexerStatus { - $collectedPaths = $this->finder->findPaths($paths); + $collectedPaths = array_merge( + $this->finder->findPaths($paths), // resolve directories recursive + $paths + ); + $collectedPaths = array_unique($collectedPaths); + $total = count($collectedPaths); $this->progressHandler->startUpdate($total); diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 08af41e..77b78d2 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -157,7 +157,7 @@ public function enrichDocument( $doc->sp_site = array_unique($sites); } catch (Exception $e) { throw new DocumentEnrichingException( - $resource->location, + $resource->toLocation(), 'Unable to set sp_site: ' . $e->getMessage(), 0, $e diff --git a/src/Service/Search/SolrResultToResourceResolver.php b/src/Service/Search/SolrResultToResourceResolver.php index cec0978..ad6121b 100644 --- a/src/Service/Search/SolrResultToResourceResolver.php +++ b/src/Service/Search/SolrResultToResourceResolver.php @@ -6,6 +6,7 @@ use Atoolo\Resource\Resource; use Atoolo\Resource\ResourceLanguage; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Exception\MissMatchingResourceFactoryException; use Exception; use Psr\Log\LoggerInterface; @@ -60,7 +61,10 @@ private function loadResource( } throw new MissMatchingResourceFactoryException( - $document->getFields()['url'] ?? '' + ResourceLocation::of( + $document->getFields()['url'] ?? '', + $lang + ) ); } } diff --git a/test/Exception/DocumentEnrichingExceptionTest.php b/test/Exception/DocumentEnrichingExceptionTest.php index 949ac52..487304b 100644 --- a/test/Exception/DocumentEnrichingExceptionTest.php +++ b/test/Exception/DocumentEnrichingExceptionTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Exception; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Exception\DocumentEnrichingException; use PHPUnit\Framework\TestCase; @@ -11,9 +12,9 @@ class DocumentEnrichingExceptionTest extends TestCase { public function testGetLocation(): void { - $e = new DocumentEnrichingException('/test.php'); + $e = new DocumentEnrichingException(ResourceLocation::of('/test.php')); $this->assertEquals( - '/test.php', + ResourceLocation::of('/test.php'), $e->getLocation(), 'unexpected location' ); diff --git a/test/Exception/MissMatchingResourceFactoryExceptionTest.php b/test/Exception/MissMatchingResourceFactoryExceptionTest.php index 7c1022e..93bdddd 100644 --- a/test/Exception/MissMatchingResourceFactoryExceptionTest.php +++ b/test/Exception/MissMatchingResourceFactoryExceptionTest.php @@ -4,6 +4,7 @@ namespace Atoolo\Search\Test\Exception; +use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Exception\MissMatchingResourceFactoryException; use PHPUnit\Framework\TestCase; @@ -11,9 +12,11 @@ class MissMatchingResourceFactoryExceptionTest extends TestCase { public function testGetLocation(): void { - $e = new MissMatchingResourceFactoryException('/test.php'); + $e = new MissMatchingResourceFactoryException( + ResourceLocation::of('/test.php') + ); $this->assertEquals( - '/test.php', + ResourceLocation::of('/test.php'), $e->getLocation(), 'unexpected location' ); From 7d7b7fa50fa1fe61952da972fa5dcfc5ecf8af7e Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Thu, 18 Apr 2024 15:50:54 +0200 Subject: [PATCH 131/145] chore: composer update --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 0e2b408..71cb9a2 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,7 @@ "require": { "php": ">=8.1 <8.4.0", "ext-intl": "*", - "atoolo/resource": "dev-feature/resource-channel", + "atoolo/resource": "dev-main", "solarium/solarium": "^6.3", "symfony/config": "^6.3 || ^7.0", "symfony/console": "^6.3 || ^7.0", From 35529bac1ba33ab61394c5403e729668f94ef20e Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 22 Apr 2024 09:50:58 +0200 Subject: [PATCH 132/145] feat: add AbsoluteDateRangeFilter filter --- .../Query/Filter/AbsoluteDateRangeFilter.php | 42 +++++++++++++++++++ .../DefaultSchema2xDocumentEnricher.php | 10 +++-- 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 src/Dto/Search/Query/Filter/AbsoluteDateRangeFilter.php diff --git a/src/Dto/Search/Query/Filter/AbsoluteDateRangeFilter.php b/src/Dto/Search/Query/Filter/AbsoluteDateRangeFilter.php new file mode 100644 index 0000000..884cab8 --- /dev/null +++ b/src/Dto/Search/Query/Filter/AbsoluteDateRangeFilter.php @@ -0,0 +1,42 @@ +formatDate($this->from) . + ' TO ' . + $this->formatDate($this->to) . + ']'; + } + + private function formatDate(?DateTime $date): string + { + if ($date === null) { + return '*'; + } + + $formatter = clone $date; + $formatter->setTimezone(new \DateTimeZone('UTC')); + return $formatter->format('Y-m-d\TH:i:s\Z'); + } +} diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php index 77b78d2..77eb8a3 100644 --- a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -116,9 +116,6 @@ public function enrichDocument( $doc->sp_generated = $this->toDateTime( $data->getInt('generated') ); - $doc->sp_date = $this->toDateTime( - $base->getInt('date') - ); $doc->sp_archive = $base->getBool('archive'); @@ -202,6 +199,13 @@ public function enrichDocument( } $doc->sp_group_path = $groupPathAsIdList; + $doc->sp_date = $this->toDateTime( + $base->getInt('date') + ); + if ($doc->sp_date !== null) { + $doc->sp_date_list = [$doc->sp_date]; + } + /** @var array $schedulingList */ $schedulingList = $metadata->getArray('scheduling'); if (!empty($schedulingList)) { From 05313ad0ac70a43a5b8c8d9f83061b61d393d1b3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 26 Apr 2024 14:50:25 +0200 Subject: [PATCH 133/145] feat: Date range Filter --- .../Query/Filter/RelativeDateRangeFilter.php | 75 +++++++++ .../Filter/AbsoluteDateRangeFilterTest.php | 44 ++++++ .../Filter/RelativeDateRangeFilterTest.php | 143 ++++++++++++++++++ .../Indexer/InternalResourceIndexerTest.php | 14 ++ 4 files changed, 276 insertions(+) create mode 100644 src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php create mode 100644 test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php create mode 100644 test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php new file mode 100644 index 0000000..7723525 --- /dev/null +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -0,0 +1,75 @@ +toSolrDateRage(); + } + + private function toSolrDateRage(): string + { + if ($this->from === null) { + $from = "NOW/DAY"; + } else { + $from = $this->toSolrIntervalSyntax($this->from); + } + + if ($this->to === null) { + $to = "NOW/DAY+1DAY-1SECOND"; + } else { + $to = $this->toSolrIntervalSyntax($this->to); + } + + return '[' . $from . ' TO ' . $to . ']'; + } + + private function toSolrIntervalSyntax(DateInterval $value): string + { + $interval = 'NOW'; + if ($value->y > 0) { + $interval = $interval . '-' . $value->y . 'YEARS'; + } + if ($value->m > 0) { + $interval = $interval . '-' . $value->m . 'MONTHS'; + } + if ($value->d > 0) { + $interval = $interval . '-' . $value->d . 'DAYS'; + } + if ($value->h > 0) { + throw new InvalidArgumentException( + 'Hours are not supported for the RelativeDateRangeFilter' + ); + } + if ($value->i > 0) { + throw new InvalidArgumentException( + 'Minutes are not supported for the RelativeDateRangeFilter' + ); + } + if ($value->s > 0) { + throw new InvalidArgumentException( + 'Seconds are not supported for the RelativeDateRangeFilter' + ); + } + + return $interval . '/DAY'; + } +} diff --git a/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php new file mode 100644 index 0000000..f90547c --- /dev/null +++ b/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php @@ -0,0 +1,44 @@ +assertEquals( + 'sp_date_list:[2021-01-01T00:00:00Z TO 2021-01-02T00:00:00Z]', + $filter->getQuery() + ); + } + + public function testGetQueryWithFrom(): void + { + $from = new \DateTime('2021-01-01 00:00:00'); + $filter = new AbsoluteDateRangeFilter($from, null, 'sp_date_list'); + $this->assertEquals( + 'sp_date_list:[2021-01-01T00:00:00Z TO *]', + $filter->getQuery() + ); + } + + public function testGetQueryWithTo(): void + { + $to = new \DateTime('2021-01-02 00:00:00'); + $filter = new AbsoluteDateRangeFilter(null, $to, 'sp_date_list'); + $this->assertEquals( + 'sp_date_list:[* TO 2021-01-02T00:00:00Z]', + $filter->getQuery() + ); + } +} diff --git a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php new file mode 100644 index 0000000..e857d93 --- /dev/null +++ b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php @@ -0,0 +1,143 @@ + + */ + public static function additionProviderForFromIntervals(): array + { + return [ + ['P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P1W', 'sp_date_list:[NOW-7DAYS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P2M', 'sp_date_list:[NOW-2MONTHS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P3Y', 'sp_date_list:[NOW-3YEARS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ]; + } + + /** + * @return array + */ + public static function additionProviderForToIntervals(): array + { + return [ + ['P1D', 'sp_date_list:[NOW/DAY TO NOW-1DAYS/DAY]'], + ['P1W', 'sp_date_list:[NOW/DAY TO NOW-7DAYS/DAY]'], + ['P2M', 'sp_date_list:[NOW/DAY TO NOW-2MONTHS/DAY]'], + ['P3Y', 'sp_date_list:[NOW/DAY TO NOW-3YEARS/DAY]'], + ]; + } + + /** + * @return array + */ + public static function additionProviderForFromAndToIntervals(): array + { + return [ + ['P1D', 'P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW-1DAYS/DAY]'], + ['P1W', 'P2M', 'sp_date_list:[NOW-7DAYS/DAY TO NOW-2MONTHS/DAY]'], + ]; + } + + /** + * @return array + */ + public static function additionProviderForInvalidIntervals(): array + { + return [ + ['PT1H'], + ['PT1M'], + ['PT1S'], + ]; + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForFromIntervals')] + public function testGetQueryWithFrom( + string $from, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + new DateInterval($from), + null, + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForToIntervals')] + public function testGetQueryWithTo( + string $to, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + null, + new DateInterval($to), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForFromAndToIntervals')] + public function testGetQueryWithFromAndTo( + string $from, + string $to, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + new DateInterval($from), + new DateInterval($to), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForInvalidIntervals')] + public function testGetQueryWithInvalidIntervals( + string $interval + ): void { + $filter = new RelativeDateRangeFilter( + null, + new DateInterval($interval), + ); + $this->expectException(InvalidArgumentException::class); + $filter->getQuery(); + } +} diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 7a435ab..2f884ce 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -226,6 +226,20 @@ public function testIndexAllWithChunks(): void $this->indexer->index(); } + public function testIndexAllWithEmptyList(): void + { + $this->finder->method('findAll') + ->willReturn([ + ]); + $this->indexerProgressHandler->expects($this->once()) + ->method('start') + ->with(0); + $this->indexerProgressHandler->expects($this->once()) + ->method('finish'); + + $this->indexer->index(); + } + public function testIndexSkipResource(): void { $this->finder->method('findAll') From d3eecb07becb0cf46a0b724ee9620e600ea4e9fa Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 26 Apr 2024 15:15:44 +0200 Subject: [PATCH 134/145] feat: add base date for relative date range filter --- .../Query/Filter/RelativeDateRangeFilter.php | 18 +++++- .../Filter/RelativeDateRangeFilterTest.php | 57 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php index 7723525..c1ef7d8 100644 --- a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -10,6 +10,7 @@ class RelativeDateRangeFilter extends Filter { public function __construct( + private readonly ?\DateTime $base, private readonly ?DateInterval $from, private readonly ?DateInterval $to, ?string $key = null @@ -28,13 +29,13 @@ public function getQuery(): string private function toSolrDateRage(): string { if ($this->from === null) { - $from = "NOW/DAY"; + $from = $this->getBaseInSolrSyntax() . "/DAY"; } else { $from = $this->toSolrIntervalSyntax($this->from); } if ($this->to === null) { - $to = "NOW/DAY+1DAY-1SECOND"; + $to = $this->getBaseInSolrSyntax() . "/DAY+1DAY-1SECOND"; } else { $to = $this->toSolrIntervalSyntax($this->to); } @@ -42,9 +43,20 @@ private function toSolrDateRage(): string return '[' . $from . ' TO ' . $to . ']'; } + private function getBaseInSolrSyntax(): string + { + if ($this->base === null) { + return 'NOW'; + } + + $formatter = clone $this->base; + $formatter->setTimezone(new \DateTimeZone('UTC')); + return $formatter->format('Y-m-d\TH:i:s\Z'); + } + private function toSolrIntervalSyntax(DateInterval $value): string { - $interval = 'NOW'; + $interval = $this->getBaseInSolrSyntax(); if ($value->y > 0) { $interval = $interval . '-' . $value->y . 'YEARS'; } diff --git a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php index e857d93..6696212 100644 --- a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php +++ b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php @@ -6,6 +6,7 @@ use Atoolo\Search\Dto\Search\Query\Filter\RelativeDateRangeFilter; use DateInterval; +use DateTime; use Exception; use InvalidArgumentException; use PHPUnit\Framework\Attributes\CoversClass; @@ -52,6 +53,36 @@ public static function additionProviderForFromAndToIntervals(): array ]; } + /** + * @return array + */ + public static function additionProviderWithBase(): array + { + return [ + [ + new DateTime('2021-01-01 00:00:00'), + 'P1D', + null, + 'sp_date_list:[2021-01-01T00:00:00Z-1DAYS/DAY' . + ' TO 2021-01-01T00:00:00Z/DAY+1DAY-1SECOND]' + ], + [ + new DateTime('2021-01-01 00:00:00'), + null, + 'P2M', + 'sp_date_list:[2021-01-01T00:00:00Z/DAY' . + ' TO 2021-01-01T00:00:00Z-2MONTHS/DAY]' + ], + [ + new DateTime('2021-01-01 00:00:00'), + 'P1W', + 'P2M', + 'sp_date_list:[2021-01-01T00:00:00Z-7DAYS/DAY' . + ' TO 2021-01-01T00:00:00Z-2MONTHS/DAY]' + ], + ]; + } + /** * @return array */ @@ -73,6 +104,7 @@ public function testGetQueryWithFrom( string $expected ): void { $filter = new RelativeDateRangeFilter( + null, new DateInterval($from), null, ); @@ -93,6 +125,7 @@ public function testGetQueryWithTo( string $expected ): void { $filter = new RelativeDateRangeFilter( + null, null, new DateInterval($to), ); @@ -114,6 +147,7 @@ public function testGetQueryWithFromAndTo( string $expected ): void { $filter = new RelativeDateRangeFilter( + null, new DateInterval($from), new DateInterval($to), ); @@ -125,6 +159,28 @@ public function testGetQueryWithFromAndTo( ); } + /** + * @throws Exception + */ + #[DataProvider('additionProviderWithBase')] + public function testGetQueryWithBase( + DateTime $base, + ?string $from, + ?string $to, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + $base, + $from === null ? null : new DateInterval($from), + $to === null ? null : new DateInterval($to), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } /** * @throws Exception @@ -134,6 +190,7 @@ public function testGetQueryWithInvalidIntervals( string $interval ): void { $filter = new RelativeDateRangeFilter( + null, null, new DateInterval($interval), ); From e4afb03fc3a8ea7b4066c9f5f67f5acb712868a9 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 26 Apr 2024 15:21:27 +0200 Subject: [PATCH 135/145] refactore: rename for from and to to before and after --- .../Query/Filter/RelativeDateRangeFilter.php | 12 +++---- .../Filter/RelativeDateRangeFilterTest.php | 36 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php index c1ef7d8..37bc886 100644 --- a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -11,8 +11,8 @@ class RelativeDateRangeFilter extends Filter { public function __construct( private readonly ?\DateTime $base, - private readonly ?DateInterval $from, - private readonly ?DateInterval $to, + private readonly ?DateInterval $before, + private readonly ?DateInterval $after, ?string $key = null ) { parent::__construct( @@ -28,16 +28,16 @@ public function getQuery(): string private function toSolrDateRage(): string { - if ($this->from === null) { + if ($this->before === null) { $from = $this->getBaseInSolrSyntax() . "/DAY"; } else { - $from = $this->toSolrIntervalSyntax($this->from); + $from = $this->toSolrIntervalSyntax($this->before); } - if ($this->to === null) { + if ($this->after === null) { $to = $this->getBaseInSolrSyntax() . "/DAY+1DAY-1SECOND"; } else { - $to = $this->toSolrIntervalSyntax($this->to); + $to = $this->toSolrIntervalSyntax($this->after); } return '[' . $from . ' TO ' . $to . ']'; diff --git a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php index 6696212..94f6f44 100644 --- a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php +++ b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php @@ -19,7 +19,7 @@ class RelativeDateRangeFilterTest extends TestCase /** * @return array */ - public static function additionProviderForFromIntervals(): array + public static function additionProviderForBeforeIntervals(): array { return [ ['P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW/DAY+1DAY-1SECOND]'], @@ -32,7 +32,7 @@ public static function additionProviderForFromIntervals(): array /** * @return array */ - public static function additionProviderForToIntervals(): array + public static function additionProviderForAfterIntervals(): array { return [ ['P1D', 'sp_date_list:[NOW/DAY TO NOW-1DAYS/DAY]'], @@ -45,7 +45,7 @@ public static function additionProviderForToIntervals(): array /** * @return array */ - public static function additionProviderForFromAndToIntervals(): array + public static function additionProviderForBeforeAndAfterIntervals(): array { return [ ['P1D', 'P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW-1DAYS/DAY]'], @@ -98,14 +98,14 @@ public static function additionProviderForInvalidIntervals(): array /** * @throws Exception */ - #[DataProvider('additionProviderForFromIntervals')] + #[DataProvider('additionProviderForBeforeIntervals')] public function testGetQueryWithFrom( - string $from, + string $before, string $expected ): void { $filter = new RelativeDateRangeFilter( null, - new DateInterval($from), + new DateInterval($before), null, ); @@ -119,15 +119,15 @@ public function testGetQueryWithFrom( /** * @throws Exception */ - #[DataProvider('additionProviderForToIntervals')] + #[DataProvider('additionProviderForAfterIntervals')] public function testGetQueryWithTo( - string $to, + string $after, string $expected ): void { $filter = new RelativeDateRangeFilter( null, null, - new DateInterval($to), + new DateInterval($after), ); $this->assertEquals( @@ -140,16 +140,16 @@ public function testGetQueryWithTo( /** * @throws Exception */ - #[DataProvider('additionProviderForFromAndToIntervals')] + #[DataProvider('additionProviderForBeforeAndAfterIntervals')] public function testGetQueryWithFromAndTo( - string $from, - string $to, + string $before, + string $after, string $expected ): void { $filter = new RelativeDateRangeFilter( null, - new DateInterval($from), - new DateInterval($to), + new DateInterval($before), + new DateInterval($after), ); $this->assertEquals( @@ -165,14 +165,14 @@ public function testGetQueryWithFromAndTo( #[DataProvider('additionProviderWithBase')] public function testGetQueryWithBase( DateTime $base, - ?string $from, - ?string $to, + ?string $before, + ?string $after, string $expected ): void { $filter = new RelativeDateRangeFilter( $base, - $from === null ? null : new DateInterval($from), - $to === null ? null : new DateInterval($to), + $before === null ? null : new DateInterval($before), + $after === null ? null : new DateInterval($after), ); $this->assertEquals( From f1799c3b972250802b1010b6a112db0a52488ef3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 26 Apr 2024 15:53:43 +0200 Subject: [PATCH 136/145] fix: after date range was wrong --- .../Query/Filter/RelativeDateRangeFilter.php | 16 +++++++------ .../Filter/RelativeDateRangeFilterTest.php | 24 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php index 37bc886..f16ae61 100644 --- a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -31,13 +31,15 @@ private function toSolrDateRage(): string if ($this->before === null) { $from = $this->getBaseInSolrSyntax() . "/DAY"; } else { - $from = $this->toSolrIntervalSyntax($this->before); + $from = $this->toSolrIntervalSyntax($this->before, '-') . + '/DAY'; } if ($this->after === null) { $to = $this->getBaseInSolrSyntax() . "/DAY+1DAY-1SECOND"; } else { - $to = $this->toSolrIntervalSyntax($this->after); + $to = $this->toSolrIntervalSyntax($this->after, '+') . + "/DAY+1DAY-1SECOND"; } return '[' . $from . ' TO ' . $to . ']'; @@ -54,17 +56,17 @@ private function getBaseInSolrSyntax(): string return $formatter->format('Y-m-d\TH:i:s\Z'); } - private function toSolrIntervalSyntax(DateInterval $value): string + private function toSolrIntervalSyntax(DateInterval $value, string $operator): string { $interval = $this->getBaseInSolrSyntax(); if ($value->y > 0) { - $interval = $interval . '-' . $value->y . 'YEARS'; + $interval = $interval . $operator . $value->y . 'YEARS'; } if ($value->m > 0) { - $interval = $interval . '-' . $value->m . 'MONTHS'; + $interval = $interval . $operator . $value->m . 'MONTHS'; } if ($value->d > 0) { - $interval = $interval . '-' . $value->d . 'DAYS'; + $interval = $interval . $operator . $value->d . 'DAYS'; } if ($value->h > 0) { throw new InvalidArgumentException( @@ -82,6 +84,6 @@ private function toSolrIntervalSyntax(DateInterval $value): string ); } - return $interval . '/DAY'; + return $interval; } } diff --git a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php index 94f6f44..f43ae0d 100644 --- a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php +++ b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php @@ -35,10 +35,10 @@ public static function additionProviderForBeforeIntervals(): array public static function additionProviderForAfterIntervals(): array { return [ - ['P1D', 'sp_date_list:[NOW/DAY TO NOW-1DAYS/DAY]'], - ['P1W', 'sp_date_list:[NOW/DAY TO NOW-7DAYS/DAY]'], - ['P2M', 'sp_date_list:[NOW/DAY TO NOW-2MONTHS/DAY]'], - ['P3Y', 'sp_date_list:[NOW/DAY TO NOW-3YEARS/DAY]'], + ['P1D', 'sp_date_list:[NOW/DAY TO NOW+1DAYS/DAY+1DAY-1SECOND]'], + ['P1W', 'sp_date_list:[NOW/DAY TO NOW+7DAYS/DAY+1DAY-1SECOND]'], + ['P2M', 'sp_date_list:[NOW/DAY TO NOW+2MONTHS/DAY+1DAY-1SECOND]'], + ['P3Y', 'sp_date_list:[NOW/DAY TO NOW+3YEARS/DAY+1DAY-1SECOND]'], ]; } @@ -48,8 +48,16 @@ public static function additionProviderForAfterIntervals(): array public static function additionProviderForBeforeAndAfterIntervals(): array { return [ - ['P1D', 'P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW-1DAYS/DAY]'], - ['P1W', 'P2M', 'sp_date_list:[NOW-7DAYS/DAY TO NOW-2MONTHS/DAY]'], + [ + 'P1D', + 'P1D', + 'sp_date_list:[NOW-1DAYS/DAY TO NOW+1DAYS/DAY+1DAY-1SECOND]' + ], + [ + 'P1W', + 'P2M', + 'sp_date_list:[NOW-7DAYS/DAY TO NOW+2MONTHS/DAY+1DAY-1SECOND]' + ], ]; } @@ -71,14 +79,14 @@ public static function additionProviderWithBase(): array null, 'P2M', 'sp_date_list:[2021-01-01T00:00:00Z/DAY' . - ' TO 2021-01-01T00:00:00Z-2MONTHS/DAY]' + ' TO 2021-01-01T00:00:00Z+2MONTHS/DAY+1DAY-1SECOND]' ], [ new DateTime('2021-01-01 00:00:00'), 'P1W', 'P2M', 'sp_date_list:[2021-01-01T00:00:00Z-7DAYS/DAY' . - ' TO 2021-01-01T00:00:00Z-2MONTHS/DAY]' + ' TO 2021-01-01T00:00:00Z+2MONTHS/DAY+1DAY-1SECOND]' ], ]; } From dceacc7b325f3744eafa2228a71e87175b0b7b5b Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Fri, 26 Apr 2024 15:56:06 +0200 Subject: [PATCH 137/145] style: fix codestyles --- src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php index f16ae61..a919819 100644 --- a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -56,8 +56,10 @@ private function getBaseInSolrSyntax(): string return $formatter->format('Y-m-d\TH:i:s\Z'); } - private function toSolrIntervalSyntax(DateInterval $value, string $operator): string - { + private function toSolrIntervalSyntax( + DateInterval $value, + string $operator + ): string { $interval = $this->getBaseInSolrSyntax(); if ($value->y > 0) { $interval = $interval . $operator . $value->y . 'YEARS'; From eff2d808135e0cb9773e7f7ce1fb27f28ab14d4a Mon Sep 17 00:00:00 2001 From: Holger Veltrup <92872893+sitepark-veltrup@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:15:42 +0200 Subject: [PATCH 138/145] Update src/Service/Indexer/IndexerProgressState.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mario Schäper <95750382+sitepark-schaeper@users.noreply.github.com> --- src/Service/Indexer/IndexerProgressState.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index f531f13..2d8332e 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -29,16 +29,16 @@ public function __construct( public function prepare(string $message): void { $this->status = new IndexerStatus( - IndexerStatusState::PREPARING, - new DateTime(), - null, - 0, - 0, - 0, - new DateTime(), - 0, - 0, - $message + state: IndexerStatusState::PREPARING, + startTime: new DateTime(), + endTime: null, + total: 0, + processed: 0, + skipped: 0, + lastUpdate: new DateTime(), + updated: 0, + errors: 0, + prepareMessage: $message, ); $this->statusStore->store( $this->getStatusStoreKey(), From 766787accd1ee11333584e577efc6de876c7f4f5 Mon Sep 17 00:00:00 2001 From: Holger Veltrup <92872893+sitepark-veltrup@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:16:04 +0200 Subject: [PATCH 139/145] Update src/Service/Indexer/IndexerProgressState.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mario Schäper <95750382+sitepark-schaeper@users.noreply.github.com> --- src/Service/Indexer/IndexerProgressState.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index 2d8332e..c5992d6 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -83,15 +83,15 @@ public function startUpdate(int $total): void $this->isUpdate = true; $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); $this->status = new IndexerStatus( - IndexerStatusState::RUNNING, - $storedStatus->startTime, - $storedStatus->endTime, - $storedStatus->total + $total, - $storedStatus->processed, - $storedStatus->skipped, - new DateTime(), - $storedStatus->updated, - $storedStatus->errors, + state: IndexerStatusState::RUNNING, + startTime: $storedStatus->startTime, + endTime: $storedStatus->endTime, + total: $storedStatus->total + $total, + processed: $storedStatus->processed, + skipped: $storedStatus->skipped, + lastUpdate: new DateTime(), + updated: $storedStatus->updated, + errors: $storedStatus->errors, ); $this->statusStore->store( $this->getStatusStoreKey(), From 7691a8332c7f273643ee0bd386a896833024c460 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 29 Apr 2024 09:14:37 +0200 Subject: [PATCH 140/145] fix: add space in error message --- src/Console/Command/Io/TypifiedInput.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Command/Io/TypifiedInput.php b/src/Console/Command/Io/TypifiedInput.php index 43af47a..0147d06 100644 --- a/src/Console/Command/Io/TypifiedInput.php +++ b/src/Console/Command/Io/TypifiedInput.php @@ -22,7 +22,7 @@ public function getStringOption(string $name): string $value = $this->input->getOption($name); if (!is_string($value)) { throw new InvalidArgumentException( - 'option' . $name . ' must be a string: ' . $value + 'option ' . $name . ' must be a string: ' . $value ); } return $value; From e2a0f2905713ac30e3cd7c1b4a255bbfef0cb755 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 29 Apr 2024 09:15:22 +0200 Subject: [PATCH 141/145] fix: typo --- src/Console/Command/MoreLikeThis.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php index 79ddf37..09d74d8 100644 --- a/src/Console/Command/MoreLikeThis.php +++ b/src/Console/Command/MoreLikeThis.php @@ -37,7 +37,7 @@ public function __construct( protected function configure(): void { $this - ->setHelp('Command to performs a more-like-this search') + ->setHelp('Command to perform a more-like-this search') ->addArgument( 'location', InputArgument::REQUIRED, From 390c6852d8557cd967cc502d93aabe017202f1a3 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 29 Apr 2024 09:29:45 +0200 Subject: [PATCH 142/145] fix: sanitize file-system path --- src/Service/Indexer/IndexerStatusStore.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php index 52ba416..602f525 100644 --- a/src/Service/Indexer/IndexerStatusStore.php +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -119,7 +119,9 @@ private function createBaseDirectory(): void private function getStatusFile(string $key): string { + $sanitizedKey = str_replace('\\', '', $key); + $sanitizedKey = basename($sanitizedKey); return $this->basedir . - '/atoolo.search.index.' . $key . ".status.json"; + '/atoolo.search.index.' . $sanitizedKey . ".status.json"; } } From b3a9b32abc5685a97b83ea13063b84252b0960fd Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 29 Apr 2024 10:09:30 +0200 Subject: [PATCH 143/145] refactor: throw UnsupportedIndexLanguageException for unsupported language --- .../UnsupportedIndexLanguageException.php | 35 +++++++++++++++++ src/Service/IndexName.php | 5 +++ .../Indexer/InternalResourceIndexer.php | 25 ++++++------ src/Service/ResourceChannelBasedIndexName.php | 16 +++++++- .../UnsupportedIndexLanguageExceptionTest.php | 38 +++++++++++++++++++ .../Indexer/InternalResourceIndexerTest.php | 8 ++++ .../ResourceChannelBasedIndexNameTest.php | 9 ++--- 7 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 src/Exception/UnsupportedIndexLanguageException.php create mode 100644 test/Exception/UnsupportedIndexLanguageExceptionTest.php diff --git a/src/Exception/UnsupportedIndexLanguageException.php b/src/Exception/UnsupportedIndexLanguageException.php new file mode 100644 index 0000000..8676722 --- /dev/null +++ b/src/Exception/UnsupportedIndexLanguageException.php @@ -0,0 +1,35 @@ +code . ': ' . $message, + $code, + $previous + ); + } + + public function getIndex(): string + { + return $this->index; + } + + public function getLang(): ResourceLanguage + { + return $this->lang; + } +} diff --git a/src/Service/IndexName.php b/src/Service/IndexName.php index 2fe9163..1d3ac99 100644 --- a/src/Service/IndexName.php +++ b/src/Service/IndexName.php @@ -5,9 +5,14 @@ namespace Atoolo\Search\Service; use Atoolo\Resource\ResourceLanguage; +use Atoolo\Search\Exception\UnsupportedIndexLanguageException; interface IndexName { + /** + * @throws UnsupportedIndexLanguageException Is thrown if no valid index + * can be determined for the language. + */ public function name(ResourceLanguage $lang): string; /** diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php index 3547f60..c633042 100644 --- a/src/Service/Indexer/InternalResourceIndexer.php +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -10,6 +10,7 @@ use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Indexer\IndexerParameter; use Atoolo\Search\Dto\Indexer\IndexerStatus; +use Atoolo\Search\Exception\UnsupportedIndexLanguageException; use Atoolo\Search\Indexer; use Exception; use Psr\Log\LoggerInterface; @@ -248,23 +249,19 @@ private function indexTranslationSplittedResources( ); foreach ($splitterResult->getLanguages() as $lang) { - $langIndex = $this->indexService->getIndex($lang); - - if ($index === $langIndex) { - $this->handleError( - 'No Index for language "' . $lang->code . '" ' . - 'found (base index: "' . $index . '")' + try { + $langIndex = $this->indexService->getIndex($lang); + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + $lang, + $langIndex, + $splitterResult->getTranslations($lang) ); + } catch (UnsupportedIndexLanguageException $e) { + $this->handleError($e->getMessage()); continue; } - - $this->indexResourcesPerLanguageIndex( - $processId, - $parameter, - $lang, - $langIndex, - $splitterResult->getTranslations($lang) - ); } } diff --git a/src/Service/ResourceChannelBasedIndexName.php b/src/Service/ResourceChannelBasedIndexName.php index f481dcc..dabf43a 100644 --- a/src/Service/ResourceChannelBasedIndexName.php +++ b/src/Service/ResourceChannelBasedIndexName.php @@ -6,6 +6,7 @@ use Atoolo\Resource\ResourceChannel; use Atoolo\Resource\ResourceLanguage; +use Atoolo\Search\Exception\UnsupportedIndexLanguageException; class ResourceChannelBasedIndexName implements IndexName { @@ -14,12 +15,17 @@ public function __construct( ) { } + /** + * @throws UnsupportedIndexLanguageException + */ public function name(ResourceLanguage $lang): string { $locale = $this->langToAvailableLocale($this->resourceChannel, $lang); + if (empty($locale)) { return $this->resourceChannel->searchIndex; } + return $this->resourceChannel->searchIndex . '-' . $locale; } @@ -40,6 +46,9 @@ public function names(): array return $names; } + /** + * @throws UnsupportedIndexLanguageException + */ private function langToAvailableLocale( ResourceChannel $resourceChannel, ResourceLanguage $lang @@ -56,6 +65,11 @@ private function langToAvailableLocale( return $availableLocale; } } - return ''; + throw new UnsupportedIndexLanguageException( + $resourceChannel->searchIndex, + $lang, + 'No valid index can be determined for the language ' . + $lang->code + ); } } diff --git a/test/Exception/UnsupportedIndexLanguageExceptionTest.php b/test/Exception/UnsupportedIndexLanguageExceptionTest.php new file mode 100644 index 0000000..f72b9d1 --- /dev/null +++ b/test/Exception/UnsupportedIndexLanguageExceptionTest.php @@ -0,0 +1,38 @@ +assertEquals( + 'test', + $e->getIndex(), + 'unexpected index' + ); + } + + public function testGetLang(): void + { + $e = new UnsupportedIndexLanguageException( + 'test', + ResourceLanguage::of('de') + ); + $this->assertEquals( + ResourceLanguage::of('de'), + $e->getLang(), + 'unexpected index' + ); + } +} diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php index 2f884ce..b55cf3f 100644 --- a/test/Service/Indexer/InternalResourceIndexerTest.php +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -8,6 +8,7 @@ use Atoolo\Resource\ResourceLoader; use Atoolo\Resource\ResourceLocation; use Atoolo\Search\Dto\Indexer\IndexerConfiguration; +use Atoolo\Search\Exception\UnsupportedIndexLanguageException; use Atoolo\Search\Service\Indexer\DocumentEnricher; use Atoolo\Search\Service\Indexer\IndexerConfigurationLoader; use Atoolo\Search\Service\Indexer\IndexerProgressHandler; @@ -117,6 +118,13 @@ public function setUp(): void if ($lang->code === 'en') { return 'test-en_US'; } + if ($lang->code === 'fr') { + throw new UnsupportedIndexLanguageException( + 'test', + $lang, + 'unsupported language' + ); + } return 'test'; }); $this->solrIndexService->method('updater') diff --git a/test/Service/ResourceChannelBasedIndexNameTest.php b/test/Service/ResourceChannelBasedIndexNameTest.php index 84a501b..9f5b1dd 100644 --- a/test/Service/ResourceChannelBasedIndexNameTest.php +++ b/test/Service/ResourceChannelBasedIndexNameTest.php @@ -6,6 +6,7 @@ use Atoolo\Resource\ResourceChannel; use Atoolo\Resource\ResourceLanguage; +use Atoolo\Search\Exception\UnsupportedIndexLanguageException; use Atoolo\Search\Service\ResourceChannelBasedIndexName; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -58,12 +59,8 @@ public function testNameWithLang(): void public function testNameWithUnsupportedLang(): void { - $this->assertEquals( - 'test', - $this->indexName->name(ResourceLanguage::of('it')), - 'The default index name should be returned ' . - 'if the language is not supported' - ); + $this->expectException(UnsupportedIndexLanguageException::class); + $this->indexName->name(ResourceLanguage::of('it')); } public function testNames(): void From 4b7cf32b887c73cf29a81585af7a7dfd496e3e27 Mon Sep 17 00:00:00 2001 From: Holger Veltrup Date: Mon, 29 Apr 2024 11:16:14 +0200 Subject: [PATCH 144/145] fix: set the query-time to 0 if none can be determined --- src/Service/Search/SolrMoreLikeThis.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php index 6987b84..9f7159e 100644 --- a/src/Service/Search/SolrMoreLikeThis.php +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -75,7 +75,7 @@ private function buildResult( offset: 0, results: $resourceList, facetGroups: [], - queryTime: $result->getQueryTime() ?? -1 + queryTime: $result->getQueryTime() ?? 0 ); } } From 2e2b71ae3e3ba86a18eee2e0dce535386b1c965d Mon Sep 17 00:00:00 2001 From: Holger Veltrup <92872893+sitepark-veltrup@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:18:19 +0200 Subject: [PATCH 145/145] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mario Schäper <95750382+sitepark-schaeper@users.noreply.github.com> --- src/Service/Indexer/IndexerProgressState.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Service/Indexer/IndexerProgressState.php b/src/Service/Indexer/IndexerProgressState.php index c5992d6..7e1ed17 100644 --- a/src/Service/Indexer/IndexerProgressState.php +++ b/src/Service/Indexer/IndexerProgressState.php @@ -59,15 +59,15 @@ public function start(int $total): void } $this->status = new IndexerStatus( - IndexerStatusState::RUNNING, - $startTime, - null, - $total, - 0, - 0, - new DateTime(), - 0, - 0 + state: IndexerStatusState::RUNNING, + startTime: $startTime, + endTime: null, + total: $total, + processed: 0, + skipped: 0, + lastUpdate: new DateTime(), + updated: 0, + errors: 0, ); $this->statusStore->store( $this->getStatusStoreKey(),