diff --git a/.gitignore b/.gitignore index f86b9c0..6fda8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !var/cache/.gitkeep /var/log/* !var/log/.gitkeep +/var/test/* /tools .phpactor.json composer.lock diff --git a/README.md b/README.md index c93069c..c7e0153 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ -[![codecov](https://codecov.io/gh/sitepark/atoolo-resource/graph/badge.svg?token=QwvDRxKEa2)](https://codecov.io/gh/sitepark/atoolo-resource) +[![codecov](https://codecov.io/gh/sitepark/atoolo-search/graph/badge.svg?token=xBMwUzm34b)](https://codecov.io/gh/sitepark/atoolo-search) +![phpstan](https://img.shields.io/badge/PHPStan-level%209-brightgreen) +![php](https://img.shields.io/badge/PHP-8.1-blue) +![php](https://img.shields.io/badge/PHP-8.2-blue) +![php](https://img.shields.io/badge/PHP-8.3-blue) # Atoolo search -Provides services with which a Solr index can be filled and searched for [resources](https://github.com/sitepark/atoolo-resource) via this index. +Provides services with which a Solr index can be filled and searched for [resources](https://github.com/sitepark/atoolo-resource) via a index. + +[Documentation](https://sitepark.github.io/atoolo-docs/develop/components/search/) diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..9c84b77 --- /dev/null +++ b/bin/console @@ -0,0 +1,20 @@ +#!/usr/bin/env php +load('commands.yml'); +$container->compile(); + +$application = $container->get(Application::class); + +$application->run(); \ No newline at end of file diff --git a/composer.json b/composer.json index cb78212..71cb9a2 100644 --- a/composer.json +++ b/composer.json @@ -3,32 +3,26 @@ "description": "Indexing und searching", "license": "MIT", "type": "library", - "authors": [{ - "name": "veltrup", - "email": "veltrup@sitepark.com" - }], - "autoload": { - "psr-4": { - "Atoolo\\Search\\": "src" - } - }, - "autoload-dev": { - "psr-4": { - "Atoolo\\Search\\Test\\": "test" + "authors": [ + { + "name": "veltrup", + "email": "veltrup@sitepark.com" } - }, - "minimum-stability": "dev", - "prefer-stable": true, + ], "require": { "php": ">=8.1 <8.4.0", - "atoolo/resource": "dev-feature/hierarchy-loader", + "ext-intl": "*", + "atoolo/resource": "dev-main", "solarium/solarium": "^6.3", - "symfony/config": "^6.3", - "symfony/console": "^6.3", - "symfony/dependency-injection": "^6.3", - "symfony/event-dispatcher": "^6.3", - "symfony/finder": "^6.3", - "symfony/yaml": "^6.3" + "symfony/config": "^6.3 || ^7.0", + "symfony/console": "^6.3 || ^7.0", + "symfony/dependency-injection": "^6.3 || ^7.0", + "symfony/event-dispatcher": "^6.3 || ^7.0", + "symfony/finder": "^6.3 || ^7.0", + "symfony/lock": "^6.3 || ^7.0", + "symfony/property-access": "^6.3 || ^7.0", + "symfony/serializer": "^6.3 || ^7.0", + "symfony/yaml": "^6.3 || ^7.0" }, "require-dev": { "dealerdirect/phpcodesniffer-composer-installer": "^1.0", @@ -36,10 +30,34 @@ "phpcompatibility/php-compatibility": "^9.3", "phpunit/phpunit": "^10.4", "roave/security-advisories": "dev-latest", - "squizlabs/php_codesniffer": "^3.7" + "squizlabs/php_codesniffer": "^3.7", + "symfony/filesystem": "^6.3 || ^7.0" + }, + "repositories": {}, + "minimum-stability": "dev", + "prefer-stable": true, + "autoload": { + "psr-4": { + "Atoolo\\Search\\": "src" + } + }, + "autoload-dev": { + "psr-4": { + "Atoolo\\Search\\Test\\": "test" + } + }, + "config": { + "allow-plugins": { + "dealerdirect/phpcodesniffer-composer-installer": true, + "infection/extension-installer": true + }, + "optimize-autoloader": true, + "preferred-install": { + "*": "dist" + }, + "sort-packages": true }, "scripts": { - "post-install-cmd": "phive --no-progress install --force-accept-unsigned --trust-gpg-keys C00543248C87FB13,4AA394086372C20A,CF1A108D0E7AE720,51C67305FFC2E5C0", "analyse": [ "@analyse:phplint", @@ -64,18 +82,7 @@ "test": [ "@test:phpunit" ], - "test:phpunit": "./tools/phpunit.phar -c phpunit.xml --coverage-text", - "test:infection": "vendor/bin/infection --threads=8 --no-progress --only-covered -s || exit 0" - }, - "config": { - "allow-plugins": { - "dealerdirect/phpcodesniffer-composer-installer": true, - "infection/extension-installer": true - }, - "optimize-autoloader": true, - "preferred-install": { - "*": "dist" - }, - "sort-packages": true + "test:infection": "vendor/bin/infection --threads=8 --no-progress --only-covered -s || exit 0", + "test:phpunit": "./tools/phpunit.phar -c phpunit.xml --coverage-text" } } diff --git a/config/commands.yml b/config/commands.yml new file mode 100644 index 0000000..faa017c --- /dev/null +++ b/config/commands.yml @@ -0,0 +1,50 @@ +services: + _defaults: + autowire: true + autoconfigure: true + _instanceof: + Symfony\Component\Console\Command\Command: + tags: ['command'] + + Atoolo\Search\Console\: + resource: '../src/Console' + + atoolo.search.indexer.console.progressBar: + class: 'Atoolo\Search\Console\Command\Io\IndexerProgressBar' + + Atoolo\Search\Console\Command\Indexer: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.indexer.console.progressBar' + - '@atoolo.search.indexer.indexerCollection' + + Atoolo\Search\Console\Command\IndexerInternalResourceUpdate: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.indexer.console.progressBar' + - '@atoolo.search.indexer.internalResourceIndexer' + + Atoolo\Search\Console\Command\Search: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.search' + + Atoolo\Search\Console\Command\Suggest: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.suggest' + + Atoolo\Search\Console\Command\MoreLikeThis: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.moreLikeThis' + + Atoolo\Search\Console\Command\DumpIndexDocument: + arguments: + - '@atoolo.resource.resourceChannelFactory' + - '@atoolo.search.indexer.indexDocumentDumper' + + Atoolo\Search\Console\Application: + public: true + arguments: + - !tagged command \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index 8544648..f2d23b0 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -5,9 +5,14 @@ bootstrap="vendor/autoload.php" executionOrder="random" cacheResultFile="var/cache/.phpunit.result.cache" - cacheDirectory="var/cache/.phpunit.cache"> + cacheDirectory="var/cache/.phpunit.cache" + displayDetailsOnTestsThatTriggerDeprecations="true" + displayDetailsOnTestsThatTriggerErrors="true" + displayDetailsOnTestsThatTriggerNotices="true" + displayDetailsOnTestsThatTriggerWarnings="true"> - + diff --git a/src/Console/Application.php b/src/Console/Application.php new file mode 100644 index 0000000..96a4b3e --- /dev/null +++ b/src/Console/Application.php @@ -0,0 +1,22 @@ + $commands + */ + public function __construct(iterable $commands = []) + { + parent::__construct(); + foreach ($commands as $command) { + $this->add($command); + } + } +} diff --git a/src/Console/Command/DumpIndexDocument.php b/src/Console/Command/DumpIndexDocument.php new file mode 100644 index 0000000..4355329 --- /dev/null +++ b/src/Console/Command/DumpIndexDocument.php @@ -0,0 +1,70 @@ +setHelp('Command to dump a index-document') + ->addArgument( + 'paths', + InputArgument::REQUIRED | InputArgument::IS_ARRAY, + 'Resources paths or directories of resources to be indexed.' + ) + ; + } + + /** + * @throws JsonException + */ + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $typedInput = new TypifiedInput($input); + + $paths = $typedInput->getArrayArgument('paths'); + + $resourceChannel = $this->channelFactory->create(); + $io = new SymfonyStyle($input, $output); + $io->title('Channel: ' . $resourceChannel->name); + + $dump = $this->dumper->dump($paths); + + foreach ($dump as $fields) { + $output->writeln(json_encode( + $fields, + JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT + )); + } + + return Command::SUCCESS; + } +} diff --git a/src/Console/Command/Indexer.php b/src/Console/Command/Indexer.php new file mode 100644 index 0000000..7c55c62 --- /dev/null +++ b/src/Console/Command/Indexer.php @@ -0,0 +1,111 @@ +setHelp('Command to fill a search index') + ->addArgument( + 'paths', + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'Resources paths or directories of resources to be indexed.' + ) + ->addOption( + 'source', + null, + InputArgument::OPTIONAL, + 'Uses only the indexer of a specific source', + '' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $typedInput = new TypifiedInput($input); + $this->output = $output; + $this->io = new SymfonyStyle($input, $output); + + $source = $typedInput->getStringOption('source'); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + foreach ($this->indexers->getIndexers() as $indexer) { + if (!empty($source) && $indexer->getSource() !== $source) { + continue; + } + if ($indexer->enabled()) { + $this->io->newLine(); + $this->io->section( + 'Index with Indexer "' . $indexer->getName() . '"' + ); + $progressHandler = $indexer->getProgressHandler(); + $this->progressBar->init($progressHandler); + $indexer->setProgressHandler($this->progressBar); + try { + $status = $indexer->index(); + } finally { + $indexer->setProgressHandler($progressHandler); + } + $this->io->newLine(2); + $this->io->section("Status"); + $this->io->text($status->getStatusLine()); + $this->io->newLine(); + $this->errorReport(); + } + } + + return Command::SUCCESS; + } + + protected function errorReport(): void + { + if (empty($this->progressBar->getErrors())) { + return; + } + $this->io->section("Error Report"); + + foreach ($this->progressBar->getErrors() as $error) { + if ($this->io->isVerbose() && $this->getApplication() !== null) { + $this->getApplication()->renderThrowable($error, $this->output); + } else { + $this->io->error($error->getMessage()); + } + } + } +} diff --git a/src/Console/Command/IndexerInternalResourceUpdate.php b/src/Console/Command/IndexerInternalResourceUpdate.php new file mode 100644 index 0000000..c710fdc --- /dev/null +++ b/src/Console/Command/IndexerInternalResourceUpdate.php @@ -0,0 +1,99 @@ +setHelp('Command to update internal resources in search index') + ->addArgument( + 'paths', + InputArgument::REQUIRED | InputArgument::IS_ARRAY, + 'Resources paths or directories of resources to be updated.' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $typedInput = new TypifiedInput($input); + $this->output = $output; + $this->io = new SymfonyStyle($input, $output); + + $paths = $typedInput->getArrayArgument('paths'); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + $this->io->section( + 'Index resource paths with Indexer "' . + $this->indexer->getName() . '"' + ); + $this->io->listing($paths); + $progressHandler = $this->indexer->getProgressHandler(); + $this->progressBar->init($progressHandler); + $this->indexer->setProgressHandler($this->progressBar); + try { + $status = $this->indexer->update($paths); + } finally { + $this->indexer->setProgressHandler($progressHandler); + } + $this->io->newLine(2); + $this->io->section("Status"); + $this->io->text($status->getStatusLine()); + $this->io->newLine(); + $this->errorReport(); + + + return Command::SUCCESS; + } + + protected function errorReport(): void + { + if (empty($this->progressBar->getErrors())) { + return; + } + $this->io->section("Error Report"); + + foreach ($this->progressBar->getErrors() as $error) { + if ($this->io->isVerbose() && $this->getApplication() !== null) { + $this->getApplication()->renderThrowable($error, $this->output); + } else { + $this->io->error($error->getMessage()); + } + } + } +} diff --git a/src/Console/Command/Io/IndexerProgressBar.php b/src/Console/Command/Io/IndexerProgressBar.php new file mode 100644 index 0000000..bf96aac --- /dev/null +++ b/src/Console/Command/Io/IndexerProgressBar.php @@ -0,0 +1,128 @@ + + */ + private array $errors = []; + + public function __construct( + private readonly OutputInterface $output = new ConsoleOutput() + ) { + } + + public function init( + IndexerProgressHandler $progressHandler + ): void { + $this->currentProgressHandler = $progressHandler; + $this->errors = []; + $this->progressBar = null; + } + + public function prepare(string $message): void + { + $this->currentProgressHandler->prepare($message); + $this->output->writeln($message); + $this->prepareLines++; + } + + public function start(int $total): void + { + if ($this->prepareLines > 0) { + $this->output->write("\x1b[" . $this->prepareLines . "A"); + } + $this->currentProgressHandler->start($total); + $this->progressBar = new ProgressBar($this->output, $total); + $this->formatProgressBar('green'); + } + + /** + * @throws ExceptionInterface + */ + public function startUpdate(int $total): void + { + $this->currentProgressHandler->startUpdate($total); + $this->progressBar = new ProgressBar($this->output, $total); + $this->formatProgressBar('green'); + } + + /** + * @throws JsonException + */ + public function advance(int $step): void + { + $this->currentProgressHandler->advance($step); + $this->progressBar?->advance($step); + } + + public function skip(int $step): void + { + $this->currentProgressHandler->skip($step); + } + + private function formatProgressBar(string $color): void + { + $this->progressBar?->setBarCharacter('•'); + $this->progressBar?->setEmptyBarCharacter('⚬'); + $this->progressBar?->setProgressCharacter('➤'); + $this->progressBar?->setFormat( + "%current%/%max% [%bar%] %percent:3s%%\n" . + " %estimated:-20s% %memory:20s%" + ); + } + + public function error(Throwable $throwable): void + { + $this->currentProgressHandler->error($throwable); + $this->formatProgressBar('red'); + $this->errors[] = $throwable; + } + + /** + * @throws JsonException + */ + public function finish(): void + { + $this->currentProgressHandler->finish(); + $this->progressBar?->finish(); + } + + /** + * @return array + */ + public function getErrors(): array + { + return $this->errors; + } + + public function getStatus(): IndexerStatus + { + return $this->currentProgressHandler->getStatus(); + } + + public function abort(): void + { + $this->currentProgressHandler->abort(); + $this->progressBar?->finish(); + } +} diff --git a/src/Console/Command/Io/TypifiedInput.php b/src/Console/Command/Io/TypifiedInput.php new file mode 100644 index 0000000..0147d06 --- /dev/null +++ b/src/Console/Command/Io/TypifiedInput.php @@ -0,0 +1,66 @@ +input->getOption($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + 'option ' . $name . ' must be a string: ' . $value + ); + } + return $value; + } + + public function getIntOption(string $name): int + { + $value = $this->input->getOption($name); + if (!is_numeric($value)) { + throw new InvalidArgumentException( + 'option ' . $name . ' must be a integer: ' . $value + ); + } + return (int)$value; + } + + public function getStringArgument(string $name): string + { + $value = $this->input->getArgument($name); + if (!is_string($value)) { + throw new InvalidArgumentException( + 'argument ' . $name . ' must be a string' + ); + } + return $value; + } + + /** + * @return string[] + */ + public function getArrayArgument(string $name): array + { + $value = $this->input->getArgument($name); + if (!is_array($value)) { + throw new InvalidArgumentException( + 'argument ' . $name . ' must be a array' + ); + } + return $value; + } +} diff --git a/src/Console/Command/MoreLikeThis.php b/src/Console/Command/MoreLikeThis.php new file mode 100644 index 0000000..09d74d8 --- /dev/null +++ b/src/Console/Command/MoreLikeThis.php @@ -0,0 +1,106 @@ +setHelp('Command to perform a more-like-this search') + ->addArgument( + 'location', + InputArgument::REQUIRED, + 'Location of the resource to which the MoreLikeThis ' . + 'search is to be applied.' + ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->input = new TypifiedInput($input); + $this->io = new SymfonyStyle($input, $output); + + $location = ResourceLocation::of( + $this->input->getStringArgument('location'), + ResourceLanguage::of( + $this->input->getStringOption('lang') + ) + ); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + $query = $this->buildQuery($location); + $result = $this->searcher->moreLikeThis($query); + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function buildQuery( + ResourceLocation $location, + ): MoreLikeThisQuery { + $filterList = []; + return new MoreLikeThisQuery( + location: $location, + filter: $filterList, + limit: 5, + fields: ['content'] + ); + } + + protected function outputResult(SearchResult $result): void + { + if ($result->total === 0) { + $this->io->text('No results found.'); + return; + } + $this->io->text($result->total . " Results:"); + foreach ($result as $resource) { + $this->io->text($resource->location); + } + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); + } +} diff --git a/src/Console/Command/Search.php b/src/Console/Command/Search.php new file mode 100644 index 0000000..47b3322 --- /dev/null +++ b/src/Console/Command/Search.php @@ -0,0 +1,119 @@ +setHelp('Command to performs a search') + ->addArgument( + 'text', + InputArgument::REQUIRED, + 'Text with which to search.' + ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + + $this->input = new TypifiedInput($input); + $this->io = new SymfonyStyle($input, $output); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + $query = $this->buildQuery($input); + + $result = $this->searcher->search($query); + + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function buildQuery(InputInterface $input): SearchQuery + { + $builder = new SearchQueryBuilder(); + + $text = $this->input->getStringArgument('text'); + $builder->text($text); + + // TODO: filter + + // TODO: facet + + return $builder->build(); + } + + protected function outputResult( + SearchResult $result + ): void { + + if ($result->total === 0) { + $this->io->text('No results found'); + return; + } + + $this->io->section('Results (' . $result->total . ')'); + foreach ($result as $resource) { + $this->io->text($resource->location); + } + + if (count($result->facetGroups) > 0) { + $this->io->section('Facets'); + foreach ($result->facetGroups as $facetGroup) { + $this->io->section($facetGroup->key); + $listing = []; + foreach ($facetGroup->facets as $facet) { + $listing[] = + $facet->key . + ' (' . $facet->hits . ')'; + } + $this->io->listing($listing); + } + } + + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); + } +} diff --git a/src/Console/Command/Suggest.php b/src/Console/Command/Suggest.php new file mode 100644 index 0000000..4d81c34 --- /dev/null +++ b/src/Console/Command/Suggest.php @@ -0,0 +1,106 @@ +setHelp('Command that performs a suggest search') + ->addArgument( + 'terms', + InputArgument::REQUIRED, + 'Suggest terms.' + ) + ->addOption( + 'lang', + null, + InputArgument::OPTIONAL, + 'Language to be used for the search. (de, en, fr, it, ...)', + '' + ) + ; + } + + protected function execute( + InputInterface $input, + OutputInterface $output + ): int { + $this->input = new TypifiedInput($input); + $this->io = new SymfonyStyle($input, $output); + $terms = $this->input->getStringArgument('terms'); + $lang = $this->input->getStringOption('lang'); + + $resourceChannel = $this->channelFactory->create(); + $this->io->title('Channel: ' . $resourceChannel->name); + + $query = $this->buildQuery($terms, $lang); + + $result = $this->search->suggest($query); + + $this->outputResult($result); + + return Command::SUCCESS; + } + + protected function buildQuery(string $terms, string $lang): SuggestQuery + { + $excludeMedia = new ObjectTypeFilter(['media'], 'media'); + $excludeMedia = $excludeMedia->exclude(); + return new SuggestQuery( + $terms, + $lang, + [ + new ArchiveFilter(), + $excludeMedia + ] + ); + } + + protected function outputResult(SuggestResult $result): void + { + if (empty($result->suggestions)) { + $this->io->text('No suggestions found'); + return; + } + + foreach ($result as $suggest) { + $this->io->text( + $suggest->term . + ' (' . $suggest->hits . ')' + ); + } + $this->io->text('Query-Time: ' . $result->queryTime . 'ms'); + } +} diff --git a/src/Dto/Indexer/IndexerConfiguration.php b/src/Dto/Indexer/IndexerConfiguration.php new file mode 100644 index 0000000..d03df38 --- /dev/null +++ b/src/Dto/Indexer/IndexerConfiguration.php @@ -0,0 +1,20 @@ +chunkSize < 10) { + throw new \InvalidArgumentException( + 'chunk size must be greater than 9' + ); + } + } +} diff --git a/src/Dto/Indexer/IndexerStatus.php b/src/Dto/Indexer/IndexerStatus.php new file mode 100644 index 0000000..7ea77aa --- /dev/null +++ b/src/Dto/Indexer/IndexerStatus.php @@ -0,0 +1,87 @@ +endTime; + if ($endTime === null || $endTime->getTimestamp() === 0) { + $endTime = new DateTime(); + } + $duration = $this->startTime->diff($endTime); + + $lastUpdate = $this->lastUpdate; + if ($lastUpdate->getTimestamp() === 0) { + $lastUpdate = $endTime; + } + + if ($this->state === IndexerStatusState::PREPARING) { + return + '[' . $this->state->name . '] ' . + 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . + 'message: ' . $this->prepareMessage; + } + + return + '[' . $this->state->name . '] ' . + 'start: ' . $this->startTime->format('d.m.Y H:i') . ', ' . + 'time: ' . $duration->format('%Hh %Im %Ss') . ', ' . + 'processed: ' . $this->processed . "/" . $this->total . ', ' . + 'skipped: ' . $this->skipped . ', ' . + 'lastUpdate: ' . $lastUpdate->format('d.m.Y H:i') . ', ' . + 'updated: ' . $this->updated . ', ' . + 'errors: ' . $this->errors; + } +} diff --git a/src/Dto/Indexer/IndexerStatusState.php b/src/Dto/Indexer/IndexerStatusState.php new file mode 100644 index 0000000..8e22865 --- /dev/null +++ b/src/Dto/Indexer/IndexerStatusState.php @@ -0,0 +1,17 @@ +formatDate($this->from) . + ' TO ' . + $this->formatDate($this->to) . + ']'; + } + + private function formatDate(?DateTime $date): string + { + if ($date === null) { + return '*'; + } + + $formatter = clone $date; + $formatter->setTimezone(new \DateTimeZone('UTC')); + return $formatter->format('Y-m-d\TH:i:s\Z'); + } +} diff --git a/src/Dto/Search/Query/Filter/AndFilter.php b/src/Dto/Search/Query/Filter/AndFilter.php new file mode 100644 index 0000000..253a492 --- /dev/null +++ b/src/Dto/Search/Query/Filter/AndFilter.php @@ -0,0 +1,29 @@ +filter as $filter) { + $query[] = $filter->getQuery(); + } + + return '(' . implode(' AND ', $query) . ')'; + } +} diff --git a/src/Dto/Search/Query/Filter/ArchiveFilter.php b/src/Dto/Search/Query/Filter/ArchiveFilter.php new file mode 100644 index 0000000..2b12b39 --- /dev/null +++ b/src/Dto/Search/Query/Filter/ArchiveFilter.php @@ -0,0 +1,20 @@ +values = $values; + parent::__construct( + $key, + $key !== null ? [$key] : [] + ); + } + + public function getQuery(): string + { + $filterValue = count($this->values) === 1 + ? $this->values[0] + : '(' . implode(' ', $this->values) . ')'; + return $this->field . ':' . $filterValue; + } + + public function exclude(): FieldFilter + { + $field = $this->field; + if (!str_starts_with($field, '-')) { + $field = '-' . $field; + } + return new FieldFilter( + $field, + $this->values, + $this->key + ); + } +} diff --git a/src/Dto/Search/Query/Filter/Filter.php b/src/Dto/Search/Query/Filter/Filter.php new file mode 100644 index 0000000..cb5edf1 --- /dev/null +++ b/src/Dto/Search/Query/Filter/Filter.php @@ -0,0 +1,22 @@ +filter->getQuery(); + } +} diff --git a/src/Dto/Search/Query/Filter/ObjectTypeFilter.php b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php new file mode 100644 index 0000000..f854b0e --- /dev/null +++ b/src/Dto/Search/Query/Filter/ObjectTypeFilter.php @@ -0,0 +1,25 @@ +filter as $filter) { + $query[] = $filter->getQuery(); + } + + return '(' . implode(' OR ', $query) . ')'; + } +} diff --git a/src/Dto/Search/Query/Filter/QueryFilter.php b/src/Dto/Search/Query/Filter/QueryFilter.php new file mode 100644 index 0000000..6a25853 --- /dev/null +++ b/src/Dto/Search/Query/Filter/QueryFilter.php @@ -0,0 +1,22 @@ +query; + } +} diff --git a/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php new file mode 100644 index 0000000..a919819 --- /dev/null +++ b/src/Dto/Search/Query/Filter/RelativeDateRangeFilter.php @@ -0,0 +1,91 @@ +toSolrDateRage(); + } + + private function toSolrDateRage(): string + { + if ($this->before === null) { + $from = $this->getBaseInSolrSyntax() . "/DAY"; + } else { + $from = $this->toSolrIntervalSyntax($this->before, '-') . + '/DAY'; + } + + if ($this->after === null) { + $to = $this->getBaseInSolrSyntax() . "/DAY+1DAY-1SECOND"; + } else { + $to = $this->toSolrIntervalSyntax($this->after, '+') . + "/DAY+1DAY-1SECOND"; + } + + return '[' . $from . ' TO ' . $to . ']'; + } + + private function getBaseInSolrSyntax(): string + { + if ($this->base === null) { + return 'NOW'; + } + + $formatter = clone $this->base; + $formatter->setTimezone(new \DateTimeZone('UTC')); + return $formatter->format('Y-m-d\TH:i:s\Z'); + } + + private function toSolrIntervalSyntax( + DateInterval $value, + string $operator + ): string { + $interval = $this->getBaseInSolrSyntax(); + if ($value->y > 0) { + $interval = $interval . $operator . $value->y . 'YEARS'; + } + if ($value->m > 0) { + $interval = $interval . $operator . $value->m . 'MONTHS'; + } + if ($value->d > 0) { + $interval = $interval . $operator . $value->d . 'DAYS'; + } + if ($value->h > 0) { + throw new InvalidArgumentException( + 'Hours are not supported for the RelativeDateRangeFilter' + ); + } + if ($value->i > 0) { + throw new InvalidArgumentException( + 'Minutes are not supported for the RelativeDateRangeFilter' + ); + } + if ($value->s > 0) { + throw new InvalidArgumentException( + 'Seconds are not supported for the RelativeDateRangeFilter' + ); + } + + return $interval; + } +} diff --git a/src/Dto/Search/Query/Filter/SiteFilter.php b/src/Dto/Search/Query/Filter/SiteFilter.php new file mode 100644 index 0000000..37b786d --- /dev/null +++ b/src/Dto/Search/Query/Filter/SiteFilter.php @@ -0,0 +1,25 @@ + + */ + private array $filter = []; + + /** + * @var array + */ + private array $facets = []; + + private QueryOperator $defaultQueryOperator = + QueryOperator::OR; + + public function __construct() + { + } + + /** + * @return $this + */ + public function text(string $text): static + { + $this->text = $text; + return $this; + } + + /** + * @return $this + */ + public function lang(string $lang): static + { + $this->lang = $lang; + return $this; + } + + /** + * @return $this + */ + public function offset(int $offset): static + { + if ($offset < 0) { + throw new \InvalidArgumentException('offset is lower then 0'); + } + $this->offset = $offset; + return $this; + } + + /** + * @return $this + */ + public function limit(int $limit): static + { + if ($limit < 0) { + throw new \InvalidArgumentException('limit is lower then 0'); + } + $this->limit = $limit; + return $this; + } + + /** + * @return $this + */ + public function sort(Criteria ...$criteriaList): static + { + foreach ($criteriaList as $criteria) { + $this->sort[] = $criteria; + } + return $this; + } + + /** + * @return $this + */ + public function filter(Filter ...$filterList): static + { + foreach ($filterList as $filter) { + if ($filter->key !== null) { + foreach ($this->filter as $existingFilter) { + if ($existingFilter->key === $filter->key) { + throw new \InvalidArgumentException( + 'filter key "' . $filter->key . + '" already exists' + ); + } + } + } + $this->filter[] = $filter; + } + return $this; + } + + /** + * @return $this + */ + public function facet(Facet ...$facetList): static + { + foreach ($facetList as $facet) { + if (isset($this->facets[$facet->key])) { + throw new \InvalidArgumentException( + 'facet key "' . $facet->key . + '" already exists' + ); + } + $this->facets[$facet->key] = $facet; + } + return $this; + } + + /** + * @return $this + */ + public function defaultQueryOperator( + QueryOperator $defaultQueryOperator + ): static { + $this->defaultQueryOperator = $defaultQueryOperator; + return $this; + } + + public function build(): SearchQuery + { + return new SearchQuery( + text: $this->text, + lang: $this->lang, + offset: $this->offset, + limit: $this->limit, + sort: $this->sort, + filter: $this->filter, + facets: array_values($this->facets), + defaultQueryOperator: $this->defaultQueryOperator + ); + } +} diff --git a/src/Dto/Search/Query/Sort/Criteria.php b/src/Dto/Search/Query/Sort/Criteria.php new file mode 100644 index 0000000..54ac7af --- /dev/null +++ b/src/Dto/Search/Query/Sort/Criteria.php @@ -0,0 +1,16 @@ + + * @codeCoverageIgnore + */ +class SearchResult implements IteratorAggregate +{ + /** + * @param Resource[] $results + * @param FacetGroup[] $facetGroups + */ + public function __construct( + public readonly int $total, + public readonly int $limit, + public readonly int $offset, + public readonly array $results, + public readonly array $facetGroups, + public readonly int $queryTime + ) { + } + + public function getIterator(): Traversable + { + return new ArrayIterator($this->results); + } +} diff --git a/src/Dto/Search/Result/SuggestResult.php b/src/Dto/Search/Result/SuggestResult.php new file mode 100644 index 0000000..9d5a27a --- /dev/null +++ b/src/Dto/Search/Result/SuggestResult.php @@ -0,0 +1,33 @@ + + * @codeCoverageIgnore + */ +class SuggestResult implements IteratorAggregate +{ + /** + * @param Suggestion[] $suggestions + * @param int $queryTime + */ + public function __construct( + public readonly array $suggestions, + public readonly int $queryTime + ) { + } + + /** + * @return ArrayIterator + */ + public function getIterator(): ArrayIterator + { + return new ArrayIterator($this->suggestions); + } +} diff --git a/src/Dto/Search/Result/Suggestion.php b/src/Dto/Search/Result/Suggestion.php new file mode 100644 index 0000000..b6b77e9 --- /dev/null +++ b/src/Dto/Search/Result/Suggestion.php @@ -0,0 +1,17 @@ +__toString() . ': ' . $message, + $code, + $previous + ); + } + + public function getLocation(): ResourceLocation + { + return $this->location; + } +} diff --git a/src/Exception/MissMatchingResourceFactoryException.php b/src/Exception/MissMatchingResourceFactoryException.php new file mode 100644 index 0000000..f381727 --- /dev/null +++ b/src/Exception/MissMatchingResourceFactoryException.php @@ -0,0 +1,29 @@ +__toString() . ': ' . $message, + $code, + $previous + ); + } + + public function getLocation(): ResourceLocation + { + return $this->location; + } +} diff --git a/src/Exception/UnexpectedResultException.php b/src/Exception/UnexpectedResultException.php new file mode 100644 index 0000000..9d4bf21 --- /dev/null +++ b/src/Exception/UnexpectedResultException.php @@ -0,0 +1,28 @@ +result, + $code, + $previous + ); + } + + public function getResult(): string + { + return $this->result; + } +} diff --git a/src/Exception/UnsupportedIndexLanguageException.php b/src/Exception/UnsupportedIndexLanguageException.php new file mode 100644 index 0000000..8676722 --- /dev/null +++ b/src/Exception/UnsupportedIndexLanguageException.php @@ -0,0 +1,35 @@ +code . ': ' . $message, + $code, + $previous + ); + } + + public function getIndex(): string + { + return $this->index; + } + + public function getLang(): ResourceLanguage + { + return $this->lang; + } +} diff --git a/src/Indexer.php b/src/Indexer.php new file mode 100644 index 0000000..f18fbd7 --- /dev/null +++ b/src/Indexer.php @@ -0,0 +1,41 @@ +indexName->name(ResourceLanguage::default()) . + '-' . $this->source; + } + + protected function getConfig(): IndexerConfiguration + { + return $this->config ??= $this->configLoader->load($this->source); + } + + public function getName(): string + { + return $this->getConfig()->name; + } + + public function getSource(): string + { + return $this->source; + } + + public function getProgressHandler(): IndexerProgressHandler + { + return $this->progressHandler; + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + $this->progressHandler = $progressHandler; + } + + public function abort(): void + { + $this->aborter->requestAbortion($this->getKey()); + } + + protected function isAbortionRequested(): bool + { + return $this->aborter->isAbortionRequested($this->getKey()); + } + + public function enabled(): bool + { + return $this->configLoader->exists($this->source); + } + + abstract public function index(): IndexerStatus; + + /** + * @inheritDoc + */ + abstract public function remove(array $idList): void; +} diff --git a/src/Service/IndexName.php b/src/Service/IndexName.php new file mode 100644 index 0000000..1d3ac99 --- /dev/null +++ b/src/Service/IndexName.php @@ -0,0 +1,25 @@ +indexer->getSource(); + } + + public function getProgressHandler(): IndexerProgressHandler + { + return $this->indexer->getProgressHandler(); + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + $this->indexer->setProgressHandler($progressHandler); + } + + public function getName(): string + { + return "Background Indexer"; + } + + /** + * @param string[] $idList + */ + public function remove(array $idList): void + { + $this->indexer->remove($idList); + } + + public function abort(): void + { + $this->indexer->abort(); + } + + public function index(): IndexerStatus + { + return $this->indexer->index(); + } + + /** + * @param array $paths + */ + public function update(array $paths): IndexerStatus + { + return $this->indexer->update($paths); + } + + /** + * @throws ExceptionInterface + */ + public function getStatus(): IndexerStatus + { + return $this->statusStore->load($this->getStatusStoreKey()); + } + + private function getStatusStoreKey(): string + { + return $this->index->name(ResourceLanguage::default()) . + '-' . $this->indexer->getSource(); + } +} diff --git a/src/Service/Indexer/ContentCollector.php b/src/Service/Indexer/ContentCollector.php new file mode 100644 index 0000000..0ff5287 --- /dev/null +++ b/src/Service/Indexer/ContentCollector.php @@ -0,0 +1,62 @@ + $matchers + */ + public function __construct(private readonly iterable $matchers) + { + } + + /** + * @param array $data + */ + public function collect(array $data): string + { + $content = $this->walk([], $data); + return implode(' ', $content); + } + + /** + * @param string[] $path + * @param array $data + * @return string[] + */ + private function walk(array $path, array $data): array + { + $contentCollections = []; + foreach ($data as $key => $value) { + if (!is_array($value)) { + continue; + } + + if (is_string($key)) { + $path[] = $key; + } + + $matcherContent = []; + foreach ($this->matchers as $matcher) { + $content = $matcher->match($path, $value); + if (!is_string($content)) { + continue; + } + $matcherContent[] = $content; + } + $contentCollections[] = $matcherContent; + $contentCollections[] = $this->walk($path, $value); + + if (is_string($key)) { + array_pop($path); + } + } + + return array_merge(...$contentCollections); + } +} diff --git a/src/Service/Indexer/DocumentEnricher.php b/src/Service/Indexer/DocumentEnricher.php new file mode 100644 index 0000000..95c0e39 --- /dev/null +++ b/src/Service/Indexer/DocumentEnricher.php @@ -0,0 +1,29 @@ +> $documentEnricherList + */ + public function __construct( + private readonly ResourceLoader $resourceLoader, + private readonly iterable $documentEnricherList + ) { + } + + /** + * @param string[] $paths + * @return array> + * Returns the raw array data of the documents to be able to + * output them as JSON, for example. + */ + public function dump(array $paths): array + { + $documents = []; + foreach ($paths as $path) { + $location = ResourceLocation::of($path); + $resource = $this->resourceLoader->load($location); + $doc = new IndexSchema2xDocument(); + $processId = 'process-id'; + + foreach ($this->documentEnricherList as $enricher) { + /** @var IndexSchema2xDocument $doc */ + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + + $documents[] = $doc->getFields(); + } + + return $documents; + } +} diff --git a/src/Service/Indexer/IndexSchema2xDocument.php b/src/Service/Indexer/IndexSchema2xDocument.php new file mode 100644 index 0000000..d3fa585 --- /dev/null +++ b/src/Service/Indexer/IndexSchema2xDocument.php @@ -0,0 +1,192 @@ + + */ + private array $metaString = []; + + /** + * @var array + */ + private array $metaText = []; + + /** + * @var array + */ + private array $metaBool = []; + + /** + * @param string|string[] $value + */ + public function setMetaString(string $name, string|array $value): void + { + $this->metaString['sp_meta_string_' . $name] = $value; + } + + /** + * @param string|string[] $value + */ + public function setMetaText(string $name, string|array $value): void + { + $this->metaText['sp_meta_text_' . $name] = $value; + } + + public function setMetaBool(string $name, bool $value): void + { + $this->metaBool['sp_meta_bool_' . $name] = $value; + } + + /** + * @return array + */ + public function getFields(): array + { + return [ + ...array_filter( + get_object_vars($this), + fn($value, $key) => !is_null($value) + && !in_array($key, self::INHERITED_FIELDS, true) + && !in_array($key, self::META_FIELDS, true), + ARRAY_FILTER_USE_BOTH + ), + ... $this->metaString, + ... $this->metaText, + ... $this->metaBool, + ]; + } +} diff --git a/src/Service/Indexer/IndexerCollection.php b/src/Service/Indexer/IndexerCollection.php new file mode 100644 index 0000000..6a9514f --- /dev/null +++ b/src/Service/Indexer/IndexerCollection.php @@ -0,0 +1,41 @@ + $indexers + */ + public function __construct( + private readonly iterable $indexers, + ) { + } + + public function getIndexer(string $source): Indexer + { + foreach ($this->indexers as $indexer) { + if ($indexer->getSource() === $source) { + return $indexer; + } + } + throw new InvalidArgumentException( + 'Indexer not found for source: ' . $source + ); + } + + /** + * @return array + */ + public function getIndexers(): array + { + return $this->indexers instanceof \Traversable + ? iterator_to_array($this->indexers) + : $this->indexers; + } +} diff --git a/src/Service/Indexer/IndexerConfigurationLoader.php b/src/Service/Indexer/IndexerConfigurationLoader.php new file mode 100644 index 0000000..fe9dfb9 --- /dev/null +++ b/src/Service/Indexer/IndexerConfigurationLoader.php @@ -0,0 +1,86 @@ + + */ + public function loadAll(): array + { + $dir = $this->resourceChannel->resourceDir . '/indexer'; + if (!is_dir($dir)) { + return []; + } + + $files = glob($dir . '/*.php') ?: []; + + $configurations = []; + foreach ($files as $file) { + $source = pathinfo($file, PATHINFO_FILENAME); + $configurations[] = $this->load($source); + } + return $configurations; + } + + private function getFile(string $source): string + { + return $this->resourceChannel->resourceDir . + '/indexer/' . $source . '.php'; + } + + public function exists(string $source): bool + { + $file = $this->getFile($source); + return file_exists($file); + } + + public function load(string $source): IndexerConfiguration + { + $file = $this->getFile($source); + + if (!file_exists($file)) { + return new IndexerConfiguration( + $source, + $source, + new DataBag([]), + ); + } + + $saveErrorReporting = error_reporting(); + + try { + error_reporting(E_ERROR | E_PARSE); + ob_start(); + $data = require $file; + if (!is_array($data)) { + throw new RuntimeException( + 'The indexer configuration ' . + $file . ' should return an array' + ); + } + + return new IndexerConfiguration( + $source, + $data['name'] ?? $source, + new DataBag($data['data'] ?? []), + ); + } finally { + ob_end_clean(); + error_reporting($saveErrorReporting); + } + } +} diff --git a/src/Service/Indexer/IndexerProgressHandler.php b/src/Service/Indexer/IndexerProgressHandler.php new file mode 100644 index 0000000..58c00b4 --- /dev/null +++ b/src/Service/Indexer/IndexerProgressHandler.php @@ -0,0 +1,22 @@ +status = new IndexerStatus( + state: IndexerStatusState::PREPARING, + startTime: new DateTime(), + endTime: null, + total: 0, + processed: 0, + skipped: 0, + lastUpdate: new DateTime(), + updated: 0, + errors: 0, + prepareMessage: $message, + ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); + } + + /** + * @throws ExceptionInterface + */ + public function start(int $total): void + { + $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); + + $startTime = new DateTime(); + if ($storedStatus->state === IndexerStatusState::PREPARING) { + $startTime = $storedStatus->startTime; + } + + $this->status = new IndexerStatus( + state: IndexerStatusState::RUNNING, + startTime: $startTime, + endTime: null, + total: $total, + processed: 0, + skipped: 0, + lastUpdate: new DateTime(), + updated: 0, + errors: 0, + ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); + } + + /** + * @throws ExceptionInterface + */ + public function startUpdate(int $total): void + { + $this->isUpdate = true; + $storedStatus = $this->statusStore->load($this->getStatusStoreKey()); + $this->status = new IndexerStatus( + state: IndexerStatusState::RUNNING, + startTime: $storedStatus->startTime, + endTime: $storedStatus->endTime, + total: $storedStatus->total + $total, + processed: $storedStatus->processed, + skipped: $storedStatus->skipped, + lastUpdate: new DateTime(), + updated: $storedStatus->updated, + errors: $storedStatus->errors, + ); + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); + } + + public function advance(int $step): void + { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } + $this->status->processed += $step; + $this->status->lastUpdate = new DateTime(); + if ($this->isUpdate) { + $this->status->updated += $step; + } + $this->statusStore->store( + $this->getStatusStoreKey(), + $this->status + ); + } + + public function skip(int $step): void + { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } + $this->status->skipped += $step; + } + + public function error(Throwable $throwable): void + { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } + $this->status->errors++; + } + + public function finish(): void + { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } + if (!$this->isUpdate) { + $this->status->endTime = new DateTime(); + } + if ($this->status->state === IndexerStatusState::RUNNING) { + $this->status->state = IndexerStatusState::FINISHED; + } + $this->statusStore->store($this->getStatusStoreKey(), $this->status); + } + + public function abort(): void + { + if ($this->status === null) { + throw new LogicException( + 'Cannot advance without starting the progress' + ); + } + $this->status->state = IndexerStatusState::ABORTED; + } + + /** + * @throws ExceptionInterface + */ + public function getStatus(): IndexerStatus + { + return $this->status + ?? $this->statusStore->load($this->getStatusStoreKey()); + } + + private function getStatusStoreKey(): string + { + return $this->index->name(ResourceLanguage::default()) . + '-' . $this->source; + } +} diff --git a/src/Service/Indexer/IndexerStatusStore.php b/src/Service/Indexer/IndexerStatusStore.php new file mode 100644 index 0000000..602f525 --- /dev/null +++ b/src/Service/Indexer/IndexerStatusStore.php @@ -0,0 +1,127 @@ +serializer = new Serializer($normalizers, $encoders); + } + + /** + * @throws ExceptionInterface + */ + public function load(string $key): IndexerStatus + { + $file = $this->getStatusFile($key); + + if (!file_exists($file)) { + return IndexerStatus::empty(); + } + + if (!is_readable($file)) { + throw new InvalidArgumentException('Cannot read file ' . $file); + } + + $json = file_get_contents($file); + if ($json === false) { + // @codeCoverageIgnoreStart + $message = 'Failed to read file ' . $file; + $error = error_get_last(); + if ($error !== null) { + $message .= ': ' . $error['message']; + } + throw new RuntimeException($message); + // @codeCoverageIgnoreEnd + } + + /** @var IndexerStatus $status */ + $status = $this + ->serializer + ->deserialize($json, IndexerStatus::class, 'json'); + + return $status; + } + + public function store(string $key, IndexerStatus $status): void + { + $this->createBaseDirectory(); + + $file = $this->getStatusFile($key); + if (file_exists($file) && !is_writable($file)) { + throw new RuntimeException( + 'File ' . $file . ' is not writable' + ); + } + $json = $this + ->serializer + ->serialize($status, 'json'); + $result = file_put_contents($file, $json); + if ($result === false) { + // @codeCoverageIgnoreStart + $message = 'Unable to write indexer-status file ' . $file; + $error = error_get_last(); + if ($error !== null) { + $message .= ': ' . $error['message']; + } + throw new RuntimeException($message); + // @codeCoverageIgnoreEnd + } + } + + private function createBaseDirectory(): void + { + if ( + !is_dir($concurrentDirectory = $this->basedir) && + ( + !@mkdir( + $concurrentDirectory, + 0777, + true + ) && + !is_dir($concurrentDirectory) + ) + ) { + throw new RuntimeException(sprintf( + 'Directory "%s" was not created', + $concurrentDirectory + )); + } + + if (!is_writable($this->basedir)) { + throw new RuntimeException( + 'Directory ' . $this->basedir . ' is not writable' + ); + } + } + + private function getStatusFile(string $key): string + { + $sanitizedKey = str_replace('\\', '', $key); + $sanitizedKey = basename($sanitizedKey); + return $this->basedir . + '/atoolo.search.index.' . $sanitizedKey . ".status.json"; + } +} diff --git a/src/Service/Indexer/IndexingAborter.php b/src/Service/Indexer/IndexingAborter.php new file mode 100644 index 0000000..1a8a34d --- /dev/null +++ b/src/Service/Indexer/IndexingAborter.php @@ -0,0 +1,34 @@ +getAbortMarkerFile($index)); + } + + public function requestAbortion(string $index): void + { + touch($this->getAbortMarkerFile($index)); + } + + public function resetAbortionRequest(string $index): void + { + unlink($this->getAbortMarkerFile($index)); + } + + private function getAbortMarkerFile(string $index): string + { + return $this->workdir . '/' . $this->type . '-' . $index . '.abort'; + } +} diff --git a/src/Service/Indexer/InternalResourceIndexer.php b/src/Service/Indexer/InternalResourceIndexer.php new file mode 100644 index 0000000..c633042 --- /dev/null +++ b/src/Service/Indexer/InternalResourceIndexer.php @@ -0,0 +1,456 @@ +> $documentEnricherList + */ + public function __construct( + private readonly iterable $documentEnricherList, + private readonly ResourceFilter $resourceFilter, + private IndexerProgressHandler $progressHandler, + private readonly LocationFinder $finder, + private readonly ResourceLoader $resourceLoader, + private readonly TranslationSplitter $translationSplitter, + private readonly SolrIndexService $indexService, + private readonly IndexingAborter $aborter, + private readonly IndexerConfigurationLoader $configLoader, + private readonly string $source, + private readonly LoggerInterface $logger = new NullLogger(), + private readonly LockFactory $lockFactory = new LockFactory( + new SemaphoreStore() + ), + ) { + } + + public function enabled(): bool + { + return true; + } + + public function getName(): string + { + return $this->getIndexerParameter()->name; + } + + public function getProgressHandler(): IndexerProgressHandler + { + return $this->progressHandler; + } + + public function setProgressHandler( + IndexerProgressHandler $progressHandler + ): void { + $this->progressHandler = $progressHandler; + } + + public function getSource(): string + { + return $this->source; + } + + /** + * @param string[] $idList + */ + public function remove(array $idList): void + { + if (empty($idList)) { + return; + } + + $this->indexService->deleteByIdListForAllLanguages( + $this->source, + $idList + ); + $this->indexService->commitForAllLanguages(); + } + + public function abort(): void + { + $this->aborter->requestAbortion($this->getBaseIndex()); + } + + /** + * Indexes an entire directory structure or only selected files + * if `paths` was specified in `$parameter`. + */ + public function index(): IndexerStatus + { + $lock = $this->lockFactory->createLock( + 'indexer.' . $this->getBaseIndex() + ); + if (!$lock->acquire()) { + $this->logger->notice('Indexer is already running', [ + 'index' => $this->getBaseIndex() + ]); + return $this->progressHandler->getStatus(); + } + $param = $this->getIndexerParameter(); + + $this->logger->info('Start indexing', [ + 'index' => $this->getBaseIndex(), + 'chunkSize' => $param->chunkSize, + 'cleanupThreshold' => $param->cleanupThreshold, + ]); + + $this->progressHandler->prepare('Collect resource locations'); + + try { + $paths = $this->finder->findAll(); + $this->deleteErrorProtocol(); + $total = count($paths); + $this->progressHandler->start($total); + + $this->indexResources($param, $this->finder->findAll()); + } finally { + $lock->release(); + $this->progressHandler->finish(); + gc_collect_cycles(); + } + + return $this->progressHandler->getStatus(); + } + + /** + * @param string[] $paths + */ + public function update(array $paths): IndexerStatus + { + $collectedPaths = array_merge( + $this->finder->findPaths($paths), // resolve directories recursive + $paths + ); + $collectedPaths = array_unique($collectedPaths); + + $total = count($collectedPaths); + $this->progressHandler->startUpdate($total); + + try { + $param = $this->loadIndexerParameter(); + $this->indexResources($param, $collectedPaths); + } finally { + $this->progressHandler->finish(); + gc_collect_cycles(); + } + + return $this->progressHandler->getStatus(); + } + + private function getIndexerParameter(): IndexerParameter + { + return $this->parameter ??= ($this->loadIndexerParameter()); + } + + private function loadIndexerParameter(): IndexerParameter + { + $config = $this->configLoader->load($this->source); + return new IndexerParameter( + $config->name, + $config->data->getInt( + 'cleanupThreshold' + ), + $config->data->getInt( + 'chunkSize', + 500 + ) + ); + } + + private function getBaseIndex(): string + { + return $this->indexService->getIndex(ResourceLanguage::default()); + } + + /** + * Indexes the resources of all passed paths. + * + * @param array $pathList + */ + private function indexResources( + IndexerParameter $parameter, + array $pathList + ): void { + if (count($pathList) === 0) { + return; + } + + $splitterResult = $this->translationSplitter->split($pathList); + $this->indexTranslationSplittedResources( + $parameter, + $splitterResult + ); + } + + /** + * There is a separate Solr index for each language. This allows + * language-specific tokenizers and other language-relevant configurations + * to be used. Via the `$splitterResult` all paths are separated according + * to their languages and can be indexed separately. Each language is + * indexed separately here. + */ + private function indexTranslationSplittedResources( + IndexerParameter $parameter, + TranslationSplitterResult $splitterResult + ): void { + + $processId = uniqid('', true); + + $index = $this->indexService->getIndex(ResourceLanguage::default()); + + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + ResourceLanguage::default(), + $index, + $splitterResult->getBases() + ); + + foreach ($splitterResult->getLanguages() as $lang) { + try { + $langIndex = $this->indexService->getIndex($lang); + $this->indexResourcesPerLanguageIndex( + $processId, + $parameter, + $lang, + $langIndex, + $splitterResult->getTranslations($lang) + ); + } catch (UnsupportedIndexLanguageException $e) { + $this->handleError($e->getMessage()); + continue; + } + } + } + + /** + * The resources for a language are indexed here. + * + * @param ResourceLocation[] $locations + */ + private function indexResourcesPerLanguageIndex( + string $processId, + IndexerParameter $parameter, + ResourceLanguage $lang, + string $index, + array $locations + ): void { + + if (empty($locations)) { + return; + } + + $offset = 0; + $successCount = 0; + + $managedIndices = $this->indexService->getManagedIndices(); + if (!in_array($index, $managedIndices)) { + $this->handleError('Index "' . $index . '" not found'); + return; + } + + while (true) { + $indexedCount = $this->indexChunks( + $processId, + $lang, + $index, + $locations, + $offset, + $parameter->chunkSize + ); + gc_collect_cycles(); + if ($indexedCount === false) { + break; + } + $successCount += $indexedCount; + $offset += $parameter->chunkSize; + } + + if ( + $parameter->cleanupThreshold > 0 && + $successCount >= $parameter->cleanupThreshold + ) { + $this->indexService->deleteExcludingProcessId( + $lang, + $this->source, + $processId + ); + } + $this->indexService->commit($lang); + } + + /** + * For performance reasons, not every resource is indexed individually, + * but the index documents are first generated from several resources. + * These are then passed to Solr for indexing via a request. These + * methods accept a chunk with all paths that are to be indexed via a + * request. + * + * @param ResourceLocation[] $locations + */ + private function indexChunks( + string $processId, + ResourceLanguage $lang, + string $index, + array $locations, + int $offset, + int $length + ): int|false { + $resourceList = $this->loadResources( + $locations, + $offset, + $length + ); + if ($resourceList === false) { + return false; + } + if ($this->aborter->isAbortionRequested($index)) { + $this->aborter->resetAbortionRequest($index); + $this->progressHandler->abort(); + return false; + } + if (empty($resourceList)) { + return 0; + } + $this->progressHandler->advance(count($resourceList)); + $result = $this->add($lang, $processId, $resourceList); + + if ($result->getStatus() !== 0) { + $this->handleError($result->getResponse()->getStatusMessage()); + return 0; + } + + return count($resourceList); + } + + /** + * @param ResourceLocation[] $locations + * @return Resource[]|false + */ + private function loadResources( + array $locations, + int $offset, + int $length + ): array|false { + + $maxLength = count($locations) - $offset; + if ($maxLength <= 0) { + return false; + } + + $end = min($length, $maxLength) + $offset; + + $resourceList = []; + for ($i = $offset; $i < $end; $i++) { + $location = $locations[$i]; + try { + $resource = $this->resourceLoader->load($location); + $resourceList[] = $resource; + } catch (Throwable $e) { + $this->handleError($e); + } + } + return $resourceList; + } + + /** + * @param array $resources + */ + private function add( + ResourceLanguage $lang, + string $processId, + array $resources + ): UpdateResult { + + $updater = $this->indexService->updater($lang); + + foreach ($resources as $resource) { + if ($this->resourceFilter->accept($resource) === false) { + $this->progressHandler->skip(1); + continue; + } + try { + /** @var IndexSchema2xDocument $doc */ + $doc = $updater->createDocument(); + foreach ($this->documentEnricherList as $enricher) { + /** @var IndexSchema2xDocument $doc */ + $doc = $enricher->enrichDocument( + $resource, + $doc, + $processId + ); + } + $updater->addDocument($doc); + } catch (Throwable $e) { + $this->handleError($e); + } + } + + // this executes the query and returns the result + return $updater->update(); + } + + private function handleError(Throwable|string $error): void + { + if (is_string($error)) { + $error = new Exception($error); + } + $this->progressHandler->error($error); + $this->logger->error( + $error->getMessage(), + [ + 'exception' => $error, + ] + ); + } + + private function deleteErrorProtocol(): void + { + $this->indexService->deleteByQuery( + ResourceLanguage::default(), + 'crawl_status:error OR crawl_status:warning' + ); + } +} diff --git a/src/Service/Indexer/LocationFinder.php b/src/Service/Indexer/LocationFinder.php new file mode 100644 index 0000000..7adae22 --- /dev/null +++ b/src/Service/Indexer/LocationFinder.php @@ -0,0 +1,103 @@ +in($this->getBasePath())->exclude('WEB-IES'); + $finder->name('*.php'); + $finder->notPath('#.*-1015t.php.*#'); // preview files + $finder->files(); + + $pathList = []; + foreach ($finder as $file) { + $pathList[] = $this->toRelativePath($file->getPathname()); + } + + sort($pathList); + + return $pathList; + } + + /** + * @param string[] $paths + * @return string[] + */ + public function findPaths(array $paths): array + { + $pathList = []; + + $directories = []; + + $finder = new Finder(); + foreach ($paths as $path) { + if (!str_starts_with($path, '/')) { + $path = '/' . $path; + } + $absolutePath = $this->getBasePath() . $path; + if (is_file($absolutePath)) { + $pathList[] = $path; + continue; + } + if (is_dir($absolutePath)) { + $directories[] = $path; + } + } + + if (empty($directories)) { + return $pathList; + } + + foreach ($directories as $directory) { + $finder->in($this->getBasePath() . $directory); + } + $finder->name('*.php'); + $finder->notPath('#.*-1015t.php.*#'); // preview files + $finder->files(); + + foreach ($finder as $file) { + $pathList[] = $this->toRelativePath($file->getPathname()); + } + + sort($pathList); + + return $pathList; + } + + private function getBasePath(): string + { + return rtrim($this->resourceChannel->resourceDir, '/'); + } + + private function toRelativePath(string $path): string + { + return substr($path, strlen($this->getBasePath())); + } +} diff --git a/src/Service/Indexer/ResourceFilter.php b/src/Service/Indexer/ResourceFilter.php new file mode 100644 index 0000000..b104ae7 --- /dev/null +++ b/src/Service/Indexer/ResourceFilter.php @@ -0,0 +1,12 @@ + [ + * 'b' => [ + * 'c' => [ + * ... + * ] + * ] + * ] + * ] + * ``` + * @param array $value Value within a data structure + * that is to be checked. + * @return string|false The extracted content or `false` if the + * content is not relevant for the search index. + */ + public function match(array $path, array $value): string|false; +} diff --git a/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php new file mode 100644 index 0000000..77eb8a3 --- /dev/null +++ b/src/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricher.php @@ -0,0 +1,422 @@ + + * @phpstan-type Email array{email:string} + * @phpstan-type EmailList array + * @phpstan-type ContactData array{ + * phoneList?:PhoneList, + * emailList:EmailList + * } + * @phpstan-type AddressData array{ + * buildingName?:string, + * street?:string, + * postOfficeBoxData?: array{ + * buildingName?:string + * } + * } + * @phpstan-type ContactPoint array{ + * contactData?:ContactData, + * addressData?:AddressData + * } + * @implements DocumentEnricher + */ +class DefaultSchema2xDocumentEnricher implements DocumentEnricher +{ + public function __construct( + private readonly SiteKitNavigationHierarchyLoader $navigationLoader, + private readonly ContentCollector $contentCollector + ) { + } + + /** + * @throws DocumentEnrichingException + */ + public function enrichDocument( + Resource $resource, + IndexDocument $doc, + string $processId + ): IndexDocument { + + $data = $resource->data; + $base = new DataBag($data->getAssociativeArray('base')); + $metadata = new DataBag($data->getAssociativeArray('metadata')); + + $doc->sp_id = $resource->id; + $doc->sp_name = $resource->name; + $doc->sp_anchor = $data->getString('anchor'); + $doc->title = $base->getString('title'); + $doc->description = $metadata->getString('description'); + + if (empty($doc->description)) { + $doc->description = $resource->data->getString( + 'metadata.intro' + ); + } + $doc->sp_objecttype = $resource->objectType; + $doc->sp_canonical = true; + $doc->crawl_process_id = $processId; + + $url = $data->getString('mediaUrl') + ?: $data->getString('url'); + $doc->id = $url; + $doc->url = $url; + + /** @var string[] $spContentType */ + $spContentType = [$resource->objectType]; + if ($data->getBool('media') !== true) { + $spContentType[] = 'article'; + } + $contentSectionTypes = $data->getArray('contentSectionTypes'); + $spContentType = array_merge($spContentType, $contentSectionTypes); + + if ($base->has('teaser.image')) { + $spContentType[] = 'teaserImage'; + } + if ($base->has('teaser.image.copyright')) { + $spContentType[] = 'teaserImageCopyright'; + } + if ($base->has('teaser.headline')) { + $spContentType[] = 'teaserHeadline'; + } + if ($base->has('teaser.text')) { + $spContentType[] = 'teaserText'; + } + $doc->sp_contenttype = $spContentType; + + $locale = $this->getLocaleFromResource($resource); + $lang = $this->toLangFromLocale($locale); + $doc->sp_language = $lang; + $doc->meta_content_language = $lang; + + $doc->sp_changed = $this->toDateTime( + $data->getInt('changed') + ); + $doc->sp_generated = $this->toDateTime( + $data->getInt('generated') + ); + + $doc->sp_archive = $base->getBool('archive'); + + $headline = $metadata->getString('headline') + ?: $base->getString('teaser.headline') + ?: $base->getString('title'); + $doc->sp_title = $headline; + + $sortHeadline = $base->getString('teaser.headline') + ?: $metadata->getString('headline') + ?: $base->getString('title'); + $doc->sp_sortvalue = $sortHeadline; + + /** @var string[] $keyword */ + $keyword = $metadata->getArray('keywords'); + $doc->keywords = $keyword; + + $doc->sp_boost_keywords = implode( + ' ', + $metadata->getArray('boostKeywords') + ); + + try { + $sites = $this->getParentSiteGroupIdList($resource); + + $navigationRoot = $this->navigationLoader->loadRoot( + $resource->toLocation() + ); + + $siteGroupId = $navigationRoot->data->getInt( + 'siteGroup.id' + ); + if ($siteGroupId !== 0) { + $sites[] = (string)$siteGroupId; + } + $doc->sp_site = array_unique($sites); + } catch (Exception $e) { + throw new DocumentEnrichingException( + $resource->toLocation(), + 'Unable to set sp_site: ' . $e->getMessage(), + 0, + $e + ); + } + + /** @var string[] $wktPrimaryList */ + $wktPrimaryList = $base->getArray('geo.wkt.primary'); + if (!empty($wktPrimaryList)) { + $doc->sp_geo_points = $wktPrimaryList; + } + + /** @var array $categoryList */ + $categoryList = $metadata->getArray('categories'); + if (!empty($categoryList)) { + $categoryIdList = []; + foreach ($categoryList as $category) { + $categoryIdList[] = (string)$category['id']; + } + $doc->sp_category = $categoryIdList; + } + + /** @var array $categoryPath */ + $categoryPath = $metadata->getArray('categoriesPath'); + if (!empty($categoryPath)) { + $categoryIdPath = []; + foreach ($categoryPath as $category) { + $categoryIdPath[] = (string)$category['id']; + } + $doc->sp_category_path = $categoryIdPath; + } + + /** @var array $groupPath */ + $groupPath = $data->getArray('groupPath'); + $groupPathAsIdList = []; + foreach ($groupPath as $group) { + $groupPathAsIdList[] = $group['id']; + } + + if (count($groupPathAsIdList) > 2) { + $doc->sp_group = $groupPathAsIdList[count($groupPathAsIdList) - 2]; + } + $doc->sp_group_path = $groupPathAsIdList; + + $doc->sp_date = $this->toDateTime( + $base->getInt('date') + ); + if ($doc->sp_date !== null) { + $doc->sp_date_list = [$doc->sp_date]; + } + + /** @var array $schedulingList */ + $schedulingList = $metadata->getArray('scheduling'); + if (!empty($schedulingList)) { + $doc->sp_date = $this->toDateTime($schedulingList[0]['from']); + $dateList = []; + $contentTypeList = []; + foreach ($schedulingList as $scheduling) { + $contentTypeList[] = explode(' ', $scheduling['contentType']); + $from = $this->toDateTime($scheduling['from']); + if ($from !== null) { + $dateList[] = $from; + } + } + $doc->sp_contenttype = array_merge( + $doc->sp_contenttype, + ...$contentTypeList + ); + $doc->sp_contenttype = array_unique($doc->sp_contenttype); + + $doc->sp_date_list = $dateList; + } + + $contentType = $base->getString( + 'mime', + 'text/html; charset=UTF-8' + ); + $doc->meta_content_type = $contentType; + + $accessType = $data->getString('access.type'); + + /** @var string[] $groups */ + $groups = $data->getArray('access.groups'); + + if ($accessType === 'allow' && !empty($groups)) { + $doc->include_groups = array_map( + fn($id): string => (string)$this->idWithoutSignature($id), + $groups + ); + } elseif ($accessType === 'deny' && !empty($groups)) { + $doc->exclude_groups = array_map( + fn($id): string => (string)$this->idWithoutSignature($id), + $groups + ); + } else { + $doc->exclude_groups = ['none']; + $doc->include_groups = ['all']; + } + + $doc->sp_source = ['internal']; + + return $this->enrichContent($resource, $doc); + } + + /** + * @template E of IndexSchema2xDocument + * @param E $doc + * @return E + */ + private function enrichContent( + Resource $resource, + IndexDocument $doc, + ): IndexDocument { + + $content = []; + $content[] = $resource->data->getString( + 'searchindexdata.content' + ); + + $content[] = $this->contentCollector->collect( + $resource->data->getArray('content') + ); + + /** @var ContactPoint $contactPoint */ + $contactPoint = $resource->data->getArray('metadata.contactPoint'); + $content[] = $this->contactPointToContent($contactPoint); + + /** @var array $categories */ + $categories = $resource->data->getArray('metadata.categories'); + foreach ($categories as $category) { + $content[] = $category['name'] ?? ''; + } + + $cleanContent = preg_replace( + '/\s+/', + ' ', + implode(' ', $content) + ); + + $doc->content = trim($cleanContent ?? ''); + + return $doc; + } + + /** + * @param ContactPoint $contactPoint + * @return string + */ + private function contactPointToContent(array $contactPoint): string + { + if (empty($contactPoint)) { + return ''; + } + + $content = []; + foreach (($contactPoint['contactData']['phoneList'] ?? []) as $phone) { + $countryCode = $phone['phone']['countryCode'] ?? ''; + if ( + !empty($countryCode) && + !in_array($countryCode, $content, true) + ) { + $content[] = '+' . $countryCode; + } + $areaCode = $phone['phone']['areaCode'] ?? ''; + if (!empty($areaCode) && !in_array($areaCode, $content, true)) { + $content[] = $areaCode; + $content[] = '0' . $areaCode; + } + $content[] = $phone['phone']['localNumber'] ?? ''; + } + foreach ($contactPoint['contactData']['emailList'] ?? [] as $email) { + $content[] = $email['email']; + } + + if (isset($contactPoint['addressData'])) { + $data = $contactPoint['addressData']; + $content[] = $data['street'] ?? ''; + $content[] = $data['buildingName'] ?? ''; + $content[] = $data['postOfficeBoxData']['buildingName'] ?? ''; + } + + return implode(' ', $content); + } + + private function idWithoutSignature(string $id): int + { + $s = substr($id, -11); + return (int)$s; + } + + private function getLocaleFromResource(Resource $resource): string + { + + $locale = $resource->data->getString('locale'); + if ($locale !== '') { + return $locale; + } + + /** @var array $groupPath */ + $groupPath = $resource->data->getArray('groupPath'); + if (!empty($groupPath)) { + $len = count($groupPath); + for ($i = $len - 1; $i >= 0; $i--) { + $group = $groupPath[$i]; + if (isset($group['locale'])) { + return $group['locale']; + } + } + } + + return 'de_DE'; + } + + private function toLangFromLocale(string $locale): string + { + if (str_contains($locale, '_')) { + $parts = explode('_', $locale); + return $parts[0]; + } + return $locale; + } + + private function toDateTime(int $timestamp): ?DateTime + { + if ($timestamp <= 0) { + return null; + } + + $dateTime = new DateTime(); + $dateTime->setTimestamp($timestamp); + return $dateTime; + } + + /** + * @param Resource $resource + * @return string[] + */ + private function getParentSiteGroupIdList(Resource $resource): array + { + /** @var array $parents */ + $parents = $this->getNavigationParents($resource); + if (empty($parents)) { + return []; + } + + $siteGroupIdList = []; + foreach ($parents as $parent) { + if (isset($parent['siteGroup']['id'])) { + $siteGroupIdList[] = $parent['siteGroup']['id']; + } + } + + return $siteGroupIdList; + } + + /** + * @return array + */ + private function getNavigationParents(Resource $resource): array + { + return $resource->data->getAssociativeArray( + 'base.trees.navigation.parents' + ); + } +} diff --git a/src/Service/Indexer/SiteKit/HeadlineMatcher.php b/src/Service/Indexer/SiteKit/HeadlineMatcher.php new file mode 100644 index 0000000..4b18549 --- /dev/null +++ b/src/Service/Indexer/SiteKit/HeadlineMatcher.php @@ -0,0 +1,29 @@ +data->getBool('noIndex'); + return $noIndex !== true; + } +} diff --git a/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php new file mode 100644 index 0000000..c8e64b2 --- /dev/null +++ b/src/Service/Indexer/SiteKit/QuoteSectionMatcher.php @@ -0,0 +1,45 @@ +toResourceLocation($path); + if ($location === null) { + continue; + } + if ($location->lang === ResourceLanguage::default()) { + $bases[] = $location; + continue; + } + if (!isset($translations[$location->lang->code])) { + $translations[$location->lang->code] = []; + } + $translations[$location->lang->code][] = $location; + } + + return new TranslationSplitterResult($bases, $translations); + } + + private function toResourceLocation(string $path): ?ResourceLocation + { + $normalizedPath = $this->normalizePath($path); + if (empty($normalizedPath)) { + return null; + } + + $pos = strrpos($normalizedPath, '.php.translations'); + if ($pos === false) { + return ResourceLocation::of($normalizedPath); + } + + $localeFilename = basename($normalizedPath); + $locale = basename($localeFilename, '.php'); + + return ResourceLocation::of( + substr($normalizedPath, 0, $pos + 4), + ResourceLanguage::of($locale) + ); + } + + /** + * A path can signal to be translated into another language via + * the URL parameter loc. For example, + * `/dir/file.php?loc=it_IT` defines that the path + * `/dir/file.php.translations/it_IT.php` is to be used. + * This method translates the URL parameter into the correct path. + */ + private function normalizePath(string $path): string + { + $queryString = parse_url($path, PHP_URL_QUERY); + if (!is_string($queryString)) { + return $path; + } + $urlPath = parse_url($path, PHP_URL_PATH); + if (!is_string($urlPath)) { + return ''; + } + parse_str($queryString, $params); + if (!isset($params['loc']) || !is_string($params['loc'])) { + return $urlPath; + } + $loc = $params['loc']; + return $urlPath . '.translations/' . $loc . ".php"; + } +} diff --git a/src/Service/Indexer/SolrIndexService.php b/src/Service/Indexer/SolrIndexService.php new file mode 100644 index 0000000..0f18227 --- /dev/null +++ b/src/Service/Indexer/SolrIndexService.php @@ -0,0 +1,126 @@ +index->name($lang); + } + + public function updater(ResourceLanguage $lang): SolrIndexUpdater + { + $client = $this->createClient($lang); + $update = $client->createUpdate(); + $update->setDocumentClass(IndexSchema2xDocument::class); + + return new SolrIndexUpdater($client, $update); + } + + public function deleteExcludingProcessId( + ResourceLanguage $lang, + string $source, + string $processId + ): void { + $this->deleteByQuery( + $lang, + '-crawl_process_id:' . $processId . ' AND ' . + ' sp_source:' . $source + ); + } + + /** + * @param string[] $idList + */ + public function deleteByIdListForAllLanguages( + string $source, + array $idList + ): void { + $this->deleteByQueryForAllLanguages( + 'sp_id:(' . implode(' ', $idList) . ') AND ' . + 'sp_source:' . $source + ); + } + + public function deleteByQueryForAllLanguages(string $query): void + { + foreach ($this->getManagedIndices() as $index) { + $client = $this->clientFactory->create($index); + $update = $client->createUpdate(); + $update->addDeleteQuery($query); + $client->update($update); + } + } + + public function deleteByQuery(ResourceLanguage $lang, string $query): void + { + $client = $this->createClient($lang); + $update = $client->createUpdate(); + $update->addDeleteQuery($query); + $client->update($update); + } + + public function commit(ResourceLanguage $lang): void + { + $client = $this->createClient($lang); + $update = $client->createUpdate(); + $update->addCommit(); + $update->addOptimize(); + $client->update($update); + } + + public function commitForAllLanguages(): void + { + foreach ($this->getManagedIndices() as $index) { + $client = $this->clientFactory->create($index); + $update = $client->createUpdate(); + $update->addCommit(); + $update->addOptimize(); + $client->update($update); + } + } + + /** + * @return string[] + */ + public function getManagedIndices(): array + { + $client = $this->createClient(ResourceLanguage::default()); + $coreAdminQuery = $client->createCoreAdmin(); + $statusAction = $coreAdminQuery->createStatus(); + $coreAdminQuery->setAction($statusAction); + + $requiredIndexes = $this->index->names(); + + $managedIndexes = []; + $response = $client->coreAdmin($coreAdminQuery); + $statusResults = $response->getStatusResults() ?? []; + foreach ($statusResults as $statusResult) { + $index = $statusResult->getCoreName(); + if (in_array($index, $requiredIndexes, true)) { + $managedIndexes[] = $index; + } + } + + return $managedIndexes; + } + + private function createClient(ResourceLanguage $lang): Client + { + return $this->clientFactory->create($this->index->name($lang)); + } +} diff --git a/src/Service/Indexer/SolrIndexUpdater.php b/src/Service/Indexer/SolrIndexUpdater.php new file mode 100644 index 0000000..1a1e0fb --- /dev/null +++ b/src/Service/Indexer/SolrIndexUpdater.php @@ -0,0 +1,43 @@ +update->createDocument(); + return $doc; + } + + public function addDocument(Document $document): void + { + $this->documents[] = $document; + } + + public function update(): UpdateResult + { + $this->update->addDocuments($this->documents); + $this->documents = []; + return $this->client->update($this->update); + } +} diff --git a/src/Service/Indexer/TranslationSplitter.php b/src/Service/Indexer/TranslationSplitter.php new file mode 100644 index 0000000..ab88453 --- /dev/null +++ b/src/Service/Indexer/TranslationSplitter.php @@ -0,0 +1,17 @@ +> $translations + */ + public function __construct( + private readonly array $bases, + private readonly array $translations + ) { + } + + /** + * @return ResourceLanguage[] + */ + public function getLanguages(): array + { + $languages = array_keys($this->translations); + sort($languages); + return array_map( + static fn($lang) + => ResourceLanguage::of($lang), + $languages + ); + } + + /** + * @return ResourceLocation[] + */ + public function getBases(): array + { + return $this->bases; + } + + /** + * @return ResourceLocation[] + */ + public function getTranslations(ResourceLanguage $lang): array + { + return $this->translations[$lang->code] ?? []; + } +} diff --git a/src/Service/ParameterSolrClientFactory.php b/src/Service/ParameterSolrClientFactory.php new file mode 100644 index 0000000..58a190f --- /dev/null +++ b/src/Service/ParameterSolrClientFactory.php @@ -0,0 +1,54 @@ +setTimeout($this->timeout); + $adapter->setProxy($this->proxy); + $eventDispatcher = new EventDispatcher(); + $config = [ + 'endpoint' => [ + $this->host => [ + 'scheme' => $this->scheme, + 'host' => $this->host, + 'port' => $this->port, + 'path' => $this->path, + 'core' => $core, + ] + ] + ]; + + // create a client instance + return new Client( + $adapter, + $eventDispatcher, + $config + ); + } +} diff --git a/src/Service/ResourceChannelBasedIndexName.php b/src/Service/ResourceChannelBasedIndexName.php new file mode 100644 index 0000000..dabf43a --- /dev/null +++ b/src/Service/ResourceChannelBasedIndexName.php @@ -0,0 +1,75 @@ +langToAvailableLocale($this->resourceChannel, $lang); + + if (empty($locale)) { + return $this->resourceChannel->searchIndex; + } + + return $this->resourceChannel->searchIndex . '-' . $locale; + } + + /** + * The returned list contains the default index name and the index + * name of all language-specific indexes. + * + * @return string[] + */ + public function names(): array + { + $names = [$this->resourceChannel->searchIndex]; + foreach ( + $this->resourceChannel->translationLocales as $locale + ) { + $names[] = $this->resourceChannel->searchIndex . '-' . $locale; + } + return $names; + } + + /** + * @throws UnsupportedIndexLanguageException + */ + private function langToAvailableLocale( + ResourceChannel $resourceChannel, + ResourceLanguage $lang + ): string { + + if ($lang === ResourceLanguage::default()) { + return ''; + } + + foreach ( + $resourceChannel->translationLocales as $availableLocale + ) { + if (str_starts_with($availableLocale, $lang->code)) { + return $availableLocale; + } + } + throw new UnsupportedIndexLanguageException( + $resourceChannel->searchIndex, + $lang, + 'No valid index can be determined for the language ' . + $lang->code + ); + } +} diff --git a/src/Service/Search/ExternalResourceFactory.php b/src/Service/Search/ExternalResourceFactory.php new file mode 100644 index 0000000..892b6cb --- /dev/null +++ b/src/Service/Search/ExternalResourceFactory.php @@ -0,0 +1,57 @@ +getField($document, 'url'); + if ($location === null) { + return false; + } + return ( + str_starts_with($location, 'http://') || + str_starts_with($location, 'https://') + ); + } + + public function create(Document $document, ResourceLanguage $lang): Resource + { + $location = $this->getField($document, 'url'); + if ($location === null) { + throw new LogicException('document should contain an url'); + } + + return new Resource( + location: $location, + id: $this->getField($document, 'sp_id') ?? '', + name: $this->getField($document, 'title') ?? '', + objectType: 'external', + lang: ResourceLanguage::of( + $this->getField($document, 'meta_content_language') + ), + data: new DataBag([]), + ); + } + + private function getField(Document $document, string $name): ?string + { + return $document->getFields()[$name] ?? null; + } +} diff --git a/src/Service/Search/InternalMediaResourceFactory.php b/src/Service/Search/InternalMediaResourceFactory.php new file mode 100644 index 0000000..ee0e3a5 --- /dev/null +++ b/src/Service/Search/InternalMediaResourceFactory.php @@ -0,0 +1,64 @@ +getMetaLocation($document, $lang); + if ($metaLocation === null) { + return false; + } + return $this->resourceLoader->exists($metaLocation); + } + + public function create(Document $document, ResourceLanguage $lang): Resource + { + $metaLocation = $this->getMetaLocation($document, $lang); + if ($metaLocation === null) { + throw new LogicException('document should contain an url'); + } + return $this->resourceLoader->load($metaLocation); + } + + private function getMetaLocation( + Document $document, + ResourceLanguage $lang + ): ?ResourceLocation { + $url = $this->getField($document, 'url'); + if ($url === null) { + return null; + } + return ResourceLocation::of( + $url . '.meta.php', + $lang + ); + } + + private function getField(Document $document, string $name): ?string + { + return $document->getFields()[$name] ?? null; + } +} diff --git a/src/Service/Search/InternalResourceFactory.php b/src/Service/Search/InternalResourceFactory.php new file mode 100644 index 0000000..a02b6e9 --- /dev/null +++ b/src/Service/Search/InternalResourceFactory.php @@ -0,0 +1,49 @@ +getUrl($document); + if ($location === null) { + return false; + } + return str_ends_with($location, '.php'); + } + + public function create(Document $document, ResourceLanguage $lang): Resource + { + $url = $this->getUrl($document); + if ($url === null) { + throw new LogicException('document should contain an url'); + } + $location = ResourceLocation::of($url, $lang); + return $this->resourceLoader->load($location); + } + + private function getUrl(Document $document): ?string + { + return $document->getFields()['url'] ?? null; + } +} diff --git a/src/Service/Search/ResourceFactory.php b/src/Service/Search/ResourceFactory.php new file mode 100644 index 0000000..3738025 --- /dev/null +++ b/src/Service/Search/ResourceFactory.php @@ -0,0 +1,31 @@ +getEDisMax(); + $edismax->setQueryFields(implode(' ', [ + 'sp_title^1.4', + 'keywords^1.2', + 'description^1.0', + 'title^1.0', + 'url^0.9', + 'content^0.8' + ])); + $edismax->setPhraseFields(implode(' ', [ + 'sp_title^1.5', + 'description^1', + 'content^0.8' + ])); + $edismax->setBoostQuery('sp_objecttype:searchTip^100'); + + return $query; + } +} diff --git a/src/Service/Search/SolrMoreLikeThis.php b/src/Service/Search/SolrMoreLikeThis.php new file mode 100644 index 0000000..9f7159e --- /dev/null +++ b/src/Service/Search/SolrMoreLikeThis.php @@ -0,0 +1,81 @@ +index->name($query->location->lang); + $client = $this->clientFactory->create($index); + $solrQuery = $this->buildSolrQuery($client, $query); + /** @var SolrMoreLikeThisResult $result */ + $result = $client->execute($solrQuery); + return $this->buildResult($result, $query->location->lang); + } + + private function buildSolrQuery( + Client $client, + MoreLikeThisQuery $query + ): SolrMoreLikeThisQuery { + + $solrQuery = $client->createMoreLikeThis(); + $solrQuery->setOmitHeader(false); + $solrQuery->setQuery('url:"' . $query->location . '"'); + $solrQuery->setMltFields($query->fields); + $solrQuery->setRows($query->limit); + $solrQuery->setMinimumTermFrequency(2); + $solrQuery->setMatchInclude(true); + $solrQuery->createFilterQuery('nomedia') + ->setQuery('-sp_objecttype:media'); + + // Filter + foreach ($query->filter as $filter) { + $filterQuery = $solrQuery->createFilterQuery($filter->key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); + } + + return $solrQuery; + } + + private function buildResult( + SolrMoreLikeThisResult $result, + ResourceLanguage $lang + ): SearchResult { + + $resourceList = $this->resultToResourceResolver + ->loadResourceList($result, $lang); + + return new SearchResult( + total: $result->getNumFound() ?? 0, + limit: 0, + offset: 0, + results: $resourceList, + facetGroups: [], + queryTime: $result->getQueryTime() ?? 0 + ); + } +} diff --git a/src/Service/Search/SolrQueryModifier.php b/src/Service/Search/SolrQueryModifier.php new file mode 100644 index 0000000..ab51430 --- /dev/null +++ b/src/Service/Search/SolrQueryModifier.php @@ -0,0 +1,17 @@ + $resourceFactoryList + */ + public function __construct( + private readonly iterable $resourceFactoryList, + private readonly LoggerInterface $logger = new NullLogger() + ) { + } + + /** + * @return array + */ + public function loadResourceList( + SelectResult $result, + ResourceLanguage $lang + ): array { + $resourceList = []; + /** @var Document $document */ + foreach ($result as $document) { + try { + $resourceList[] = $this->loadResource($document, $lang); + } catch (Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + } + } + return $resourceList; + } + + private function loadResource( + Document $document, + ResourceLanguage $lang + ): Resource { + + foreach ($this->resourceFactoryList as $resourceFactory) { + if ($resourceFactory->accept($document, $lang)) { + return $resourceFactory->create($document, $lang); + } + } + + throw new MissMatchingResourceFactoryException( + ResourceLocation::of( + $document->getFields()['url'] ?? '', + $lang + ) + ); + } +} diff --git a/src/Service/Search/SolrSearch.php b/src/Service/Search/SolrSearch.php new file mode 100644 index 0000000..07a06e8 --- /dev/null +++ b/src/Service/Search/SolrSearch.php @@ -0,0 +1,312 @@ + $solrQueryModifierList + */ + public function __construct( + private readonly IndexName $index, + private readonly SolrClientFactory $clientFactory, + private readonly SolrResultToResourceResolver $resultToResourceResolver, + private readonly iterable $solrQueryModifierList = [] + ) { + } + + public function search(SearchQuery $query): SearchResult + { + $lang = ResourceLanguage::of($query->lang); + $index = $this->index->name($lang); + $client = $this->clientFactory->create($index); + + $solrQuery = $this->buildSolrQuery($client, $query); + /** @var SelectResult $result */ + $result = $client->execute($solrQuery); + return $this->buildResult($query, $result, $lang); + } + + private function buildSolrQuery( + Client $client, + SearchQuery $query + ): SolrSelectQuery { + + $solrQuery = $client->createSelect(); + + // supplements the query with standard values, e.g. for boosting + foreach ($this->solrQueryModifierList as $solrQueryModifier) { + $solrQuery = $solrQueryModifier->modify($solrQuery); + } + + $solrQuery->setStart($query->offset); + $solrQuery->setRows($query->limit); + + // to get query-time + $solrQuery->setOmitHeader(false); + + $this->addSortToSolrQuery($solrQuery, $query->sort); + $this->addRequiredFieldListToSolrQuery($solrQuery); + $this->addTextFilterToSolrQuery($solrQuery, $query->text); + $this->addQueryDefaultOperatorToSolrQuery( + $solrQuery, + $query->defaultQueryOperator + ); + $this->addFilterQueriesToSolrQuery( + $solrQuery, + $query->filter + ); + $this->addFacetListToSolrQuery( + $solrQuery, + $query->facets + ); + + return $solrQuery; + } + + /** + * @param Criteria[] $criteriaList + */ + private function addSortToSolrQuery( + SolrSelectQuery $solrQuery, + array $criteriaList + ): void { + $sorts = []; + foreach ($criteriaList as $criteria) { + if ($criteria instanceof Name) { + $field = 'sp_name'; + } elseif ($criteria instanceof Headline) { + $field = 'sp_title'; + } elseif ($criteria instanceof Date) { + $field = 'sp_date'; + } elseif ($criteria instanceof Natural) { + $field = 'sp_sortvalue'; + } elseif ($criteria instanceof Score) { + $field = 'score'; + } else { + throw new InvalidArgumentException( + 'unsupported sort criteria: ' . get_class($criteria) + ); + } + + $direction = strtolower($criteria->direction->name); + + $sorts[$field] = $direction; + } + $solrQuery->setSorts($sorts); + } + + private function addRequiredFieldListToSolrQuery( + SolrSelectQuery $solrQuery + ): void { + $solrQuery->setFields(['url', 'title', 'sp_id']); + } + + private function addTextFilterToSolrQuery( + SolrSelectQuery $solrQuery, + string $text + ): void { + if (empty($text)) { + return; + } + $terms = explode(' ', $text); + $terms = array_map( + fn ($term) => $solrQuery->getHelper()->escapeTerm(trim($term)), + $terms + ); + $text = implode(' ', $terms); + $solrQuery->setQuery($text); + } + + private function addQueryDefaultOperatorToSolrQuery( + SolrSelectQuery $solrQuery, + QueryOperator $operator + ): void { + $solrQuery->setQueryDefaultOperator( + $operator === QueryOperator::OR + ? SolrSelectQuery::QUERY_OPERATOR_OR + : SolrSelectQuery::QUERY_OPERATOR_AND + ); + } + + /** + * @param Filter[] $filterList + */ + private function addFilterQueriesToSolrQuery( + SolrSelectQuery $solrQuery, + array $filterList + ): void { + + foreach ($filterList as $filter) { + $key = $filter->key ?? uniqid('', true); + $filterQuery = $solrQuery->createFilterQuery($key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); + } + } + + /** + * @param \Atoolo\Search\Dto\Search\Query\Facet\Facet[] $facetList + */ + private function addFacetListToSolrQuery( + SolrSelectQuery $solrQuery, + array $facetList + ): void { + foreach ($facetList as $facet) { + if ($facet instanceof FacetField) { + $this->addFacetFieldToSolrQuery($solrQuery, $facet); + } elseif ($facet instanceof FacetQuery) { + $this->addFacetQueryToSolrQuery($solrQuery, $facet); + } elseif ($facet instanceof FacetMultiQuery) { + $this->addFacetMultiQueryToSolrQuery($solrQuery, $facet); + } else { + throw new InvalidArgumentException( + 'Unsupported facet-class ' . get_class($facet) + ); + } + } + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-field/ + */ + private function addFacetFieldToSolrQuery( + SolrSelectQuery $solrQuery, + FacetField $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + $field = $facet->field; + // https://solr.apache.org/guide/solr/latest/query-guide/faceting.html#tagging-and-excluding-filters + if ($facet->excludeFilter !== null) { + $field = '{!ex=' . $facet->excludeFilter . '}' . $field; + } + /** @var Field $solariumFacet */ + $solariumFacet = $facetSet->createFacetField($facet->key); + $solariumFacet + ->setField($field) + ->setTerms($facet->terms); + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-query/ + */ + private function addFacetQueryToSolrQuery( + SolrSelectQuery $solrQuery, + FacetQuery $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + $facetSet->createFacetQuery($facet->key) + ->setQuery($facet->query); + } + + /** + * https://solarium.readthedocs.io/en/stable/queries/select-query/building-a-select-query/components/facetset-component/facet-multiquery/ + */ + private function addFacetMultiQueryToSolrQuery( + SolrSelectQuery $solrQuery, + FacetMultiQuery $facet + ): void { + $facetSet = $solrQuery->getFacetSet(); + $solrFacet = $facetSet->createFacetMultiQuery($facet->key); + foreach ($facet->queries as $facetQuery) { + $solrFacet->createQuery( + $facetQuery->key, + $facetQuery->query + ); + } + } + + private function buildResult( + SearchQuery $query, + SelectResult $result, + ResourceLanguage $lang + ): SearchResult { + + $resourceList = $this->resultToResourceResolver + ->loadResourceList($result, $lang); + $facetGroupList = $this->buildFacetGroupList($query, $result); + + return new SearchResult( + total:$result->getNumFound() ?? 0, + limit: $query->limit, + offset: $query->offset, + results: $resourceList, + facetGroups: $facetGroupList, + queryTime: $result->getQueryTime() ?? 0 + ); + } + + /** + * @return FacetGroup[] + */ + private function buildFacetGroupList( + SearchQuery $query, + SelectResult $result + ): array { + + $facetSet = $result->getFacetSet(); + if ($facetSet === null) { + return []; + } + + $facetGroupList = []; + foreach ($query->facets as $facet) { + /** @var ?\Solarium\Component\Result\Facet\Field $resultFacet */ + $resultFacet = $facetSet->getFacet($facet->key); + if ($resultFacet === null) { + continue; + } + $facetGroupList[] = $this->buildFacetGroup( + $facet->key, + $resultFacet + ); + } + return $facetGroupList; + } + + private function buildFacetGroup( + string $key, + \Solarium\Component\Result\Facet\Field $solrFacet + ): FacetGroup { + $facetList = []; + foreach ($solrFacet as $value => $count) { + if (!is_int($count)) { + throw new InvalidArgumentException( + 'facet count should be a int: ' . $count + ); + } + $facetList[] = new Facet((string)$value, $count); + } + return new FacetGroup($key, $facetList); + } +} diff --git a/src/Service/Search/SolrSuggest.php b/src/Service/Search/SolrSuggest.php new file mode 100644 index 0000000..1557590 --- /dev/null +++ b/src/Service/Search/SolrSuggest.php @@ -0,0 +1,141 @@ +> + * } + * } + */ +class SolrSuggest implements Suggest +{ + private const INDEX_SUGGEST_FIELD = 'raw_content'; + + public function __construct( + private readonly IndexName $index, + private readonly SolrClientFactory $clientFactory + ) { + } + + /** + * @throws UnexpectedResultException + */ + public function suggest(SuggestQuery $query): SuggestResult + { + $lang = ResourceLanguage::of($query->lang); + $index = $this->index->name($lang); + $client = $this->clientFactory->create($index); + + $solrQuery = $this->buildSolrQuery($client, $query); + $solrResult = $client->select($solrQuery); + return $this->buildResult($solrResult); + } + + private function buildSolrQuery( + Client $client, + SuggestQuery $query + ): SolrSelectQuery { + $solrQuery = $client->createSelect(); + $solrQuery->addParam("spellcheck", "true"); + $solrQuery->addParam("spellcheck.accuracy", "0.6"); + $solrQuery->addParam("spellcheck.onlyMorePopular", "false"); + $solrQuery->addParam("spellcheck.count", "15"); + $solrQuery->addParam("spellcheck.maxCollations", "5"); + $solrQuery->addParam("spellcheck.maxCollationTries", "15"); + $solrQuery->addParam("spellcheck.collate", "true"); + $solrQuery->addParam("spellcheck.collateExtendedResults", "true"); + $solrQuery->addParam("spellcheck.extendedResults", "true"); + $solrQuery->addParam("facet", "true"); + $solrQuery->addParam("facet.sort", "count"); + $solrQuery->addParam("facet.method", "enum"); + $solrQuery->addParam( + "facet.prefix", + $query->text + ); + $solrQuery->addParam("facet.limit", $query->limit); + $solrQuery->addParam("facet.field", self::INDEX_SUGGEST_FIELD); + + $solrQuery->setOmitHeader(false); + $solrQuery->setStart(0); + $solrQuery->setRows(0); + + // Filter + foreach ($query->filter as $filter) { + $filterQuery = $solrQuery->createFilterQuery($filter->key); + $filterQuery->setQuery($filter->getQuery()); + $filterQuery->setTags($filter->tags); + } + + return $solrQuery; + } + + private function buildResult( + SolrSelectResult $solrResult + ): SuggestResult { + $suggestions = $this->parseSuggestion( + $solrResult->getResponse()->getBody() + ); + return new SuggestResult( + $suggestions, + $solrResult->getQueryTime() ?? 0 + ); + } + + /** + * @throws UnexpectedResultException + * @return Suggestion[] + */ + private function parseSuggestion( + string $responseBody + ): array { + try { + /** @var SolariumResponse $json */ + $json = json_decode( + $responseBody, + true, + 5, + JSON_THROW_ON_ERROR + ); + $facets = + $json['facet_counts']['facet_fields'][self::INDEX_SUGGEST_FIELD] + ?? []; + + $len = count($facets); + + $suggestions = []; + for ($i = 0; $i < $len - 1; $i += 2) { + $term = $facets[$i]; + $hits = (int)$facets[$i + 1]; + $suggestions[] = new Suggestion($term, $hits); + } + + return $suggestions; + } catch (JsonException $e) { + throw new UnexpectedResultException( + $responseBody, + "Invalid JSON for suggest result", + 0, + $e + ); + } + } +} diff --git a/src/Service/SolrClientFactory.php b/src/Service/SolrClientFactory.php new file mode 100644 index 0000000..ce26af6 --- /dev/null +++ b/src/Service/SolrClientFactory.php @@ -0,0 +1,16 @@ +createStub( + ResourceChannelFactory::class + ); + $indexer = $this->createStub( + InternalResourceIndexer::class + ); + $indexers = new IndexerCollection([$indexer]); + $progressBar = $this->createStub(IndexerProgressBar::class); + $application = new Application([ + new Indexer($resourceChannelFactory, $progressBar, $indexers) + ]); + $command = $application->get('search:indexer'); + $this->assertInstanceOf( + Indexer::class, + $command, + 'unexpected indexer command' + ); + } +} diff --git a/test/Console/Command/DumpIndexDocumentTest.php b/test/Console/Command/DumpIndexDocumentTest.php new file mode 100644 index 0000000..95de790 --- /dev/null +++ b/test/Console/Command/DumpIndexDocumentTest.php @@ -0,0 +1,89 @@ +createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + + $dumper = $this->createStub(IndexDocumentDumper::class); + $dumper->method('dump') + ->willReturn([ + ['sp_id' => '123'] + ]); + + $dumperCommand = new DumpIndexDocument( + $resourceChannelFactory, + $dumper + ); + + $application = new Application([ + $dumperCommand + ]); + + $command = $application->find('search:dump-index-document'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $this->commandTester->execute([ + 'paths' => ['test.php'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $this->resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $indexer = $this->createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub(IndexerProgressBar::class); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $this->commandTester = new CommandTester($command); + } + + public function testExecuteIndexPath(): void + { + $this->commandTester->execute([ + // pass arguments to the helper + 'paths' => ['a.php', 'b.php'] + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $commandTester = new CommandTester($command); + + $commandTester->execute([ 'paths' => ['a.php']]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'errortest', + $output, + 'error message expected' + ); + } + + + /** + * @throws Exception + */ + public function testExecuteIndexWithErrorsAndStackTrace(): void + { + + $indexer = $this->createStub( + InternalResourceIndexer::class + ); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new IndexerInternalResourceUpdate( + $this->resourceChannelFactory, + $progressBar, + $indexer, + ); + + $application = new Application([$command]); + + $command = $application->find( + 'search:indexer:update-internal-resources' + ); + $commandTester = new CommandTester($command); + + $commandTester->execute( + [ + 'paths' => ['a.php'] + ], + [ + 'verbosity' => OutputInterface::VERBOSITY_VERBOSE + ] + ); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'Exception trace', + $output, + 'error message should contains stack trace' + ); + } +} diff --git a/test/Console/Command/IndexerTest.php b/test/Console/Command/IndexerTest.php new file mode 100644 index 0000000..8f596e1 --- /dev/null +++ b/test/Console/Command/IndexerTest.php @@ -0,0 +1,238 @@ +resourceChannelFactory = $this->createStub( + ResourceChannelFactory::class + ); + $this->resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $indexerA = $this->createStub( + \Atoolo\Search\Indexer::class + ); + $indexerA->method('enabled') + ->willReturn(true); + $indexerA->method('getSource') + ->willReturn('indexer_a'); + $indexerA->method('getName') + ->willReturn('Indexer A'); + $indexerB = $this->createStub( + \Atoolo\Search\Indexer::class + ); + $indexerB->method('enabled') + ->willReturn(false); + $indexerB->method('getSource') + ->willReturn('indexer_b'); + $indexerB->method('getName') + ->willReturn('Indexer B'); + $indexers = new IndexerCollection([ + $indexerA, + $indexerB + ]); + $progressBar = $this->createStub(IndexerProgressBar::class); + + $command = new Indexer( + $this->resourceChannelFactory, + $progressBar, + $indexers, + ); + + $application = new Application([$command]); + + $command = $application->find('search:indexer'); + $this->commandTester = new CommandTester($command); + } + + public function testExecuteAllEnabledIndexer(): void + { + $this->commandTester->execute([]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<commandTester->execute( + [ + '--source' => 'indexer_a' + ] + ); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + \Atoolo\Search\Indexer::class + ); + $indexer->method('enabled') + ->willReturn(true); + $indexers = new IndexerCollection([$indexer]); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new Indexer( + $this->resourceChannelFactory, + $progressBar, + $indexers, + ); + + $application = new Application([$command]); + + $command = $application->find('search:indexer'); + $commandTester = new CommandTester($command); + + $commandTester->execute([]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'errortest', + $output, + 'error message expected' + ); + } + + + /** + * @throws Exception + */ + public function testExecuteIndexWithErrorsAndStackTrace(): void + { + + $indexer = $this->createStub( + \Atoolo\Search\Indexer::class + ); + $indexer->method('enabled') + ->willReturn(true); + $indexers = new IndexerCollection([$indexer]); + $progressBar = $this->createStub( + IndexerProgressBar::class + ); + $progressBar + ->method('getErrors') + ->willReturn([new \Exception('errortest')]); + + $command = new Indexer( + $this->resourceChannelFactory, + $progressBar, + $indexers, + ); + + $application = new Application([$command]); + + $command = $application->find('search:indexer'); + $commandTester = new CommandTester($command); + + $commandTester->execute([], [ + 'verbosity' => OutputInterface::VERBOSITY_VERBOSE + ]); + + $commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $commandTester->getDisplay(); + $this->assertStringContainsString( + 'Exception trace', + $output, + 'error message should contains stack trace' + ); + } +} diff --git a/test/Console/Command/Io/IndexerProgressBarTest.php b/test/Console/Command/Io/IndexerProgressBarTest.php new file mode 100644 index 0000000..e338f14 --- /dev/null +++ b/test/Console/Command/Io/IndexerProgressBarTest.php @@ -0,0 +1,175 @@ +createMock(OutputInterface::class); + $this->progressHandler = $this->createMock(IndexerProgressHandler::class); + $this->progressBar = new IndexerProgressBar($output); + $this->progressBar->init($this->progressHandler); + } + + public function testPrepare(): void + { + $this->progressHandler + ->expects($this->once()) + ->method('prepare') + ->with('test'); + $this->progressBar->prepare('test'); + } + + public function testStart(): void + { + $this->progressHandler + ->expects($this->once()) + ->method('start') + ->with(10); + $this->progressBar->start(10); + } + + public function testStartAfterPrepare(): void + { + $this->progressBar->prepare('test'); + + $this->progressHandler + ->expects($this->once()) + ->method('start') + ->with(10); + $this->progressBar->start(10); + } + + /** + * @throws ExceptionInterface + */ + public function testStartUpdate(): void + { + $this->progressHandler + ->expects($this->once()) + ->method('startUpdate') + ->with(10); + $this->progressBar->startUpdate(10); + } + + /** + * @phpcs:disable Generic.Files.LineLength + * @throws JsonException + */ + public function testAdvance(): void + { + + $this->progressBar->start(10); + + $this->progressHandler + ->expects($this->once()) + ->method('advance') + ->with(1); + + $this->progressBar->advance(1); + } + + public function testSkip(): void + { + $this->progressBar->start(10); + + $this->progressHandler + ->expects($this->once()) + ->method('skip') + ->with(10); + + $this->progressBar->skip(10); + } + + public function testError(): void + { + + $this->progressBar->start(10); + + $e = new Exception('test'); + + $this->progressHandler + ->expects($this->once()) + ->method('error') + ->with($e); + + $this->progressBar->error($e); + } + + public function testGetError(): void + { + $this->progressBar->start(10); + + $e = new Exception('test'); + $this->progressBar->error($e); + + $this->assertCount( + 1, + $this->progressBar->getErrors(), + 'unexpected error count' + ); + } + + /** + * @throws JsonException + */ + public function testFinish(): void + { + + $this->progressBar->start(10); + + $this->progressHandler + ->expects($this->once()) + ->method('finish'); + $this->progressBar->finish(); + } + + public function testAbort(): void + { + $this->progressBar->start(10); + + $this->progressHandler + ->expects($this->once()) + ->method('abort'); + $this->progressBar->abort(); + } + + /** + * @throws \PHPUnit\Framework\MockObject\Exception + */ + public function testGetStatus(): void + { + $status = $this->createStub(IndexerStatus::class); + $this->progressHandler->method('getStatus')->willReturn($status); + $this->progressHandler + ->expects($this->once()) + ->method('getStatus') + ->willReturn($status); + $this->progressBar->getStatus(); + } +} diff --git a/test/Console/Command/Io/TypifiedInputTest.php b/test/Console/Command/Io/TypifiedInputTest.php new file mode 100644 index 0000000..d6ae49f --- /dev/null +++ b/test/Console/Command/Io/TypifiedInputTest.php @@ -0,0 +1,130 @@ +createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 123, + $input->getIntOption('a'), + 'unexpected option value' + ); + } + + public function testGetIntOptWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getIntOption('a'); + } + + public function testGetStringOption(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 'abc', + $input->getStringOption('a'), + 'unexpected option value' + ); + } + + public function testGetStringOptWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getOption') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getStringOption('a'); + } + + public function testGetStringArgument(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->assertEquals( + 'abc', + $input->getStringArgument('a'), + 'unexpected argument value' + ); + } + + public function testGetStringArgumentWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn(123); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getStringArgument('a'); + } + + public function testGetArrayArgument(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn(['a', 'b', 'c']); + + $input = new TypifiedInput($symfonyInput); + $this->assertEquals( + ['a', 'b', 'c'], + $input->getArrayArgument('a'), + 'unexpected argument value' + ); + } + + public function testGetArrayArgumentWithInvalidValue(): void + { + $symfonyInput = $this->createStub(InputInterface::class); + $symfonyInput + ->method('getArgument') + ->willReturn('abc'); + + $input = new TypifiedInput($symfonyInput); + + $this->expectException(InvalidArgumentException::class); + $input->getArrayArgument('a'); + } +} diff --git a/test/Console/Command/MoreLikeThisTest.php b/test/Console/Command/MoreLikeThisTest.php new file mode 100644 index 0000000..4c2517d --- /dev/null +++ b/test/Console/Command/MoreLikeThisTest.php @@ -0,0 +1,161 @@ +createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $resultResource = new Resource( + '/test2.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [], + 10 + ); + $this->solrMoreLikeThis = $this->createStub(SolrMoreLikeThis::class); + + $command = new MoreLikeThis( + $resourceChannelFactory, + $this->solrMoreLikeThis + ); + + $application = new Application([$command]); + + $command = $application->find('search:mlt'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + + $resultResource = new Resource( + '/test2.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [], + 10 + ); + $this->solrMoreLikeThis->method('moreLikeThis') + ->willReturn($result); + + $this->commandTester->execute([ + 'location' => '/test.php' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<solrMoreLikeThis->method('moreLikeThis') + ->willReturn($result); + + $this->commandTester->execute([ + 'location' => '/test.php' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + + $this->solrSearch = $this->createStub(SolrSearch::class); + $command = new Search($resourceChannelFactory, $this->solrSearch); + + $application = new Application([$command]); + + $command = $application->find('search:search'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + + $resultResource = new Resource( + '/test.php', + '', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); + $result = new SearchResult( + 1, + 1, + 0, + [$resultResource], + [new FacetGroup( + 'objectType', + [new Facet( + 'content', + 1 + )] + )], + 10 + ); + + $this->solrSearch->method('search') + ->willReturn($result); + + $this->commandTester->execute([ + 'text' => 'test abc' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<solrSearch->method('search') + ->willReturn($result); + + $this->commandTester->execute([ + 'text' => 'test abc' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<createStub( + ResourceChannelFactory::class + ); + $resourceChannelFactory->method('create') + ->willReturn($resourceChannel); + $this->solrSuggest = $this->createStub(SolrSuggest::class); + + $command = new Suggest($resourceChannelFactory, $this->solrSuggest); + + $application = new Application([$command]); + + $command = $application->find('search:suggest'); + $this->commandTester = new CommandTester($command); + } + + public function testExecute(): void + { + $result = new SuggestResult( + [ + new Suggestion('security', 10), + new Suggestion('section', 5) + ], + 10 + ); + $this->solrSuggest->method('suggest') + ->willReturn($result); + + $this->commandTester->execute([ + 'terms' => 'sec' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<solrSuggest->method('suggest') + ->willReturn($result); + + $this->commandTester->execute([ + 'terms' => 'sec' + ]); + + $this->commandTester->assertCommandIsSuccessful(); + + // the output of the command in the console + $output = $this->commandTester->getDisplay(); + $this->assertEquals( + <<expectException(InvalidArgumentException::class); + new IndexerParameter( + '', + 0, + 9 + ); + } +} diff --git a/test/Dto/Indexer/IndexerStatusTest.php b/test/Dto/Indexer/IndexerStatusTest.php new file mode 100644 index 0000000..30458f6 --- /dev/null +++ b/test/Dto/Indexer/IndexerStatusTest.php @@ -0,0 +1,147 @@ +setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $lastUpdate = new DateTime(); + $lastUpdate->setDate(2024, 1, 31); + $lastUpdate->setTime(13, 17, 12); + + $this->status = new IndexerStatus( + IndexerStatusState::FINISHED, + $startTime, + $endTime, + 10, + 5, + 4, + $lastUpdate, + 6, + 2 + ); + } + + public function testGetStatusLine(): void + { + $this->assertEquals( + '[FINISHED] ' . + 'start: 31.01.2024 11:15, ' . + 'time: 01h 01m 01s, ' . + 'processed: 5/10, ' . + 'skipped: 4, ' . + 'lastUpdate: 31.01.2024 13:17, ' . + 'updated: 6, ' . + 'errors: 2', + $this->status->getStatusLine(), + "unexpected status line" + ); + } + + public function testGetStatusLineForPreparing(): void + { + + $startTime = new DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $status = new IndexerStatus( + IndexerStatusState::PREPARING, + $startTime, + $endTime, + 10, + 5, + 4, + $startTime, + 6, + 2, + 'prepare message' + ); + + $this->assertEquals( + '[PREPARING] ' . + 'start: 31.01.2024 11:15, ' . + 'time: 01h 01m 01s, ' . + 'message: prepare message', + $status->getStatusLine(), + "unexpected status line" + ); + } + + public function testEmpty(): void + { + $status = IndexerStatus::empty(); + + $dateTimePattern = '[0-9]{2}\.[0-9]{2}\.[0-9]{4} [0-9]{2}:[0-9]{2}'; + $patter = '/\[UNKNOWN] ' . + 'start: ' . $dateTimePattern . ', ' . + 'time: 00h 00m 00s, ' . + 'processed: 0\/0, ' . + 'skipped: 0, ' . + 'lastUpdate: ' . $dateTimePattern . ', ' . + 'updated: 0, ' . + 'errors: 0' . + '/'; + + $this->assertMatchesRegularExpression( + $patter, + $status->getStatusLine(), + "unexpected status line for empty status" + ); + } + + public function testStatusLineWithoutEndTime(): void + { + $startTime = new DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $lastUpdate = new DateTime(); + $lastUpdate->setTimestamp(0); + + $status = new IndexerStatus( + IndexerStatusState::UNKNOWN, + $startTime, + null, + 0, + 0, + 0, + $lastUpdate, + 0, + 0 + ); + + $dateTimePattern = '[0-9]{2}\.[0-9]{2}\.[0-9]{4} [0-9]{2}:[0-9]{2}'; + $patter = '/lastUpdate: ' . $dateTimePattern . ', /'; + + $this->assertMatchesRegularExpression( + $patter, + $status->getStatusLine(), + "unexpected status line without endTime" + ); + } +} diff --git a/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php new file mode 100644 index 0000000..f90547c --- /dev/null +++ b/test/Dto/Search/Query/Filter/AbsoluteDateRangeFilterTest.php @@ -0,0 +1,44 @@ +assertEquals( + 'sp_date_list:[2021-01-01T00:00:00Z TO 2021-01-02T00:00:00Z]', + $filter->getQuery() + ); + } + + public function testGetQueryWithFrom(): void + { + $from = new \DateTime('2021-01-01 00:00:00'); + $filter = new AbsoluteDateRangeFilter($from, null, 'sp_date_list'); + $this->assertEquals( + 'sp_date_list:[2021-01-01T00:00:00Z TO *]', + $filter->getQuery() + ); + } + + public function testGetQueryWithTo(): void + { + $to = new \DateTime('2021-01-02 00:00:00'); + $filter = new AbsoluteDateRangeFilter(null, $to, 'sp_date_list'); + $this->assertEquals( + 'sp_date_list:[* TO 2021-01-02T00:00:00Z]', + $filter->getQuery() + ); + } +} diff --git a/test/Dto/Search/Query/Filter/AndFilterTest.php b/test/Dto/Search/Query/Filter/AndFilterTest.php new file mode 100644 index 0000000..dea1659 --- /dev/null +++ b/test/Dto/Search/Query/Filter/AndFilterTest.php @@ -0,0 +1,35 @@ +createStub(Filter::class); + $a->method('getQuery') + ->willReturn('a'); + + $b = $this->createStub(Filter::class); + $b->method('getQuery') + ->willReturn('b'); + + $and = new AndFilter([$a, $b]); + + assertEquals( + '(a AND b)', + $and->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/ArchiveFilterTest.php b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php new file mode 100644 index 0000000..022d96d --- /dev/null +++ b/test/Dto/Search/Query/Filter/ArchiveFilterTest.php @@ -0,0 +1,23 @@ +assertEquals( + '-sp_archive:true', + $filter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/FieldFilterTest.php b/test/Dto/Search/Query/Filter/FieldFilterTest.php new file mode 100644 index 0000000..13d45ae --- /dev/null +++ b/test/Dto/Search/Query/Filter/FieldFilterTest.php @@ -0,0 +1,51 @@ +expectException(InvalidArgumentException::class); + new FieldFilter('test', []); + } + + public function testGetQueryWithOneField(): void + { + $field = new FieldFilter('test', ['a']); + $this->assertEquals( + 'test:a', + $field->getQuery(), + 'unexpected query' + ); + } + + public function testGetQueryWithTwoFields(): void + { + $field = new FieldFilter('test', ['a', 'b']); + $this->assertEquals( + 'test:(a b)', + $field->getQuery(), + 'unexpected query' + ); + } + + public function testExclude(): void + { + $field = new FieldFilter('test', ['a']); + $exclude = $field->exclude(); + $this->assertEquals( + '-test:a', + $exclude->getQuery(), + 'unexpected exclude query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/NotFilterTest.php b/test/Dto/Search/Query/Filter/NotFilterTest.php new file mode 100644 index 0000000..a4bfecb --- /dev/null +++ b/test/Dto/Search/Query/Filter/NotFilterTest.php @@ -0,0 +1,28 @@ +createStub(Filter::class); + $filter->method('getQuery') + ->willReturn('a:b'); + $notFilter = new NotFilter($filter); + + $this->assertEquals( + 'NOT a:b', + $notFilter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/OrFilterTest.php b/test/Dto/Search/Query/Filter/OrFilterTest.php new file mode 100644 index 0000000..5d29ae5 --- /dev/null +++ b/test/Dto/Search/Query/Filter/OrFilterTest.php @@ -0,0 +1,35 @@ +createStub(Filter::class); + $a->method('getQuery') + ->willReturn('a'); + + $b = $this->createStub(Filter::class); + $b->method('getQuery') + ->willReturn('b'); + + $and = new OrFilter([$a, $b]); + + assertEquals( + '(a OR b)', + $and->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/QueryFilterTest.php b/test/Dto/Search/Query/Filter/QueryFilterTest.php new file mode 100644 index 0000000..d0a1315 --- /dev/null +++ b/test/Dto/Search/Query/Filter/QueryFilterTest.php @@ -0,0 +1,23 @@ +assertEquals( + 'a:b', + $filter->getQuery(), + 'unexpected query' + ); + } +} diff --git a/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php new file mode 100644 index 0000000..f43ae0d --- /dev/null +++ b/test/Dto/Search/Query/Filter/RelativeDateRangeFilterTest.php @@ -0,0 +1,208 @@ + + */ + public static function additionProviderForBeforeIntervals(): array + { + return [ + ['P1D', 'sp_date_list:[NOW-1DAYS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P1W', 'sp_date_list:[NOW-7DAYS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P2M', 'sp_date_list:[NOW-2MONTHS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ['P3Y', 'sp_date_list:[NOW-3YEARS/DAY TO NOW/DAY+1DAY-1SECOND]'], + ]; + } + + /** + * @return array + */ + public static function additionProviderForAfterIntervals(): array + { + return [ + ['P1D', 'sp_date_list:[NOW/DAY TO NOW+1DAYS/DAY+1DAY-1SECOND]'], + ['P1W', 'sp_date_list:[NOW/DAY TO NOW+7DAYS/DAY+1DAY-1SECOND]'], + ['P2M', 'sp_date_list:[NOW/DAY TO NOW+2MONTHS/DAY+1DAY-1SECOND]'], + ['P3Y', 'sp_date_list:[NOW/DAY TO NOW+3YEARS/DAY+1DAY-1SECOND]'], + ]; + } + + /** + * @return array + */ + public static function additionProviderForBeforeAndAfterIntervals(): array + { + return [ + [ + 'P1D', + 'P1D', + 'sp_date_list:[NOW-1DAYS/DAY TO NOW+1DAYS/DAY+1DAY-1SECOND]' + ], + [ + 'P1W', + 'P2M', + 'sp_date_list:[NOW-7DAYS/DAY TO NOW+2MONTHS/DAY+1DAY-1SECOND]' + ], + ]; + } + + /** + * @return array + */ + public static function additionProviderWithBase(): array + { + return [ + [ + new DateTime('2021-01-01 00:00:00'), + 'P1D', + null, + 'sp_date_list:[2021-01-01T00:00:00Z-1DAYS/DAY' . + ' TO 2021-01-01T00:00:00Z/DAY+1DAY-1SECOND]' + ], + [ + new DateTime('2021-01-01 00:00:00'), + null, + 'P2M', + 'sp_date_list:[2021-01-01T00:00:00Z/DAY' . + ' TO 2021-01-01T00:00:00Z+2MONTHS/DAY+1DAY-1SECOND]' + ], + [ + new DateTime('2021-01-01 00:00:00'), + 'P1W', + 'P2M', + 'sp_date_list:[2021-01-01T00:00:00Z-7DAYS/DAY' . + ' TO 2021-01-01T00:00:00Z+2MONTHS/DAY+1DAY-1SECOND]' + ], + ]; + } + + /** + * @return array + */ + public static function additionProviderForInvalidIntervals(): array + { + return [ + ['PT1H'], + ['PT1M'], + ['PT1S'], + ]; + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForBeforeIntervals')] + public function testGetQueryWithFrom( + string $before, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + null, + new DateInterval($before), + null, + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForAfterIntervals')] + public function testGetQueryWithTo( + string $after, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + null, + null, + new DateInterval($after), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForBeforeAndAfterIntervals')] + public function testGetQueryWithFromAndTo( + string $before, + string $after, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + null, + new DateInterval($before), + new DateInterval($after), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderWithBase')] + public function testGetQueryWithBase( + DateTime $base, + ?string $before, + ?string $after, + string $expected + ): void { + $filter = new RelativeDateRangeFilter( + $base, + $before === null ? null : new DateInterval($before), + $after === null ? null : new DateInterval($after), + ); + + $this->assertEquals( + $expected, + $filter->getQuery(), + 'unexpected query' + ); + } + + /** + * @throws Exception + */ + #[DataProvider('additionProviderForInvalidIntervals')] + public function testGetQueryWithInvalidIntervals( + string $interval + ): void { + $filter = new RelativeDateRangeFilter( + null, + null, + new DateInterval($interval), + ); + $this->expectException(InvalidArgumentException::class); + $filter->getQuery(); + } +} diff --git a/test/Dto/Search/Query/SearchQueryBuilderTest.php b/test/Dto/Search/Query/SearchQueryBuilderTest.php new file mode 100644 index 0000000..40f2e5e --- /dev/null +++ b/test/Dto/Search/Query/SearchQueryBuilderTest.php @@ -0,0 +1,135 @@ +builder = new SearchQueryBuilder(); + } + + public function testSetText(): void + { + $this->builder->text('abc'); + $query = $this->builder->build(); + $this->assertEquals('abc', $query->text, 'unexpected text'); + } + + public function testSetLang(): void + { + $this->builder->lang('en'); + $query = $this->builder->build(); + $this->assertEquals('en', $query->lang, 'unexpected lang'); + } + + public function testSetOffset(): void + { + $this->builder->offset(10); + $query = $this->builder->build(); + $this->assertEquals(10, $query->offset, 'unexpected offset'); + } + + public function testSetInvalidOffset(): void + { + $this->expectException(InvalidArgumentException::class); + $this->builder->offset(-1); + } + + public function testSetLimit(): void + { + $this->builder->limit(10); + $query = $this->builder->build(); + $this->assertEquals(10, $query->limit, 'unexpected limit'); + } + + public function testSetInvalidLimit(): void + { + $this->expectException(InvalidArgumentException::class); + $this->builder->limit(-1); + } + + /** + * @throws Exception + */ + public function testSetSort(): void + { + $criteria = $this->createStub(Criteria::class); + + $this->builder->sort($criteria); + $query = $this->builder->build(); + $this->assertEquals([$criteria], $query->sort, 'unexpected sort'); + } + + public function testSetFilter(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + $this->builder->filter($filter); + $query = $this->builder->build(); + $this->assertEquals([$filter], $query->filter, 'unexpected filter'); + } + + public function testSetTwoFilterWithSameKey(): void + { + $filterA = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + $filterB = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test']) + ->getMock(); + + $this->expectException(InvalidArgumentException::class); + $this->builder->filter($filterA, $filterB); + } + + public function testSetFacet(): void + { + $facet = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + $this->builder->facet($facet); + $query = $this->builder->build(); + $this->assertEquals([$facet], $query->facets, 'unexpected facets'); + } + + public function testSetTwoFacetSWithSameKey(): void + { + $facetA = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + $facetB = $this->getMockBuilder(Facet::class) + ->setConstructorArgs(['test', null]) + ->getMock(); + + $this->expectException(InvalidArgumentException::class); + $this->builder->facet($facetA, $facetB); + } + + public function testSetQueryDefaultOperator(): void + { + $this->builder->defaultQueryOperator(QueryOperator::AND); + $query = $this->builder->build(); + $this->assertEquals( + QueryOperator::AND, + $query->defaultQueryOperator, + 'unexpected queryDefaultOperator' + ); + } +} diff --git a/test/Exception/DocumentEnrichingExceptionTest.php b/test/Exception/DocumentEnrichingExceptionTest.php new file mode 100644 index 0000000..487304b --- /dev/null +++ b/test/Exception/DocumentEnrichingExceptionTest.php @@ -0,0 +1,22 @@ +assertEquals( + ResourceLocation::of('/test.php'), + $e->getLocation(), + 'unexpected location' + ); + } +} diff --git a/test/Exception/MissMatchingResourceFactoryExceptionTest.php b/test/Exception/MissMatchingResourceFactoryExceptionTest.php new file mode 100644 index 0000000..93bdddd --- /dev/null +++ b/test/Exception/MissMatchingResourceFactoryExceptionTest.php @@ -0,0 +1,24 @@ +assertEquals( + ResourceLocation::of('/test.php'), + $e->getLocation(), + 'unexpected location' + ); + } +} diff --git a/test/Exception/UnexpectedResultExceptionTest.php b/test/Exception/UnexpectedResultExceptionTest.php new file mode 100644 index 0000000..60cb767 --- /dev/null +++ b/test/Exception/UnexpectedResultExceptionTest.php @@ -0,0 +1,21 @@ +assertEquals( + 'test', + $e->getResult(), + 'unexpected result' + ); + } +} diff --git a/test/Exception/UnsupportedIndexLanguageExceptionTest.php b/test/Exception/UnsupportedIndexLanguageExceptionTest.php new file mode 100644 index 0000000..f72b9d1 --- /dev/null +++ b/test/Exception/UnsupportedIndexLanguageExceptionTest.php @@ -0,0 +1,38 @@ +assertEquals( + 'test', + $e->getIndex(), + 'unexpected index' + ); + } + + public function testGetLang(): void + { + $e = new UnsupportedIndexLanguageException( + 'test', + ResourceLanguage::of('de') + ); + $this->assertEquals( + ResourceLanguage::of('de'), + $e->getLang(), + 'unexpected index' + ); + } +} diff --git a/test/Service/AbstractIndexerTest.php b/test/Service/AbstractIndexerTest.php new file mode 100644 index 0000000..434721e --- /dev/null +++ b/test/Service/AbstractIndexerTest.php @@ -0,0 +1,125 @@ +createMock(IndexName::class); + $indexName->method('name') + ->willReturn('www'); + $this->progressHandler = $this->createMock( + IndexerProgressHandler::class + ); + $this->aborter = $this->createMock(IndexingAborter::class); + $config = new IndexerConfiguration( + 'test', + 'Test', + new DataBag([]) + ); + $this->configLoader = $this->createMock( + IndexerConfigurationLoader::class + ); + $this->configLoader->method('load') + ->willReturn($config); + $this->indexer = new TextIndexer( + $indexName, + $this->progressHandler, + $this->aborter, + $this->configLoader, + 'test' + ); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Test', + $this->indexer->getName(), + 'The name of the indexer should be "Test"' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'test', + $this->indexer->getSource(), + 'The source of the indexer should be "test"' + ); + } + + public function testGetProgressHandler(): void + { + $this->assertEquals( + $this->progressHandler, + $this->indexer->getProgressHandler(), + 'The progress handler should be the ' . + 'same as the one passed to the constructor' + ); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->indexer->setProgressHandler($progressHandler); + + $this->assertEquals( + $progressHandler, + $this->indexer->getProgressHandler(), + 'The progress handler should be the ' . + 'same as the one passed to the setProgressHandler method' + ); + } + + public function testAbort(): void + { + + $this->aborter->expects($this->once()) + ->method('requestAbortion') + ->with('www-test'); + $this->indexer->abort(); + } + + public function testEnabled(): void + { + + $this->configLoader->expects($this->once()) + ->method('exists') + ->with('test'); + $this->indexer->enabled(); + } + + public function testIsAbortionRequested(): void + { + + $this->aborter->expects($this->once()) + ->method('isAbortionRequested') + ->with('www-test'); + $this->indexer->isAbortionRequested(); + } +} diff --git a/test/Service/Indexer/BackgroundIndexerTest.php b/test/Service/Indexer/BackgroundIndexerTest.php new file mode 100644 index 0000000..f69dd79 --- /dev/null +++ b/test/Service/Indexer/BackgroundIndexerTest.php @@ -0,0 +1,145 @@ +progressHandler = $this->createStub( + IndexerProgressHandler::class + ); + $this->indexName = $this->createMock(IndexName::class); + $this->internalResourceIndexer = $this->createMock( + InternalResourceIndexer::class + ); + $this->internalResourceIndexer->method('getSource') + ->willReturn('internal'); + $this->internalResourceIndexer->method('getProgressHandler') + ->willReturn($this->progressHandler); + $this->statusStore = $this->createMock(IndexerStatusStore::class); + $this->indexer = new BackgroundIndexer( + $this->internalResourceIndexer, + $this->indexName, + $this->statusStore + ); + } + + public function testRemove(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('remove'); + $this->indexer->remove(['123']); + } + + public function testAbort(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('abort'); + $this->indexer->abort(); + } + + public function testIndex(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('index'); + $this->indexer->index(); + } + + public function testUpdate(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('update'); + $this->indexer->update(['/index.php']); + } + + public function testIndexIfLocked(): void + { + $lockFactory = $this->createStub(LockFactory::class); + $indexer = new BackgroundIndexer( + $this->createStub(InternalResourceIndexer::class), + $this->createStub(IndexName::class), + $this->statusStore + ); + + $lock = $this->createStub(SharedLockInterface::class); + $lock->method('acquire') + ->willReturn(false); + $lockFactory->method('createLock') + ->willReturn($lock); + + $this->internalResourceIndexer->expects($this->exactly(0)) + ->method('index'); + + $indexer->index(); + } + + public function testGetStatus(): void + { + $this->statusStore->expects($this->once()) + ->method('load'); + $this->indexer->getStatus(); + } + + public function testEnable(): void + { + $this->assertTrue( + $this->indexer->enabled(), + 'indexer should always be enabled' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'internal', + $this->indexer->getSource(), + 'Unexpected source' + ); + } + + public function testGetProgressHandler(): void + { + $this->internalResourceIndexer->expects($this->once()) + ->method('getProgressHandler'); + $this->indexer->getProgressHandler(); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->internalResourceIndexer->expects($this->once()) + ->method('setProgressHandler'); + $this->indexer->setProgressHandler($progressHandler); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Background Indexer', + $this->indexer->getName(), + 'Unexpected name' + ); + } +} diff --git a/test/Service/Indexer/ContentCollectorTest.php b/test/Service/Indexer/ContentCollectorTest.php new file mode 100644 index 0000000..9e73e41 --- /dev/null +++ b/test/Service/Indexer/ContentCollectorTest.php @@ -0,0 +1,48 @@ + [ + [ + "model" => [ + "richText" => [ + "normalized" => true, + "modelType" => "html.richText", + "text" => "

Ein Text

" + ] + ] + ] + ] + ]; + $content = $collector->collect($data); + + $this->assertEquals('

Ein Text

', $content, 'unexpected content'); + } +} diff --git a/test/Service/Indexer/DocumentEnrichter.php b/test/Service/Indexer/DocumentEnrichter.php new file mode 100644 index 0000000..c5fed38 --- /dev/null +++ b/test/Service/Indexer/DocumentEnrichter.php @@ -0,0 +1,9 @@ +createStub(ResourceLoader::class); + $documentEnricher = $this->createStub(DocumentEnricher::class); + $documentEnricher->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + $doc->sp_id = '123'; + return $doc; + }); + $dumper = new IndexDocumentDumper( + $resourceLoader, + [$documentEnricher] + ); + + $dump = $dumper->dump(['/test.php']); + + $this->assertEquals( + [['sp_id' => '123']], + $dump, + 'unexpected dump' + ); + } +} diff --git a/test/Service/Indexer/IndexSchema2xDocumentTest.php b/test/Service/Indexer/IndexSchema2xDocumentTest.php new file mode 100644 index 0000000..a0e9a36 --- /dev/null +++ b/test/Service/Indexer/IndexSchema2xDocumentTest.php @@ -0,0 +1,61 @@ +sp_id = '123'; + + $this->assertEquals( + ['sp_id' => '123'], + $doc->getFields(), + 'unexpected fields' + ); + } + + public function testSetMetaString(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaString('myname', 'myvalue'); + + $this->assertEquals( + ['sp_meta_string_myname' => 'myvalue'], + $doc->getFields(), + 'unexpected meta fields' + ); + } + + public function testSetMetaText(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaText('myname', 'myvalue'); + + $this->assertEquals( + ['sp_meta_text_myname' => 'myvalue'], + $doc->getFields(), + 'unexpected meta fields' + ); + } + + public function testSetMetaBool(): void + { + $doc = new IndexSchema2xDocument(); + $doc->setMetaBool('myname', true); + + $this->assertEquals( + ['sp_meta_bool_myname' => true], + $doc->getFields(), + 'unexpected meta fields' + ); + } +} diff --git a/test/Service/Indexer/IndexerCollectionTest.php b/test/Service/Indexer/IndexerCollectionTest.php new file mode 100644 index 0000000..88be22c --- /dev/null +++ b/test/Service/Indexer/IndexerCollectionTest.php @@ -0,0 +1,52 @@ +createStub(Indexer::class); + $indexer->method('getSource')->willReturn('test'); + $indexers = new IndexerCollection([$indexer]); + $this->assertNotNull($indexers->getIndexer('test')); + } + + public function testGetMissingIndexer(): void + { + $indexers = new IndexerCollection([]); + $this->expectException(\InvalidArgumentException::class); + $indexers->getIndexer('test'); + } + + public function testGetIndexers(): void + { + $indexer = $this->createStub(Indexer::class); + $indexers = new IndexerCollection([$indexer]); + $this->assertCount( + 1, + $indexers->getIndexers(), + 'unexpected number of indexers' + ); + } + + public function testGetIndexersWithIterable(): void + { + $indexer = $this->createStub(Indexer::class); + $indexers = new IndexerCollection(new ArrayIterator([$indexer])); + $this->assertCount( + 1, + $indexers->getIndexers(), + 'unexpected number of indexers' + ); + } +} diff --git a/test/Service/Indexer/IndexerConfigurationLoaderTest.php b/test/Service/Indexer/IndexerConfigurationLoaderTest.php new file mode 100644 index 0000000..c61f907 --- /dev/null +++ b/test/Service/Indexer/IndexerConfigurationLoaderTest.php @@ -0,0 +1,142 @@ +createLoader( + self::RESOURCE_BASE . '/with-internal' + ); + $this->assertTrue( + $loader->exists('internal'), + 'Internal config should exist' + ); + } + + public function testLoad(): void + { + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); + + $config = $loader->load('internal'); + + $expected = new IndexerConfiguration( + 'internal', + 'Internal Indexer', + new DataBag([ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ]), + ); + + $this->assertEquals( + $expected, + $config, + 'unexpected config' + ); + } + + public function testLoadNotExists(): void + { + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); + + $config = $loader->load('not-exists'); + + $expected = new IndexerConfiguration( + 'not-exists', + 'not-exists', + new DataBag([ + ]), + ); + + $this->assertEquals( + $expected, + $config, + 'unexpected config' + ); + } + + public function testLoadAll(): void + { + $loader = $this->createLoader( + self::RESOURCE_BASE . '/with-internal' + ); + + $expected = new IndexerConfiguration( + 'internal', + 'Internal Indexer', + new DataBag([ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ]), + ); + + $this->assertEquals( + [$expected], + $loader->loadAll(), + 'unexpected config' + ); + } + + public function testLoadAllNotADirectory(): void + { + $loader = $this->createLoader( + self::RESOURCE_BASE . '/not-a-directory' + ); + + $this->assertEquals( + [], + $loader->loadAll(), + 'should return empty array' + ); + } + + public function testLoadAllWithConfigReturnString(): void + { + $loader = $this->createLoader( + self::RESOURCE_BASE . '/return-string' + ); + + $this->expectException(RuntimeException::class); + $loader->loadAll(); + } + + private function createLoader( + string $resourceDir + ): IndexerConfigurationLoader { + $resourceChannel = new ResourceChannel( + '', + '', + '', + '', + false, + '', + '', + '', + $resourceDir, + '', + [] + ); + return new IndexerConfigurationLoader($resourceChannel); + } +} diff --git a/test/Service/Indexer/IndexerProgressStateTest.php b/test/Service/Indexer/IndexerProgressStateTest.php new file mode 100644 index 0000000..00ae371 --- /dev/null +++ b/test/Service/Indexer/IndexerProgressStateTest.php @@ -0,0 +1,245 @@ +createMock(IndexName::class); + $indexName->method('name')->willReturn('test'); + + $this->status = IndexerStatus::empty(); + + $this->statusStore = $this->createMock(IndexerStatusStore::class); + $that = $this; + $this->statusStore->method('load') + ->willReturnCallback(function () use ($that) { + return $that->status; + }); + $this->state = new IndexerProgressState( + $indexName, + $this->statusStore, + 'source' + ); + } + + public function testPrepare(): void + { + $this->state->prepare('prepare message'); + + $this->assertMatchesRegularExpression( + '/\[PREPARING].*prepare message.*/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testStart(): void + { + $this->state->start(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testStartAfterPrepare(): void + { + + $startTime = new \DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new \DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $this->status = new IndexerStatus( + IndexerStatusState::PREPARING, + $startTime, + $endTime, + 0, + 0, + 0, + $endTime, + 0, + 0, + 'prepare message' + ); + + $this->state->start(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*start: 31\.01\.2024 11:15,.*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testUpdate(): void + { + + $this->statusStore + ->method('load') + ->willReturn(IndexerStatus::empty()); + + $this->state->startUpdate(10); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 0\/10,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testAdvanceWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->advance(1); + } + + public function testAdvanceForUpdate(): void + { + + $this->statusStore + ->method('load') + ->willReturn(IndexerStatus::empty()); + + $this->state->startUpdate(10); + + $this->statusStore->expects($this->once()) + ->method('store'); + + $this->state->advance(1); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*processed: 1\/10,.*updated: 1,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testSkip(): void + { + + $this->state->start(10); + + $this->state->skip(1); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*skipped: 1,/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testSkipWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->skip(1); + } + + public function testError(): void + { + $this->state->start(10); + + $this->state->error(new Exception('test')); + + $this->assertMatchesRegularExpression( + '/\[RUNNING].*errors: 1$/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testErrorWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->error(new Exception('test')); + } + + public function testFinish(): void + { + $this->state->start(10); + + $this->statusStore->expects($this->once()) + ->method('store'); + + $this->state->finish(); + + $this->assertMatchesRegularExpression( + '/\[FINISHED]/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testFinishWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->finish(); + } + + public function testAbort(): void + { + $this->state->start(10); + + $this->state->abort(); + + $this->assertMatchesRegularExpression( + '/\[ABORTED]/', + $this->state->getStatus()->getStatusLine(), + "unexpected status line" + ); + } + + public function testAbortWithoutStart(): void + { + $this->expectException(LogicException::class); + $this->state->abort(); + } + + public function testGetStatus(): void + { + $indexName = $this->createMock(IndexName::class); + $indexName->method('name')->willReturn('test'); + $status = $this->createMock(IndexerStatus::class); + $statusStore = $this->createMock(IndexerStatusStore::class); + $statusStore->method('load') + ->willReturn($status); + + $state = new IndexerProgressState( + $indexName, + $statusStore, + 'source' + ); + + $this->assertEquals( + $status, + $state->getStatus(), + 'unexpected status' + ); + } +} diff --git a/test/Service/Indexer/IndexerStatusStoreTest.php b/test/Service/Indexer/IndexerStatusStoreTest.php new file mode 100644 index 0000000..18badf8 --- /dev/null +++ b/test/Service/Indexer/IndexerStatusStoreTest.php @@ -0,0 +1,213 @@ +exists(self::TEST_DIR)) { + $filesystem->chmod(self::TEST_DIR, 0777, 0000, true); + $filesystem->remove(self::TEST_DIR); + } + $filesystem->mkdir(self::TEST_DIR); + } + + public function testStore(): void + { + $status = $this->createIndexerStatus(); + + $store = new IndexerStatusStore(self::TEST_DIR); + $store->store('test', $status); + + $json = file_get_contents( + self::TEST_DIR . '/atoolo.search.index.test.status.json' + ); + + $expected = + '{' . + '"state":"FINISHED",' . + '"startTime":"2024-01-31T11:15:10+00:00",' . + '"endTime":"2024-01-31T12:16:11+00:00",' . + '"total":10,' . + '"processed":5,' . + '"skipped":4,' . + '"lastUpdate":"2024-01-31T13:17:12+00:00",' . + '"updated":6,' . + '"errors":2,' . + '"prepareMessage":""' . + '}'; + + $this->assertEquals($expected, $json, 'unexpected json string'); + } + + public function testStoreWithNonExistsBaseDir(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/not-exists'; + $store = new IndexerStatusStore($baseDir); + + $store->store('test', $status); + + $this->assertDirectoryExists( + $baseDir, + 'non exists basedir should be created' + ); + } + + public function testStoreWithNonWritableBaseDir(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/non-writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($baseDir); + $filesystem->chmod($baseDir, 0000); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + + public function testStoreWithNonWritableStatusFile(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($baseDir); + + $file = $baseDir . '/atoolo.search.index.test-not-writable.status.json'; + touch($file); + $filesystem->chmod($file, 0000); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test-not-writable', $status); + } + + public function testStoreWithBaseDirNoADirectory(): void + { + $status = $this->createIndexerStatus(); + $baseDir = self::TEST_DIR . '/non-dir'; + touch($baseDir); + + $store = new IndexerStatusStore($baseDir); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + + public function testStoreWithNonCreatableBaseDir(): void + { + $status = $this->createIndexerStatus(); + $nonWritable = self::TEST_DIR . '/non-writable'; + + $filesystem = new Filesystem(); + $filesystem->mkdir($nonWritable); + $filesystem->chmod($nonWritable, 0000); + + $store = new IndexerStatusStore($nonWritable . '/subdir'); + + $this->expectException(RuntimeException::class); + $store->store('test', $status); + } + + public function testLoad(): void + { + $baseDir = __DIR__ . '/../../resources/' . + 'Service/Indexer/IndexerStatusStore'; + + $store = new IndexerStatusStore($baseDir); + + $status = $store->load('test'); + + $expected = $this->createIndexerStatus(); + $this->assertEquals( + $expected, + $status, + 'unexpected status' + ); + } + + public function testLoadFileNotExists(): void + { + $baseDir = __DIR__ . '/../../resources/' . + 'Service/Indexer/IndexerStatusStore'; + + $store = new IndexerStatusStore($baseDir); + + $status = $store->load('test-not-exists'); + + $this->assertEquals( + 0, + $status->total, + 'empty status expected' + ); + } + + /** + * @throws ExceptionInterface + */ + public function testLoadFileNotReadable(): void + { + $file = self::TEST_DIR . '/' . + 'atoolo.search.index.test-not-readable.status.json'; + + $filesystem = new Filesystem(); + $filesystem->touch($file); + $filesystem->chmod($file, 0000); + + $store = new IndexerStatusStore(self::TEST_DIR); + + $this->expectException(InvalidArgumentException::class); + $store->load('test-not-readable'); + } + + private function createIndexerStatus(): IndexerStatus + { + + $startTime = new \DateTime(); + $startTime->setDate(2024, 1, 31); + $startTime->setTime(11, 15, 10); + + $endTime = new \DateTime(); + $endTime->setDate(2024, 1, 31); + $endTime->setTime(12, 16, 11); + + $lastUpdate = new \DateTime(); + $lastUpdate->setDate(2024, 1, 31); + $lastUpdate->setTime(13, 17, 12); + + return new IndexerStatus( + IndexerStatusState::FINISHED, + $startTime, + $endTime, + 10, + 5, + 4, + $lastUpdate, + 6, + 2 + ); + } +} diff --git a/test/Service/Indexer/IndexingAborterTest.php b/test/Service/Indexer/IndexingAborterTest.php new file mode 100644 index 0000000..35afb76 --- /dev/null +++ b/test/Service/Indexer/IndexingAborterTest.php @@ -0,0 +1,65 @@ +file = $workdir . '/background-indexer-test.abort'; + if (file_exists($this->file)) { + unlink($this->file); + } + $this->aborter = new IndexingAborter($workdir, 'background-indexer'); + } + + public function testIsAbortionRequested(): void + { + $this->assertFalse( + $this->aborter->isAbortionRequested('test'), + 'should not aborted' + ); + } + public function testIsAbortionRequestedWithExistsMarkerFile(): void + { + touch($this->file); + $this->assertTrue( + $this->aborter->isAbortionRequested('test'), + 'should not aborted' + ); + } + + public function testRequestAbortion(): void + { + $this->aborter->requestAbortion('test'); + $this->assertFileExists( + $this->file, + 'requestAbortion call should create file' + ); + } + + public function testResetAbortionRequest(): void + { + touch($this->file); + $this->aborter->resetAbortionRequest('test'); + $this->assertFileDoesNotExist( + $this->file, + 'resetAbortionRequest call should remove file' + ); + } +} diff --git a/test/Service/Indexer/InternalResourceIndexerTest.php b/test/Service/Indexer/InternalResourceIndexerTest.php new file mode 100644 index 0000000..b55cf3f --- /dev/null +++ b/test/Service/Indexer/InternalResourceIndexerTest.php @@ -0,0 +1,480 @@ +indexerFilter = $this->createMock( + ResourceFilter::class + ); + + $this->indexerProgressHandler = $this->createMock( + IndexerProgressHandler::class + ); + $this->finder = $this->createMock(LocationFinder::class); + $this->documentEnricher = $this->createMock(DocumentEnricher::class); + $this->documentEnricher + ->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + return $doc; + }); + $this->translationSplitter = new SubDirTranslationSplitter(); + $this->resourceLoader = $this->createStub(ResourceLoader::class); + $this->resourceLoader->method('load') + ->willReturnCallback(function ($location) { + return new Resource( + $location->location, + '', + '', + '', + $location->lang, + new DataBag([]) + ); + }); + $this->solrIndexService = $this->createMock(SolrIndexService::class); + $this->updateResult = $this->createStub(UpdateResult::class); + $this->updater = $this->createMock(SolrIndexUpdater::class); + $this->updater->method('update')->willReturn($this->updateResult); + $this->updater->method('createDocument')->willReturn( + new IndexSchema2xDocument() + ); + $this->solrIndexService->method('getManagedIndices') + ->willReturnCallback(function () { + return $this->availableIndexes; + }); + $this->solrIndexService->method('getIndex') + ->willReturnCallback(function ($lang) { + if ($lang->code === 'en') { + return 'test-en_US'; + } + if ($lang->code === 'fr') { + throw new UnsupportedIndexLanguageException( + 'test', + $lang, + 'unsupported language' + ); + } + return 'test'; + }); + $this->solrIndexService->method('updater') + ->willReturn($this->updater); + $this->aborter = $this->createMock(IndexingAborter::class); + $this->indexerConfiguration = new IndexerConfiguration( + 'test-source', + 'Indexer-Name', + new DataBag([ + 'cleanupThreshold' => 10, + 'chunkSize' => 10 + ]) + ); + $this->indexerConfigurationLoader = $this->createMock( + IndexerConfigurationLoader::class + ); + $this->indexerConfigurationLoader->method('load') + ->willReturn($this->indexerConfiguration); + + $this->lockFactory = new LockFactory(new SemaphoreStore()); + $this->logger = $this->createMock(LoggerInterface::class); + + $this->indexer = new InternalResourceIndexer( + [ $this->documentEnricher ], + $this->indexerFilter, + $this->indexerProgressHandler, + $this->finder, + $this->resourceLoader, + $this->translationSplitter, + $this->solrIndexService, + $this->aborter, + $this->indexerConfigurationLoader, + 'test-source', + $this->logger, + $this->lockFactory, + ); + } + + public function testAbort(): void + { + $this->aborter->expects($this->once()) + ->method('requestAbortion') + ->with('test'); + + $this->indexer->abort(); + } + + public function testRemove(): void + { + $this->solrIndexService->expects($this->once()) + ->method('deleteByIdListForAllLanguages'); + + $this->indexer->remove(['123']); + } + + public function testRemoveEmpty(): void + { + $this->solrIndexService->expects($this->exactly(0)) + ->method('deleteByIdListForAllLanguages'); + + $this->indexer->remove([]); + } + + public function testIndexAllWithChunks(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/b.php.translations/en_US.php', + '/a/c.php', + '/a/c.php.translations/fr_FR.php', + '/a/d.php', + '/a/e.php', + '/a/f.php', + '/a/g.php', + '/a/h.php', + '/a/i.php', + '/a/j.php', + '/a/k.php', + '/a/l.php', + '/a/error.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + $this->indexerFilter->method('accept') + ->willReturn(true); + + $this->documentEnricher + ->method('enrichDocument') + ->willReturnCallback(function ($resource, $doc) { + if ($resource->location === '/a/error.php') { + throw new RuntimeException('test'); + } + return $doc; + }); + + $this->updater->expects($this->exactly(12)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(3)) + ->method('update'); + + $this->indexerProgressHandler->expects($this->exactly(2)) + ->method('error'); + + $this->indexer->index(); + } + + public function testIndexAllWithEmptyList(): void + { + $this->finder->method('findAll') + ->willReturn([ + ]); + $this->indexerProgressHandler->expects($this->once()) + ->method('start') + ->with(0); + $this->indexerProgressHandler->expects($this->once()) + ->method('finish'); + + $this->indexer->index(); + } + + public function testIndexSkipResource(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->indexerFilter->method('accept') + ->willReturnCallback(function (Resource $resource) { + return ($resource->location !== '/a/b.php'); + }); + + $this->updater->expects($this->exactly(1)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); + + $this->indexer->index(); + } + + public function testAborted(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->aborter->method('isAbortionRequested') + ->willReturn(true); + + $this->aborter->expects($this->once()) + ->method('resetAbortionRequest'); + + $this->indexerProgressHandler->expects($this->once()) + ->method('abort'); + + $this->indexer->index(); + } + + public function testWithUnsuccessfulStatus(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(500); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $this->indexer->index(); + } + + public function testWithInvalidResource(): void + { + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + ]); + + $this->resourceLoader->method('load') + ->willThrowException( + new InvalidResourceException( + ResourceLocation::of('/a/b.php') + ) + ); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $this->indexer->index(); + } + + public function testUpdate(): void + { + $this->finder->method('findPaths') + ->willReturn([ + '/a/b.php', + '/a/c.php' + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->indexerFilter->method('accept') + ->willReturn(true); + + $this->updater->expects($this->exactly(2)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); + + $this->indexer->update([ + '/a/b.php', + '/a/c.php' + ]); + } + + public function testUpdateOtherLang(): void + { + $this->finder->method('findPaths') + ->willReturn([ + '/a/b.php.translations/en_US.php', + ]); + + $this->updateResult->method('getStatus') + ->willReturn(0); + + $this->indexerFilter->method('accept') + ->willReturn(true); + + $this->updater->expects($this->exactly(1)) + ->method('addDocument'); + + $this->updater->expects($this->exactly(1)) + ->method('update'); + + $this->indexer->update([ + '/a/b.php.translations/en_US.php', + ]); + } + + public function testUpdateWithParameter(): void + { + $this->finder->expects($this->once()) + ->method('findPaths') + ->with($this->equalTo(['?a=b'])); + + $this->indexer->update([ + '?a=b' + ]); + } + + public function testWithoutAvailableIndexes(): void + { + + $this->availableIndexes = []; + $this->finder->method('findAll') + ->willReturn([ + '/a/b.php', + '/a/c.php', + '/a/d.php', + '/a/e.php', + '/a/f.php', + '/a/g.php', + '/a/h.php', + '/a/i.php', + '/a/j.php', + '/a/k.php', + '/a/l.php' + ]); + + $this->indexerProgressHandler->expects($this->once()) + ->method('error'); + + $this->indexer->index(); + } + + public function testEnabled(): void + { + $this->assertTrue( + $this->indexer->enabled(), + 'indexer should be always enabled' + ); + } + + public function testGetName(): void + { + $this->assertEquals( + 'Indexer-Name', + $this->indexer->getName(), + 'unexpected Indexer Name' + ); + } + + public function testGetProgressHandler(): void + { + $this->assertEquals( + $this->indexerProgressHandler, + $this->indexer->getProgressHandler(), + 'unexpected progress handler' + ); + } + + public function testSetProgressHandler(): void + { + $progressHandler = $this->createStub(IndexerProgressHandler::class); + $this->indexer->setProgressHandler($progressHandler); + $this->assertEquals( + $progressHandler, + $this->indexer->getProgressHandler(), + 'unexpected progress handler' + ); + } + + public function testGetSource(): void + { + $this->assertEquals( + 'test-source', + $this->indexer->getSource(), + 'unexpected source' + ); + } + + public function testLock(): void + { + $this->logger->expects($this->once()) + ->method('notice') + ->with('Indexer is already running', [ + 'index' => 'test' + ]); + $lock = $this->lockFactory->createLock('indexer.test'); + try { + $lock->acquire(); + $this->indexer->index(); + } finally { + $lock->release(); + } + } +} diff --git a/test/Service/Indexer/LocationFinderTest.php b/test/Service/Indexer/LocationFinderTest.php new file mode 100644 index 0000000..10cc69b --- /dev/null +++ b/test/Service/Indexer/LocationFinderTest.php @@ -0,0 +1,85 @@ +locationFinder = new LocationFinder($resourceChannel); + } + + public function testFindAll(): void + { + $locations = $this->locationFinder->findAll(); + $this->assertEquals( + [ + '/a.php', + '/b/c.php', + '/f.pdf.media.php' + ], + $locations, + 'unexpected locations' + ); + } + + public function testFindPathsWithFile(): void + { + $locations = $this->locationFinder->findPaths( + [ + 'a.php' + ] + ); + $this->assertEquals( + [ + '/a.php', + ], + $locations, + 'unexpected locations' + ); + } + + public function testFindPathsWithDirectory(): void + { + $locations = $this->locationFinder->findPaths( + [ + '/b', + ] + ); + $this->assertEquals( + [ + '/b/c.php', + ], + $locations, + 'unexpected locations' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php new file mode 100644 index 0000000..214448f --- /dev/null +++ b/test/Service/Indexer/SiteKit/DefaultSchema2xDocumentEnricherTest.php @@ -0,0 +1,773 @@ +createStub( + SiteKitNavigationHierarchyLoader::class + ); + $navigationLoader + ->method('loadRoot') + ->willReturnCallback(function ($location) { + if ($location->location === 'throwException') { + throw new InvalidResourceException($location); + } + return $this->createResource([ + 'siteGroup' => ['id' => 999] + ]); + }); + $contentCollector = $this->createStub(ContentCollector::class); + $contentCollector + ->method('collect') + ->willReturn('collected content'); + + $this->enricher = new DefaultSchema2xDocumentEnricher( + $navigationLoader, + $contentCollector + ); + } + + public function testEnrichSpId(): void + { + $resource = new Resource( + '', + '123', + '', + '', + ResourceLanguage::default(), + new DataBag([]) + ); + $doc = $this->enrichWithResource($resource); + $this->assertEquals('123', $doc->sp_id, 'unexpected id'); + } + + public function testEnrichName(): void + { + $resource = $this->createResource([ + 'name' => 'test' + ]); + $doc = $this->enrichWithResource($resource); + $this->assertEquals('test', $doc->sp_name, 'unexpected name'); + } + + public function testEnrichObjectType(): void + { + $resource = $this->createResource([ + 'objectType' => 'test' + ]); + $doc = $this->enrichWithResource($resource); + $this->assertEquals( + 'test', + $doc->sp_objecttype, + 'unexpected objectType' + ); + } + + public function testEnrichAnchor(): void + { + $doc = $this->enrichWithData(['anchor' => 'abc']); + $this->assertEquals('abc', $doc->sp_anchor, 'unexpected ancohr'); + } + + public function testEnrichTitle(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'abc']]); + $this->assertEquals('abc', $doc->title, 'unexpected title'); + } + + public function testEnrichDescriptionWithInto(): void + { + $doc = $this->enrichWithData(['metadata' => ['intro' => 'abc']]); + $this->assertEquals( + 'abc', + $doc->description, + 'unexpected description' + ); + } + + public function testEnrichCanonical(): void + { + $doc = $this->enrichWithData([]); + $this->assertTrue( + $doc->sp_canonical, + 'unexpected canonical' + ); + } + + public function testEnrichCrawlProcessId(): void + { + $resource = $this->createResource([]); + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + $this->assertEquals( + $doc->crawl_process_id, + 'progress-id', + 'unexpected progress id' + ); + } + + public function testEnrichId(): void + { + $doc = $this->enrichWithData(['url' => '/test.php']); + $this->assertEquals( + '/test.php', + $doc->id, + 'unexpected id' + ); + } + + public function testEnrichUrl(): void + { + $doc = $this->enrichWithData(['url' => '/test.php']); + $this->assertEquals( + '/test.php', + $doc->url, + 'unexpected url' + ); + } + + public function testEnrichMediaId(): void + { + $doc = $this->enrichWithData(['mediaUrl' => '/test.php']); + $this->assertEquals( + '/test.php', + $doc->id, + 'unexpected id' + ); + } + + public function testEnrichMediaUrl(): void + { + $doc = $this->enrichWithData(['mediaUrl' => '/test.php']); + $this->assertEquals( + '/test.php', + $doc->url, + 'unexpected url' + ); + } + + public function testEnrichSpContentType(): void + { + $doc = $this->enrichWithData([ + 'objectType' => 'content', + 'contentSectionTypes' => ['text', 'linkList'], + 'base' => [ + 'teaser' => [ + 'headline' => 'test', + 'image' => [ + 'copyright' => 'test' + ], + 'text' => 'test' + ] + ] + ]); + $this->assertEquals( + [ + 'content', + 'article', + 'text', + 'linkList', + 'teaserImage', + 'teaserImageCopyright', + 'teaserHeadline', + 'teaserText' + ], + $doc->sp_contenttype, + 'unexpected sp_contenttype' + ); + } + + public function testEnrichDefaultLanguage(): void + { + $doc = $this->enrichWithData([]); + $this->assertEquals( + 'de', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguage(): void + { + $doc = $this->enrichWithData(['locale' => 'en_US']); + $this->assertEquals( + 'en', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguageWithShortLocale(): void + { + $doc = $this->enrichWithData(['locale' => 'en']); + $this->assertEquals( + 'en', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichLanguageOverGroupPath(): void + { + $doc = $this->enrichWithData([ + 'groupPath' => [ + ['id' => 1, 'locale' => 'fr_FR'], + ['id' => 2, 'locale' => 'it_IT'] + ] + ]); + $this->assertEquals( + 'it', + $doc->sp_language, + 'unexpected language' + ); + } + + public function testEnrichDefaultMetaContentLanguage(): void + { + $doc = $this->enrichWithData([]); + $this->assertEquals( + 'de', + $doc->meta_content_language, + 'unexpected language' + ); + } + + public function testEnrichChanged(): void + { + $doc = $this->enrichWithData(['changed' => 1708932236]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_changed, + 'unexpected sp_changed' + ); + } + + public function testEnrichGenerated(): void + { + $doc = $this->enrichWithData(['generated' => 1708932236]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_generated, + 'unexpected sp_generated' + ); + } + + public function testEnrichDate(): void + { + $doc = $this->enrichWithData(['base' => ['date' => 1708932236]]); + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_date, + 'unexpected sp_generated' + ); + } + + public function testEnrichArchive(): void + { + $doc = $this->enrichWithData(['base' => ['archive' => true]]); + $this->assertTrue( + $doc->sp_archive, + 'unexpected language' + ); + } + + public function testEnrichSpTitle(): void + { + $doc = $this->enrichWithData(['metadata' => ['headline' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSpTitleWithTeaserHeadlineFallback(): void + { + $doc = $this->enrichWithData(['base' => [ + 'title' => 'test', + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSpTitleWithTeaserTitleFallback(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_title, + 'unexpected sp_title' + ); + } + + public function testEnrichSortValue(): void + { + $doc = $this->enrichWithData(['base' => [ + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichSortValueWithHeadlineFallback(): void + { + $doc = $this->enrichWithData(['base' => [ + 'title' => 'test', + 'teaser' => [ + 'headline' => 'test' + ] + ]]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichSortValueWithTitleFallback(): void + { + $doc = $this->enrichWithData(['base' => ['title' => 'test']]); + $this->assertEquals( + 'test', + $doc->sp_sortvalue, + 'unexpected sp_sortvalue' + ); + } + + public function testEnrichKeywords(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'keywords' => ['abc', 'cde'] + ]]); + $this->assertEquals( + ['abc', 'cde'], + $doc->keywords, + 'unexpected keywords' + ); + } + + public function testEnrichBoostKeywords(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'boostKeywords' => ['abc', 'cde'] + ]]); + $this->assertEquals( + 'abc cde', + $doc->sp_boost_keywords, + 'unexpected keywords' + ); + } + + public function testEnrichSpSites(): void + { + $doc = $this->enrichWithData(['base' => [ + 'trees' => [ + 'navigation' => [ + 'parents' => [ + ['siteGroup' => ['id' => '123']], + ['siteGroup' => ['id' => '456']] + ] + ] + ] + ]]); + $this->assertEquals( + ['123', '456', '999'], + $doc->sp_site, + 'unexpected keywords' + ); + } + + public function testEnrichSpSitesWithInvalidRootResource(): void + { + $resource = $this->createResource([ + 'url' => 'throwException' + ]); + + $this->expectException(DocumentEnrichingException::class); + $this->enrichWithResource($resource); + } + + public function testEnrichGeoPoints(): void + { + $doc = $this->enrichWithData(['base' => [ + 'geo' => [ + 'wkt' => [ + 'primary' => [ + 'value' => 'test' + ] + ] + ] + ]]); + $this->assertEquals( + ['value' => 'test'], + $doc->sp_geo_points, + 'unexpected sp_geo_points' + ); + } + + public function testEnrichCategories(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'categories' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + ['1', '2', '3'], + $doc->sp_category, + 'unexpected sp_category' + ); + } + + public function testEnrichCategoryPath(): void + { + $doc = $this->enrichWithData(['metadata' => [ + 'categoriesPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]]); + $this->assertEquals( + ['1', '2', '3'], + $doc->sp_category_path, + 'unexpected sp_category_path' + ); + } + + public function testEnrichSpGroup(): void + { + $doc = $this->enrichWithData([ + 'groupPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]); + $this->assertEquals( + 2, + $doc->sp_group, + 'unexpected sp_group' + ); + } + + public function testEnrichSpGroupPath(): void + { + $doc = $this->enrichWithData([ + 'groupPath' => [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ] + ]); + $this->assertEquals( + [1, 2, 3], + $doc->sp_group_path, + 'unexpected sp_group_path' + ); + } + + public function testEnrichDateViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'base' => ['date' => 1707549836], + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test'], + ['from' => 1709105036, 'contentType' => 'test'], + ] + ] + ]); + + $expected = new DateTime(); + $expected->setTimestamp(1708932236); + + $this->assertEquals( + $expected, + $doc->sp_date, + 'unexpected sp_date' + ); + } + + public function testEnrichContentTypeViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'objectType' => 'content', + 'base' => ['date' => 1707549836], + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test1'], + ['from' => 1709105036, 'contentType' => 'test2'], + ['from' => 1707981836, 'contentType' => 'test1'], + ] + ] + ]); + + $this->assertEquals( + ['content', 'article', 'test1', 'test2'], + $doc->sp_contenttype, + 'unexpected sp_contenttype' + ); + } + + public function testEnrichDateListViaScheduling(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'scheduling' => [ + ['from' => 1708932236, 'contentType' => 'test'], + ['from' => 1709105036, 'contentType' => 'test'], + ] + ] + ]); + + $dateA = new DateTime(); + $dateA->setTimestamp(1708932236); + $dateB = new DateTime(); + $dateB->setTimestamp(1709105036); + + $this->assertEquals( + [$dateA, $dateB], + $doc->sp_date_list, + 'unexpected sp_date' + ); + } + + public function testEnrichDefaultMetaContentType(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + 'text/html; charset=UTF-8', + $doc->meta_content_type, + 'unexpected meta_content_type' + ); + } + + public function testEnrichIncludeGroups(): void + { + $doc = $this->enrichWithData([ + 'access' => [ + 'type' => 'allow', + 'groups' => ['100010100000001028'] + ] + ]); + + $this->assertEquals( + ['1028'], + $doc->include_groups, + 'unexpected include_groups' + ); + } + + public function testEnrichIncludeAllGroups(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + ['all'], + $doc->include_groups, + 'unexpected include_groups' + ); + } + + public function testEnrichExcludeGroups(): void + { + $doc = $this->enrichWithData([ + 'access' => [ + 'type' => 'deny', + 'groups' => ['100010100000001028'] + ] + ]); + + $this->assertEquals( + ['1028'], + $doc->exclude_groups, + 'unexpected exclude_groups' + ); + } + + public function testEnrichNonExcludeGroups(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + ['none'], + $doc->exclude_groups, + 'unexpected exclude_groups' + ); + } + + public function testEnrichMetaContentType(): void + { + $doc = $this->enrichWithData([ + 'base' => ['mime' => 'application/pdf'] + ]); + + $this->assertEquals( + 'application/pdf', + $doc->meta_content_type, + 'unexpected meta_content_type' + ); + } + + public function testEnrichInternal(): void + { + $doc = $this->enrichWithData([]); + + $this->assertEquals( + ['internal'], + $doc->sp_source, + 'unexpected sp_source' + ); + } + + public function testEnrichContent(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'categories' => [ + ['id' => 1, 'name' => 'CategoryA'], + ['id' => 2, 'name' => 'CategoryB'] + ] + ], + 'searchindexdata' => ['content' => 'abc'] + ]); + + $this->assertEquals( + 'abc collected content CategoryA CategoryB', + $doc->content, + 'unexpected content' + ); + } + + public function testEnrichContactPointContent(): void + { + $doc = $this->enrichWithData([ + 'metadata' => [ + 'contactPoint' => [ + 'contactData' => [ + 'phoneList' => [ + ['phone' => [ + 'countryCode' => '49', + 'areaCode' => '251', + 'localNumber' => '123' + ]], + ['phone' => [ + 'countryCode' => '49', + 'areaCode' => '2571', + 'localNumber' => '456' + ]] + ], + 'emailList' => [ + ['email' => 'test1@sitepark.com'], + ['email' => 'test2@sitepark.com'] + ] + ], + 'addressData' => [ + 'street' => 'Neubrückenstr', + 'buildingName' => 'Pressehaus', + 'postOfficeBoxData' => [ + 'buildingName' => 'Sitepark' + ] + ] + ], + ] + ]); + + $this->assertEquals( + 'collected content +49 251 0251 123 +49 2571 02571 456 ' . + 'test1@sitepark.com test2@sitepark.com ' . + 'Neubrückenstr Pressehaus Sitepark', + $doc->content, + 'unexpected content' + ); + } + + private function enrichWithResource( + Resource $resource + ): IndexSchema2xDocument { + /** @var IndexSchema2xDocument $doc */ + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + return $doc; + } + + /** + * @param array> $data + */ + private function enrichWithData( + array $data + ): IndexSchema2xDocument { + $resource = $this->createResource($data); + /** @var IndexSchema2xDocument $doc */ + $doc = $this->enricher->enrichDocument( + $resource, + new IndexSchema2xDocument(), + 'progress-id' + ); + return $doc; + } + + /** + * @param array> $data + */ + private function createResource(array $data): Resource + { + return new Resource( + $data['url'] ?? '', + $data['id'] ?? '123', + $data['name'] ?? '', + $data['objectType'] ?? '', + ResourceLanguage::of($data['locale'] ?? ''), + new DataBag($data) + ); + } +} diff --git a/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php new file mode 100644 index 0000000..1c23d99 --- /dev/null +++ b/test/Service/Indexer/SiteKit/HeadlineMatcherTest.php @@ -0,0 +1,58 @@ + "Überschrift" + ]; + + $content = $matcher->match(['items', 'model'], $value); + + $this->assertEquals('Überschrift', $content, 'unexpected headline'); + } + + public function testMatcherNotMachedPathToShort(): void + { + $matcher = new HeadlineMatcher(); + + $value = [ + "headline" => "Überschrift" + ]; + + $content = $matcher->match(['model'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNotMachedNoModel(): void + { + $matcher = new HeadlineMatcher(); + + $value = [ + "headline" => "Überschrift" + ]; + + $content = $matcher->match(['items', 'modelX'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/NoIndexFilterTest.php b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php new file mode 100644 index 0000000..5dc31a9 --- /dev/null +++ b/test/Service/Indexer/SiteKit/NoIndexFilterTest.php @@ -0,0 +1,50 @@ +assertTrue( + $filter->accept($resource), + "resource should be accepted" + ); + } + + public function testAcceptWithNoIndex(): void + { + $resource = new Resource( + 'test', + 'test', + 'test', + 'test', + ResourceLanguage::default(), + new DataBag(['noIndex' => true]) + ); + $filter = new NoIndexFilter(); + $this->assertFalse( + $filter->accept($resource), + "resource should not be accepted" + ); + } +} diff --git a/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php new file mode 100644 index 0000000..a3d50a3 --- /dev/null +++ b/test/Service/Indexer/SiteKit/QuoteSectionMatcherTest.php @@ -0,0 +1,113 @@ + "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEquals( + 'Quote-Text Citation', + $content, + 'unexpected quote text' + ); + } + public function testMatcherNoMatchPathToShort(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchNoItems(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['itemsX'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchInvalidType(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quoteX", + "model" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNoMatchMissingModel(): void + { + $matcher = new QuoteSectionMatcher(); + + $value = [ + "type" => "quote", + "modelX" => [ + "quote" => "Quote-Text", + "citation" => "Citation" + ] + ]; + + $content = $matcher->match(['items'], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php new file mode 100644 index 0000000..07a0443 --- /dev/null +++ b/test/Service/Indexer/SiteKit/RichtTextMatcherTest.php @@ -0,0 +1,64 @@ + true, + "modelType" => "html.richText", + "text" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEquals('Ein Text', $content, 'unexpected content'); + } + + public function testMatcherNotMatchedInvalidType(): void + { + $matcher = new RichtTextMatcher(); + + $value = [ + "normalized" => true, + "modelType" => "html.richTextX", + "text" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } + + public function testMatcherNotMatchedTextMissing(): void + { + $matcher = new RichtTextMatcher(); + + $value = [ + "normalized" => true, + "modelType" => "html.richText", + "textX" => "

Ein Text

" + ]; + + $content = $matcher->match([], $value); + + $this->assertEmpty( + $content, + 'should not find any content' + ); + } +} diff --git a/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php new file mode 100644 index 0000000..a50a3d6 --- /dev/null +++ b/test/Service/Indexer/SiteKit/SubDirTranslationSplitterTest.php @@ -0,0 +1,120 @@ +split(['/a/b.php']); + + $this->assertEquals( + ['/a/b.php'], + $result->getBases(), + 'unexpected bases' + ); + } + + public function testLanguages(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php', + '/a/b.php.translations/it_IT.php', + '/a/b.php.translations/en_US.php' + ]); + + $this->assertEquals( + [ + ResourceLanguage::of('en_US'), + ResourceLanguage::of('it_IT'), + ], + $result->getLanguages(), + 'unexpected locales' + ); + } + + public function testGetTranlsations(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php', + '/a/b.php.translations/it_IT.php', + '/a/b.php.translations/en_US.php', + '/c/d.php', + '/c/d.php.translations/it_IT.php', + '/c/d.php.translations/en_US.php' + ]); + + $lang = ResourceLanguage::of('it_IT'); + $translations = $result->getTranslations($lang); + + $expected = [ + ResourceLocation::of('/a/b.php', $lang), + ResourceLocation::of('/c/d.php', $lang), + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } + + public function testSplitWithLocParameter(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/a/b.php?loc=en_US', + ]); + + $lang = ResourceLanguage::of('en_US'); + $translations = $result->getTranslations($lang); + + $expected = [ + ResourceLocation::of('/a/b.php', $lang) + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } + + public function testSplitWithOutPath(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '?a=b', + ]); + + $this->assertEquals( + [], + $result->getBases(), + 'bases should be empty' + ); + } + + public function testSplitWithUnsupportedParameter(): void + { + $splitter = new SubDirTranslationSplitter(); + $result = $splitter->split([ + '/test.php?a=b', + ]); + + $this->assertEquals( + ['/test.php'], + $result->getBases(), + 'unexpected bases' + ); + } +} diff --git a/test/Service/Indexer/SolrIndexServiceTest.php b/test/Service/Indexer/SolrIndexServiceTest.php new file mode 100644 index 0000000..e4c2bf2 --- /dev/null +++ b/test/Service/Indexer/SolrIndexServiceTest.php @@ -0,0 +1,119 @@ +createStub(StatusResult::class); + $statusResult->method('getCoreName')->willReturn('test'); + $statusResultEn = $this->createStub(StatusResult::class); + $statusResultEn->method('getCoreName')->willReturn('test-en_US'); + $response = $this->createStub(CoreAdminResult::class); + $response->method('getStatusResults')->willReturn([ + $statusResult, + $statusResultEn + ]); + + $this->client = $this->createMock(Client::class); + $this->client->method('coreAdmin')->willReturn($response); + $this->indexName = $this->createMock(IndexName::class); + $this->indexName->method('name')->willReturn('test', 'test-en_US'); + $this->indexName->method('names')->willReturn(['test', 'test-en_US']); + $this->factory = $this->createMock(SolrClientFactory::class); + $this->factory->method('create')->willReturn($this->client); + $this->indexService = new SolrIndexService( + $this->indexName, + $this->factory + ); + } + public function testUpdater(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->updater(ResourceLanguage::default()); + } + + public function testGetIndex(): void + { + $index = $this->indexService->getIndex(ResourceLanguage::default()); + $this->assertEquals( + 'test', + $index, + 'Index name should be returned' + ); + } + + public function testDeleteExcludingProcessId(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->deleteExcludingProcessId( + ResourceLanguage::default(), + 'test', + 'test' + ); + } + + public function testDeleteByIdListForAllLanguages(): void + { + $this->client->expects($this->exactly(2))->method('createUpdate'); + $this->indexService->deleteByIdListForAllLanguages('test', ['test']); + } + + public function testByQuery(): void + { + $this->client->expects($this->once())->method('createUpdate'); + $this->indexService->deleteByQuery( + ResourceLanguage::default(), + 'test' + ); + } + + public function testCommit(): void + { + $this->client->expects($this->once())->method('update'); + $this->indexService->commit(ResourceLanguage::default()); + } + + public function testCommitForAllLanguages(): void + { + $this->client->expects($this->exactly(2))->method('update'); + $this->indexService->commitForAllLanguages(); + } + + public function testGetManagedIndexes(): void + { + $statusResult = $this->createStub(StatusResult::class); + $statusResult->method('getCoreName')->willReturn('test'); + $response = $this->createStub(CoreAdminResult::class); + $response->method('getStatusResults')->willReturn([$statusResult]); + $this->client->method('coreAdmin')->willReturn($response); + + $indices = $this->indexService->getManagedIndices(); + + $this->assertEquals( + ['test', 'test-en_US'], + $indices, + 'Indices should be returned' + ); + } +} diff --git a/test/Service/Indexer/SolrIndexUpdaterTest.php b/test/Service/Indexer/SolrIndexUpdaterTest.php new file mode 100644 index 0000000..66c6db5 --- /dev/null +++ b/test/Service/Indexer/SolrIndexUpdaterTest.php @@ -0,0 +1,52 @@ +updateQuery = $this->createMock(UpdateQuery::class); + $this->updateQuery->method('createDocument')->willReturn( + new IndexSchema2xDocument() + ); + $this->client = $this->createMock(Client::class); + $this->updater = new SolrIndexUpdater( + $this->client, + $this->updateQuery + ); + } + + public function testCreateDocument(): void + { + $this->updateQuery->expects($this->once())->method('createDocument'); + $this->updater->createDocument(); + } + + public function testAddAndUpdate(): void + { + $doc = $this->updater->createDocument(); + $this->updater->addDocument($doc); + $this->updateQuery->expects($this->once()) + ->method('addDocuments') + ->with([$doc]); + $this->updater->update(); + } +} diff --git a/test/Service/Indexer/TranslationSplitterResultTest.php b/test/Service/Indexer/TranslationSplitterResultTest.php new file mode 100644 index 0000000..ffe279c --- /dev/null +++ b/test/Service/Indexer/TranslationSplitterResultTest.php @@ -0,0 +1,93 @@ +assertEquals( + [ + ResourceLocation::of('/a/b.php') + ], + $result->getBases(), + 'unexpected bases' + ); + } + + public function testLanguages(): void + { + $it = ResourceLanguage::of('it_IT'); + $en = ResourceLanguage::of('en_US'); + + $result = new TranslationSplitterResult( + [], + [ + $it->code => [ResourceLocation::of('/a/b.php', $it)], + $en->code => [ResourceLocation::of('/a/b.php', $en)], + ] + ); + + $this->assertEquals( + [$en, $it], + $result->getLanguages(), + 'unexpected languages' + ); + } + + public function testGetTranslations(): void + { + $it = ResourceLanguage::of('it_IT'); + $en = ResourceLanguage::of('en_US'); + $result = new TranslationSplitterResult( + [], + [ + $it->code => [ + ResourceLocation::of('/a/b.php', $it), + ResourceLocation::of('/c/d.php', $it), + ], + $en->code => [ + ResourceLocation::of('/a/b.php', $en), + ResourceLocation::of('/a/b.php', $en), + ] + ] + ); + + $translations = $result->getTranslations($it); + + $expected = [ + ResourceLocation::of('/a/b.php', $it), + ResourceLocation::of('/c/d.php', $it), + ]; + + $this->assertEquals( + $expected, + $translations, + 'unexpected translations' + ); + } + public function testGetMissingTranslations(): void + { + $result = new TranslationSplitterResult([], []); + $lang = ResourceLanguage::of('en_US'); + + $this->assertEquals( + [], + $result->getTranslations($lang), + 'empty array expected' + ); + } +} diff --git a/test/Service/ResourceChannelBasedIndexNameTest.php b/test/Service/ResourceChannelBasedIndexNameTest.php new file mode 100644 index 0000000..9f5b1dd --- /dev/null +++ b/test/Service/ResourceChannelBasedIndexNameTest.php @@ -0,0 +1,74 @@ +indexName = new ResourceChannelBasedIndexName( + $resourceChannel + ); + } + + public function testName(): void + { + $this->assertEquals( + 'test', + $this->indexName->name(ResourceLanguage::default()), + 'The default index name should be returned ' . + 'if no language is given' + ); + } + + public function testNameWithLang(): void + { + $this->assertEquals( + 'test-en_US', + $this->indexName->name(ResourceLanguage::of('en')), + 'The language-specific index name should be returned ' . + 'if a language is given' + ); + } + + public function testNameWithUnsupportedLang(): void + { + $this->expectException(UnsupportedIndexLanguageException::class); + $this->indexName->name(ResourceLanguage::of('it')); + } + + public function testNames(): void + { + $this->assertEquals( + ['test', 'test-en_US'], + $this->indexName->names(), + 'All index names should be returned' + ); + } +} diff --git a/test/Service/Search/ExternalResourceFactoryTest.php b/test/Service/Search/ExternalResourceFactoryTest.php new file mode 100644 index 0000000..8d2e218 --- /dev/null +++ b/test/Service/Search/ExternalResourceFactoryTest.php @@ -0,0 +1,99 @@ +factory = new ExternalResourceFactory(); + } + + public function testAcceptHttps(): void + { + $document = $this->createDocument('https://www.sitepark.com'); + $this->assertTrue( + $this->factory->accept($document, ResourceLanguage::default()), + 'should be accepted' + ); + } + + public function testAcceptHttp(): void + { + $document = $this->createDocument('http://www.sitepark.com'); + $this->assertTrue( + $this->factory->accept($document, ResourceLanguage::default()), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document, ResourceLanguage::default()), + 'should be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('https://www.sitepark.com'); + $resource = $this->factory->create( + $document, + ResourceLanguage::of('en') + ); + + $this->assertEquals( + 'https://www.sitepark.com', + $resource->location, + 'unexpected location' + ); + } + + public function testCreateWithName(): void + { + $document = $this->createDocument('https://www.sitepark.com', 'Test'); + $resource = $this->factory->create( + $document, + ResourceLanguage::of('en') + ); + + $this->assertEquals( + 'Test', + $resource->name, + 'unexpected name' + ); + } + + public function testCreateWithMissingUrl(): void + { + $document = $this->createStub(Document::class); + + $this->expectException(\LogicException::class); + $this->factory->create($document, ResourceLanguage::of('en')); + } + + private function createDocument(string $url, string $title = ''): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url, + 'title' => $title + ]); + return $document; + } +} diff --git a/test/Service/Search/InternalMediaResourceFactoryTest.php b/test/Service/Search/InternalMediaResourceFactoryTest.php new file mode 100644 index 0000000..0528021 --- /dev/null +++ b/test/Service/Search/InternalMediaResourceFactoryTest.php @@ -0,0 +1,100 @@ +resourceLoader = $this->createStub( + ResourceLoader::class + ); + $this->factory = new InternalMediaResourceFactory( + $this->resourceLoader + ); + } + + public function testAccept(): void + { + $document = $this->createDocument('/image.jpg'); + $this->resourceLoader + ->method('exists') + ->willReturn(true); + + $this->assertTrue( + $this->factory->accept($document, ResourceLanguage::default()), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document, ResourceLanguage::default()), + 'should not be accepted' + ); + } + + public function testAcceptNotExists(): void + { + $document = $this->createDocument('/image.jpg'); + $this->resourceLoader + ->method('exists') + ->willReturn(false); + + $this->assertFalse( + $this->factory->accept($document, ResourceLanguage::default()), + 'should not be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('/image.jpg'); + $resource = $this->createStub(Resource::class); + $this->resourceLoader + ->method('load') + ->willReturn($resource); + + $this->assertEquals( + $resource, + $this->factory->create($document, ResourceLanguage::of('en')), + 'unexpected resource' + ); + } + + public function testCreateWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->expectException(LogicException::class); + $this->factory->create($document, ResourceLanguage::of('de')); + } + + private function createDocument(string $url): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url + ]); + return $document; + } +} diff --git a/test/Service/Search/InternalResourceFactoryTest.php b/test/Service/Search/InternalResourceFactoryTest.php new file mode 100644 index 0000000..f78a8e7 --- /dev/null +++ b/test/Service/Search/InternalResourceFactoryTest.php @@ -0,0 +1,92 @@ +resourceLoader = $this->createStub( + ResourceLoader::class + ); + $this->factory = new InternalResourceFactory( + $this->resourceLoader + ); + } + + public function testAccept(): void + { + $document = $this->createDocument('/test.php'); + $this->assertTrue( + $this->factory->accept($document, ResourceLanguage::default()), + 'should be accepted' + ); + } + + public function testAcceptWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->assertFalse( + $this->factory->accept($document, ResourceLanguage::default()), + 'should not be accepted' + ); + } + + public function testAcceptWithWrongUrl(): void + { + $document = $this->createDocument('/test.txt'); + $this->assertFalse( + $this->factory->accept($document, ResourceLanguage::default()), + 'should not be accepted' + ); + } + + public function testCreate(): void + { + $document = $this->createDocument('/test.php'); + $resource = $this->createStub(Resource::class); + $this->resourceLoader + ->method('load') + ->willReturn($resource); + + $this->assertEquals( + $resource, + $this->factory->create($document, ResourceLanguage::of('de')), + 'unexpected resource' + ); + } + + public function testCreateWithoutUrl(): void + { + $document = $this->createStub(Document::class); + $this->expectException(LogicException::class); + $this->factory->create($document, ResourceLanguage::of('de')); + } + + private function createDocument(string $url): Document + { + $document = $this->createStub(Document::class); + $document + ->method('getFields') + ->willReturn([ + 'url' => $url + ]); + return $document; + } +} diff --git a/test/Service/Search/SiteKit/DefaultBoostModifierTest.php b/test/Service/Search/SiteKit/DefaultBoostModifierTest.php new file mode 100644 index 0000000..0a08579 --- /dev/null +++ b/test/Service/Search/SiteKit/DefaultBoostModifierTest.php @@ -0,0 +1,22 @@ +modify($query); + + $this->assertNotNull($modifiedQuery->getDisMax()->getBoostQueries()); + } +} diff --git a/test/Service/Search/SolrMoreLikeThisTest.php b/test/Service/Search/SolrMoreLikeThisTest.php new file mode 100644 index 0000000..4195f9d --- /dev/null +++ b/test/Service/Search/SolrMoreLikeThisTest.php @@ -0,0 +1,84 @@ +createStub(IndexName::class); + $clientFactory = $this->createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrMoreLikeThisQuery::class); + $filterQuery = new FilterQuery(); + $query->method('createFilterQuery')->willReturn($filterQuery); + + $client->method('createMoreLikeThis')->willReturn($query); + + $result = $this->createStub(SolrMoreLikeThisResult::class); + $client->method('execute')->willReturn($result); + + $this->resource = $this->createStub(Resource::class); + + $resultToResourceResolver = $this->createStub( + SolrResultToResourceResolver::class + ); + $resultToResourceResolver + ->method('loadResourceList') + ->willReturn([$this->resource]); + + $this->searcher = new SolrMoreLikeThis( + $indexName, + $clientFactory, + $resultToResourceResolver + ); + } + + public function testMoreLikeThis(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new MoreLikeThisQuery( + ResourceLocation::of('/test.php'), + [$filter] + ); + + $searchResult = $this->searcher->moreLikeThis($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } +} diff --git a/test/Service/Search/SolrResultToResourceResolverTest.php b/test/Service/Search/SolrResultToResourceResolverTest.php new file mode 100644 index 0000000..93cf6c6 --- /dev/null +++ b/test/Service/Search/SolrResultToResourceResolverTest.php @@ -0,0 +1,96 @@ +createStub(Document::class); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resource = $this->createStub(Resource::class); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(true); + $resourceFactory->method('create')->willReturn($resource); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); + + $this->assertEquals( + [$resource], + $resourceList, + 'unexpected resourceList' + ); + } + + public function testLoadResourceListWithoutAcceptedFactory(): void + { + $document = $this->createStub(Document::class); + $document->method('getFields')->willReturn(['url' => 'test']); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(false); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); + + $this->assertEmpty( + $resourceList, + 'resourceList should be empty' + ); + } + + public function testLoadResourceListWithoutAcceptedFactoryNoUrl(): void + { + $document = $this->createStub(Document::class); + $result = $this->createStub(SelectResult::class); + $result->method('getIterator')->willReturn( + new ArrayIterator([$document]) + ); + + $resourceFactory = $this->createStub(ResourceFactory::class); + $resourceFactory->method('accept')->willReturn(false); + + $resolver = new SolrResultToResourceResolver([$resourceFactory]); + + $resourceList = $resolver->loadResourceList( + $result, + ResourceLanguage::default() + ); + + $this->assertEmpty( + $resourceList, + 'resourceList should be empty' + ); + } +} diff --git a/test/Service/Search/SolrSearchTest.php b/test/Service/Search/SolrSearchTest.php new file mode 100644 index 0000000..94d6ac1 --- /dev/null +++ b/test/Service/Search/SolrSearchTest.php @@ -0,0 +1,365 @@ +createStub(IndexName::class); + $clientFactory = $this->createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrSelectQuery::class); + + $query->method('createFilterQuery') + ->willReturn(new FilterQuery()); + $query->method('getFacetSet') + ->willReturn(new FacetSet()); + + $client->method('createSelect')->willReturn($query); + + $this->result = $this->createStub(SelectResult::class); + $client->method('execute')->willReturn($this->result); + + $this->resource = $this->createStub(Resource::class); + + $resultToResourceResolver = $this->createStub( + SolrResultToResourceResolver::class + ); + + $solrQueryModifier = $this->createStub(SolrQueryModifier::class); + $solrQueryModifier->method('modify')->willReturn($query); + + $resultToResourceResolver + ->method('loadResourceList') + ->willReturn([$this->resource]); + + $this->searcher = new SolrSearch( + $indexName, + $clientFactory, + $resultToResourceResolver, + [$solrQueryModifier], + ); + } + + public function testSelectEmpty(): void + { + $query = new SearchQuery( + '', + '', + 0, + 1, + [ + ], + [], + [], + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithText(): void + { + $query = new SearchQuery( + 'cat dog', + '', + 0, + 10, + [ + ], + [], + [], + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithSort(): void + { + $query = new SearchQuery( + '', + '', + 0, + 10, + [ + new Name(), + new Headline(), + new Date(), + new Natural(), + new Score(), + ], + [], + [], + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithInvalidSort(): void + { + $sort = $this->createStub(Criteria::class); + + $query = new SearchQuery( + '', + '', + 0, + 10, + [$sort], + [], + [], + QueryOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->search($query); + } + + public function testSelectWithAndDefaultOperator(): void + { + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [], + [], + QueryOperator::AND + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithFilter(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [$filter], + [], + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithFacets(): void + { + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [], + $facets, + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + [$this->resource], + $searchResult->results, + 'unexpected results' + ); + } + + public function testSelectWithInvalidFacets(): void + { + + $facets = [ + $this->createStub(Facet::class) + ]; + + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [], + $facets, + QueryOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->search($query); + } + + public function testResultFacets(): void + { + + $facet = new Field([ + 'content' => 10, + 'media' => 5 + ]); + $facetSet = new \Solarium\Component\Result\FacetSet([ + 'objectType' => $facet + ]); + + $this->result->method('getFacetSet') + ->willReturn($facetSet); + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [], + $facets, + QueryOperator::OR + ); + + $searchResult = $this->searcher->search($query); + + $this->assertEquals( + 'objectType', + $searchResult->facetGroups[0]->key, + 'unexpected results' + ); + } + + public function testInvalidResultFacets(): void + { + + $facet = new Field([ + 'content' => 'nonint', + ]); + $facetSet = new \Solarium\Component\Result\FacetSet([ + 'objectType' => $facet + ]); + + $this->result->method('getFacetSet') + ->willReturn($facetSet); + + $facets = [ + new ObjectTypeFacet('objectType', ['content'], 'ob'), + new FacetQuery('query', 'sp_id:123', 'ob'), + new FacetMultiQuery( + 'multiquery', + [new FacetQuery('query', 'sp_id:123', null)], + 'ob' + ) + ]; + + $query = new SearchQuery( + '', + '', + 0, + 10, + [], + [], + $facets, + QueryOperator::OR + ); + + $this->expectException(InvalidArgumentException::class); + $this->searcher->search($query); + } +} diff --git a/test/Service/Search/SolrSuggestTest.php b/test/Service/Search/SolrSuggestTest.php new file mode 100644 index 0000000..734420d --- /dev/null +++ b/test/Service/Search/SolrSuggestTest.php @@ -0,0 +1,133 @@ +createStub(IndexName::class); + $clientFactory = $this->createStub( + SolrClientFactory::class + ); + $client = $this->createStub(Client::class); + $clientFactory->method('create')->willReturn($client); + + $query = $this->createStub(SolrSelectQuery::class); + + $query->method('createFilterQuery') + ->willReturn(new FilterQuery()); + + $client->method('createSelect')->willReturn($query); + + $this->result = $this->createStub(SelectResult::class); + $client->method('select')->willReturn($this->result); + + $this->searcher = new SolrSuggest($indexName, $clientFactory); + } + + public function testSuggest(): void + { + $filter = $this->getMockBuilder(Filter::class) + ->setConstructorArgs(['test', []]) + ->getMock(); + + $query = new SuggestQuery( + 'myindex', + 'cat', + [$filter] + ); + + $response = new Response(<<result->method('getResponse')->willReturn($response); + + $suggestResult = $this->searcher->suggest($query); + + $expected = [ + new Suggestion('category', 10), + new Suggestion('catalog', 5), + ]; + + $this->assertEquals( + $expected, + $suggestResult->suggestions, + 'unexpected suggestion' + ); + } + + public function testEmptySuggest(): void + { + $query = new SuggestQuery( + 'myindex', + 'cat', + ); + + $response = new Response(<<result->method('getResponse')->willReturn($response); + + $suggestResult = $this->searcher->suggest($query); + + $this->assertEmpty( + $suggestResult->suggestions, + 'suggestion should be empty' + ); + } + + public function testInvalidSuggestResponse(): void + { + $query = new SuggestQuery( + 'myindex', + 'cat', + ); + + $response = new Response("none json"); + + $this->result->method('getResponse')->willReturn($response); + + $this->expectException(UnexpectedResultException::class); + $this->searcher->suggest($query); + } +} diff --git a/test/Service/SolrParameterClientFactoryTest.php b/test/Service/SolrParameterClientFactoryTest.php new file mode 100644 index 0000000..6f0fc2c --- /dev/null +++ b/test/Service/SolrParameterClientFactoryTest.php @@ -0,0 +1,24 @@ +create('myindex'); + $this->assertNotNull($client, 'client instance expected'); + } +} diff --git a/test/Service/TextIndexer.php b/test/Service/TextIndexer.php new file mode 100644 index 0000000..7612695 --- /dev/null +++ b/test/Service/TextIndexer.php @@ -0,0 +1,34 @@ +progressHandler->getStatus(); + } + + /** + * @inheritDoc + */ + public function remove(array $idList): void + { + } + + /** + * make this method public for testing + */ + public function isAbortionRequested(): bool + { + return parent::isAbortionRequested(); + } +} diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer b/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/not-a-directory/indexer @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php b/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php new file mode 100644 index 0000000..05d0e7d --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/return-string/indexer/return-string.php @@ -0,0 +1,3 @@ + 'Internal Indexer', + 'data' => [ + 'cleanupThreshold' => 1000, + 'chunkSize' => 500, + ], +]; diff --git a/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php b/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php new file mode 100644 index 0000000..196f10a --- /dev/null +++ b/test/resources/Service/Indexer/IndexerConfigurationLoader/without-source/indexer/without-source.php @@ -0,0 +1,5 @@ + 'Without Source' +]; diff --git a/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json new file mode 100644 index 0000000..212e4b2 --- /dev/null +++ b/test/resources/Service/Indexer/IndexerStatusStore/atoolo.search.index.test.status.json @@ -0,0 +1,11 @@ +{ + "state": "FINISHED", + "startTime": "2024-01-31T11:15:10+00:00", + "endTime": "2024-01-31T12:16:11+00:00", + "total": 10, + "processed": 5, + "skipped": 4, + "lastUpdate": "2024-01-31T13:17:12+00:00", + "updated": 6, + "errors": 2 +} diff --git a/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php b/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php new file mode 100644 index 0000000..174d7fd --- /dev/null +++ b/test/resources/Service/Indexer/LocationFinder/WEB-IES/d.php @@ -0,0 +1,3 @@ +