feat: add FULL TEXT SEARCH support (#235)

CHANGELOG.md

@@ -1,5 +1,6 @@
 # v8.3.0 (2024-11-08)
 
+- add support for full text search (#235)
 - add support for IDENTITY columns (#243)
 - consolidate schema options formatting (#241)
 - add support for invisible columns (#240)

README.md

@@ -349,6 +349,31 @@ $schemaBuilder->create('user_items', function (Blueprint $table) {
 });
 ```
 
+### Full Text Search
+
+Spanner supports [Full Text Search](https://cloud.google.com/spanner/docs/full-text-search) which allows you to search for text in columns.
+
+You can define a token list column and a search index for the column as below.
+
+```php
+$schemaBuilder->create('user', function (Blueprint $table) {
+    $table->uuid('id')->primary();
+    $table->string('name');
+    // adds an invisible column for full text search
+    $table->tokenList('UserNameTokens', TokenizerFunction::FullText, 'name', ['language_tag' => 'en']);
+
+    // adds a SEARCH INDEX
+    $table->fullText(['UserNameTokens']);
+});
+```
+
+Once the schema has been applied, you can use the search methods in the query builder to search for text in the columns as below.
+
+```php
+User::query()->searchFullText('UserNameTokens', 'John OR Kevin', ['enhance_query' => true])->get();
+```
+
+The methods available are `searchFullText`, `searchSubstring`, and `searchNgrams`.
 
 ### Secondary Index Options
 

src/Query/Builder.php

@@ -29,6 +29,7 @@ class Builder extends BaseBuilder
 {
     use Concerns\SetsRequestTimeouts;
     use Concerns\UsesDataBoost;
+    use Concerns\UsesFullTextSearch;
     use Concerns\UsesMutations;
     use Concerns\UsesPartitionedDml;
     use Concerns\UsesStaleReads;

src/Query/Concerns/UsesFullTextSearch.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * Copyright 2019 Colopl Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Colopl\Spanner\Query\Concerns;
+
+trait UsesFullTextSearch
+{
+    /**
+     * @param string $tokens
+     * @param string $query
+     * @param array<string, scalar> $options
+     * @param string $boolean
+     * @return $this
+     */
+    public function searchFullText(
+        string $tokens,
+        string $query,
+        array $options = [],
+        string $boolean = 'and',
+    ): static
+    {
+        $this->addSearchCondition('SearchFullText', $tokens, $query, $options, $boolean);
+        return $this;
+    }
+
+    /**
+     * @param string $tokens
+     * @param string $query
+     * @param array<string, scalar> $options
+     * @param string $boolean
+     * @return $this
+     */
+    public function searchNgrams(
+        string $tokens,
+        string $query,
+        array $options = [],
+        string $boolean = 'and',
+    ): static
+    {
+        $this->addSearchCondition('SearchNgrams', $tokens, $query, $options, $boolean);
+        return $this;
+    }
+
+    /**
+     * @param string $tokens
+     * @param string $query
+     * @param array<string, scalar> $options
+     * @param string $boolean
+     * @return $this
+     */
+    public function searchSubstring(
+        string $tokens,
+        string $query,
+        array $options = [],
+        string $boolean = 'and',
+    ): static
+    {
+        $this->addSearchCondition('SearchSubstring', $tokens, $query, $options, $boolean);
+        return $this;
+    }
+
+    /**
+     * @param string $type
+     * @param string $tokens
+     * @param string $query
+     * @param array<string, scalar> $options
+     * @param string $boolean
+     * @return void
+     */
+    protected function addSearchCondition(
+        string $type,
+        string $tokens,
+        string $query,
+        array $options = [],
+        string $boolean = 'and',
+    ): void
+    {
+        $this->wheres[] = [
+            'type' => $type,
+            'tokens' => $tokens,
+            'boolean' => $boolean,
+            'query' => $query,
+            'options' => $options,
+        ];
+        $this->addBinding($query);
+    }
+}

src/Query/Grammar.php

@@ -141,6 +141,53 @@ protected function whereInUnnest(Builder $query, $where)
             . ' in unnest(?)';
     }
 
+    /**
+     * @param Builder $query
+     * @param array{ tokens: string, query: string, options: array<string, scalar> } $where
+     * @return string
+     */
+    protected function whereSearchFullText(Builder $query, array $where): string
+    {
+        return $this->buildSearchFunction('search', $where);
+    }
+
+    /**
+     * @param Builder $query
+     * @param array{ tokens: string, query: string, options: array<string, scalar> } $where
+     * @return string
+     */
+    protected function whereSearchNgrams(Builder $query, array $where): string
+    {
+        return $this->buildSearchFunction('search_ngrams', $where);
+    }
+
+    /**
+     * @param Builder $query
+     * @param array{ tokens: string, query: string, options: array<string, scalar> } $where
+     * @return string
+     */
+    protected function whereSearchSubstring(Builder $query, array $where): string
+    {
+        return $this->buildSearchFunction('search_substring', $where);
+    }
+
+    /**
+     * @param string $function
+     * @param array{ tokens: string, query: string, options: array<string, scalar> } $where
+     * @return string
+     */
+    protected function buildSearchFunction(string $function, array $where): string
+    {
+        $tokens = $this->wrap($where['tokens']);
+        $rawQuery = $where['query'];
+        $options = $where['options'];
+        return $function . '(' . implode(', ', array_filter([
+            $tokens,
+            $this->quoteString($rawQuery),
+            $this->formatOptions($options, ' => '),
+        ])) . ')';
+    }
+
     /**
      * @inheritDoc
      */

src/Schema/Blueprint.php

@@ -244,6 +244,22 @@ public function timestampArray($column)
         ]);
     }
 
+    /**
+     * @param string $column
+     * @param TokenizerFunction $function
+     * @param string $target
+     * @param array $options
+     * @return ColumnDefinition
+     */
+    public function tokenList(string $column, TokenizerFunction $function, string $target, array $options = []): ColumnDefinition
+    {
+        return $this->addColumn('tokenList', $column, [
+            'function' => $function,
+            'target' => $target,
+            'options' => $options,
+        ])->invisible()->nullable();
+    }
+
     /**
      * @deprecated use interleaveInParent instead.
      * @param string $parentTableName
@@ -269,6 +285,27 @@ public function interleaveInParent(string $table): InterleaveDefinition
         return $command;
     }
 
+    /**
+     * @param string|list<string> $columns
+     * @param string|null $name
+     * @param string|null $algorithm
+     * @return SearchIndexDefinition
+     */
+    public function fullText($columns, $name = null, $algorithm = null)
+    {
+        $type = 'fullText';
+        $columns = (array) $columns;
+
+        $this->commands[] = $command = new SearchIndexDefinition([
+            'name' => $type,
+            'index' => $name ?? $this->createIndexName($type, $columns),
+            'columns' => $columns,
+            'algorithm' => $algorithm,
+        ]);
+
+        return $command;
+    }
+
     /**
      * @see https://cloud.google.com/spanner/docs/ttl#defining_a_row_deletion_policy
      * @param string $column

src/Schema/Grammar.php

@@ -24,7 +24,6 @@
 use Illuminate\Database\Query\Expression;
 use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Database\Schema\Grammars\Grammar as BaseGrammar;
-use Illuminate\Support\Arr;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Fluent;
 use Illuminate\Support\Str;
@@ -166,6 +165,60 @@ public function compileDropColumn(Blueprint $blueprint, Fluent $command)
         );
     }
 
+    /**
+     * Compile a fulltext index key command.
+     *
+     * @param Blueprint $blueprint
+     * @param SearchIndexDefinition $command
+     * @return string
+     */
+    public function compileFullText(Blueprint $blueprint, Fluent $command): string
+    {
+        $schema = sprintf('create search index %s on %s(%s)',
+            $this->wrap($command->index),
+            $this->wrapTable($blueprint),
+            $this->columnize($command->columns),
+        );
+
+        $schema .= $this->addStoringToIndex($command);
+
+        $partitionBy = (array) $command->partitionBy;
+        if (count($partitionBy) > 0) {
+            $schema .= ' partition by ' . $this->columnize($partitionBy);
+        }
+
+        if (isset($command->orderBy)) {
+            $schema .= ' order by ';
+            foreach ($command->orderBy as $column => $order) {
+                $schema .= is_string($column)
+                    ? $this->wrap($column) . ' ' . $order
+                    : $this->wrap($order);
+            }
+        }
+
+        $schema .= $this->addInterleaveToIndex($command);
+
+        $schema .= $command->getOptions() !== []
+            ? ' options (' . $this->formatOptions($command->getOptions()) . ')'
+            : '';
+
+        return $schema;
+    }
+
+    /**
+     * Compile a drop fulltext index command.
+     *
+     * @param Blueprint $blueprint
+     * @param Fluent<string, mixed>&object{ index: string } $command
+     * @return string
+     */
+    public function compileDropFullText(Blueprint $blueprint, Fluent $command): string
+    {
+        return sprintf('drop search index %s',
+            $this->wrap($command->index),
+        );
+    }
+
     /**
      * @param Blueprint $blueprint
      * @param RowDeletionPolicyDefinition $command
@@ -719,6 +772,18 @@ protected function typeBoolean(Fluent $column)
         return 'bool';
     }
 
+    /**
+     * @param Fluent<string, mixed>&object{ function: TokenizerFunction, target: string, options: array<string, scalar> } $column
+     * @return string
+     */
+    protected function typeTokenList(Fluent $column): string
+    {
+        return 'tokenlist as (' . $column->function->value . '(' . implode(', ', array_filter([
+            $this->wrap($column->target),
+            $this->formatOptions($column->options, ' => '),
+        ])) . '))';
+    }
+
     /**
      * Get the SQL for an invisible column modifier.
      *

src/Schema/SearchIndexDefinition.php

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Copyright 2019 Colopl Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Colopl\Spanner\Schema;
+
+/**
+ * @property string $name
+ * @property string $index
+ * @property list<string> $columns
+ * @property string|list<string> $partitionBy
+ * @property list<string>|array<string, string> $orderBy
+ * @property bool|null $sortOrderSharding
+ * @property bool|null $disableAutomaticUidColumn
+ * @method $this partitionBy(string|string[] $columns)
+ * @method $this orderBy(string|string[] $columns)
+ * @method $this sortOrderSharding(bool $toggle = true)
+ * @method $this disableAutomaticUidColumn(bool $toggle = true)
+ */
+class SearchIndexDefinition extends IndexDefinition
+{
+    /**
+     * @return array{ sortOrderSharding?: bool, disableAutomaticUidColumn?: bool }
+     */
+    public function getOptions(): array
+    {
+        return array_filter([
+            'sortOrderSharding' => $this->sortOrderSharding,
+            'disableAutomaticUidColumn' => $this->disableAutomaticUidColumn,
+        ], static fn($v) => $v !== null);
+    }
+}