Refactor Redis Storage and add JSON support (#78)

PR introduces a new `BaseStorage` class, subclassed by `HashStorage` and
`JsonStorage`, to handle the underlying data structures (as well as I/O)
for Redis. Includes:
- New storage classes and type enums
- New `key_separator` param for better use-customizability when
constructing redis keys
- Updated documentation and new user guide
- Updated docstrings and examples

---------

Co-authored-by: Sam Partee <sam.partee@redis.com>

docs/_extension/gallery_directive.py

@@ -1,12 +1,12 @@
 """A directive to generate a gallery of images from structured data.
 
-Generating a gallery of images that are all the same size is a common
-pattern in documentation, and this can be cumbersome if the gallery is
-generated programmatically. This directive wraps this particular use-case
-in a helper-directive to generate it with a single YAML configuration file.
+Generating a gallery of images that are all the same size is a common pattern in
+documentation, and this can be cumbersome if the gallery is generated
+programmatically. This directive wraps this particular use-case in a helper-
+directive to generate it with a single YAML configuration file.
 
-It currently exists for maintainers of the pydata-sphinx-theme,
-but might be abstracted into a standalone package if it proves useful.
+It currently exists for maintainers of the pydata-sphinx-theme, but might be
+abstracted into a standalone package if it proves useful.
 """
 from pathlib import Path
 from typing import Any, Dict, List

docs/api/searchindex.rst

@@ -13,18 +13,22 @@ SearchIndex
 .. autosummary::
 
     SearchIndex.__init__
+    SearchIndex.client
+    SearchIndex.name
+    SearchIndex.prefix
+    SearchIndex.key_separator
+    SearchIndex.storage_type
     SearchIndex.from_yaml
     SearchIndex.from_dict
     SearchIndex.from_existing
+    SearchIndex.connect
+    SearchIndex.create
+    SearchIndex.load
     SearchIndex.search
     SearchIndex.query
-    SearchIndex.create
     SearchIndex.delete
-    SearchIndex.load
-    SearchIndex.client
-    SearchIndex.connect
-    SearchIndex.disconnect
     SearchIndex.info
+    SearchIndex.disconnect
 
 
 
@@ -44,17 +48,22 @@ AsyncSearchIndex
 .. autosummary::
 
     AsyncSearchIndex.__init__
+    AsyncSearchIndex.client
+    AsyncSearchIndex.name
+    AsyncSearchIndex.prefix
+    AsyncSearchIndex.key_separator
+    AsyncSearchIndex.storage_type
     AsyncSearchIndex.from_yaml
     AsyncSearchIndex.from_dict
     AsyncSearchIndex.from_existing
+    AsyncSearchIndex.connect
+    AsyncSearchIndex.create
+    AsyncSearchIndex.load
     AsyncSearchIndex.search
     AsyncSearchIndex.query
-    AsyncSearchIndex.create
     AsyncSearchIndex.delete
-    AsyncSearchIndex.load
-    AsyncSearchIndex.connect
-    AsyncSearchIndex.disconnect
     AsyncSearchIndex.info
+    AsyncSearchIndex.disconnect
 
 
 

docs/examples/openai_qna.ipynb

@@ -46,7 +46,7 @@
    "source": [
     "# first we need to install a few things\n",
     "\n",
-    "!pip install pandas wget tenacity tiktoken openai"
+    "!pip install pandas wget tenacity tiktoken openai==0.28.1"
    ]
   },
   {

docs/user_guide/getting_started_01.ipynb

@@ -127,6 +127,8 @@
     "index:\n",
     "  name: user_index\n",
     "  prefix: user\n",
+    "  storage_type: hash\n",
+    "  key_separator: ':'\n",
     "\n",
     "fields:\n",
     "    # define tag fields\n",
@@ -162,6 +164,8 @@
     "    \"index\": {\n",
     "        \"name\": \"user_index\",\n",
     "        \"prefix\": \"user\",\n",
+    "        \"storage_type\": \"hash\",\n",
+    "        \"key_separator\": \":\"\n",
     "    },\n",
     "    \"fields\": {\n",
     "        \"tag\": [{\"name\": \"credit_score\"}],\n",
@@ -217,8 +221,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m16:03:01\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m   Indices:\n",
-      "\u001b[32m16:03:01\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m   1. user_index\n"
+      "\u001b[32m22:49:46\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m   Indices:\n",
+      "\u001b[32m22:49:46\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m   1. user_index\n"
      ]
     }
    ],
@@ -465,7 +469,7 @@
    ],
    "source": [
     "# create a new SearchIndex instance from an existing index\n",
-    "existing_index = SearchIndex.from_existing(\"user_index\", \"redis://localhost:6379\")\n",
+    "existing_index = SearchIndex.from_existing(name=\"user_index\", redis_url=\"redis://localhost:6379\")\n",
     "\n",
     "# run the same query\n",
     "results = existing_index.query(query)\n",
@@ -583,7 +587,10 @@
     {
      "data": {
       "text/plain": [
-       "{'index': {'name': 'user_index', 'prefix': 'user'},\n",
+       "{'index': {'name': 'user_index',\n",
+       "  'prefix': 'user',\n",
+       "  'storage_type': 'hash',\n",
+       "  'key_separator': ':'},\n",
        " 'fields': {'tag': [{'name': 'credit_score'}],\n",
        "  'text': [{'name': 'job'}],\n",
        "  'numeric': [{'name': 'age'}],\n",
@@ -612,7 +619,10 @@
     {
      "data": {
       "text/plain": [
-       "{'index': {'name': 'user_index', 'prefix': 'user'},\n",
+       "{'index': {'name': 'user_index',\n",
+       "  'prefix': 'user',\n",
+       "  'storage_type': 'hash',\n",
+       "  'key_separator': ':'},\n",
        " 'fields': {'tag': [{'name': 'credit_score'}, {'name': 'job'}],\n",
        "  'text': [],\n",
        "  'numeric': [{'name': 'age'}],\n",
@@ -725,7 +735,7 @@
       "│ offsets_per_term_avg        │ 0           │\n",
       "│ records_per_doc_avg         │ 4           │\n",
       "│ sortable_values_size_mb     │ 0           │\n",
-      "│ total_indexing_time         │ 0.59        │\n",
+      "│ total_indexing_time         │ 1.738       │\n",
       "│ total_inverted_index_blocks │ 7           │\n",
       "│ vector_index_sz_mb          │ 0.235603    │\n",
       "╰─────────────────────────────┴─────────────╯\n"