dev-minor

.github/actions/run-sdk-prompt-management-tests/action.yml

@@ -0,0 +1,42 @@
+name: 'Run SDK Prompt Management Tests'
+description: 'Runs SDK prompt management tests for R2R'
+runs:
+  using: "composite"
+  steps:
+    # First run basic prompt operations
+    - name: Add prompt test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_add_prompt
+
+    - name: Get prompt test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_get_prompt
+
+    - name: Get all prompts test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_get_all_prompts
+
+    - name: Update prompt test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_update_prompt
+
+    # Then run error handling and access control tests
+    - name: Prompt error handling test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_prompt_error_handling
+
+    - name: Prompt access control test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_prompt_access_control
+
+    # Finally run deletion test
+    - name: Delete prompt test (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_delete_prompt

.github/workflows/r2r-full-integration-deep-dive-tests.yml

@@ -36,6 +36,3 @@ jobs:
 
       - name: Start R2R Full server
         uses: ./.github/actions/start-r2r-full
-
-      - name: Run Test Zerox
-        uses: ./.github/actions/run-script-zerox-tests

.github/workflows/r2r-full-py-integration-tests-graphrag.yml

@@ -16,6 +16,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:

.github/workflows/r2r-full-py-integration-tests-mac-and-windows.yml

@@ -6,6 +6,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:

.github/workflows/r2r-full-py-integration-tests.yml

@@ -16,6 +16,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:
@@ -27,6 +28,7 @@ jobs:
           - sdk-retrieval
           - sdk-auth
           - sdk-collections
+          - sdk-prompts
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       TELEMETRY_ENABLED: 'false'
@@ -56,29 +58,27 @@ jobs:
       - name: Run CLI Ingestion Tests
         if: matrix.test_category == 'cli-ingestion'
         uses: ./.github/actions/run-cli-ingestion-tests
-        continue-on-error: true
 
       - name: Run CLI Retrieval Tests
         if: matrix.test_category == 'cli-retrieval'
         uses: ./.github/actions/run-cli-retrieval-tests
-        continue-on-error: true
 
       - name: Run SDK Ingestion Tests
         if: matrix.test_category == 'sdk-ingestion'
         uses: ./.github/actions/run-sdk-ingestion-tests
-        continue-on-error: true
 
       - name: Run SDK Retrieval Tests
         if: matrix.test_category == 'sdk-retrieval'
         uses: ./.github/actions/run-sdk-retrieval-tests
-        continue-on-error: true
 
       - name: Run SDK Auth Tests
         if: matrix.test_category == 'sdk-auth'
         uses: ./.github/actions/run-sdk-auth-tests
-        continue-on-error: true
 
       - name: Run SDK Collections Tests
         if: matrix.test_category == 'sdk-collections'
         uses: ./.github/actions/run-sdk-collections-tests
-        continue-on-error: true
+
+      - name: Run SDK Prompt Tests
+        if: matrix.test_category == 'sdk-prompts'
+        uses: ./.github/actions/run-sdk-prompt-management-tests

.github/workflows/r2r-light-py-integration-tests-graphrag.yml

@@ -18,6 +18,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:

.github/workflows/r2r-light-py-integration-tests-mac-and-windows.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:

.github/workflows/r2r-light-py-integration-tests.yml

@@ -18,6 +18,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
 
     strategy:
       matrix:
@@ -29,6 +30,7 @@ jobs:
           - sdk-retrieval
           - sdk-auth
           - sdk-collections
+          - sdk-prompts
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       TELEMETRY_ENABLED: 'false'
@@ -59,29 +61,27 @@ jobs:
       - name: Run CLI Ingestion Tests
         if: matrix.test_category == 'cli-ingestion'
         uses: ./.github/actions/run-cli-ingestion-tests
-        continue-on-error: true
 
       - name: Run CLI Retrieval Tests
         if: matrix.test_category == 'cli-retrieval'
         uses: ./.github/actions/run-cli-retrieval-tests
-        continue-on-error: true
 
       - name: Run SDK Ingestion Tests
         if: matrix.test_category == 'sdk-ingestion'
         uses: ./.github/actions/run-sdk-ingestion-tests
-        continue-on-error: true
 
       - name: Run SDK Retrieval Tests
         if: matrix.test_category == 'sdk-retrieval'
         uses: ./.github/actions/run-sdk-retrieval-tests
-        continue-on-error: true
 
       - name: Run SDK Auth Tests
         if: matrix.test_category == 'sdk-auth'
         uses: ./.github/actions/run-sdk-auth-tests
-        continue-on-error: true
 
       - name: Run SDK Collections Tests
         if: matrix.test_category == 'sdk-collections'
         uses: ./.github/actions/run-sdk-collections-tests
-        continue-on-error: true
+
+      - name: Run SDK Prompt Tests
+        if: matrix.test_category == 'sdk-prompts'
+        uses: ./.github/actions/run-sdk-prompt-management-tests

docs/cookbooks/graphrag.mdx

@@ -99,9 +99,6 @@ excluded_parsers = ["mp4"]
     semantic_similarity_threshold = 0.7
     generation_config = { model = "openai/gpt-4o-mini" }
 
-  [ingestion.extra_parsers]
-    pdf = "zerox"
-
 [database]
 provider = "postgres"
 batch_size = 256
@@ -204,9 +201,6 @@ max_characters = 1_024
 combine_under_n_chars = 128
 overlap = 256
 
-    [ingestion.extra_parsers]
-    pdf = "zerox"
-
 [orchestration]
 provider = "hatchet"
 kg_creation_concurrency_lipmit = 32

js/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "r2r-js",
-  "version": "0.3.11",
+  "version": "0.3.12",
   "description": "",
   "main": "dist/index.js",
   "browser": "dist/index.browser.js",

py/cli/command_group.py

@@ -2,7 +2,7 @@
 from asyncclick import pass_context
 from asyncclick.exceptions import Exit
 
-from r2r import R2RAsyncClient
+from sdk import R2RAsyncClient
 
 
 @click.group()

py/cli/commands/ingestion.py

@@ -11,7 +11,7 @@
 from cli.command_group import cli
 from cli.utils.param_types import JSON
 from cli.utils.timer import timer
-from core.base.abstractions import IndexMeasure, IndexMethod, VectorTableName
+from shared.abstractions import IndexMeasure, IndexMethod, VectorTableName
 
 
 async def ingest_files_from_urls(client, urls):
@@ -243,6 +243,7 @@ async def create_vector_index(
     index_measure,
     index_arguments,
     index_name,
+    index_column,
     no_concurrent,
 ):
     """Create a vector index for similarity search."""
@@ -254,6 +255,7 @@ async def create_vector_index(
             index_measure=index_measure,
             index_arguments=index_arguments,
             index_name=index_name,
+            index_column=index_column,
             concurrently=not no_concurrent,
         )
     click.echo(json.dumps(response, indent=2))

py/core/__init__.py

@@ -134,6 +134,9 @@
     # Crypto provider
     "CryptoConfig",
     "CryptoProvider",
+    # Email provider
+    "EmailConfig",
+    "EmailProvider",
     # Database providers
     "DatabaseConfig",
     "DatabaseProvider",
@@ -192,9 +195,9 @@
     "AudioParser",
     "DOCXParser",
     "ImageParser",
-    "PDFParser",
+    "VLMPDFParser",
+    "BasicPDFParser",
     "PDFParserUnstructured",
-    "PDFParserMarker",
     "PPTParser",
     # Structured parsers
     "CSVParser",
@@ -233,6 +236,7 @@
     # Embeddings
     "LiteLLMEmbeddingProvider",
     "OpenAIEmbeddingProvider",
+    "OllamaEmbeddingProvider",
     # LLM
     "OpenAICompletionProvider",
     "LiteLLMCompletionProvider",

py/core/base/__init__.py

@@ -106,6 +106,9 @@
     # Crypto provider
     "CryptoConfig",
     "CryptoProvider",
+    # Email provider
+    "EmailConfig",
+    "EmailProvider",
     # Database providers
     "DatabaseConfig",
     "DatabaseProvider",

py/core/base/parsers/base_parser.py

@@ -3,14 +3,11 @@
 from abc import ABC, abstractmethod
 from typing import AsyncGenerator, Generic, TypeVar
 
-from ..abstractions import DataType
-
 T = TypeVar("T")
 
 
 class AsyncParser(ABC, Generic[T]):
+
     @abstractmethod
-    async def ingest(
-        self, data: T, **kwargs
-    ) -> AsyncGenerator[DataType, None]:
+    async def ingest(self, data: T, **kwargs) -> AsyncGenerator[str, None]:
         pass

py/core/base/providers/__init__.py

@@ -16,6 +16,7 @@
     UserHandler,
     VectorHandler,
 )
+from .email import EmailConfig, EmailProvider
 from .embedding import EmbeddingConfig, EmbeddingProvider
 from .ingestion import ChunkingStrategy, IngestionConfig, IngestionProvider
 from .llm import CompletionConfig, CompletionProvider
@@ -36,6 +37,9 @@
     # Crypto provider
     "CryptoConfig",
     "CryptoProvider",
+    # Email provider
+    "EmailConfig",
+    "EmailProvider",
     # Database providers
     "DatabaseConnectionManager",
     "DocumentHandler",

py/core/base/providers/auth.py

@@ -10,6 +10,8 @@
 from ..api.models import UserResponse
 from .base import Provider, ProviderConfig
 from .crypto import CryptoProvider
+from .database import DatabaseProvider
+from .email import EmailProvider
 
 logger = logging.getLogger()
 
@@ -33,8 +35,17 @@ def validate_config(self) -> None:
 
 class AuthProvider(Provider, ABC):
     security = HTTPBearer(auto_error=False)
-
-    def __init__(self, config: AuthConfig, crypto_provider: CryptoProvider):
+    crypto_provider: CryptoProvider
+    email_provider: EmailProvider
+    database_provider: DatabaseProvider
+
+    def __init__(
+        self,
+        config: AuthConfig,
+        crypto_provider: CryptoProvider,
+        database_provider: DatabaseProvider,
+        email_provider: EmailProvider,
+    ):
         if not isinstance(config, AuthConfig):
             raise ValueError(
                 "AuthProvider must be initialized with an AuthConfig"
@@ -43,6 +54,8 @@ def __init__(self, config: AuthConfig, crypto_provider: CryptoProvider):
         self.admin_email = config.default_admin_email
         self.admin_password = config.default_admin_password
         self.crypto_provider = crypto_provider
+        self.database_provider = database_provider
+        self.email_provider = email_provider
         super().__init__(config)
         self.config: AuthConfig = config  # for type hinting