Merge pull request #1692 from SciPhi-AI/feature/improve-collection-lo…

…gic-add-regr-tests

Feature/improve collection logic add regr tests

.github/actions/setup-python-full/action.yml

@@ -1,34 +1,80 @@
 name: 'Setup Python for R2R Full'
 description: 'Sets up Python and installs R2R dependencies for full installation'
+
 inputs:
   os:
     description: 'Operating system'
     required: true
+  python-version:
+    description: 'Python version to use'
+    required: false
+    default: '3.12'
+  poetry-version:
+    description: 'Poetry version to install'
+    required: false
+    default: '1.7.1'
+  r2r-version:
+    description: 'R2R CLI & SDK version to install'
+    required: false
+    default: 'latest'
+
 runs:
   using: "composite"
   steps:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.12'
-
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+
     - name: Install R2R CLI & Python SDK
       shell: bash
       run: |
-        pip install r2r
-
-    - name: Install Poetry and dependencies on Ubuntu and macOS
-      if: inputs.os == 'ubuntu-latest' || inputs.os == 'macos-latest'
+        if [ "${{ inputs.r2r-version }}" = "latest" ]; then
+          pip install r2r
+        else
+          pip install r2r==${{ inputs.r2r-version }}
+        fi
+    
+    - name: Install Poetry
       shell: bash
       run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        cd py && poetry install -E core -E ingestion-bundle
-
-    - name: Install Poetry and dependencies on Windows
-      if: inputs.os == 'windows-latest'
-      shell: cmd
+        curl -sSL https://install.python-poetry.org | python3 - --version ${{ inputs.poetry-version }}
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+    
+    - name: Set Poetry config
+      shell: bash
+      run: |
+        poetry config virtualenvs.create true
+        poetry config virtualenvs.in-project true
+    
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pypoetry
+          py/.venv
+        key: ${{ runner.os }}-poetry-${{ hashFiles('py/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+    
+    - name: Verify pip cache directory
+      shell: bash
+      run: |
+        mkdir -p ~/.cache/pip
+    
+    - name: Install dependencies
+      shell: bash
+      working-directory: py
+      run: |
+        if [ ! -f "poetry.lock" ]; then
+          echo "No poetry.lock found, creating one..."
+          poetry lock
+        fi
+        poetry install -E core -E ingestion-bundle --no-interaction
+    
+    - name: Verify installation
+      shell: bash
       run: |
-        python -c "import urllib.request; print(urllib.request.urlopen('https://install.python-poetry.org').read().decode())" > install-poetry.py
-        python install-poetry.py
-        echo %USERPROFILE%\AppData\Roaming\Python\Scripts >> %GITHUB_PATH%
-        cd py && poetry install -E core -E ingestion-bundle
+        python -c "import r2r; print(f'R2R version: {r2r.__version__}')"
+        poetry --version

.github/actions/setup-python-light/action.yml

@@ -1,30 +1,56 @@
 name: 'Setup Python for R2R Light'
 description: 'Sets up Python environment and installs dependencies using Poetry'
+
 inputs:
   os:
     description: 'Operating system'
     required: true
+  python-version:
+    description: 'Python version to use'
+    required: false
+    default: '3.12'
+  poetry-version:
+    description: 'Poetry version to install'
+    required: false
+    default: '1.7.1'
+
 runs:
   using: "composite"
   steps:
     - name: Set up Python environment
       uses: actions/setup-python@v4
       with:
-        python-version: '3.12'
-
-    - name: Install Poetry and dependencies on Ubuntu and macOS
-      if: inputs.os == 'ubuntu-latest' || inputs.os == 'macos-latest'
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+
+    - name: Install Poetry
       shell: bash
       run: |
-        curl -sSL https://install.python-poetry.org | python3 -
-        cd py && poetry install -E core -E ingestion-bundle
-
-    - name: Install Poetry and dependencies on Windows
-      if: inputs.os == 'windows-latest'
-      shell: pwsh
+        curl -sSL https://install.python-poetry.org | python3 - --version ${{ inputs.poetry-version }}
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+    
+    - name: Set Poetry config
+      shell: bash
+      run: |
+        poetry config virtualenvs.create true
+        poetry config virtualenvs.in-project true
+    
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pypoetry
+          py/.venv
+        key: ${{ runner.os }}-poetry-${{ hashFiles('py/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+    
+    - name: Install dependencies
+      shell: bash
+      working-directory: py
       run: |
-        (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
-        $env:PATH += ";$env:APPDATA\Python\Scripts"
-        poetry --version
-        cd py
-        poetry install -E core -E ingestion-bundle
+        if [ ! -f "poetry.lock" ]; then
+          echo "No poetry.lock found, creating one..."
+          poetry lock
+        fi
+        poetry install -E core -E ingestion-bundle --no-interaction

.github/workflows/build-cluster-docker.yml → .github/workflows/build-cluster-service-docker.yml

@@ -1,4 +1,4 @@
-name: Build and Publish Cluster Docker Image
+name: Build and Publish Cluster Service Docker Image
 
 on:
   workflow_dispatch:

.github/workflows/build-unst-docker.yml → .github/workflows/build-unst-service-docker.yml

@@ -1,4 +1,4 @@
-name: Build and Publish Unstructured Docker Image
+name: Build and Publish Unstructured Service Docker Image
 
 on:
   workflow_dispatch: