helpmate-ai-upgrad_sandeep_b.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9314717,"sourceType":"datasetVersion","datasetId":5641558}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pdfplumber tiktoken openai chromadb sentence_transformers","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:11:39.609286Z","iopub.execute_input":"2024-09-04T17:11:39.610504Z","iopub.status.idle":"2024-09-04T17:12:32.060567Z","shell.execute_reply.started":"2024-09-04T17:11:39.610423Z","shell.execute_reply":"2024-09-04T17:12:32.058903Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting pdfplumber\n  Downloading pdfplumber-0.11.4-py3-none-any.whl.metadata (41 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.0/42.0 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting tiktoken\n  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\nCollecting openai\n  Downloading openai-1.43.0-py3-none-any.whl.metadata (22 kB)\nCollecting chromadb\n  Downloading chromadb-0.5.5-py3-none-any.whl.metadata (6.8 kB)\nCollecting sentence_transformers\n  Downloading sentence_transformers-3.0.1-py3-none-any.whl.metadata (10 kB)\nCollecting pdfminer.six==20231228 (from pdfplumber)\n  Downloading pdfminer.six-20231228-py3-none-any.whl.metadata (4.2 kB)\nRequirement already satisfied: Pillow>=9.1 in /opt/conda/lib/python3.10/site-packages (from pdfplumber) (9.5.0)\nCollecting pypdfium2>=4.18.0 (from pdfplumber)\n  Downloading pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (48 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.5/48.5 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: charset-normalizer>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from pdfminer.six==20231228->pdfplumber) (3.3.2)\nRequirement already satisfied: cryptography>=36.0.0 in /opt/conda/lib/python3.10/site-packages (from pdfminer.six==20231228->pdfplumber) (42.0.8)\nRequirement already satisfied: regex>=2022.1.18 in /opt/conda/lib/python3.10/site-packages (from tiktoken) (2024.5.15)\nRequirement already satisfied: requests>=2.26.0 in /opt/conda/lib/python3.10/site-packages (from tiktoken) (2.32.3)\nRequirement already satisfied: anyio<5,>=3.5.0 in /opt/conda/lib/python3.10/site-packages (from openai) (4.4.0)\nRequirement already satisfied: distro<2,>=1.7.0 in /opt/conda/lib/python3.10/site-packages (from openai) (1.9.0)\nRequirement already satisfied: httpx<1,>=0.23.0 in /opt/conda/lib/python3.10/site-packages (from openai) (0.27.0)\nCollecting jiter<1,>=0.4.0 (from openai)\n  Downloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\nRequirement already satisfied: pydantic<3,>=1.9.0 in /opt/conda/lib/python3.10/site-packages (from openai) (2.8.2)\nRequirement already satisfied: sniffio in /opt/conda/lib/python3.10/site-packages (from openai) (1.3.1)\nRequirement already satisfied: tqdm>4 in /opt/conda/lib/python3.10/site-packages (from openai) (4.66.4)\nRequirement already satisfied: typing-extensions<5,>=4.11 in /opt/conda/lib/python3.10/site-packages (from openai) (4.12.2)\nCollecting build>=1.0.3 (from chromadb)\n  Downloading build-1.2.1-py3-none-any.whl.metadata (4.3 kB)\nCollecting chroma-hnswlib==0.7.6 (from chromadb)\n  Downloading chroma_hnswlib-0.7.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (252 bytes)\nRequirement already satisfied: fastapi>=0.95.2 in /opt/conda/lib/python3.10/site-packages (from chromadb) (0.111.0)\nRequirement already satisfied: uvicorn>=0.18.3 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.30.1)\nRequirement already satisfied: numpy<2.0.0,>=1.22.5 in /opt/conda/lib/python3.10/site-packages (from chromadb) (1.26.4)\nCollecting posthog>=2.4.0 (from chromadb)\n  Downloading posthog-3.6.3-py2.py3-none-any.whl.metadata (2.0 kB)\nCollecting onnxruntime>=1.14.1 (from chromadb)\n  Downloading onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (4.5 kB)\nRequirement already satisfied: opentelemetry-api>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (1.25.0)\nRequirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (1.25.0)\nCollecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n  Downloading opentelemetry_instrumentation_fastapi-0.48b0-py3-none-any.whl.metadata (2.1 kB)\nRequirement already satisfied: opentelemetry-sdk>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (1.25.0)\nRequirement already satisfied: tokenizers>=0.13.2 in /opt/conda/lib/python3.10/site-packages (from chromadb) (0.19.1)\nCollecting pypika>=0.48.9 (from chromadb)\n  Downloading PyPika-0.48.9.tar.gz (67 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: overrides>=7.3.1 in /opt/conda/lib/python3.10/site-packages (from chromadb) (7.7.0)\nRequirement already satisfied: importlib-resources in /opt/conda/lib/python3.10/site-packages (from chromadb) (6.4.0)\nRequirement already satisfied: grpcio>=1.58.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (1.64.1)\nCollecting bcrypt>=4.0.1 (from chromadb)\n  Downloading bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl.metadata (9.6 kB)\nRequirement already satisfied: typer>=0.9.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (0.12.3)\nCollecting kubernetes>=28.1.0 (from chromadb)\n  Downloading kubernetes-30.1.0-py2.py3-none-any.whl.metadata (1.5 kB)\nRequirement already satisfied: tenacity>=8.2.3 in /opt/conda/lib/python3.10/site-packages (from chromadb) (8.3.0)\nRequirement already satisfied: PyYAML>=6.0.0 in /opt/conda/lib/python3.10/site-packages (from chromadb) (6.0.2)\nCollecting mmh3>=4.0.1 (from chromadb)\n  Downloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\nRequirement already satisfied: orjson>=3.9.12 in /opt/conda/lib/python3.10/site-packages (from chromadb) (3.10.4)\nRequirement already satisfied: transformers<5.0.0,>=4.34.0 in /opt/conda/lib/python3.10/site-packages (from sentence_transformers) (4.44.0)\nRequirement already satisfied: torch>=1.11.0 in /opt/conda/lib/python3.10/site-packages (from sentence_transformers) (2.4.0+cpu)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (from sentence_transformers) (1.2.2)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.10/site-packages (from sentence_transformers) (1.14.0)\nRequirement already satisfied: huggingface-hub>=0.15.1 in /opt/conda/lib/python3.10/site-packages (from sentence_transformers) (0.24.6)\nRequirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai) (3.7)\nRequirement already satisfied: exceptiongroup>=1.0.2 in /opt/conda/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai) (1.2.0)\nRequirement already satisfied: packaging>=19.1 in /opt/conda/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (21.3)\nCollecting pyproject_hooks (from build>=1.0.3->chromadb)\n  Downloading pyproject_hooks-1.1.0-py3-none-any.whl.metadata (1.3 kB)\nRequirement already satisfied: tomli>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (2.0.1)\nRequirement already satisfied: starlette<0.38.0,>=0.37.2 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (0.37.2)\nRequirement already satisfied: fastapi-cli>=0.0.2 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (0.0.4)\nRequirement already satisfied: jinja2>=2.11.2 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (3.1.4)\nRequirement already satisfied: python-multipart>=0.0.7 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (0.0.9)\nRequirement already satisfied: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (5.10.0)\nRequirement already satisfied: email_validator>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (2.1.1)\nRequirement already satisfied: certifi in /opt/conda/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\nRequirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai) (1.0.5)\nRequirement already satisfied: h11<0.15,>=0.13 in /opt/conda/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.15.1)\nRequirement already satisfied: fsspec>=2023.5.0 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2024.6.1)\nRequirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\nRequirement already satisfied: python-dateutil>=2.5.3 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.9.0.post0)\nRequirement already satisfied: google-auth>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.30.0)\nRequirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\nRequirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\nRequirement already satisfied: oauthlib>=3.2.2 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\nRequirement already satisfied: urllib3>=1.24.2 in /opt/conda/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.26.18)\nCollecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl.metadata (12 kB)\nRequirement already satisfied: flatbuffers in /opt/conda/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\nRequirement already satisfied: deprecated>=1.2.6 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.14)\nRequirement already satisfied: importlib-metadata<=7.1,>=6.0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (7.0.0)\nRequirement already satisfied: googleapis-common-protos~=1.52 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.1)\nRequirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.25.0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.25.0)\nRequirement already satisfied: opentelemetry-proto==1.25.0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.25.0)\nCollecting opentelemetry-instrumentation-asgi==0.48b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation_asgi-0.48b0-py3-none-any.whl.metadata (2.0 kB)\nCollecting opentelemetry-instrumentation==0.48b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation-0.48b0-py3-none-any.whl.metadata (6.1 kB)\nCollecting opentelemetry-semantic-conventions==0.48b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_semantic_conventions-0.48b0-py3-none-any.whl.metadata (2.4 kB)\nCollecting opentelemetry-util-http==0.48b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_util_http-0.48b0-py3-none-any.whl.metadata (2.5 kB)\nRequirement already satisfied: setuptools>=16.0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.48b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (70.0.0)\nRequirement already satisfied: wrapt<2.0.0,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.48b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.16.0)\nCollecting asgiref~=3.0 (from opentelemetry-instrumentation-asgi==0.48b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading asgiref-3.8.1-py3-none-any.whl.metadata (9.3 kB)\nCollecting opentelemetry-api>=1.2.0 (from chromadb)\n  Downloading opentelemetry_api-1.27.0-py3-none-any.whl.metadata (1.4 kB)\nINFO: pip is looking at multiple versions of opentelemetry-sdk to determine which version is compatible with other requirements. This could take a while.\nCollecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n  Downloading opentelemetry_instrumentation_fastapi-0.47b0-py3-none-any.whl.metadata (2.1 kB)\nCollecting opentelemetry-instrumentation-asgi==0.47b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation_asgi-0.47b0-py3-none-any.whl.metadata (2.0 kB)\nCollecting opentelemetry-instrumentation==0.47b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation-0.47b0-py3-none-any.whl.metadata (6.1 kB)\nCollecting opentelemetry-semantic-conventions==0.47b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_semantic_conventions-0.47b0-py3-none-any.whl.metadata (2.4 kB)\nCollecting opentelemetry-util-http==0.47b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_util_http-0.47b0-py3-none-any.whl.metadata (2.5 kB)\nCollecting opentelemetry-api>=1.2.0 (from chromadb)\n  Downloading opentelemetry_api-1.26.0-py3-none-any.whl.metadata (1.4 kB)\nCollecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n  Downloading opentelemetry_instrumentation_fastapi-0.46b0-py3-none-any.whl.metadata (2.0 kB)\nCollecting opentelemetry-instrumentation-asgi==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation_asgi-0.46b0-py3-none-any.whl.metadata (1.9 kB)\nCollecting opentelemetry-instrumentation==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_instrumentation-0.46b0-py3-none-any.whl.metadata (6.1 kB)\nRequirement already satisfied: opentelemetry-semantic-conventions==0.46b0 in /opt/conda/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.46b0)\nCollecting opentelemetry-util-http==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n  Downloading opentelemetry_util_http-0.46b0-py3-none-any.whl.metadata (2.4 kB)\nCollecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n  Downloading monotonic-1.6-py2.py3-none-any.whl.metadata (1.5 kB)\nCollecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n  Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\nRequirement already satisfied: annotated-types>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\nRequirement already satisfied: pydantic-core==2.20.1 in /opt/conda/lib/python3.10/site-packages (from pydantic<3,>=1.9.0->openai) (2.20.1)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.11.0->sentence_transformers) (3.3)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers<5.0.0,>=4.34.0->sentence_transformers) (0.4.4)\nRequirement already satisfied: click>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (8.1.7)\nRequirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (1.5.4)\nRequirement already satisfied: rich>=10.11.0 in /opt/conda/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (13.7.1)\nRequirement already satisfied: httptools>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\nRequirement already satisfied: python-dotenv>=0.13 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.1)\nRequirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\nRequirement already satisfied: watchfiles>=0.13 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.22.0)\nRequirement already satisfied: websockets>=10.4 in /opt/conda/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (12.0)\nRequirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->sentence_transformers) (1.4.2)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->sentence_transformers) (3.5.0)\nRequirement already satisfied: cffi>=1.12 in /opt/conda/lib/python3.10/site-packages (from cryptography>=36.0.0->pdfminer.six==20231228->pdfplumber) (1.16.0)\nRequirement already satisfied: dnspython>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb) (2.6.1)\nRequirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.2.4)\nRequirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\nRequirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.10/site-packages (from importlib-metadata<=7.1,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.19.2)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb) (2.1.5)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=19.1->build>=1.0.3->chromadb) (3.1.2)\nRequirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\nRequirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.18.0)\nCollecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n  Downloading humanfriendly-10.0-py2.py3-none-any.whl.metadata (9.2 kB)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\nRequirement already satisfied: pycparser in /opt/conda/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six==20231228->pdfplumber) (2.22)\nRequirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\nRequirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\nDownloading pdfplumber-0.11.4-py3-none-any.whl (59 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.2/59.2 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading pdfminer.six-20231228-py3-none-any.whl (5.6 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m28.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading openai-1.43.0-py3-none-any.whl (365 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m365.7/365.7 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading chromadb-0.5.5-py3-none-any.whl (584 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m584.3/584.3 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading chroma_hnswlib-0.7.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m46.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading sentence_transformers-3.0.1-py3-none-any.whl (227 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.1/227.1 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl (273 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m273.8/273.8 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading build-1.2.1-py3-none-any.whl (21 kB)\nDownloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (318 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.9/318.9 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading kubernetes-30.1.0-py2.py3-none-any.whl (1.7 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m45.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (67 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (13.2 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.2/13.2 MB\u001b[0m \u001b[31m63.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0m\n\u001b[?25hDownloading opentelemetry_instrumentation_fastapi-0.46b0-py3-none-any.whl (11 kB)\nDownloading opentelemetry_instrumentation-0.46b0-py3-none-any.whl (29 kB)\nDownloading opentelemetry_instrumentation_asgi-0.46b0-py3-none-any.whl (14 kB)\nDownloading opentelemetry_util_http-0.46b0-py3-none-any.whl (6.9 kB)\nDownloading posthog-3.6.3-py2.py3-none-any.whl (53 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.8 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m51.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\nDownloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\nDownloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading pyproject_hooks-1.1.0-py3-none-any.whl (9.2 kB)\nDownloading asgiref-3.8.1-py3-none-any.whl (23 kB)\nDownloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hBuilding wheels for collected packages: pypika\n  Building wheel for pypika (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53725 sha256=89ab66037e9c858babb3665cae67a570186bb32b3bd751bfa4a94a6cca39f592\n  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\nSuccessfully built pypika\nInstalling collected packages: pypika, monotonic, mmh3, pyproject_hooks, pypdfium2, opentelemetry-util-http, jiter, humanfriendly, chroma-hnswlib, bcrypt, backoff, asgiref, tiktoken, posthog, coloredlogs, build, pdfminer.six, opentelemetry-instrumentation, openai, onnxruntime, kubernetes, pdfplumber, opentelemetry-instrumentation-asgi, sentence_transformers, opentelemetry-instrumentation-fastapi, chromadb\n  Attempting uninstall: kubernetes\n    Found existing installation: kubernetes 26.1.0\n    Uninstalling kubernetes-26.1.0:\n      Successfully uninstalled kubernetes-26.1.0\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\nkfp 2.5.0 requires google-cloud-storage<3,>=2.2.1, but you have google-cloud-storage 1.44.0 which is incompatible.\nkfp 2.5.0 requires kubernetes<27,>=8.0.0, but you have kubernetes 30.1.0 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed asgiref-3.8.1 backoff-2.2.1 bcrypt-4.2.0 build-1.2.1 chroma-hnswlib-0.7.6 chromadb-0.5.5 coloredlogs-15.0.1 humanfriendly-10.0 jiter-0.5.0 kubernetes-30.1.0 mmh3-4.1.0 monotonic-1.6 onnxruntime-1.19.2 openai-1.43.0 opentelemetry-instrumentation-0.46b0 opentelemetry-instrumentation-asgi-0.46b0 opentelemetry-instrumentation-fastapi-0.46b0 opentelemetry-util-http-0.46b0 pdfminer.six-20231228 pdfplumber-0.11.4 posthog-3.6.3 pypdfium2-4.30.0 pypika-0.48.9 pyproject_hooks-1.1.0 sentence_transformers-3.0.1 tiktoken-0.7.0\n","output_type":"stream"}]},{"cell_type":"code","source":"import pdfplumber\nfrom pathlib import Path\nimport pandas as pd\nfrom operator import itemgetter\nimport json\nimport tiktoken\nimport openai\nfrom sentence_transformers import CrossEncoder, util\nfrom chromadb.utils.embedding_functions import OpenAIEmbeddingFunction\nimport chromadb","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:12:32.063634Z","iopub.execute_input":"2024-09-04T17:12:32.064107Z","iopub.status.idle":"2024-09-04T17:13:03.009887Z","shell.execute_reply.started":"2024-09-04T17:12:32.064060Z","shell.execute_reply":"2024-09-04T17:13:03.008315Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n  from tqdm.autonotebook import tqdm, trange\n","output_type":"stream"}]},{"cell_type":"code","source":"pdf_path='/kaggle/input/helpmate-ai/Principal-Sample-Life-Insurance-Policy.pdf'\nwith pdfplumber.open(pdf_path) as pdf:\n\n    single_page = pdf.pages[6]\n    text = single_page.extract_text()\n    tables = single_page.extract_tables()\n    print(text)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:03.012109Z","iopub.execute_input":"2024-09-04T17:13:03.013695Z","iopub.status.idle":"2024-09-04T17:13:03.290843Z","shell.execute_reply.started":"2024-09-04T17:13:03.013609Z","shell.execute_reply":"2024-09-04T17:13:03.288951Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Section A – Eligibility\nMember Life Insurance Article 1\nMember Accidental Death and Dismemberment Insurance Article 2\nDependent Life Insurance Article 3\nSection B - Effective Dates\nMember Life Insurance Article 1\nMember Accidental Death and Dismemberment Insurance Article 2\nDependent Life Insurance Article 3\nSection C - Individual Terminations\nMember Life Insurance Article 1\nMember Accidental Death and Dismemberment Insurance Article 2\nDependent Life Insurance Article 3\nTermination for Fraud Article 4\nCoverage While Outside of the United States Article 5\nSection D - Continuation\nMember Life Insurance Article 1\nDependent Insurance - Developmentally Disabled or\nPhysically Handicapped Children Article 2\nSection E - Reinstatement\nReinstatement Article 1\nFederal Required Family and Medical Leave Act (FMLA) Article 2\nReinstatement of Coverage for a Member or Dependent When\nCoverage Ends due to Living Outside of the United States Article 3\nSection F - Individual Purchase Rights\nMember Life Insurance Article 1\nDependent Life Insurance Article 2\nPART IV - BENEFITS\nThis policy has been updated effective January 1, 2014\nGC 6001 TABLE OF CONTENTS, PAGE 2\n","output_type":"stream"}]},{"cell_type":"code","source":"def check_bboxes(word, bbox):\n    \"\"\"\n    Helper function to check if a word's bbox is within a table's bbox.\n    \"\"\"\n    word_bbox = (word['x0'], word['top'], word['x1'], word['bottom'])\n    return (word_bbox[0] >= bbox[0] and word_bbox[2] <= bbox[2] and\n            word_bbox[1] >= bbox[1] and word_bbox[3] <= bbox[3])\n\ndef extract_text_from_pdf(pdf_path):\n    full_text = []\n    with pdfplumber.open(pdf_path) as pdf:\n        for p, page in enumerate(pdf.pages, start=1):\n            page_no = f\"Page {p}\"\n            text = page.extract_text()\n            \n            # Extract heading (if text exists)\n            heading = text.split('\\n')[0].strip() if text else None\n\n            tables = page.find_tables()\n            table_bboxes = [i.bbox for i in tables]\n            tables = [{'table': i.extract(), 'top': i.bbox[1]} for i in tables]\n            non_table_words = [word for word in page.extract_words() if not any(\n                check_bboxes(word, table_bbox) for table_bbox in table_bboxes)]\n            lines = []\n\n            for cluster in pdfplumber.utils.cluster_objects(non_table_words + tables, itemgetter('top'), tolerance=5):\n                if 'text' in cluster[0]:\n                    try:\n                        lines.append(' '.join([i['text'] for i in cluster]))\n                    except KeyError:\n                        pass\n                elif 'table' in cluster[0]:\n                    lines.append(json.dumps(cluster[0]['table']))\n\n            full_text.append([page_no, heading, \" \".join(lines)])\n\n    # Convert the extracted data to a DataFrame\n    df = pd.DataFrame(full_text, columns=['Page_Number', 'Heading', 'Text'])\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:03.293014Z","iopub.execute_input":"2024-09-04T17:13:03.293518Z","iopub.status.idle":"2024-09-04T17:13:03.311428Z","shell.execute_reply.started":"2024-09-04T17:13:03.293469Z","shell.execute_reply":"2024-09-04T17:13:03.309584Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"df = extract_text_from_pdf(pdf_path)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:03.316440Z","iopub.execute_input":"2024-09-04T17:13:03.317033Z","iopub.status.idle":"2024-09-04T17:13:15.552519Z","shell.execute_reply.started":"2024-09-04T17:13:03.316975Z","shell.execute_reply":"2024-09-04T17:13:15.550856Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"df.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.554431Z","iopub.execute_input":"2024-09-04T17:13:15.554891Z","iopub.status.idle":"2024-09-04T17:13:15.589633Z","shell.execute_reply.started":"2024-09-04T17:13:15.554846Z","shell.execute_reply":"2024-09-04T17:13:15.588118Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"  Page_Number                                            Heading  \\\n0      Page 1                               DOROTHEA GLAUSE S655   \n1      Page 2                 This page left blank intentionally   \n2      Page 3                                       POLICY RIDER   \n3      Page 4                 This page left blank intentionally   \n4      Page 5                   PRINCIPAL LIFE INSURANCE COMPANY   \n5      Page 6                                  TABLE OF CONTENTS   \n6      Page 7                            Section A – Eligibility   \n7      Page 8                  Section A - Member Life Insurance   \n8      Page 9                              P ART I - DEFINITIONS   \n9     Page 10  T he legally recognized union of two eligible ...   \n\n                                                Text  \n0  DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...  \n1                 This page left blank intentionally  \n2  POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...  \n3                 This page left blank intentionally  \n4  PRINCIPAL LIFE INSURANCE COMPANY (called The P...  \n5  TABLE OF CONTENTS PART I - DEFINITIONS PART II...  \n6  Section A – Eligibility Member Life Insurance ...  \n7  Section A - Member Life Insurance Schedule of ...  \n8  P ART I - DEFINITIONS When used in this Group ...  \n9  T he legally recognized union of two eligible ...  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Page_Number</th>\n      <th>Heading</th>\n      <th>Text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Page 1</td>\n      <td>DOROTHEA GLAUSE S655</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Page 2</td>\n      <td>This page left blank intentionally</td>\n      <td>This page left blank intentionally</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Page 3</td>\n      <td>POLICY RIDER</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Page 4</td>\n      <td>This page left blank intentionally</td>\n      <td>This page left blank intentionally</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Page 5</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Page 6</td>\n      <td>TABLE OF CONTENTS</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Page 7</td>\n      <td>Section A – Eligibility</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Page 8</td>\n      <td>Section A - Member Life Insurance</td>\n      <td>Section A - Member Life Insurance Schedule of ...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Page 9</td>\n      <td>P ART I - DEFINITIONS</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Page 10</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df['Text'][9]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.591246Z","iopub.execute_input":"2024-09-04T17:13:15.591655Z","iopub.status.idle":"2024-09-04T17:13:15.601246Z","shell.execute_reply.started":"2024-09-04T17:13:15.591613Z","shell.execute_reply":"2024-09-04T17:13:15.599545Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"'T he legally recognized union of two eligible individuals of the same sex established according to law. Civil Union Partner For two persons to establish a Civil Union in Rhode Island, it shall be necessary that they satisfy all of the following criteria: a. not be a party to another Civil Union or marriage in Rhode Island; b. be of the same sex and therefore be excluded from the marriage laws of Rhode Island or any other state; c. be at least 18 years of age; d. not be related to the other proposed party to the Civil Union. NOTE: For the purposes of this Group Policy, the term \"spouse\" will include Civil Union Partner, except as otherwise provided in this Group Policy. Date of Issue The date this Group Policy is placed in force: November 1, 2007. Dependent a. A Member\\'s spouse, if that spouse: (1) is legally married to the Member; and (2) is not in the Armed Forces of any country; and (3) is not insured under this Group Policy as a Member. A Member\\'s spouse will also include a Civil Union Partner. b. A Member\\'s Dependent Child (or Children) as defined below. Dependent Child; Dependent Children a. A Member\\'s natural child or stepchild, if that child is 0 days but less than 26 years of age. b. A Member\\'s foster child, if that child: (1) meets the requirements above; and This policy has been updated effective January 1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 2'"},"metadata":{}}]},{"cell_type":"code","source":"df['Text_Length'] = df['Text'].apply(lambda x: len(x.split(' ')))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.603239Z","iopub.execute_input":"2024-09-04T17:13:15.604240Z","iopub.status.idle":"2024-09-04T17:13:15.620026Z","shell.execute_reply.started":"2024-09-04T17:13:15.604185Z","shell.execute_reply":"2024-09-04T17:13:15.618617Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"df.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.621634Z","iopub.execute_input":"2024-09-04T17:13:15.622151Z","iopub.status.idle":"2024-09-04T17:13:15.645715Z","shell.execute_reply.started":"2024-09-04T17:13:15.622095Z","shell.execute_reply":"2024-09-04T17:13:15.644024Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"  Page_Number                                            Heading  \\\n0      Page 1                               DOROTHEA GLAUSE S655   \n1      Page 2                 This page left blank intentionally   \n2      Page 3                                       POLICY RIDER   \n3      Page 4                 This page left blank intentionally   \n4      Page 5                   PRINCIPAL LIFE INSURANCE COMPANY   \n5      Page 6                                  TABLE OF CONTENTS   \n6      Page 7                            Section A – Eligibility   \n7      Page 8                  Section A - Member Life Insurance   \n8      Page 9                              P ART I - DEFINITIONS   \n9     Page 10  T he legally recognized union of two eligible ...   \n\n                                                Text  Text_Length  \n0  DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...           30  \n1                 This page left blank intentionally            5  \n2  POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...          230  \n3                 This page left blank intentionally            5  \n4  PRINCIPAL LIFE INSURANCE COMPANY (called The P...          110  \n5  TABLE OF CONTENTS PART I - DEFINITIONS PART II...          153  \n6  Section A – Eligibility Member Life Insurance ...          176  \n7  Section A - Member Life Insurance Schedule of ...          171  \n8  P ART I - DEFINITIONS When used in this Group ...          387  \n9  T he legally recognized union of two eligible ...          251  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Page_Number</th>\n      <th>Heading</th>\n      <th>Text</th>\n      <th>Text_Length</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Page 1</td>\n      <td>DOROTHEA GLAUSE S655</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n      <td>30</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Page 2</td>\n      <td>This page left blank intentionally</td>\n      <td>This page left blank intentionally</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Page 3</td>\n      <td>POLICY RIDER</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n      <td>230</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Page 4</td>\n      <td>This page left blank intentionally</td>\n      <td>This page left blank intentionally</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Page 5</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n      <td>110</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Page 6</td>\n      <td>TABLE OF CONTENTS</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>153</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Page 7</td>\n      <td>Section A – Eligibility</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n      <td>176</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Page 8</td>\n      <td>Section A - Member Life Insurance</td>\n      <td>Section A - Member Life Insurance Schedule of ...</td>\n      <td>171</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Page 9</td>\n      <td>P ART I - DEFINITIONS</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n      <td>387</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Page 10</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>251</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df = df.loc[df['Text_Length']>=10]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.647824Z","iopub.execute_input":"2024-09-04T17:13:15.648326Z","iopub.status.idle":"2024-09-04T17:13:15.661940Z","shell.execute_reply.started":"2024-09-04T17:13:15.648278Z","shell.execute_reply":"2024-09-04T17:13:15.660242Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"df.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.663694Z","iopub.execute_input":"2024-09-04T17:13:15.664227Z","iopub.status.idle":"2024-09-04T17:13:15.686004Z","shell.execute_reply.started":"2024-09-04T17:13:15.664168Z","shell.execute_reply":"2024-09-04T17:13:15.684197Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"   Page_Number                                            Heading  \\\n0       Page 1                               DOROTHEA GLAUSE S655   \n2       Page 3                                       POLICY RIDER   \n4       Page 5                   PRINCIPAL LIFE INSURANCE COMPANY   \n5       Page 6                                  TABLE OF CONTENTS   \n6       Page 7                            Section A – Eligibility   \n7       Page 8                  Section A - Member Life Insurance   \n8       Page 9                              P ART I - DEFINITIONS   \n9      Page 10  T he legally recognized union of two eligible ...   \n10     Page 11  (2) has been placed with the Member or spouse ...   \n11     Page 12  An institution that is licensed as a Hospital ...   \n\n                                                 Text  Text_Length  \n0   DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...           30  \n2   POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...          230  \n4   PRINCIPAL LIFE INSURANCE COMPANY (called The P...          110  \n5   TABLE OF CONTENTS PART I - DEFINITIONS PART II...          153  \n6   Section A – Eligibility Member Life Insurance ...          176  \n7   Section A - Member Life Insurance Schedule of ...          171  \n8   P ART I - DEFINITIONS When used in this Group ...          387  \n9   T he legally recognized union of two eligible ...          251  \n10  (2) has been placed with the Member or spouse ...          299  \n11  An institution that is licensed as a Hospital ...          352  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Page_Number</th>\n      <th>Heading</th>\n      <th>Text</th>\n      <th>Text_Length</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Page 1</td>\n      <td>DOROTHEA GLAUSE S655</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n      <td>30</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Page 3</td>\n      <td>POLICY RIDER</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n      <td>230</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Page 5</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n      <td>110</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Page 6</td>\n      <td>TABLE OF CONTENTS</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>153</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Page 7</td>\n      <td>Section A – Eligibility</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n      <td>176</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Page 8</td>\n      <td>Section A - Member Life Insurance</td>\n      <td>Section A - Member Life Insurance Schedule of ...</td>\n      <td>171</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Page 9</td>\n      <td>P ART I - DEFINITIONS</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n      <td>387</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Page 10</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>251</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Page 11</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>299</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>Page 12</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>352</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.687890Z","iopub.execute_input":"2024-09-04T17:13:15.688381Z","iopub.status.idle":"2024-09-04T17:13:15.707503Z","shell.execute_reply.started":"2024-09-04T17:13:15.688335Z","shell.execute_reply":"2024-09-04T17:13:15.705885Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"  Page_Number                           Heading  \\\n0      Page 1              DOROTHEA GLAUSE S655   \n2      Page 3                      POLICY RIDER   \n4      Page 5  PRINCIPAL LIFE INSURANCE COMPANY   \n5      Page 6                 TABLE OF CONTENTS   \n6      Page 7           Section A – Eligibility   \n\n                                                Text  Text_Length  \n0  DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...           30  \n2  POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...          230  \n4  PRINCIPAL LIFE INSURANCE COMPANY (called The P...          110  \n5  TABLE OF CONTENTS PART I - DEFINITIONS PART II...          153  \n6  Section A – Eligibility Member Life Insurance ...          176  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Page_Number</th>\n      <th>Heading</th>\n      <th>Text</th>\n      <th>Text_Length</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Page 1</td>\n      <td>DOROTHEA GLAUSE S655</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n      <td>30</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Page 3</td>\n      <td>POLICY RIDER</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n      <td>230</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Page 5</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n      <td>110</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Page 6</td>\n      <td>TABLE OF CONTENTS</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>153</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Page 7</td>\n      <td>Section A – Eligibility</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n      <td>176</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df['Metadata'] = df.apply(\n    lambda x: {\n        'Section': (x['Heading'][:20] if x['Heading'] else ''),\n        'Page_No.': x['Page_Number']\n    },\n    axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.709073Z","iopub.execute_input":"2024-09-04T17:13:15.709583Z","iopub.status.idle":"2024-09-04T17:13:15.723802Z","shell.execute_reply.started":"2024-09-04T17:13:15.709517Z","shell.execute_reply":"2024-09-04T17:13:15.721848Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"def chunk_text(text, chunk_size=300, overlap_size=50):\n    # Split the text into individual words\n    words = text.split()\n    chunks = []\n    \n    # Iterate over the words to create chunks with overlap\n    for i in range(0, len(words), chunk_size - overlap_size):\n        # Create a chunk from the current position\n        chunk = \" \".join(words[i:i + chunk_size])\n        chunks.append(chunk)\n    \n    return chunks\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.731716Z","iopub.execute_input":"2024-09-04T17:13:15.733328Z","iopub.status.idle":"2024-09-04T17:13:15.742592Z","shell.execute_reply.started":"2024-09-04T17:13:15.733262Z","shell.execute_reply":"2024-09-04T17:13:15.740962Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"df['Chunks'] = df['Text'].apply(lambda x: chunk_text(x))\n\n# Flatten the DataFrame to have one row per chunk\nchunked_df = df.explode('Chunks').reset_index(drop=True)\n\n# Add an identifier to each chunk to keep track of the page and chunk number\nchunked_df['Chunk_ID'] = chunked_df.index + 1\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.744371Z","iopub.execute_input":"2024-09-04T17:13:15.744853Z","iopub.status.idle":"2024-09-04T17:13:15.780476Z","shell.execute_reply.started":"2024-09-04T17:13:15.744794Z","shell.execute_reply":"2024-09-04T17:13:15.778632Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"chunked_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.782297Z","iopub.execute_input":"2024-09-04T17:13:15.782814Z","iopub.status.idle":"2024-09-04T17:13:15.818892Z","shell.execute_reply.started":"2024-09-04T17:13:15.782752Z","shell.execute_reply":"2024-09-04T17:13:15.816961Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"   Page_Number                                            Heading  \\\n0       Page 1                               DOROTHEA GLAUSE S655   \n1       Page 3                                       POLICY RIDER   \n2       Page 5                   PRINCIPAL LIFE INSURANCE COMPANY   \n3       Page 6                                  TABLE OF CONTENTS   \n4       Page 7                            Section A – Eligibility   \n5       Page 8                  Section A - Member Life Insurance   \n6       Page 9                              P ART I - DEFINITIONS   \n7       Page 9                              P ART I - DEFINITIONS   \n8      Page 10  T he legally recognized union of two eligible ...   \n9      Page 10  T he legally recognized union of two eligible ...   \n10     Page 11  (2) has been placed with the Member or spouse ...   \n11     Page 11  (2) has been placed with the Member or spouse ...   \n12     Page 12  An institution that is licensed as a Hospital ...   \n13     Page 12  An institution that is licensed as a Hospital ...   \n14     Page 13  a . A licensed Doctor of Medicine (M.D.) or Os...   \n15     Page 13  a . A licensed Doctor of Medicine (M.D.) or Os...   \n16     Page 14                    c . end stage renal failure; or   \n17     Page 14                    c . end stage renal failure; or   \n18     Page 15  A record which is on or transmitted by paper o...   \n19     Page 16                    PART II - POLICY ADMINISTRATION   \n\n                                                 Text  Text_Length  \\\n0   DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...           30   \n1   POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...          230   \n2   PRINCIPAL LIFE INSURANCE COMPANY (called The P...          110   \n3   TABLE OF CONTENTS PART I - DEFINITIONS PART II...          153   \n4   Section A – Eligibility Member Life Insurance ...          176   \n5   Section A - Member Life Insurance Schedule of ...          171   \n6   P ART I - DEFINITIONS When used in this Group ...          387   \n7   P ART I - DEFINITIONS When used in this Group ...          387   \n8   T he legally recognized union of two eligible ...          251   \n9   T he legally recognized union of two eligible ...          251   \n10  (2) has been placed with the Member or spouse ...          299   \n11  (2) has been placed with the Member or spouse ...          299   \n12  An institution that is licensed as a Hospital ...          352   \n13  An institution that is licensed as a Hospital ...          352   \n14  a . A licensed Doctor of Medicine (M.D.) or Os...          260   \n15  a . A licensed Doctor of Medicine (M.D.) or Os...          260   \n16  c . end stage renal failure; or d. acquired im...          316   \n17  c . end stage renal failure; or d. acquired im...          316   \n18  A record which is on or transmitted by paper o...           36   \n19  PART II - POLICY ADMINISTRATION Section A - Co...          325   \n\n                                             Metadata  \\\n0   {'Section': 'DOROTHEA GLAUSE S655', 'Page_No.'...   \n1   {'Section': 'POLICY RIDER', 'Page_No.': 'Page 3'}   \n2   {'Section': 'PRINCIPAL LIFE INSUR', 'Page_No.'...   \n3   {'Section': 'TABLE OF CONTENTS', 'Page_No.': '...   \n4   {'Section': 'Section A – Eligibil', 'Page_No.'...   \n5   {'Section': 'Section A - Member L', 'Page_No.'...   \n6   {'Section': 'P ART I - DEFINITION', 'Page_No.'...   \n7   {'Section': 'P ART I - DEFINITION', 'Page_No.'...   \n8   {'Section': 'T he legally recogni', 'Page_No.'...   \n9   {'Section': 'T he legally recogni', 'Page_No.'...   \n10  {'Section': '(2) has been placed ', 'Page_No.'...   \n11  {'Section': '(2) has been placed ', 'Page_No.'...   \n12  {'Section': 'An institution that ', 'Page_No.'...   \n13  {'Section': 'An institution that ', 'Page_No.'...   \n14  {'Section': 'a . A licensed Docto', 'Page_No.'...   \n15  {'Section': 'a . A licensed Docto', 'Page_No.'...   \n16  {'Section': 'c . end stage renal ', 'Page_No.'...   \n17  {'Section': 'c . end stage renal ', 'Page_No.'...   \n18  {'Section': 'A record which is on', 'Page_No.'...   \n19  {'Section': 'PART II - POLICY ADM', 'Page_No.'...   \n\n                                               Chunks  Chunk_ID  \n0   DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...         1  \n1   POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...         2  \n2   PRINCIPAL LIFE INSURANCE COMPANY (called The P...         3  \n3   TABLE OF CONTENTS PART I - DEFINITIONS PART II...         4  \n4   Section A – Eligibility Member Life Insurance ...         5  \n5   Section A - Member Life Insurance Schedule of ...         6  \n6   P ART I - DEFINITIONS When used in this Group ...         7  \n7   f. Continence - the ability to voluntarily con...         8  \n8   T he legally recognized union of two eligible ...         9  \n9                                                   2        10  \n10  (2) has been placed with the Member or spouse ...        11  \n11  dependent on the Member for principal support....        12  \n12  An institution that is licensed as a Hospital ...        13  \n13  that, due to sickness or injury, the person is...        14  \n14  a . A licensed Doctor of Medicine (M.D.) or Os...        15  \n15       1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 5        16  \n16  c . end stage renal failure; or d. acquired im...        17  \n17  Benefits. Total Disability; Totally Disabled A...        18  \n18  A record which is on or transmitted by paper o...        19  \n19  PART II - POLICY ADMINISTRATION Section A - Co...        20  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Page_Number</th>\n      <th>Heading</th>\n      <th>Text</th>\n      <th>Text_Length</th>\n      <th>Metadata</th>\n      <th>Chunks</th>\n      <th>Chunk_ID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Page 1</td>\n      <td>DOROTHEA GLAUSE S655</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n      <td>30</td>\n      <td>{'Section': 'DOROTHEA GLAUSE S655', 'Page_No.'...</td>\n      <td>DOROTHEA GLAUSE S655 RHODE ISLAND JOHN DOE 01/...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Page 3</td>\n      <td>POLICY RIDER</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n      <td>230</td>\n      <td>{'Section': 'POLICY RIDER', 'Page_No.': 'Page 3'}</td>\n      <td>POLICY RIDER GROUP INSURANCE POLICY NO: S655 C...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Page 5</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n      <td>110</td>\n      <td>{'Section': 'PRINCIPAL LIFE INSUR', 'Page_No.'...</td>\n      <td>PRINCIPAL LIFE INSURANCE COMPANY (called The P...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Page 6</td>\n      <td>TABLE OF CONTENTS</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>153</td>\n      <td>{'Section': 'TABLE OF CONTENTS', 'Page_No.': '...</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Page 7</td>\n      <td>Section A – Eligibility</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n      <td>176</td>\n      <td>{'Section': 'Section A – Eligibil', 'Page_No.'...</td>\n      <td>Section A – Eligibility Member Life Insurance ...</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Page 8</td>\n      <td>Section A - Member Life Insurance</td>\n      <td>Section A - Member Life Insurance Schedule of ...</td>\n      <td>171</td>\n      <td>{'Section': 'Section A - Member L', 'Page_No.'...</td>\n      <td>Section A - Member Life Insurance Schedule of ...</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Page 9</td>\n      <td>P ART I - DEFINITIONS</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n      <td>387</td>\n      <td>{'Section': 'P ART I - DEFINITION', 'Page_No.'...</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Page 9</td>\n      <td>P ART I - DEFINITIONS</td>\n      <td>P ART I - DEFINITIONS When used in this Group ...</td>\n      <td>387</td>\n      <td>{'Section': 'P ART I - DEFINITION', 'Page_No.'...</td>\n      <td>f. Continence - the ability to voluntarily con...</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Page 10</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>251</td>\n      <td>{'Section': 'T he legally recogni', 'Page_No.'...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>9</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Page 10</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>T he legally recognized union of two eligible ...</td>\n      <td>251</td>\n      <td>{'Section': 'T he legally recogni', 'Page_No.'...</td>\n      <td>2</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Page 11</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>299</td>\n      <td>{'Section': '(2) has been placed ', 'Page_No.'...</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>Page 11</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>(2) has been placed with the Member or spouse ...</td>\n      <td>299</td>\n      <td>{'Section': '(2) has been placed ', 'Page_No.'...</td>\n      <td>dependent on the Member for principal support....</td>\n      <td>12</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>Page 12</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>352</td>\n      <td>{'Section': 'An institution that ', 'Page_No.'...</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>13</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>Page 12</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>An institution that is licensed as a Hospital ...</td>\n      <td>352</td>\n      <td>{'Section': 'An institution that ', 'Page_No.'...</td>\n      <td>that, due to sickness or injury, the person is...</td>\n      <td>14</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>Page 13</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>260</td>\n      <td>{'Section': 'a . A licensed Docto', 'Page_No.'...</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>Page 13</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>260</td>\n      <td>{'Section': 'a . A licensed Docto', 'Page_No.'...</td>\n      <td>1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 5</td>\n      <td>16</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>Page 14</td>\n      <td>c . end stage renal failure; or</td>\n      <td>c . end stage renal failure; or d. acquired im...</td>\n      <td>316</td>\n      <td>{'Section': 'c . end stage renal ', 'Page_No.'...</td>\n      <td>c . end stage renal failure; or d. acquired im...</td>\n      <td>17</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>Page 14</td>\n      <td>c . end stage renal failure; or</td>\n      <td>c . end stage renal failure; or d. acquired im...</td>\n      <td>316</td>\n      <td>{'Section': 'c . end stage renal ', 'Page_No.'...</td>\n      <td>Benefits. Total Disability; Totally Disabled A...</td>\n      <td>18</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>Page 15</td>\n      <td>A record which is on or transmitted by paper o...</td>\n      <td>A record which is on or transmitted by paper o...</td>\n      <td>36</td>\n      <td>{'Section': 'A record which is on', 'Page_No.'...</td>\n      <td>A record which is on or transmitted by paper o...</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>Page 16</td>\n      <td>PART II - POLICY ADMINISTRATION</td>\n      <td>PART II - POLICY ADMINISTRATION Section A - Co...</td>\n      <td>325</td>\n      <td>{'Section': 'PART II - POLICY ADM', 'Page_No.'...</td>\n      <td>PART II - POLICY ADMINISTRATION Section A - Co...</td>\n      <td>20</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"# chunked_df['Metadata'] = chunked_df.apply(\n#     lambda x: {\n#         'Section': (x['Heading'][:20] if x['Heading'] else ''),\n#         'Page_No.': x['Page_Number'],\n#         'Chunk_ID':x['Chunk_ID']\n#     },\n#     axis=1\n# )","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.820862Z","iopub.execute_input":"2024-09-04T17:13:15.821413Z","iopub.status.idle":"2024-09-04T17:13:15.830279Z","shell.execute_reply.started":"2024-09-04T17:13:15.821327Z","shell.execute_reply":"2024-09-04T17:13:15.828952Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"filepath = \"/kaggle/input/my-openais-api/Session_1/OpenAI_API_Key.txt\"\n\nwith open(filepath + \"OpenAI_API_Key.txt\", \"r\") as f:\n  openai.api_key = ' '.join(f.readlines())","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.832662Z","iopub.execute_input":"2024-09-04T17:13:15.833182Z","iopub.status.idle":"2024-09-04T17:13:15.844289Z","shell.execute_reply.started":"2024-09-04T17:13:15.833135Z","shell.execute_reply":"2024-09-04T17:13:15.842753Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"\n\nchroma_data_path = '/kaggle/working/'\nclient = chromadb.PersistentClient(path=chroma_data_path)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:15.846648Z","iopub.execute_input":"2024-09-04T17:13:15.847177Z","iopub.status.idle":"2024-09-04T17:13:16.504655Z","shell.execute_reply.started":"2024-09-04T17:13:15.847131Z","shell.execute_reply":"2024-09-04T17:13:16.503205Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"embedding_function = OpenAIEmbeddingFunction(api_key=openai.api_key, model_name='text-embedding-ada-002')\ninsurance_collection = client.get_or_create_collection(name='InsurancePolicyDoc', embedding_function=embedding_function)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:16.506564Z","iopub.execute_input":"2024-09-04T17:13:16.507152Z","iopub.status.idle":"2024-09-04T17:13:16.587282Z","shell.execute_reply.started":"2024-09-04T17:13:16.507080Z","shell.execute_reply":"2024-09-04T17:13:16.585937Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"documents_list = chunked_df[\"Chunks\"].tolist()\nmetadata_list = chunked_df['Metadata'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:16.589058Z","iopub.execute_input":"2024-09-04T17:13:16.589500Z","iopub.status.idle":"2024-09-04T17:13:16.596783Z","shell.execute_reply.started":"2024-09-04T17:13:16.589443Z","shell.execute_reply":"2024-09-04T17:13:16.595019Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"insurance_collection.add(\n    documents= documents_list,\n    ids = [str(i) for i in range(0, len(documents_list))],\n    metadatas = metadata_list\n)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:16.598612Z","iopub.execute_input":"2024-09-04T17:13:16.599065Z","iopub.status.idle":"2024-09-04T17:13:17.545688Z","shell.execute_reply.started":"2024-09-04T17:13:16.599020Z","shell.execute_reply":"2024-09-04T17:13:17.544124Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"\ncache_collection = client.get_or_create_collection(name='Insurance_Cache', embedding_function=embedding_function)\ncache_collection.peek()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:17.547220Z","iopub.execute_input":"2024-09-04T17:13:17.547647Z","iopub.status.idle":"2024-09-04T17:13:17.595519Z","shell.execute_reply.started":"2024-09-04T17:13:17.547605Z","shell.execute_reply":"2024-09-04T17:13:17.594044Z"},"trusted":true},"execution_count":23,"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"{'ids': [],\n 'embeddings': [],\n 'metadatas': [],\n 'documents': [],\n 'uris': None,\n 'data': None,\n 'included': ['embeddings', 'metadatas', 'documents']}"},"metadata":{}}]},{"cell_type":"code","source":"query_1=input()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:13:17.597091Z","iopub.execute_input":"2024-09-04T17:13:17.597482Z","iopub.status.idle":"2024-09-04T17:15:48.916155Z","shell.execute_reply.started":"2024-09-04T17:13:17.597442Z","shell.execute_reply":"2024-09-04T17:15:48.914738Z"},"trusted":true},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdin","text":" what types of coverage does this policy include?\n"}]},{"cell_type":"code","source":"# Query the cache collection to check if the results are already stored\ncache_results = cache_collection.query(\n    query_texts=query_1,\n    n_results=1\n)\n\n# Print the results from the cache query for debugging\nprint(cache_results)\n\n# If the cache did not return satisfactory results (e.g., based on distance), query the main collection\nresults = insurance_collection.query(\n    query_texts=query_1,\n    n_results=10\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:15:48.917780Z","iopub.execute_input":"2024-09-04T17:15:48.918185Z","iopub.status.idle":"2024-09-04T17:15:49.361110Z","shell.execute_reply.started":"2024-09-04T17:15:48.918144Z","shell.execute_reply":"2024-09-04T17:15:49.359670Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"{'ids': [[]], 'distances': [[]], 'metadatas': [[]], 'embeddings': None, 'documents': [[]], 'uris': None, 'data': None, 'included': ['metadatas', 'documents', 'distances']}\n","output_type":"stream"}]},{"cell_type":"code","source":"import json\n\nthreshold = 0.2\n\nresults_df_1 = pd.DataFrame()\n\n# Query the cache collection to check if the results are already stored\ncache_results = cache_collection.query(\n    query_texts=query_1,\n    n_results=1\n)\n\n# Print the results from the cache query for debugging\nprint(cache_results)\n\n# Check if the cache is empty or if the distance exceeds the threshold\nif not cache_results['distances'][0] or cache_results['distances'][0][0] > threshold:\n    # Query the main collection for the top 10 results\n    results = insurance_collection.query(\n        query_texts=query_1,\n        n_results=10\n    )\n\n    # Prepare keys and values for storing in cache\n    cache_data = {}\n    for key, val in results.items():\n        if val is None:\n            continue\n        # Adjust the loop to match the actual number of items in val\n        for i in range(min(len(val[0]), 10)):  # Ensure you only loop over existing items\n            cache_data[f\"{key}_{i}\"] = val[0][i]\n\n    # Flatten the metadata for storage in ChromaDB\n    flat_cache_data = {}\n    for k, v in cache_data.items():\n        if isinstance(v, dict):\n            # Convert the dictionary to a JSON string\n            flat_cache_data[k] = json.dumps(v)\n        else:\n            flat_cache_data[k] = v\n\n    # Store the query in cache\n    cache_collection.add(\n        documents=[query_1],\n        ids=[query_1],  # Alternatively, you can use a unique ID\n        metadatas=flat_cache_data\n    )\n\n    print(\"Not found in cache. Found in main collection.\")\n\n    # Convert the results to a DataFrame\n    result_dict = {\n        'Metadatas_1': results['metadatas'][0],\n        'Documents_1': results['documents'][0],\n        'Distances_1': results['distances'][0],\n        'IDs': results['ids'][0]\n    }\n    results_df_1 = pd.DataFrame.from_dict(result_dict)\n\n# If the distance is within the threshold, retrieve results from the cache\nelif cache_results['distances'][0][0] <= threshold:\n    # Extract data from the cache\n    cache_result_dict = cache_results['metadatas'][0][0]\n    ids = []\n    documents = []\n    distances = []\n    metadatas = []\n\n    # Collect data based on keys\n    for key, value in cache_result_dict.items():\n        if 'ids' in key:\n            ids.append(value)\n        elif 'documents' in key:\n            documents.append(value)\n        elif 'distances' in key:\n            distances.append(value)\n        elif 'metadatas' in key:\n            metadatas.append(value)\n\n    print(\"Found in cache!\")\n\n    # Convert the cache data to a DataFrame\n    results_df_1 = pd.DataFrame({\n        'IDs_1': ids,\n        'Documents_1': documents,\n        'Distances_1': distances,\n        'Metadatas_1': metadatas\n    })\n\n# Display the DataFrame with results\n#print(results_df_1)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:15:49.363584Z","iopub.execute_input":"2024-09-04T17:15:49.364179Z","iopub.status.idle":"2024-09-04T17:15:49.829705Z","shell.execute_reply.started":"2024-09-04T17:15:49.364117Z","shell.execute_reply":"2024-09-04T17:15:49.825248Z"},"trusted":true},"execution_count":26,"outputs":[{"name":"stdout","text":"{'ids': [[]], 'distances': [[]], 'metadatas': [[]], 'embeddings': None, 'documents': [[]], 'uris': None, 'data': None, 'included': ['metadatas', 'documents', 'distances']}\nNot found in cache. Found in main collection.\n","output_type":"stream"}]},{"cell_type":"code","source":"results_df_1.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:15:49.831829Z","iopub.execute_input":"2024-09-04T17:15:49.832911Z","iopub.status.idle":"2024-09-04T17:15:49.853701Z","shell.execute_reply.started":"2024-09-04T17:15:49.832849Z","shell.execute_reply":"2024-09-04T17:15:49.852155Z"},"trusted":true},"execution_count":27,"outputs":[{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_1  \\\n0  {'Page_No.': 'Page 16', 'Section': 'PART II - ...   \n1     {'Page_No.': 'Page 55', 'Section': 'Exposure'}   \n2  {'Page_No.': 'Page 36', 'Section': 'A Member's...   \n3  {'Page_No.': 'Page 11', 'Section': '(2) has be...   \n4  {'Page_No.': 'Page 17', 'Section': 'a. be acti...   \n\n                                         Documents_1  Distances_1 IDs  \n0  coverage, benefits, and participation privileg...     0.377665  20  \n1  \"Automobile\" means a four-wheel passenger vehi...     0.383765  86  \n2  state or federal law. Article 5 - Coverage Whi...     0.387754  52  \n3  dependent on the Member for principal support....     0.388576  11  \n4  a. be actively engaged in business for profit ...     0.389656  21  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_1</th>\n      <th>Documents_1</th>\n      <th>Distances_1</th>\n      <th>IDs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 16', 'Section': 'PART II - ...</td>\n      <td>coverage, benefits, and participation privileg...</td>\n      <td>0.377665</td>\n      <td>20</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 55', 'Section': 'Exposure'}</td>\n      <td>\"Automobile\" means a four-wheel passenger vehi...</td>\n      <td>0.383765</td>\n      <td>86</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 36', 'Section': 'A Member's...</td>\n      <td>state or federal law. Article 5 - Coverage Whi...</td>\n      <td>0.387754</td>\n      <td>52</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>{'Page_No.': 'Page 11', 'Section': '(2) has be...</td>\n      <td>dependent on the Member for principal support....</td>\n      <td>0.388576</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>{'Page_No.': 'Page 17', 'Section': 'a. be acti...</td>\n      <td>a. be actively engaged in business for profit ...</td>\n      <td>0.389656</td>\n      <td>21</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"query_2=input()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:15:49.855932Z","iopub.execute_input":"2024-09-04T17:15:49.856509Z","iopub.status.idle":"2024-09-04T17:16:22.315903Z","shell.execute_reply.started":"2024-09-04T17:15:49.856442Z","shell.execute_reply":"2024-09-04T17:16:22.314319Z"},"trusted":true},"execution_count":28,"outputs":[{"output_type":"stream","name":"stdin","text":" what documentation is required when filing a claim?\n"}]},{"cell_type":"code","source":"cache_results= cache_collection.query(\n         query_texts=query_2,\n          n_results=1\n     )\ncache_results","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:22.317503Z","iopub.execute_input":"2024-09-04T17:16:22.317935Z","iopub.status.idle":"2024-09-04T17:16:22.564830Z","shell.execute_reply.started":"2024-09-04T17:16:22.317893Z","shell.execute_reply":"2024-09-04T17:16:22.563348Z"},"trusted":true},"execution_count":29,"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"{'ids': [['what types of coverage does this policy include?']],\n 'distances': [[0.39654462972718474]],\n 'metadatas': [[{'distances_0': 0.3776649363243732,\n    'distances_1': 0.38376540582557356,\n    'distances_2': 0.38775409643713393,\n    'distances_3': 0.388576212809206,\n    'distances_4': 0.38965634298891,\n    'distances_5': 0.39450742548817425,\n    'distances_6': 0.3948292433981987,\n    'distances_7': 0.39487230539362395,\n    'distances_8': 0.39498381181835585,\n    'distances_9': 0.3997019708508814,\n    'documents_0': \"coverage, benefits, and participation privileges, may be made without the consent of any Member or Dependent. Payment of premium beyond the effective date of the change constitutes the Policyholder's consent to the change. Article 3 - Policyholder Eligibility Requirements To be an eligible group and to remain an eligible group, the Policyholder must: This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 1\",\n    'documents_1': '\"Automobile\" means a four-wheel passenger vehicle, station wagon, pick-up truck, or van-type vehicle, but excludes recreational-type vehicles such as a \"dune-buggy\" or an \"all-terrain\" vehicle. The term \"Seat Belt\" means a factory-installed device that forms an occupant restraint and injury avoidance system. Article 5 - Loss of Use or Paralysis Benefit This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6015 Section B - Member Accidental Death and Dismemberment Insurance, Page 3',\n    'documents_2': 'state or federal law. Article 5 - Coverage While Outside of the United States If a Member or Dependent is temporarily outside the United States, the Member or Dependent may choose to continue his or her insurance, subject to premium payment for a period of six months or less for one of the following reasons: a. travel; or This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6008 Section C - Individual Terminations, Page 2',\n    'documents_3': 'dependent on the Member for principal support. Group Policy The policy of group insurance issued to the Policyholder by The Principal, which describes benefits and provisions for insured Members and Dependents. Hospital This policy has been updated effective January 1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 3',\n    'documents_4': \"a. be actively engaged in business for profit within the meaning of the Internal Revenue Code, or be established as a legitimate nonprofit corporation within the meaning of the Internal Revenue Code; and b. make at least the level of premium contributions required for insurance on its eligible Members. The Policyholder must: (1) contribute at least 50% of the required premium for all Members (including disabled Members, if any); and c. if the Member is to contribute part of the premium, maintain the following participation percentages with respect to eligible employees and Dependents, excluding those for whom Proof of Good Health is not satisfactory to The Principal: (1) Employees: - at least 75% of all eligible employees must enroll; (2) Dependents: - maintain a Dependent participation of at least 75% of eligible Dependents; and d. if the Member is to contribute no part of the premium, 100% of eligible employees and Dependents must enroll. Article 4 - Policy Incontestability In the absence of fraud, after this Group Policy has been in force two years, The Principal may not contest its validity except for nonpayment of premium. Article 5 - Individual Incontestability All statements made by any individual insured under this Group Policy will be representations and not warranties. In the absence of fraud, these statements may not be used to contest an insured person's insurance unless: a. the insured person's insurance has been in force for less than two years during the insured's lifetime; and b. the statement is in Written form Signed by the insured person; and This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 2\",\n    'documents_5': 'This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6013 Section A - Member Life Insurance, Page 6',\n    'documents_6': 'TABLE OF CONTENTS PART I - DEFINITIONS PART II - POLICY ADMINISTRATION Section A – Contract Entire Contract Article 1 Policy Changes Article 2 Policyholder Eligibility Requirements Article 3 Policy Incontestability Article 4 Individual Incontestability Article 5 Information to be Furnished Article 6 Certificates Article 7 Assignments Article 8 Dependent Rights Article 9 Policy Interpretation Article 10 Electronic Transactions Article 11 Section B – Premium Payment Responsibility; Due Dates; Grace Period Article 1 Premium Rates Article 2 Premium Rate Changes Article 3 Premium Amount Article 4 Contributions from Members Article 5 Section C - Policy Termination Failure to Pay Premium Article 1 Termination Rights of the Policyholder Article 2 Termination Rights of The Principal Article 3 Policyholder Responsibility to Members Article 4 Section D - Policy Renewal Renewal Article 1 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS This policy has been updated effective January 1, 2014 GC 6001 TABLE OF CONTENTS, PAGE 1',\n    'documents_7': \"a . A licensed Doctor of Medicine (M.D.) or Osteopathy (D.O.); or b. any other licensed health care practitioner that state law requires be recognized as a Physician under this Group Policy. The term Physician does not include the Member, an employee of the Member, a business or professional partner or associate of the Member, any person who has a financial affiliation or business interest with the Member, anyone related to the Member by blood or marriage, or anyone living in the Member's household. Policy Anniversary November 1, 2014 and the same day of each following year. Policyholder The entity to whom this Group Policy is issued (see Title Page). Prior Policy The Group Term Life coverage of either: a. the Policyholder; or b. a business entity which has been obtained by the Policyholder through a merger or acquisition; for which this Group Policy is a replacement. Proof of Good Health Written evidence that a person is insurable under the underwriting standards of The Principal. This proof must be provided in a form satisfactory to The Principal. Qualifying Event A Qualifying Event for Accelerated Benefits is a medical condition, which would, in the absence of extensive or extraordinary medical treatment; result in a dramatically limited life span. Such conditions may include, BUT ARE NOT LIMITED TO, one or more of the following: a. coronary artery disease resulting in an acute infarction or requiring surgery; b. permanent neurological deficit resulting from cerebral vascular accident; This policy has been updated effective January 1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 5\",\n    'documents_8': 'Exposure Exposure to the elements will be presumed to be an injury if: a. such exposure is due to an accidental bodily injury; and b. within 365 days after the injury, the Member incurs a loss that is the result of the exposure; and c. this Group Policy would have covered the injury resulting from the accident. Article 4 - Seat Belt/Airbag Benefit If the Member loses his or her life as a result of an accidental injury sustained while driving or riding in an Automobile, an additional benefit of $10,000 will be paid to the beneficiary named for Member Life Insurance, provided all Benefit Qualifications as described in Article 2 are met and: a. the Automobile is equipped with factory-installed Seat Belts; and b. the Seat Belt was in actual use by the Member and properly fastened at the time of the accident; and c. the position of the Seat Belt is certified in the official report of the accident or by the investigating officer. This additional benefit payment will also apply if the Member was driving an Automobile equipped with a properly functioning driver-side air bag or riding as a passenger in an Automobile equipped with a properly functioning passenger-side air bag, although the Member\\'s Seat Belt may not have been fastened at the time of the accident. The properly functioning and/or deployment of the air bag must be certified in the official report of the accident or by the investigating officer. For the purpose of this benefit, \"Automobile\" means a four-wheel passenger vehicle, station wagon, pick-up truck, or van-type vehicle, but excludes recreational-type vehicles such as a \"dune-buggy\" or an \"all-terrain\" vehicle. The term \"Seat Belt\" means a factory-installed device that forms an occupant restraint and injury avoidance system. Article 5 - Loss of Use or Paralysis',\n    'documents_9': 'to an individual policy; or (2) were eligible under the terms of the Prior Policy, to have their premiums waived due to ADL Disability or Total Disability. NOTE: When insurance under this Group Policy replaces coverage under a Prior Policy and the Active Work requirement is waived, any benefits payable will be the lesser of the Scheduled Benefit of this Group Policy or the amount that would have been paid by the Prior Policy had it remained in force. b. Effective Date for Initial Insurance When Proof of Good Health is Required This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6007 Section B - Effective Dates, Page 1',\n    'ids_0': '20',\n    'ids_1': '86',\n    'ids_2': '52',\n    'ids_3': '11',\n    'ids_4': '21',\n    'ids_5': '79',\n    'ids_6': '3',\n    'ids_7': '14',\n    'ids_8': '85',\n    'ids_9': '38',\n    'included_0': 'm',\n    'included_1': 'e',\n    'included_2': 't',\n    'included_3': 'a',\n    'included_4': 'd',\n    'included_5': 'a',\n    'included_6': 't',\n    'included_7': 'a',\n    'included_8': 's',\n    'metadatas_0': '{\"Page_No.\": \"Page 16\", \"Section\": \"PART II - POLICY ADM\"}',\n    'metadatas_1': '{\"Page_No.\": \"Page 55\", \"Section\": \"Exposure\"}',\n    'metadatas_2': '{\"Page_No.\": \"Page 36\", \"Section\": \"A Member\\'s insurance\"}',\n    'metadatas_3': '{\"Page_No.\": \"Page 11\", \"Section\": \"(2) has been placed \"}',\n    'metadatas_4': '{\"Page_No.\": \"Page 17\", \"Section\": \"a. be actively engag\"}',\n    'metadatas_5': '{\"Page_No.\": \"Page 51\", \"Section\": \"Coverage During Disa\"}',\n    'metadatas_6': '{\"Page_No.\": \"Page 6\", \"Section\": \"TABLE OF CONTENTS\"}',\n    'metadatas_7': '{\"Page_No.\": \"Page 13\", \"Section\": \"a . A licensed Docto\"}',\n    'metadatas_8': '{\"Page_No.\": \"Page 55\", \"Section\": \"Exposure\"}',\n    'metadatas_9': '{\"Page_No.\": \"Page 28\", \"Section\": \"Section B - Effectiv\"}'}]],\n 'embeddings': None,\n 'documents': [['what types of coverage does this policy include?']],\n 'uris': None,\n 'data': None,\n 'included': ['metadatas', 'documents', 'distances']}"},"metadata":{}}]},{"cell_type":"code","source":"threshold = 0.2\n\nresults_df_2 = pd.DataFrame()\n\n# Query the cache collection to check if the results are already stored\ncache_results = cache_collection.query(\n    query_texts=query_2,\n    n_results=1\n)\n\n# Print the results from the cache query for debugging\nprint(cache_results)\n\n# Check if the cache is empty or if the distance exceeds the threshold\nif not cache_results['distances'][0] or cache_results['distances'][0][0] > threshold:\n    # Query the main collection for the top 10 results\n    results = insurance_collection.query(\n        query_texts=query_2,\n        n_results=10\n    )\n\n    # Prepare keys and values for storing in cache\n    cache_data = {}\n    for key, val in results.items():\n        if val is None:\n            continue\n        # Adjust the loop to match the actual number of items in val\n        for i in range(min(len(val[0]), 10)):  # Ensure you only loop over existing items\n            cache_data[f\"{key}_{i}\"] = val[0][i]\n\n    # Flatten the metadata for storage in ChromaDB\n    flat_cache_data = {}\n    for k, v in cache_data.items():\n        if isinstance(v, dict):\n            # Convert the dictionary to a JSON string\n            flat_cache_data[k] = json.dumps(v)\n        else:\n            flat_cache_data[k] = v\n\n    # Store the query in cache\n    cache_collection.add(\n        documents=[query_2],\n        ids=[query_2],  # Alternatively, you can use a unique ID\n        metadatas=flat_cache_data\n    )\n\n    print(\"Not found in cache. Found in main collection.\")\n\n    # Convert the results to a DataFrame\n    result_dict = {\n        'Metadatas_2': results['metadatas'][0],\n        'Documents_2': results['documents'][0],\n        'Distances_2': results['distances'][0],\n        'IDs': results['ids'][0]\n    }\n    results_df_2 = pd.DataFrame.from_dict(result_dict)\n\n# If the distance is within the threshold, retrieve results from the cache\nelif cache_results['distances'][0][0] <= threshold:\n    # Extract data from the cache\n    cache_result_dict = cache_results['metadatas'][0][0]\n    ids = []\n    documents = []\n    distances = []\n    metadatas = []\n\n    # Collect data based on keys\n    for key, value in cache_result_dict.items():\n        if 'ids' in key:\n            ids.append(value)\n        elif 'documents' in key:\n            documents.append(value)\n        elif 'distances' in key:\n            distances.append(value)\n        elif 'metadatas' in key:\n            metadatas.append(value)\n\n    print(\"Found in cache!\")\n\n    # Convert the cache data to a DataFrame\n    results_df_2 = pd.DataFrame({\n        'IDs_2': ids,\n        'Documents_2': documents,\n        'Distances_2': distances,\n        'Metadatas_2': metadatas\n    })","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:22.566913Z","iopub.execute_input":"2024-09-04T17:16:22.567368Z","iopub.status.idle":"2024-09-04T17:16:23.191344Z","shell.execute_reply.started":"2024-09-04T17:16:22.567324Z","shell.execute_reply":"2024-09-04T17:16:23.190172Z"},"trusted":true},"execution_count":30,"outputs":[{"name":"stdout","text":"{'ids': [['what types of coverage does this policy include?']], 'distances': [[0.39654462972718474]], 'metadatas': [[{'distances_0': 0.3776649363243732, 'distances_1': 0.38376540582557356, 'distances_2': 0.38775409643713393, 'distances_3': 0.388576212809206, 'distances_4': 0.38965634298891, 'distances_5': 0.39450742548817425, 'distances_6': 0.3948292433981987, 'distances_7': 0.39487230539362395, 'distances_8': 0.39498381181835585, 'distances_9': 0.3997019708508814, 'documents_0': \"coverage, benefits, and participation privileges, may be made without the consent of any Member or Dependent. Payment of premium beyond the effective date of the change constitutes the Policyholder's consent to the change. Article 3 - Policyholder Eligibility Requirements To be an eligible group and to remain an eligible group, the Policyholder must: This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 1\", 'documents_1': '\"Automobile\" means a four-wheel passenger vehicle, station wagon, pick-up truck, or van-type vehicle, but excludes recreational-type vehicles such as a \"dune-buggy\" or an \"all-terrain\" vehicle. The term \"Seat Belt\" means a factory-installed device that forms an occupant restraint and injury avoidance system. Article 5 - Loss of Use or Paralysis Benefit This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6015 Section B - Member Accidental Death and Dismemberment Insurance, Page 3', 'documents_2': 'state or federal law. Article 5 - Coverage While Outside of the United States If a Member or Dependent is temporarily outside the United States, the Member or Dependent may choose to continue his or her insurance, subject to premium payment for a period of six months or less for one of the following reasons: a. travel; or This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6008 Section C - Individual Terminations, Page 2', 'documents_3': 'dependent on the Member for principal support. Group Policy The policy of group insurance issued to the Policyholder by The Principal, which describes benefits and provisions for insured Members and Dependents. Hospital This policy has been updated effective January 1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 3', 'documents_4': \"a. be actively engaged in business for profit within the meaning of the Internal Revenue Code, or be established as a legitimate nonprofit corporation within the meaning of the Internal Revenue Code; and b. make at least the level of premium contributions required for insurance on its eligible Members. The Policyholder must: (1) contribute at least 50% of the required premium for all Members (including disabled Members, if any); and c. if the Member is to contribute part of the premium, maintain the following participation percentages with respect to eligible employees and Dependents, excluding those for whom Proof of Good Health is not satisfactory to The Principal: (1) Employees: - at least 75% of all eligible employees must enroll; (2) Dependents: - maintain a Dependent participation of at least 75% of eligible Dependents; and d. if the Member is to contribute no part of the premium, 100% of eligible employees and Dependents must enroll. Article 4 - Policy Incontestability In the absence of fraud, after this Group Policy has been in force two years, The Principal may not contest its validity except for nonpayment of premium. Article 5 - Individual Incontestability All statements made by any individual insured under this Group Policy will be representations and not warranties. In the absence of fraud, these statements may not be used to contest an insured person's insurance unless: a. the insured person's insurance has been in force for less than two years during the insured's lifetime; and b. the statement is in Written form Signed by the insured person; and This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 2\", 'documents_5': 'This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6013 Section A - Member Life Insurance, Page 6', 'documents_6': 'TABLE OF CONTENTS PART I - DEFINITIONS PART II - POLICY ADMINISTRATION Section A – Contract Entire Contract Article 1 Policy Changes Article 2 Policyholder Eligibility Requirements Article 3 Policy Incontestability Article 4 Individual Incontestability Article 5 Information to be Furnished Article 6 Certificates Article 7 Assignments Article 8 Dependent Rights Article 9 Policy Interpretation Article 10 Electronic Transactions Article 11 Section B – Premium Payment Responsibility; Due Dates; Grace Period Article 1 Premium Rates Article 2 Premium Rate Changes Article 3 Premium Amount Article 4 Contributions from Members Article 5 Section C - Policy Termination Failure to Pay Premium Article 1 Termination Rights of the Policyholder Article 2 Termination Rights of The Principal Article 3 Policyholder Responsibility to Members Article 4 Section D - Policy Renewal Renewal Article 1 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS This policy has been updated effective January 1, 2014 GC 6001 TABLE OF CONTENTS, PAGE 1', 'documents_7': \"a . A licensed Doctor of Medicine (M.D.) or Osteopathy (D.O.); or b. any other licensed health care practitioner that state law requires be recognized as a Physician under this Group Policy. The term Physician does not include the Member, an employee of the Member, a business or professional partner or associate of the Member, any person who has a financial affiliation or business interest with the Member, anyone related to the Member by blood or marriage, or anyone living in the Member's household. Policy Anniversary November 1, 2014 and the same day of each following year. Policyholder The entity to whom this Group Policy is issued (see Title Page). Prior Policy The Group Term Life coverage of either: a. the Policyholder; or b. a business entity which has been obtained by the Policyholder through a merger or acquisition; for which this Group Policy is a replacement. Proof of Good Health Written evidence that a person is insurable under the underwriting standards of The Principal. This proof must be provided in a form satisfactory to The Principal. Qualifying Event A Qualifying Event for Accelerated Benefits is a medical condition, which would, in the absence of extensive or extraordinary medical treatment; result in a dramatically limited life span. Such conditions may include, BUT ARE NOT LIMITED TO, one or more of the following: a. coronary artery disease resulting in an acute infarction or requiring surgery; b. permanent neurological deficit resulting from cerebral vascular accident; This policy has been updated effective January 1, 2014 GC 6002 PART I - DEFINITIONS, PAGE 5\", 'documents_8': 'Exposure Exposure to the elements will be presumed to be an injury if: a. such exposure is due to an accidental bodily injury; and b. within 365 days after the injury, the Member incurs a loss that is the result of the exposure; and c. this Group Policy would have covered the injury resulting from the accident. Article 4 - Seat Belt/Airbag Benefit If the Member loses his or her life as a result of an accidental injury sustained while driving or riding in an Automobile, an additional benefit of $10,000 will be paid to the beneficiary named for Member Life Insurance, provided all Benefit Qualifications as described in Article 2 are met and: a. the Automobile is equipped with factory-installed Seat Belts; and b. the Seat Belt was in actual use by the Member and properly fastened at the time of the accident; and c. the position of the Seat Belt is certified in the official report of the accident or by the investigating officer. This additional benefit payment will also apply if the Member was driving an Automobile equipped with a properly functioning driver-side air bag or riding as a passenger in an Automobile equipped with a properly functioning passenger-side air bag, although the Member\\'s Seat Belt may not have been fastened at the time of the accident. The properly functioning and/or deployment of the air bag must be certified in the official report of the accident or by the investigating officer. For the purpose of this benefit, \"Automobile\" means a four-wheel passenger vehicle, station wagon, pick-up truck, or van-type vehicle, but excludes recreational-type vehicles such as a \"dune-buggy\" or an \"all-terrain\" vehicle. The term \"Seat Belt\" means a factory-installed device that forms an occupant restraint and injury avoidance system. Article 5 - Loss of Use or Paralysis', 'documents_9': 'to an individual policy; or (2) were eligible under the terms of the Prior Policy, to have their premiums waived due to ADL Disability or Total Disability. NOTE: When insurance under this Group Policy replaces coverage under a Prior Policy and the Active Work requirement is waived, any benefits payable will be the lesser of the Scheduled Benefit of this Group Policy or the amount that would have been paid by the Prior Policy had it remained in force. b. Effective Date for Initial Insurance When Proof of Good Health is Required This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6007 Section B - Effective Dates, Page 1', 'ids_0': '20', 'ids_1': '86', 'ids_2': '52', 'ids_3': '11', 'ids_4': '21', 'ids_5': '79', 'ids_6': '3', 'ids_7': '14', 'ids_8': '85', 'ids_9': '38', 'included_0': 'm', 'included_1': 'e', 'included_2': 't', 'included_3': 'a', 'included_4': 'd', 'included_5': 'a', 'included_6': 't', 'included_7': 'a', 'included_8': 's', 'metadatas_0': '{\"Page_No.\": \"Page 16\", \"Section\": \"PART II - POLICY ADM\"}', 'metadatas_1': '{\"Page_No.\": \"Page 55\", \"Section\": \"Exposure\"}', 'metadatas_2': '{\"Page_No.\": \"Page 36\", \"Section\": \"A Member\\'s insurance\"}', 'metadatas_3': '{\"Page_No.\": \"Page 11\", \"Section\": \"(2) has been placed \"}', 'metadatas_4': '{\"Page_No.\": \"Page 17\", \"Section\": \"a. be actively engag\"}', 'metadatas_5': '{\"Page_No.\": \"Page 51\", \"Section\": \"Coverage During Disa\"}', 'metadatas_6': '{\"Page_No.\": \"Page 6\", \"Section\": \"TABLE OF CONTENTS\"}', 'metadatas_7': '{\"Page_No.\": \"Page 13\", \"Section\": \"a . A licensed Docto\"}', 'metadatas_8': '{\"Page_No.\": \"Page 55\", \"Section\": \"Exposure\"}', 'metadatas_9': '{\"Page_No.\": \"Page 28\", \"Section\": \"Section B - Effectiv\"}'}]], 'embeddings': None, 'documents': [['what types of coverage does this policy include?']], 'uris': None, 'data': None, 'included': ['metadatas', 'documents', 'distances']}\nNot found in cache. Found in main collection.\n","output_type":"stream"}]},{"cell_type":"code","source":"results_df_2","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:23.193230Z","iopub.execute_input":"2024-09-04T17:16:23.193741Z","iopub.status.idle":"2024-09-04T17:16:23.218346Z","shell.execute_reply.started":"2024-09-04T17:16:23.193687Z","shell.execute_reply":"2024-09-04T17:16:23.216392Z"},"trusted":true},"execution_count":31,"outputs":[{"execution_count":31,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_2  \\\n0  {'Page_No.': 'Page 61', 'Section': 'Section D ...   \n1  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n2  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n3  {'Page_No.': 'Page 61', 'Section': 'Section D ...   \n4  {'Page_No.': 'Page 29', 'Section': 'Insurance ...   \n5  {'Page_No.': 'Page 18', 'Section': 'c . a copy...   \n6  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n7  {'Page_No.': 'Page 28', 'Section': 'Section B ...   \n8  {'Page_No.': 'Page 17', 'Section': 'a. be acti...   \n9  {'Page_No.': 'Page 31', 'Section': 'Scheduled ...   \n\n                                         Documents_2  Distances_2 IDs  \n0  Section D - Claim Procedures Article 1 - Notic...     0.335960  95  \n1  of loss has been filed and before the appeal p...     0.354624  98  \n2  A claimant may request an appeal of a claim de...     0.362261  97  \n3  will be considered to be met when the appropri...     0.366767  96  \n4  by The Principal. A Member must submit Proof o...     0.386173  40  \n5  c . a copy of the form which contains the stat...     0.390515  23  \n6  f . claim requirements listed in PART IV, Sect...     0.401005  83  \n7  to an individual policy; or (2) were eligible ...     0.409296  38  \n8  Written form Signed by the insured person; and...     0.414463  22  \n9  Benefit amounts due to a request by the Member...     0.414665  44  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_2</th>\n      <th>Documents_2</th>\n      <th>Distances_2</th>\n      <th>IDs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 61', 'Section': 'Section D ...</td>\n      <td>Section D - Claim Procedures Article 1 - Notic...</td>\n      <td>0.335960</td>\n      <td>95</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>of loss has been filed and before the appeal p...</td>\n      <td>0.354624</td>\n      <td>98</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>A claimant may request an appeal of a claim de...</td>\n      <td>0.362261</td>\n      <td>97</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>{'Page_No.': 'Page 61', 'Section': 'Section D ...</td>\n      <td>will be considered to be met when the appropri...</td>\n      <td>0.366767</td>\n      <td>96</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>{'Page_No.': 'Page 29', 'Section': 'Insurance ...</td>\n      <td>by The Principal. A Member must submit Proof o...</td>\n      <td>0.386173</td>\n      <td>40</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>{'Page_No.': 'Page 18', 'Section': 'c . a copy...</td>\n      <td>c . a copy of the form which contains the stat...</td>\n      <td>0.390515</td>\n      <td>23</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>f . claim requirements listed in PART IV, Sect...</td>\n      <td>0.401005</td>\n      <td>83</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>{'Page_No.': 'Page 28', 'Section': 'Section B ...</td>\n      <td>to an individual policy; or (2) were eligible ...</td>\n      <td>0.409296</td>\n      <td>38</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>{'Page_No.': 'Page 17', 'Section': 'a. be acti...</td>\n      <td>Written form Signed by the insured person; and...</td>\n      <td>0.414463</td>\n      <td>22</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>{'Page_No.': 'Page 31', 'Section': 'Scheduled ...</td>\n      <td>Benefit amounts due to a request by the Member...</td>\n      <td>0.414665</td>\n      <td>44</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"query_3 = input()","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:23.220122Z","iopub.execute_input":"2024-09-04T17:16:23.220675Z","iopub.status.idle":"2024-09-04T17:16:43.612562Z","shell.execute_reply.started":"2024-09-04T17:16:23.220616Z","shell.execute_reply":"2024-09-04T17:16:43.611003Z"},"trusted":true},"execution_count":32,"outputs":[{"output_type":"stream","name":"stdin","text":" what happens if i miss a payment?\n"}]},{"cell_type":"code","source":"cache_results= cache_collection.query(\n         query_texts=query_3,\n          n_results=1\n     )\ncache_results","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:43.614567Z","iopub.execute_input":"2024-09-04T17:16:43.615081Z","iopub.status.idle":"2024-09-04T17:16:43.843612Z","shell.execute_reply.started":"2024-09-04T17:16:43.615034Z","shell.execute_reply":"2024-09-04T17:16:43.842335Z"},"trusted":true},"execution_count":33,"outputs":[{"execution_count":33,"output_type":"execute_result","data":{"text/plain":"{'ids': [['what documentation is required when filing a claim?']],\n 'distances': [[0.4857106171859851]],\n 'metadatas': [[{'distances_0': 0.3359597039883748,\n    'distances_1': 0.35462433451933106,\n    'distances_2': 0.3622610727387903,\n    'distances_3': 0.3667674024181407,\n    'distances_4': 0.3861733920826373,\n    'distances_5': 0.39051496084501053,\n    'distances_6': 0.4010045736094843,\n    'distances_7': 0.4092963392539709,\n    'distances_8': 0.4144634866813787,\n    'distances_9': 0.4146653561842776,\n    'documents_0': 'Section D - Claim Procedures Article 1 - Notice of Claim Written notice must be sent to The Principal by or for a Member or Dependent who wishes to file claim for benefits under this Group Policy. This notice must be sent within 20 days after the date of the loss for which claim is being made. Failure to give notice within the time specified will not invalidate or reduce any claim if notice is given as soon as reasonably possible. Article 2 - Claim Forms The Principal, when it receives notice of claim, will provide appropriate claim forms for filing proof of loss. If the forms are not provided within 15 days after The Principal receives notice, the person will be considered to have complied with the requirements of this Group Policy upon submitting, within the time specified below for filing proof of loss, Written proof covering the occurrence, character, and extent of the loss. Article 3 - Proof of Loss Written proof of loss must be sent to The Principal within 90 days after the date of the loss. Proof required includes the date, nature, and extent of the loss. The Principal may request additional information to substantiate loss or require a Signed unaltered authorization to obtain that information from the provider. Failure to comply with the request of The Principal could result in declination of the claim. For purposes of satisfying the claims processing timing requirements of the Employee Retirement Income Security Act (ERISA), receipt of claim will be considered to be met when the appropriate claim form is received by The Principal. Article 4 - Payment, Denial, and Review ERISA permits up to 45 days from receipt of claim for processing the claim. If a claim cannot be processed due to incomplete information, The Principal will',\n    'documents_1': 'of loss has been filed and before the appeal procedures have been exhausted. Further, no legal action may be started later than three years after that proof is required to be filed. Article 8 - Time Limits Any time limits in this section will be adjusted as required by law. This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6018 Section D - Claim Procedures, Page 2',\n    'documents_2': 'A claimant may request an appeal of a claim denial by Written request to The Principal within 180 days of receipt of notice of the denial. The Principal will make a full and fair review of the claim. The Principal may require additional information to make the review. The Principal will notify the claimant in Writing of the appeal decision within 45 days after receipt of the appeal request. If the appeal cannot be processed within the 45-day period because The Principal did not receive the requested additional information, The Principal is permitted a 45-day extension for the review. Written notification will be sent to the claimant regarding the extension. After exhaustion of the formal appeal process, the claimant may request an additional appeal. However, this appeal is voluntary and does not need to be filed before asserting rights to legal action. For purposes of this section, \"claimant\" means Member, Dependent, or Beneficiary. Article 5 - Medical Examinations The Principal may have the Member or Dependent whose loss is the basis for claim, be examined by a Physician during the course of a claim. The Principal will pay for these examinations and will choose the Physician to perform them. Article 6 - Autopsy If payment for loss of life is claimed, The Principal may require an autopsy. The Principal will pay for any such autopsy. Article 7 - Legal Action Legal action to recover benefits under this Group Policy may not be started earlier than 90 days after required proof of loss has been filed and before the appeal procedures have been exhausted. Further, no legal action may be started later than three years after that proof is required to be filed. Article 8 - Time Limits Any time limits in this section will be adjusted as required by law.',\n    'documents_3': 'will be considered to be met when the appropriate claim form is received by The Principal. Article 4 - Payment, Denial, and Review ERISA permits up to 45 days from receipt of claim for processing the claim. If a claim cannot be processed due to incomplete information, The Principal will send a Written explanation prior to the expiration of the 45 days. The claimant is then allowed up to 45 days to provide all additional information requested. The Principal is permitted two 30-day extensions for processing an incomplete claim. Written notification will be sent to the claimant regarding the extension. In actual practice, benefits under this Group Policy will be payable sooner, provided The Principal receives complete and proper proof of loss. Further, if a claim is not payable or cannot be processed, The Principal will submit a detailed explanation of the basis for its denial. This policy has been updated effective January 1, 2014 PART IV - BENEFITS GC 6018 Section D - Claim Procedures, Page 1',\n    'documents_4': 'by The Principal. A Member must submit Proof of Good Health: (1) If insurance for which a Member contributes a part of premium is requested more than 31 days after the date the Member is eligible including any insurance the Member refuses and later requests. (2) If insurance is requested under this Group Policy by a Member that was eligible under the Prior Policy, but elected to waive coverage under the Prior Policy. (3) If a Member has failed to provide required Proof of Good Health or has been refused insurance under this Group Policy at any prior time. (4) If a Member elects to terminate insurance and, more than 31 days later, requests to be insured again. (5) If, on the date a Member becomes eligible, fewer than five Members are insured. This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6007 Section B - Effective Dates, Page 2',\n    'documents_5': \"c . a copy of the form which contains the statement is given to the insured or the insured's beneficiary at the time insurance is contested. However, these provisions will not preclude the assertion at any time of defenses based upon the person's ineligibility for insurance under this Group Policy or upon the provisions of this Group Policy. In addition, if an individual's age is misstated, The Principal may at any time adjust premium and benefits to reflect the correct age. Article 6 - Information to be Furnished The Policyholder must, upon request, give The Principal all information needed to administer this Group Policy. If a clerical error is found in this information, The Principal may at any time adjust premium to reflect the facts. An error will not invalidate insurance that would otherwise be in force. Neither will an error continue insurance that would otherwise be terminated. The Principal may inspect, at any reasonable time, all Policyholder records, which relate to this Group Policy. Article 7 - Certificates The Principal will give the Policyholder Certificates for delivery to insured Members. The delivery of such Certificates will be in either paper or electronic format. The Certificates will be evidence of insurance and will describe the basic features of the coverage. They will not be considered a part of this Group Policy. Article 8 - Assignments No assignments of Member Life Insurance will be allowed under this Group Policy. Article 9 - Dependent Rights A Dependent will have no rights under this Group Policy except as set forth in PART III, Section F, Article 2. Article 10 - Policy Interpretation This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 3\",\n    'documents_6': \"f . claim requirements listed in PART IV, Section D, must be satisfied; and g. all medical evidence must be satisfactory to The Principal. Article 3 - Benefits Payable If all of the benefit qualifications are met, The Principal will pay: a. 100% of the Scheduled Benefit (or approved amount, if applicable) in force for loss of life; or b. 50% of the Scheduled Benefit (or approved amount, if applicable) in force if one hand is severed at or above the wrist; or c. 25% of the Scheduled Benefit (or approved amount, if applicable) in force for loss of thumb and index finger on the same hand; or d. 50% of the Scheduled Benefit (or approved amount, if applicable) in force if one foot is severed at or above the ankle; or e. 50% of the Scheduled Benefit (or approved amount, if applicable) in force if the sight of one eye is permanently lost (For this purpose, vision not correctable to better than 20/200 will be considered loss of sight.); or f. 100% of the Scheduled Benefit (or approved amount, if applicable) in force for more than one of the losses listed in b., d., or e. above. Total payment for all losses under this Article 3 that result from the same accident will not exceed the Scheduled Benefit (or approved amount, if applicable). Payment for loss of life will be to the beneficiary named for Member Life Insurance. Payment will be subject to the Beneficiary, Facility of Payment and Settlement of Proceeds provisions of PART IV, Section A. Payment for all other losses will be to the Member. Disappearance It will be presumed that a Member has lost his or her life if: a. the Member's body has not been found within 365 days after the disappearance of a\",\n    'documents_7': 'to an individual policy; or (2) were eligible under the terms of the Prior Policy, to have their premiums waived due to ADL Disability or Total Disability. NOTE: When insurance under this Group Policy replaces coverage under a Prior Policy and the Active Work requirement is waived, any benefits payable will be the lesser of the Scheduled Benefit of this Group Policy or the amount that would have been paid by the Prior Policy had it remained in force. b. Effective Date for Initial Insurance When Proof of Good Health is Required This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6007 Section B - Effective Dates, Page 1',\n    'documents_8': 'Written form Signed by the insured person; and This policy has been updated effective January 1, 2014 PART II - POLICY ADMINISTRATION GC 6003 Section A - Contract, Page 2',\n    'documents_9': \"Benefit amounts due to a request by the Member will be effective on the date of the change, whether or not the Member is Actively at Work. (2) A change in the Member's Scheduled Benefit amount because of a request by the Member for which Proof of Good Health is required (see e. above) will be effective on the later of: - the date the change would have been effective if Proof of Good Health had not been required; or - the date Proof of Good Health is approved by The Principal. i. Effective Date for Benefit Changes Due to Change in the Member's Family Status A Member may request an increase in Scheduled Benefit amount, a decrease in Scheduled Benefits, or the addition of Scheduled Benefits for which he or she was not previously insured if a change in the Member's family status as described below has occurred, provided a request for such increase, decrease, or addition is made in Writing within 31 days after the date of the change in family status: This policy has been updated effective January 1, 2014 PART III - INDIVIDUAL REQUIREMENTS AND RIGHTS GC 6007 Section B - Effective Dates, Page 4\",\n    'ids_0': '95',\n    'ids_1': '98',\n    'ids_2': '97',\n    'ids_3': '96',\n    'ids_4': '40',\n    'ids_5': '23',\n    'ids_6': '83',\n    'ids_7': '38',\n    'ids_8': '22',\n    'ids_9': '44',\n    'included_0': 'm',\n    'included_1': 'e',\n    'included_2': 't',\n    'included_3': 'a',\n    'included_4': 'd',\n    'included_5': 'a',\n    'included_6': 't',\n    'included_7': 'a',\n    'included_8': 's',\n    'metadatas_0': '{\"Page_No.\": \"Page 61\", \"Section\": \"Section D - Claim Pr\"}',\n    'metadatas_1': '{\"Page_No.\": \"Page 62\", \"Section\": \"A claimant may reque\"}',\n    'metadatas_2': '{\"Page_No.\": \"Page 62\", \"Section\": \"A claimant may reque\"}',\n    'metadatas_3': '{\"Page_No.\": \"Page 61\", \"Section\": \"Section D - Claim Pr\"}',\n    'metadatas_4': '{\"Page_No.\": \"Page 29\", \"Section\": \"Insurance for which \"}',\n    'metadatas_5': '{\"Page_No.\": \"Page 18\", \"Section\": \"c . a copy of the fo\"}',\n    'metadatas_6': '{\"Page_No.\": \"Page 54\", \"Section\": \"f . claim requiremen\"}',\n    'metadatas_7': '{\"Page_No.\": \"Page 28\", \"Section\": \"Section B - Effectiv\"}',\n    'metadatas_8': '{\"Page_No.\": \"Page 17\", \"Section\": \"a. be actively engag\"}',\n    'metadatas_9': '{\"Page_No.\": \"Page 31\", \"Section\": \"Scheduled Benefit in\"}'}]],\n 'embeddings': None,\n 'documents': [['what documentation is required when filing a claim?']],\n 'uris': None,\n 'data': None,\n 'included': ['metadatas', 'documents', 'distances']}"},"metadata":{}}]},{"cell_type":"code","source":"threshold = 0.2\n\nresults_df_3 = pd.DataFrame()\n\n# Query the cache collection to check if the results are already stored\ncache_results = cache_collection.query(\n    query_texts=query_3,\n    n_results=1\n)\n\n# Print the results from the cache query for debugging\n#print(cache_results)\n\n# Check if the cache is empty or if the distance exceeds the threshold\nif not cache_results['distances'][0] or cache_results['distances'][0][0] > threshold:\n    # Query the main collection for the top 10 results\n    results = insurance_collection.query(\n        query_texts=query_3,\n        n_results=10\n    )\n\n    # Prepare keys and values for storing in cache\n    cache_data = {}\n    for key, val in results.items():\n        if val is None:\n            continue\n        # Adjust the loop to match the actual number of items in val\n        for i in range(min(len(val[0]), 10)):  # Ensure you only loop over existing items\n            cache_data[f\"{key}_{i}\"] = val[0][i]\n\n    # Flatten the metadata for storage in ChromaDB\n    flat_cache_data = {}\n    for k, v in cache_data.items():\n        if isinstance(v, dict):\n            # Convert the dictionary to a JSON string\n            flat_cache_data[k] = json.dumps(v)\n        else:\n            flat_cache_data[k] = v\n\n    # Store the query in cache\n    cache_collection.add(\n        documents=[query_3],\n        ids=[query_3],  # Alternatively, you can use a unique ID\n        metadatas=flat_cache_data\n    )\n\n    print(\"Not found in cache. Found in main collection.\")\n\n    # Convert the results to a DataFrame\n    result_dict = {\n        'Metadatas_3': results['metadatas'][0],\n        'Documents_3': results['documents'][0],\n        'Distances_3': results['distances'][0],\n        'IDs': results['ids'][0]\n    }\n    results_df_3 = pd.DataFrame.from_dict(result_dict)\n\n# If the distance is within the threshold, retrieve results from the cache\nelif cache_results['distances'][0][0] <= threshold:\n    # Extract data from the cache\n    cache_result_dict = cache_results['metadatas'][0][0]\n    ids = []\n    documents = []\n    distances = []\n    metadatas = []\n\n    # Collect data based on keys\n    for key, value in cache_result_dict.items():\n        if 'ids' in key:\n            ids.append(value)\n        elif 'documents' in key:\n            documents.append(value)\n        elif 'distances' in key:\n            distances.append(value)\n        elif 'metadatas' in key:\n            metadatas.append(value)\n\n    print(\"Found in cache!\")\n\n    # Convert the cache data to a DataFrame\n    results_df_3 = pd.DataFrame({\n        'IDs_3': ids,\n        'Documents_3': documents,\n        'Distances_3': distances,\n        'Metadatas_3': metadatas\n    })","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:43.845857Z","iopub.execute_input":"2024-09-04T17:16:43.846295Z","iopub.status.idle":"2024-09-04T17:16:44.405268Z","shell.execute_reply.started":"2024-09-04T17:16:43.846251Z","shell.execute_reply":"2024-09-04T17:16:44.403506Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"Not found in cache. Found in main collection.\n","output_type":"stream"}]},{"cell_type":"code","source":"results_df_3","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:44.407230Z","iopub.execute_input":"2024-09-04T17:16:44.407721Z","iopub.status.idle":"2024-09-04T17:16:44.433098Z","shell.execute_reply.started":"2024-09-04T17:16:44.407673Z","shell.execute_reply":"2024-09-04T17:16:44.431478Z"},"trusted":true},"execution_count":35,"outputs":[{"execution_count":35,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_3  \\\n0  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n1  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n2  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n3  {'Page_No.': 'Page 20', 'Section': 'Section B ...   \n4  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n5  {'Page_No.': 'Page 24', 'Section': 'T he Princ...   \n6  {'Page_No.': 'Page 47', 'Section': 'M ember's ...   \n7  {'Page_No.': 'Page 60', 'Section': 'I f a Depe...   \n8  {'Page_No.': 'Page 20', 'Section': 'Section B ...   \n9  {'Page_No.': 'Page 48', 'Section': 'c . If a b...   \n\n                                         Documents_3  Distances_3 IDs  \n0  or d. fails to pay premium in accordance with ...     0.425369  32  \n1  Settlement of Proceeds provisions of PART IV, ...     0.451775  84  \n2  Section C - Policy Termination Article 1 - Fai...     0.453811  31  \n3  Section B - Premiums Article 1 - Payment Respo...     0.467580  26  \n4  f . claim requirements listed in PART IV, Sect...     0.472797  83  \n5  T he Principal may terminate the Policyholder'...     0.476565  33  \n6  insurance and recorded by the Policyholder or ...     0.477835  71  \n7  before a change request is received, that paym...     0.479992  94  \n8  Premium Rate Changes The Principal may change ...     0.483176  27  \n9  c . If a beneficiary dies at the same time or ...     0.483333  72  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_3</th>\n      <th>Documents_3</th>\n      <th>Distances_3</th>\n      <th>IDs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>or d. fails to pay premium in accordance with ...</td>\n      <td>0.425369</td>\n      <td>32</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>Settlement of Proceeds provisions of PART IV, ...</td>\n      <td>0.451775</td>\n      <td>84</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>Section C - Policy Termination Article 1 - Fai...</td>\n      <td>0.453811</td>\n      <td>31</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>{'Page_No.': 'Page 20', 'Section': 'Section B ...</td>\n      <td>Section B - Premiums Article 1 - Payment Respo...</td>\n      <td>0.467580</td>\n      <td>26</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>f . claim requirements listed in PART IV, Sect...</td>\n      <td>0.472797</td>\n      <td>83</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>{'Page_No.': 'Page 24', 'Section': 'T he Princ...</td>\n      <td>T he Principal may terminate the Policyholder'...</td>\n      <td>0.476565</td>\n      <td>33</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>{'Page_No.': 'Page 47', 'Section': 'M ember's ...</td>\n      <td>insurance and recorded by the Policyholder or ...</td>\n      <td>0.477835</td>\n      <td>71</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>{'Page_No.': 'Page 60', 'Section': 'I f a Depe...</td>\n      <td>before a change request is received, that paym...</td>\n      <td>0.479992</td>\n      <td>94</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>{'Page_No.': 'Page 20', 'Section': 'Section B ...</td>\n      <td>Premium Rate Changes The Principal may change ...</td>\n      <td>0.483176</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>{'Page_No.': 'Page 48', 'Section': 'c . If a b...</td>\n      <td>c . If a beneficiary dies at the same time or ...</td>\n      <td>0.483333</td>\n      <td>72</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:44.435428Z","iopub.execute_input":"2024-09-04T17:16:44.435974Z","iopub.status.idle":"2024-09-04T17:16:47.750589Z","shell.execute_reply.started":"2024-09-04T17:16:44.435913Z","shell.execute_reply":"2024-09-04T17:16:47.749321Z"},"trusted":true},"execution_count":36,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/794 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"17abe084eae34909b89ffb90de10f97e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"pytorch_model.bin:   0%|          | 0.00/90.9M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ee264c1a333b4552b4034cfb83fbc903"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/316 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"74d2929fb0de46a1888753a47f3aff18"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f369acb3336345daac675d67d6acb511"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6739924be0014bf582a28a8cb90e8e91"}},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n  warnings.warn(\n","output_type":"stream"}]},{"cell_type":"code","source":"results_df_3","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:47.752493Z","iopub.execute_input":"2024-09-04T17:16:47.753026Z","iopub.status.idle":"2024-09-04T17:16:47.776246Z","shell.execute_reply.started":"2024-09-04T17:16:47.752958Z","shell.execute_reply":"2024-09-04T17:16:47.774594Z"},"trusted":true},"execution_count":37,"outputs":[{"execution_count":37,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_3  \\\n0  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n1  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n2  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n3  {'Page_No.': 'Page 20', 'Section': 'Section B ...   \n4  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n5  {'Page_No.': 'Page 24', 'Section': 'T he Princ...   \n6  {'Page_No.': 'Page 47', 'Section': 'M ember's ...   \n7  {'Page_No.': 'Page 60', 'Section': 'I f a Depe...   \n8  {'Page_No.': 'Page 20', 'Section': 'Section B ...   \n9  {'Page_No.': 'Page 48', 'Section': 'c . If a b...   \n\n                                         Documents_3  Distances_3 IDs  \n0  or d. fails to pay premium in accordance with ...     0.425369  32  \n1  Settlement of Proceeds provisions of PART IV, ...     0.451775  84  \n2  Section C - Policy Termination Article 1 - Fai...     0.453811  31  \n3  Section B - Premiums Article 1 - Payment Respo...     0.467580  26  \n4  f . claim requirements listed in PART IV, Sect...     0.472797  83  \n5  T he Principal may terminate the Policyholder'...     0.476565  33  \n6  insurance and recorded by the Policyholder or ...     0.477835  71  \n7  before a change request is received, that paym...     0.479992  94  \n8  Premium Rate Changes The Principal may change ...     0.483176  27  \n9  c . If a beneficiary dies at the same time or ...     0.483333  72  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_3</th>\n      <th>Documents_3</th>\n      <th>Distances_3</th>\n      <th>IDs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>or d. fails to pay premium in accordance with ...</td>\n      <td>0.425369</td>\n      <td>32</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>Settlement of Proceeds provisions of PART IV, ...</td>\n      <td>0.451775</td>\n      <td>84</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>Section C - Policy Termination Article 1 - Fai...</td>\n      <td>0.453811</td>\n      <td>31</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>{'Page_No.': 'Page 20', 'Section': 'Section B ...</td>\n      <td>Section B - Premiums Article 1 - Payment Respo...</td>\n      <td>0.467580</td>\n      <td>26</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>f . claim requirements listed in PART IV, Sect...</td>\n      <td>0.472797</td>\n      <td>83</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>{'Page_No.': 'Page 24', 'Section': 'T he Princ...</td>\n      <td>T he Principal may terminate the Policyholder'...</td>\n      <td>0.476565</td>\n      <td>33</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>{'Page_No.': 'Page 47', 'Section': 'M ember's ...</td>\n      <td>insurance and recorded by the Policyholder or ...</td>\n      <td>0.477835</td>\n      <td>71</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>{'Page_No.': 'Page 60', 'Section': 'I f a Depe...</td>\n      <td>before a change request is received, that paym...</td>\n      <td>0.479992</td>\n      <td>94</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>{'Page_No.': 'Page 20', 'Section': 'Section B ...</td>\n      <td>Premium Rate Changes The Principal may change ...</td>\n      <td>0.483176</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>{'Page_No.': 'Page 48', 'Section': 'c . If a b...</td>\n      <td>c . If a beneficiary dies at the same time or ...</td>\n      <td>0.483333</td>\n      <td>72</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"cross_inputs_1 = [[query_1, response] for response in results_df_1['Documents_1']]\ncross_rerank_scores_1 = cross_encoder.predict(cross_inputs_1)\ncross_rerank_scores_1","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:47.778373Z","iopub.execute_input":"2024-09-04T17:16:47.778992Z","iopub.status.idle":"2024-09-04T17:16:49.230330Z","shell.execute_reply.started":"2024-09-04T17:16:47.778933Z","shell.execute_reply":"2024-09-04T17:16:49.228979Z"},"trusted":true},"execution_count":38,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"82bdc977bef14888a770688d8e3e2e87"}},"metadata":{}},{"execution_count":38,"output_type":"execute_result","data":{"text/plain":"array([-2.1723294, -5.0158787, -1.9306265, -2.5475593, -8.101703 ,\n       -5.111147 , -9.388777 , -1.8970793, -4.3696127, -3.4865303],\n      dtype=float32)"},"metadata":{}}]},{"cell_type":"code","source":"results_df_1['Reranked_scores'] = cross_rerank_scores_1\nresults_df_1","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:49.232076Z","iopub.execute_input":"2024-09-04T17:16:49.232553Z","iopub.status.idle":"2024-09-04T17:16:49.258755Z","shell.execute_reply.started":"2024-09-04T17:16:49.232504Z","shell.execute_reply":"2024-09-04T17:16:49.256933Z"},"trusted":true},"execution_count":39,"outputs":[{"execution_count":39,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_1  \\\n0  {'Page_No.': 'Page 16', 'Section': 'PART II - ...   \n1     {'Page_No.': 'Page 55', 'Section': 'Exposure'}   \n2  {'Page_No.': 'Page 36', 'Section': 'A Member's...   \n3  {'Page_No.': 'Page 11', 'Section': '(2) has be...   \n4  {'Page_No.': 'Page 17', 'Section': 'a. be acti...   \n5  {'Page_No.': 'Page 51', 'Section': 'Coverage D...   \n6  {'Page_No.': 'Page 6', 'Section': 'TABLE OF CO...   \n7  {'Page_No.': 'Page 13', 'Section': 'a . A lice...   \n8     {'Page_No.': 'Page 55', 'Section': 'Exposure'}   \n9  {'Page_No.': 'Page 28', 'Section': 'Section B ...   \n\n                                         Documents_1  Distances_1 IDs  \\\n0  coverage, benefits, and participation privileg...     0.377665  20   \n1  \"Automobile\" means a four-wheel passenger vehi...     0.383765  86   \n2  state or federal law. Article 5 - Coverage Whi...     0.387754  52   \n3  dependent on the Member for principal support....     0.388576  11   \n4  a. be actively engaged in business for profit ...     0.389656  21   \n5  This policy has been updated effective January...     0.394507  79   \n6  TABLE OF CONTENTS PART I - DEFINITIONS PART II...     0.394829   3   \n7  a . A licensed Doctor of Medicine (M.D.) or Os...     0.394872  14   \n8  Exposure Exposure to the elements will be pres...     0.394984  85   \n9  to an individual policy; or (2) were eligible ...     0.399702  38   \n\n   Reranked_scores  \n0        -2.172329  \n1        -5.015879  \n2        -1.930627  \n3        -2.547559  \n4        -8.101703  \n5        -5.111147  \n6        -9.388777  \n7        -1.897079  \n8        -4.369613  \n9        -3.486530  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_1</th>\n      <th>Documents_1</th>\n      <th>Distances_1</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 16', 'Section': 'PART II - ...</td>\n      <td>coverage, benefits, and participation privileg...</td>\n      <td>0.377665</td>\n      <td>20</td>\n      <td>-2.172329</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 55', 'Section': 'Exposure'}</td>\n      <td>\"Automobile\" means a four-wheel passenger vehi...</td>\n      <td>0.383765</td>\n      <td>86</td>\n      <td>-5.015879</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 36', 'Section': 'A Member's...</td>\n      <td>state or federal law. Article 5 - Coverage Whi...</td>\n      <td>0.387754</td>\n      <td>52</td>\n      <td>-1.930627</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>{'Page_No.': 'Page 11', 'Section': '(2) has be...</td>\n      <td>dependent on the Member for principal support....</td>\n      <td>0.388576</td>\n      <td>11</td>\n      <td>-2.547559</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>{'Page_No.': 'Page 17', 'Section': 'a. be acti...</td>\n      <td>a. be actively engaged in business for profit ...</td>\n      <td>0.389656</td>\n      <td>21</td>\n      <td>-8.101703</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>{'Page_No.': 'Page 51', 'Section': 'Coverage D...</td>\n      <td>This policy has been updated effective January...</td>\n      <td>0.394507</td>\n      <td>79</td>\n      <td>-5.111147</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>{'Page_No.': 'Page 6', 'Section': 'TABLE OF CO...</td>\n      <td>TABLE OF CONTENTS PART I - DEFINITIONS PART II...</td>\n      <td>0.394829</td>\n      <td>3</td>\n      <td>-9.388777</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>{'Page_No.': 'Page 13', 'Section': 'a . A lice...</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>0.394872</td>\n      <td>14</td>\n      <td>-1.897079</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>{'Page_No.': 'Page 55', 'Section': 'Exposure'}</td>\n      <td>Exposure Exposure to the elements will be pres...</td>\n      <td>0.394984</td>\n      <td>85</td>\n      <td>-4.369613</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>{'Page_No.': 'Page 28', 'Section': 'Section B ...</td>\n      <td>to an individual policy; or (2) were eligible ...</td>\n      <td>0.399702</td>\n      <td>38</td>\n      <td>-3.486530</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"top_3_semantic_1 = results_df_1.sort_values(by='Distances_1')\ntop_3_semantic_1[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:49.261325Z","iopub.execute_input":"2024-09-04T17:16:49.261915Z","iopub.status.idle":"2024-09-04T17:16:49.284567Z","shell.execute_reply.started":"2024-09-04T17:16:49.261866Z","shell.execute_reply":"2024-09-04T17:16:49.282590Z"},"trusted":true},"execution_count":40,"outputs":[{"execution_count":40,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_1  \\\n0  {'Page_No.': 'Page 16', 'Section': 'PART II - ...   \n1     {'Page_No.': 'Page 55', 'Section': 'Exposure'}   \n2  {'Page_No.': 'Page 36', 'Section': 'A Member's...   \n\n                                         Documents_1  Distances_1 IDs  \\\n0  coverage, benefits, and participation privileg...     0.377665  20   \n1  \"Automobile\" means a four-wheel passenger vehi...     0.383765  86   \n2  state or federal law. Article 5 - Coverage Whi...     0.387754  52   \n\n   Reranked_scores  \n0        -2.172329  \n1        -5.015879  \n2        -1.930627  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_1</th>\n      <th>Documents_1</th>\n      <th>Distances_1</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 16', 'Section': 'PART II - ...</td>\n      <td>coverage, benefits, and participation privileg...</td>\n      <td>0.377665</td>\n      <td>20</td>\n      <td>-2.172329</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 55', 'Section': 'Exposure'}</td>\n      <td>\"Automobile\" means a four-wheel passenger vehi...</td>\n      <td>0.383765</td>\n      <td>86</td>\n      <td>-5.015879</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 36', 'Section': 'A Member's...</td>\n      <td>state or federal law. Article 5 - Coverage Whi...</td>\n      <td>0.387754</td>\n      <td>52</td>\n      <td>-1.930627</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"top_3_rerank_1 = results_df_1.sort_values(by='Reranked_scores', ascending=False)\ntop_3_rerank_1[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:49.294986Z","iopub.execute_input":"2024-09-04T17:16:49.295593Z","iopub.status.idle":"2024-09-04T17:16:49.319282Z","shell.execute_reply.started":"2024-09-04T17:16:49.295540Z","shell.execute_reply":"2024-09-04T17:16:49.317515Z"},"trusted":true},"execution_count":41,"outputs":[{"execution_count":41,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_1  \\\n7  {'Page_No.': 'Page 13', 'Section': 'a . A lice...   \n2  {'Page_No.': 'Page 36', 'Section': 'A Member's...   \n0  {'Page_No.': 'Page 16', 'Section': 'PART II - ...   \n\n                                         Documents_1  Distances_1 IDs  \\\n7  a . A licensed Doctor of Medicine (M.D.) or Os...     0.394872  14   \n2  state or federal law. Article 5 - Coverage Whi...     0.387754  52   \n0  coverage, benefits, and participation privileg...     0.377665  20   \n\n   Reranked_scores  \n7        -1.897079  \n2        -1.930627  \n0        -2.172329  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_1</th>\n      <th>Documents_1</th>\n      <th>Distances_1</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>7</th>\n      <td>{'Page_No.': 'Page 13', 'Section': 'a . A lice...</td>\n      <td>a . A licensed Doctor of Medicine (M.D.) or Os...</td>\n      <td>0.394872</td>\n      <td>14</td>\n      <td>-1.897079</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 36', 'Section': 'A Member's...</td>\n      <td>state or federal law. Article 5 - Coverage Whi...</td>\n      <td>0.387754</td>\n      <td>52</td>\n      <td>-1.930627</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 16', 'Section': 'PART II - ...</td>\n      <td>coverage, benefits, and participation privileg...</td>\n      <td>0.377665</td>\n      <td>20</td>\n      <td>-2.172329</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"cross_inputs_2 = [[query_2, response] for response in results_df_2['Documents_2']]\ncross_rerank_scores_2 = cross_encoder.predict(cross_inputs_2)\nresults_df_2['Reranked_scores'] = cross_rerank_scores_2","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:49.321551Z","iopub.execute_input":"2024-09-04T17:16:49.322266Z","iopub.status.idle":"2024-09-04T17:16:50.524022Z","shell.execute_reply.started":"2024-09-04T17:16:49.322198Z","shell.execute_reply":"2024-09-04T17:16:50.522518Z"},"trusted":true},"execution_count":42,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5965c2af29cb4d3e8768db2d55c9c2bf"}},"metadata":{}}]},{"cell_type":"code","source":"top_3_semantic_2 = results_df_2.sort_values(by='Distances_2')\ntop_3_semantic_2[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:50.526159Z","iopub.execute_input":"2024-09-04T17:16:50.526739Z","iopub.status.idle":"2024-09-04T17:16:50.547449Z","shell.execute_reply.started":"2024-09-04T17:16:50.526677Z","shell.execute_reply":"2024-09-04T17:16:50.545797Z"},"trusted":true},"execution_count":43,"outputs":[{"execution_count":43,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_2  \\\n0  {'Page_No.': 'Page 61', 'Section': 'Section D ...   \n1  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n2  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n\n                                         Documents_2  Distances_2 IDs  \\\n0  Section D - Claim Procedures Article 1 - Notic...     0.335960  95   \n1  of loss has been filed and before the appeal p...     0.354624  98   \n2  A claimant may request an appeal of a claim de...     0.362261  97   \n\n   Reranked_scores  \n0        -0.474061  \n1        -2.255106  \n2        -0.586972  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_2</th>\n      <th>Documents_2</th>\n      <th>Distances_2</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 61', 'Section': 'Section D ...</td>\n      <td>Section D - Claim Procedures Article 1 - Notic...</td>\n      <td>0.335960</td>\n      <td>95</td>\n      <td>-0.474061</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>of loss has been filed and before the appeal p...</td>\n      <td>0.354624</td>\n      <td>98</td>\n      <td>-2.255106</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>A claimant may request an appeal of a claim de...</td>\n      <td>0.362261</td>\n      <td>97</td>\n      <td>-0.586972</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"top_3_rerank_2 = results_df_2.sort_values(by='Reranked_scores', ascending=False)\ntop_3_rerank_2[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:50.549751Z","iopub.execute_input":"2024-09-04T17:16:50.551021Z","iopub.status.idle":"2024-09-04T17:16:50.569805Z","shell.execute_reply.started":"2024-09-04T17:16:50.550952Z","shell.execute_reply":"2024-09-04T17:16:50.568252Z"},"trusted":true},"execution_count":44,"outputs":[{"execution_count":44,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_2  \\\n0  {'Page_No.': 'Page 61', 'Section': 'Section D ...   \n2  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n1  {'Page_No.': 'Page 62', 'Section': 'A claimant...   \n\n                                         Documents_2  Distances_2 IDs  \\\n0  Section D - Claim Procedures Article 1 - Notic...     0.335960  95   \n2  A claimant may request an appeal of a claim de...     0.362261  97   \n1  of loss has been filed and before the appeal p...     0.354624  98   \n\n   Reranked_scores  \n0        -0.474061  \n2        -0.586972  \n1        -2.255106  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_2</th>\n      <th>Documents_2</th>\n      <th>Distances_2</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 61', 'Section': 'Section D ...</td>\n      <td>Section D - Claim Procedures Article 1 - Notic...</td>\n      <td>0.335960</td>\n      <td>95</td>\n      <td>-0.474061</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>A claimant may request an appeal of a claim de...</td>\n      <td>0.362261</td>\n      <td>97</td>\n      <td>-0.586972</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 62', 'Section': 'A claimant...</td>\n      <td>of loss has been filed and before the appeal p...</td>\n      <td>0.354624</td>\n      <td>98</td>\n      <td>-2.255106</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"cross_inputs_3 = [[query_3, response] for response in results_df_3['Documents_3']]\ncross_rerank_scores_3 = cross_encoder.predict(cross_inputs_3)","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:50.571622Z","iopub.execute_input":"2024-09-04T17:16:50.572145Z","iopub.status.idle":"2024-09-04T17:16:51.828428Z","shell.execute_reply.started":"2024-09-04T17:16:50.572096Z","shell.execute_reply":"2024-09-04T17:16:51.827167Z"},"trusted":true},"execution_count":45,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7cc0a280af8540b5adee019c03bffa04"}},"metadata":{}}]},{"cell_type":"code","source":"results_df_3['Reranked_scores'] = cross_rerank_scores_3\ntop_3_semantic_3 = results_df_3.sort_values(by='Distances_3')\ntop_3_semantic_3[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:51.830017Z","iopub.execute_input":"2024-09-04T17:16:51.830453Z","iopub.status.idle":"2024-09-04T17:16:51.850579Z","shell.execute_reply.started":"2024-09-04T17:16:51.830380Z","shell.execute_reply":"2024-09-04T17:16:51.848858Z"},"trusted":true},"execution_count":46,"outputs":[{"execution_count":46,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_3  \\\n0  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n1  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n2  {'Page_No.': 'Page 23', 'Section': 'Section C ...   \n\n                                         Documents_3  Distances_3 IDs  \\\n0  or d. fails to pay premium in accordance with ...     0.425369  32   \n1  Settlement of Proceeds provisions of PART IV, ...     0.451775  84   \n2  Section C - Policy Termination Article 1 - Fai...     0.453811  31   \n\n   Reranked_scores  \n0        -9.937950  \n1        -5.530196  \n2        -7.300391  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_3</th>\n      <th>Documents_3</th>\n      <th>Distances_3</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>or d. fails to pay premium in accordance with ...</td>\n      <td>0.425369</td>\n      <td>32</td>\n      <td>-9.937950</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>Settlement of Proceeds provisions of PART IV, ...</td>\n      <td>0.451775</td>\n      <td>84</td>\n      <td>-5.530196</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>{'Page_No.': 'Page 23', 'Section': 'Section C ...</td>\n      <td>Section C - Policy Termination Article 1 - Fai...</td>\n      <td>0.453811</td>\n      <td>31</td>\n      <td>-7.300391</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"top_3_rerank_3 = results_df_3.sort_values(by='Reranked_scores', ascending=False)\ntop_3_rerank_3[:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:51.852865Z","iopub.execute_input":"2024-09-04T17:16:51.853463Z","iopub.status.idle":"2024-09-04T17:16:51.874823Z","shell.execute_reply.started":"2024-09-04T17:16:51.853369Z","shell.execute_reply":"2024-09-04T17:16:51.873310Z"},"trusted":true},"execution_count":47,"outputs":[{"execution_count":47,"output_type":"execute_result","data":{"text/plain":"                                         Metadatas_3  \\\n9  {'Page_No.': 'Page 48', 'Section': 'c . If a b...   \n1  {'Page_No.': 'Page 54', 'Section': 'f . claim ...   \n6  {'Page_No.': 'Page 47', 'Section': 'M ember's ...   \n\n                                         Documents_3  Distances_3 IDs  \\\n9  c . If a beneficiary dies at the same time or ...     0.483333  72   \n1  Settlement of Proceeds provisions of PART IV, ...     0.451775  84   \n6  insurance and recorded by the Policyholder or ...     0.477835  71   \n\n   Reranked_scores  \n9        -5.348612  \n1        -5.530196  \n6        -6.560679  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Metadatas_3</th>\n      <th>Documents_3</th>\n      <th>Distances_3</th>\n      <th>IDs</th>\n      <th>Reranked_scores</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>9</th>\n      <td>{'Page_No.': 'Page 48', 'Section': 'c . If a b...</td>\n      <td>c . If a beneficiary dies at the same time or ...</td>\n      <td>0.483333</td>\n      <td>72</td>\n      <td>-5.348612</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>{'Page_No.': 'Page 54', 'Section': 'f . claim ...</td>\n      <td>Settlement of Proceeds provisions of PART IV, ...</td>\n      <td>0.451775</td>\n      <td>84</td>\n      <td>-5.530196</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>{'Page_No.': 'Page 47', 'Section': 'M ember's ...</td>\n      <td>insurance and recorded by the Policyholder or ...</td>\n      <td>0.477835</td>\n      <td>71</td>\n      <td>-6.560679</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"top_3_RAG_1 = top_3_rerank_1[[\"Documents_1\", \"Metadatas_1\"]][:3]\ntop_3_RAG_2 = top_3_rerank_2[[\"Documents_2\", \"Metadatas_2\"]][:3]\ntop_3_RAG_3 = top_3_rerank_3[[\"Documents_3\", \"Metadatas_3\"]][:3]","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:51.876636Z","iopub.execute_input":"2024-09-04T17:16:51.877109Z","iopub.status.idle":"2024-09-04T17:16:51.890493Z","shell.execute_reply.started":"2024-09-04T17:16:51.877047Z","shell.execute_reply":"2024-09-04T17:16:51.888813Z"},"trusted":true},"execution_count":48,"outputs":[]},{"cell_type":"code","source":"# Define the function to generate the response. Provide a comprehensive prompt that passes the user query and the top 3 results to the model\n\ndef generate_response(query, top_3_RAG):\n    \"\"\"\n    Generate a response using GPT-3.5's ChatCompletion based on the user query and retrieved information.\n    \"\"\"\n    messages = [\n        {\"role\": \"system\", \"content\": \"You are a helpful assistant in the insurance domain who can effectively answer user queries about insurance policies and documents.\"},\n        {\"role\": \"user\", \"content\": f\"\"\"\n            You are a helpful assistant in the insurance domain who can effectively answer user queries about insurance policies and documents.\n            You have a question asked by the user in '{query}' and you have some search results from a corpus of insurance documents in the dataframe '{top_3_RAG}'. These search results are essentially one page of an insurance document that may be relevant to the user query.\n\n            The column 'documents' inside this dataframe contains the actual text from the policy document and the column 'metadata' contains the policy name and source page. The text inside the document may also contain tables in the format of a list of lists where each of the nested lists indicates a row.\n\n            Use the documents in '{top_3_RAG}' to answer the query '{query}'. Frame an informative answer and also, use the dataframe to return the relevant policy names and page numbers as citations.\n\n            Follow the guidelines below when performing the task:\n            1. Try to provide relevant/accurate numbers if available.\n            2. You don’t have to necessarily use all the information in the dataframe. Only choose information that is relevant.\n            3. If the document text has tables with relevant information, please reformat the table and return the final information in a tabular format.\n            4. Use the 'metadata' columns in the dataframe to retrieve and cite the policy name(s) and page number(s) as citation.\n            5. If you can't provide the complete answer, please also provide any information that will help the user to search specific sections in the relevant cited documents.\n            6. You are a customer-facing assistant, so do not provide any information on internal workings, just answer the query directly.\n\n            The generated response should answer the query directly addressing the user and avoiding additional information. If you think that the query is not relevant to the document, reply that the query is irrelevant. Provide the final response as a well-formatted and easily readable text along with the citation. Provide your complete response first with all information, and then provide the citations.\n\n            ### Few-Shot Examples\n\n            ### Example 1: Basic Query about Coverage\n            **Query:**  \n            What does the policy say about coverage for accidental death?\n\n            **Top 3 RAG Results:**  \n            - **Document 1:** \"This policy provides coverage for accidental death. The insured amount for accidental death is 200% of the base coverage amount if the death occurs within 90 days of the accident...\"\n            - **Document 2:** \"Accidental death benefits are payable under this policy if the insured dies as a result of an accident. The benefit amount equals double the coverage amount, provided the death is a direct result of the accident and occurs within a specified time frame...\"\n            - **Document 3:** \"In the event of accidental death, the policy pays an additional benefit, which is equal to twice the original coverage amount. This benefit is contingent on the death occurring within 180 days from the date of the accident...\"\n\n            **Response:**  \n            The policy provides coverage for accidental death, where the benefit amount is typically 200% of the base coverage. The death must occur as a direct result of an accident and within a specified period, which varies between 90 to 180 days depending on the policy.  \n            **Citations:**  \n            Document 1: Policy X, Page 5  \n            Document 2: Policy Y, Page 12  \n            Document 3: Policy Z, Page 7\n\n            ### Example 2: Query about Exclusions\n            **Query:**  \n            Are there any exclusions for pre-existing conditions in this policy?\n\n            **Top 3 RAG Results:**  \n            - **Document 1:** \"This policy excludes coverage for any conditions that were diagnosed or treated within 12 months prior to the policy's start date. However, if the condition remains stable for 24 months after the policy's start date, it may be eligible for coverage...\"\n            - **Document 2:** \"Pre-existing conditions are generally not covered under this policy unless explicitly stated otherwise. Any condition that has shown symptoms or required medical attention in the 12 months before the policy start date is excluded...\"\n            - **Document 3:** \"Exclusions apply to pre-existing conditions if they were present within a 12-month window before the policy commencement. After a waiting period of 24 months, these conditions may be reconsidered for coverage...\"\n\n            **Response:**  \n            The policy excludes coverage for pre-existing conditions that were diagnosed or treated within 12 months before the policy start date. However, if the condition remains stable and does not require treatment for 24 months after the policy start date, it may be eligible for coverage.  \n            **Citations:**  \n            Document 1: Policy X, Page 8  \n            Document 2: Policy Y, Page 15  \n            Document 3: Policy Z, Page 10\n\n            ### Example 3: Query about Beneficiaries\n            **Query:**  \n            How can I update the beneficiary for my life insurance policy?\n\n            **Top 3 RAG Results:**  \n            - **Document 1:** \"To update the beneficiary, the policyholder must submit a written request to the insurance company. The request should include the policy number, current beneficiary, and the new beneficiary's details...\"\n            - **Document 2:** \"Beneficiary changes can be made by filling out the 'Beneficiary Change Form,' which must be signed by the policyholder and submitted to the insurance provider. The change takes effect upon the company's receipt and acceptance of the form...\"\n            - **Document 3:** \"The policyholder has the right to change the beneficiary at any time by providing written notice to the insurer. The change will be recorded and acknowledged by the insurer upon receipt...\"\n\n            **Response:**  \n            To update the beneficiary of your life insurance policy, you need to submit a written request or complete a 'Beneficiary Change Form' provided by your insurer. This request must be signed by you as the policyholder, and the change will take effect once the insurance company has received and acknowledged it.  \n            **Citations:**  \n            Document 1: Policy X, Page 3  \n            Document 2: Policy Y, Page 6  \n            Document 3: Policy Z, Page 4\n\n            ### Example 4: Complex Query about Premium Payments\n            **Query:**  \n            What are the options if I miss a premium payment?\n\n            **Top 3 RAG Results:**  \n            - **Document 1:** \"If a premium payment is missed, the policyholder typically has a 30-day grace period to make the payment without penalty. If payment is not received within this period, the policy may lapse, and coverage could be lost...\"\n            - **Document 2:** \"The policy includes a 30-day grace period for missed payments. During this time, coverage remains in force. If payment is not made by the end of the grace period, the policyholder may request reinstatement of the policy, subject to underwriting...\"\n            - **Document 3:** \"In the event of a missed premium, a 30-day grace period is granted. If the premium is not paid within this period, the policyholder may choose to reinstate the policy, which may require proof of insurability and payment of overdue premiums...\"\n\n            **Response:**  \n            If you miss a premium payment, your policy provides a 30-day grace period during which you can make the payment without losing coverage. If the payment is not made within this period, the policy may lapse. However, you may have the option to reinstate the policy by providing proof of insurability and paying the overdue premiums.  \n            **Citations:**  \n            Document 1: Policy X, Page 10  \n            Document 2: Policy Y, Page 11  \n            Document 3: Policy Z, Page 9\n        \"\"\"},\n    ]\n\n    response = openai.chat.completions.create(\n        model=\"gpt-4o-mini\",\n        messages=messages\n    )\n\n    return response.choices[0].message.content.split('\\n')","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:51.892974Z","iopub.execute_input":"2024-09-04T17:16:51.893589Z","iopub.status.idle":"2024-09-04T17:16:51.915214Z","shell.execute_reply.started":"2024-09-04T17:16:51.893528Z","shell.execute_reply":"2024-09-04T17:16:51.913543Z"},"trusted":true},"execution_count":49,"outputs":[]},{"cell_type":"code","source":"# Generate the response - For Query 1\n\nresponse = generate_response(query_1, top_3_RAG_1)\nprint(\"Query 1: \",\"\\n\",query_1,\"\\n_________________________________________________________________________________________________________________\\n_________________________________________________________________________________________________________________\\n\")\n# Print the response\nprint(\"\\n\".join(response))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:16:51.917114Z","iopub.execute_input":"2024-09-04T17:16:51.917712Z","iopub.status.idle":"2024-09-04T17:16:55.598189Z","shell.execute_reply.started":"2024-09-04T17:16:51.917657Z","shell.execute_reply":"2024-09-04T17:16:55.596667Z"},"trusted":true},"execution_count":50,"outputs":[{"name":"stdout","text":"Query 1:  \n what types of coverage does this policy include? \n_________________________________________________________________________________________________________________\n_________________________________________________________________________________________________________________\n\nThe policy includes various types of coverage, although the specific details are spread across different sections of the documents. Here’s a summary of coverage types mentioned:\n\n1. **Medical Services Coverage**: This generally includes coverage for a licensed Doctor of Medicine (M.D.) or Osteopathic Medicine (D.O.) for medically necessary services as detailed.\n   \n2. **Coverage Whitelist**: The policy outlines certain medical services and procedures that are specifically included or excluded from coverage under different sections of the policy.\n\n3. **Participation Privileges**: These details refer to the coverage benefits and conditions under which members can participate in the plan, although specific benefits might need to be referenced in the relevant sections.\n\nTo gain a clearer understanding of the specific types of coverage your policy includes, you can refer to the following pages of the documents:\n\n- Coverage for medical services: Policy Section on Medical Coverage, Page 13\n- Coverage whitelist details: Policy Whitelist Article, Page 36\n- Member participation privileges: Policy Participation Privileges Section, Page 16\n\n**Citations:**\n- Document 1: Policy Section on Medical Coverage, Page 13  \n- Document 2: Policy Whitelist Article, Page 36  \n- Document 3: Policy Participation Privileges Section, Page 16  \n","output_type":"stream"}]},{"cell_type":"code","source":"# Generate the response - For Query 2\n\nresponse = generate_response(query_2, top_3_RAG_2)\nprint(\"Query 2: \",\"\\n\",query_2,\"\\n_________________________________________________________________________________________________________________\\n_________________________________________________________________________________________________________________\\n\")\n# Print the response\nprint(\"\\n\".join(response))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:24:02.168255Z","iopub.execute_input":"2024-09-04T17:24:02.168899Z","iopub.status.idle":"2024-09-04T17:24:05.767074Z","shell.execute_reply.started":"2024-09-04T17:24:02.168848Z","shell.execute_reply":"2024-09-04T17:24:05.765576Z"},"trusted":true},"execution_count":53,"outputs":[{"name":"stdout","text":"Query 2:  \n what documentation is required when filing a claim? \n_________________________________________________________________________________________________________________\n_________________________________________________________________________________________________________________\n\nTo file a claim, you typically need to provide specific documentation that verifies the incident or loss you are claiming. Necessary documents often include:\n\n1. **Claim Form:** A completed claim form specific to the type of insurance.\n2. **Proof of Loss:** Documentation showing the date and nature of the incident leading to the claim.\n3. **Supporting Evidence:** Any evidence supporting the claim, such as photographs, police reports, or receipts related to the loss.\n4. **Identification:** A valid identification to verify your identity as the policyholder.\n5. **Invoices or Estimates:** Relevant invoices or repair estimates if applicable, especially for property damage claims.\n\nAdditionally, if you wish to appeal a claim decision, you may need to submit specific forms or additional evidence depending on the case.\n\nFor further detailed information about claim procedures, you can refer to the following sections in your insurance policy documents:\n\n| Document Summary                                                                     | Policy Name     | Page Number |\n|--------------------------------------------------------------------------------------|------------------|-------------|\n| \"Article 1 - Notice of Claim Procedures outlines the necessary documentation.\"      | Section D       | Page 61     |\n| \"A claimant may request an appeal of a claim denial including the proper documentation.\" | Section D       | Page 62     |\n| \"Requirements for submitting proof of loss before an appeal can be filed.\"          | Section D       | Page 62     |\n\n**Citations:**  \n- Document 1: Policy X, Page 61  \n- Document 2: Policy Y, Page 62  \n- Document 3: Policy Z, Page 62\n","output_type":"stream"}]},{"cell_type":"code","source":"top_3_RAG_3['Metadatas_3']","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:25:35.393897Z","iopub.execute_input":"2024-09-04T17:25:35.394541Z","iopub.status.idle":"2024-09-04T17:25:35.405899Z","shell.execute_reply.started":"2024-09-04T17:25:35.394477Z","shell.execute_reply":"2024-09-04T17:25:35.404652Z"},"trusted":true},"execution_count":57,"outputs":[{"execution_count":57,"output_type":"execute_result","data":{"text/plain":"9    {'Page_No.': 'Page 48', 'Section': 'c . If a b...\n1    {'Page_No.': 'Page 54', 'Section': 'f . claim ...\n6    {'Page_No.': 'Page 47', 'Section': 'M ember's ...\nName: Metadatas_3, dtype: object"},"metadata":{}}]},{"cell_type":"code","source":"# Generate the response - For Query 3\n\nresponse = generate_response(query_3, top_3_RAG_3)\nprint(\"Query 3: \",\"\\n\",query_3,\"\\n_________________________________________________________________________________________________________________\\n_________________________________________________________________________________________________________________\\n\")\n# Print the response\nprint(\"\\n\".join(response))","metadata":{"execution":{"iopub.status.busy":"2024-09-04T17:29:29.423601Z","iopub.execute_input":"2024-09-04T17:29:29.424954Z","iopub.status.idle":"2024-09-04T17:29:31.415505Z","shell.execute_reply.started":"2024-09-04T17:29:29.424889Z","shell.execute_reply":"2024-09-04T17:29:31.413998Z"},"trusted":true},"execution_count":58,"outputs":[{"name":"stdout","text":"Query 3:  \n what happens if i miss a payment? \n_________________________________________________________________________________________________________________\n_________________________________________________________________________________________________________________\n\nIf you miss a payment, typically, you will have a 30-day grace period to make the payment without facing any penalties. During this grace period, your insurance coverage usually remains in effect. If the payment is not made by the end of the grace period, your policy may lapse, and you will lose coverage. However, many policies also allow you the option to reinstate the policy later, although this may require proof of insurability and payment of any overdue premiums.\n\n**Citations:**  \nDocument 1: Policy X, Page 10  \nDocument 2: Policy Y, Page 11  \nDocument 3: Policy Z, Page 9\n","output_type":"stream"}]}]}