force_GPU_localrag_no_rewrite.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import ollama\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "from openai import OpenAI\n",
    "import argparse\n",
    "\n",
    "# ANSI escape codes for colors\n",
    "PINK = \"\\033[95m\"\n",
    "CYAN = \"\\033[96m\"\n",
    "YELLOW = \"\\033[93m\"\n",
    "NEON_GREEN = \"\\033[92m\"\n",
    "RESET_COLOR = \"\\033[0m\"\n",
    "\n",
    "# Path to save embeddings and file modification time\n",
    "EMBEDDINGS_FILE = \"embeddings.pt\"\n",
    "MOD_TIME_FILE = \"vault_mod_time.json\"\n",
    "\n",
    "\n",
    "# Constants\n",
    "EMBEDDINGS_DIR = \"Embeddings\"\n",
    "MOD_TIME_FILE = os.path.join(EMBEDDINGS_DIR, \"mod_times.json\")\n",
    "\n",
    "\n",
    "# Function to open a file and return its contents as a string\n",
    "def open_file(filepath):\n",
    "    with open(filepath, \"r\", encoding=\"utf-8\") as infile:\n",
    "        return infile.read()\n",
    "\n",
    "\n",
    "# Function to get relevant context from the vault based on user input\n",
    "def get_relevant_context(rewritten_input, vault_embeddings, vault_content, top_k=3):\n",
    "    if vault_embeddings.nelement() == 0:  # Check if the tensor has any elements\n",
    "        return []\n",
    "    # Encode the rewritten input\n",
    "    input_embedding = ollama.embeddings(\n",
    "        model=\"mxbai-embed-large\", prompt=rewritten_input\n",
    "    )[\"embedding\"]\n",
    "    # Compute cosine similarity between the input and vault embeddings\n",
    "    cos_scores = torch.cosine_similarity(\n",
    "        torch.tensor(input_embedding).unsqueeze(0), vault_embeddings\n",
    "    )\n",
    "    # Adjust top_k if it's greater than the number of available scores\n",
    "    top_k = min(top_k, len(cos_scores))\n",
    "    # Sort the scores and get the top-k indices\n",
    "    top_indices = torch.topk(cos_scores, k=top_k)[1].tolist()\n",
    "    # Get the corresponding context from the vault\n",
    "    relevant_context = [vault_content[idx].strip() for idx in top_indices]\n",
    "    return relevant_context\n",
    "\n",
    "\n",
    "# Function to interact with the Ollama model\n",
    "def ollama_chat(\n",
    "    user_input,\n",
    "    system_message,\n",
    "    vault_embeddings,\n",
    "    vault_content,\n",
    "    ollama_model,\n",
    "    conversation_history,\n",
    "):\n",
    "    # Get relevant context from the vault\n",
    "    relevant_context = get_relevant_context(\n",
    "        user_input, vault_embeddings, vault_content, top_k=3\n",
    "    )\n",
    "    if relevant_context:\n",
    "        # Convert list to a single string with newlines between items\n",
    "        context_str = \"\\n\".join(relevant_context)\n",
    "        print(\"Context Pulled from Documents: \\n\\n\" + CYAN + context_str + RESET_COLOR)\n",
    "    else:\n",
    "        print(CYAN + \"No relevant context found.\" + RESET_COLOR)\n",
    "\n",
    "    # Prepare the user's input by concatenating it with the relevant context\n",
    "    user_input_with_context = user_input\n",
    "    if relevant_context:\n",
    "        user_input_with_context = context_str + \"\\n\\n\" + user_input\n",
    "\n",
    "    # Append the user's input to the conversation history\n",
    "    conversation_history.append({\"role\": \"user\", \"content\": user_input_with_context})\n",
    "\n",
    "    # Create a message history including the system message and the conversation history\n",
    "    messages = [{\"role\": \"system\", \"content\": system_message}, *conversation_history]\n",
    "\n",
    "    # Send the completion request to the Ollama model\n",
    "    response = client.chat.completions.create(model=ollama_model, messages=messages)\n",
    "\n",
    "    # Append the model's response to the conversation history\n",
    "    conversation_history.append(\n",
    "        {\"role\": \"assistant\", \"content\": response.choices[0].message.content}\n",
    "    )\n",
    "\n",
    "    # Return the content of the response from the model\n",
    "    return response.choices[0].message.content\n",
    "\n",
    "\n",
    "# Function to save embeddings and file modification time\n",
    "def save_embeddings(embeddings, mod_time):\n",
    "    torch.save(embeddings, EMBEDDINGS_FILE)\n",
    "    with open(MOD_TIME_FILE, \"w\") as f:\n",
    "        json.dump({\"mod_time\": mod_time}, f)\n",
    "\n",
    "\n",
    "\n",
    "# Function to load embeddings and file modification time\n",
    "def load_embeddings(vault_name):\n",
    "    embeddings_file = os.path.join(EMBEDDINGS_DIR, f\"{vault_name}_embeddings.pt\")\n",
    "    if os.path.exists(\"embeddings.pt\") and os.path.exists(MOD_TIME_FILE):\n",
    "        embeddings = torch.load(embeddings_file)\n",
    "        with open(MOD_TIME_FILE, \"r\") as f:\n",
    "            mod_time_data = json.load(f)\n",
    "        mod_time = mod_time_data.get(vault_name)\n",
    "        return embeddings, mod_time\n",
    "    return None, None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Function to generate embeddings for vault content with checkpointing and logging\n",
    "def generate_embeddings(vault_data, vault_name, start_idx=0):\n",
    "    vault_embeddings = []\n",
    "    progress_log = os.path.join(EMBEDDINGS_DIR, f\"{vault_name}_progress.json\")\n",
    "\n",
    "    # Load progress log if exists\n",
    "    if os.path.exists(progress_log):\n",
    "        with open(progress_log, \"r\") as f:\n",
    "            progress_data = json.load(f)\n",
    "        start_idx = progress_data.get(\"last_processed_idx\", 0)\n",
    "        print(f\"Resuming from index {start_idx}\")\n",
    "    else:\n",
    "        start_idx = 0\n",
    "\n",
    "    for entry in tqdm(vault_data, desc=\"Generating embeddings\"):\n",
    "        file_path = entry[\"file_name\"]\n",
    "        modification_time = entry[\"modification_time\"]\n",
    "        chunks = entry[\"chunks\"]\n",
    "\n",
    "        for chunk in chunks:\n",
    "            content = chunk[\"text\"]\n",
    "            chunk_id = chunk[id]\n",
    "\n",
    "            # Check GPU temperature before processing each content\n",
    "            while (\n",
    "                check_gpu_temperature() > 51\n",
    "            ):  # Adjust temperature threshold as needed\n",
    "                print(\"GPU temperature is too high. Pausing until temperature drops...\")\n",
    "                time.sleep(30)  # Sleep for 30 seconds before rechecking\n",
    "\n",
    "            response = ollama.embeddings(model=\"mxbai-embed-large\", prompt=content)\n",
    "            if \"embedding\" in response:\n",
    "                vault_embeddings.append(response[\"embedding\"])\n",
    "            else:\n",
    "                print(f\"Failed to get embedding for content: {content}\")\n",
    "\n",
    "            # Save checkpoint every 10 embeddings or at the end of each entry\n",
    "            if len(vault_embeddings) % 10 == 0 or entry == vault_data[-1]:\n",
    "                checkpoint_path = os.path.join(\n",
    "                    EMBEDDINGS_DIR, f\"{vault_name}_embeddings.pt\"\n",
    "                )\n",
    "                # Check file modification time\n",
    "                current_mod_time = os.path.getmtime(file_path)\n",
    "                save_embeddings(\n",
    "                    torch.tensor(vault_embeddings), current_mod_time, vault_name\n",
    "                )\n",
    "                vault_embeddings = []\n",
    "                # with open(progress_log, \"w\") as f:\n",
    "                #    json.dump({\"last_processed_idx\": id}, f)\n",
    "\n",
    "    return vault_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"vault.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "    vault_data = json.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(vault_data)\n",
    "\n",
    "vault_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vault_name = \"vault\"\n",
    "start_idx=0\n",
    "EMBEDDINGS_DIR = \"Embeddings\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vault_embeddings = []\n",
    "progress_log = os.path.join(EMBEDDINGS_DIR, f\"embeddings_generation_of_{vault_name}_progress.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Load progress log if exists\n",
    "if os.path.exists(progress_log):\n",
    "    with open(progress_log, \"r\") as f:\n",
    "        progress_data = json.load(f)\n",
    "    last_processed_chunk_id = progress_data.get(\"last_processed_chunk_id\")\n",
    "    file_path = progress_data.get(\"file_path\")\n",
    "    print(f\"Last processed chunk ID: {last_processed_chunk_id}\")\n",
    "    print(f\"File path: {file_path}\")\n",
    "    # Optionally, you can use these values in your further processing\n",
    "    # For example:\n",
    "    # start_idx = progress_data.get(\"last_processed_idx\", 0)\n",
    "    # print(f\"Resuming from index {start_idx}\")\n",
    "else:\n",
    "    print(f\"Progress log '{progress_log}' does not exist.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "progress_log =\"Embeddings/embeddings_generation_of_vault_progress.json\"\n",
    "\n",
    "# Load progress log if exists\n",
    "if os.path.exists(progress_log):\n",
    "    print(\"exists\")\n",
    "    with open(progress_log, \"r\") as f:\n",
    "        progress_data = json.load(f)\n",
    "    start_chunk_id = progress_data.get(\"last_processed_chunk_id\")\n",
    "    start_file_name = progress_data.get(\"file_name\")\n",
    "    print(f\"Resuming from index {start_chunk_id}\")\n",
    "    print(f\"Resuming from index {start_file_name}\")\n",
    "else:\n",
    "    start_idx = 0\n",
    "    print(f\"doesnt exist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_file = os.path.join(EMBEDDINGS_DIR, \"vault_embeddings.pt\")\n",
    "    # Load existing embeddings if they exist\n",
    "if os.path.exists(embeddings_file):\n",
    "    existing_embeddings = torch.load(embeddings_file)\n",
    "    #print(type(existing_embeddings))\n",
    "    print(len(existing_embeddings))\n",
    "\n",
    "    chunk_id = \"ec483f61ef5354dc35446dc5b75ee9b585a3ec8a175f4419602db36b9a533949\"\n",
    "    for dicts in existing_embeddings:\n",
    "        if chunk_id in my_dict:\n",
    "            print(chunk_id, \"is in alredy generated embeddings file\")\n",
    "            pass\n",
    "        else:\n",
    "            content = chunk[\"text\"]\n",
    "            chunk_id = chunk[\"id\"]\n",
    "            print(content, chunk_id)\n",
    "            \"\"\"\n",
    "            # Check GPU temperature before processing each content\n",
    "            while (\n",
    "                check_gpu_temperature() > 51\n",
    "            ):  # Adjust temperature threshold as needed\n",
    "                print(\"GPU temperature is too high. Pausing until temperature drops...\")\n",
    "                time.sleep(30)  # Sleep for 30 seconds before rechecking\n",
    "\n",
    "            \"\"\"\n",
    "\n",
    "            response = ollama.embeddings(model=\"mxbai-embed-large\", prompt=content)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving in txt format!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_embeddings_to_txt(new_embeddings):\n",
    "    os.makedirs(EMBEDDINGS_DIR, exist_ok=True)\n",
    "    embeddings_file = os.path.join(EMBEDDINGS_DIR, \"vault_embeddings.txt\")\n",
    "\n",
    "    # Extract the embedding values (assuming they are tensors)\n",
    "    embedding_values = [embedding.tolist() for embedding in new_embeddings]\n",
    "\n",
    "    # Save the embedding values to a text file\n",
    "    with open(embeddings_file, \"w\") as txt_file:\n",
    "        for values in embedding_values:\n",
    "            txt_file.write(\" \".join(str(val) for val in values) + \"\\n\")\n",
    "\n",
    "# Usage example:\n",
    "# new_embeddings = ...  # Your list of embeddings\n",
    "# save_embeddings_to_txt(new_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving in pt format!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Function to save embeddings\n",
    "def save_embeddings(new_embeddings):\n",
    "    os.makedirs(EMBEDDINGS_DIR, exist_ok=True)\n",
    "    embeddings_file = os.path.join(EMBEDDINGS_DIR, \"vault_embeddings.pt\")\n",
    "    # Load existing embeddings if they exist\n",
    "    if os.path.exists(embeddings_file):\n",
    "        existing_embeddings = torch.load(embeddings_file)\n",
    "        existing_embeddings.extend(new_embeddings)\n",
    "        updated_embeddings = existing_embeddings\n",
    "    else:\n",
    "        updated_embeddings = new_embeddings\n",
    "\n",
    "    # Save the updated embeddings\n",
    "    torch.save(updated_embeddings, embeddings_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_embeddings_to_txt(new_embeddings):\n",
    "    os.makedirs(EMBEDDINGS_DIR, exist_ok=True)\n",
    "    embeddings_file = os.path.join(EMBEDDINGS_DIR, \"vault_embeddings.txt\")\n",
    "\n",
    "    # Extract the embedding values (assuming they are tensors)\n",
    "    embedding_values = [embedding.tolist() for embedding in new_embeddings]\n",
    "\n",
    "    # Save the embedding values to a text file\n",
    "    with open(embeddings_file, \"w\") as txt_file:\n",
    "        for values in embedding_values:\n",
    "            txt_file.write(\" \".join(str(val) for val in values) + \"\\n\")\n",
    "\n",
    "# Usage example:\n",
    "# new_embeddings = ...  # Your list of embeddings\n",
    "# save_embeddings_to_txt(new_embeddings)\n",
    "\n",
    "#the following is generate_embeddings function!\n",
    "\n",
    "vault_embeddings = []\n",
    "embeddings_file = os.path.join(EMBEDDINGS_DIR, \"vault_embeddings.pt\")\n",
    "    # Load existing embeddings if they exist\n",
    "if os.path.exists(embeddings_file):\n",
    "    existing_embeddings = torch.load(embeddings_file)\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "for entry in tqdm(vault_data, desc=\"Generating embeddings\"):\n",
    "    file_path = entry[\"file_name\"]\n",
    "    modification_time = entry[\"modification_time\"]\n",
    "    chunks = entry[\"chunks\"]\n",
    "\n",
    "    for chunk in chunks:\n",
    "        content = chunk[\"text\"]\n",
    "        chunk_id = chunk[\"id\"]\n",
    "        print(content, chunk_id)\n",
    "        \"\"\"\n",
    "        # Check GPU temperature before processing each content\n",
    "        while (\n",
    "            check_gpu_temperature() > 51\n",
    "        ):  # Adjust temperature threshold as needed\n",
    "            print(\"GPU temperature is too high. Pausing until temperature drops...\")\n",
    "            time.sleep(30)  # Sleep for 30 seconds before rechecking\n",
    "\n",
    "        \"\"\"\n",
    "        for dicts in existing_embeddings:\n",
    "            if chunk_id in my_dict:\n",
    "                print(chunk_id, \"is in alredy generated embeddings file\")\n",
    "                pass\n",
    "            else:\n",
    "                content = chunk[\"text\"]\n",
    "                chunk_id = chunk[\"id\"]\n",
    "                print(content, chunk_id)\n",
    "                \"\"\"\n",
    "                # Check GPU temperature before processing each content\n",
    "                while (\n",
    "                    check_gpu_temperature() > 51\n",
    "                ):  # Adjust temperature threshold as needed\n",
    "                    print(\"GPU temperature is too high. Pausing until temperature drops...\")\n",
    "                    time.sleep(30)  # Sleep for 30 seconds before rechecking\n",
    "\n",
    "                \"\"\"\n",
    "\n",
    "    \n",
    "                response = ollama.embeddings(model=\"mxbai-embed-large\", prompt=content)\n",
    "                if \"embedding\" in response:\n",
    "                    vault_embeddings.append({chunk_id: response[\"embedding\"]})\n",
    "                else:\n",
    "                    print(f\"Failed to get embedding for content: {content}\")\n",
    "                # Save checkpoint every 10 embeddings or at the end of each entry\n",
    "                if len(vault_embeddings) % 10 == 0 or entry == vault_data[-1]:\n",
    "                    checkpoint_path = os.path.join(\n",
    "                        EMBEDDINGS_DIR, f\"{vault_name}_embeddings.pt\"\n",
    "                    )\n",
    "                    # Check file modification time\n",
    "                    current_mod_time = os.path.getmtime(file_path)\n",
    "                    checkpoint_path = os.path.join(EMBEDDINGS_DIR, f\"{vault_name}_embeddings.pt\")\n",
    "                    \n",
    "                    save_embeddings(vault_embeddings)\n",
    "                    \n",
    "                    vault_embeddings = []\n",
    "                    # Construct the data to save\n",
    "                    data_to_save = {\n",
    "                        \"last_processed_chunk_id\": chunk_id,\n",
    "                        \"file_name\": file_path  # Include the file_path here\n",
    "                    }\n",
    "                    # Save data to JSON file\n",
    "                    with open(progress_log, \"w\") as f:\n",
    "                        json.dump(data_to_save, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "vault_files = [\n",
    "    f for f in os.listdir() if f.startswith(\"vault\") and f.endswith(\".txt\")\n",
    "]\n",
    "\n",
    "vault_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for vault_file in vault_files:\n",
    "    vault_name = os.path.splitext(vault_file)[0]\n",
    "    vault_content = open_file(vault_file).splitlines()\n",
    "\n",
    "    print(len(vault_content))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "saved_embeddings = torch.load(r\"Embeddings\\vault_embeddings.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_embeddings.shape\n",
    "\n",
    "print(\"Size of first dimension:\", saved_embeddings.size(0))  # Size of the first dimension\n",
    "print(\"Size of second dimension:\", saved_embeddings.size(1))  # Size of the second dimensionsaved_embeddings.size(2))  # Size of the third dimension\n",
    "# Print the first 10 elements\n",
    "\n",
    "print(saved_embeddings[1])\n",
    "\n",
    "print(saved_embeddings[2])\n",
    "\n",
    "\n",
    "print(\"Number of dimensions:\", saved_embeddings.dim())  # Number of dimensions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.exists(\n",
    "    r\"C:\\Users\\deletable\\OneDrive\\easy-local-rag\\Embeddings\\vault_embeddings.pt\"\n",
    ") and len(vault_content) == saved_embeddings.size(0):\n",
    "    print(f\"Loaded saved embeddings for {vault_name}\")\n",
    "    vault_embeddings = saved_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_saved_embeddings, saved_mod_time = load_embeddings(vault_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "combined_tensor = None\n",
    "\n",
    "combined_tensor = torch.cat((old_saved_embeddings, saved_embeddings))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "if saved_embeddings.numel() > 0 and current_mod_time == saved_mod_time:\n",
    "    print(f\"Loaded saved embeddings for {vault_name}\")\n",
    "    vault_embeddings = saved_embeddings\n",
    "else:\n",
    "    print(f\"Generating new embeddings for {vault_name}\")\n",
    "    vault_embeddings = generate_embeddings(vault_content, vault_name)\n",
    "    vault_embeddings_tensor = torch.tensor(vault_embeddings).to(device)\n",
    "\n",
    "    save_embeddings(vault_embeddings_tensor, current_mod_time, vault_name)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "# Ensure the embeddings are in tensor format\n",
    "if not isinstance(vault_embeddings, torch.Tensor):\n",
    "    vault_embeddings_tensor = torch.tensor(vault_embeddings).to(device)\n",
    "else:\n",
    "    vault_embeddings_tensor = vault_embeddings\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Clean up NVML\n",
    "pynvml.nvmlShutdown()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Parse command-line arguments\n",
    "parser = argparse.ArgumentParser(description=\"Ollama Chat\")\n",
    "parser.add_argument(\n",
    "    \"--model\", default=\"llama3\", help=\"Ollama model to use (default: llama3)\"\n",
    ")\n",
    "args = parser.parse_args()\n",
    "\n",
    "# Configuration for the Ollama API client\n",
    "client = OpenAI(base_url=\"http://localhost:11434/v1\", api_key=\"llama3\")\n",
    "\n",
    "# Load the vault content\n",
    "vault_content = []\n",
    "if os.path.exists(\"vault.txt\"):\n",
    "    with open(\"vault.txt\", \"r\", encoding=\"utf-8\") as vault_file:\n",
    "        vault_content = vault_file.readlines()\n",
    "\n",
    "\n",
    "# Conversation loop\n",
    "conversation_history = []\n",
    "system_message = \"You are a helpful assistant that is an expert at extracting the most useful information from a given text\"\n",
    "\n",
    "while True:\n",
    "    user_input = input(\n",
    "        YELLOW\n",
    "        + \"Ask a question about your documents (or type 'quit' to exit): \"\n",
    "        + RESET_COLOR\n",
    "    )\n",
    "    if user_input.lower() == \"quit\":\n",
    "        break\n",
    "\n",
    "    response = ollama_chat(\n",
    "        user_input,\n",
    "        system_message,\n",
    "        vault_embeddings_tensor,\n",
    "        vault_content,\n",
    "        args.model,\n",
    "        conversation_history,\n",
    "    )\n",
    "    print(NEON_GREEN + \"Response: \\n\\n\" + response + RESET_COLOR)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ollama\n",
    "\n",
    "    \n",
    "stream = ollama.chat(model='phi3', messages=[\n",
    "    {'role': 'user', 'content': 'Why is the sky blue?'},\n",
    "], stream=True)\n",
    "\n",
    "for chunk in stream:\n",
    "    print(chunk['message']['content'], end='', flush=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Function to load vault content from JSON file and create a dictionary\n",
    "def load_vault_content(json_file):\n",
    "    vault_content = {}\n",
    "    with open(json_file, \"r\", encoding=\"utf-8\") as f:\n",
    "        data = json.load(f)\n",
    "        for entry in tqdm(data, desc=\"Loading vault content\"):\n",
    "            for chunk in entry[\"chunks\"]:\n",
    "                chunk_id = chunk[\"id\"]\n",
    "                chunk_text = chunk[\"text\"].strip()\n",
    "                vault_content[chunk_id] = chunk_text\n",
    "    return vault_content\n",
    "\n",
    "# Example usage\n",
    "json_file = \"vault.json\"\n",
    "vault_content = load_vault_content(json_file)\n",
    "\n",
    "# Print the vault content to verify\n",
    "type(vault_content)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(vault_content{5})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "# Function to load embeddings and file modification time\n",
    "def load_embeddings():\n",
    "    embeddings_file = r\"Embeddings\\vault_embeddings.pt\"\n",
    "    if os.path.exists(embeddings_file):\n",
    "        embeddings = torch.load(embeddings_file)\n",
    "        return embeddings\n",
    "    return None, None\n",
    "\n",
    "vault_embeddings = load_embeddings()\n",
    "\n",
    "len(vault_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vault_content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rewritten_input = \" pelvic in ammatory dz PIP proximal interphalangeal PKU phenylketonuria PMB postmenopausal haemorrhage PMS premenstrual syndrome PO per os (Latin for by mouth) PoP progester1-only pill POP plaster of Paris PPH postpartum haemorrhage PR per ?\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_embedding = ollama.embeddings(\n",
    "    model=\"mxbai-embed-large\", prompt=rewritten_input\n",
    ")[\"embedding\"]\n",
    "\n",
    "print (input_embedding)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a tensor from input_embedding\n",
    "input_embedding_tensor = torch.tensor(input_embedding).unsqueeze(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Prepare embeddings and ids lists\n",
    "embeddings_list = []\n",
    "ids_list = []\n",
    "\n",
    "for embedding_dict in vault_embeddings:\n",
    "    for chunk_id, embedding in embedding_dict.items():\n",
    "        embeddings_list.append(embedding)\n",
    "        ids_list.append(chunk_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a tensor from the embeddings list\n",
    "vault_embeddings_tensor = torch.tensor(embeddings_list)\n",
    "\n",
    "# Compute cosine similarity between the input and vault embeddings\n",
    "cos_scores = torch.cosine_similarity(input_embedding_tensor, vault_embeddings_tensor)\n",
    "\n",
    "# Adjust top_k if it's greater than the number of available scores\n",
    "top_k = min(3, len(cos_scores))\n",
    "\n",
    "# Sort the scores and get the top-k indices\n",
    "top_indices = torch.topk(cos_scores, k=top_k)[1].tolist()\n",
    "\n",
    "# Get the corresponding chunk_ids from the top indices\n",
    "top_chunk_ids = [ids_list[idx] for idx in top_indices]\n",
    "\n",
    "print(top_chunk_ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(vault_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assume relevant_context is already a list of strings obtained from the get_relevant_context function\n",
    "relevant_context = [vault_content[chunk_id].strip() for chunk_id in top_chunk_ids]\n",
    "\n",
    "# Join all the relevant context strings into a single string with newlines\n",
    "combined_relevant_context = \"\\n\".join(relevant_context)\n",
    "\n",
    "print(combined_relevant_context)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "# Assuming json_file contains valid JSON data\n",
    "with open(\"vault.json\", \"r\", encoding=\"utf-8\") as json_file:\n",
    "    data = json.load(json_file)\n",
    "\n",
    "# Create a set to store encountered IDs\n",
    "seen_ids = set()\n",
    "\n",
    "# Iterate through the dictionaries\n",
    "for file_dict in data:\n",
    "    for chunk in file_dict[\"chunks\"]:\n",
    "        chunk_id = chunk[\"id\"]\n",
    "        if chunk_id in seen_ids:\n",
    "            print(f\"Duplicate ID found: {chunk_id}\")\n",
    "        else:\n",
    "            seen_ids.add(chunk_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = \"The dictum\"\n",
    "\n",
    "\n",
    "print(response)\n",
    "print(len(response))\n",
    "sentence = response.split(\".\",maxsplit=1)[0]\n",
    "response = \"\".join(response[1:])\n",
    "\n",
    "print(sentence)\n",
    "\n",
    "print(response)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from semantic_text_splitter import TextSplitter\n",
    "\n",
    "# Maximum number of characters in a chunk\n",
    "max_characters = 1000\n",
    "# Optionally can also have the splitter not trim whitespace for you\n",
    "splitter = TextSplitter(max_characters)\n",
    "# splitter = TextSplitter(max_characters, trim=False)\n",
    "\n",
    "chunks = splitter.chunks(\"your document text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "oracle = pipeline(model=\"medicalai/ClinicalBERT\")\n",
    "oracle(question=\"Where do I live?\", context=\"My name is Wolfgang and I live in Berlin\")\n",
    "{'score': 0.9191, 'start': 34, 'end': 40, 'answer': 'Berlin'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "how to find collections in the milvus db\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, list_collections\n",
    "\n",
    "def list_all_collections():\n",
    "    # Connect to Milvus\n",
    "    connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "    \n",
    "    # List all collections\n",
    "    collections = list_collections()\n",
    "    \n",
    "    # Print the names of all collections\n",
    "    print(\"Collections in Milvus:\")\n",
    "    for collection in collections:\n",
    "        print(collection)\n",
    "\n",
    "# Call the function to list all collections\n",
    "list_all_collections()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, Collection\n",
    "import pandas as pd\n",
    "\n",
    "def check_milvus_collection(collection_name):\n",
    "    # Connect to Milvus\n",
    "    connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "    \n",
    "    # Load the collection\n",
    "    collection = Collection(collection_name)\n",
    "\n",
    "    \n",
    "    # Print some information about the collection\n",
    "    print(f\"Collection name: {collection.name}\")\n",
    "    print(f\"Number of entities: {collection.num_entities}\")\n",
    "    \n",
    "    # Load the first few entities to inspect\n",
    "    results = collection.query(expr=\"id != ''\", limit=10)\n",
    "    \n",
    "    # Convert the results to a DataFrame for easier viewing\n",
    "    df = pd.DataFrame(results)\n",
    "    print(df.head())\n",
    "\n",
    "# Call the function with the name of your collection\n",
    "check_milvus_collection(\"html_chunks\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "how to delete a Collection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import utility\n",
    "\n",
    "# Replace 'your_collection_name' with the name of your collection\n",
    "utility.drop_collection(\"html_chunks\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import utility\n",
    "\n",
    "# Replace 'your_collection_name' with the name of your collection\n",
    "utility.drop_collection(\"example_collection\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "see what are all the collection presnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List all collections\n",
    "collections = utility.list_collections()\n",
    "\n",
    "# Drop each collection\n",
    "for collection_name in collections:\n",
    "    if collection_name:\n",
    "        print(f\"Collection {collection_name} successfully.\")\n",
    "    else :\n",
    "        print(\"no collections in the database\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "print(collection.schema)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import Collection, Index\n",
    "\n",
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "\n",
    "# Define the index parameters\n",
    "index_params = {\n",
    "    \"index_type\": \"IVF_FLAT\",\n",
    "    \"params\": {\"nlist\": 128},\n",
    "    \"metric_type\": \"L2\"\n",
    "}\n",
    "\n",
    "# Create the index\n",
    "index = Index(collection, \"embedding\", index_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, utility\n",
    "\n",
    "connections.connect(\n",
    "        alias=\"default\",\n",
    "        host=\"localhost\",\n",
    "        port=\"19530\",\n",
    "        max_receive_message_size=2**30,  # Set to 1 GB (adjust as needed)\n",
    "        max_send_message_size=2**30,  # Set to 1 GB (adjust as needed)\n",
    "    )\n",
    "\n",
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "\n",
    "# Load the collection into memory\n",
    "collection.load()\n",
    "\n",
    "# Define the query expression\n",
    "expr = \"serial == 0\"  # Example expression to get entities where serial is 0\n",
    "\n",
    "# Specify the fields you want to retrieve\n",
    "output_fields = [\"id\", \"text\", \"file_name\", \"modification_time\", \"serial\", \"embedding\"]\n",
    "\n",
    "# Execute the query\n",
    "results = collection.query(expr, output_fields=output_fields)\n",
    "\n",
    "# Print the results\n",
    "for result in results:\n",
    "    print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(result_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('Semantic_vault.json', 'r') as file:\n",
    "    data = json.load(file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import (\n",
    "    connections,\n",
    "    Collection,\n",
    "    FieldSchema,\n",
    "    CollectionSchema,\n",
    "    DataType,\n",
    "    utility,\n",
    ")\n",
    "\n",
    "connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "\n",
    "collection_name = \"html_chunks\"\n",
    "\n",
    "id_field = FieldSchema(\n",
    "    name=\"id\", dtype=DataType.VARCHAR, is_primary=True, max_length=64\n",
    ")\n",
    "text_field = FieldSchema(name=\"text\", dtype=DataType.VARCHAR, max_length=65535)\n",
    "file_name_field = FieldSchema(\n",
    "    name=\"file_name\", dtype=DataType.VARCHAR, max_length=65535\n",
    ")\n",
    "modification_time_field = FieldSchema(\n",
    "    name=\"modification_time\", dtype=DataType.FLOAT\n",
    ")\n",
    "serial_field = FieldSchema(name=\"serial\", dtype=DataType.INT64)\n",
    "embedding_field = FieldSchema(\n",
    "    name=\"embedding\", dtype=DataType.FLOAT_VECTOR, dim=1024\n",
    ")  # Adjust dim as per your embeddings\n",
    "\n",
    "schema = CollectionSchema(\n",
    "    fields=[\n",
    "        id_field,\n",
    "        text_field,\n",
    "        file_name_field,\n",
    "        modification_time_field,\n",
    "        serial_field,\n",
    "        embedding_field,\n",
    "    ],\n",
    "    description=\"HTML text chunks\",\n",
    ")\n",
    "\n",
    "if utility.has_collection(collection_name):\n",
    "    existing_collection = Collection(collection_name)\n",
    "    existing_schema = existing_collection.schema\n",
    "\n",
    "    if existing_schema != schema:\n",
    "        raise ValueError(\n",
    "            \"The existing collection schema does not match the provided schema.\"\n",
    "        )\n",
    "\n",
    "else:\n",
    "    collection = Collection(name=collection_name, schema=schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def insert_in_batches(collection, entities, batch_size=1000):\n",
    "    total = len(entities[\"id\"])\n",
    "    for start in tqdm(range(0, total, batch_size), desc=\"Inserting batches\"):\n",
    "        end = min(start + batch_size, total)\n",
    "        batch = {\n",
    "            \"id\": entities[\"id\"][start:end],\n",
    "            \"text\": entities[\"text\"][start:end],\n",
    "            \"file_name\": entities[\"file_name\"][start:end],\n",
    "            \"modification_time\": entities[\"modification_time\"][start:end],\n",
    "            \"serial\": entities[\"serial\"][start:end],\n",
    "            \"embedding\": entities[\"embedding\"][start:end]\n",
    "        }\n",
    "        collection.insert([\n",
    "            batch[\"id\"],\n",
    "            batch[\"text\"],\n",
    "            batch[\"file_name\"],\n",
    "            batch[\"modification_time\"],\n",
    "            batch[\"serial\"],\n",
    "            batch[\"embedding\"]\n",
    "        ])\n",
    "        print(f\"Inserted batch {start // batch_size + 1} of {total // batch_size + 1}\")\n",
    "\n",
    "\n",
    "\n",
    "# Connect to Milvus\n",
    "connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "\n",
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "\n",
    "# Prepare the data\n",
    "entities = {\n",
    "    \"id\": [],\n",
    "    \"text\": [],\n",
    "    \"file_name\": [],\n",
    "    \"modification_time\": [],\n",
    "    \"serial\": [],\n",
    "    \"embedding\": []\n",
    "}\n",
    "\n",
    "dummy_embedding = [0.0] * 1024  # Dummy embedding with dimension 1024\n",
    "\n",
    "for item in data:\n",
    "    file_name = item[\"file_name\"]\n",
    "    modification_time = item[\"modification_time\"]\n",
    "    for chunk in item[\"chunks\"]:\n",
    "        entities[\"id\"].append(chunk[\"id\"])\n",
    "        entities[\"text\"].append(chunk[\"text\"])\n",
    "        entities[\"file_name\"].append(file_name)\n",
    "        entities[\"modification_time\"].append(modification_time)\n",
    "        entities[\"serial\"].append(0)  # Assuming serial is 0 for all\n",
    "        entities[\"embedding\"].append(dummy_embedding)\n",
    "\n",
    "# Insert data into Milvus in batches\n",
    "insert_in_batches(collection, entities)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*updateingembeddings*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Load the tensor from the .pt file\n",
    "embeddings_tensor = torch.load('Embeddings\\\\mxbai-embed-large_vault_embeddings.pt')\n",
    "\n",
    "# Convert the tensor back to a dictionary\n",
    "embeddings_data = {f'embedding{i+1}': embedding.tolist() for i, embedding in enumerate(embeddings_tensor)}\n",
    "\n",
    "print(embeddings_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Load the tensor from the .pt file\n",
    "embeddings_tensor = torch.load('Embeddings\\\\mxbai-embed-large_vault_embeddings.pt')\n",
    "\n",
    "# Specify the number of top embeddings to load\n",
    "top_n = 5\n",
    "\n",
    "# Slice the tensor to get the top N embeddings\n",
    "top_embeddings_tensor = embeddings_tensor[1]\n",
    "\n",
    "\n",
    "print(top_embeddings_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Slice the tensor to get the top N embeddings\n",
    "top_embeddings_tensor = embeddings_tensor[1]\n",
    "\n",
    "\n",
    "print(top_embeddings_tensor)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import Collection, Index\n",
    "\n",
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "\n",
    "# Define the index parameters\n",
    "index_params = {\n",
    "    \"index_type\": \"IVF_FLAT\",\n",
    "    \"params\": {\"nlist\": 128},\n",
    "    \"metric_type\": \"L2\"\n",
    "}\n",
    "\n",
    "# Create the index\n",
    "index = Index(collection, \"embedding\", index_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, Collection\n",
    "\n",
    "connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from pymilvus import connections, Collection\n",
    "\n",
    "# Connect to Milvus\n",
    "connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "\n",
    "# Connect to the collection\n",
    "collection = Collection(\"html_chunks\")\n",
    "\n",
    "# Load the collection into memory\n",
    "collection.load()\n",
    "\n",
    "# Prepare the update data\n",
    "update_data = [\n",
    "    {\"id\":\"0089abb82785319d86a1f289b195ff11aec07d9af5b7689a43770e63490c20dc\", \"embedding\":[-0.7828123569488525, -0.27709338068962097, 0.41511330008506775, 0.9022812247276306, -0.6322506070137024, 0.25069254636764526, -0.2767322361469269, 0.36378464102745056, 0.867172360420227, 0.29119518399238586, -0.15393337607383728, 0.25213611125946045, -0.030258454382419586, -0.27532655000686646, -0.8751985430717468, -0.3354091942310333, 0.08982079476118088, 0.40445676445961, -0.05755322426557541, -0.20795336365699768, 1.0453933477401733, 0.06438960880041122, 0.14159871637821198, -0.44276347756385803, -0.7332879900932312, 0.44683006405830383, -0.6011638641357422, -0.43703967332839966, 0.9407795071601868, 1.2600488662719727, -0.9180915951728821, -0.19145409762859344, 0.3684351146221161, -0.6113595962524414, -0.6186680793762207, -0.06702829897403717, 0.44037437438964844, 0.0577254593372345, -0.6090666055679321, -0.67686527967453, -0.14073005318641663, 0.791888952255249, 1.2983211278915405, -0.6797413229942322, -0.9377477169036865, 0.026104621589183807, 0.002500586211681366, 0.6734619736671448, -0.3600749373435974, -0.4859426021575928, 0.28935375809669495, -0.25887036323547363, -0.20125725865364075, -1.1320416927337646, 0.12654437124729156, -0.41371965408325195, 0.19836944341659546, 0.8286101818084717, 0.4089277386665344, -0.6084116697311401, 0.305874228477478, 0.8362943530082703, 1.3196208477020264, -0.4272621273994446, 0.14155367016792297, 1.1993752717971802, -0.7608587741851807, -0.24938003718852997, 0.4166712164878845, -0.34164080023765564, -0.7164101600646973, 1.3966069221496582, 0.1842719316482544, -0.3932875990867615, -0.08720587193965912, -0.0024885423481464386, -0.0983353704214096, 0.07610379159450531, 0.10145093500614166, 0.8245826959609985, 0.2206858992576599, 0.46613121032714844, 0.5964999794960022, 0.8553804755210876, 0.047342654317617416, -0.6342425346374512, 0.44146043062210083, -0.07817327231168747, 0.08509775251150131, -0.4073849320411682, -0.570923924446106, 0.4592619240283966, 0.059260960668325424, 0.8091621398925781, 0.16043901443481445, 0.6404651999473572, -0.7015365362167358, -0.4782419800758362, -0.09655443578958511, -0.11392519623041153, 0.3022889196872711, 1.1102445125579834, -0.3002280294895172, 0.5474172830581665, -0.6330676674842834, -0.5207372307777405, 0.35849103331565857, -0.6403241753578186, -0.6694198846817017, -0.2248014658689499, 0.6102932095527649, 0.6094659566879272, -0.2199753224849701, -0.12408298254013062, -0.25588691234588623, 0.7326384782791138, 0.11910589039325714, 0.010322792455554008, 0.17390039563179016, 0.39949628710746765, 0.2885652780532837, -0.4026263654232025, 0.6853739023208618, -0.8465296626091003, -1.062707781791687, -0.8048808574676514, 0.14866429567337036, -0.0200975239276886, 0.33638861775398254, -1.0627965927124023, -0.20597469806671143, -0.6585509181022644, 0.3964637517929077, -0.42271727323532104, 0.3422508239746094, -0.33586516976356506, 0.5504766702651978, 0.11341515183448792, 0.43062320351600647, 0.38355928659439087, 0.1631673276424408, 0.028494618833065033, 0.09033548831939697, 1.0734225511550903, 0.29145586490631104, 0.3386431932449341, 0.05573185533285141, 0.38668256998062134, -0.21481506526470184, 0.7326793670654297, 0.46962404251098633, 0.6254472136497498, 0.4054642915725708, -0.9464871287345886, -0.3126782178878784, -0.7804150581359863, -0.6361397504806519, 1.1802926063537598, 0.2841724455356598, 0.2932494282722473, -0.7587368488311768, 0.14325524866580963, -0.21557894349098206, -0.2519146502017975, 0.2892794609069824, 0.4353052079677582, 0.05374610424041748, -0.4044302999973297, 0.2772212028503418, 0.44893187284469604, 0.8168901801109314, -0.5159703493118286, 0.05127984285354614, -0.07906079292297363, 0.07874259352684021, -0.19184830784797668, 0.024883270263671875, -0.2994977831840515, -0.8908796310424805, 0.2894689738750458, -1.033141851425171, -1.1443676948547363, 0.027115348726511, -0.4413950443267822, -0.8843006491661072, 0.2633334994316101, 0.16155856847763062, -0.18143001198768616, -0.4340452253818512, -0.13534517586231232, 0.6675559878349304, 0.8297298550605774, -1.620424509048462, -0.0478750541806221, -0.7924292683601379, 0.10492290556430817, -1.1588443517684937, 0.5396803021430969, -0.23328852653503418, -0.5367871522903442, -0.8641131520271301, 0.04577129706740379, -0.9081952571868896, -0.6130248308181763, 0.21835562586784363, 0.00754740834236145, 0.5138764977455139, 0.385618656873703, 0.7304354310035706, -0.24746915698051453, 1.4227275848388672, 0.4114985466003418, -0.9272089004516602, -0.20446808636188507, 0.7575680613517761, -0.04947645217180252, 0.36690831184387207, -0.0753171369433403, -0.48955339193344116, 0.16400061547756195, 0.6950822472572327, 0.796556830406189, 0.10567907989025116, 1.092645287513733, -0.11592575162649155, 0.864288866519928, -0.13690586388111115, 0.5976158380508423, 0.15143530070781708, -0.10309546440839767, -0.28697264194488525, 1.5222625732421875, -0.029410772025585175, -0.27701815962791443, -0.2898039221763611, 0.4311669170856476, 0.2761283814907074, 0.01433992013335228, 0.4125245213508606, 0.6000123023986816, 0.5534193515777588, 0.31543514132499695, 0.8913508057594299, 0.711327850818634, 0.44659700989723206, -0.47488290071487427, -0.5959708094596863, -1.0232748985290527, -0.9673599004745483, -0.1320626437664032, -0.9237433075904846, 0.4658123254776001, -0.4616546034812927, -0.5537431240081787, -0.756135880947113, -0.2055044174194336, 0.8977864384651184, 0.9900720119476318, -1.073256015777588, -0.05154156684875488, -0.13122674822807312, 0.14755132794380188, 0.17332904040813446, 0.15981528162956238, -0.3830607533454895, 0.10646172612905502, 0.32112863659858704, -0.2840203642845154, -0.17590004205703735, -0.07977893203496933, -0.9956254959106445, -1.0929687023162842, 0.351550817489624, -0.3931729197502136, -0.45285850763320923, 0.6257066130638123, -0.15066084265708923, -0.5823119878768921, 1.1720454692840576, -0.6299132108688354, -0.6330057382583618, -0.7318710684776306, 0.6776159405708313, -0.1377057433128357, -0.07928688079118729, 0.574948787689209, -0.9032381176948547, -0.3774559199810028, -0.05741565674543381, -0.23508578538894653, -0.7008081674575806, -0.8436916470527649, -0.40787258744239807, -0.738305926322937, -0.20799748599529266, 0.06622184813022614, 0.4232769310474396, -0.5065373182296753, 0.19286414980888367, -0.6509308815002441, -0.24057413637638092, -0.04327694699168205, -1.1149649620056152, 0.32449042797088623, 0.04027672857046127, -0.009925730526447296, -0.00860348716378212, 0.035241805016994476, 0.961402416229248, 1.6239004135131836, -1.3194670677185059, -0.0843912810087204, 0.26747560501098633, 0.11685553193092346, -1.0494153499603271, 0.2685088515281677, 0.4690292179584503, -0.3495185673236847, -0.2709217667579651, -0.02148091048002243, -0.9055293798446655, 0.667119562625885, -0.4028824269771576, -0.058205414563417435, -0.6161108613014221, -0.08350023627281189, -0.055240094661712646, -0.7080701589584351, 0.9326834678649902, -0.7189999222755432, 0.015568023547530174, -0.6325771808624268, -0.48769688606262207, 0.4339579939842224, 0.06462158262729645, 1.0564589500427246, 0.012237174436450005, -0.28099945187568665, -0.09105395525693893, 1.1428085565567017, 0.003152664750814438, -0.365811288356781, 0.24031777679920197, 0.46469855308532715, 0.06441424041986465, -0.4390372335910797, 1.1711890697479248, -0.21651363372802734, -0.26394930481910706, 0.3631729483604431, -0.10780630260705948, 0.09698575735092163, -0.2340589463710785, -0.8349785804748535, 0.4507956802845001, 0.7355408668518066, -1.2916113138198853, 0.3771459460258484, -0.13844043016433716, 0.061914242804050446, 1.2471542358398438, 0.9479784965515137, 0.48663467168807983, 0.7191925644874573, 0.25303512811660767, -1.073193073272705, 0.16893619298934937, 0.835830569267273, 0.32706326246261597, 0.2556898891925812, 0.5823195576667786, -0.5997410416603088, 0.08555889874696732, 0.36556774377822876, -1.1713024377822876, -0.6900826096534729, 0.8452309370040894, 0.16897399723529816, 0.3335384726524353, -0.8841701745986938, 0.6400192379951477, -0.38393110036849976, 0.10511837154626846, 0.08530358970165253, -0.8921894431114197, -0.7649389505386353, -0.26284167170524597, 0.8040898442268372, -0.9934847354888916, 0.13527844846248627, -0.00034137204056605697, -0.13388687372207642, 0.7602734565734863, 0.19115203619003296, -0.5571087598800659, -1.1135334968566895, 1.2016669511795044, 1.0383418798446655, -0.05373189598321915, 0.0072942450642585754, 0.9476677179336548, -0.12187536060810089, 0.005340047180652618, 0.6960304379463196, -0.4471643567085266, 0.22554655373096466, 0.022479910403490067, -0.1371382176876068, 0.045324329286813736, 0.19908663630485535, -0.9888361692428589, -0.1708393692970276, -0.7932950258255005, 0.8040523529052734, 0.18941457569599152, 0.38939759135246277, 0.35048311948776245, -0.46439129114151, -0.6405993700027466, -0.44786250591278076, -0.8720367550849915, -0.4702659845352173, -1.059401273727417, 0.169228196144104, 0.6573431491851807, -0.15779396891593933, 0.41328224539756775, 0.4089913070201874, 0.32797449827194214, 0.2709249258041382, 0.5829678177833557, -0.7375612258911133, -0.8174483180046082, 0.48164117336273193, -0.28550973534584045, 0.438120573759079, -0.5372886657714844, 0.6216517686843872, 0.3498506546020508, -0.6326940059661865, 0.3916647136211395, 0.5389883518218994, -0.29212018847465515, -0.13139373064041138, 0.37030261754989624, 0.8290108442306519, 1.1654198169708252, 0.7834715843200684, 0.8289585709571838, 0.281147301197052, -0.10776208341121674, -0.22225645184516907, 1.0595126152038574, -0.47823986411094666, 0.558658242225647, 0.5952408313751221, 0.6904051303863525, -0.3643483817577362, 0.23490102589130402, -0.7168416976928711, 0.08201944082975388, -0.21190235018730164, 1.0949116945266724, -0.4312162697315216, -0.21731089055538177, 0.8046319484710693, -0.34188300371170044, -0.3342248201370239, 0.34150010347366333, 0.5644885897636414, 0.22329670190811157, 0.5374613404273987, 0.6638902425765991, -0.04241419583559036, 0.6314306855201721, -1.264382243156433, 0.17592641711235046, -0.08156540989875793, 0.585129976272583, -0.2954152226448059, 0.7137546539306641, 1.0163776874542236, 0.3597449064254761, -0.010748045518994331, -0.06771355122327805, -0.41270333528518677, 0.04926379770040512, 0.32313305139541626, -0.22411608695983887, -0.11568377912044525, -0.18440884351730347, -0.044718608260154724, -0.41990965604782104, 0.43218889832496643, -1.1676831245422363, -0.1465362012386322, -0.9497988820075989, -0.4961434602737427, 0.48876118659973145, 0.01912865787744522, -0.20438435673713684, -0.9661914110183716, -0.8275837898254395, 0.5886679291725159, -0.24904929101467133, -0.8241468071937561, -1.2563287019729614, 0.42271888256073, -0.21833771467208862, -0.18834301829338074, -0.35647517442703247, 0.21790724992752075, -0.160165473818779, 0.28961777687072754, 1.2050076723098755, 0.5882022380828857, 0.6677361130714417, -0.7755512595176697, -0.6698856949806213, 0.3085930347442627, 0.4423472285270691, -0.8612925410270691, -0.7086820006370544, 0.9501177072525024, -0.07202187180519104, 0.2954002618789673, -0.3254663944244385, 0.026675090193748474, 0.025175977498292923, -0.11680497229099274, -0.16134566068649292, -1.1736458539962769, -0.17515845596790314, -0.22735998034477234, -1.1782408952713013, 0.23448622226715088, 0.555097222328186, 1.0272732973098755, -0.12091849744319916, -0.49476057291030884, -0.6139771342277527, -0.3111691474914551, -0.6111134886741638, -0.1311991959810257, 0.08840072154998779, -0.7535379528999329, -0.423729807138443, 0.9652791619300842, -0.9326862096786499, -0.030271410942077637, -0.2672901749610901, 0.572633683681488, 0.5541442632675171, 0.06106690689921379, -0.25328925251960754, 0.10050307214260101, 0.2205144464969635, -0.0029218830168247223, 0.37970659136772156, 1.0893585681915283, -0.48965054750442505, -0.28187936544418335, -0.45352500677108765, 0.05686216801404953, 0.23815053701400757, 0.9108790755271912, -0.8947011232376099, -0.3918737471103668, 0.6078469753265381, -0.46113941073417664, -0.25768133997917175, 0.028962820768356323, 0.5540531873703003, 0.7688850164413452, 1.1604217290878296, -0.2545185089111328, -0.3496750295162201, -0.29876038432121277, -0.6075044870376587, 0.10560159385204315, -0.5743539929389954, -0.37509840726852417, 0.6476843953132629, -0.7451499700546265, 0.909752368927002, -0.4916507303714752, 1.3429774045944214, 0.8164942264556885, 0.5551773309707642, -1.136173963546753, -0.88737952709198, -0.18391817808151245, 0.7150078415870667, -1.311277985572815, -0.646000862121582, -0.28940948843955994, -0.8243460655212402, -0.6619224548339844, -0.6285117268562317, -0.4861736595630646, -0.544216513633728, -0.7019585371017456, 1.073026180267334, -0.3136158287525177, 0.46049872040748596, -0.30617138743400574, 0.011663413606584072, 0.03099004179239273, 0.46234264969825745, -0.17282962799072266, -0.22173409163951874, 0.1355520337820053, 0.05259473994374275, 0.35731786489486694, 1.1285725831985474, -0.5342696309089661, 0.36401888728141785, 0.19066904485225677, 0.5359245538711548, -0.1151067465543747, -0.7907774448394775, 0.34285256266593933, 0.03209856152534485, -0.18068496882915497, -0.7659915089607239, 0.37322714924812317, -0.5482969284057617, -0.9801268577575684, -0.7808899879455566, -0.13044746220111847, -0.03961855173110962, -0.12683773040771484, -0.46861815452575684, 0.16096775233745575, 0.48979854583740234, 0.07882393151521683, 0.18732787668704987, 1.2765270471572876, -0.3927789628505707, 0.32367679476737976, 1.0622624158859253, -0.48433879017829895, 0.14840534329414368, -1.0559195280075073, -0.5786721706390381, -0.19665919244289398, -0.23280677199363708, 0.07061266154050827, -0.6196882724761963, -0.20694640278816223, -0.21130990982055664, 0.14239314198493958, 0.7344328165054321, 0.21972158551216125, 0.2129114270210266, 0.15087349712848663, -0.0742127075791359, -0.41161251068115234, 0.4481009840965271, -0.6592992544174194, 0.3445277512073517, -0.06725998222827911, -1.0093241930007935, -0.39873993396759033, 1.0417283773422241, -0.2408204823732376, 0.1389636993408203, -1.0223289728164673, -0.5833547711372375, -0.39379027485847473, -0.45204979181289673, -0.6208251714706421, -0.8309588432312012, -0.4838869571685791, -0.8137210011482239, -0.02119668386876583, 0.38687852025032043, 0.07786019891500473, -0.3280502259731293, -0.47039806842803955, 0.05917166918516159, 0.006624951958656311, 0.47081729769706726, -0.653079628944397, 0.11899271607398987, -0.6001037955284119, -0.4698660373687744, 0.30799829959869385, 0.7207427620887756, -0.46462497115135193, -0.8700082302093506, -0.693430483341217, 1.330476999282837, -0.24625979363918304, 0.5697460174560547, -0.4665035307407379, -0.17081290483474731, 0.4641020596027374, 0.23030589520931244, -0.8305099606513977, 0.9053868055343628, 1.9754174947738647, -0.4045619070529938, 0.1424213945865631, 0.09013987332582474, -0.7641696929931641, -0.43348047137260437, 0.254970520734787, 0.39023301005363464, -0.6019237041473389, -0.11761455237865448, -0.10652146488428116, -0.619891345500946, -0.5585184097290039, -0.39253541827201843, -0.14359170198440552, -0.02045685052871704, 0.08113674819469452, -0.47777172923088074, -0.7695105075836182, 0.7146340608596802, -0.2978198528289795, -0.026410579681396484, -0.679672360420227, 0.5441126823425293, 0.4229040741920471, -0.4900919198989868, 0.10135972499847412, -0.16382941603660583, -0.4112054705619812, 0.5304985642433167, -0.422262579202652, 0.16837505996227264, -0.21960686147212982, 0.07616384327411652, -0.43301618099212646, -0.2341102510690689, 0.1110062301158905, 0.1406770646572113, 0.05324694514274597, 1.0372394323349, 0.43421730399131775, 0.20339930057525635, -0.0714939683675766, 0.542420506477356, -0.6587614417076111, -0.09479944407939911, -0.37277328968048096, -0.24222326278686523, 0.332794189453125, -0.21185892820358276, -0.34475618600845337, -0.8007310628890991, -0.6896945834159851, -0.027323976159095764, -0.37570860981941223, 0.10788998007774353, -0.5305667519569397, 0.30824315547943115, 0.7836369276046753, 0.5023688673973083, -0.6462723016738892, 0.8511403203010559, -0.39739662408828735, 0.6937305927276611, -0.08644947409629822, -0.4637433588504791, 0.18260690569877625, 0.33808445930480957, -0.17987027764320374, -0.006979070603847504, -0.12938007712364197, 0.20369721949100494, 0.20433661341667175, 0.36284542083740234, 0.5256175994873047, 0.339283287525177, -0.052791714668273926, 1.0536131858825684, -0.8461920022964478, 0.6701087951660156, 0.003691643476486206, 0.35625624656677246, 0.12332391738891602, -0.6573437452316284, -0.05978787690401077, -0.18037474155426025, 1.1996794939041138, 0.4259011745452881, 0.1476624608039856, 0.8975593447685242, 0.0557866096496582, -0.6699909567832947, 0.22699102759361267, 0.5546096563339233, 0.5099072456359863, 0.8300193548202515, 0.9974905252456665, -1.2952442169189453, 0.13909275829792023, 0.5345084071159363, 0.1415347009897232, 0.04033251851797104, 0.8311087489128113, 0.3860299587249756, -0.3479769825935364, 0.09967685490846634, 0.49587464332580566, -0.5085391998291016, 0.34436240792274475, -0.8622831702232361, 0.6187289953231812, -0.022166062146425247, -0.1321491301059723, 0.0720633715391159, -0.5104812383651733, 0.5399727821350098, 0.012869700789451599, -0.23510035872459412, 0.7465575933456421, 0.39066028594970703, 0.6935822367668152, -0.7358700037002563, -1.0268124341964722, -0.41557225584983826, 0.5272613763809204, 0.36830899119377136, -0.08851656317710876, -0.17620517313480377, 0.617353081703186, -0.6875759363174438, -0.9804831743240356, -0.699802577495575, 0.01282181590795517, 0.35673341155052185, 0.2078530192375183, 0.25989022850990295, -0.693341851234436, 0.328828901052475, 0.5572656989097595, -0.23232977092266083, 1.2812203168869019, -1.2430477142333984, -0.564706027507782, -0.6317421197891235, -0.05283181369304657, -1.0940709114074707, -0.08549457788467407, 0.42516979575157166, -0.4948086142539978, 0.7227563261985779, 0.4729738235473633, -0.29119032621383667, 0.08366723358631134, -0.35979485511779785, 0.8140606880187988, 0.657080352306366, 0.05383536219596863, 0.514124870300293, -1.2432138919830322, -0.7750939726829529, 0.4888150095939636, -0.3858971893787384, -0.8024272918701172, -0.44721364974975586, 0.5933371186256409, -0.46949437260627747, -0.1451990157365799, 0.4638333320617676, 0.709322988986969, 0.12979909777641296, 0.2850637435913086, -0.7411597371101379, 0.4654315710067749, 1.1913286447525024, -0.15056781470775604, 0.2919842600822449, -0.3896973133087158, 0.5137627124786377, 0.48908865451812744, -0.24162420630455017, -0.3973892629146576, -0.9040055274963379, 0.031022492796182632, -0.3406449854373932, 1.2701733112335205, 0.0048257410526275635, -0.053963929414749146, -0.09831936657428741, -0.2770848870277405, -0.5249350666999817, -0.6511154770851135, -0.5731566548347473, 0.04406149685382843, 0.9729418158531189, 1.0695326328277588, -0.1138397604227066, 0.743656575679779, -0.4605046510696411, -0.02104860544204712, 1.3837884664535522, 0.5222346186637878, -0.08241817355155945, 1.0239756107330322, 0.09018297493457794, 0.3138029873371124, -0.06273296475410461, 0.08198478817939758, -0.28149712085723877, -0.5058165788650513, -0.3699208199977875, -0.1168360561132431, -1.2314711809158325, -0.8921575546264648, -0.15453292429447174, 0.3605525493621826, 1.106641173362732, -0.3346380293369293, -1.2032334804534912, -0.27449116110801697, -0.10209019482135773, -1.43972909450531, -0.1842588484287262, -0.803627073764801, 0.5417440533638, -0.09008854627609253, -0.6216168999671936, 0.5643336176872253, -0.8007320165634155, 3.5021815299987793, 1.016852855682373, 0.5111916065216064, -0.06203165650367737, 0.3394480347633362, 1.3156750202178955, 0.7115001082420349, -0.6001495718955994, -0.583597719669342, -1.116575002670288, -0.23857945203781128, -0.5605703592300415, 0.20842598378658295, -0.12406795471906662, -0.07421883940696716, 0.3580460846424103, -0.47767090797424316, 1.2727248668670654, 1.255099892616272, -1.141005277633667, -1.0805120468139648, 0.3944804072380066, 0.24160261452198029, 0.4390304684638977, -0.42183157801628113, 0.5386890172958374, 0.5698067545890808, 0.2089328169822693, 0.3864547908306122, -0.28130751848220825, -0.1943625807762146, -1.0028913021087646, 0.1828201562166214, -0.2802167534828186, -0.21930807828903198, 0.9968680739402771, -0.17843198776245117, -0.8042709827423096, 0.026393946260213852, -0.45423999428749084, -0.060831256210803986, 0.674354076385498, 0.2047150731086731, 0.20027579367160797, 0.03743268549442291, 0.6377115249633789, -0.19428640604019165, -0.5843294262886047, 1.161407232284546, 0.003731667995452881, 0.9762888550758362, -0.1507333666086197, -0.3480280637741089, 0.3626687228679657, -0.5249403715133667, 0.0167134590446949, -0.1862075924873352, 0.03579982370138168, 0.012735847383737564, -0.37850436568260193, 0.5220426917076111, 0.8285608887672424, -0.9965909719467163, 0.003847181797027588, 0.07117022573947906, 0.2738375663757324, 0.2219880223274231, 1.2206134796142578, 0.09095965325832367, -0.43485915660858154, 0.24298234283924103, -0.3875679075717926, -0.4092029333114624, 0.014655247330665588, -0.14599615335464478, 0.21616807579994202, 0.4090018570423126, -0.20996913313865662, -0.08457768708467484, -0.0279887355864048, 0.48800235986709595, 0.002125512808561325, -0.7597452402114868, 0.16583509743213654, 0.5458806157112122, 0.28260183334350586, -0.4618920683860779, -0.4035499691963196, -0.7964058518409729, 0.48726287484169006, -0.016892068088054657, 0.7664936780929565, -0.039836250245571136, 0.3039447069168091, 0.6841843724250793]}\n",
    "]\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "# Specify the number of top embeddings to load\n",
    "top_n = 5\n",
    "\n",
    "# Iterate through the top N embeddings\n",
    "for idx, embedding_dict in enumerate(tqdm(embeddings_tensor[:top_n])):\n",
    "    for id, embedding in embedding_dict.items():\n",
    "        update_data[\"id\"] = id\n",
    "        update_data[\"embedding\"] = embedding\n",
    "\n",
    "\"\"\"\n",
    "print(update_data)\n",
    "\n",
    "# Update the embeddings in Milvus with tqdm progress bar\n",
    "collection.insert(data=update_data)\n",
    "\n",
    "\n",
    "print(\"Embeddings updated successfully.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example data matching the schema\n",
    "update_data = [\n",
    "    {\"id\": 1,  \"embedding\": [0.1, 0.2, 0.3, ...]},\n",
    "    {\"id\": 2,  \"vector\": [0.4, 0.5, 0.6, ...]}\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "{'0089abb82785319d86a1f289b195ff11aec07d9af5b7689a43770e63490c20dc': [-0.7214671969413757, -0.16116780042648315], \n",
    "'ac3a3817216183abf66ef7904b63de7d9e2590a477c3c94b4a1a741f2591ac19': [-0.7828123569488525, -0.5159667730331421], \n",
    "'b7ad834cca68dd2d26ce5a3bba4d424a9b4c2216a1be407a648e5549e73a1320': [0.7405392527580261, -0.9613822102546692]}\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import Collection\n",
    "\n",
    "# Assuming you have already created a collection\n",
    "collection = Collection(name=\"html_chunks\")\n",
    "\n",
    "# Get the schema of the collection\n",
    "schema = collection.schema\n",
    "\n",
    "# Extract field names\n",
    "field_names = [field.name for field in schema.fields]\n",
    "\n",
    "print(\"Field names:\", field_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType\n",
    "\n",
    "# Connect to Milvus\n",
    "connections.connect(host='localhost', port='19530')\n",
    "\n",
    "# Define fields\n",
    "fields = [\n",
    "    FieldSchema(name=\"id\", dtype=DataType.INT64, is_primary=True ),\n",
    "    FieldSchema(name=\"vector\", dtype=DataType.FLOAT_VECTOR, dim=2)\n",
    "]\n",
    "\n",
    "# Enable dynamic fields in the schema definition\n",
    "schema = CollectionSchema(fields, enable_dynamic_field=True)\n",
    "\n",
    "# Create the collection with the schema\n",
    "collection = Collection(name=\"example_collection\", schema=schema)\n",
    "\n",
    "# Now you can insert data with dynamic fields\n",
    "data = [\n",
    "    {\"id\": 1, \"vector\": [0.1,0.1]},\n",
    "    {\"id\": 2, \"vector\": [0.4,0.1]}\n",
    "]\n",
    "\n",
    "# Insert data into the collection\n",
    "collection.insert(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now you can insert data with dynamic fields\n",
    "data = [\n",
    "    {\"id\": 1, \"vector\": [0.1,0.1]},\n",
    "    {\"id\": 2, \"vector\": [0.4,0.1]}\n",
    "]\n",
    "\n",
    "# Insert data into the collection\n",
    "collection.upsert(data)\n",
    "\n",
    "\n",
    "\n",
    "# Extract field names\n",
    "field_names = [field.name for field in schema.fields]\n",
    "print(\"Field names:\", field_names)\n",
    "\n",
    "# Get collection statistics\n",
    "stats = collection.num_entities\n",
    "print(\"Number of entities:\", stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, Collection\n",
    "\n",
    "# Connect to Milvus\n",
    "connections.connect(host='localhost', port='19530')\n",
    "\n",
    "# Get the collection you want to delete\n",
    "collection = Collection(name=\"html_chunks\")\n",
    "\n",
    "# Drop the collection\n",
    "collection.drop()\n",
    "\n",
    "print(\"Collection deleted successfully.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import connections, Collection\n",
    "\n",
    "# Connect to Milvus\n",
    "connections.connect(host='localhost', port='19530')\n",
    "\n",
    "# Get the collection\n",
    "collection = Collection(name=\"example_collection\")\n",
    "\n",
    "collection.load()\n",
    "\n",
    "# Define the query vector\n",
    "query_vector = [[0.1, 0.1]]\n",
    "\n",
    "# Perform the search\n",
    "search_params = {\"metric_type\": \"L2\", \"params\": {\"nprobe\": 10}}\n",
    "results = collection.search(data=query_vector, anns_field=\"vector\", param=search_params, limit=10)\n",
    "\n",
    "# Print the results\n",
    "for result in results:\n",
    "    print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import MilvusClient\n",
    "\n",
    "client = MilvusClient(\"default.db\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fast language models are artificial intelligence (AI) models that can process and generate human-like language quickly and efficiently. The importance of fast language models lies in their ability to revolutionize various industries and aspects of our lives. Here are some reasons why fast language models are crucial:\n",
      "\n",
      "1. **Improved User Experience**: Fast language models enable applications to respond rapidly to user input, providing a seamless and interactive experience. This is particularly important for applications like chatbots, virtual assistants, and language translation systems, where quick responses are essential for effective communication.\n",
      "2. **Real-time Processing**: Fast language models can process and analyze large amounts of text data in real-time, making them ideal for applications that require immediate insights, such as sentiment analysis, entity recognition, and topic modeling.\n",
      "3. **Enhanced Customer Service**: Fast language models can help companies respond quickly to customer inquiries, improving customer satisfaction and reducing response times. This leads to increased customer loyalty and retention.\n",
      "4. **Increased Productivity**: Fast language models can automate many tasks, such as text summarization, language translation, and content generation, freeing up human resources to focus on more strategic and creative tasks.\n",
      "5. **Competitive Advantage**: Companies that leverage fast language models can gain a competitive edge over their competitors by providing faster and more accurate responses, improving customer experience, and increasing operational efficiency.\n",
      "6. **Improved Accessibility**: Fast language models can enable people with disabilities, language barriers, or limited literacy to communicate more effectively. For example, language translation systems can facilitate communication between people who speak different languages.\n",
      "7. **Advanced Research and Development**: Fast language models can accelerate research in various fields, such as natural language processing, machine learning, and artificial intelligence. They can also facilitate the development of new applications and services that were previously unimaginable.\n",
      "8. **Cost Savings**: Fast language models can reduce operational costs by automating tasks, improving efficiency, and reducing the need for human intervention.\n",
      "9. **Enhanced Decision-Making**: Fast language models can provide businesses with real-time insights and analytics, enabling data-driven decision-making and improving overall business performance.\n",
      "10. **Future-Proofing**: As language models continue to evolve, fast language models will be essential for keeping pace with the increasingly complex and dynamic nature of human language.\n",
      "\n",
      "In summary, fast language models are crucial for improving user experiences, enhancing customer service, increasing productivity, and gaining a competitive advantage. They also have the potential to drive innovation, improve accessibility, and reduce costs, making them an essential component of many industries and applications.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from groq import Groq\n",
    "\n",
    "client = Groq(\n",
    "    api_key = \"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\",\n",
    ")\n",
    "\n",
    "chat_completion = client.chat.completions.create(\n",
    "    messages=[\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": \"Explain the importance of fast language models\",\n",
    "        }\n",
    "    ],\n",
    "    model=\"llama3-70b-8192\",\n",
    ")\n",
    "\n",
    "print(chat_completion.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import asyncio\n",
    "import edge_tts\n",
    "from pydub import AudioSegment\n",
    "from pydub.playback import play\n",
    "import nest_asyncio\n",
    "\n",
    "# Apply the nest_asyncio patch\n",
    "nest_asyncio.apply()\n",
    "\n",
    "async def text_to_speech_and_play(text, voice=\"en-GB-MiaNeural\", speed=1):\n",
    "    try:\n",
    "        rate = \"+\" + str(int((speed - 1) * 100)) + \"%\"\n",
    "        communicate = edge_tts.Communicate(text, voice, rate=rate)\n",
    "        await communicate.save(\"output.mp3\")\n",
    "\n",
    "        # Load the saved audio file\n",
    "        sound = AudioSegment.from_file(\"output.mp3\", format=\"mp3\")\n",
    "\n",
    "        # Play the audio\n",
    "        play(sound)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error occurred during playback: {e}\")\n",
    "\n",
    "# Example usage\n",
    "text = \"Hello, this is a test of the edge-tts library for text-to-speech conversion.\"\n",
    "asyncio.run(text_to_speech_and_play(text, speed=1.2))  # Increase speed to 1.5x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(torch.cuda.is_available())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'num_loaded_entities'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\deletable\\OneDrive\\easy-local-rag\\force_GPU_localrag_no_rewrite.ipynb Cell 76\u001b[0m line \u001b[0;36m1\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y135sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m collection_load_state \u001b[39m=\u001b[39m utility\u001b[39m.\u001b[39mloading_progress(collection_name)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y135sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39m# Handle different load states\u001b[39;00m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y135sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m \u001b[39mif\u001b[39;00m collection_load_state[\u001b[39m\"\u001b[39;49m\u001b[39mnum_loaded_entities\u001b[39;49m\u001b[39m\"\u001b[39;49m] \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y135sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m     \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCollection \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mcollection_name\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m state: NotLoad\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y135sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m \u001b[39melif\u001b[39;00m collection_load_state[\u001b[39m\"\u001b[39m\u001b[39mnum_loaded_entities\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m<\u001b[39m collection_load_state[\u001b[39m\"\u001b[39m\u001b[39mnum_total_entities\u001b[39m\u001b[39m\"\u001b[39m]:\n",
      "\u001b[1;31mKeyError\u001b[0m: 'num_loaded_entities'"
     ]
    }
   ],
   "source": [
    "from pymilvus import connections, utility\n",
    "\n",
    "# Specify the collection name you want to check\n",
    "collection_name = \"html_chunks\"\n",
    "\n",
    "# Set up a Milvus client\n",
    "connections.connect(\"default\", host=\"localhost\", port=\"19530\")\n",
    "\n",
    "# Get the load state of the collection\n",
    "collection_load_state = utility.loading_progress(collection_name)\n",
    "\n",
    "# Handle different load states\n",
    "if collection_load_state[\"num_loaded_entities\"] == 0:\n",
    "    print(f\"Collection '{collection_name}' state: NotLoad\")\n",
    "elif collection_load_state[\"num_loaded_entities\"] < collection_load_state[\"num_total_entities\"]:\n",
    "    print(f\"Collection '{collection_name}' state: Loading\")\n",
    "elif collection_load_state[\"num_loaded_entities\"] == collection_load_state[\"num_total_entities\"]:\n",
    "    print(f\"Collection '{collection_name}' state: Loaded\")\n",
    "else:\n",
    "    print(f\"Collection '{collection_name}' state: NotExist\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Docker is running\n"
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "\n",
    "def is_docker_running():\n",
    "    try:\n",
    "        # Run the `docker info` command to check if Docker is running\n",
    "        result = subprocess.run(['docker', 'info'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
    "        if result.returncode == 0:\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "    except FileNotFoundError:\n",
    "        # Docker command not found\n",
    "        return False\n",
    "\n",
    "# Example usage\n",
    "if is_docker_running():\n",
    "    print(\"Docker is running\")\n",
    "else:\n",
    "    print(\"Docker is not running\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Tried to import the tflite runtime, but it was not found. Trying to switching to onnxruntime instead, if appropriate models are available.\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Tried to import the tflite runtime for provided tflite models, but it was not found. Please install it using `pip install tflite-runtime`",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\openwakeword\\model.py:114\u001b[0m, in \u001b[0;36mModel.__init__\u001b[1;34m(self, wakeword_models, class_mapping_dicts, enable_speex_noise_suppression, vad_threshold, custom_verifier_models, custom_verifier_threshold, inference_framework, **kwargs)\u001b[0m\n\u001b[0;32m    113\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 114\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mtflite_runtime\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39minterpreter\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mtflite\u001b[39;00m\n\u001b[0;32m    116\u001b[0m     \u001b[39mdef\u001b[39;00m \u001b[39mtflite_predict\u001b[39m(tflite_interpreter, input_index, output_index, x):\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tflite_runtime'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\deletable\\OneDrive\\easy-local-rag\\force_GPU_localrag_no_rewrite.ipynb Cell 78\u001b[0m line \u001b[0;36m6\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y140sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mnumpy\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mnp\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y140sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39m# Initialize OpenWakeWord\u001b[39;00m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y140sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m model \u001b[39m=\u001b[39m oww\u001b[39m.\u001b[39;49mModel(model_path\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m)  \u001b[39m# Default path loads available models\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y140sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39m# Add or configure the wake word if it's not pre-loaded\u001b[39;00m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y140sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m wake_word \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mhey llama\u001b[39m\u001b[39m\"\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\openwakeword\\utils.py:686\u001b[0m, in \u001b[0;36mre_arg.<locals>.decorator.<locals>.wrapped\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    683\u001b[0m         logging\u001b[39m.\u001b[39mwarning(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mDEPRECATION: keyword argument \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mk\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m is no longer valid and \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    684\u001b[0m                         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mwill be removed in future releases. Use \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mkwarg_map[k]\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m instead.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    685\u001b[0m     new_kwargs[kwarg_map\u001b[39m.\u001b[39mget(k, k)] \u001b[39m=\u001b[39m v\n\u001b[1;32m--> 686\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mnew_kwargs)\n",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\openwakeword\\model.py:130\u001b[0m, in \u001b[0;36mModel.__init__\u001b[1;34m(self, wakeword_models, class_mapping_dicts, enable_speex_noise_suppression, vad_threshold, custom_verifier_models, custom_verifier_threshold, inference_framework, **kwargs)\u001b[0m\n\u001b[0;32m    128\u001b[0m             wakeword_models \u001b[39m=\u001b[39m [i\u001b[39m.\u001b[39mreplace(\u001b[39m'\u001b[39m\u001b[39m.tflite\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39m.onnx\u001b[39m\u001b[39m'\u001b[39m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m wakeword_models]\n\u001b[0;32m    129\u001b[0m         \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> 130\u001b[0m             \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mTried to import the tflite runtime for provided tflite models, but it was not found. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    131\u001b[0m                              \u001b[39m\"\u001b[39m\u001b[39mPlease install it using `pip install tflite-runtime`\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    133\u001b[0m \u001b[39mif\u001b[39;00m inference_framework \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39monnx\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m    134\u001b[0m     \u001b[39mtry\u001b[39;00m:\n",
      "\u001b[1;31mValueError\u001b[0m: Tried to import the tflite runtime for provided tflite models, but it was not found. Please install it using `pip install tflite-runtime`"
     ]
    }
   ],
   "source": [
    "import openwakeword as oww\n",
    "import sounddevice as sd\n",
    "import numpy as np\n",
    "\n",
    "# Initialize OpenWakeWord\n",
    "model = oww.Model(model_path=None)  # Default path loads available models\n",
    "\n",
    "# Add or configure the wake word if it's not pre-loaded\n",
    "wake_word = \"hey llama\"\n",
    "model.add_phrase(wake_word)\n",
    "\n",
    "# Define a callback function to handle audio stream\n",
    "def audio_callback(indata, frames, time, status):\n",
    "    if status:\n",
    "        print(status)\n",
    "\n",
    "    # Convert the audio input to the format expected by OpenWakeWord\n",
    "    audio_data = np.frombuffer(indata, dtype=np.float32)\n",
    "    \n",
    "    # Check if the wake word is detected\n",
    "    if model(audio_data):\n",
    "        print(\"Wake word detected!\")\n",
    "        # Insert the action to be performed after wake word detection\n",
    "        interact_with_ollama()\n",
    "\n",
    "# Function to interact with Ollama after wake word detection\n",
    "def interact_with_ollama():\n",
    "    print(\"Listening for commands...\")\n",
    "\n",
    "    # Your existing logic to interact with the Ollama model goes here\n",
    "    user_input = input(\"Enter your message: \")\n",
    "    if user_input.lower() == \"exit\":\n",
    "        return\n",
    "    # Call your existing function to interact with the model\n",
    "    # chat_with_model(user_input, ...)\n",
    "\n",
    "# Start streaming from the microphone\n",
    "def start_audio_stream():\n",
    "    with sd.InputStream(callback=audio_callback, channels=1, samplerate=16000):\n",
    "        print(\"Listening for wake word...\")\n",
    "        while True:\n",
    "            sd.sleep(1000)\n",
    "\n",
    "# Start the wake word detection\n",
    "if __name__ == \"__main__\":\n",
    "    start_audio_stream()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\deletable\\Google Drive\\goodman Gilman 12e\\I. General Principles\\6. Drug Metabolism.htm\n",
      "\n",
      "C:\\Users\\deletable\\Google Drive\\goodman Gilman 12e\\I. General Principles\\6. Drug Metabolism.htm\n",
      "\n",
      "C:\\Users\\deletable\\Google Drive\\Medicine books\\Kaplan Psychiatry\\Kaplan and Sadocks Comprehensive Textbook of Psychiatry_files\\34.14_Carbamazepine.htm\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'os' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\deletable\\OneDrive\\easy-local-rag\\force_GPU_localrag_no_rewrite.ipynb Cell 79\u001b[0m line \u001b[0;36m3\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=27'>28</a>\u001b[0m \u001b[39mprint\u001b[39m(result)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=29'>30</a>\u001b[0m \u001b[39m# Extracting just the file names and making them clickable\u001b[39;00m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=30'>31</a>\u001b[0m texts \u001b[39m=\u001b[39m [os\u001b[39m.\u001b[39;49mpath\u001b[39m.\u001b[39;49mbasename(item[\u001b[39m'\u001b[39;49m\u001b[39mfile_name\u001b[39;49m\u001b[39m'\u001b[39;49m]) \u001b[39mfor\u001b[39;49;00m item \u001b[39min\u001b[39;49;00m search_result[\u001b[39m'\u001b[39;49m\u001b[39mmetadatas\u001b[39;49m\u001b[39m'\u001b[39;49m][\u001b[39m0\u001b[39;49m]]\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=32'>33</a>\u001b[0m \u001b[39m# Create clickable links (formatting may vary based on your terminal)\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=33'>34</a>\u001b[0m clickable_texts \u001b[39m=\u001b[39m [\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\\033\u001b[39;00m\u001b[39m]8;;file://\u001b[39m\u001b[39m{\u001b[39;00mitem[\u001b[39m'\u001b[39m\u001b[39mfile_name\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m}\u001b[39;00m\u001b[39m\\033\u001b[39;00m\u001b[39m\\\\\u001b[39;00m\u001b[39m{\u001b[39;00mos\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mbasename(item[\u001b[39m'\u001b[39m\u001b[39mfile_name\u001b[39m\u001b[39m'\u001b[39m])\u001b[39m}\u001b[39;00m\u001b[39m\\033\u001b[39;00m\u001b[39m]8;;\u001b[39m\u001b[39m\\033\u001b[39;00m\u001b[39m\\\\\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39mfor\u001b[39;00m item \u001b[39min\u001b[39;00m search_result[\u001b[39m'\u001b[39m\u001b[39mmetadatas\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]]\n",
      "\u001b[1;32mc:\\Users\\deletable\\OneDrive\\easy-local-rag\\force_GPU_localrag_no_rewrite.ipynb Cell 79\u001b[0m line \u001b[0;36m3\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=27'>28</a>\u001b[0m \u001b[39mprint\u001b[39m(result)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=29'>30</a>\u001b[0m \u001b[39m# Extracting just the file names and making them clickable\u001b[39;00m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=30'>31</a>\u001b[0m texts \u001b[39m=\u001b[39m [os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mbasename(item[\u001b[39m'\u001b[39m\u001b[39mfile_name\u001b[39m\u001b[39m'\u001b[39m]) \u001b[39mfor\u001b[39;00m item \u001b[39min\u001b[39;00m search_result[\u001b[39m'\u001b[39m\u001b[39mmetadatas\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]]\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=32'>33</a>\u001b[0m \u001b[39m# Create clickable links (formatting may vary based on your terminal)\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/deletable/OneDrive/easy-local-rag/force_GPU_localrag_no_rewrite.ipynb#Y141sZmlsZQ%3D%3D?line=33'>34</a>\u001b[0m clickable_texts \u001b[39m=\u001b[39m [\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\\033\u001b[39;00m\u001b[39m]8;;file://\u001b[39m\u001b[39m{\u001b[39;00mitem[\u001b[39m'\u001b[39m\u001b[39mfile_name\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m}\u001b[39;00m\u001b[39m\\033\u001b[39;00m\u001b[39m\\\\\u001b[39;00m\u001b[39m{\u001b[39;00mos\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mbasename(item[\u001b[39m'\u001b[39m\u001b[39mfile_name\u001b[39m\u001b[39m'\u001b[39m])\u001b[39m}\u001b[39;00m\u001b[39m\\033\u001b[39;00m\u001b[39m]8;;\u001b[39m\u001b[39m\\033\u001b[39;00m\u001b[39m\\\\\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39mfor\u001b[39;00m item \u001b[39min\u001b[39;00m search_result[\u001b[39m'\u001b[39m\u001b[39mmetadatas\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]]\n",
      "\u001b[1;31mNameError\u001b[0m: name 'os' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "import chromadb\n",
    "from chromadb.config import DEFAULT_TENANT, DEFAULT_DATABASE, Settings\n",
    "\n",
    "top_k = 3\n",
    "\n",
    "client = chromadb.PersistentClient(\n",
    "    settings=Settings(),\n",
    "    tenant=DEFAULT_TENANT,\n",
    "    database=DEFAULT_DATABASE,\n",
    ")\n",
    "\n",
    "# Get or create the collection\n",
    "collection_name = \"html_chunks\"\n",
    "collection = client.get_or_create_collection(name=collection_name)\n",
    "\"\"\"\n",
    "[-0.6356295943260193, 0.30443644523620605, 0.20280691981315613, 0.31135398149490356, -0.5034267902374268, 0.2658042907714844, -0.8977582454681396, 0.024995647370815277, -0.3600396513938904, 0.3522745966911316, -0.7889750003814697, -0.41680046916007996, 0.36920347809791565, -0.3339357078075409, -0.4677542448043823, -0.053941547870635986, 0.20812828838825226, -0.018922142684459686, 0.27403122186660767, 0.16290877759456635, 0.56324\n",
    "\"\"\"\n",
    "\n",
    "# Perform similarity search\n",
    "search_result = collection.query(\n",
    "    query_embeddings=[[-0.6356295943260193, 0.30443644523620605, 0.20280691981315613, 0.31135398149490356, -0.5034267902374268, 0.2658042907714844, -0.8977582454681396, 0.024995647370815277, -0.3600396513938904, 0.3522745966911316, -0.7889750003814697, -0.41680046916007996, 0.36920347809791565, -0.3339357078075409, -0.4677542448043823, -0.053941547870635986, 0.20812828838825226, -0.018922142684459686, 0.27403122186660767, 0.16290877759456635, 0.5632448196411133, 0.5337744355201721, -0.6507474780082703, -0.1623895913362503, 0.03621070086956024, -0.12580707669258118, 0.9864926338195801, -0.7188980579376221, 1.0028506517410278, 0.06788843870162964, -0.06789693236351013, -0.513510525226593, 0.7000594735145569, -0.8249821662902832, 0.4538975656032562, -0.546875, 0.16801097989082336, -0.16040831804275513, 0.2166394144296646, 0.22505341470241547, 0.3404287099838257, 0.5449384450912476, 0.6172062158584595, 0.14243784546852112, -0.9649701714515686, -0.4360896944999695, -0.13985326886177063, -0.4795413613319397, -0.015353974886238575, 0.44867584109306335, -0.21299761533737183, 0.22973376512527466, 0.3451249301433563, -0.05053294077515602, 0.14284515380859375, 0.6392150521278381, -0.0024933461099863052, -0.18543016910552979, -0.16475313901901245, 0.7157194018363953, 0.28231513500213623, 0.2975705862045288, 0.10309626162052155, -0.6298861503601074, 0.6494780778884888, 1.3590682744979858, -0.4988357424736023, -0.08849099278450012, 0.06397508084774017, -0.5411152839660645, 0.5053094625473022, -0.527260959148407, -0.267977774143219, -0.004141584038734436, -1.1163352727890015, -0.31634730100631714, 0.2004377245903015, 0.4070836901664734, 0.6347278952598572, -0.12162886559963226, -0.5323264598846436, 0.24483294785022736, 0.8039793968200684, -0.26549071073532104, -0.6790412068367004, 1.0549616813659668, -0.38765212893486023, 0.6657972931861877, -0.4320064187049866, 0.1686834841966629, -0.22565987706184387, 0.6205726265907288, -0.145903542637825, -0.004813186824321747, -0.5697376728057861, 0.6937528252601624, -1.1816707849502563, -0.2811824679374695, 0.6447184085845947, 0.3280942738056183, -0.02387501671910286, -0.3259039521217346, -0.3660866320133209, 0.5150827765464783, -0.7968448400497437, -0.01477990671992302, 0.9045580625534058, 0.08435997366905212, -1.1397532224655151, -0.3221482038497925, -0.16645489633083344, 0.5347068905830383, -0.9190102815628052, -0.5464086532592773, 0.19465412199497223, 0.3516436219215393, 0.1691083163022995, 0.16593614220619202, -1.0752745866775513, 0.6059240698814392, 0.6021680235862732, -0.42770296335220337, 0.5329254269599915, -0.07441462576389313, 0.548255980014801, 0.26084601879119873, 0.02948981523513794, 0.9036267995834351, -0.9133635759353638, -0.15525943040847778, -0.3696145713329315, -0.22493115067481995, 0.36625269055366516, 0.05808873847126961, 0.2069566547870636, 0.02149207517504692, -0.08984746038913727, 0.5690158009529114, -0.1290569007396698, -0.5877832174301147, 0.6144800186157227, 1.3212443590164185, -0.412470281124115, 1.109380841255188, -0.08998868614435196, 0.8545292615890503, -0.02220149338245392, -0.4932103753089905, -0.3816225528717041, 0.4552813470363617, 0.4955340623855591, -0.50367271900177, -0.4818374216556549, 0.29898715019226074, -0.768745481967926, 0.28148898482322693, -0.1698964238166809, 0.8639934062957764, -0.4461216330528259, -0.01920936070382595, -0.5453159213066101, 0.14661797881126404, 0.3911826014518738, 0.26281923055648804, -0.9050867557525635, 0.5990042686462402, -0.634423017501831, 0.5916415452957153, -1.4138869047164917, -0.8449857234954834, 0.9598507285118103, -0.399242103099823, -0.20747287571430206, 0.16230559349060059, 0.07988214492797852, 0.2841717004776001, 0.7050260901451111, 0.37181124091148376, 0.47310999035835266, 0.2519637942314148, -0.11916328221559525, -0.4511711001396179, 0.49862655997276306, 0.8796514272689819, -0.5261385440826416, 0.32042253017425537, 0.5895456671714783, 0.778410017490387, -0.27875205874443054, -0.27702653408050537, 0.4393885135650635, 0.5934813022613525, -0.9575445652008057, 0.07971829921007156, -0.4269765615463257, 0.2677367925643921, -1.310658574104309, 0.7807843685150146, -0.05410294234752655, -0.4448789358139038, -0.19834062457084656, 0.3195054233074188, -0.6442908048629761, -0.1757310926914215, 0.24003776907920837, 0.4516682028770447, -0.2950725853443146, 0.7867997884750366, 0.6048092842102051, 0.8357210159301758, 1.0454741716384888, 0.5385309457778931, -0.005026469938457012, -0.05221301317214966, -0.023670606315135956, -0.26159849762916565, -0.003160431981086731, 0.19431866705417633, -0.1938720941543579, 0.29677629470825195, -0.16733889281749725, 0.12277771532535553, 0.729619562625885, 0.46725863218307495, 0.4523926079273224, 0.7979772686958313, 0.6215299367904663, 0.6624202132225037, -0.2538185715675354, -0.021305091679096222, -0.08324340730905533, 0.21318107843399048, 0.09809932112693787, 0.5983430743217468, -0.7681514620780945, -0.11565998196601868, 0.6108688116073608, 0.45912864804267883, 0.061092548072338104, -0.5476665496826172, 0.3025435507297516, 0.3165351450443268, 0.8110254406929016, -0.39104723930358887, 0.26497167348861694, -0.10361117124557495, -0.386578232049942, -0.4510408043861389, -0.5720421671867371, 0.6509406566619873, -0.39842289686203003, 1.079506516456604, 0.8186306953430176, 0.5665601491928101, -0.17532020807266235, -0.8176208734512329, 0.5578778386116028, -0.2676686644554138, -0.6669120192527771, -0.36930811405181885, 0.1818803995847702, 0.5343548059463501, -0.7481024861335754, 0.11871972680091858, 0.38534167408943176, 1.1003361940383911, 0.020617520436644554, 0.7635549306869507, -0.2808392345905304, 0.10431066155433655, -0.6449786424636841, -0.7170246839523315, -1.184220790863037, 0.19789054989814758, -0.46233609318733215, -0.3038467466831207, -0.1182783842086792, 0.11086414754390717, 0.030545204877853394, -0.49134311079978943, 0.3610839247703552, -0.29419010877609253, 0.7959008812904358, 0.117757648229599, -0.3101486265659332, 0.11459718644618988, -0.3724847435951233, 0.8782446980476379, -0.14071723818778992, 0.11639556288719177, -0.29548490047454834, -0.019688043743371964, -1.095634937286377, -0.2287035584449768, 0.43737727403640747, -0.340981662273407, 0.363934725522995, 0.19415058195590973, -0.18098999559879303, 0.21197694540023804, -0.54720538854599, -0.17787878215312958, -0.18333540856838226, -0.3493005931377411, 0.1160973608493805, 0.663384199142456, 0.20583248138427734, 0.44187071919441223, 0.7165032029151917, 0.18555229902267456, -0.15695522725582123, -0.015582896769046783, 0.2144198715686798, 0.4859273433685303, -0.9782689809799194, 0.10316941142082214, 0.8631366491317749, 0.4430571496486664, -0.027217544615268707, -0.593692421913147, -0.3229045271873474, -0.05989491939544678, -0.48413562774658203, -0.04212408512830734, 0.9258332848548889, 0.515265166759491, 0.29866600036621094, -1.1789438724517822, -0.005110146477818489, 0.19201251864433289, -0.4767605662345886, -0.7806530594825745, -0.09268759191036224, 0.052438538521528244, -0.09465068578720093, 0.6759461760520935, -0.39966240525245667, 0.01192743144929409, 0.25929561257362366, 0.29158538579940796, -0.24538180232048035, 0.04296952858567238, 0.07123641669750214, 0.5614143013954163, 0.597213089466095, 0.4515298306941986, 0.36818405985832214, -0.04988826811313629, -0.1065785139799118, -0.10970988869667053, -0.4028429388999939, 0.376934289932251, 0.46930158138275146, 0.1866099238395691, -0.022927626967430115, 0.05722881853580475, 0.3203786611557007, -0.4935929775238037, 0.3631371855735779, 0.1976204812526703, -0.31426680088043213, -0.51776123046875, 0.33471256494522095, 0.6075355410575867, -0.24345871806144714, -0.8583675026893616, 0.9246248006820679, -0.2949109375476837, 0.2583586573600769, -0.4159403145313263, 0.45417070388793945, 0.11249470710754395, -0.26048174500465393, -0.29865550994873047, -0.25096502900123596, -1.0323820114135742, 0.15998464822769165, -0.12356597185134888, 0.9243184328079224, -1.0097675323486328, 0.32005441188812256, -0.4988463819026947, 1.173769235610962, 0.044698990881443024, 0.21519474685192108, 0.004016496241092682, -0.9313920140266418, 1.0093644857406616, -0.17423591017723083, 0.5343116521835327, -0.29523029923439026, 0.19120752811431885, 0.936451256275177, -0.30376872420310974, -1.4660130739212036, -0.6526172757148743, 0.34731921553611755, 0.3749508261680603, -0.08389432728290558, -0.11721187829971313, 0.0518893226981163, 0.4601473808288574, 0.6623015403747559, -0.008302115835249424, -0.11927421391010284, -0.035679012537002563, -0.12708131968975067, 0.4758574664592743, 0.22526592016220093, -0.03148771449923515, -0.3212296962738037, 0.7070837616920471, 0.18626387417316437, 0.20492003858089447, 0.2761208415031433, 0.43320590257644653, -0.9353894591331482, 0.14910297095775604, -0.08525629341602325, 0.0038226544857025146, -0.4441908597946167, -0.1661405861377716, -0.747835636138916, -0.07370395958423615, -0.458076536655426, -0.2509487271308899, -0.38089022040367126, -0.44214168190956116, -0.0560910701751709, 0.8272272348403931, 0.11659994721412659, -0.6896398663520813, 0.15009744465351105, -0.22595205903053284, -0.44584527611732483, 0.022752799093723297, 0.3025949001312256, -0.07590273767709732, -0.1107652336359024, 0.07741949707269669, -0.030375152826309204, -0.7736625075340271, 0.18909406661987305, 0.1410152018070221, 0.01013525202870369, 0.2827032804489136, 1.404625654220581, 0.22915199398994446, 0.22299335896968842, -0.3764198124408722, -0.009057460352778435, -0.4517810046672821, 0.6066561937332153, 0.2738346755504608, -0.8375136852264404, 0.29799985885620117, -0.3193308711051941, -0.260483980178833, 0.1502709686756134, -0.21340228617191315, -0.040987446904182434, -0.7318118810653687, 0.6686577200889587, 0.1904929280281067, -0.05169554054737091, 0.7766177654266357, -0.206859290599823, 0.24793019890785217, 0.222746804356575, 0.6418781280517578, 0.2600628733634949, 0.6845930814743042, -0.11765843629837036, -0.43175435066223145, 0.1564560979604721, -0.018162619322538376, -0.38884425163269043, 1.4618372917175293, 0.21781711280345917, 0.1021563857793808, -0.49117711186408997, 0.1602267026901245, -0.09037631750106812, -0.3311360478401184, -0.11723226308822632, -0.5122638940811157, -0.5164554715156555, -0.4842739701271057, 0.13169686496257782, 0.233375683426857, 0.9024423956871033, -0.14903762936592102, 0.16380390524864197, 0.45096898078918457, -0.4379960894584656, -0.7008974552154541, -0.2922176420688629, -0.2841012477874756, 0.005074694752693176, -0.15587103366851807, 1.428929090499878, -0.6168954372406006, -0.286298006772995, -0.6602852940559387, -0.40621402859687805, -0.6621814370155334, -0.4419342577457428, 0.33044174313545227, 0.013556335121393204, -0.46048951148986816, -0.4870924949645996, 0.2428191900253296, 0.5363143682479858, -0.6077966094017029, 0.8114022612571716, -0.3070627748966217, 0.07562687247991562, 0.13693329691886902, -0.7023646235466003, 0.3106388747692108, 0.1445322334766388, -0.13522571325302124, 0.05644333362579346, -0.0438406765460968, -0.6632128953933716, 0.13819994032382965, 0.7871968746185303, -0.6784119009971619, -0.11232555657625198, -1.6336243152618408, 0.7522643208503723, -0.7834001183509827, -0.1765022724866867, -0.5867566466331482, -0.3020797073841095, -0.1647185981273651, 0.5323359370231628, 0.9770557880401611, 0.48638850450515747, 0.07223464548587799, -0.31390973925590515, -0.018670495599508286, -0.14777885377407074, -0.11998744308948517, -0.8896119594573975, -0.7015586495399475, -0.9194685220718384, 0.2111382782459259, 0.3016636371612549, -0.44098103046417236, -0.2077137678861618, 0.560275673866272, 0.2597046494483948, -0.19569921493530273, -0.43230128288269043, -0.5371386408805847, -0.5379911065101624, 0.5993067026138306, -0.14817021787166595, 0.02637903206050396, -0.6650094389915466, 0.11296067386865616, -0.17674022912979126, 0.42388758063316345, 0.44789639115333557, -0.22775350511074066, 0.21773399412631989, -0.37888103723526, 0.8082832098007202, 0.3991642892360687, -0.5500142574310303, -0.3134237229824066, 0.21801814436912537, -0.7288543581962585, 0.43829602003097534, -0.5350544452667236, -0.1368848979473114, -0.6682260632514954, -1.4703024625778198, 0.8854314684867859, 0.08771681785583496, 0.07819792628288269, 0.3846578598022461, -0.3278235197067261, 0.26518967747688293, -0.7480456829071045, 1.0855956077575684, -0.12699507176876068, -0.7509798407554626, -0.6588702201843262, 0.6340868473052979, 0.8851154446601868, -0.4257168769836426, -0.503755509853363, -0.0940280333161354, 0.5390630960464478, -0.06265701353549957, -0.5197319984436035, -0.7388554215431213, -0.7729600071907043, 0.0060540903359651566, 0.9390827417373657, 0.324600487947464, 0.8628858327865601, 0.1712632030248642, -0.033721037209033966, 0.13698810338974, -0.03017233870923519, 0.282376229763031, -0.14472520351409912, 0.2875179946422577, 0.22093889117240906, 0.049793098121881485, -0.40104755759239197, -0.2714282274246216, 0.008337259292602539, -0.23010584712028503, -0.341207355260849, 0.36184337735176086, 0.029922710731625557, -1.079835057258606, 0.6473352313041687, -0.12561064958572388, -0.4855438470840454, -0.25615501403808594, 0.546730637550354, 0.12279000878334045, -0.8737667798995972, 0.0957171767950058, 0.2902389168739319, 0.6890752911567688, 0.03351587802171707, -0.17063283920288086, 0.2979131042957306, -0.4374488890171051, 0.6736552715301514, -0.6454824209213257, 0.5213987231254578, -0.20446136593818665, -0.950809121131897, 0.7589296102523804, -0.3076208829879761, 0.012500826269388199, -0.5468019247055054, 0.7306820154190063, -0.08256392180919647, -0.5232702493667603, 0.1307363361120224, -0.2663514316082001, -0.08290831744670868, 0.2086411714553833, 0.21990752220153809, -0.15789233148097992, -0.11033633351325989, -0.1428796648979187, 0.33171316981315613, 0.143754780292511, -0.37248921394348145, -0.9867744445800781, 0.21646566689014435, 0.7588269710540771, -0.4285925030708313, -0.7239342331886292, -1.364531397819519, 0.6106533408164978, 0.027091272175312042, -0.5254396200180054, -0.14599883556365967, -0.2322789430618286, -0.983940601348877, -0.47773733735084534, -0.5251028537750244, -0.8377848863601685, 0.2102552354335785, 0.023716866970062256, -0.446683406829834, 0.2001984417438507, -0.5517518520355225, 0.9283438920974731, -0.11489316821098328, 0.16406917572021484, -1.0824949741363525, 0.2849193215370178, -0.8829202055931091, -0.5927539467811584, -0.15358389914035797, -0.039714328944683075, -0.06891696900129318, -0.302928626537323, -0.09119056165218353, 0.0957060232758522, 0.3444592356681824, 0.10698999464511871, -0.23019877076148987, -0.04852449893951416, 0.07632453739643097, -1.0777418613433838, -0.18246212601661682, 0.4407045841217041, -0.9484966993331909, 0.12614640593528748, 0.3467027544975281, 0.21330639719963074, 1.1891840696334839, 0.3433381915092468, 0.10491400957107544, -0.5290055871009827, -0.6450383067131042, 0.2582719027996063, -0.3070856034755707, -1.2089899778366089, -0.6051779389381409, 0.2951289713382721, -0.9710834622383118, -0.37152111530303955, -0.952155590057373, -0.26107969880104065, -0.05090443044900894, 0.6933125853538513, -0.6890741586685181, 0.10946905612945557, -0.14336638152599335, -0.06184860318899155, -0.09613153338432312, 1.1214008331298828, 0.27132973074913025, 0.024517595767974854, 0.4660326838493347, -0.584222674369812, -0.20549774169921875, 0.6471846699714661, -0.4817051291465759, 0.42979198694229126, -0.28896671533584595, 0.2469446361064911, -0.1409761905670166, 0.49655622243881226, -0.2590743899345398, -0.33039698004722595, 0.25905659794807434, 0.569513201713562, 0.43976718187332153, -0.03700309246778488, -0.3654700815677643, -0.46205729246139526, -0.663243293762207, -0.06134049594402313, -0.3262154757976532, 0.25772252678871155, -0.13822638988494873, -0.10836063325405121, -0.38024020195007324, 0.18240530788898468, -0.5312700271606445, 0.2971050441265106, 0.06476420909166336, 0.5139998197555542, -0.023338057100772858, -0.1828857958316803, 0.69744873046875, 0.6493849158287048, -0.5880656838417053, 0.18603257834911346, 0.3649701178073883, 0.005371298640966415, -0.30936264991760254, -0.2619839310646057, -0.11533337831497192, 0.8874173760414124, 0.4622904658317566, -0.5862589478492737, -0.13684016466140747, -0.1617700159549713, -0.28405052423477173, 0.20821534097194672, -0.16388723254203796, -0.8301358222961426, -1.1848950386047363, -0.11947742104530334, -0.4703727662563324, -0.1559082418680191, -0.9917868375778198, 0.13105008006095886, -0.014321278780698776, -0.1444891393184662, -0.46907132863998413, -0.010132748633623123, 0.8551155924797058, 0.29468250274658203, 0.019679509103298187, 0.543715238571167, 1.173114538192749, -0.40101924538612366, 0.19440263509750366, 0.05437031388282776, 0.1218879222869873, 0.5549723505973816, -0.003191642463207245, -0.20085757970809937, 0.12353415787220001, 0.7567529678344727, -0.37419313192367554, 0.14106884598731995, 0.14888450503349304, -0.20837244391441345, -0.11447983980178833, -0.00033619627356529236, -0.5416716933250427, 0.6498554348945618, 0.4731717109680176, 0.0800071731209755, -0.2503551244735718, 0.4939233064651489, -0.3914499282836914, 0.8099855780601501, 0.33711177110671997, -0.2004203498363495, -1.1891320943832397, -0.15059004724025726, -0.2696514129638672, 0.771365761756897, 0.1803327202796936, -0.09942473471164703, -0.14238086342811584, -0.5588799715042114, 0.7556812763214111, 0.5195324420928955, 0.1181153655052185, 0.6277140974998474, 0.08855551481246948, -0.8417577743530273, -0.5606439709663391, 0.7647442817687988, 0.8403286337852478, 0.3920360803604126, -0.036455847322940826, 0.24877291917800903, -0.6560336947441101, 0.5588562488555908, -0.5675591230392456, -0.043635472655296326, 0.6340799331665039, 0.2689315676689148, -0.5026146173477173, -0.7258998155593872, 0.7554991245269775, -0.44978564977645874, 0.17890247702598572, 0.49349305033683777, 0.07539166510105133, 0.24494680762290955, -0.044873252511024475, -0.07409633696079254, 1.5578782558441162, 0.4973260760307312, 0.0797274261713028, -0.8621687889099121, 0.6363574266433716, 0.19618692994117737, 0.16286104917526245, 0.6584277749061584, 0.17166227102279663, -0.7608713507652283, -0.29641497135162354, -0.49752238392829895, 0.6210159659385681, -0.09452828019857407, 0.3628080487251282, -0.7435675859451294, -0.3920072615146637, 0.34434616565704346, -0.6346385478973389, -0.7489935755729675, 0.0032858997583389282, 0.10242318361997604, -0.1867717206478119, 0.11608196794986725, 0.7346366047859192, 0.3640761077404022, 0.2832483947277069, -0.3645772337913513, -0.2608571946620941, -0.215126171708107, -0.5664611458778381, -0.7556113600730896, 0.42286068201065063, -0.25128450989723206, -0.3750276267528534, -0.2571601867675781, -0.4207020699977875, -0.4393715560436249, 0.19497926533222198, -0.6915786862373352, -0.29131650924682617, 0.1663101464509964, 0.20882119238376617, 0.2933160960674286, 0.11405909061431885, -0.4083311855792999, 0.1890886276960373, -0.3053746223449707, 0.4177943468093872, -0.27505940198898315, 1.0924288034439087, 0.0039606839418411255, 0.006342977285385132, -0.3030489683151245, -0.14650602638721466, 0.29217272996902466, -0.7084975242614746, -0.3428809642791748, -0.48579713702201843, 0.08228877186775208, -0.592500627040863, -0.4595879316329956, 0.5094588994979858, 0.5250687599182129, -0.18069645762443542, -0.5042603611946106, -0.695640504360199, 0.4261232316493988, 0.030743397772312164, 0.014631934463977814, 0.028413884341716766, 0.09668408334255219, 0.25397056341171265, 0.3529687225818634, 0.3868337571620941, -0.855126142501831, 4.235260009765625, 0.9560744762420654, 0.6035435795783997, 0.42477405071258545, 0.6662796139717102, 0.43407297134399414, 0.7622981071472168, -0.15140068531036377, 0.10365205258131027, -0.22116076946258545, -1.3133785724639893, 0.38363268971443176, 0.26724016666412354, 0.37172359228134155, 0.18615609407424927, -0.05587582290172577, -0.5500074028968811, 0.7178911566734314, -0.20556575059890747, -0.5297616720199585, -0.15155205130577087, -0.2553306818008423, -0.4816346764564514, 0.351627379655838, 0.20019638538360596, -0.5606828927993774, -0.2223605513572693, 0.5079123377799988, 0.11755114793777466, -0.6610434055328369, -0.14568625390529633, -0.7494722008705139, 0.35892629623413086, -0.5179727077484131, -0.59076327085495, 0.7619636058807373, 0.4519079029560089, -0.48193150758743286, 0.34680861234664917, -0.34397777915000916, 0.10551229119300842, 0.21564921736717224, 0.28877541422843933, -0.07116256654262543, -0.18951821327209473, 0.5492172837257385, -0.32368409633636475, -0.12838426232337952, 0.2746536135673523, -0.3042498826980591, 1.0889232158660889, -1.3389211893081665, 0.3156415820121765, -0.2736278772354126, -0.7778304815292358, -0.3222159445285797, 0.24777215719223022, -0.7749565839767456, 0.3537411689758301, 0.5297192335128784, 1.1156270503997803, -0.09839405119419098, -0.7189709544181824, 0.5046054720878601, 0.11321356892585754, -0.373166024684906, 0.2454410195350647, 0.4880473017692566, -0.053457438945770264, -0.5199326872825623, 0.09813719987869263, 0.0841895118355751, -0.11548393964767456, -1.069744348526001, 0.10653255879878998, 0.5705379247665405, 0.09658079594373703, 0.28702378273010254, 0.7485814690589905, -0.2249084860086441, 0.11067014932632446, -0.08582058548927307, -0.200260192155838, -0.5628126859664917, 0.5252023935317993, 0.09158893674612045, 0.12103450298309326, 0.021535858511924744, -0.720868706703186, 0.158065065741539, 0.38065338134765625, -0.49424225091934204, 0.03765846788883209, -0.7396163940429688, -0.5127720832824707]],\n",
    "    n_results=top_k,\n",
    "    include=[\"metadatas\"],  # Fields to return in the search results\n",
    ")\n",
    "\n",
    "# Extracting the text fields and joining them with a double newline separator\n",
    "texts = [item['file_name'] for item in search_result['metadatas'][0]]\n",
    "result = \"\\n\\n\".join(texts)\n",
    "\n",
    "print(result)\n",
    "\n",
    "# Extracting just the file names and making them clickable\n",
    "texts = [os.path.basename(item['file_name']) for item in search_result['metadatas'][0]]\n",
    "\n",
    "# Create clickable links (formatting may vary based on your terminal)\n",
    "clickable_texts = [f\"\\033]8;;file://{item['file_name']}\\033\\\\{os.path.basename(item['file_name'])}\\033]8;;\\033\\\\\" for item in search_result['metadatas'][0]]\n",
    "\n",
    "result = \"\\n\\n\".join(clickable_texts)\n",
    "\n",
    "print(result)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "delete_collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Collection html_chunks does not exist.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[9], line 7\u001b[0m\n\u001b[0;32m      4\u001b[0m client \u001b[38;5;241m=\u001b[39m chromadb\u001b[38;5;241m.\u001b[39mPersistentClient(path\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mUsers\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mdeletable\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mOneDrive\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124measy-local-rag\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mchroma\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m      6\u001b[0m \u001b[38;5;66;03m# Delete the collection\u001b[39;00m\n\u001b[1;32m----> 7\u001b[0m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete_collection\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhtml_chunks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\chromadb\\api\\client.py:198\u001b[0m, in \u001b[0;36mClient.delete_collection\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m    193\u001b[0m \u001b[38;5;129m@override\u001b[39m\n\u001b[0;32m    194\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete_collection\u001b[39m(\n\u001b[0;32m    195\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    196\u001b[0m     name: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m    197\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 198\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_server\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete_collection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    199\u001b[0m \u001b[43m        \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    200\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtenant\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtenant\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    201\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdatabase\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdatabase\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    202\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\chromadb\\telemetry\\opentelemetry\\__init__.py:146\u001b[0m, in \u001b[0;36mtrace_method.<locals>.decorator.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    144\u001b[0m \u001b[38;5;28;01mglobal\u001b[39;00m tracer, granularity\n\u001b[0;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m trace_granularity \u001b[38;5;241m<\u001b[39m granularity:\n\u001b[1;32m--> 146\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    147\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m tracer:\n\u001b[0;32m    148\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m f(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\deletable\\OneDrive\\Windows_software\\openai whisper\\openai\\Lib\\site-packages\\chromadb\\api\\segment.py:326\u001b[0m, in \u001b[0;36mSegmentAPI.delete_collection\u001b[1;34m(self, name, tenant, database)\u001b[0m\n\u001b[0;32m    324\u001b[0m         \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_collection_cache[existing[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mid]\n\u001b[0;32m    325\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 326\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCollection \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not exist.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mValueError\u001b[0m: Collection html_chunks does not exist."
     ]
    }
   ],
   "source": [
    "import chromadb\n",
    "\n",
    "# Initialize the client\n",
    "client = chromadb.PersistentClient(path=r\"C:\\Users\\deletable\\OneDrive\\easy-local-rag\\chroma\")\n",
    "\n",
    "# Delete the collection\n",
    "client.delete_collection(\"html_chunks\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (openai)",
   "language": "python",
   "name": "openai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}