Attempt to support LM Studio, blocked by LiteLLM

SciPhi-AI · Jan 9, 2025 · 493a7cd · 493a7cd
1 parent 5ee250b
commit 493a7cd
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 3 deletions.
diff --git a/py/core/configs/lm_studio.toml b/py/core/configs/lm_studio.toml
@@ -0,0 +1,61 @@
+[agent]
+system_instruction_name = "rag_agent"
+tool_names = ["local_search"]
+
+  [agent.generation_config]
+  model = "lm_studio/llama-3.2-3b-instruct"
+
+[completion]
+provider = "litellm"
+concurrent_request_limit = 1
+
+  [completion.generation_config]
+  model = "lm_studio/llama-3.2-3b-instruct"
+  temperature = 0.1
+  top_p = 1
+  max_tokens_to_sample = 1_024
+  stream = false
+  add_generation_kwargs = { }
+
+[embedding]
+provider = "litellm"
+base_model = "lm_studio/text-embedding-nomic-embed-text-v1.5"
+base_dimension = nan
+batch_size = 128
+add_title_as_prefix = true
+concurrent_request_limit = 2
+
+[database]
+provider = "postgres"
+
+  [database.graph_creation_settings]
+    graph_entity_description_prompt = "graphrag_entity_description"
+    entity_types = [] # if empty, all entities are extracted
+    relation_types = [] # if empty, all relations are extracted
+    fragment_merge_count = 4 # number of fragments to merge into a single extraction
+    max_knowledge_relationships = 100
+    max_description_input_length = 65536
+    generation_config = { model = "lm_studio/llama-3.2-3b-instruct" } # and other params, model used for relationshipt extraction
+
+  [database.graph_enrichment_settings]
+    community_reports_prompt = "graphrag_community_reports"
+    max_summary_input_length = 65536
+    generation_config = { model = "lm_studio/llama-3.2-3b-instruct" } # and other params, model used for node description and graph clustering
+    leiden_params = {}
+
+  [database.graph_search_settings]
+    generation_config = { model = "lm_studio/llama-3.2-3b-instruct" }
+
+
+[orchestration]
+provider = "simple"
+
+
+[ingestion]
+vision_img_model = "lm_studio/llama3.2-vision"
+vision_pdf_model = "lm_studio/llama3.2-vision"
+chunks_for_document_summary = 16
+document_summary_model = "lm_studio/llama-3.2-3b-instruct"
+
+  [ingestion.extra_parsers]
+    pdf = "zerox"
diff --git a/py/poetry.lock b/py/poetry.lock
diff --git a/py/pyproject.toml b/py/pyproject.toml
@@ -61,7 +61,7 @@ future = { version = "^1.0.0", optional = true }
 graspologic = { version = "^3.4.1", optional = true }
 gunicorn = { version = "^21.2.0", optional = true }
 hatchet-sdk = { version = "^0.38.0", optional = true }
-litellm = { version = "^1.52.8", optional = true }
+litellm = { version = "^1.57.4", optional = true }
 networkx = { version = "^3.3", optional = true }
 ollama = { version = "^0.3.1", optional = true }
 passlib = { version = "^1.7.4", optional = true }

diff --git a/py/r2r/compose.full.yaml b/py/r2r/compose.full.yaml
@@ -345,6 +345,9 @@ services:
       # Ollama
       - OLLAMA_API_BASE=${OLLAMA_API_BASE:-http://host.docker.internal:11434}
 
+      # LM Studio
+      - LM_STUDIO_API_BASE=${LM_STUDIO_API_BASE:-http://host.docker.internal:1234}
+
       # Huggingface
       - HUGGINGFACE_API_BASE=${HUGGINGFACE_API_BASE:-http://host.docker.internal:8080}
       - HUGGINGFACE_API_KEY=${HUGGINGFACE_API_KEY}

diff --git a/py/r2r/compose.full_with_replicas.yaml b/py/r2r/compose.full_with_replicas.yaml
@@ -343,6 +343,9 @@ services:
       # Ollama
       - OLLAMA_API_BASE=${OLLAMA_API_BASE:-http://host.docker.internal:11434}
 
+      # LM Studio
+      - LM_STUDIO_API_BASE=${LM_STUDIO_API_BASE:-http://host.docker.internal:1234}
+
       # Huggingface
       - HUGGINGFACE_API_BASE=${HUGGINGFACE_API_BASE:-http://host.docker.internal:8080}
       - HUGGINGFACE_API_KEY=${HUGGINGFACE_API_KEY}

diff --git a/py/r2r/compose.yaml b/py/r2r/compose.yaml
@@ -101,6 +101,9 @@ services:
       # Ollama
       - OLLAMA_API_BASE=${OLLAMA_API_BASE:-http://host.docker.internal:11434}
 
+      # LM Studio
+      - LM_STUDIO_API_BASE=${LM_STUDIO_API_BASE:-http://host.docker.internal:1234}
+
     networks:
       - r2r-network
     healthcheck: