feat: add multimodal retrieval

.gitignore

@@ -10,4 +10,7 @@
 .env.production.local
 
 # devbox stuff
-.devbox/*
+.devbox/*
+
+# python cache
+__pycache__

Makefile

@@ -6,6 +6,7 @@ build:
 	bun run build
 
 dev:
+	docker-compose -p preprocessor -f ./preprocessor/docker-compose.yml up -d
 	bun dev
 
 local-up: db
@@ -22,10 +23,11 @@ db-down-prune:
 	docker-compose -p chatbot_builder -f ./build/local/docker-compose.yml down --volumes
 
 db-migrate:
-	bun drizzle-kit push:pg
+	bun drizzle-kit push
 
 db-seed:
 	bun src/server/db/migration/seed.ts
 
 lint-staged:
 	bun lint-staged
+

build/local/docker-compose.yml

@@ -15,3 +15,4 @@ volumes:
 networks:
   default:
     name: chatbot-builder-network
+    external: true

drizzle.config.ts

@@ -4,9 +4,13 @@ import { env } from '~/env'
 
 export default {
   schema: './src/server/db/migration/schema.ts',
-  driver: 'pg',
+  dialect: 'postgresql',
   dbCredentials: {
-    connectionString: env.DATABASE_URL,
+    url: env.DATABASE_URL,
+  },
+  migrations: {
+    table: 'migrations',
+    schema: 'public',
   },
   // tablesFilter: ["chatbot-builder_*"],
   out: './src/server/db/migrations',

package.json

@@ -16,7 +16,9 @@
     "@dwarvesf/react-hooks": "^0.8.2",
     "@dwarvesf/react-utils": "^0.4.2",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/community": "^0.2.3",
+    "@langchain/community": "^0.2.13",
+    "@langchain/core": "^0.2.9",
+    "@langchain/openai": "^0.2.0",
     "@mochi-ui/core": "^0.14.3",
     "@mochi-ui/icons": "^0.8.1",
     "@mochi-ui/theme": "^0.20.1",
@@ -28,12 +30,14 @@
     "@trpc/next": "next",
     "@trpc/react-query": "next",
     "@trpc/server": "next",
+    "@types/pg": "^8.11.6",
     "@types/react-color": "^3.0.12",
     "@vercel/blob": "^0.23.2",
     "async": "^3.2.5",
+    "buffer": "^6.0.3",
     "clsx": "^2.1.1",
     "dayjs": "^1.11.11",
-    "drizzle-orm": "^0.29.4",
+    "drizzle-orm": "^0.31.0",
     "fast-xml-parser": "^4.3.6",
     "jsdom": "^24.0.0",
     "langchain": "^0.2.2",
@@ -42,12 +46,14 @@
     "next": "^14.1.3",
     "next-auth": "^4.24.6",
     "openai": "^4.42.0",
+    "pg": "^8.12.0",
     "postgres": "^3.4.4",
     "react": "18.2.0",
     "react-color": "^2.19.3",
     "react-dom": "18.2.0",
     "react-dropzone": "^14.2.3",
     "react-hook-form": "^7.51.3",
+    "sharp": "^0.33.4",
     "superjson": "^2.2.1",
     "tailwindcss-animate": "^1.0.7",
     "use-debounce": "^10.0.1",
@@ -68,7 +74,7 @@
     "@typescript-eslint/parser": "^7.1.1",
     "cheerio": "^1.0.0-rc.12",
     "domhandler": "^5.0.3",
-    "drizzle-kit": "^0.20.14",
+    "drizzle-kit": "^0.22.7",
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.1.3",
     "eslint-plugin-drizzle": "^0.2.3",

preprocessor/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12
+
+WORKDIR /preprocessor
+
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 poppler-utils tesseract-ocr -y
+
+COPY ./requirements.txt /preprocessor/requirements.txt
+
+RUN pip install --no-cache-dir -r /preprocessor/requirements.txt
+
+COPY . /preprocessor/
+
+EXPOSE 80
+
+CMD ["fastapi", "run", "main.py", "--port", "80"]

preprocessor/app/config/config.py

@@ -0,0 +1,11 @@
+import os
+
+from dotenv import load_dotenv
+from pydantic_settings import BaseSettings
+
+load_dotenv()
+
+class Settings(BaseSettings):
+    database_url: str =os.getenv("DATABASE_URL")
+
+settings = Settings()

preprocessor/app/db/database.py

@@ -0,0 +1,19 @@
+from app.config.config import settings
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+SQLALCHEMY_DATABASE_URL = settings.database_url
+
+engine = create_engine(SQLALCHEMY_DATABASE_URL)
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()