From 8aa61f00a9fa3d8f307bf68b0f23d846b37794be Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 6 Jan 2025 14:27:52 +0800 Subject: [PATCH 1/3] chore: add rye format on pre-commit hook --- backend/.pre-commit-config.yaml | 9 +++++++++ backend/pyproject.toml | 4 +++- backend/requirements-dev.lock | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 backend/.pre-commit-config.yaml diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml new file mode 100644 index 00000000..08d980ae --- /dev/null +++ b/backend/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.8.6 + hooks: + # Run the linter. + - id: ruff + # Run the formatter. + - id: ruff-format \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3cb92fcc..c8e08514 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -67,4 +67,6 @@ requires-python = ">= 3.8" managed = true virtual = true universal = true -dev-dependencies = [] +dev-dependencies = [ + "pre-commit>=4.0.1", +] diff --git a/backend/requirements-dev.lock b/backend/requirements-dev.lock index 68b6f3f4..4151b0b5 100644 --- a/backend/requirements-dev.lock +++ b/backend/requirements-dev.lock @@ -71,6 +71,8 @@ certifi==2024.6.2 cffi==1.16.0 # via argon2-cffi-bindings # via cryptography +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -122,6 +124,8 @@ dill==0.3.7 dirtyjson==1.0.8 # via llama-index-core # via llama-index-legacy +distlib==0.3.9 + # via virtualenv distro==1.9.0 # via anthropic # via openai @@ -150,6 +154,7 @@ fastavro==1.9.5 filelock==3.15.1 # via huggingface-hub # via transformers + # via virtualenv flower==2.0.1 frozenlist==1.4.1 # via aiohttp @@ -256,6 +261,8 @@ huggingface-hub==0.23.3 # via transformers humanize==4.9.0 # via flower +identify==2.6.5 + # via pre-commit idna==3.7 # via anyio # via email-validator @@ -415,6 +422,8 @@ nltk==3.9.1 # via llama-index # via llama-index-core # via llama-index-legacy +nodeenv==1.9.1 + # via pre-commit numpy==1.26.4 # via datasets # via langchain @@ -485,11 +494,14 @@ pillow==10.3.0 # via llama-index-core # via llama-index-llms-gemini # via python-pptx +platformdirs==4.3.6 + # via virtualenv playwright==1.45.1 pluggy==1.5.0 # via pytest portalocker==2.10.1 # via deepeval +pre-commit==4.0.1 prometheus-client==0.20.0 # via flower prompt-toolkit==3.0.47 @@ -595,6 +607,7 @@ pyyaml==6.0.1 # via langchain-core # via llama-index-core # via optuna + # via pre-commit # via transformers # via uvicorn ragas==0.2.6 @@ -751,6 +764,8 @@ vine==5.1.0 # via amqp # via celery # via kombu +virtualenv==20.28.1 + # via pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 From 5e298ade5808fc5932cd8df8f32e74deb251a098 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 6 Jan 2025 15:15:06 +0800 Subject: [PATCH 2/3] add ruff format action --- .github/workflows/backend-test.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/backend-test.yml diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml new file mode 100644 index 00000000..317d89f1 --- /dev/null +++ b/.github/workflows/backend-test.yml @@ -0,0 +1,30 @@ +name: Backend Test + +on: + push: + branches: + - main + paths: + - backend/** + pull_request: + branches: + - main + paths: + - backend/** + +jobs: + backend-test: + name: Backend Test + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./backend + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install Ruff Format + uses: astral-sh/ruff-action@v3 + - name: Run Ruff Check + run: ruff check + - name: Run Ruff Format + run: ruff format From ba9688023a0a1f14574d33a0d874ed9841768e99 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 6 Jan 2025 15:32:08 +0800 Subject: [PATCH 3/3] fix --- backend/app/alembic/versions/04d4f05116ed_.py | 1 - backend/app/alembic/versions/10f36e8a25c4_.py | 3 --- backend/app/alembic/versions/197bc8be72d1_.py | 2 -- backend/app/alembic/versions/27a6723b767a_.py | 3 --- .../749767db5505_add_recommend_questions.py | 2 -- backend/app/alembic/versions/94b198e20946_.py | 1 - .../alembic/versions/a54f966436ce_evaluation.py | 2 -- backend/app/alembic/versions/a8c79553c9f6_.py | 2 -- backend/app/alembic/versions/ac6e4d58580d_.py | 1 - backend/app/alembic/versions/bd17a4ebccc5_.py | 1 - backend/app/alembic/versions/c7f016a904c1_.py | 2 -- backend/app/alembic/versions/dfee070b8abd_.py | 2 -- backend/app/alembic/versions/e32f1e546eec_.py | 1 - backend/app/alembic/versions/eb0b85608c0a_.py | 2 -- backend/app/api/admin_routes/evaluation/models.py | 2 -- .../admin_routes/knowledge_base/document/routes.py | 2 +- .../app/api/admin_routes/knowledge_base/routes.py | 2 +- backend/app/api/admin_routes/llm/routes.py | 2 +- backend/app/api/main.py | 3 +-- backend/app/api/routes/chat.py | 1 - backend/app/api/routes/document.py | 3 +-- backend/app/auth/users.py | 2 +- backend/app/core/config.py | 2 +- backend/app/evaluation/evals.py | 2 -- .../app/evaluation/evaluators/e2e_rag_evaluator.py | 2 +- backend/app/exceptions.py | 14 +++++++------- backend/app/models/chat_engine.py | 1 - backend/app/models/embed_model.py | 2 +- .../models/knowledge_base_scoped/table_naming.py | 4 ++-- backend/app/models/relationship.py | 4 ++-- backend/app/rag/chat.py | 2 +- backend/app/rag/embed_model_option.py | 11 +++++------ backend/app/rag/knowledge_graph/base.py | 6 +++--- backend/app/rag/knowledge_graph/intent.py | 2 +- backend/app/rag/knowledge_graph/prerequisite.py | 2 +- backend/app/rag/retrieve.py | 2 +- backend/app/repositories/chat.py | 4 ++-- backend/app/repositories/chunk.py | 2 +- backend/app/site_settings/__init__.py | 1 - backend/app/tasks/evaluate.py | 3 +-- backend/app/tasks/knowledge_base.py | 5 ++--- backend/app/utils/dspy.py | 1 - backend/main.py | 4 ++-- backend/pyproject.toml | 4 ++++ backend/requirements-dev.lock | 1 + 45 files changed, 45 insertions(+), 78 deletions(-) diff --git a/backend/app/alembic/versions/04d4f05116ed_.py b/backend/app/alembic/versions/04d4f05116ed_.py index ccb2bdbe..67b6562d 100644 --- a/backend/app/alembic/versions/04d4f05116ed_.py +++ b/backend/app/alembic/versions/04d4f05116ed_.py @@ -9,7 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from app.models.base import AESEncryptedColumn diff --git a/backend/app/alembic/versions/10f36e8a25c4_.py b/backend/app/alembic/versions/10f36e8a25c4_.py index e9db112d..7e6a6e39 100644 --- a/backend/app/alembic/versions/10f36e8a25c4_.py +++ b/backend/app/alembic/versions/10f36e8a25c4_.py @@ -7,9 +7,6 @@ """ from alembic import op -import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/197bc8be72d1_.py b/backend/app/alembic/versions/197bc8be72d1_.py index 5065cf1a..ac36b9aa 100644 --- a/backend/app/alembic/versions/197bc8be72d1_.py +++ b/backend/app/alembic/versions/197bc8be72d1_.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/27a6723b767a_.py b/backend/app/alembic/versions/27a6723b767a_.py index 7fdc7929..7de95ba9 100644 --- a/backend/app/alembic/versions/27a6723b767a_.py +++ b/backend/app/alembic/versions/27a6723b767a_.py @@ -8,9 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType -from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. revision = "27a6723b767a" diff --git a/backend/app/alembic/versions/749767db5505_add_recommend_questions.py b/backend/app/alembic/versions/749767db5505_add_recommend_questions.py index 850e8a81..5cdc7808 100644 --- a/backend/app/alembic/versions/749767db5505_add_recommend_questions.py +++ b/backend/app/alembic/versions/749767db5505_add_recommend_questions.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/94b198e20946_.py b/backend/app/alembic/versions/94b198e20946_.py index f71d0d17..893f566b 100644 --- a/backend/app/alembic/versions/94b198e20946_.py +++ b/backend/app/alembic/versions/94b198e20946_.py @@ -9,7 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/a54f966436ce_evaluation.py b/backend/app/alembic/versions/a54f966436ce_evaluation.py index 259a0cf1..093e97ce 100644 --- a/backend/app/alembic/versions/a54f966436ce_evaluation.py +++ b/backend/app/alembic/versions/a54f966436ce_evaluation.py @@ -9,8 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType -from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. revision = "a54f966436ce" diff --git a/backend/app/alembic/versions/a8c79553c9f6_.py b/backend/app/alembic/versions/a8c79553c9f6_.py index 8dc51db5..b768beb5 100644 --- a/backend/app/alembic/versions/a8c79553c9f6_.py +++ b/backend/app/alembic/versions/a8c79553c9f6_.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/ac6e4d58580d_.py b/backend/app/alembic/versions/ac6e4d58580d_.py index 57f4408d..18f207c5 100644 --- a/backend/app/alembic/versions/ac6e4d58580d_.py +++ b/backend/app/alembic/versions/ac6e4d58580d_.py @@ -9,7 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/bd17a4ebccc5_.py b/backend/app/alembic/versions/bd17a4ebccc5_.py index 7104b345..7fab9bfe 100644 --- a/backend/app/alembic/versions/bd17a4ebccc5_.py +++ b/backend/app/alembic/versions/bd17a4ebccc5_.py @@ -9,7 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/c7f016a904c1_.py b/backend/app/alembic/versions/c7f016a904c1_.py index e278745c..7603c0df 100644 --- a/backend/app/alembic/versions/c7f016a904c1_.py +++ b/backend/app/alembic/versions/c7f016a904c1_.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/dfee070b8abd_.py b/backend/app/alembic/versions/dfee070b8abd_.py index 55f308e9..c54804e5 100644 --- a/backend/app/alembic/versions/dfee070b8abd_.py +++ b/backend/app/alembic/versions/dfee070b8abd_.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/backend/app/alembic/versions/e32f1e546eec_.py b/backend/app/alembic/versions/e32f1e546eec_.py index a7403a1d..a21c7140 100644 --- a/backend/app/alembic/versions/e32f1e546eec_.py +++ b/backend/app/alembic/versions/e32f1e546eec_.py @@ -9,7 +9,6 @@ from alembic import op import sqlalchemy as sa import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from app.models.base import AESEncryptedColumn diff --git a/backend/app/alembic/versions/eb0b85608c0a_.py b/backend/app/alembic/versions/eb0b85608c0a_.py index 449b49f3..4fe1b475 100644 --- a/backend/app/alembic/versions/eb0b85608c0a_.py +++ b/backend/app/alembic/versions/eb0b85608c0a_.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa -import sqlmodel.sql.sqltypes -from tidb_vector.sqlalchemy import VectorType from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/backend/app/api/admin_routes/evaluation/models.py b/backend/app/api/admin_routes/evaluation/models.py index 50406170..caa6b783 100644 --- a/backend/app/api/admin_routes/evaluation/models.py +++ b/backend/app/api/admin_routes/evaluation/models.py @@ -5,8 +5,6 @@ from fastapi_pagination import Params from pydantic import BaseModel -from app.models import EvaluationTask - class CreateEvaluationTask(BaseModel): name: str diff --git a/backend/app/api/admin_routes/knowledge_base/document/routes.py b/backend/app/api/admin_routes/knowledge_base/document/routes.py index 1a0c8559..a5d4fa45 100644 --- a/backend/app/api/admin_routes/knowledge_base/document/routes.py +++ b/backend/app/api/admin_routes/knowledge_base/document/routes.py @@ -113,7 +113,7 @@ def remove_kb_document( ) graph_repo.delete_orphaned_entities(session) - logger.info(f"Deleted orphaned entities successfully.") + logger.info("Deleted orphaned entities successfully.") chunk_repo.delete_by_document(session, document_id) logger.info(f"Deleted chunks of document #{document_id} successfully.") diff --git a/backend/app/api/admin_routes/knowledge_base/routes.py b/backend/app/api/admin_routes/knowledge_base/routes.py index e8e40f61..e2e362b8 100644 --- a/backend/app/api/admin_routes/knowledge_base/routes.py +++ b/backend/app/api/admin_routes/knowledge_base/routes.py @@ -36,7 +36,7 @@ build_kg_index_for_chunk, build_index_for_document, ) -from app.repositories import knowledge_base_repo, data_source_repo, chat_engine_repo +from app.repositories import knowledge_base_repo, data_source_repo from app.tasks.knowledge_base import ( import_documents_for_knowledge_base, stats_for_knowledge_base, diff --git a/backend/app/api/admin_routes/llm/routes.py b/backend/app/api/admin_routes/llm/routes.py index f0e4a158..bf5d36cf 100644 --- a/backend/app/api/admin_routes/llm/routes.py +++ b/backend/app/api/admin_routes/llm/routes.py @@ -8,7 +8,7 @@ from app.api.deps import CurrentSuperuserDep, SessionDep from app.exceptions import InternalServerError, LLMNotFound -from app.models import AdminLLM, LLM, ChatEngine, DataSource, KnowledgeBase +from app.models import AdminLLM, LLM, ChatEngine, KnowledgeBase from app.rag.chat_config import get_llm from app.rag.llm_option import LLMOption, admin_llm_options from app.repositories.llm import llm_repo diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 6f29bc39..70ba5d26 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter from app.api.routes import ( @@ -45,7 +45,6 @@ ) from app.auth.users import auth_backend, fastapi_users -from app.api.deps import current_superuser api_router = APIRouter() api_router.include_router(index.router, tags=["index"]) diff --git a/backend/app/api/routes/chat.py b/backend/app/api/routes/chat.py index 21265197..eace3344 100644 --- a/backend/app/api/routes/chat.py +++ b/backend/app/api/routes/chat.py @@ -12,7 +12,6 @@ from fastapi_pagination import Params, Page from app.api.deps import SessionDep, OptionalUserDep, CurrentUserDep -from app.rag.chat_config import ChatEngineConfig from app.repositories import chat_repo from app.models import Chat, ChatUpdate from app.rag.chat import ( diff --git a/backend/app/api/routes/document.py b/backend/app/api/routes/document.py index 94caeb95..08028f60 100644 --- a/backend/app/api/routes/document.py +++ b/backend/app/api/routes/document.py @@ -1,6 +1,5 @@ -from fastapi import FastAPI, HTTPException, APIRouter +from fastapi import HTTPException, APIRouter from fastapi.responses import StreamingResponse -from sqlmodel import Session from app.api.deps import SessionDep from app.repositories import document_repo from app.file_storage import get_file_storage diff --git a/backend/app/auth/users.py b/backend/app/auth/users.py index f8ee407a..058e36ec 100644 --- a/backend/app/auth/users.py +++ b/backend/app/auth/users.py @@ -16,7 +16,7 @@ from fastapi_users.exceptions import UserAlreadyExists, UserNotExists from sqlmodel.ext.asyncio.session import AsyncSession -from app.core.config import settings, Environment +from app.core.config import settings from app.core.db import get_db_async_session from app.models import User, UserSession from app.auth.db import get_user_db, get_user_session_db diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 248d0236..187167ce 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -140,7 +140,7 @@ def _validate_secrets(self) -> Self: secret = self.SECRET_KEY if not secret: raise ValueError( - f"Please set a secret key using the SECRET_KEY environment variable." + "Please set a secret key using the SECRET_KEY environment variable." ) min_length = 32 diff --git a/backend/app/evaluation/evals.py b/backend/app/evaluation/evals.py index e447a25d..0dd01a51 100644 --- a/backend/app/evaluation/evals.py +++ b/backend/app/evaluation/evals.py @@ -22,8 +22,6 @@ ) import pandas as pd from ragas.metrics import ( - LLMContextRecall, - Faithfulness, FactualCorrectness, SemanticSimilarity, ) diff --git a/backend/app/evaluation/evaluators/e2e_rag_evaluator.py b/backend/app/evaluation/evaluators/e2e_rag_evaluator.py index 45c9e0e3..6d60a550 100644 --- a/backend/app/evaluation/evaluators/e2e_rag_evaluator.py +++ b/backend/app/evaluation/evaluators/e2e_rag_evaluator.py @@ -1,5 +1,5 @@ import time -from typing import Any, Optional, Sequence, List, Mapping +from typing import Optional, Sequence, Mapping from llama_index.core.evaluation.base import EvaluationResult from deepeval import evaluate from deepeval.test_case import LLMTestCase diff --git a/backend/app/exceptions.py b/backend/app/exceptions.py index e596232f..29d4973f 100644 --- a/backend/app/exceptions.py +++ b/backend/app/exceptions.py @@ -41,7 +41,7 @@ class DefaultLLMNotFound(LLMException): status_code = 404 def __init__(self): - self.detail = f"default llm is not found" + self.detail = "default llm is not found" # Embedding model @@ -62,7 +62,7 @@ class DefaultEmbeddingModelNotFound(EmbeddingModelException): status_code = 404 def __init__(self): - self.detail = f"default embedding model is not found" + self.detail = "default embedding model is not found" # Reranker model @@ -83,7 +83,7 @@ class DefaultRerankerModelNotFound(RerankerModelException): status_code = 404 def __init__(self): - self.detail = f"default reranker model is not found" + self.detail = "default reranker model is not found" # Knowledge base @@ -113,28 +113,28 @@ class KBNoLLMConfigured(KBException): status_code = 500 def __init__(self): - self.detail = f"must configured a LLM for knowledge base" + self.detail = "must configured a LLM for knowledge base" class KBNoEmbedModelConfigured(KBException): status_code = 500 def __init__(self): - self.detail = f"must configured a embedding model for knowledge base" + self.detail = "must configured a embedding model for knowledge base" class KBNoVectorIndexConfigured(KBException): status_code = 500 def __init__(self): - self.detail = f"must configured vector index as one of the index method for knowledge base, which is required for now" + self.detail = "must configured vector index as one of the index method for knowledge base, which is required for now" class KBNotAllowedUpdateEmbedModel(KBException): status_code = 500 def __init__(self): - self.detail = f"update embedding model is not allowed once the knowledge base has been created" + self.detail = "update embedding model is not allowed once the knowledge base has been created" class KBIsUsedByChatEngines(KBException): diff --git a/backend/app/models/chat_engine.py b/backend/app/models/chat_engine.py index c7890b28..ccb1ba05 100644 --- a/backend/app/models/chat_engine.py +++ b/backend/app/models/chat_engine.py @@ -8,7 +8,6 @@ JSON, DateTime, Relationship as SQLRelationship, - SQLModel, ) from .base import UpdatableBaseModel diff --git a/backend/app/models/embed_model.py b/backend/app/models/embed_model.py index 3d94b1f7..3e690150 100644 --- a/backend/app/models/embed_model.py +++ b/backend/app/models/embed_model.py @@ -1,6 +1,6 @@ from typing import Optional, Any -from sqlmodel import Field, Column, JSON, String, SQLModel +from sqlmodel import Field, Column, JSON, String from .base import UpdatableBaseModel, AESEncryptedColumn from app.types import EmbeddingProvider diff --git a/backend/app/models/knowledge_base_scoped/table_naming.py b/backend/app/models/knowledge_base_scoped/table_naming.py index e9631c60..9739d1ea 100644 --- a/backend/app/models/knowledge_base_scoped/table_naming.py +++ b/backend/app/models/knowledge_base_scoped/table_naming.py @@ -45,7 +45,7 @@ def get_kb_vector_dims(kb: KnowledgeBase): vector_dimension = kb.embedding_model.vector_dimension else: logger.warning( - f"This knowledge base doesn't configured a embedding model or this vector vector_dimension " - f"of the embedding model is miss." + "This knowledge base doesn't configured a embedding model or this vector vector_dimension " + "of the embedding model is miss." ) return vector_dimension diff --git a/backend/app/models/relationship.py b/backend/app/models/relationship.py index eb3aff88..c6556bdd 100644 --- a/backend/app/models/relationship.py +++ b/backend/app/models/relationship.py @@ -102,14 +102,14 @@ class KBRelationship(PatchSQLModel, table=True, registry=ctx.registry): source_entity_id: int = Field(foreign_key=f"{entities_table_name}.id") source_entity: entity_model = SQLRelationship( sa_relationship_kwargs={ - "primaryjoin": f"KBRelationship.source_entity_id == KBEntity.id", + "primaryjoin": "KBRelationship.source_entity_id == KBEntity.id", "lazy": "joined", }, ) target_entity_id: int = Field(foreign_key=f"{entities_table_name}.id") target_entity: entity_model = SQLRelationship( sa_relationship_kwargs={ - "primaryjoin": f"KBRelationship.target_entity_id == KBEntity.id", + "primaryjoin": "KBRelationship.target_entity_id == KBEntity.id", "lazy": "joined", }, ) diff --git a/backend/app/rag/chat.py b/backend/app/rag/chat.py index 814882dc..f58c8397 100644 --- a/backend/app/rag/chat.py +++ b/backend/app/rag/chat.py @@ -920,7 +920,7 @@ def _external_chat(self) -> Generator[ChatEvent | str, None, None]: db_assistant_message.post_verification_result_url = ( post_verification_result_url ) - except Exception as e: + except Exception: logger.error( "Specific error occurred during post verification job.", exc_info=True ) diff --git a/backend/app/rag/embed_model_option.py b/backend/app/rag/embed_model_option.py index e463221f..f308a58b 100644 --- a/backend/app/rag/embed_model_option.py +++ b/backend/app/rag/embed_model_option.py @@ -2,7 +2,6 @@ from pydantic import BaseModel from app.types import EmbeddingProvider -from app.core.config import settings class EmbeddingModelOption(BaseModel): @@ -27,7 +26,7 @@ class EmbeddingModelOption(BaseModel): provider_description="The OpenAI API provides a simple interface for developers to create an intelligence layer in their applications, powered by OpenAI's state of the art models.", provider_url="https://platform.openai.com", default_embedding_model="text-embedding-3-small", - embedding_model_description=f"Find more information about OpenAI Embedding at https://platform.openai.com/docs/guides/embeddings", + embedding_model_description="Find more information about OpenAI Embedding at https://platform.openai.com/docs/guides/embeddings", credentials_display_name="OpenAI API Key", credentials_description="The API key of OpenAI, you can find it in https://platform.openai.com/api-keys", credentials_type="str", @@ -39,7 +38,7 @@ class EmbeddingModelOption(BaseModel): provider_description="Jina AI provides multimodal, bilingual long-context embeddings for search and RAG", provider_url="https://jina.ai/embeddings/", default_embedding_model="jina-embeddings-v2-base-en", - embedding_model_description=f"Find more information about Jina AI Embeddings at https://jina.ai/embeddings/", + embedding_model_description="Find more information about Jina AI Embeddings at https://jina.ai/embeddings/", credentials_display_name="Jina API Key", credentials_description="The API key of Jina, you can find it in https://jina.ai/embeddings/", credentials_type="str", @@ -51,7 +50,7 @@ class EmbeddingModelOption(BaseModel): provider_description="Cohere provides industry-leading large language models (LLMs) and RAG capabilities tailored to meet the needs of enterprise use cases that solve real-world problems.", provider_url="https://cohere.com/embeddings", default_embedding_model="embed-multilingual-v3.0", - embedding_model_description=f"Documentation: https://docs.cohere.com/docs/cohere-embed", + embedding_model_description="Documentation: https://docs.cohere.com/docs/cohere-embed", credentials_display_name="Cohere API Key", credentials_description="You can get one from https://dashboard.cohere.com/api-keys", credentials_type="str", @@ -79,7 +78,7 @@ class EmbeddingModelOption(BaseModel): provider_description="Ollama is a lightweight framework for building and running large language models and embed models.", provider_url="https://ollama.com", default_embedding_model="nomic-embed-text", - embedding_model_description=f"Documentation: https://ollama.com/blog/embedding-models", + embedding_model_description="Documentation: https://ollama.com/blog/embedding-models", default_config={ "api_base": "http://localhost:11434", }, @@ -107,7 +106,7 @@ class EmbeddingModelOption(BaseModel): provider_description="Gitee AI is a third-party model provider that offers ready-to-use cutting-edge model APIs for AI developers.", provider_url="https://ai.gitee.com", default_embedding_model="bge-large-zh-v1.5", - embedding_model_description=f"Find more information about Gitee AI Embeddings at https://ai.gitee.com/docs/openapi/v1#tag/%E7%89%B9%E5%BE%81%E6%8A%BD%E5%8F%96/POST/embeddings", + embedding_model_description="Find more information about Gitee AI Embeddings at https://ai.gitee.com/docs/openapi/v1#tag/%E7%89%B9%E5%BE%81%E6%8A%BD%E5%8F%96/POST/embeddings", credentials_display_name="Gitee AI API Key", credentials_description="The API key of Gitee AI, you can find it in https://ai.gitee.com/dashboard/settings/tokens", credentials_type="str", diff --git a/backend/app/rag/knowledge_graph/base.py b/backend/app/rag/knowledge_graph/base.py index 3bd042fe..0aea5da7 100644 --- a/backend/app/rag/knowledge_graph/base.py +++ b/backend/app/rag/knowledge_graph/base.py @@ -8,8 +8,8 @@ from sqlmodel import Session from llama_index.core.data_structs import IndexLPG -from llama_index.core.callbacks import CallbackManager, trace_method -from llama_index.core.callbacks.schema import CBEventType, EventPayload +from llama_index.core.callbacks import CallbackManager +from llama_index.core.callbacks.schema import EventPayload from llama_index.core.indices.base import BaseIndex from llama_index.core.storage.docstore.types import RefDocInfo from llama_index.core.storage.storage_context import StorageContext @@ -319,7 +319,7 @@ def process_query(sub_query): relationship_meta_filters=relationship_meta_filters, session=tmp_session, ) - except Exception as exc: + except Exception: tmp_session.rollback() traceback.print_exc() raise diff --git a/backend/app/rag/knowledge_graph/intent.py b/backend/app/rag/knowledge_graph/intent.py index c428bda2..a73c5857 100644 --- a/backend/app/rag/knowledge_graph/intent.py +++ b/backend/app/rag/knowledge_graph/intent.py @@ -1,6 +1,6 @@ import logging import dspy -from dspy.functional import TypedChainOfThought, TypedPredictor +from dspy.functional import TypedPredictor from typing import List, Optional from pydantic import BaseModel, Field from llama_index.core.tools import FunctionTool diff --git a/backend/app/rag/knowledge_graph/prerequisite.py b/backend/app/rag/knowledge_graph/prerequisite.py index 0d00d295..ee098e78 100644 --- a/backend/app/rag/knowledge_graph/prerequisite.py +++ b/backend/app/rag/knowledge_graph/prerequisite.py @@ -1,6 +1,6 @@ import logging import dspy -from dspy.functional import TypedChainOfThought, TypedPredictor +from dspy.functional import TypedChainOfThought from typing import List, Optional from pydantic import BaseModel, Field diff --git a/backend/app/rag/retrieve.py b/backend/app/rag/retrieve.py index 8a155088..5013a34f 100644 --- a/backend/app/rag/retrieve.py +++ b/backend/app/rag/retrieve.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional, Type +from typing import List, Type from llama_index.core import VectorStoreIndex from llama_index.core.schema import NodeWithScore diff --git a/backend/app/repositories/chat.py b/backend/app/repositories/chat.py index bb072316..0affd683 100644 --- a/backend/app/repositories/chat.py +++ b/backend/app/repositories/chat.py @@ -194,8 +194,8 @@ def chat_trend_by_origin( origins.add(row.origin) stats = [] - for date, origin_counts in date_origin_counts.items(): - stat = {"date": date} + for d, origin_counts in date_origin_counts.items(): + stat = {"date": d} for origin in origins: stat[origin] = origin_counts[origin] stats.append(stat) diff --git a/backend/app/repositories/chunk.py b/backend/app/repositories/chunk.py index 0008fdb8..457c2c22 100644 --- a/backend/app/repositories/chunk.py +++ b/backend/app/repositories/chunk.py @@ -1,4 +1,4 @@ -from typing import Type, Optional +from typing import Type from sqlalchemy import func, delete from sqlmodel import Session, select, SQLModel diff --git a/backend/app/site_settings/__init__.py b/backend/app/site_settings/__init__.py index c9e14e7e..b6642457 100644 --- a/backend/app/site_settings/__init__.py +++ b/backend/app/site_settings/__init__.py @@ -2,7 +2,6 @@ import threading import logging from sqlmodel import Session, select -from datetime import datetime, UTC from app.models import SiteSetting as DBSiteSetting from app.core.db import engine diff --git a/backend/app/tasks/evaluate.py b/backend/app/tasks/evaluate.py index 5dd861f0..43f5adbc 100644 --- a/backend/app/tasks/evaluate.py +++ b/backend/app/tasks/evaluate.py @@ -1,7 +1,6 @@ import logging import traceback -import httpx from llama_index.core.base.llms.types import ChatMessage from app.celery import app as celery_app @@ -150,7 +149,7 @@ def evaluate_task(evaluation_task_item: EvaluationTaskItem): show_progress=False, ) - logger.debug(f"eval_result to_pandas") + logger.debug("eval_result to_pandas") result_list = eval_result.to_pandas().to_dict(orient="records") logger.debug(f"result list {result_list}") if len(result_list) != 1: diff --git a/backend/app/tasks/knowledge_base.py b/backend/app/tasks/knowledge_base.py index ffb5bce5..97a5d9a0 100644 --- a/backend/app/tasks/knowledge_base.py +++ b/backend/app/tasks/knowledge_base.py @@ -1,6 +1,6 @@ from celery.utils.log import get_task_logger from sqlalchemy import delete -from sqlmodel import Session, select +from sqlmodel import Session from app.celery import app as celery_app from app.core.db import engine @@ -10,7 +10,6 @@ KnowledgeBaseDataSource, DataSource, ) -from app.models.knowledge_base import KnowledgeBase from app.rag.datasource import get_data_source_loader from app.repositories import knowledge_base_repo, document_repo from .build_index import build_index_for_document @@ -196,7 +195,7 @@ def purge_kb_datasource_related_resources(kb_id: int, datasource_id: int): ) graph_repo.delete_orphaned_entities(session) - logger.info(f"Deleted orphaned entities successfully.") + logger.info("Deleted orphaned entities successfully.") chunk_repo.delete_by_datasource(session, datasource_id) logger.info(f"Deleted chunks from data source #{datasource_id} successfully.") diff --git a/backend/app/utils/dspy.py b/backend/app/utils/dspy.py index 8b19bb95..94e0870d 100644 --- a/backend/app/utils/dspy.py +++ b/backend/app/utils/dspy.py @@ -2,7 +2,6 @@ import datetime import hashlib from typing import Any, Literal -import logging import dspy import requests diff --git a/backend/main.py b/backend/main.py index 9ed9df40..3392336b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -147,7 +147,7 @@ def runserver(host, port): @click.option( "--tidb-ai-chat-engine", default="default", - help=f"TiDB AI chat engine, default=default", + help="TiDB AI chat engine, default=default", ) def runeval(dataset, llm_provider, run_name, tidb_ai_chat_engine): from app.evaluation.evals import Evaluation @@ -176,7 +176,7 @@ def runeval(dataset, llm_provider, run_name, tidb_ai_chat_engine): @click.option( "--tidb-ai-chat-engine", default="default", - help=f"TiDB AI chat engine, default=default", + help="TiDB AI chat engine, default=default", ) @click.option("--run-size", default=30, help="Run size, default=30") def runeval_dataset(csv, llm_provider, run_name, tidb_ai_chat_engine, run_size): diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c8e08514..078bf1b3 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -69,4 +69,8 @@ virtual = true universal = true dev-dependencies = [ "pre-commit>=4.0.1", + "ruff>=0.8.6", ] + +[tool.ruff.lint] +ignore = ["E711", "E712", "F811", "F821", "F841"] diff --git a/backend/requirements-dev.lock b/backend/requirements-dev.lock index 4151b0b5..3242838b 100644 --- a/backend/requirements-dev.lock +++ b/backend/requirements-dev.lock @@ -644,6 +644,7 @@ rich==13.7.1 # via typer rsa==4.9 # via google-auth +ruff==0.8.6 s3transfer==0.10.2 # via boto3 safetensors==0.4.3