chore(ci): add ruff lint and format on git hook and github action

.github/workflows/backend-test.yml

@@ -0,0 +1,30 @@
+name: Backend Test
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - backend/**
+  pull_request:
+    branches:
+      - main
+    paths:
+      - backend/**
+
+jobs:
+  backend-test:
+    name: Backend Test
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./backend
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Ruff Format
+        uses: astral-sh/ruff-action@v3
+      - name: Run Ruff Check
+        run: ruff check
+      - name: Run Ruff Format
+        run: ruff format

backend/.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.8.6
+  hooks:
+    # Run the linter.
+    - id: ruff
+    # Run the formatter.
+    - id: ruff-format

backend/app/alembic/versions/04d4f05116ed_.py

@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from app.models.base import AESEncryptedColumn
 
 

backend/app/alembic/versions/10f36e8a25c4_.py

@@ -7,9 +7,6 @@
 """
 
 from alembic import op
-import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/197bc8be72d1_.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/27a6723b767a_.py

@@ -8,9 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
-from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.
 revision = "27a6723b767a"

backend/app/alembic/versions/749767db5505_add_recommend_questions.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/94b198e20946_.py

@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/a54f966436ce_evaluation.py

@@ -9,8 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
-from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.
 revision = "a54f966436ce"

backend/app/alembic/versions/a8c79553c9f6_.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/ac6e4d58580d_.py

@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/bd17a4ebccc5_.py

@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/c7f016a904c1_.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/dfee070b8abd_.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.

backend/app/alembic/versions/e32f1e546eec_.py

@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from app.models.base import AESEncryptedColumn
 
 

backend/app/alembic/versions/eb0b85608c0a_.py

@@ -8,8 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-import sqlmodel.sql.sqltypes
-from tidb_vector.sqlalchemy import VectorType
 from sqlalchemy.dialects import mysql
 
 # revision identifiers, used by Alembic.

backend/app/api/admin_routes/evaluation/models.py

@@ -5,8 +5,6 @@
 from fastapi_pagination import Params
 from pydantic import BaseModel
 
-from app.models import EvaluationTask
-
 
 class CreateEvaluationTask(BaseModel):
     name: str

backend/app/api/admin_routes/knowledge_base/document/routes.py

@@ -113,7 +113,7 @@ def remove_kb_document(
         )
 
         graph_repo.delete_orphaned_entities(session)
-        logger.info(f"Deleted orphaned entities successfully.")
+        logger.info("Deleted orphaned entities successfully.")
 
         chunk_repo.delete_by_document(session, document_id)
         logger.info(f"Deleted chunks of document #{document_id} successfully.")

backend/app/api/admin_routes/knowledge_base/routes.py

@@ -36,7 +36,7 @@
     build_kg_index_for_chunk,
     build_index_for_document,
 )
-from app.repositories import knowledge_base_repo, data_source_repo, chat_engine_repo
+from app.repositories import knowledge_base_repo, data_source_repo
 from app.tasks.knowledge_base import (
     import_documents_for_knowledge_base,
     stats_for_knowledge_base,

backend/app/api/admin_routes/llm/routes.py

@@ -8,7 +8,7 @@
 
 from app.api.deps import CurrentSuperuserDep, SessionDep
 from app.exceptions import InternalServerError, LLMNotFound
-from app.models import AdminLLM, LLM, ChatEngine, DataSource, KnowledgeBase
+from app.models import AdminLLM, LLM, ChatEngine, KnowledgeBase
 from app.rag.chat_config import get_llm
 from app.rag.llm_option import LLMOption, admin_llm_options
 from app.repositories.llm import llm_repo

backend/app/api/main.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter
 
 
 from app.api.routes import (
@@ -45,7 +45,6 @@
 )
 
 from app.auth.users import auth_backend, fastapi_users
-from app.api.deps import current_superuser
 
 api_router = APIRouter()
 api_router.include_router(index.router, tags=["index"])

backend/app/api/routes/chat.py

@@ -12,7 +12,6 @@
 from fastapi_pagination import Params, Page
 
 from app.api.deps import SessionDep, OptionalUserDep, CurrentUserDep
-from app.rag.chat_config import ChatEngineConfig
 from app.repositories import chat_repo
 from app.models import Chat, ChatUpdate
 from app.rag.chat import (

backend/app/api/routes/document.py

@@ -1,6 +1,5 @@
-from fastapi import FastAPI, HTTPException, APIRouter
+from fastapi import HTTPException, APIRouter
 from fastapi.responses import StreamingResponse
-from sqlmodel import Session
 from app.api.deps import SessionDep
 from app.repositories import document_repo
 from app.file_storage import get_file_storage

backend/app/auth/users.py

@@ -16,7 +16,7 @@
 from fastapi_users.exceptions import UserAlreadyExists, UserNotExists
 from sqlmodel.ext.asyncio.session import AsyncSession
 
-from app.core.config import settings, Environment
+from app.core.config import settings
 from app.core.db import get_db_async_session
 from app.models import User, UserSession
 from app.auth.db import get_user_db, get_user_session_db

backend/app/core/config.py

@@ -140,7 +140,7 @@ def _validate_secrets(self) -> Self:
         secret = self.SECRET_KEY
         if not secret:
             raise ValueError(
-                f"Please set a secret key using the SECRET_KEY environment variable."
+                "Please set a secret key using the SECRET_KEY environment variable."
             )
 
         min_length = 32

backend/app/evaluation/evals.py

@@ -22,8 +22,6 @@
 )
 import pandas as pd
 from ragas.metrics import (
-    LLMContextRecall,
-    Faithfulness,
     FactualCorrectness,
     SemanticSimilarity,
 )

backend/app/evaluation/evaluators/e2e_rag_evaluator.py

@@ -1,5 +1,5 @@
 import time
-from typing import Any, Optional, Sequence, List, Mapping
+from typing import Optional, Sequence, Mapping
 from llama_index.core.evaluation.base import EvaluationResult
 from deepeval import evaluate
 from deepeval.test_case import LLMTestCase

backend/app/exceptions.py

@@ -41,7 +41,7 @@ class DefaultLLMNotFound(LLMException):
     status_code = 404
 
     def __init__(self):
-        self.detail = f"default llm is not found"
+        self.detail = "default llm is not found"
 
 
 # Embedding model
@@ -62,7 +62,7 @@ class DefaultEmbeddingModelNotFound(EmbeddingModelException):
     status_code = 404
 
     def __init__(self):
-        self.detail = f"default embedding model is not found"
+        self.detail = "default embedding model is not found"
 
 
 # Reranker model
@@ -83,7 +83,7 @@ class DefaultRerankerModelNotFound(RerankerModelException):
     status_code = 404
 
     def __init__(self):
-        self.detail = f"default reranker model is not found"
+        self.detail = "default reranker model is not found"
 
 
 # Knowledge base
@@ -113,28 +113,28 @@ class KBNoLLMConfigured(KBException):
     status_code = 500
 
     def __init__(self):
-        self.detail = f"must configured a LLM for knowledge base"
+        self.detail = "must configured a LLM for knowledge base"
 
 
 class KBNoEmbedModelConfigured(KBException):
     status_code = 500
 
     def __init__(self):
-        self.detail = f"must configured a embedding model for knowledge base"
+        self.detail = "must configured a embedding model for knowledge base"
 
 
 class KBNoVectorIndexConfigured(KBException):
     status_code = 500
 
     def __init__(self):
-        self.detail = f"must configured vector index as one of the index method for knowledge base, which is required for now"
+        self.detail = "must configured vector index as one of the index method for knowledge base, which is required for now"
 
 
 class KBNotAllowedUpdateEmbedModel(KBException):
     status_code = 500
 
     def __init__(self):
-        self.detail = f"update embedding model is not allowed once the knowledge base has been created"
+        self.detail = "update embedding model is not allowed once the knowledge base has been created"
 
 
 class KBIsUsedByChatEngines(KBException):

backend/app/models/chat_engine.py

@@ -8,7 +8,6 @@
     JSON,
     DateTime,
     Relationship as SQLRelationship,
-    SQLModel,
 )
 
 from .base import UpdatableBaseModel

backend/app/models/embed_model.py

@@ -1,6 +1,6 @@
 from typing import Optional, Any
 
-from sqlmodel import Field, Column, JSON, String, SQLModel
+from sqlmodel import Field, Column, JSON, String
 
 from .base import UpdatableBaseModel, AESEncryptedColumn
 from app.types import EmbeddingProvider

backend/app/models/knowledge_base_scoped/table_naming.py

@@ -45,7 +45,7 @@ def get_kb_vector_dims(kb: KnowledgeBase):
         vector_dimension = kb.embedding_model.vector_dimension
     else:
         logger.warning(
-            f"This knowledge base doesn't configured a embedding model or this vector vector_dimension "
-            f"of the embedding model is miss."
+            "This knowledge base doesn't configured a embedding model or this vector vector_dimension "
+            "of the embedding model is miss."
         )
     return vector_dimension

backend/app/models/relationship.py

@@ -102,14 +102,14 @@ class KBRelationship(PatchSQLModel, table=True, registry=ctx.registry):
         source_entity_id: int = Field(foreign_key=f"{entities_table_name}.id")
         source_entity: entity_model = SQLRelationship(
             sa_relationship_kwargs={
-                "primaryjoin": f"KBRelationship.source_entity_id == KBEntity.id",
+                "primaryjoin": "KBRelationship.source_entity_id == KBEntity.id",
                 "lazy": "joined",
             },
         )
         target_entity_id: int = Field(foreign_key=f"{entities_table_name}.id")
         target_entity: entity_model = SQLRelationship(
             sa_relationship_kwargs={
-                "primaryjoin": f"KBRelationship.target_entity_id == KBEntity.id",
+                "primaryjoin": "KBRelationship.target_entity_id == KBEntity.id",
                 "lazy": "joined",
             },
         )

backend/app/rag/chat.py

@@ -920,7 +920,7 @@ def _external_chat(self) -> Generator[ChatEvent | str, None, None]:
             db_assistant_message.post_verification_result_url = (
                 post_verification_result_url
             )
-        except Exception as e:
+        except Exception:
             logger.error(
                 "Specific error occurred during post verification job.", exc_info=True
             )