From 867c9f4a22e252181c6f3b4f523e3fb7940d6b5d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:00:59 +0000 Subject: [PATCH] [pre-commit.ci] Add auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/safety_and_evaluation.rst | 6 +++--- eval/collect_rag_outputs.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/source/safety_and_evaluation.rst b/docs/source/safety_and_evaluation.rst index 9b4a082..c4a283d 100644 --- a/docs/source/safety_and_evaluation.rst +++ b/docs/source/safety_and_evaluation.rst @@ -21,7 +21,7 @@ Synthetic Dataset Generation The evaluation utilizes a synthetic dataset of 1000 queries, carefully structured to test different aspects of the system: - **Regular Situations (80%)**: Divided equally among three levels of detail: - + - Low: 1-2 sentences, brief queries with minimal context - Medium: 2-3 sentences with moderate background information - High: Detailed paragraph with extensive context @@ -150,7 +150,7 @@ Performance Metrics RAGAS Metrics By Category ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Note: These metrics were obtained using a synthetic dataset specifically generated from services in the Greater Toronto Area (GTA). +Note: These metrics were obtained using a synthetic dataset specifically generated from services in the Greater Toronto Area (GTA). The RAG system evaluated used specialized prompts that differ marginally from the current system. .. list-table:: @@ -264,4 +264,4 @@ To enable re-ranking in your API calls, simply set the `rerank` parameter to `tr "longitude": -79.3832, "radius": 5000, "rerank": true - } \ No newline at end of file + } diff --git a/eval/collect_rag_outputs.py b/eval/collect_rag_outputs.py index c8d8919..611e662 100644 --- a/eval/collect_rag_outputs.py +++ b/eval/collect_rag_outputs.py @@ -2,7 +2,7 @@ import asyncio import json import logging -from typing import Dict, List, Any +from typing import Dict, Any import aiohttp from tqdm.asyncio import tqdm_asyncio @@ -27,7 +27,8 @@ async def fetch_recommendation( "query": query["query"], "answer": result["message"], "context": [ - str(service["id"]) for service in (result.get("services", []) or []) + str(service["id"]) + for service in (result.get("services", []) or []) ], "ground_truth": query["context"], } @@ -39,7 +40,9 @@ async def fetch_recommendation( return None -async def process_samples(samples_file: str, output_file: str, batch_size: int = 5) -> None: +async def process_samples( + samples_file: str, output_file: str, batch_size: int = 5 +) -> None: """Process samples in batches and save results.""" # Load samples with open(samples_file, "r") as f: @@ -90,4 +93,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main()