Merge pull request #16 from VectorInstitute/add_support_for_empower_api

Add support for empower API

README.md

@@ -82,18 +82,35 @@ docker compose --env-file .env.development --profile frontend -f docker-compose.
 
 ### 📥 Data setup
 
-#### Download service data (211 API)
+#### Test data
 
-**GTA data**
+If you want to test the system without real data, you can generate some dummy testing data:
 
 ```bash
-python3 scripts/download_data.py --api-key $YOUR_211_API_KEY --dataset on --is-gta --data-dir <path_to_data_dir>
+python3 scripts/generate_test_data.py
 ```
 
-**Ontario-wide data**
+#### Download service data
+
+If you are using the 211 API or Empower's API, make sure you check with them to see if the API keys are
+configured correctly for the geography of interest.
+
+**GTA data (211 API)**
+
+```bash
+python3 scripts/download_211_data.py --api-key $YOUR_211_API_KEY --dataset on --is-gta --data-dir <path_to_data_dir>
+```
+
+**Ontario-wide data (211 API)**
+
+```bash
+python3 scripts/download_211_data.py --api-key $YOUR_211_API_KEY --dataset on --data-dir <path_to_data_dir>
+```
+
+**Empower API data**
 
 ```bash
-python3 scripts/download_data.py --api-key $YOUR_211_API_KEY --dataset on --data-dir <path_to_data_dir>
+python3 scripts/download_empower_data.py --api-key $YOUR_EMPOWER_API_KEY --data-dir <path_to_data_dir>
 ```
 
 #### Upload data and embeddings

docs/source/index.md

@@ -91,18 +91,35 @@ docker compose --env-file .env.development --profile frontend -f docker-compose.
 
 ### 📥 Data setup
 
-#### Download service data (211 API)
+#### Test data
 
-**GTA data**
+If you want to test the system without real data, you can generate some dummy testing data:
 
 ```bash
-python3 scripts/download_data.py --api-key $YOUR_211_API_KEY --dataset on --is-gta --data-dir <path_to_data_dir>
+python3 scripts/generate_test_data.py
 ```
 
-**Ontario-wide data**
+#### Download service data
+
+If you are using the 211 API or Empower's API, make sure you check with them to see if the API keys are
+configured correctly for the geography of interest.
+
+**GTA data (211 API)**
+
+```bash
+python3 scripts/download_211_data.py --api-key $YOUR_211_API_KEY --dataset on --is-gta --data-dir <path_to_data_dir>
+```
+
+**Ontario-wide data (211 API)**
+
+```bash
+python3 scripts/download_211_data.py --api-key $YOUR_211_API_KEY --dataset on --data-dir <path_to_data_dir>
+```
+
+**Empower API data**
 
 ```bash
-python3 scripts/download_data.py --api-key $YOUR_211_API_KEY --dataset on --data-dir <path_to_data_dir>
+python3 scripts/download_empower_data.py --api-key $YOUR_EMPOWER_API_KEY --data-dir <path_to_data_dir>
 ```
 
 #### Upload data and embeddings

eval/collect_rag_outputs.py

@@ -2,19 +2,18 @@
 import asyncio
 import json
 import logging
-from typing import Dict, Any
+from typing import Dict, Any, List, Optional
 
 import aiohttp
 from tqdm.asyncio import tqdm_asyncio
 
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 async def fetch_recommendation(
     session: aiohttp.ClientSession, query: Dict[str, Any], endpoint: str
-) -> Dict[str, Any]:
+) -> Optional[Dict[str, Any]]:
     """Fetch recommendation from the RAG system API."""
     try:
         async with session.post(
@@ -41,19 +40,19 @@ async def fetch_recommendation(
 
 
 async def process_samples(
-    samples_file: str, output_file: str, batch_size: int = 5
+    samples_file: str, output_file: str, endpoint: str, batch_size: int = 5
 ) -> None:
     """Process samples in batches and save results."""
     # Load samples
     with open(samples_file, "r") as f:
         samples = json.load(f)
 
-    results = []
+    results: List[Dict[str, Any]] = []
     async with aiohttp.ClientSession() as session:
         # Process in batches
         for i in range(0, len(samples), batch_size):
             batch = samples[i : i + batch_size]
-            tasks = [fetch_recommendation(session, query) for query in batch]
+            tasks = [fetch_recommendation(session, query, endpoint) for query in batch]
             batch_results = await tqdm_asyncio.gather(*tasks)
             results.extend([r for r in batch_results if r is not None])
 
@@ -89,7 +88,9 @@ def main() -> None:
 
     args = parser.parse_args()
 
-    asyncio.run(process_samples(args.input, args.output, args.batch_size))
+    asyncio.run(
+        process_samples(args.input, args.output, args.endpoint, args.batch_size)
+    )
 
 
 if __name__ == "__main__":