Merge pull request #1066 from kga245/reduce-docker-logging

Logging update. Fixed regression and update to Docker log handling.

backend/server/websocket_manager.py

@@ -10,6 +10,7 @@
 from gpt_researcher.utils.enum import ReportType, Tone
 from multi_agents.main import run_research_task
 from gpt_researcher.actions import stream_output  # Import stream_output
+from backend.server.server_utils import CustomLogsHandler
 
 
 class WebSocketManager:
@@ -78,10 +79,21 @@ async def chat(self, message, websocket):
 async def run_agent(task, report_type, report_source, source_urls, document_urls, tone: Tone, websocket, headers=None, config_path=""):
     """Run the agent."""
     start_time = datetime.datetime.now()
-    # Instead of running the agent directly run it through the different report type classes
+
+    # Create logs handler for this research task
+    logs_handler = CustomLogsHandler(websocket, task)
+
+    # Initialize researcher based on report type
     if report_type == "multi_agents":
-        report = await run_research_task(query=task, websocket=websocket, stream_output=stream_output, tone=tone, headers=headers)
+        report = await run_research_task(
+            query=task, 
+            websocket=logs_handler,  # Use logs_handler instead of raw websocket
+            stream_output=stream_output, 
+            tone=tone, 
+            headers=headers
+        )
         report = report.get("report", "")
+
     elif report_type == ReportType.DetailedReport.value:
         researcher = DetailedReport(
             query=task,
@@ -91,10 +103,11 @@ async def run_agent(task, report_type, report_source, source_urls, document_urls
             document_urls=document_urls,
             tone=tone,
             config_path=config_path,
-            websocket=websocket,
+            websocket=logs_handler,  # Use logs_handler instead of raw websocket
             headers=headers
         )
         report = await researcher.run()
+
     else:
         researcher = BasicReport(
             query=task,
@@ -104,15 +117,9 @@ async def run_agent(task, report_type, report_source, source_urls, document_urls
             document_urls=document_urls,
             tone=tone,
             config_path=config_path,
-            websocket=websocket,
+            websocket=logs_handler,  # Use logs_handler instead of raw websocket
             headers=headers
         )
         report = await researcher.run()
 
-    # measure time
-    end_time = datetime.datetime.now()
-    await websocket.send_json(
-        {"type": "logs", "output": f"\nTotal run time: {end_time - start_time}\n"}
-    )
-
     return report

docker-compose.yml

@@ -7,6 +7,9 @@ services:
       OPENAI_API_KEY: ${OPENAI_API_KEY}
       TAVILY_API_KEY: ${TAVILY_API_KEY}
       LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
+      LOGGING_LEVEL: INFO
+    volumes:
+      - ./outputs:/usr/src/app/outputs
     restart: always
     ports:
       - 8000:8000
@@ -15,13 +18,15 @@ services:
     image: gptresearcher/gptr-nextjs
     stdin_open: true
     environment:
-      - CHOKIDAR_USEPOLLING=true
+      CHOKIDAR_USEPOLLING: true
+      LOGGING_LEVEL: INFO
     build:
       dockerfile: Dockerfile.dev
       context: frontend/nextjs
     volumes:
       - /app/node_modules
       - ./frontend/nextjs:/app
+      - ./outputs:/app/outputs
     restart: always
     ports:
       - 3000:3000
@@ -33,6 +38,7 @@ services:
       OPENAI_API_KEY: ${OPENAI_API_KEY}
       TAVILY_API_KEY: ${TAVILY_API_KEY}
       LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
+      LOGGING_LEVEL: INFO
     profiles: ["test"]
     command: >
       /bin/sh -c "

main.py

@@ -18,6 +18,11 @@
     ]
 )
 
+# Suppress verbose fontTools logging
+logging.getLogger('fontTools').setLevel(logging.WARNING)
+logging.getLogger('fontTools.subset').setLevel(logging.WARNING)
+logging.getLogger('fontTools.ttLib').setLevel(logging.WARNING)
+
 # Create logger instance
 logger = logging.getLogger(__name__)
 

tests/test_logging_output.py

@@ -0,0 +1,49 @@
+import pytest
+import asyncio
+from pathlib import Path
+import json
+import logging
+from fastapi import WebSocket
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class TestWebSocket(WebSocket):
+    def __init__(self):
+        self.events = []
+
+    async def accept(self):
+        pass
+
+    async def send_json(self, event):
+        logger.info(f"WebSocket received event: {event}")
+        self.events.append(event)
+
+@pytest.mark.asyncio
+async def test_log_output_file():
+    """Test to verify logs are properly written to output file"""
+    from gpt_researcher.agent import GPTResearcher
+
+    # 1. Setup like the main app
+    websocket = TestWebSocket()
+    await websocket.accept()
+
+    # 2. Initialize researcher like main app
+    query = "What is the capital of France?"
+    researcher = GPTResearcher(query=query, websocket=websocket)
+
+    # 3. Run research
+    await researcher.conduct_research()
+
+    # 4. Verify events were captured
+    logger.info(f"Events captured: {len(websocket.events)}")
+    assert len(websocket.events) > 0, "No events were captured"
+
+    # 5. Check output file
+    output_dir = Path("outputs")
+    output_files = list(output_dir.glob(f"task_*_{query.replace(' ', '_')[:50]}.json"))
+    assert len(output_files) > 0, "No output file was created"
+
+    with open(output_files[-1]) as f:
+        data = json.load(f)
+        assert len(data.get('events', [])) > 0, "No events in output file"