Adding the TTS code

.gitignore

@@ -150,3 +150,7 @@ dmypy.json
 *.webm
 #calender json
 /core/tools/calendarjson
+
+
+#Redis data
+redis_data

core/static/index.html

@@ -12,6 +12,7 @@
     <script src="/static/socketEvents.js"></script>
     <script  src="/static/index.js" defer></script>
     <script src="/static/eventlisteners.js"></script>
+    <script src="/static/tts.js"></script>
     <link rel="stylesheet" href="/static/index.css">
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>

core/static/tts.js

@@ -0,0 +1,187 @@
+// TODO:  Remove random debugging stuff
+// Check if socket already exists, if not create it
+const ttsSocket = (() => {
+    const config = {
+        websocketServer: 'http://localhost:5000'
+    };
+
+    return io(config.websocketServer, {
+        transports: ['websocket', 'polling'],
+        reconnection: true,
+        reconnectionAttempts: 5,
+        reconnectionDelay: 1000,
+        timeout: 10000,
+        autoConnect: true,
+    });
+})();
+
+console.log('Attempting to connect to server...');
+
+let audioContext;
+let isProcessing = false;
+let audioQueue = [];
+let expectedSentenceId = 1;
+
+// Update all socket references to ttsSocket
+ttsSocket.onAny((eventName, ...args) => {
+    console.log(`Received event: ${eventName}`, args);
+});
+
+ttsSocket.on('connecting', () => {
+    console.log('Attempting to connect...');
+});
+
+ttsSocket.on('connect', () => {
+    console.log('Connected to remote WebSocket server:', ttsSocket.io.uri);
+    console.log('Connected to server with ID:', ttsSocket.id);
+    console.log('Transport type:', ttsSocket.io.engine.transport.name);
+});
+
+ttsSocket.on('connect_error', (error) => {
+    console.error('Connection error:', error);
+    console.log('Failed connecting to:', ttsSocket.io.uri);
+    console.log('Transport type:', ttsSocket.io.engine.transport.name);
+});
+
+ttsSocket.on('connect_timeout', () => {
+    console.error('Connection timeout');
+});
+
+ttsSocket.on('reconnect_attempt', (attemptNumber) => {
+    console.log(`Reconnection attempt ${attemptNumber}`);
+});
+
+ttsSocket.on('disconnect', () => {
+    console.log('Disconnected from server');
+});
+
+async function initAudioContext() {
+    audioContext = new (window.AudioContext || window.webkitAudioContext)();
+}
+
+async function processAudioChunk(audioData, sentenceId) {
+    try {
+        console.log(`Processing audio chunk for sentence ${sentenceId}`);
+        const arrayBuffer = new Uint8Array(audioData).buffer;
+        const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+        const source = audioContext.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(audioContext.destination);
+
+        // Add an event listener for when the audio finishes playing
+        source.onended = () => {
+            console.log(`Finished playing sentence ${sentenceId}`);
+            expectedSentenceId++;
+            isProcessing = false;
+            processQueuedAudio(); // Process next chunk if available
+        };
+
+        source.start();
+        isProcessing = true;
+
+    } catch (error) {
+        console.error('Error processing audio chunk:', error);
+        isProcessing = false;
+        processQueuedAudio(); // Try next chunk on error
+    }
+}
+
+function processQueuedAudio() {
+    if (isProcessing || audioQueue.length === 0) return;
+
+    // Sort queue by sentence ID
+    audioQueue.sort((a, b) => a.sentenceId - b.sentenceId);
+
+    // Process next chunk if it matches expected ID
+    const nextChunk = audioQueue[0];
+    if (nextChunk.sentenceId === expectedSentenceId) {
+        audioQueue.shift(); // Remove from queue
+        processAudioChunk(nextChunk.audioData, nextChunk.sentenceId);
+    }
+}
+
+// Socket.IO event handler
+ttsSocket.on('audio_stream', async (data) => {
+    console.log('Received audio_stream event:', {
+        sentenceId: data.sentence_id,
+        dataLength: data.audio_data.length
+    });
+
+    if (!audioContext) {
+        console.log('Initializing audio context');
+        await initAudioContext();
+    }
+
+    const audioData = new Uint8Array(data.audio_data);
+    const sentenceId = data.sentence_id;
+
+    // Reset state if this is the start of a new generation
+    if (sentenceId === 1) {
+        console.log('New text generation - resetting client state');
+        expectedSentenceId = 1;
+        audioQueue = [];
+        isProcessing = false;
+    }
+
+    console.log(`Queueing audio chunk ${sentenceId}`);
+
+    // Queue the audio chunk
+    audioQueue.push({
+        audioData: audioData,
+        sentenceId: sentenceId,
+        timestamp: Date.now()
+    });
+
+    console.log(`Current queue length: ${audioQueue.length}`);
+    // Try to process queued audio
+    processQueuedAudio();
+});
+
+// Initialize audio context on user interaction
+document.addEventListener('click', async () => {
+    if (!audioContext) {
+        await initAudioContext();
+    }
+    if (audioContext.state === 'suspended') {
+        await audioContext.resume();
+    }
+});
+
+ttsSocket.on('test', (data) => {
+    console.log('Received test message:', data);
+});
+
+ttsSocket.on('connect', () => {
+    console.log('Connected to remote WebSocket server:', ttsSocket.io.uri);
+    console.log('Connected to server with ID:', ttsSocket.id);
+    console.log('Transport type:', ttsSocket.io.engine.transport.name);
+});
+
+ttsSocket.on('connect_error', (error) => {
+    console.error('Connection error:', error);
+    console.log('Failed connecting to:', ttsSocket.io.uri);
+    console.log('Transport type:', ttsSocket.io.engine.transport.name);
+});
+
+ttsSocket.on('connect_timeout', () => {
+    console.error('Connection timeout');
+});
+
+ttsSocket.on('reconnect_attempt', (attemptNumber) => {
+    console.log(`Reconnection attempt ${attemptNumber}`);
+});
+
+ttsSocket.on('disconnect', () => {
+    console.log('Disconnected from server');
+});
+
+ttsSocket.on('connect', () => {
+    console.log('Connected to remote WebSocket server:', ttsSocket.io.uri);
+    console.log('Transport type:', ttsSocket.io.engine.transport.name);
+});
+
+ttsSocket.on('connect_error', (error) => {
+    console.error('Connection error:', error);
+    console.log('Failed connecting to:', ttsSocket.io.uri);
+}); 

docker-compose.yml

@@ -43,6 +43,38 @@ services:
     stop_signal: SIGINT
     ports:
       - "3001:3001"
+  redis:
+    image: redis:latest
+    container_name: redis_audio
+    ports:
+      - "6379:6379"
+    command: >
+      redis-server 
+      --appendonly yes 
+      --save 60 1 
+      --save 300 100 
+      --save 900 1000
+      --maxmemory 1000mb
+    volumes:
+      - ./redis_data:/data
+  tts:
+    build:
+      context: ./textToSpeech
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    ports:
+      - "5000:5000"
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - NARKEE_API_KEY=${NARKEE_API_KEY}
+    depends_on:
+      - redis 
+
+
+
+
+
 
 networks:
   backend:

textToSpeech/Dockerfile

@@ -1,25 +1,8 @@
-# Use an official Python runtime as a parent image
-FROM python:3.9-slim
-
-# Set the working directory in the container
+FROM python:3.12-alpine
 WORKDIR /app
+COPY requirements.txt .
+RUN pip install -r requirements.txt
 
-# Copy the current directory contents into the container at /app
-COPY . /app
-
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    espeak-ng \
-    libespeak-ng1 \
-    ffmpeg \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install any needed packages specified in requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Make port 5000 available to the world outside this container
-EXPOSE 5000
-
-# Run ttssend.py when the container launches
-CMD ["python", "tts_server.py"]
+COPY app /app
 
+CMD ["python", "/app/app.py"]