Merge branch 'dev' into main

.flake8

@@ -1,4 +1,5 @@
 [flake8]
-max-line-length = 120
-exclude = .venv, _pycache_, migrations
-ignore = E501,F401,F811,F841,E203,E231,W503
+max-line-length = 88
+extend-ignore = E501
+exclude = venv, frontend
+ignore = E203, W503

.github/workflows/deploy.yml

@@ -0,0 +1,190 @@
+name: CI-Validate Deployment
+
+on:
+    push:
+      branches:
+        - main
+    schedule:
+      - cron: '0 6,18 * * *'  # Runs at 6:00 AM and 6:00 PM GMT
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Setup Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version  # Verify installation
+
+      - name: Login to Azure
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
+
+      - name: Install Bicep CLI
+        run: az bicep install
+
+      - name: Generate Resource Group Name
+        id: generate_rg_name
+        run: |
+          echo "Generating a unique resource group name..."
+          TIMESTAMP=$(date +%Y%m%d%H%M%S)
+          COMMON_PART="pslautomationRes"
+          UNIQUE_RG_NAME="${COMMON_PART}${TIMESTAMP}"
+          echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
+          echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}" 
+      
+      - name: Check and Create Resource Group
+        id: check_create_rg
+        run: |
+          set -e  
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "false" ]; then
+            echo "Resource group does not exist. Creating..."
+            az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; }
+          else
+            echo "Resource group already exists."
+          fi
+
+      - name: Generate Unique Solution Prefix
+        id: generate_solution_prefix
+        run: |
+          set -e
+          COMMON_PART="pslr"
+          TIMESTAMP=$(date +%s)  
+          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3) 
+          UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}"
+          echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
+          echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}" 
+
+      - name: Deploy Bicep Template
+        id: deploy
+        run: |
+          set -e
+          az deployment group create \
+            --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
+            --template-file infrastructure/deployment.json \
+            --parameters \
+                HostingPlanName="${{ env.SOLUTION_PREFIX }}-plan" \
+                ApplicationInsightsName="appins-${{ env.SOLUTION_PREFIX }}" \
+                WebsiteName="webapp-${{ env.SOLUTION_PREFIX }}" \
+                CosmosDBName="db-cosmos-${{ env.SOLUTION_PREFIX }}" \
+                CosmosDBRegion="NorthCentralUS" \
+                AzureSearchService="search-${{ env.SOLUTION_PREFIX }}" \
+                AzureOpenAIResource="aoai-${{ env.SOLUTION_PREFIX }}" \
+                WorkspaceName="worksp-${{ env.SOLUTION_PREFIX }}" 
+      
+      - name: Delete Bicep Deployment
+        if: success()
+        run: |
+          set -e  
+          echo "Checking if resource group exists..."
+          rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
+          if [ "$rg_exists" = "true" ]; then
+            echo "Resource group exists. Cleaning..."
+            az group delete \
+                --name ${{ env.RESOURCE_GROUP_NAME }} \
+                --yes \
+                --no-wait
+            echo "Resource group deleted...  ${{ env.RESOURCE_GROUP_NAME }}"
+          else
+            echo "Resource group does not exist."
+          fi
+
+      - name: Wait for Resource Deletion to Complete
+        run: |
+          # List of resources to check based on SOLUTION_PREFIX
+          resources_to_check=(
+            "aoai-${{ env.SOLUTION_PREFIX }}"
+            "appins-${{ env.SOLUTION_PREFIX }}"
+            "db-cosmos-${{ env.SOLUTION_PREFIX }}"
+            "${{ env.SOLUTION_PREFIX }}-plan"
+            "search-${{ env.SOLUTION_PREFIX }}"
+            "webapp-${{ env.SOLUTION_PREFIX }}"
+            "worksp-${{ env.SOLUTION_PREFIX }}"
+          )
+
+          # Get the list of resources in YAML format
+          resource_list=$(az resource list --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} --output yaml)
+          
+          # Maximum number of retries and retry intervals
+          max_retries=3
+          retry_intervals=(30 60 120)
+          retries=0
+          
+          while true; do
+            resource_found=false
+            # Iterate through the resources to check
+            for resource in "${resources_to_check[@]}"; do
+              echo "Checking resource: $resource"
+              if echo "$resource_list" | grep -q "name: $resource"; then
+                echo "Resource '$resource' exists in the subscription."
+                resource_found=true
+              else
+                echo "Resource '$resource' does not exist in the subscription."
+              fi
+            done
+
+            # If any resource exists, retry
+            if [ "$resource_found" = true ]; then
+              retries=$((retries + 1))
+              if [ "$retries" -ge "$max_retries" ]; then
+                echo "Maximum retry attempts reached. Exiting."
+                break
+              else
+                echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..."
+                sleep ${retry_intervals[$retries-1]}
+              fi
+            else
+              echo "No resources found. Exiting."
+              break
+            fi
+          done
+
+      - name: Purging the Resources
+        if: success()
+        run: |
+          set -e
+          # Purging resources based on solution prefix
+          echo "Purging resources..."
+
+          # List of resources to purge
+          resources_to_purge=(
+            "aoai-${{ env.SOLUTION_PREFIX }}"
+            "appins-${{ env.SOLUTION_PREFIX }}"
+            "db-cosmos-${{ env.SOLUTION_PREFIX }}"
+            "${{ env.SOLUTION_PREFIX }}-plan"
+            "search-${{ env.SOLUTION_PREFIX }}"
+            "webapp-${{ env.SOLUTION_PREFIX }}"
+            "worksp-${{ env.SOLUTION_PREFIX }}"
+          )
+          
+          for resource in "${resources_to_purge[@]}"; do
+            echo "Purging resource: $resource"
+            if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/uksouth/deletedAccounts/$resource --verbose; then
+              echo "Failed to purge resource: $resource"
+            else
+              echo "Purged the resource: $resource"
+            fi
+          done
+
+          echo "Resource purging completed successfully"
+
+      - name: Send Notification on Failure
+        if: failure()
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>We would like to inform you that the Research Assistant Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
+          }
+          EOF
+          )
+
+          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"

.github/workflows/docker-build-and-push.yml

@@ -5,6 +5,8 @@ on:
     types: [closed]
     branches:
       - main
+      - dev
+      - demo
   workflow_dispatch: # Add this line to enable manual triggering
 
 jobs:
@@ -20,13 +22,40 @@ jobs:
         uses: docker/setup-buildx-action@v1
 
       - name: Log in to Azure Container Registry
-        uses: azure/docker-login@v1
+        if: ${{ github.ref_name == 'main' }}
+        uses: azure/docker-login@v2
         with:
           login-server: ${{ secrets.ACR_LOGIN_SERVER }}
           username: ${{ secrets.ACR_USERNAME }}
           password: ${{ secrets.ACR_PASSWORD }}
 
+      - name: Log in to Azure Container Registry (Dev/Demo)
+        if: ${{ github.ref_name == 'dev' || github.ref_name == 'demo' }}
+        uses: azure/docker-login@v2
+        with:
+          login-server: ${{ secrets.ACR_DEV_LOGIN_SERVER }}
+          username: ${{ secrets.ACR_DEV_USERNAME }}
+          password: ${{ secrets.ACR_DEV_PASSWORD }}
+
+      - name: Set Docker image tag
+        id: docker_tag
+        run: |
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "TAG=latest" >> $GITHUB_ENV
+          elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
+            echo "TAG=dev" >> $GITHUB_ENV
+          elif [[ "${{ github.ref }}" == "refs/heads/demo" ]]; then
+            echo "TAG=demo" >> $GITHUB_ENV
+          fi
+
       - name: Build and push Docker image
+        if: ${{ github.ref_name == 'main' }}
+        run: |
+          docker build -t ${{ secrets.ACR_LOGIN_SERVER }}/webapp:${{ env.TAG }} -f WebApp.Dockerfile .
+          docker push ${{ secrets.ACR_LOGIN_SERVER }}/webapp:${{ env.TAG }}
+
+      - name: Build and push Docker image (Dev/Demo)
+        if: ${{ github.ref_name == 'dev' || github.ref_name == 'demo' }}
         run: |
           docker build -t ${{ secrets.ACR_LOGIN_SERVER }}/webapp:latest -f WebApp.Dockerfile .
           docker push ${{ secrets.ACR_LOGIN_SERVER }}/webapp:latest

.github/workflows/pylint.yml

@@ -20,16 +20,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # Step 3: Run all code quality checks
-      - name: Run Code Quality Checks
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          echo "Fixing imports with Isort..."
-          python -m isort --verbose .
-          echo "Formatting code with Black..."
-          python -m black --verbose .
-          echo "Running Flake8..."
-          python -m flake8 --config=.flake8 --verbose .
+          pip install -r requirements-dev.txt
+          
+      # Step 3: Run all code quality checks
+      - name: Pylint
+        run: |
           echo "Running Pylint..."
-          python -m pylint --rcfile=.pylintrc --verbose .
+          python -m flake8 --config=.flake8 --verbose .
+          

.github/workflows/tests.yml

@@ -0,0 +1,111 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - main  # Trigger on push to the main branch
+  pull_request:
+    branches:
+      - main  # Trigger on pull requests to the main branch
+    types:
+      - opened
+      - ready_for_review
+      - reopened
+      - synchronize
+
+jobs:
+  backend_tests:
+    name: Backend Tests
+    runs-on: ubuntu-latest  # Use the latest Ubuntu runner
+
+    steps:
+      - uses: actions/checkout@v4  # Checkout the repository
+
+      # Set up Python environment for Backend
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"  # Set Python version
+
+      - name: Install Backend Dependencies
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install coverage pytest-cov
+          python -m pip install azure-keyvault-secrets
+
+      - name: Run Backend Tests with Coverage
+        run: |
+          if python -m pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term-missing --junitxml=coverage-junit.xml; then
+            echo "Tests completed, checking coverage."
+            # Only fail if coverage does not meet criteria
+            if [ -f coverage.xml ]; then
+              COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); coverage = root.find('coverage').get('lines').split('%')[0]; print(float(coverage))")
+              if (( $(echo "$COVERAGE < 80" | bc -l) )); then
+                echo "Coverage is below 80%, failing the job."
+                exit 1
+              fi
+            fi
+          else
+            echo "No tests found, skipping coverage check."
+          fi
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: backend-coverage
+          path: |
+            coverage.xml  # Correct path to backend coverage
+            coverage-junit.xml  # Correct path to backend JUnit report
+            htmlcov/  # Correct path to backend HTML coverage report
+
+  frontend_tests:
+    name: Frontend Tests
+    runs-on: ubuntu-latest  # Use the latest Ubuntu runner
+
+    steps:
+      - uses: actions/checkout@v4  # Checkout the repository
+
+      # Set up Node.js environment for Frontend
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'  # Set the Node.js version
+
+      - name: Cache npm dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      
+      - name: Navigate to frontend directory
+        run: cd frontend
+
+      - name: Install Frontend Dependencies
+        run: |
+          cd frontend  # Change to the frontend directory
+          npm install  # Install dependencies from frontend/package.json
+
+      - name: Run Frontend Tests with Coverage
+        run: |
+          cd frontend  # Change to the frontend directory
+          if npm run test -- --coverage; then
+            echo "Tests completed, checking coverage."
+            # Check coverage report and ensure it meets threshold
+            if [ -f frontend/coverage/lcov-report/index.html ]; then
+              COVERAGE=$(cat frontend/coverage/lcov-report/index.html | grep -oP 'total: \K[0-9]+(\.[0-9]+)?')
+              if (( $(echo "$COVERAGE < 80" | bc -l) )); then
+                echo "Coverage is below 80%, failing the job."
+                exit 1
+              fi
+            fi
+          else
+            echo "No tests found, skipping coverage check."
+          fi
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: frontend-coverage
+          path: |
+            frontend/coverage/  # Correct path to frontend coverage
+            frontend/coverage/lcov-report/  # Correct path to frontend lcov report