Ci/speed up ci

.github/workflows/ci.yml

@@ -21,28 +21,20 @@ jobs:
         ! git log origin/master..HEAD --oneline --pretty=format:%s | \
           grep -Ev '^(build|chore|ci|docs|feat|fix|perf|style|refactor|test):|^Merge '
 
-  quality:
-    name: Quality Checks
+  lint:
+    name: Lint Checks
     needs: commit-check
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        task: [lint, test, build]
-
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
     - name: Set up Python 3.12
-      # Skip Python setup for build task
-      if: matrix.task != 'build'
       uses: actions/setup-python@v5
       with:
         python-version: '3.12'
 
-    - name: Cache Python dependencies
-      # Skip cache for build task
-      if: matrix.task != 'build'
+    - name: Cache Poetry dependencies
       uses: actions/cache@v4
       with:
         path: ~/.cache/poetry
@@ -51,33 +43,22 @@ jobs:
           ${{ runner.os }}-poetry-
 
     - name: Install dependencies
-      # Skip dependencies for build task
-      if: matrix.task != 'build'
       run: |
         curl -sSL https://install.python-poetry.org | python3 -
         poetry config virtualenvs.create false
         poetry install
 
-    - name: Set up Docker
-      if: matrix.task == 'build'
-      uses: docker/setup-buildx-action@v3
-
-    - name: Set up Docker Compose
-      if: matrix.task == 'build'
-      run: |
-        docker compose version
-
-    - name: Run ${{ matrix.task }}
-      run: make ${{ matrix.task }}
+    - name: Run lint
+      run: make lint
 
     - name: Generate Pylint Badge
-      if: matrix.task == 'lint' && github.event_name == 'push' && github.ref == 'refs/heads/master'
+      if: github.event_name == 'push' && github.ref == 'refs/heads/master'
       run: |
         SCORE=$(pylint portfolio_analytics --output-format=text **/*.py | sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p')
         echo "PYLINT_SCORE=$SCORE" >> $GITHUB_ENV
 
     - name: Create Pylint Badge
-      if: matrix.task == 'lint' && github.event_name == 'push' && github.ref == 'refs/heads/master'
+      if: github.event_name == 'push' && github.ref == 'refs/heads/master'
       uses: schneegans/dynamic-badges-action@v1.7.0
       with:
         auth: ${{ secrets.GIST_SECRET }}
@@ -87,23 +68,80 @@ jobs:
         message: ${{ env.PYLINT_SCORE }}
         color: ${{ env.PYLINT_SCORE >= 9 && 'brightgreen' || env.PYLINT_SCORE >= 7 && 'yellow' || 'red' }}
 
+  test:
+    name: Run Tests
+    needs: commit-check
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/poetry
+        key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
+    - name: Install dependencies
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        poetry config virtualenvs.create false
+        poetry install
+
+    - name: Run tests
+      run: make test
+
     - name: Upload coverage reports
-      # Only run for test task
-      if: matrix.task == 'test'
       uses: codecov/codecov-action@v4
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         files: ./coverage.xml
-        fail_ci_if_error: false  # TODO: set to true
+        fail_ci_if_error: false
 
-  integration:
-    name: Integration Tests
-    needs: quality
+  build:
+    name: Build Docker Images
+    needs: commit-check
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        component: [api, dashboard]
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/poetry
+        key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
+    - name: Install dependencies
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        poetry config virtualenvs.create false
+        poetry install
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
+
+    - name: Run build
+      run: make build
 
+  integration-api:
+    name: API Integration Tests
+    needs: [lint, test, build]
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
@@ -113,6 +151,14 @@ jobs:
       with:
         python-version: '3.12'
 
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/poetry
+        key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
     - name: Install dependencies
       run: |
         curl -sSL https://install.python-poetry.org | python3 -
@@ -122,16 +168,31 @@ jobs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v3
 
-    - name: Set up Docker Compose
+    - name: Start API service and run integration tests
       run: |
-        docker compose version
+        docker compose up -d --pull never
+        pytest tests/integration -v -m api_integration
+        docker compose down
+
+  integration-dashboard:
+    name: Dashboard Integration Tests
+    needs: [lint, test, build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
 
-    - name: Start ${{ matrix.component }} service and run integration tests
-      run: make test-${{ matrix.component }}
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
+
+    - name: Start Dashboard service and run integration tests
+      run: |
+        docker build -t dash-app-tests . -f tests/integration/test_dashboard.Dockerfile
+        docker run --rm dash-app-tests
 
   release:
     name: Release
-    needs: [commit-check, quality, integration]
+    needs: [integration-api, integration-dashboard]
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     permissions:
@@ -167,3 +228,37 @@ jobs:
       run: |
         git pull --rebase
         python -m semantic_release publish --patch
+
+  publish:
+    name: Publish Docker Images
+    needs: release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
+
+    - name: Get latest git tag
+      run: |
+        git fetch --tags
+        TAG=$(git describe --tags --abbrev=0)
+        echo "RELEASE_VERSION=${TAG}" >> $GITHUB_ENV
+
+    - name: Build and push application images
+      run: |
+        # Build images
+        docker compose build
+
+        # Tag with version and latest
+        docker tag ghcr.io/gbourniq/portfolio_analytics/api ghcr.io/gbourniq/portfolio_analytics/api:${{ env.RELEASE_VERSION }}
+        docker tag ghcr.io/gbourniq/portfolio_analytics/api ghcr.io/gbourniq/portfolio_analytics/api:latest
+        docker tag ghcr.io/gbourniq/portfolio_analytics/dashboard ghcr.io/gbourniq/portfolio_analytics/dashboard:${{ env.RELEASE_VERSION }}
+        docker tag ghcr.io/gbourniq/portfolio_analytics/dashboard ghcr.io/gbourniq/portfolio_analytics/dashboard:latest
+
+        # Push all tags
+        docker push ghcr.io/gbourniq/portfolio_analytics/api:${{ env.RELEASE_VERSION }}
+        docker push ghcr.io/gbourniq/portfolio_analytics/api:latest
+        docker push ghcr.io/gbourniq/portfolio_analytics/dashboard:${{ env.RELEASE_VERSION }}
+        docker push ghcr.io/gbourniq/portfolio_analytics/dashboard:latest

README.md

@@ -46,8 +46,14 @@ The dashboard analyzes portfolio positions across multiple stock exchanges, acce
 
 ### Quick Start with Docker
 
+Run containers with images built locally
 ```bash
-make up
+docker-compose up -d --build
+```
+
+Run containers with packaged images from the Github Container Registry
+```bash
+GIT_TAG=v0.1.24 docker-compose up -d --pull
 ```
 
 ### Local Development

api.Dockerfile

@@ -1,32 +1,17 @@
-FROM python:3.12-slim
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app/portfolio_analytics
+FROM ghcr.io/gbourniq/portfolio_analytics/python-base:3.12-slim
 
 # Add labels
-LABEL maintainer="guillaume.bournique@gmail.com" \
-description="Portfolio Analytics API"
-
-# Install curl for healthcheck
-RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user
-RUN groupadd -r appuser && useradd -r -g appuser appuser
-
-WORKDIR /app
+LABEL description="Portfolio Analytics API"
 
 # Install dependencies
 COPY pyproject.toml ./pyproject.toml
-RUN python3.12 -m pip install poetry==1.8.5 && \
-    poetry config virtualenvs.create false && \
-    poetry install --only main,api
+RUN poetry install --only main,api
 
 # Copy source code to the container
 COPY portfolio_analytics/common/utils portfolio_analytics/common/utils
 COPY portfolio_analytics/api portfolio_analytics/api
 
-# Set ownership
+# Set ownership and switch to non-root user
 RUN chown -R appuser:appuser /app
 USER appuser
 

base.Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONPATH=/app/portfolio_analytics
+
+# Add labels
+LABEL maintainer="guillaume.bournique@gmail.com" \
+    description="Base Python image for Portfolio Analytics" \
+    org.opencontainers.image.source="https://github.com/gbourniq/portfolio_analytics"
+
+# Install common system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    htop \
+    vim \
+    procps \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
+WORKDIR /app
+
+# Install poetry
+RUN python3.12 -m pip install poetry==1.8.5 && \
+    poetry config virtualenvs.create false

dashboard.Dockerfile

@@ -1,33 +1,17 @@
-FROM python:3.12-slim
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONPATH=/app/portfolio_analytics
+FROM ghcr.io/gbourniq/portfolio_analytics/python-base:3.12-slim
 
 # Add labels
-LABEL maintainer="guillaume.bournique@gmail.com" \
-description="Portfolio Analytics Dashboard"
-
-# Install curl for healthcheck
-RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user
-RUN groupadd -r appuser && useradd -r -g appuser appuser
-
-WORKDIR /app
+LABEL description="Portfolio Analytics Dashboard"
 
 # Install dependencies
 COPY pyproject.toml ./pyproject.toml
-RUN python3.12 -m pip install poetry==1.8.5 && \
-    poetry config virtualenvs.create false && \
-    poetry install --only main,dashboard
+RUN poetry install --only main,dashboard
 
 # Copy source code to the container
 COPY portfolio_analytics/common/utils portfolio_analytics/common/utils
 COPY portfolio_analytics/dashboard portfolio_analytics/dashboard
 
-# Set ownership
+# Set ownership and switch to non-root user
 RUN chown -R appuser:appuser /app
 USER appuser