diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..ad94101 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,62 @@ +# CI Workflows + +This repository contains several CI workflows designed for deployment in a self-hosted runner. Below is a description of each workflow and its purpose. + +## Workflows + +### `ci-build.yml` +**Trigger:** Automatically on every pull request or Manually + +- Builds the `depa-training-encfs`, `depa-training`, and other containers required for the COVID scenario. +- Runs containers locally to perform preprocessing, save the model, and train the model. + +### `ci.yml` +**Trigger:** Manually + +- Prepares data and model for deployment. +- Creates Azure Storage and Key Vault if they do not already exist. +- Imports data and model encryption keys with key release policies. +- Encrypts the data and model. +- Uploads the encrypted data and model to Azure Storage. +- Deploys CCR on Azure Container Instances (ACI) and trains the model. + +### `ci-local.yml` +**Trigger:** Manually + +- Pulls containers from Azure Container Registry (ACR). +- Runs containers locally to perform preprocessing, save the model, and train the model. + +### `release.yml` +**Trigger:** Release Event + +- Builds the `depa-training-encfs`, `depa-training`, contract service container, and other containers required for the COVID scenario. +- Pushes the built containers to ACR. + +### `contract-service.yml` +**Trigger:** Manually + +- Builds the contract service container. +- Deploys the contract service. + +## Steps to Deploy Self-Hosted Runner + +1. **Set up Recommended OIDC Authentication:** + - Follow the [official guide](https://learn.microsoft.com/en-us/azure/app-service/deploy-github-actions?tabs=openid%2Cpython%2Caspnetcore#set-up-a-github-actions-workflow-manually) to authenticate GitHub Actions with Azure services using the OIDC approach. + +2. **Assign Necessary Permissions:** + - For the service principal created during the above step (or an external one), assign the following permissions: + - Contributor + - Custom role with `Microsoft.Authorization/GetRoleAssignment` and `Microsoft.Authorization/CreateRoleAssignment` + +3. **Create a New Self-Hosted Runner:** + - Navigate to `Settings` > `Actions` > `Runners` in your GitHub repository. + - Create a "New self-hosted Runner". + +4. **Set Up the Azure VM:** + - Create an Azure VM with at least 16GB RAM, 4 CPUs, and 128GB SSD. + - Follow the instructions provided in the self-hosted runner setup to configure the action runner on your Azure VM. + +--- + +By following these instructions, you can set up and utilize the CI workflows in your self-hosted runner to automate and manage the deployment processes for your projects. + diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 0000000..2f06667 --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,60 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Build and Test Images + +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + build-and-test-image: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Update submodules + run: git submodule update --init --recursive + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.19.x' + + - name: Install jq + run: sudo apt install -y jq + + - name: Install make + run: sudo apt install make + + - name: Install wheel + run: pip install wheel + + - name: Build encrypted filesystem artifacta, contract ledger client & depa-training container + run: ci/build.sh + + - name: Build container images + run: cd ${{ github.workspace }}/scenarios/covid && ./ci/build.sh + + - name: Run pre-processing + run: cd ./scenarios/covid/deployment/docker && ./preprocess.sh + + - name: Run model saving + run: cd ./scenarios/covid/deployment/docker && ./save-model.sh + + - name: Run training + run: cd ./scenarios/covid/deployment/docker && ./train.sh + diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 17198ff..5595509 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -9,14 +9,18 @@ name: Deploy COVID scenario in Docker -on: [workflow_dispatch] +on: + workflow_dispatch: env: CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} jobs: deploy-ccr-covid: runs-on: [self-hosted, linux, X64] + steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 @@ -24,9 +28,10 @@ jobs: - name: Update submodules run: git submodule update --init --recursive - - name: Login to Docker Hub - uses: docker/login-action@v2 + - name: Login to ACR + uses: docker/login-action@v3 with: + registry: ${{ vars.CONTAINER_REGISTRY }} username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} @@ -41,6 +46,6 @@ jobs: - name: Run model saving run: cd ./scenarios/covid/deployment/docker && ./save-model.sh - + - name: Run training run: cd ./scenarios/covid/deployment/docker && ./train.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe919db..7273ad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ env: CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} AZURE_STORAGE_ACCOUNT_NAME: ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} AZURE_ICMR_CONTAINER_NAME: ${{ vars.AZURE_ICMR_CONTAINER_NAME }} AZURE_COWIN_CONTAINER_NAME: ${{ vars.AZURE_COWIN_CONTAINER_NAME }} @@ -31,6 +32,11 @@ env: TOOLS_HOME: ${{ github.workspace }}/external/confidential-sidecar-containers/tools DATA_DIRECTORY: ${{ github.workspace}}/scenarios/covid/data CONTRACT_SERVICE_URL: ${{ vars.CONTRACT_SERVICE_URL }} + + +permissions: + id-token: write + contents: read jobs: deploy-ccr-covid-aci: @@ -55,11 +61,14 @@ jobs: - name: Log in with Azure uses: azure/login@v1 with: - creds: '${{ secrets.AZURE_CREDENTIALS }}' + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: + registry: ${{ vars.CONTAINER_REGISTRY }} username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} @@ -78,6 +87,12 @@ jobs: - name: Pull container images for generating policy run: cd ${{ github.workspace }}/ci && ./pull-containers.sh + - name: create storage and containers + run: cd ${{ github.workspace }}/scenarios/covid/data && ./1-create-storage-containers.sh + + - name: create azure key vault + run: cd ${{ github.workspace }}/scenarios/covid/data && ./2-create-akv.sh + - name: Import data and model encryption keys with key release policies run: cd ${{ github.workspace }}/scenarios/covid/data && ./3-import-keys.sh @@ -88,7 +103,7 @@ jobs: run: cd ${{ github.workspace }}/scenarios/covid/data && ./5-upload-encrypted-data.sh - name: Run training - run: cd ${{ github.workspace }}/scenarios/covid/deployment/aci && ./deploy.sh -c ${{ github.event.inputs.contract }} -q ../../config/query_config.json -m ../../config/model_config.json + run: cd ${{ github.workspace }}/scenarios/covid/deployment/aci && ./deploy.sh -c ${{ github.event.inputs.contract }} -p ../../config/pipeline_config.json - name: Dump training container logs run: sleep 200 && az container logs --name depa-training-covid --resource-group $AZURE_RESOURCE_GROUP --container-name depa-training @@ -99,6 +114,9 @@ jobs: - name: Download and decrypt model run: cd ${{ github.workspace }}/scenarios/covid/data && ./6-download-decrypt-model.sh + - name: Clean up resource group and all resources + run: az group delete --yes --name $AZURE_RESOURCE_GROUP + - name: Cleanup data directory run: sudo rm -rf $DATA_DIRECTORY if: ${{ always() }} diff --git a/scenarios/covid/data/1-create-storage-containers.sh b/scenarios/covid/data/1-create-storage-containers.sh index 101f380..e6e2a43 100755 --- a/scenarios/covid/data/1-create-storage-containers.sh +++ b/scenarios/covid/data/1-create-storage-containers.sh @@ -1,34 +1,66 @@ #!/bin/bash +# +echo "Checking if resource group $AZURE_RESOURCE_GROUP exists..." +RG_EXISTS=$(az group exists --name $AZURE_RESOURCE_GROUP) -az group create \ - --location westeurope \ - --name $AZURE_RESOURCE_GROUP - -az storage account create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --name $AZURE_STORAGE_ACCOUNT_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_ICMR_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_COWIN_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_INDEX_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_MODEL_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_OUTPUT_CONTAINER_NAME +if [ "$RG_EXISTS" == "false" ]; then + echo "Resource group $AZURE_RESOURCE_GROUP does not exist. Creating it now..." + # Create the resource group + az group create --name $AZURE_RESOURCE_GROUP --location $AZURE_LOCATION +else + echo "Resource group $AZURE_RESOURCE_GROUP already exists. Skipping creation." +fi + +#echo "Check if storage account $STORAGE_ACCOUNT_NAME exists..." +STORAGE_ACCOUNT_EXISTS=$(az storage account check-name --name $AZURE_STORAGE_ACCOUNT_NAME --query "nameAvailable" --output tsv) + +if [ "$STORAGE_ACCOUNT_EXISTS" == "true" ]; then + echo "Storage account $AZURE_STORAGE_ACCOUNT_NAME does not exist. Creating it now..." + az storage account create --resource-group $AZURE_RESOURCE_GROUP --name $AZURE_STORAGE_ACCOUNT_NAME +else + echo "Storage account $AZURE_STORAGE_ACCOUNT_NAME exists" +fi + +# Get the storage account key +ACCOUNT_KEY=$(az storage account keys list --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv) + + +# Check if the ICMR container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_ICMR_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_ICMR_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_ICMR_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the COWIN container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_COWIN_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_COWIN_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_COWIN_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the INDEX container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_INDEX_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_INDEX_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_INDEX_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the MODEL container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_MODEL_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_MODEL_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_MODEL_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the OUTPUT container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_OUTPUT_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_OUTPUT_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_OUTPUT_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi diff --git a/scenarios/covid/data/2-create-akv.sh b/scenarios/covid/data/2-create-akv.sh index 6cbcb46..27e9110 100755 --- a/scenarios/covid/data/2-create-akv.sh +++ b/scenarios/covid/data/2-create-akv.sh @@ -2,17 +2,24 @@ set -e -echo CREATING $AZURE_KEYVAULT_ENDPOINT in resouce group $AZURE_RESOURCE_GROUP -echo $AZURE_RESOURCE_GROUP - + echo CREATING $AZURE_KEYVAULT_ENDPOINT in resouce group $AZURE_RESOURCE_GROUP + if [[ "$AZURE_KEYVAULT_ENDPOINT" == *".vault.azure.net" ]]; then - # Create Azure key vault with RBAC authorization AZURE_AKV_RESOURCE_NAME=`echo $AZURE_KEYVAULT_ENDPOINT | awk '{split($0,a,"."); print a[1]}'` - az keyvault create --name $AZURE_AKV_RESOURCE_NAME --resource-group $AZURE_RESOURCE_GROUP --sku "Premium" --enable-rbac-authorization - # Assign RBAC roles to the resource owner so they can import keys - AKV_SCOPE=`az keyvault show --name $AZURE_AKV_RESOURCE_NAME --query id --output tsv` - az role assignment create --role "Key Vault Crypto Officer" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE - az role assignment create --role "Key Vault Crypto User" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + # Check if the Key Vault already exists + echo "Checking if Key Vault $KEY_VAULT_NAME exists..." + KEY_VAULT_EXISTS=$(az keyvault list --resource-group $AZURE_RESOURCE_GROUP --query "[?name=='$AZURE_AKV_RESOURCE_NAME'].name" --output tsv) + if [ -z "$KEY_VAULT_EXISTS" ]; then + echo "Key Vault $KEY_VAULT_NAME does not exist. Creating it now..." + # Create Azure key vault with RBAC authorization + az keyvault create --name $AZURE_AKV_RESOURCE_NAME --resource-group $AZURE_RESOURCE_GROUP --sku "Premium" --enable-rbac-authorization + # Assign RBAC roles to the resource owner so they can import keys + AKV_SCOPE=`az keyvault show --name $AZURE_AKV_RESOURCE_NAME --query id --output tsv` + az role assignment create --role "Key Vault Crypto Officer" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + az role assignment create --role "Key Vault Crypto User" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + else + echo "Key Vault $AZURE_KEYVAULT_ENDPOINT exists" + fi else echo "Automated creation of key vaults is supported only for vaults" -fi \ No newline at end of file +fi diff --git a/scenarios/covid/deployment/aci/deploy.sh b/scenarios/covid/deployment/aci/deploy.sh index a8eee35..cad9068 100755 --- a/scenarios/covid/deployment/aci/deploy.sh +++ b/scenarios/covid/deployment/aci/deploy.sh @@ -140,13 +140,20 @@ echo $TMP > /tmp/aci-parameters.json echo Deploying training clean room... -az group create \ - --location westeurope \ - --name $AZURE_RESOURCE_GROUP +echo "Checking if resource group $AZURE_RESOURCE_GROUP exists..." +RG_EXISTS=$(az group exists --name $AZURE_RESOURCE_GROUP) + +if [ "$RG_EXISTS" == "false" ]; then + echo "Resource group $AZURE_RESOURCE_GROUP does not exist. Creating it now..." + # Create the resource group + az group create --name $AZURE_RESOURCE_GROUP --location $AZURE_LOCATION +else + echo "Resource group $AZURE_RESOURCE_GROUP already exists. Skipping creation." +fi az deployment group create \ --resource-group $AZURE_RESOURCE_GROUP \ --template-file arm-template.json \ --parameters @/tmp/aci-parameters.json -echo Deployment complete. \ No newline at end of file +echo Deployment complete.