azure-pipelines.yml

trigger:
- main

variables:
  subscriptionId: '38ca6696-5c82-4571-b2af-bf3f256cf663'
  resourceGroupName: 'condeploymenttest09'  # note that it will be parsed to max 19 lowercase alphanumeric chars anyway
  location: 'Poland Central'  # changing this should be considered carefully (check supported functionalities and assigned compute quotas)
  azconnection: 'Azure subscription 1 ($(subscriptionId))'

stages:
  - stage: Continuous_Integration
    jobs:
    - job: Continuous_Integration
      pool:
        vmImage: ubuntu-latest
      steps:
      - task: UsePythonVersion@0
        inputs:
          versionSpec: '3.10'
          addToPath: true
    
      - script: |
          python -m pip install --upgrade pip
          pip install flake8 isort mypy black pylint
        displayName: 'Install Python code quality tools'
    
      - script: |
          flake8 . || echo "Code style issues"
        displayName: 'Check code style with flake8'
    
      - script: |
          isort --check-only . --diff || echo "Import order issues"
        displayName: 'Check import order with isort'
    
      - script: |
          mypy . --ignore-missing-imports || echo "Type checking issues"
        displayName: 'Perform type check with mypy'
    
      - script: |
          black --check . || echo "Code format issues"
        displayName: 'Check code format with black'
    
      - script: |
          pylint . || echo "Code quality issues"
        displayName: 'Check code quality with pylint'

      - script: |
          python -m pip install --upgrade pip
          pip install -r $(Build.SourcesDirectory)/function_api/function_app/requirements.txt --target="$(Build.SourcesDirectory)/function_api/function_app/.python_packages/lib/site-packages"
        displayName: 'Install dependencies for Function App'

      - script: |
          cd $(Build.SourcesDirectory)/function_api/function_app
          zip -r $(Build.ArtifactStagingDirectory)/functionapp.zip .
        displayName: 'Pack function for Function App'

      - script: |
          mkdir -p $(Build.ArtifactStagingDirectory)/services
          tar -czf $(Build.ArtifactStagingDirectory)/services/airbyte.tar.gz airbyte
          tar -czf $(Build.ArtifactStagingDirectory)/services/kafka.tar.gz kafka
          tar -czf $(Build.ArtifactStagingDirectory)/services/airflow.tar.gz airflow
          tar -czf $(Build.ArtifactStagingDirectory)/services/mlflow.tar.gz mlflow
        displayName: 'Pack folders'

      - publish: $(Build.ArtifactStagingDirectory)/functionapp.zip
        artifact: FunctionAppArtifact
        displayName: 'Publish Function App Artifact'

      - publish: $(Build.ArtifactStagingDirectory)/services
        artifact: CompressedFolders
        displayName: 'Publish packed folders Artifact'

  - stage: Continuous_Deployment
    jobs:
    - job: Continuous_Deployment
      pool:
        vmImage: ubuntu-latest 
      steps:
      - download: current
        artifact: FunctionAppArtifact
        displayName: 'Download Function App Artifact'        

      - download: current
        artifact: CompressedFolders
        displayName: 'Download packed folders Artifact'        

      - powershell: |
          $rgn = "$(resourceGroupName)".ToLower() -replace '[^a-z0-9]', ''
          $rgn = $rgn.Substring(0, [System.Math]::Min($rgn.Length, 19))
          Write-Host "##vso[task.setvariable variable=resourceGroupName]$rgn"
        displayName: 'Assign parsed value to resourceGroupName pipeline variable'

      - powershell: |
          $tsan = "$(resourceGroupName)" + "1qe3p"
          $tcn = "configs"
          Write-Host "##vso[task.setvariable variable=tempStorageAccountName]$tsan"
          Write-Host "##vso[task.setvariable variable=tempContainerName]$tcn"
        displayName: 'Assign parsed value to tempStorageAccountName and tempContainerName pipeline variables'

      - powershell: |
          $san = "$(resourceGroupName)" + "3de90"
          $cn = "kafkadata"
          Write-Host "##vso[task.setvariable variable=storageAccountName]$san"
          Write-Host "##vso[task.setvariable variable=containerName]$cn"
        displayName: 'Assign parsed value to storageAccountName and containerName pipeline variables'

      - powershell: |
          $fan = "$(resourceGroupName)" + "2xa4c"
          Write-Host "##vso[task.setvariable variable=functionAppName]$fan"
        displayName: 'Assign parsed value to functionAppName pipeline variable'

      - powershell: |
          $apim = "$(resourceGroupName)" + "a1l45"
          Write-Host "##vso[task.setvariable variable=apiManagementName]$apim"
        displayName: 'Assign parsed value to apiManagementName pipeline variable'

      - powershell: |
          $loc = "$(location)"
          $loc = $loc.ToLower() -replace ' ', ''
          Write-Host "##vso[task.setvariable variable=lowercaseLocation]$loc"
        displayName: 'Convert location to lowercase and remove spaces'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy temporary Azure Blob Storage'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/blob_storage/temp_blob_storage_template.json'
          overrideParameters: '-storageAccountName "$(tempStorageAccountName)" -containerName "$(tempContainerName)"'
          deploymentMode: 'Incremental'

      - task: AzureCLI@2
        displayName: 'Upload tar.gz files to temporary storage'  
        inputs:
          azureSubscription: $(azconnection)
          scriptType: 'bash'
          scriptLocation: 'inlineScript'
          inlineScript: |
            az storage blob upload --account-name $(tempStorageAccountName) --container-name $(tempContainerName) --file $(Pipeline.Workspace)/CompressedFolders/airbyte.tar.gz --name airbyte.tar.gz --overwrite true
            az storage blob upload --account-name $(tempStorageAccountName) --container-name $(tempContainerName) --file $(Pipeline.Workspace)/CompressedFolders/kafka.tar.gz --name kafka.tar.gz --overwrite true
            az storage blob upload --account-name $(tempStorageAccountName) --container-name $(tempContainerName) --file $(Pipeline.Workspace)/CompressedFolders/airflow.tar.gz --name airflow.tar.gz --overwrite true
            az storage blob upload --account-name $(tempStorageAccountName) --container-name $(tempContainerName) --file $(Pipeline.Workspace)/CompressedFolders/mlflow.tar.gz --name mlflow.tar.gz --overwrite true

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Azure ML Studio Workspace and dependencies'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/mlstudio/azureml_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/mlstudio/azureml_parameters.json'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Azure Blob Storage'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/blob_storage/blob_storage_template.json'
          overrideParameters: '-storageAccountName "$(storageAccountName)" -containerName "$(containerName)"'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Kafka with its infrastructure'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/kafka_infra/kafka_infra_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/kafka_infra/kafka_infra_parameters.json'
          overrideParameters: '-tempStorageAccountName "$(tempStorageAccountName)" -tempContainerName "$(tempContainerName)"'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Airbyte with its infrastructure'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airbyte_infra/airbyte_infra_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/airbyte_infra/airbyte_infra_parameters.json'
          overrideParameters: '-tempStorageAccountName "$(tempStorageAccountName)" -tempContainerName "$(tempContainerName)"'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy MLflow with its infrastructure'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/mlflow_infra/mlflow_infra_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/mlflow_infra/mlflow_infra_parameters.json'
          overrideParameters: '-tempStorageAccountName "$(tempStorageAccountName)" -tempContainerName "$(tempContainerName)"'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Airflow with its infrastructure'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_infra_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_infra_parameters.json'
          overrideParameters: '-tempStorageAccountName "$(tempStorageAccountName)" -tempContainerName "$(tempContainerName)"'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy VNet Peering between Airbyte and Kafka'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/vnet_peering_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/airbyte_kafka_vnet_peering_parameters.json'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy VNet Peering between Kafka and Airflow'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/vnet_peering_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/kafka_airflow_vnet_peering_parameters.json'
          deploymentMode: 'Incremental'
      
      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy VNet Peering between Airflow and MLflow'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/vnet_peering_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/vnet_peerings/airflow_mlflow_vnet_peering_parameters.json'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Private Endpoint from Airflow to Azure Blob Storage'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_storage_private_endpoint_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_storage_private_endpoint_parameters.json'
          overrideParameters: '-storageAccountName "$(storageAccountName)"'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Private Endpoint from Airflow to Azure ML Studio Workspace'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_mlstudio_private_endpoint_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_mlstudio_private_endpoint_parameters.json'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy private DNS zone to connect Airflow to Azure Blob Storage'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/blob_storage_private_link_template.json'
          overrideParameters: '-storageAccountName "$(storageAccountName)"'
          deploymentMode: 'Incremental'

      - task: AzurePowerShell@5
        displayName: 'Assign Azure ML Workspace ID to pipeline variable'
        inputs:
          azureSubscription: $(azconnection)
          ScriptType: 'InlineScript'
          Inline: |
            $workspace = Get-AzResource -ResourceGroupName "$(resourceGroupName)" -ResourceType "Microsoft.MachineLearningServices/workspaces" -Name "mlserving"
            $wid = $workspace.Properties.workspaceId
            Write-Host "##vso[task.setvariable variable=workspaceId]$wid"
          azurePowerShellVersion: 'Latest'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy private DNS zone to connect Airflow to Azure ML Studio Workspace'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/azureml_private_link_template.json'
          overrideParameters: '-mlworkspaceId "$(workspaceId)"'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Virtual Network for Function App'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/function_app/functionapp_vnet_template.json'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Azure Function App'   
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: '$(subscriptionId)'
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/function_app/function_app_template.json'
          overrideParameters: '-name "$(functionAppName)"'
          deploymentMode: 'Incremental'

      - task: AzureFunctionApp@2
        displayName: 'Deploy packed function to Azure Function App'  
        inputs:
          connectedServiceNameARM: $(azconnection)
          appType: 'functionAppLinux'
          appName: $(functionAppName)
          package: '$(Pipeline.Workspace)/FunctionAppArtifact/functionapp.zip'
          runtimeStack: 'PYTHON|3.10'
          deploymentMethod: 'auto'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy Private Endpoint from Function App to Azure ML Studio Workspace'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: $(resourceGroupName)
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/airflow_infra/airflow_mlstudio_private_endpoint_template.json'
          csmParametersFile: '$(Build.SourcesDirectory)/arm_templates/function_app/function_azureml_private_endpoint_parameters.json'
          deploymentMode: 'Incremental'

      - task: AzureResourceManagerTemplateDeployment@3
        displayName: 'Deploy private DNS zone to connect Function App to Azure ML Studio Workspace'
        inputs:
          deploymentScope: 'Resource Group'
          azureResourceManagerConnection: $(azconnection)
          subscriptionId: $(subscriptionId)
          action: 'Create Or Update Resource Group'
          resourceGroupName: '$(resourceGroupName)functionappdns'
          location: $(location)
          templateLocation: 'Linked artifact'
          csmFile: '$(Build.SourcesDirectory)/arm_templates/function_app/function_azureml_private_link_template.json'
          overrideParameters: '-mlworkspaceId "$(workspaceId)" -resourceGroup "$(resourceGroupName)"'
          deploymentMode: 'Incremental'

      - task: AzurePowerShell@5
        displayName: 'Roles assignment'
        inputs:
          azureSubscription: $(azconnection)
          ScriptType: 'FilePath'
          ScriptPath: 'scripts/roles_assignment.ps1'
          ScriptArguments: '-resourceGroupName "$(resourceGroupName)" -storageAccountName "$(storageAccountName)" -functionAppName "$(functionAppName)"'
          azurePowerShellVersion: 'LatestVersion'

      - script: chmod +x function_api/apimanagement.sh
        displayName: 'Make apimanagement.sh executable'

      - task: AzureCLI@2
        displayName: 'Deploy API Management service and API for Function App'
        inputs:
          azureSubscription: $(azconnection)
          scriptType: 'bash'
          scriptLocation: 'inlineScript'
          inlineScript: |
            cd function_api
            ./apimanagement.sh $(resourceGroupName) $(lowercaseLocation) $(apiManagementName) $(functionAppName)

      - task: AzurePowerShell@5
        displayName: 'Inbound policy assignment to function API'
        inputs:
          azureSubscription: $(azconnection)
          ScriptType: 'InlineScript'
          Inline: |
            $apimContext = New-AzApiManagementContext -ResourceGroupName "$(resourceGroupName)" -ServiceName "$(apiManagementName)"
            $policy = "<policies> " +
                      "<inbound> " +
                      "<set-query-parameter name='code' exists-action='override'> " +
                      "<value>{{FunctionKey}}</value> " +
                      "</set-query-parameter> " +
                      "<set-header name='Content-Type' exists-action='override'> " +
                      "<value>application/json</value> " +
                      "</set-header> " +
                      "</inbound> " +
                      "<backend> " +
                      "<base /> " +
                      "</backend> " +
                      "<outbound> " +
                      "<base /> " +
                      "</outbound> " +
                      "<on-error> " +
                      "<base /> " +
                      "</on-error> " +
                      "</policies>"
            Set-AzApiManagementPolicy -Context $apimContext -ApiId "function" -Policy $policy -OperationId "post-operation"
          azurePowerShellVersion: 'Latest'