diff --git a/.github/ops-files/use-cflinuxfs3.yml b/.github/ops-files/use-cflinuxfs3.yml new file mode 100644 index 00000000000..5e52e190372 --- /dev/null +++ b/.github/ops-files/use-cflinuxfs3.yml @@ -0,0 +1,105 @@ +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/diego/droplet_destinations/cflinuxfs3? + value: /home/vcap +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/diego/lifecycle_bundles/buildpack~1cflinuxfs3? + value: buildpack_app_lifecycle/buildpack_app_lifecycle.tgz +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/stacks/name=cflinuxfs4:before + value: + description: Cloud Foundry Linux-based filesystem (Ubuntu 18.04) + name: cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/0:before + value: + name: staticfile_buildpack + package: staticfile-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/0:after + value: + name: java_buildpack + package: java-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/1:after + value: + name: ruby_buildpack + package: ruby-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/2:after + value: + name: dotnet_core_buildpack + package: dotnet-core-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/3:after + value: + name: nodejs_buildpack + package: nodejs-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/4:after + value: + name: go_buildpack + package: go-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/5:after + value: + name: python_buildpack + package: python-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/6:after + value: + name: php_buildpack + package: php-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/7:after + value: + name: nginx_buildpack + package: nginx-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/8:after + value: + name: r_buildpack + package: r-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/install_buildpacks/9:after + value: + name: binary_buildpack + package: binary-buildpack-cflinuxfs3 +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/diego/droplet_destinations/cflinuxfs3? + value: /home/vcap +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/diego/lifecycle_bundles/buildpack~1cflinuxfs3? + value: buildpack_app_lifecycle/buildpack_app_lifecycle.tgz +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/diego/droplet_destinations/cflinuxfs3? + value: /home/vcap +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/diego/lifecycle_bundles/buildpack~1cflinuxfs3? + value: buildpack_app_lifecycle/buildpack_app_lifecycle.tgz +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cc_deployment_updater/properties/cc/diego/droplet_destinations/cflinuxfs3? + value: /home/vcap +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cc_deployment_updater/properties/cc/diego/lifecycle_bundles/buildpack~1cflinuxfs3? + value: buildpack_app_lifecycle/buildpack_app_lifecycle.tgz +- type: replace + path: /instance_groups/name=diego-cell/jobs/name=cflinuxfs3-rootfs-setup? + value: + name: cflinuxfs3-rootfs-setup + properties: + cflinuxfs3-rootfs: + trusted_certs: + - ((diego_instance_identity_ca.ca)) + - ((credhub_tls.ca)) + - ((uaa_ssl.ca)) + release: cflinuxfs3 +- type: replace + path: /instance_groups/name=diego-cell/jobs/name=rep/properties/diego/rep/preloaded_rootfses/0:before + value: cflinuxfs3:/var/vcap/packages/cflinuxfs3/rootfs.tar +- type: replace + path: /releases/name=cflinuxfs4:before + value: + name: cflinuxfs3 + sha1: 5463400cd5490e9d847326668d504a8833cf3e4e + url: https://bosh.io/d/github.com/cloudfoundry/cflinuxfs3-release?v=0.369.0 + version: 0.369.0 diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index fed1851a000..e8a80a5915c 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -22,6 +22,15 @@ on: name: required: true type: string + pool-name: + type: string + default: ${{ vars.SHEPHERD_POOL_NAME }} + pool-namespace: + type: string + default: 'tas-devex' + is-pr: + type: boolean + default: true jobs: run-integration-tests: @@ -32,7 +41,7 @@ jobs: container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest steps: - uses: LouisBrunner/checks-action@v2.0.0 - if: always() + if: always() && inputs.is-pr id: check with: token: ${{ secrets.GITHUB_TOKEN }} @@ -86,19 +95,21 @@ jobs: name: Claim Environment env: account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} - pool_name: ${{ vars.SHEPHERD_POOL_NAME }} + pool_name: ${{ inputs.pool-name }} + pool_namespace: ${{ inputs.pool-namespace }} run: | shepherd login service-account ${account_token} - lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace official --namespace tas-devex --json | jq -r .id) - # Give somtime for the lease to complete. Shepherd may take upto an 3 hours to create an env + echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace tas-devex --namespace ${pool_namespace}" + lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace tas-devex --namespace ${pool_namespace} --json | jq -r .id) + # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env # if the pool is empty. count=0 while [ $count -lt 360 ] ; do sleep 30 - status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) + status=$(shepherd get lease ${lease_id} --namespace ${pool_namespace} --json | jq -r .status) if [ $status == "LEASED" ] ; then - shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json + shepherd get lease ${lease_id} --namespace ${pool_namespace} --json | jq .output > metadata.json break elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then echo "There was an error obtaining the lease. Lease status is ${status}." @@ -112,7 +123,7 @@ jobs: env_name=$(jq -r .name metadata.json) cat metadata.json | jq -r '.name' echo "lease-id=$lease_id" >> "${GITHUB_OUTPUT}" - + - name: Install Tools run: | if [[ ${{ inputs.os }} =~ "windows" ]] @@ -156,6 +167,7 @@ jobs: # deploy bosh -d cf manifest > /tmp/manifest.yml bosh interpolate /tmp/manifest.yml \ + -o .github/ops-files/use-cflinuxfs3.yml \ -o cf-deployment/operations/add-persistent-isolation-segment-diego-cell.yml \ -o cli-ci/ci/infrastructure/operations/add-oidc-provider.yml \ -o cli-ci/ci/infrastructure/operations/add-uaa-client-credentials.yml \ @@ -170,57 +182,24 @@ jobs: - name: Deploy MIN CAPI with Isolation Segment and OIDC Provider if: ${{ inputs.capi-version != 'edge' }} run: | - # TODO: Make this actually deploy min capi - # Creates vars files - mkdir vars-files - echo "cs = ${{ secrets.CLIENT_SECRET }}" - cat << EOF > vars-files/vars.yml - client-secret: ${{ secrets.CLIENT_SECRET }} - EOF - - # Copy Ops files - mkdir ops-files - cp cf-deployment/operations/test/add-persistent-isolation-segment-diego-cell.yml ops-files/ - cp cli-ci/ci/infrastructure/operations/add-oidc-provider.yml ops-files/ - cp cli-ci/ci/infrastructure/operations/add-uaa-client-credentials.yml ops-files/ - cp cli-ci/ci/infrastructure/operations/diego-cell-instances.yml ops-files/ - cp cli-ci/ci/infrastructure/operations/use-latest-ruby-buildpack.yml ops-files/ - - # Deletes CF-D env_name=$(jq -r .name metadata.json) jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv eval "$(bbl print-env --metadata-file metadata.json)" - bosh -d cf delete-deployment -n - - # Deploy CF-D - mkdir toolsmiths-env - cp metadata.json toolsmiths-env/metadata - cat metadata.json | jq -r .name > toolsmiths-env/name - export VARS_FILES="vars.yml" - export MANIFEST_FILE="cf-deployment.yml" - export SYSTEM_DOMAIN="" - export REGENERATE_CREDENTIALS=false - export DEPLOY_WITH_UPTIME_MEASUREMENTS=false - export MEASURE_SYSLOG_AVAILABILITY=false - export TCP_DOMAIN="" - export AVAILABLE_PORT="" - export FAIL_ON_DOWNTIME=false - export APP_PUSHABILITY_THRESHOLD=0 - export HTTP_AVAILABILITY_THRESHOLD=0 - export RECENT_LOGS_THRESHOLD=0 - export STREAMING_LOGS_THRESHOLD=0 - export APP_SYSLOG_AVAILABILITY_THRESHOLD=0 - export USE_SINGLE_APP_INSTANCE=false - export BOSH_DEPLOY_ARGS="" - export BOSH_LITE=false - export BBL_JSON_CONFIG="" - export SKIP_STEMCELL_UPLOAD=false - export OPS_FILES="add-persistent-isolation-segment-diego-cell.yml \ - add-uaa-client-credentials.yml \ - diego-cell-instances.yml \ - add-oidc-provider.yml \ - use-latest-ruby-buildpack.yml" - ./cf-deployment-concourse-tasks/bosh-deploy/task + + # deploy + bosh -d cf manifest > /tmp/manifest.yml + bosh interpolate /tmp/manifest.yml \ + -o cf-deployment/operations/test/add-persistent-isolation-segment-diego-cell.yml \ + -o cli-ci/ci/infrastructure/operations/add-oidc-provider.yml \ + -o cli-ci/ci/infrastructure/operations/add-uaa-client-credentials.yml \ + -o cli-ci/ci/infrastructure/operations/diego-cell-instances.yml \ + -o cli-ci/ci/infrastructure/operations/use-latest-ruby-buildpack.yml \ + -v client-secret="${{ secrets.CLIENT_SECRET }}" \ + > ./director.yml + + bosh -d cf deploy director.yml -n + echo "Deployed CAPI version:" + bosh -d cf releases | grep capi - name: Set Up Go uses: actions/setup-go@v5 @@ -293,9 +272,9 @@ jobs: shepherd login service-account ${account_token} set -x shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex - + - uses: LouisBrunner/checks-action@v2.0.0 - if: always() + if: always() && inputs.is-pr with: token: ${{ secrets.GITHUB_TOKEN }} check_id: ${{ steps.check.outputs.check_id }} diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index b49c93963c7..baa7a4ad2a5 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -4,6 +4,17 @@ run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ githu on: workflow_dispatch: + inputs: + workflow: + description: Tests to run + required: true + type: choice + options: + - run-integration-tests-cf-env + - run-integration-tests-cf-env-with-client-creds + - run-integration-tests-cf-env-with-min-capi + # - run-integration-windows + # - run-integration-windows-client-credentials workflow_run: workflows: - "Tests" @@ -13,40 +24,44 @@ on: jobs: run-integration-tests-cf-env: name: Integration tests - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.workflow == 'run-integration-tests-cf-env') || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge run-with-client-creds: false os: ubuntu-latest name: Integration + is-pr: ${{ github.event_name != 'workflow_dispatch' }} secrets: inherit run-integration-tests-cf-env-with-client-creds: name: client creds - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.workflow == 'run-integration-tests-cf-env-with-client-creds') || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge run-with-client-creds: true os: ubuntu-latest name: Integration client creds + is-pr: ${{ github.event_name != 'workflow_dispatch' }} secrets: inherit - # run-integration-tests-cf-env-with-min-capi: - # name: MIN CAPI - # # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - # uses: ./.github/workflows/tests-integration-reusable.yml - # with: - # capi-version: min - # run-with-client-creds: false - # os: ubuntu-latest - # name: Integration MIN CAPI - # secrets: inherit + run-integration-tests-cf-env-with-min-capi: + name: MIN CAPI + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.workflow == 'run-integration-tests-cf-env-with-min-capi') || github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/tests-integration-reusable.yml + with: + capi-version: min + run-with-client-creds: false + os: ubuntu-latest + name: Integration MIN CAPI + pool-name: cfd_16_11_0 + is-pr: ${{ github.event_name != 'workflow_dispatch' }} + secrets: inherit #run-integration-windows: # name: Windows - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ (github.event_name == 'workflow_dispatch' && inputs.workflow == 'run-integration-windows') || github.event.workflow_run.conclusion == 'success' }} # uses: ./.github/workflows/tests-integration-reusable.yml # with: # capi-version: edge @@ -57,7 +72,7 @@ jobs: #run-integration-windows-client-credentials: # name: Windows with client credentials - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ (github.event_name == 'workflow_dispatch' && inputs.workflow == 'run-integration-windows-client-credentials') || github.event.workflow_run.conclusion == 'success' }} # uses: ./.github/workflows/tests-integration-reusable.yml # with: # capi-version: edge