From 7aaf4361ba9059cd88e6d7d26f8cf9433dfd8fae Mon Sep 17 00:00:00 2001 From: Okan Kocabalkanli Date: Wed, 4 Oct 2023 11:51:43 -0400 Subject: [PATCH] Added Benchmarking to Binary Build --- .github/workflows/test-build.yml | 245 ++--------------------- packaging/uniformBuild/uniformBuilder.go | 35 ++-- 2 files changed, 45 insertions(+), 235 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index b629a41fae..73920fec66 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -55,7 +55,7 @@ jobs: MakeBinary: name: 'MakeBinary' runs-on: ubuntu-latest - environment: Build-Instance +# environment: Build-Instance permissions: id-token: write contents: read @@ -110,229 +110,28 @@ jobs: with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} passphrase: ${{ secrets.PASSPHRASE }} -# -# - name: Build Binaries -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# run: make amazon-cloudwatch-agent-linux amazon-cloudwatch-agent-windows package-rpm package-deb package-win -# -# - name: Sign Build Files -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# run: for f in $(find build/bin/); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done - -# -# - name: Upload to s3 -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# # Copy the RPM to .../amazon_linux/... because BETA customers expect it there. -# run: | -# echo "BucketKey: ${{ secrets.S3_INTEGRATION_BUCKET}} ${{ inputs.BucketKey }}" -# aws s3 cp build/bin s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }} --recursive -# aws s3 cp build/bin/linux/amd64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm -# aws s3 cp build/bin/linux/arm64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/arm64/latest/amazon-cloudwatch-agent.rpm -# -# - name: Login ECR -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# id: login-ecr -# uses: aws-actions/amazon-ecr-login@v1 -# -# - name: Set up Docker Buildx -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# uses: docker/setup-buildx-action@v1 -# -# - name: Set up QEMU -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# uses: docker/setup-qemu-action@v1 -# -# # Build dir is ignored in our .dockerignore thus need to copy to another dir. -# - name: Copy Binary For Agent Image Build -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# run: cp -r build/bin/linux/* . -# -# - name: Build Cloudwatch Agent Image -# uses: docker/build-push-action@v4 -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# with: -# file: amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/localdeb/Dockerfile -# context: . -# push: true -# tags: | -# ${{ steps.login-ecr.outputs.registry }}/${{ inputs.ContainerRepositoryNameAndTag }} -# platforms: linux/amd64, linux/arm64 + BinaryBenchmarkTest: + name: BinaryBenchmarkTest + runs-on: ubuntu-latest + needs: [MakeBinary] + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + - name: Download RPM + run: | + aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/linux/amd64/amazon-cloudwatch-agent.deb . + - name: GetRPMSize + run: ls -lh + - name: Measure Installation Time + run : time sudo dpkg -i -E ./amazon-cloudwatch-agent.deb -# MakeMSIZip: -# name: 'MakeMSIZip' -# runs-on: ubuntu-latest -# needs: [MakeBinary] -# permissions: -# id-token: write -# contents: read -# steps: -# - uses: actions/checkout@v3 -# with: -# repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} -# -# - name: Set up Go 1.x -# uses: actions/setup-go@v4 -# with: -# go-version: ~1.19.6 -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} -# aws-region: us-west-2 -# -# - name: Cache win zip -# id: cached_win_zip -# uses: actions/cache@v3 -# with: -# key: "cached_win_zip_${{ github.sha }}" -# path: go.mod -# -# - name: Copy binary -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false -# run: | -# aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }} . --recursive -# - name: Unzip -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false -# run: | -# sudo apt install unzip -# unzip windows/amd64/amazon-cloudwatch-agent.zip -d windows-agent -# - name: Create msi dep folder and copy deps -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false -# run: | -# export version=$(cat CWAGENT_VERSION) -# echo cw agent version $version -# mkdir msi_dep -# cp -r msi/tools/. msi_dep/ -# cp -r windows-agent/amazon-cloudwatch-agent/. msi_dep/ -# go run msi/tools/msiversion/msiversionconverter.go $version msi_dep/amazon-cloudwatch-agent.wxs '' -# go run msi/tools/msiversion/msiversionconverter.go $version msi_dep/manifest.json __VERSION__ -# -# - name: Zip -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false -# run: | -# sudo apt install zip -# zip buildMSI.zip msi_dep/* -# -# - name: Upload zip -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false -# run: aws s3 cp buildMSI.zip s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/buildMSI.zip -# -# MakeMacPkg: -# name: 'MakeMacPkg' -# runs-on: macos-11 -# permissions: -# id-token: write -# contents: read -# steps: -# - uses: actions/checkout@v3 -# with: -# path: cwa -# fetch-depth: 0 -# -# - uses: actions/checkout@v3 -# with: -# repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} -# path: test -# -# - name: Set up Go 1.x -# uses: actions/setup-go@v4 -# with: -# go-version: ~1.19.6 -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} -# aws-region: us-west-2 -# -# - name: Cache binaries -# id: cached_binaries -# uses: actions/cache@v3 -# with: -# key: "cached-binaries-${{ runner.os }}-${{ inputs.BucketKey }}" -# path: go.mod -# -# - name: Cache pkg -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# uses: actions/cache@v3 -# with: -# path: | -# ~/Library/Caches/go-build -# ~/go/pkg/mod -# key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} -# -# - name: Build Binaries -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# working-directory: cwa -# run: make amazon-cloudwatch-agent-darwin package-darwin -# -# - name: Copy binary -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# working-directory: cwa -# run: | -# echo cw agent version $(cat CWAGENT_VERSION) -# cp -r build/bin/darwin/amd64/. /tmp/ -# cp -r build/bin/darwin/arm64/. /tmp/arm64/ -# cp build/bin/CWAGENT_VERSION /tmp/CWAGENT_VERSION -# -# - name: Create pkg dep folder and copy deps -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# working-directory: test -# run: | -# cp -r pkg/tools/. /tmp/ -# cp -r pkg/tools/. /tmp/arm64/ -# -# - name: Build And Upload PKG -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# working-directory: /tmp/ -# run : | -# chmod +x create_pkg.sh -# chmod +x arm64/create_pkg.sh -# ./create_pkg.sh ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} "nosha" amd64 -# cd arm64 -# ./create_pkg.sh ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} "nosha" arm64 -# -# BuildMSI: -# name: 'BuildMSI' -# runs-on: windows-latest -# needs: [MakeMSIZip] -# permissions: -# id-token: write -# contents: read -# steps: -# - uses: actions/checkout@v3 -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} -# aws-region: us-west-2 -# -# - name: Cache msi -# id: cached_msi -# uses: actions/cache@v3 -# with: -# key: "cached_msi_${{ github.sha }}" -# path: go.mod -# -# # Using the env variable returns "" for bucket name thus use the secret -# - name: Copy msi -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_msi.outputs.cache-hit == false -# run: aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/buildMSI.zip . -# -# - name: Create msi -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_msi.outputs.cache-hit == false -# run : | -# curl -OLS https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311.exe -# .\wix311.exe /install /quiet /norestart -# $wixToolsetBinPath = ";C:\Program Files (x86)\WiX Toolset v3.11\bin;" -# $env:PATH = $env:PATH + $wixToolsetBinPath -# Expand-Archive buildMSI.zip -Force -# cd buildMSI/msi_dep -# .\create_msi.ps1 "nosha" ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} -# -# #GH actions set up gpg only works on ubuntu as of this commit date + # #GH actions set up gpg only works on ubuntu as of this commit date GPGSignMacAndWindowsPackage: name: 'SignMacAndWindowsPackage' runs-on: ubuntu-latest diff --git a/packaging/uniformBuild/uniformBuilder.go b/packaging/uniformBuild/uniformBuilder.go index 92a56093b8..abefc5cf3d 100644 --- a/packaging/uniformBuild/uniformBuilder.go +++ b/packaging/uniformBuild/uniformBuilder.go @@ -81,7 +81,7 @@ func (rbm *RemoteBuildManager) BuildCWAAgent(gitUrl string, branch string, commi if err != nil { return err } - if isAlreadyBuilt := rbm.CheckS3(commitHash); isAlreadyBuilt { + if isAlreadyBuilt := rbm.fileExistsInS3(commitHash); isAlreadyBuilt { fmt.Println("\033Found cache skipping build") return nil } @@ -97,7 +97,7 @@ func (rbm *RemoteBuildManager) BuildCWAAgent(gitUrl string, branch string, commi // Windows func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string) error { - //rbm.CheckS3(fmt.) + //rbm.fileExistsInS3(fmt.) //@TODO add cache //@TODO add os check command := mergeCommands( @@ -113,8 +113,13 @@ func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string return rbm.RunCommand(command, instanceName, fmt.Sprintf("Making MSI zip file for %s", commitHash)) } func (rbm *RemoteBuildManager) BuildMSI(instanceName string, commitHash string) error { - //@TODO add cache - //@TODO add os check + if err := rbm.instanceManager.insertOSRequirement(instanceName, WINDOWS); err != nil { + return err + } + if isAlreadyBuilt := rbm.fileExistsInS3(fmt.Sprintf("%s/amazon-cloudwatch-agent.msi", commitHash)); isAlreadyBuilt { + fmt.Println("\033Found cache skipping build") + return nil + } command := mergeCommandsWin( CopyMsi(commitHash), "Expand-Archive buildMSI.zip -DestinationPat C:\\buildMSI -Force", @@ -127,8 +132,13 @@ func (rbm *RemoteBuildManager) BuildMSI(instanceName string, commitHash string) // / MACOS ------------ func (rbm *RemoteBuildManager) MakeMacPkg(instanceName string, commitHash string) error { - //@TODO add cache - //@TODO add os check + if err := rbm.instanceManager.insertOSRequirement(instanceName, MACOS); err != nil { + return err + } + if isAlreadyBuilt := rbm.fileExistsInS3(fmt.Sprintf("%s/amd64/amazon-cloudwatch-agent.pkg", commitHash)); isAlreadyBuilt { + fmt.Println("\033Found cache skipping build") + return nil + } command := mergeCommands( CloneGitRepo(MAIN_REPO, "main"), "cd ccwa", @@ -169,8 +179,8 @@ func initEnvCmd(os OS) string { } // CACHE COMMANDS -func (rbm *RemoteBuildManager) CheckS3(targetFile string) bool { - return false //DOESNT WORK FOR NOW forcing an already existing cache +func (rbm *RemoteBuildManager) fileExistsInS3(targetFile string) bool { + fmt.Printf("Checking for %s cache \n", targetFile) input := &s3.ListObjectsV2Input{ Bucket: aws.String(S3_INTEGRATION_BUCKET), Prefix: aws.String(targetFile), @@ -180,12 +190,13 @@ func (rbm *RemoteBuildManager) CheckS3(targetFile string) bool { if err.Error() == "NotFound: Not Found" { fmt.Printf("Object %s does not exist in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile) } else { - panic(err) + fmt.Println(err) } - } else { - fmt.Printf("Object %s exists in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile) + return false } - return false + fmt.Printf("Object %s exists in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile) + return true + } func main() {