Skip to content

Commit

Permalink
Added Benchmarking to Binary Build
Browse files Browse the repository at this point in the history
  • Loading branch information
@okankoAMZ
okankoAMZ committed Oct 5, 2023
1 parent cadd0b6 commit 7aaf436
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 235 deletions.
245 changes: 22 additions & 223 deletions .github/workflows/test-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
MakeBinary:
name: 'MakeBinary'
runs-on: ubuntu-latest
environment: Build-Instance
# environment: Build-Instance
permissions:
id-token: write
contents: read
Expand Down Expand Up @@ -110,229 +110,28 @@ jobs:
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
passphrase: ${{ secrets.PASSPHRASE }}
#
# - name: Build Binaries
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# run: make amazon-cloudwatch-agent-linux amazon-cloudwatch-agent-windows package-rpm package-deb package-win
#
# - name: Sign Build Files
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# run: for f in $(find build/bin/); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done

#
# - name: Upload to s3
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# # Copy the RPM to .../amazon_linux/... because BETA customers expect it there.
# run: |
# echo "BucketKey: ${{ secrets.S3_INTEGRATION_BUCKET}} ${{ inputs.BucketKey }}"
# aws s3 cp build/bin s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }} --recursive
# aws s3 cp build/bin/linux/amd64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
# aws s3 cp build/bin/linux/arm64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/arm64/latest/amazon-cloudwatch-agent.rpm
#
# - name: Login ECR
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# id: login-ecr
# uses: aws-actions/amazon-ecr-login@v1
#
# - name: Set up Docker Buildx
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# uses: docker/setup-buildx-action@v1
#
# - name: Set up QEMU
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# uses: docker/setup-qemu-action@v1
#
# # Build dir is ignored in our .dockerignore thus need to copy to another dir.
# - name: Copy Binary For Agent Image Build
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# run: cp -r build/bin/linux/* .
#
# - name: Build Cloudwatch Agent Image
# uses: docker/build-push-action@v4
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# with:
# file: amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/localdeb/Dockerfile
# context: .
# push: true
# tags: |
# ${{ steps.login-ecr.outputs.registry }}/${{ inputs.ContainerRepositoryNameAndTag }}
# platforms: linux/amd64, linux/arm64
BinaryBenchmarkTest:
name: BinaryBenchmarkTest
runs-on: ubuntu-latest
needs: [MakeBinary]
permissions:
id-token: write
contents: read
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
aws-region: us-west-2
- name: Download RPM
run: |
aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/linux/amd64/amazon-cloudwatch-agent.deb .
- name: GetRPMSize
run: ls -lh
- name: Measure Installation Time
run : time sudo dpkg -i -E ./amazon-cloudwatch-agent.deb

# MakeMSIZip:
# name: 'MakeMSIZip'
# runs-on: ubuntu-latest
# needs: [MakeBinary]
# permissions:
# id-token: write
# contents: read
# steps:
# - uses: actions/checkout@v3
# with:
# repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}}
#
# - name: Set up Go 1.x
# uses: actions/setup-go@v4
# with:
# go-version: ~1.19.6
#
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v2
# with:
# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
# aws-region: us-west-2
#
# - name: Cache win zip
# id: cached_win_zip
# uses: actions/cache@v3
# with:
# key: "cached_win_zip_${{ github.sha }}"
# path: go.mod
#
# - name: Copy binary
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false
# run: |
# aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }} . --recursive
# - name: Unzip
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false
# run: |
# sudo apt install unzip
# unzip windows/amd64/amazon-cloudwatch-agent.zip -d windows-agent
# - name: Create msi dep folder and copy deps
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false
# run: |
# export version=$(cat CWAGENT_VERSION)
# echo cw agent version $version
# mkdir msi_dep
# cp -r msi/tools/. msi_dep/
# cp -r windows-agent/amazon-cloudwatch-agent/. msi_dep/
# go run msi/tools/msiversion/msiversionconverter.go $version msi_dep/amazon-cloudwatch-agent.wxs '<version>'
# go run msi/tools/msiversion/msiversionconverter.go $version msi_dep/manifest.json __VERSION__
#
# - name: Zip
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false
# run: |
# sudo apt install zip
# zip buildMSI.zip msi_dep/*
#
# - name: Upload zip
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_win_zip.outputs.cache-hit == false
# run: aws s3 cp buildMSI.zip s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/buildMSI.zip
#
# MakeMacPkg:
# name: 'MakeMacPkg'
# runs-on: macos-11
# permissions:
# id-token: write
# contents: read
# steps:
# - uses: actions/checkout@v3
# with:
# path: cwa
# fetch-depth: 0
#
# - uses: actions/checkout@v3
# with:
# repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}}
# path: test
#
# - name: Set up Go 1.x
# uses: actions/setup-go@v4
# with:
# go-version: ~1.19.6
#
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v2
# with:
# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
# aws-region: us-west-2
#
# - name: Cache binaries
# id: cached_binaries
# uses: actions/cache@v3
# with:
# key: "cached-binaries-${{ runner.os }}-${{ inputs.BucketKey }}"
# path: go.mod
#
# - name: Cache pkg
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# uses: actions/cache@v3
# with:
# path: |
# ~/Library/Caches/go-build
# ~/go/pkg/mod
# key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
#
# - name: Build Binaries
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# working-directory: cwa
# run: make amazon-cloudwatch-agent-darwin package-darwin
#
# - name: Copy binary
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# working-directory: cwa
# run: |
# echo cw agent version $(cat CWAGENT_VERSION)
# cp -r build/bin/darwin/amd64/. /tmp/
# cp -r build/bin/darwin/arm64/. /tmp/arm64/
# cp build/bin/CWAGENT_VERSION /tmp/CWAGENT_VERSION
#
# - name: Create pkg dep folder and copy deps
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# working-directory: test
# run: |
# cp -r pkg/tools/. /tmp/
# cp -r pkg/tools/. /tmp/arm64/
#
# - name: Build And Upload PKG
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
# working-directory: /tmp/
# run : |
# chmod +x create_pkg.sh
# chmod +x arm64/create_pkg.sh
# ./create_pkg.sh ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} "nosha" amd64
# cd arm64
# ./create_pkg.sh ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} "nosha" arm64
#
# BuildMSI:
# name: 'BuildMSI'
# runs-on: windows-latest
# needs: [MakeMSIZip]
# permissions:
# id-token: write
# contents: read
# steps:
# - uses: actions/checkout@v3
#
# - name: Configure AWS Credentials
# uses: aws-actions/configure-aws-credentials@v2
# with:
# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
# aws-region: us-west-2
#
# - name: Cache msi
# id: cached_msi
# uses: actions/cache@v3
# with:
# key: "cached_msi_${{ github.sha }}"
# path: go.mod
#
# # Using the env variable returns "" for bucket name thus use the secret
# - name: Copy msi
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_msi.outputs.cache-hit == false
# run: aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/buildMSI.zip .
#
# - name: Create msi
# if: contains(inputs.BucketKey, 'test') == false || steps.cached_msi.outputs.cache-hit == false
# run : |
# curl -OLS https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311.exe
# .\wix311.exe /install /quiet /norestart
# $wixToolsetBinPath = ";C:\Program Files (x86)\WiX Toolset v3.11\bin;"
# $env:PATH = $env:PATH + $wixToolsetBinPath
# Expand-Archive buildMSI.zip -Force
# cd buildMSI/msi_dep
# .\create_msi.ps1 "nosha" ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}
#
# #GH actions set up gpg only works on ubuntu as of this commit date
# #GH actions set up gpg only works on ubuntu as of this commit date
GPGSignMacAndWindowsPackage:
name: 'SignMacAndWindowsPackage'
runs-on: ubuntu-latest
Expand Down
35 changes: 23 additions & 12 deletions packaging/uniformBuild/uniformBuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func (rbm *RemoteBuildManager) BuildCWAAgent(gitUrl string, branch string, commi
if err != nil {
return err
}
if isAlreadyBuilt := rbm.CheckS3(commitHash); isAlreadyBuilt {
if isAlreadyBuilt := rbm.fileExistsInS3(commitHash); isAlreadyBuilt {
fmt.Println("\033Found cache skipping build")
return nil
}
Expand All @@ -97,7 +97,7 @@ func (rbm *RemoteBuildManager) BuildCWAAgent(gitUrl string, branch string, commi

// Windows
func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string) error {
//rbm.CheckS3(fmt.)
//rbm.fileExistsInS3(fmt.)
//@TODO add cache
//@TODO add os check
command := mergeCommands(
Expand All @@ -113,8 +113,13 @@ func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string
return rbm.RunCommand(command, instanceName, fmt.Sprintf("Making MSI zip file for %s", commitHash))
}
func (rbm *RemoteBuildManager) BuildMSI(instanceName string, commitHash string) error {
//@TODO add cache
//@TODO add os check
if err := rbm.instanceManager.insertOSRequirement(instanceName, WINDOWS); err != nil {
return err
}
if isAlreadyBuilt := rbm.fileExistsInS3(fmt.Sprintf("%s/amazon-cloudwatch-agent.msi", commitHash)); isAlreadyBuilt {
fmt.Println("\033Found cache skipping build")
return nil
}
command := mergeCommandsWin(
CopyMsi(commitHash),
"Expand-Archive buildMSI.zip -DestinationPat C:\\buildMSI -Force",
Expand All @@ -127,8 +132,13 @@ func (rbm *RemoteBuildManager) BuildMSI(instanceName string, commitHash string)

// / MACOS ------------
func (rbm *RemoteBuildManager) MakeMacPkg(instanceName string, commitHash string) error {
//@TODO add cache
//@TODO add os check
if err := rbm.instanceManager.insertOSRequirement(instanceName, MACOS); err != nil {
return err
}
if isAlreadyBuilt := rbm.fileExistsInS3(fmt.Sprintf("%s/amd64/amazon-cloudwatch-agent.pkg", commitHash)); isAlreadyBuilt {
fmt.Println("\033Found cache skipping build")
return nil
}
command := mergeCommands(
CloneGitRepo(MAIN_REPO, "main"),
"cd ccwa",
Expand Down Expand Up @@ -169,8 +179,8 @@ func initEnvCmd(os OS) string {
}

// CACHE COMMANDS
func (rbm *RemoteBuildManager) CheckS3(targetFile string) bool {
return false //DOESNT WORK FOR NOW forcing an already existing cache
func (rbm *RemoteBuildManager) fileExistsInS3(targetFile string) bool {
fmt.Printf("Checking for %s cache \n", targetFile)
input := &s3.ListObjectsV2Input{
Bucket: aws.String(S3_INTEGRATION_BUCKET),
Prefix: aws.String(targetFile),
Expand All @@ -180,12 +190,13 @@ func (rbm *RemoteBuildManager) CheckS3(targetFile string) bool {
if err.Error() == "NotFound: Not Found" {
fmt.Printf("Object %s does not exist in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile)
} else {
panic(err)
fmt.Println(err)
}
} else {
fmt.Printf("Object %s exists in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile)
return false
}
return false
fmt.Printf("Object %s exists in bucket %s\n", S3_INTEGRATION_BUCKET, targetFile)
return true

}

func main() {
Expand Down

0 comments on commit 7aaf436

Please sign in to comment.