From cadd0b682f7d74da043d5ea917d524d0edaa257f Mon Sep 17 00:00:00 2001 From: Okan Kocabalkanli Date: Mon, 2 Oct 2023 14:12:32 -0400 Subject: [PATCH] Integration Test --- .github/workflows/integration-test.yml | 2 +- .github/workflows/test-build.yml | 116 +++++++++--------- packaging/uniformBuild/aws_utils.go | 36 +++++- packaging/uniformBuild/commands.go | 33 +++-- packaging/uniformBuild/constant.go | 8 +- packaging/uniformBuild/scripts/actions.py | 29 ----- packaging/uniformBuild/uniformBuilder.go | 42 +++---- packaging/uniformBuild/uniformBuilder_test.go | 12 +- 8 files changed, 139 insertions(+), 139 deletions(-) delete mode 100644 packaging/uniformBuild/scripts/actions.py diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 782c315151..b964046d44 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -49,7 +49,7 @@ jobs: with: ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" BucketKey: "integration-test/binary/${{ github.sha }}" - PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + PackageBucketKey: "integration-test/binary/${{ github.sha }}" GenerateTestMatrix: name: 'GenerateTestMatrix' diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index aaf026b906..b629a41fae 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -7,9 +7,9 @@ env: CWA_GITHUB_REPO_NAME: "aws/amazon-cloudwatch-agent" on: - push: - branches: - - uniform-build-env +# push: +## branches: +### - uniform-build-env workflow_dispatch: inputs: ContainerRepositoryNameAndTag: @@ -93,7 +93,7 @@ jobs: if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false run: | cd packaging/uniformBuild - ./uniformBuild -r "https://github.com/${{env.CWA_GITHUB_REPO_NAME}}.git" -b "main" -c "${{github.sha}}" -a "${{secrets.AWS_TEST_ACCOUNT_ID}}" + ./uniformBuild -r "https://github.com/${{env.CWA_GITHUB_REPO_NAME}}.git" -b "main" -c "${{inputs.BucketKey}}" -a "${{secrets.AWS_TEST_ACCOUNT_ID}}" # - name: Cache go ^^^ make it not main # # Only skip for integration builds not release builds. # if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false @@ -103,13 +103,13 @@ jobs: # ~/go/pkg/mod # ~/.cache/go-build # key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} -# -# - name: Import GPG Key -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false -# uses: crazy-max/ghaction-import-gpg@v5 -# with: -# gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} -# passphrase: ${{ secrets.PASSPHRASE }} + + - name: Import GPG Key + if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PASSPHRASE }} # # - name: Build Binaries # if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false @@ -118,7 +118,7 @@ jobs: # - name: Sign Build Files # if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false # run: for f in $(find build/bin/); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done -# + # # - name: Upload to s3 # if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false @@ -333,49 +333,49 @@ jobs: # .\create_msi.ps1 "nosha" ${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }} # # #GH actions set up gpg only works on ubuntu as of this commit date -# GPGSignMacAndWindowsPackage: -# name: 'SignMacAndWindowsPackage' -# runs-on: ubuntu-latest -# needs: [BuildMSI, MakeMacPkg] -# permissions: -# id-token: write -# contents: read -# steps: -# - uses: actions/checkout@v3 -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} -# aws-region: us-west-2 -# -# - name: Cache sig -# id: cached_sig -# uses: actions/cache@v3 -# with: -# key: "cached_sig_${{ github.sha }}" -# path: go.mod -# -# - name: Download from s3 -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_sig.outputs.cache-hit == false -# run: | -# mkdir -p packages/amd64 -# mkdir packages/arm64 -# aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amazon-cloudwatch-agent.msi ./packages/amazon-cloudwatch-agent.msi -# aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amd64/amazon-cloudwatch-agent.pkg ./packages/amd64/amazon-cloudwatch-agent.pkg -# aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/arm64/amazon-cloudwatch-agent.pkg ./packages/arm64/amazon-cloudwatch-agent.pkg -# - name: Import GPG Key -# uses: crazy-max/ghaction-import-gpg@v5 -# with: -# gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} -# passphrase: ${{ secrets.PASSPHRASE }} -# -# - name: Sign Build Files -# run: for f in $(find packages/); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done -# -# - name: Upload to s3 -# if: contains(inputs.BucketKey, 'test') == false || steps.cached_sig.outputs.cache-hit == false -# run: | -# aws s3 cp packages/amazon-cloudwatch-agent.msi.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amazon-cloudwatch-agent.msi.sig -# aws s3 cp packages/amd64/amazon-cloudwatch-agent.pkg.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amd64/amazon-cloudwatch-agent.pkg.sig -# aws s3 cp packages/arm64/amazon-cloudwatch-agent.pkg.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/arm64/amazon-cloudwatch-agent.pkg.sig + GPGSignMacAndWindowsPackage: + name: 'SignMacAndWindowsPackage' + runs-on: ubuntu-latest + needs: [MakeBinary] + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + + - name: Cache sig + id: cached_sig + uses: actions/cache@v3 + with: + key: "cached_sig_${{ github.sha }}" + path: go.mod + + - name: Download from s3 + if: contains(inputs.BucketKey, 'test') == false || steps.cached_sig.outputs.cache-hit == false + run: | + mkdir -p packages/amd64 + mkdir packages/arm64 + aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amazon-cloudwatch-agent.msi ./packages/amazon-cloudwatch-agent.msi + aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amd64/amazon-cloudwatch-agent.pkg ./packages/amd64/amazon-cloudwatch-agent.pkg + aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/arm64/amazon-cloudwatch-agent.pkg ./packages/arm64/amazon-cloudwatch-agent.pkg + - name: Import GPG Key + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PASSPHRASE }} + + - name: Sign Build Files + run: for f in $(find packages/); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done + + - name: Upload to s3 + if: contains(inputs.BucketKey, 'test') == false || steps.cached_sig.outputs.cache-hit == false + run: | + aws s3 cp packages/amazon-cloudwatch-agent.msi.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amazon-cloudwatch-agent.msi.sig + aws s3 cp packages/amd64/amazon-cloudwatch-agent.pkg.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amd64/amazon-cloudwatch-agent.pkg.sig + aws s3 cp packages/arm64/amazon-cloudwatch-agent.pkg.sig s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/arm64/amazon-cloudwatch-agent.pkg.sig diff --git a/packaging/uniformBuild/aws_utils.go b/packaging/uniformBuild/aws_utils.go index 6a5957db62..e45d133953 100644 --- a/packaging/uniformBuild/aws_utils.go +++ b/packaging/uniformBuild/aws_utils.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( @@ -5,6 +8,12 @@ import ( "encoding/json" "errors" "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -12,11 +21,6 @@ import ( ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" "github.com/aws/smithy-go" "github.com/schollz/progressbar/v3" - "sort" - "strconv" - "strings" - "sync" - "time" ) var ( @@ -150,7 +154,7 @@ func (imng *InstanceManager) Close() error { wg.Add(1) go func(client *ec2.Client, instanceID string) { defer wg.Done() - err := StopInstanceCmd(client, instanceID) + err := TerminateInstanceCmd(client, instanceID) time.Sleep(10 * time.Second) if err != nil { return @@ -286,6 +290,26 @@ func StopInstanceCmd(client *ec2.Client, instanceID string) error { fmt.Println("Stopped instance with ID " + instanceID) return nil } + +func TerminateInstanceCmd(client *ec2.Client, instanceID string) error { + //@TODO:Change to terminate + + input := &ec2.TerminateInstancesInput{ + InstanceIds: []string{ + instanceID, + }, + DryRun: aws.Bool(false), + } + + _, err := client.TerminateInstances(context.TODO(), input) + if err != nil { + fmt.Println("Got an error terminating the instance") + return err + } + fmt.Println("Stopped instance with ID " + instanceID) + return nil +} + func enforceCommentLimit(s string) string { const commentCharLimit = 100 if len(s) > commentCharLimit { diff --git a/packaging/uniformBuild/commands.go b/packaging/uniformBuild/commands.go index 1f81ac9566..412863a820 100644 --- a/packaging/uniformBuild/commands.go +++ b/packaging/uniformBuild/commands.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( @@ -33,35 +36,33 @@ func MakeBuild() string { ) return command } -func UploadToS3(commitHash string) string { - BucketKey := commitHash +func UploadToS3(key string) string { command := mergeCommands( - fmt.Sprintf("echo 'BucketKey: %s %s'", + fmt.Sprintf("echo 'key: %s %s'", S3_INTEGRATION_BUCKET, - BucketKey, + key, ), "cd ccwa", fmt.Sprintf("aws s3 cp build/bin s3://%s/%s --recursive", S3_INTEGRATION_BUCKET, - BucketKey, + key, ), fmt.Sprintf("aws s3 cp build/bin/linux/amd64/amazon-cloudwatch-agent.rpm s3://%s/%s/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm", S3_INTEGRATION_BUCKET, - BucketKey, + key, ), fmt.Sprintf("aws s3 cp build/bin/linux/arm64/amazon-cloudwatch-agent.rpm s3://%s/%s/amazon_linux/arm64/latest/amazon-cloudwatch-agent.rpm", S3_INTEGRATION_BUCKET, - BucketKey, + key, ), ) return command } -func CopyBinary(commitHash string) string { - BucketKey := commitHash +func CopyBinary(key string) string { command := mergeCommands(fmt.Sprintf( "aws s3 cp s3://%s/%s . --recursive", S3_INTEGRATION_BUCKET, - BucketKey, + key, ), ) return command @@ -79,19 +80,18 @@ func MakeMSI() string { "go run msi/tools/msiversion/msiversionconverter.go $version msi_dep/manifest.json __VERSION__", ) } -func CopyMsi(commitHash string) string { +func CopyMsi(key string) string { return fmt.Sprintf( "aws s3 cp s3://%s/%s/buildMSI.zip .", S3_INTEGRATION_BUCKET, - commitHash, + key, ) } -func UploadMSI(commitHash string) string { - BucketKey := commitHash +func UploadMSI(key string) string { return fmt.Sprintf( "aws s3 cp buildMSI.zip s3://%s/%s/buildMSI.zip", S3_INTEGRATION_BUCKET, - BucketKey, + key, ) } @@ -126,9 +126,8 @@ func CreatePkgCopyDeps() string { "cd ..", ) } -func BuildAndUploadMac(commitHash string) string { +func BuildAndUploadMac(key string) string { bucket := S3_INTEGRATION_BUCKET - key := commitHash return mergeCommands( "cd /tmp/", "chmod +x create_pkg.sh", diff --git a/packaging/uniformBuild/constant.go b/packaging/uniformBuild/constant.go index 82ed18aca7..961b43f964 100644 --- a/packaging/uniformBuild/constant.go +++ b/packaging/uniformBuild/constant.go @@ -1,15 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( - "github.com/aws/aws-sdk-go-v2/service/ec2/types" "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2/types" ) type OS string const TEST_REPO = "https://github.com/aws/amazon-cloudwatch-agent-test" const MAIN_REPO = "https://github.com/aws/amazon-cloudwatch-agent.git" -const S3_INTEGRATION_BUCKET = "uniform-build-env-test" +const S3_INTEGRATION_BUCKET = "cloudwatch-agent-integration-bucket" const BUILD_ARN = "arn:aws:iam::506463145083:instance-profile/Uniform-Build-Env-Instance-Profile" const COMMAND_TRACKING_TIMEOUT = 20 * time.Minute const COMMAND_TRACKING_INTERVAL = 1 * time.Second diff --git a/packaging/uniformBuild/scripts/actions.py b/packaging/uniformBuild/scripts/actions.py deleted file mode 100644 index d2bb186231..0000000000 --- a/packaging/uniformBuild/scripts/actions.py +++ /dev/null @@ -1,29 +0,0 @@ -import requests - -OWNER ="aws" -REPO = "amazon-cloudwatch-agent" -GITHUB_API = f"https://api.github.com/repos/{OWNER}/{REPO}/actions" -id = "6202732096" -GITHUB_API_RUN = GITHUB_API + f"/runs/{id}" -GITHUB_API_JOB = GITHUB_API_RUN + "/jobs?per_page=100" -data = requests.get(GITHUB_API_RUN).json() -print(data) -print(data['status'],data['conclusion']) -if data.get('conclusion') =='failure': - jobs = [] - for page in range(3): - jobdata = requests.get(GITHUB_API_JOB+f"&page={page}").json() - jobs += jobdata.get('jobs') - print(len(jobs)) - failed = [] - job_ids = [] - for job in jobs: - id = job.get('id') - if id in job_ids: - continue - # print(job.get("name"),job.get("conclusion")) - if job.get("conclusion") == 'failure': - failed.append(job.get("name")) - # print(job.get("name"),"failed") - job_ids.append(id) - print(len(failed),"\n".join(failed)) \ No newline at end of file diff --git a/packaging/uniformBuild/uniformBuilder.go b/packaging/uniformBuild/uniformBuilder.go index e1e0af8fcc..92a56093b8 100644 --- a/packaging/uniformBuild/uniformBuilder.go +++ b/packaging/uniformBuild/uniformBuilder.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( @@ -5,12 +8,13 @@ import ( "errors" "flag" "fmt" + "strings" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/ssm" "golang.org/x/sync/errgroup" - "strings" ) // This is the main struct that is managing the build process @@ -26,6 +30,9 @@ var DEFAULT_INSTANCE_GUIDE = map[string]OS{ "MacPkgMaker": MACOS, "WindowsMSIBuilder": WINDOWS, } +var LINUX_TEST_INSTANCE_GUIDE = map[string]OS{ + "MainBuildEnv": LINUX, +} var MACOS_TEST_INSTANCE_GUIDE = map[string]OS{ "MacPkgMaker": MACOS, } @@ -91,6 +98,8 @@ func (rbm *RemoteBuildManager) BuildCWAAgent(gitUrl string, branch string, commi // Windows func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string) error { //rbm.CheckS3(fmt.) + //@TODO add cache + //@TODO add os check command := mergeCommands( CloneGitRepo(TEST_REPO, "main"), "cd ccwa", @@ -104,21 +113,22 @@ func (rbm *RemoteBuildManager) MakeMsiZip(instanceName string, commitHash string return rbm.RunCommand(command, instanceName, fmt.Sprintf("Making MSI zip file for %s", commitHash)) } func (rbm *RemoteBuildManager) BuildMSI(instanceName string, commitHash string) error { - //@TODO needs windows ami + //@TODO add cache + //@TODO add os check command := mergeCommandsWin( - //CopyMsi(commitHash), - //"Expand-Archive buildMSI.zip -Force", + CopyMsi(commitHash), + "Expand-Archive buildMSI.zip -DestinationPat C:\\buildMSI -Force", "cd C:\\buildMSI\\msi_dep", fmt.Sprintf(".\\create_msi.ps1 \"nosha\" %s/%s", S3_INTEGRATION_BUCKET, commitHash), ) - rbm.RunCommand(CopyMsi(commitHash), instanceName, "copy msi") - rbm.RunCommand("Expand-Archive buildMSI.zip -DestinationPat C:\\buildMSI -Force", instanceName, "unzip msi.zip") + return rbm.RunCommand(command, instanceName, fmt.Sprintf("Making MSI Build file for %s", commitHash)) } // / MACOS ------------ func (rbm *RemoteBuildManager) MakeMacPkg(instanceName string, commitHash string) error { - //@TODO needs mac ami + //@TODO add cache + //@TODO add os check command := mergeCommands( CloneGitRepo(MAIN_REPO, "main"), "cd ccwa", @@ -158,8 +168,6 @@ func initEnvCmd(os OS) string { } -// - // CACHE COMMANDS func (rbm *RemoteBuildManager) CheckS3(targetFile string) bool { return false //DOESNT WORK FOR NOW forcing an already existing cache @@ -225,19 +233,9 @@ func main() { }) if err := eg.Wait(); err != nil { fmt.Printf("Failed because: %s \n", err) + return } - //err = rbm.MakeMsiZip("WindowsMSIPacker", comment) - //if err != nil { - // panic(err) - //} - //time.Sleep(7 * time.Minute) - //err = rbm.MakeMacPkg("MacPkgMaker", comment) - //if err != nil { - // panic(err) - //} - //err = rbm.BuildMSI("WindowsMSIBuilder", comment) - //if err != nil { - // panic(err) - //} + fmt.Printf("\033[32mSuccesfully\033[0m built CWA from %s with %s branch, check \033[32m%s \033[0m bucket with \033[1;32m%s\033[0m hash", + repo, branch, S3_INTEGRATION_BUCKET, comment) } diff --git a/packaging/uniformBuild/uniformBuilder_test.go b/packaging/uniformBuild/uniformBuilder_test.go index c034db48f2..7ed34f5903 100644 --- a/packaging/uniformBuild/uniformBuilder_test.go +++ b/packaging/uniformBuild/uniformBuilder_test.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( @@ -5,13 +8,14 @@ import ( "context" "flag" "fmt" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/stretchr/testify/require" "os" "testing" "time" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/stretchr/testify/require" + "github.com/aws/aws-sdk-go-v2/config" ) @@ -49,7 +53,7 @@ func TestSupportedAmis(t *testing.T) { } func TestEc2Generation(t *testing.T) { - rbm := CreateRemoteBuildManager(DEFAULT_INSTANCE_GUIDE, accountID) + rbm := CreateRemoteBuildManager(LINUX_TEST_INSTANCE_GUIDE, accountID) fmt.Println(rbm.ssmClient) defer rbm.Close() }