Add option to skip pull from cache (#47)

* Skip pull on cache hit * add newlines * fix if * update readme * rename var * revert quote change * fix env var * set plugin tester version * Update README.md * add test * update return * Update README.md * Update ecr-registry-provider.bash * Update gcr-registry-provider.bash * Update README.md * fix tests * Update tests/ecr-registry-provider.bats Co-authored-by: Ryan Ling <ryan@outlook.com.au> * fix erroneously commited format change --------- Co-authored-by: Ryan Ling <ryan@outlook.com.au>
seek-oss · Sep 18, 2023 · 5ede176 · 5ede176
1 parent 72f5aea
commit 5ede176
Show file tree

Hide file tree

Showing 8 changed files with 104 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -291,6 +291,19 @@ steps:
       - my-custom-plugin#v1.0.0:
 ```
 
+### Skipping image pull from cache
+
+By default, this plugin will pull the image when a cache hit is found. In scenarios where you may be using a caching step to ensure that an image exists for future steps, this may not be required. You can use `skip-pull-from-cache` to allow the plugin to exit early without pulling the image.
+
+```yaml
+steps:
+  - label: Build Cache
+    command: ':'
+    plugins:
+      - seek-oss/docker-ecr-cache#v2.0.0:
+          skip-pull-from-cache: true
+```
+
 ### AWS ECR specific configuration
 
 #### Specifying an ECR repository name

diff --git a/hooks/lib/ecr-registry-provider.bash b/hooks/lib/ecr-registry-provider.bash
@@ -33,6 +33,21 @@ ecr_exists() {
     --query 'repositories[0].registryId'
 }
 
+image_exists() {
+  local repository_name="$(get_ecr_repository_name)"
+  local image_tag="${1}"
+  local image_meta="$(aws ecr list-images \
+    --repository-name "${repository_name}" \
+    --query "imageIds[?imageTag=='${image_tag}'].imageTag" \
+    --output text)"
+
+  if [ "$image_meta" == "$image_tag" ]; then
+    true
+  else
+    false
+  fi
+}
+
 get_ecr_arn() {
   local repository_name="${1}"
   aws ecr describe-repositories \

diff --git a/hooks/lib/gcr-registry-provider.bash b/hooks/lib/gcr-registry-provider.bash
@@ -18,3 +18,8 @@ get_registry_url() {
   fi
   echo "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_HOSTNAME}/${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_GCP_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_NAME:-"$(get_default_image_name)"}"
 }
+
+image_exists() {
+  # TODO - implement check for cache in GCR
+  false
+}
diff --git a/hooks/lib/stub-registry-provider.bash b/hooks/lib/stub-registry-provider.bash
@@ -15,3 +15,7 @@ get_registry_url() {
 compute_tag() {
   echo "stubbed-computed-tag"
 }
+
+image_exists() {
+  echo "stubbed image_exists"
+}
diff --git a/hooks/pre-command b/hooks/pre-command
@@ -36,7 +36,13 @@ read_build_args
 secrets_args=()
 read_secrets
 
+if [ "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE:-}" == "true" ] && image_exists "$tag"; then
+  echo "Image exists, skipping pull"
+  exit 0;
+fi
+
 echo "--- Pulling image"
+
 if ! docker pull "${image}:${tag}"; then
   echo '--- Building image'
   image_build_args=(

diff --git a/plugin.yml b/plugin.yml
@@ -33,4 +33,6 @@ configuration:
       type: string
     region:
       type: string
+    skip-pull-from-cache:
+      type: boolean
   required: []
diff --git a/tests/ecr-registry-provider.bats b/tests/ecr-registry-provider.bats
@@ -69,14 +69,14 @@ pre_command_hook="$PWD/hooks/pre-command"
     "pull 1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \
     "build --file=Dockerfile --tag=1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \
     "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \
-    "push ${repository_uri}:deadbee : echo pushed deadbeef" \
+    "push ${repository_uri}:deadbee : echo pushed deadbee" \
     "push ${repository_uri}:latest : echo pushed latest"
 
   stub sha1sum \
     "Dockerfile : echo 'sha1sum(Dockerfile)'" \
     ": echo sha1sum" \
     ": echo sha1sum" \
-    ": echo deadbeef"
+    ": echo deadbee"
 
   run "${pre_command_hook}"
 
@@ -87,7 +87,7 @@ pre_command_hook="$PWD/hooks/pre-command"
   assert_output --partial "tag existing resource"
   assert_output --partial "put lifecycle policy"
   assert_output --partial "tagged latest"
-  assert_output --partial "pushed deadbeef"
+  assert_output --partial "pushed deadbee"
   assert_output --partial "pushed latest"
 
   unstub aws
@@ -115,14 +115,14 @@ pre_command_hook="$PWD/hooks/pre-command"
     "pull 1234567891012.dkr.ecr.eu-west-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \
     "build --file=Dockerfile --tag=1234567891012.dkr.ecr.eu-west-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \
     "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \
-    "push ${repository_uri}:deadbee : echo pushed deadbeef" \
+    "push ${repository_uri}:deadbee : echo pushed deadbee" \
     "push ${repository_uri}:latest : echo pushed latest"
 
   stub sha1sum \
     "Dockerfile : echo 'sha1sum(Dockerfile)'" \
     ": echo sha1sum" \
     ": echo sha1sum" \
-    ": echo deadbeef"
+    ": echo deadbee"
 
   run "${pre_command_hook}"
 
@@ -133,7 +133,7 @@ pre_command_hook="$PWD/hooks/pre-command"
   assert_output --partial "tag existing resource"
   assert_output --partial "put lifecycle policy"
   assert_output --partial "tagged latest"
-  assert_output --partial "pushed deadbeef"
+  assert_output --partial "pushed deadbee"
   assert_output --partial "pushed latest"
 
   unstub aws
@@ -163,14 +163,14 @@ pre_command_hook="$PWD/hooks/pre-command"
     "pull 1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \
     "build --file=Dockerfile --tag=1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \
     "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \
-    "push ${repository_uri}:deadbee : echo pushed deadbeef" \
+    "push ${repository_uri}:deadbee : echo pushed deadbee" \
     "push ${repository_uri}:latest : echo pushed latest"
 
   stub sha1sum \
     "Dockerfile : echo 'sha1sum(Dockerfile)'" \
     ": echo sha1sum" \
     ": echo sha1sum" \
-    ": echo deadbeef"
+    ": echo deadbee"
 
   run "${pre_command_hook}"
 
@@ -181,9 +181,48 @@ pre_command_hook="$PWD/hooks/pre-command"
   assert_output --partial "tag existing resource"
   assert_output --partial "put lifecycle policy"
   assert_output --partial "tagged latest"
-  assert_output --partial "pushed deadbeef"
+  assert_output --partial "pushed deadbee"
   assert_output --partial "pushed latest"
 
+  unstub aws
+  unstub docker
+  unstub sha1sum
+}
+
+@test "ECR: Calls list-images to check existence of cache" {
+  export AWS_DEFAULT_REGION="ap-southeast-2"
+  export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGION="ap-southeast-1"
+  export BUILDKITE_ORGANIZATION_SLUG="example-org"
+  export BUILDKITE_PIPELINE_SLUG="example-pipeline"
+  export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE="true"
+  local expected_repository_name="build-cache/example-org/example-pipeline"
+  local repository_uri="1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/${expected_repository_name}"
+
+  stub aws \
+    "sts get-caller-identity --query Account --output text : echo 1234567891012" \
+    "ecr get-login-password --region ap-southeast-1 : echo secure-ecr-password" \
+    "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \
+    "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:ap-southeast-1:1234567891012:repository/${expected_repository_name}" \
+    "ecr tag-resource --resource-arn arn:aws:ecr:ap-southeast-1:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \
+    "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \
+    "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo ${repository_uri}" \
+    "ecr list-images --repository-name ${expected_repository_name} --query imageIds[?imageTag==\'deadbee\'].imageTag --output text : echo 'deadbee'"
+
+  stub docker \
+    "login --username AWS --password-stdin 1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com : echo logging in to docker"
+
+  stub sha1sum \
+    "Dockerfile : echo 'sha1sum(Dockerfile)'" \
+    ": echo sha1sum" \
+    ": echo sha1sum" \
+    ": echo deadbee"
+
+  run "${pre_command_hook}"
+
+  assert_success
+  assert_output --partial "logging in to docker"
+  assert_output --partial "looked up repository"
+  assert_output --partial "Image exists, skipping pull"
   unstub aws
   unstub docker
   unstub sha1sum

diff --git a/tests/pre-command.bats b/tests/pre-command.bats
@@ -97,3 +97,14 @@ pre_command_hook="$PWD/hooks/pre-command"
   unstub mktemp
   unstub docker
 }
+
+@test "Exits 0 if skip-pull-on-cache and image exists" {
+  export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub"
+  export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE="true"
+  local repository_uri="pretend.host/path/segment/image"
+
+  run "${pre_command_hook}"
+
+  assert_success
+  assert_line "Image exists, skipping pull"
+}