Add support to cache on specific json keys (#51)

* Add support to cache on specific json keys * Tweaks * add nested access * bump versions * nit * Tweak readme * tweak * tweak * sort * phrasing
seek-oss · Apr 9, 2024 · f8e70fb · f8e70fb
1 parent 5560666
commit f8e70fb
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 19 deletions.
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,3 @@
+{
+  "singleQuote": true
+}
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ RUN echo 'my expensive build step'
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1
+      - seek-oss/docker-ecr-cache#v2.2.0
       - docker#v3.12.0
 ```
 
@@ -52,7 +52,7 @@ RUN npm install
 steps:
   - command: npm test
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           cache-on:
             - package.json # avoid cache hits on stale lockfiles
             - package-lock.json
@@ -67,7 +67,7 @@ The `cache-on` property also supports Bash globbing with `globstar`:
 steps:
   - command: npm test
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           cache-on:
             - '**/package.json' # monorepo with multiple manifest files
             - yarn.lock
@@ -76,6 +76,27 @@ steps:
             - /workdir/node_modules
 ```
 
+It also supports caching on specific JSON keys which can be specified following a `#` character using [jq syntax](https://jqlang.github.io/jq/manual/#object-identifier-index). This requires [jq](https://jqlang.github.io/jq/) to be installed on the build agent. This implementation works by matching on the first `.json#` substring.
+
+A given entry cannot contain both bash globbing and a jq path.
+
+```yaml
+steps:
+  - command: pnpm test
+    plugins:
+      - seek-oss/docker-ecr-cache#v2.2.0:
+          cache-on:
+            - .npmrc
+            - package.json#.dependencies
+            - package.json#.devDependencies
+            - package.json#.packageManager
+            - package.json#.pnpm.overrides
+            - pnpm-lock.yaml
+      - docker#v3.12.0:
+          volumes:
+            - /workdir/node_modules
+```
+
 ### Using another Dockerfile
 
 It's possible to specify the Dockerfile to use by:
@@ -84,7 +105,7 @@ It's possible to specify the Dockerfile to use by:
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           dockerfile: my-dockerfile
       - docker#v3.12.0
 ```
@@ -95,7 +116,7 @@ Alternatively, Dockerfile can be embedded inline:
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           dockerfile-inline: |
             FROM node:16-alpine
             WORKDIR /workdir
@@ -124,7 +145,7 @@ steps:
       --build-arg BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG
       --file Dockerfile.secondary
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1
+      - seek-oss/docker-ecr-cache#v2.2.0
 ```
 
 Your `Dockerfile.secondary` can then [dynamically use these args](https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact):
@@ -152,7 +173,7 @@ stage to run commands against:
 steps:
   - command: cargo test
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           target: build-deps
       - docker#v3.12.0
 ```
@@ -167,7 +188,7 @@ The `context` property can be used to specify a different path.
 steps:
   - command: cargo test
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           dockerfile: dockerfiles/test/Dockerfile
           context: '.'
       - docker#v3.12.0
@@ -196,7 +217,7 @@ steps:
     env:
       ARG_1: wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           build-args:
             - ARG_1
             - ARG_2=such
@@ -211,7 +232,7 @@ steps:
     env:
       ARG_1: wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           additional-build-args: '--ssh= default=\$SSH_AUTH_SOCK'
       - docker#v3.12.0
 ```
@@ -239,7 +260,7 @@ steps:
     env:
       SECRET: wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           secrets:
             - SECRET
       - docker#v3.12.0
@@ -255,7 +276,7 @@ steps:
     plugins:
       - seek-oss/private-npm#v1.2.0:
           env: SECRET
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           secrets:
             - id=npmrc,src=.npmrc
       - docker#v3.12.0
@@ -273,7 +294,7 @@ By default images are kept in ECR for up to 30 days. This can be changed by spec
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           max-age-days: 7
       - docker#v3.12.0
 ```
@@ -286,7 +307,7 @@ By default, image name and computed tag are exported to the Docker buildkite plu
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           export-env-variable: BUILDKITE_PLUGIN_MY_CUSTOM_PLUGIN_CACHE_IMAGE
       - my-custom-plugin#v1.0.0:
 ```
@@ -300,7 +321,7 @@ steps:
   - label: Build Cache
     command: ':'
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           skip-pull-from-cache: true
 ```
 
@@ -316,7 +337,7 @@ optionally use a custom repository name:
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           ecr-name: my-unique-repository-name
           ecr-tags:
             Key: Value
@@ -332,7 +353,7 @@ By default, the plugin uses the region specified in the `AWS_DEFAULT_REGION` env
 steps:
   - command: echo wow
     plugins:
-      - seek-oss/docker-ecr-cache#v2.1.1:
+      - seek-oss/docker-ecr-cache#v2.2.0:
           region: ap-southeast-2
       - docker#v3.12.0
 ```
@@ -374,7 +395,7 @@ Example:
 ```yaml
 - command: echo wow
   plugins:
-    - seek-oss/docker-ecr-cache#v2.1.1:
+    - seek-oss/docker-ecr-cache#v2.2.0:
         registry-provider: gcr
         gcp-project: foo-bar-123456
 ```

diff --git a/hooks/lib/stdlib.bash b/hooks/lib/stdlib.bash
@@ -100,7 +100,18 @@ compute_tag() {
     echoerr "${glob}"
     for file in ${glob}; do
       echoerr "+ ${file}"
-      sums+="$(sha1sum "${file}")"
+      if [[ "${file}" == *.json#* ]]; then
+        # Extract the file path and keys from the pattern
+        file_path="${file%%#*}"
+        keys=${file#*#}
+
+        # Read the JSON file and calculate sha1sum only for the specified keys
+        value=$(jq -r "${keys}" "${file_path}")
+        sums+="$(echo -n "${value}" | sha1sum)"
+      else
+        # Calculate sha1sum for the whole file
+        sums+="$(sha1sum "${file}")"
+      fi
     done
   done
 

diff --git a/tests/stdlib.bats b/tests/stdlib.bats
@@ -2,6 +2,7 @@
 
 # export UNAME_STUB_DEBUG=/dev/tty
 # export SHA1SUM_STUB_DEBUG=/dev/tty
+# export JQ_STUB_DEBUG=/dev/tty
 
 load "$BATS_PLUGIN_PATH/load.bash"
 load "$PWD/hooks/lib/stdlib.bash"
@@ -159,3 +160,36 @@ pre_command_hook="$PWD/hooks/pre-command"
   unstub uname
   unstub sha1sum
 }
+
+@test "Can compute image tag with cache-on on specific json keys" {
+  # this var leaks in via pre-command
+  target="my-multi-stage-container"
+  export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_CACHE_ON_1="test-package.json#.dependencies"
+
+  stub uname \
+    "-m : echo my-architecture" \
+    "-m : echo my-architecture"
+  stub jq \
+    "-r .dependencies test-package.json : echo '{\"test\":\"123\"}'"
+  stub sha1sum \
+    "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \
+    ": echo sha1sum(target: my-multi-stage-container)" \
+    ": echo sha1sum(uname: my-architecture)" \
+    ": echo sha1sum(jq: .dependencies)" \
+    ": echo sha1sum(hashes so far)"
+
+  run compute_tag "pretend-dockerfile"
+
+  assert_success
+  assert_line "--- Computing tag"
+  assert_line "DOCKERFILE"
+  assert_line "+ pretend-dockerfile:my-multi-stage-container"
+  assert_line "ARCHITECTURE"
+  assert_line "+ my-architecture"
+  assert_line "BUILD_ARGS"
+  assert_line "CACHE_ON"
+
+  unstub uname
+  unstub jq
+  unstub sha1sum
+}