Bake with Tacoinfra Remote Signer using AWS KMS (#467)

* Pull flextesa image from oxheadalpha docker repo

* remove false comment

* indentation

* Reformat signer structure + add tacoinfra signer

* Pin utils Docker image

Often python docker images have changes pushed using the same tag. The digest is different and hence we have to rebuild our utils image locally every time this happens. We pin the digest to enforce the use of single python version. We should be updating the base image manually when we choose to do so.

* Working tezos-k8s remote signers

* Put service account name in correct spot

* Handle tacoinfra signer in config-generator

* Functioning signer

* check signer_url exists before creating url

* Check for secret key before checking for remote signer

* Simplify getting key type

* Rename tezos-k8s signers to octez signers

* Allow specifying env vars for tacoinfra signer container

* Separate definition of diff signer types

* Add comments

* Don't allow replicas of tacoinfra signer to be configurable

* Remove tacoinfra secret

* Reorder functions and rename signer account

* Fail if account is undefined

* Rename container from tezos-signer to octez-signer

* Remove secret volume

* pick accounts field only

* security context enhancements + use pvc for file ratchet

* Add create-keys-json.py script

* Use oxheadalpha docker hub tacoinfra image

* Remove temp env vars

* Remove extra #

* Update helm chart diff test

* Exit on error + cleanup octez-node.sh

* Put back correct python image

* Fix tests

* make image pull policy configurable on other containers

* update tacoinfra signer image

* update tests

* updates

* Add namespace to metadata

* WIP CI docker caching

* Debug chown

* WIP docker cache and temporarily turn of testing helm charts

* Temp turn off check-lint-and-format and publish mkchain job

* WIP

* Fix adding security context capabilities not working

The k8s docs here https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ say:

"""
Note: Linux capability constants have the form CAP_XXX. But when you list capabilities in your container manifest, you must omit the CAP_ portion of the constant. For example, to add CAP_SYS_TIME, include SYS_TIME in your list of capabilities.
"""

On EKS clusters we are running with version 1.22, writing "CAP_" works. It doesn't seem this rule was enforced However with my testing on 1.25 this breaks and we need to remove "CAP_".

* Regen tests

* Rename init container

* update test

.github/workflows/ci.yml

@@ -147,31 +147,66 @@ jobs:
       matrix: ${{fromJson(needs.list_containers_to_publish.outputs.matrix)}}
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: 'true'
-
-    - name: Login to registry
-      run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Docker meta
-      id: meta
-      uses: docker/metadata-action@v3
-      with:
-        images: ghcr.io/${{ github.repository_owner }}/tezos-k8s-${{ matrix.container }}
-        tags: |
-          type=ref,event=branch
-          type=ref,event=pr
-          type=match,pattern=([0-9]+\.[0-9]+\.[0-9]+),group=1
-
-    - name: Push ${{ matrix.container }} container to GHCR
-      uses: docker/build-push-action@v2
-      with:
-        push: true
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-        file: ${{ matrix.container }}/Dockerfile
-        context: ${{ matrix.container}}/.
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+          submodules: "true"
+
+      # We configure docker image caching for faster builds. See:
+      # https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+        with:
+          install: true
+
+      - name: Login to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-multi-buildx-${{ matrix.container }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx-${{ matrix.container }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/tezos-k8s-${{ matrix.container }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=match,pattern=([0-9]+\.[0-9]+\.[0-9]+),group=1
+
+      - name: Push ${{ matrix.container }} container to GHCR
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ${{ matrix.container }}/Dockerfile
+          context: ${{ matrix.container}}/.
+
+          # Cache settings
+          builder: ${{ steps.buildx.outputs.name }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          # Note the mode=max here
+          # More: https://github.com/moby/buildkit#--export-cache-options
+          # And: https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   lint_helm_charts:
     runs-on: ubuntu-latest

charts/tezos/scripts/octez-node.sh

@@ -1,6 +1,6 @@
-set -x
+#!/bin/sh
 
-set
+set -xe
 
 # ensure we can run octez-client commands without specifying client dir
 ln -s /var/tezos/client /home/tezos/.tezos-client

charts/tezos/scripts/tacoinfra/create-keys-json.py

@@ -0,0 +1,40 @@
+"""Get the public key hashes of the accounts provided via the signer's
+ConfigMap. Create json objects with the hashes as the keys and write them to
+keys.json. The signer will read the file to determine keys it is signing for."""
+
+import json
+import logging
+import sys
+from os import path
+
+from pytezos import Key
+
+config_path = "./signer-config"
+accounts_json_path = f"{config_path}/accounts.json"
+
+if not path.isfile(accounts_json_path):
+    logging.warning("accounts.json file not found. Exiting.")
+    sys.exit(0)
+
+keys = {}
+
+with open(accounts_json_path, "r") as accounts_file:
+    accounts = json.load(accounts_file)
+    for account in accounts:
+        key = Key.from_encoded_key(account["key"])
+        if key.is_secret:
+            raise ValueError(
+                f"'{account['account_name']}' account's key is not a public key."
+            )
+        keys[key.public_key_hash()] = {
+            "account_name": account["account_name"],
+            "public_key": account["key"],
+            "key_id": account["key_id"],
+        }
+
+logging.info(f"Writing keys to {config_path}/keys.json...")
+with open(f"{config_path}/keys.json", "w") as keys_file:
+    keys_json = json.dumps(keys, indent=2)
+    print(keys_json, file=keys_file)
+    logging.info(f"Wrote keys.")
+    logging.debug(f"Keys: {keys_json}")

charts/tezos/templates/_helpers.tpl

@@ -3,12 +3,12 @@
   Returns a string "true" or empty string which is falsey.
 */}}
 {{- define "tezos.doesZerotierConfigExist" -}}
-{{- $zerotier_config := .Values.zerotier_config | default dict }}
-{{- if and ($zerotier_config.zerotier_network)  ($zerotier_config.zerotier_token) }}
-{{- "true" }}
-{{- else }}
-{{- "" }}
-{{- end }}
+  {{- $zerotier_config := .Values.zerotier_config | default dict }}
+  {{- if and ($zerotier_config.zerotier_network)  ($zerotier_config.zerotier_token) }}
+    {{- "true" }}
+  {{- else }}
+    {{- "" }}
+  {{- end }}
 {{- end }}
 
 {{/*
@@ -19,20 +19,11 @@
   Returns a string "true" or empty string which is falsey.
 */}}
 {{- define "tezos.shouldWaitForDNSNode" -}}
-{{- if and (not .Values.is_invitation) (hasKey .Values.node_config_network "genesis")}}
-{{- "true" }}
-{{- else }}
-{{- "" }}
-{{- end }}
-{{- end }}
-
-{{- define "tezos.shouldDeploySignerStatefulset" -}}
-{{- $signers := .Values.signers | default dict }}
-{{- if and (not .Values.is_invitation) ($signers | len) }}
-{{- "true" }}
-{{- else }}
-{{- "" }}
-{{- end }}
+  {{- if and (not .Values.is_invitation) (hasKey .Values.node_config_network "genesis")}}
+    {{- "true" }}
+  {{- else }}
+    {{- "" }}
+  {{- end }}
 {{- end }}
 
 {{/*
@@ -41,12 +32,12 @@
   Returns a string "true" or empty string which is falsey.
 */}}
 {{- define "tezos.shouldActivateProtocol" -}}
-{{ $activation := .Values.activation | default dict }}
-{{- if and ($activation.protocol_hash)  ($activation.protocol_parameters) }}
-{{- "true" }}
-{{- else }}
-{{- "" }}
-{{- end }}
+  {{ $activation := .Values.activation | default dict }}
+  {{- if and ($activation.protocol_hash)  ($activation.protocol_parameters) }}
+    {{- "true" }}
+  {{- else }}
+    {{- "" }}
+  {{- end }}
 {{- end }}
 
 {{/*
@@ -92,7 +83,6 @@
   for its node class. All identities for all instances of the node
   class will be stored in it. Each instance will look up its identity
   values by its hostname, e.g. archive-node-0.
-  Returns a string "true" or empty string which is falsey.
 */}}
 {{- define "tezos.includeNodeIdentitySecret" }}
   {{- range $index, $config := $.node_vals.instances }}
@@ -149,3 +139,49 @@ metadata:
     {{- "" }}
   {{- end }}
 {{- end }}
+
+{{- /* Make sure only a single signer signs for an account */}}
+{{- define "tezos.checkDupeSignerAccounts" }}
+  {{- $accountNames := dict }}
+  {{- range $signer := concat list
+    (values (.Values.octezSigners | default dict ))
+    (values (.Values.tacoinfraSigners | default dict ))
+  }}
+
+    {{- range $account := $signer.accounts }}
+      {{- if hasKey $accountNames $account }}
+        {{- fail (printf "Account '%s' is specified by more than one remote signer" $account) }}
+      {{- else }}
+        {{- $_ := set $accountNames $account "" }}
+      {{- end }}
+    {{- end }}
+
+  {{- end }}
+{{- end }}
+
+{{- define "tezos.hasKeyPrefix" }}
+  {{- $keyPrefixes := list "edsk" "edpk" "spsk" "sppk" "p2sk" "p2pk" }}
+  {{- has (substr 0 4 .) $keyPrefixes | ternary "true" "" }}
+{{- end }}
+
+{{- define "tezos.hasKeyHashPrefix" }}
+  {{- $keyHashPrefixes := list "tz1" "tz2" "tz3" }}
+  {{- has (substr 0 3 .) $keyHashPrefixes | ternary "true" "" }}
+{{- end }}
+
+{{- define "tezos.hasSecretKeyPrefix" }}
+  {{- if not (include "tezos.hasKeyPrefix" .key) }}
+    {{- fail (printf "'%s' account's key is not a valid key." .account_name) }}
+  {{- end }}
+  {{- substr 2 4 .key | eq "sk" | ternary "true" "" }}
+{{- end }}
+
+{{- define "tezos.validateAccountKeyPrefix" }}
+  {{- if (not (or
+      (include "tezos.hasKeyPrefix" .key)
+      (include "tezos.hasKeyHashPrefix" .key)
+    )) }}
+    {{- fail (printf "'%s' account's key is not a valid key or key hash." .account_name) }}
+  {{- end }}
+  {{- "true" }}
+{{- end }}

charts/tezos/templates/configs.yaml

@@ -1,4 +1,8 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tezos-config
+  namespace: {{ .Release.Namespace }}
 data:
   CHAIN_NAME: "{{ .Values.node_config_network.chain_name }}"
   CHAIN_PARAMS: |
@@ -40,13 +44,23 @@ data:
 {{- end }}
 {{- $nodes_copy | mustToPrettyJson | indent 4 }}
 
-  SIGNERS: |
-{{ .Values.signers | mustToPrettyJson | indent 4 }}
-kind: ConfigMap
-metadata:
-  name: tezos-config
-  namespace: {{ .Release.Namespace }}
+  OCTEZ_SIGNERS: |
+{{- $octezSigners := dict }}
+{{- range $signerName, $signerConfig := .Values.octezSigners }}
+    {{- $_ := set $signerConfig "name" $signerName }}
+    {{- $podName := print $.Values.octez_signer_statefulset.name "-" (len $octezSigners) }}
+    {{- $_ := set $octezSigners $podName $signerConfig }}
+{{- end }}
+{{ $octezSigners | default dict | mustToPrettyJson | indent 4 }}
+  TACOINFRA_SIGNERS: |
+{{- $tacoinfraSigners := dict }}
+{{- range $signerName, $signerConfig := .Values.tacoinfraSigners }}
+    {{- $_ := set $tacoinfraSigners $signerName (pick $signerConfig "accounts") }}
+{{- end }}
+{{ $tacoinfraSigners | default dict | mustToPrettyJson | indent 4 }}
+
 ---
+
 {{- if (include "tezos.doesZerotierConfigExist" .) }}
 apiVersion: v1
 data:
@@ -60,6 +74,7 @@ metadata:
   namespace: {{ .Release.Namespace }}
 {{- end }}
 ---
+
 apiVersion: v1
 data:
   ACCOUNTS: |

charts/tezos/templates/signer.yaml → charts/tezos/templates/octez-signer.yaml

@@ -1,23 +1,39 @@
-{{- if (include "tezos.shouldDeploySignerStatefulset" .) }}
+{{- $octezSigners := .Values.octezSigners | default dict }}
+{{- if and (not .Values.is_invitation) (len $octezSigners) }}
+  {{- include "tezos.checkDupeSignerAccounts" $ }}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.octez_signer_statefulset.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterIP: None
+  ports:
+    - port: 6732
+      name: signer
+  selector:
+    app: {{ .Values.octez_signer_statefulset.name }}
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: {{ .Values.signer_statefulset.name }}
+  name: {{ .Values.octez_signer_statefulset.name }}
   namespace: {{ .Release.Namespace }}
 spec:
   podManagementPolicy: Parallel
-  replicas: {{ .Values.signers | len }}
-  serviceName: {{ .Values.signer_statefulset.name }}
+  replicas: {{ len $octezSigners }}
+  serviceName: {{ .Values.octez_signer_statefulset.name }}
   selector:
     matchLabels:
-      app: {{ .Values.signer_statefulset.name }}
+      app: {{ .Values.octez_signer_statefulset.name }}
   template:
     metadata:
       labels:
-        app: {{ .Values.signer_statefulset.name }}
+        app: {{ .Values.octez_signer_statefulset.name }}
     spec:
       containers:
-      - name: tezos-signer
+      - name: octez-signer
         image: "{{ .Values.images.octez }}"
         imagePullPolicy: IfNotPresent
         ports:
@@ -47,7 +63,7 @@ spec:
               fieldRef:
                 fieldPath: metadata.name
           - name: MY_POD_TYPE
-            value: {{ .Values.signer_statefulset.pod_type }}
+            value: {{ .Values.octez_signer_statefulset.pod_type }}
         volumeMounts:
           - mountPath: /var/tezos
             name: var-volume
@@ -62,17 +78,4 @@ spec:
           secret:
             secretName: tezos-secret
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Values.signer_statefulset.name }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  clusterIP: None
-  ports:
-    - port: 6732
-      name: signer
-  selector:
-    app: {{ .Values.signer_statefulset.name }}
----
 {{- end }}