diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f4de3814..a4f348fac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,31 +147,66 @@ jobs: matrix: ${{fromJson(needs.list_containers_to_publish.outputs.matrix)}} steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: 'true' - - - name: Login to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Docker meta - id: meta - uses: docker/metadata-action@v3 - with: - images: ghcr.io/${{ github.repository_owner }}/tezos-k8s-${{ matrix.container }} - tags: | - type=ref,event=branch - type=ref,event=pr - type=match,pattern=([0-9]+\.[0-9]+\.[0-9]+),group=1 - - - name: Push ${{ matrix.container }} container to GHCR - uses: docker/build-push-action@v2 - with: - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - file: ${{ matrix.container }}/Dockerfile - context: ${{ matrix.container}}/. + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 1 + submodules: "true" + + # We configure docker image caching for faster builds. See: + # https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@master + with: + install: true + + - name: Login to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-multi-buildx-${{ matrix.container }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-multi-buildx-${{ matrix.container }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/${{ github.repository_owner }}/tezos-k8s-${{ matrix.container }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=match,pattern=([0-9]+\.[0-9]+\.[0-9]+),group=1 + + - name: Push ${{ matrix.container }} container to GHCR + uses: docker/build-push-action@v2 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + file: ${{ matrix.container }}/Dockerfile + context: ${{ matrix.container}}/. + + # Cache settings + builder: ${{ steps.buildx.outputs.name }} + cache-from: type=local,src=/tmp/.buildx-cache + # Note the mode=max here + # More: https://github.com/moby/buildkit#--export-cache-options + # And: https://github.com/docker/buildx#--cache-tonametypetypekeyvalue + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache lint_helm_charts: runs-on: ubuntu-latest diff --git a/charts/tezos/scripts/octez-node.sh b/charts/tezos/scripts/octez-node.sh index 8b9d7432d..a93502d9d 100644 --- a/charts/tezos/scripts/octez-node.sh +++ b/charts/tezos/scripts/octez-node.sh @@ -1,6 +1,6 @@ -set -x +#!/bin/sh -set +set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client diff --git a/charts/tezos/scripts/tacoinfra/create-keys-json.py b/charts/tezos/scripts/tacoinfra/create-keys-json.py new file mode 100644 index 000000000..7730fe86d --- /dev/null +++ b/charts/tezos/scripts/tacoinfra/create-keys-json.py @@ -0,0 +1,40 @@ +"""Get the public key hashes of the accounts provided via the signer's +ConfigMap. Create json objects with the hashes as the keys and write them to +keys.json. The signer will read the file to determine keys it is signing for.""" + +import json +import logging +import sys +from os import path + +from pytezos import Key + +config_path = "./signer-config" +accounts_json_path = f"{config_path}/accounts.json" + +if not path.isfile(accounts_json_path): + logging.warning("accounts.json file not found. Exiting.") + sys.exit(0) + +keys = {} + +with open(accounts_json_path, "r") as accounts_file: + accounts = json.load(accounts_file) + for account in accounts: + key = Key.from_encoded_key(account["key"]) + if key.is_secret: + raise ValueError( + f"'{account['account_name']}' account's key is not a public key." + ) + keys[key.public_key_hash()] = { + "account_name": account["account_name"], + "public_key": account["key"], + "key_id": account["key_id"], + } + +logging.info(f"Writing keys to {config_path}/keys.json...") +with open(f"{config_path}/keys.json", "w") as keys_file: + keys_json = json.dumps(keys, indent=2) + print(keys_json, file=keys_file) + logging.info(f"Wrote keys.") + logging.debug(f"Keys: {keys_json}") diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index 34cdf5143..57a08d659 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -3,12 +3,12 @@ Returns a string "true" or empty string which is falsey. */}} {{- define "tezos.doesZerotierConfigExist" -}} -{{- $zerotier_config := .Values.zerotier_config | default dict }} -{{- if and ($zerotier_config.zerotier_network) ($zerotier_config.zerotier_token) }} -{{- "true" }} -{{- else }} -{{- "" }} -{{- end }} + {{- $zerotier_config := .Values.zerotier_config | default dict }} + {{- if and ($zerotier_config.zerotier_network) ($zerotier_config.zerotier_token) }} + {{- "true" }} + {{- else }} + {{- "" }} + {{- end }} {{- end }} {{/* @@ -19,20 +19,11 @@ Returns a string "true" or empty string which is falsey. */}} {{- define "tezos.shouldWaitForDNSNode" -}} -{{- if and (not .Values.is_invitation) (hasKey .Values.node_config_network "genesis")}} -{{- "true" }} -{{- else }} -{{- "" }} -{{- end }} -{{- end }} - -{{- define "tezos.shouldDeploySignerStatefulset" -}} -{{- $signers := .Values.signers | default dict }} -{{- if and (not .Values.is_invitation) ($signers | len) }} -{{- "true" }} -{{- else }} -{{- "" }} -{{- end }} + {{- if and (not .Values.is_invitation) (hasKey .Values.node_config_network "genesis")}} + {{- "true" }} + {{- else }} + {{- "" }} + {{- end }} {{- end }} {{/* @@ -41,12 +32,12 @@ Returns a string "true" or empty string which is falsey. */}} {{- define "tezos.shouldActivateProtocol" -}} -{{ $activation := .Values.activation | default dict }} -{{- if and ($activation.protocol_hash) ($activation.protocol_parameters) }} -{{- "true" }} -{{- else }} -{{- "" }} -{{- end }} + {{ $activation := .Values.activation | default dict }} + {{- if and ($activation.protocol_hash) ($activation.protocol_parameters) }} + {{- "true" }} + {{- else }} + {{- "" }} + {{- end }} {{- end }} {{/* @@ -92,7 +83,6 @@ for its node class. All identities for all instances of the node class will be stored in it. Each instance will look up its identity values by its hostname, e.g. archive-node-0. - Returns a string "true" or empty string which is falsey. */}} {{- define "tezos.includeNodeIdentitySecret" }} {{- range $index, $config := $.node_vals.instances }} @@ -149,3 +139,49 @@ metadata: {{- "" }} {{- end }} {{- end }} + +{{- /* Make sure only a single signer signs for an account */}} +{{- define "tezos.checkDupeSignerAccounts" }} + {{- $accountNames := dict }} + {{- range $signer := concat list + (values (.Values.octezSigners | default dict )) + (values (.Values.tacoinfraSigners | default dict )) + }} + + {{- range $account := $signer.accounts }} + {{- if hasKey $accountNames $account }} + {{- fail (printf "Account '%s' is specified by more than one remote signer" $account) }} + {{- else }} + {{- $_ := set $accountNames $account "" }} + {{- end }} + {{- end }} + + {{- end }} +{{- end }} + +{{- define "tezos.hasKeyPrefix" }} + {{- $keyPrefixes := list "edsk" "edpk" "spsk" "sppk" "p2sk" "p2pk" }} + {{- has (substr 0 4 .) $keyPrefixes | ternary "true" "" }} +{{- end }} + +{{- define "tezos.hasKeyHashPrefix" }} + {{- $keyHashPrefixes := list "tz1" "tz2" "tz3" }} + {{- has (substr 0 3 .) $keyHashPrefixes | ternary "true" "" }} +{{- end }} + +{{- define "tezos.hasSecretKeyPrefix" }} + {{- if not (include "tezos.hasKeyPrefix" .key) }} + {{- fail (printf "'%s' account's key is not a valid key." .account_name) }} + {{- end }} + {{- substr 2 4 .key | eq "sk" | ternary "true" "" }} +{{- end }} + +{{- define "tezos.validateAccountKeyPrefix" }} + {{- if (not (or + (include "tezos.hasKeyPrefix" .key) + (include "tezos.hasKeyHashPrefix" .key) + )) }} + {{- fail (printf "'%s' account's key is not a valid key or key hash." .account_name) }} + {{- end }} + {{- "true" }} +{{- end }} diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index 4912edf3b..84ea560de 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -1,4 +1,8 @@ apiVersion: v1 +kind: ConfigMap +metadata: + name: tezos-config + namespace: {{ .Release.Namespace }} data: CHAIN_NAME: "{{ .Values.node_config_network.chain_name }}" CHAIN_PARAMS: | @@ -40,13 +44,23 @@ data: {{- end }} {{- $nodes_copy | mustToPrettyJson | indent 4 }} - SIGNERS: | -{{ .Values.signers | mustToPrettyJson | indent 4 }} -kind: ConfigMap -metadata: - name: tezos-config - namespace: {{ .Release.Namespace }} + OCTEZ_SIGNERS: | +{{- $octezSigners := dict }} +{{- range $signerName, $signerConfig := .Values.octezSigners }} + {{- $_ := set $signerConfig "name" $signerName }} + {{- $podName := print $.Values.octez_signer_statefulset.name "-" (len $octezSigners) }} + {{- $_ := set $octezSigners $podName $signerConfig }} +{{- end }} +{{ $octezSigners | default dict | mustToPrettyJson | indent 4 }} + TACOINFRA_SIGNERS: | +{{- $tacoinfraSigners := dict }} +{{- range $signerName, $signerConfig := .Values.tacoinfraSigners }} + {{- $_ := set $tacoinfraSigners $signerName (pick $signerConfig "accounts") }} +{{- end }} +{{ $tacoinfraSigners | default dict | mustToPrettyJson | indent 4 }} + --- + {{- if (include "tezos.doesZerotierConfigExist" .) }} apiVersion: v1 data: @@ -60,6 +74,7 @@ metadata: namespace: {{ .Release.Namespace }} {{- end }} --- + apiVersion: v1 data: ACCOUNTS: | diff --git a/charts/tezos/templates/signer.yaml b/charts/tezos/templates/octez-signer.yaml similarity index 71% rename from charts/tezos/templates/signer.yaml rename to charts/tezos/templates/octez-signer.yaml index 9c462e3fa..9a42c2a8b 100644 --- a/charts/tezos/templates/signer.yaml +++ b/charts/tezos/templates/octez-signer.yaml @@ -1,23 +1,39 @@ -{{- if (include "tezos.shouldDeploySignerStatefulset" .) }} +{{- $octezSigners := .Values.octezSigners | default dict }} +{{- if and (not .Values.is_invitation) (len $octezSigners) }} + {{- include "tezos.checkDupeSignerAccounts" $ }} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.octez_signer_statefulset.name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + ports: + - port: 6732 + name: signer + selector: + app: {{ .Values.octez_signer_statefulset.name }} +--- apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ .Values.signer_statefulset.name }} + name: {{ .Values.octez_signer_statefulset.name }} namespace: {{ .Release.Namespace }} spec: podManagementPolicy: Parallel - replicas: {{ .Values.signers | len }} - serviceName: {{ .Values.signer_statefulset.name }} + replicas: {{ len $octezSigners }} + serviceName: {{ .Values.octez_signer_statefulset.name }} selector: matchLabels: - app: {{ .Values.signer_statefulset.name }} + app: {{ .Values.octez_signer_statefulset.name }} template: metadata: labels: - app: {{ .Values.signer_statefulset.name }} + app: {{ .Values.octez_signer_statefulset.name }} spec: containers: - - name: tezos-signer + - name: octez-signer image: "{{ .Values.images.octez }}" imagePullPolicy: IfNotPresent ports: @@ -47,7 +63,7 @@ spec: fieldRef: fieldPath: metadata.name - name: MY_POD_TYPE - value: {{ .Values.signer_statefulset.pod_type }} + value: {{ .Values.octez_signer_statefulset.pod_type }} volumeMounts: - mountPath: /var/tezos name: var-volume @@ -62,17 +78,4 @@ spec: secret: secretName: tezos-secret --- -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.signer_statefulset.name }} - namespace: {{ .Release.Namespace }} -spec: - clusterIP: None - ports: - - port: 6732 - name: signer - selector: - app: {{ .Values.signer_statefulset.name }} ---- {{- end }} diff --git a/charts/tezos/templates/tacoinfra-remote-signer/_helpers.tpl b/charts/tezos/templates/tacoinfra-remote-signer/_helpers.tpl new file mode 100644 index 000000000..5c5087fcc --- /dev/null +++ b/charts/tezos/templates/tacoinfra-remote-signer/_helpers.tpl @@ -0,0 +1,23 @@ + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "tacoinfra-remote-signer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "tacoinfra-remote-signer.labels" -}} +helm.sh/chart: {{ include "tacoinfra-remote-signer.chart" . }} +{{ include "tacoinfra-remote-signer.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "tacoinfra-remote-signer.selectorLabels" -}} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/tezos/templates/tacoinfra-remote-signer/main.yaml b/charts/tezos/templates/tacoinfra-remote-signer/main.yaml new file mode 100644 index 000000000..5cf2c126c --- /dev/null +++ b/charts/tezos/templates/tacoinfra-remote-signer/main.yaml @@ -0,0 +1,222 @@ +{{- range $signerName, $signerConfig := .Values.tacoinfraSigners }} + {{- include "tezos.checkDupeSignerAccounts" $ }} + {{- $_ := set $ "signerName" $signerName }} + {{- $_ := set $ "signerConfig" $signerConfig }} + + {{- include "tacoinfra-remote-signer.serviceAccount" $ }} + +apiVersion: v1 +kind: Service +metadata: + name: {{ $signerName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "tacoinfra-remote-signer.labels" $ | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 5000 + selector: + appType: tacoinfra-remote-signer + signerName: {{ $signerName }} +--- + +apiVersion: v1 +kind: Secret +metadata: + name: {{ $signerName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "tacoinfra-remote-signer.labels" $ | nindent 4 }} +data: + accounts.json: | +{{- $accountKeys := list }} +{{- range $accountName := $signerConfig.accounts }} + {{- $accounts := default dict $.Values.accounts }} + {{- $account := get $accounts $accountName | default dict }} + {{- if not $account }} + {{- fail (printf "Account '%s' is undefined." $accountName) }} + {{- end }} + + {{- $_ := set $account "account_name" $accountName }} + + {{- if not (and ($account.key) ($account.key_id)) }} + {{- fail (printf "Account '%s' requires 'key' and 'key_id' values." $accountName) }} + {{- end }} + {{- if (include "tezos.hasSecretKeyPrefix" $account) }} + {{- fail (printf "'%s' account's key is not a public key." $accountName) }} + {{- end }} + + {{- $accountKeys = append $accountKeys (pick $account "account_name" "key" "key_id") }} +{{- end }} +{{- $accountKeys | toJson | b64enc | nindent 4 }} +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: file-ratchet + namespace: {{ $.Release.Namespace }} + labels: + {{- include "tacoinfra-remote-signer.labels" $ | nindent 4 }} +spec: + resources: + requests: + storage: 8Ki + accessModes: + - ReadWriteOnce +--- + + {{- $signerImage := default dict $signerConfig.image }} + {{- $signerImageName := default $.Values.images.tacoinfraRemoteSigner $signerImage.name }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $signerName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "tacoinfra-remote-signer.labels" $ | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + appType: tacoinfra-remote-signer + signerName: {{ $signerName }} + {{- include "tacoinfra-remote-signer.selectorLabels" $ | nindent 6 }} + template: + metadata: + {{- with $signerConfig.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + appType: tacoinfra-remote-signer + signerName: {{ $signerName }} + {{- include "tacoinfra-remote-signer.selectorLabels" $ | nindent 8 }} + spec: + {{- with $signerConfig.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ default "" $signerConfig.serviceAccountName }} + securityContext: + {{- with $signerConfig.podSecurityContext }} + {{- toYaml . | nindent 8 }} + {{- end }} + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + initContainers: + {{- /* Secret mount is always mounted as ro fs. + Copy it to empty dir to make it writeable. + */}} + - name: copy-accounts + image: {{ $signerImageName }} + imagePullPolicy: {{ default "IfNotPresent" $signerImage.pullPolicy }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + capabilities: + drop: + - ALL + add: + - CHOWN # chown + - FOWNER # chmod + - DAC_OVERRIDE # cp + command: ["/bin/sh", "-c"] + args: + - | + set -ex + cp /etc/signer-config/* /app/signer-config + chown -R 999:999 /app/signer-config /etc/file_ratchets + chmod 770 /app/signer-config + chmod 770 /etc/file_ratchets + volumeMounts: + - name: signer-secret + mountPath: /etc/signer-config + - name: signer-config + mountPath: /app/signer-config + - name: file-ratchet + mountPath: /etc/file_ratchets + - name: create-keys-json + image: {{ $signerImageName }} + imagePullPolicy: {{ default "IfNotPresent" $signerImage.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: ["python"] + args: + - "-c" + - | +{{ tpl ($.Files.Get "scripts/tacoinfra/create-keys-json.py") $ | indent 13 }} + volumeMounts: + - name: signer-config + mountPath: /app/signer-config + containers: + - name: remote-signer + image: {{ $signerImageName }} + imagePullPolicy: {{ default "IfNotPresent" $signerImage.pullPolicy }} + args: ["kms"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + {{- with $signerConfig.securityContext }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 5000 + volumeMounts: + - name: file-ratchet + mountPath: /etc/file_ratchets + - name: signer-config + mountPath: /app/signer-config + readOnly: true + env: + {{- range $name, $value := $signerConfig.env }} + - name: {{ $name }} + value: {{ $value }} + {{- end }} + + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + {{- with $signerConfig.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: signer-secret + secret: + secretName: {{ $signerName }} + - name: signer-config + emptyDir: {} + - name: file-ratchet + persistentVolumeClaim: + claimName: file-ratchet + {{- with $signerConfig.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $signerConfig.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $signerConfig.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + +--- +{{- end -}} diff --git a/charts/tezos/templates/tacoinfra-remote-signer/serviceaccount.yaml b/charts/tezos/templates/tacoinfra-remote-signer/serviceaccount.yaml new file mode 100644 index 000000000..8615a5388 --- /dev/null +++ b/charts/tezos/templates/tacoinfra-remote-signer/serviceaccount.yaml @@ -0,0 +1,23 @@ +{{- define "tacoinfra-remote-signer.serviceAccount" }} +{{- $serviceAccount := $.signerConfig.serviceAccount | default dict }} + +{{- if or $serviceAccount.create (not (hasKey $serviceAccount "create")) }} + +{{- /* Set the SA name on $.signerConfig for the deployment to access */ -}} +{{- $_ := set $.signerConfig "serviceAccountName" ($serviceAccount.name | default $.signerName) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $serviceAccount.name | default $.signerName }} + namespace: {{ $.Release.Namespace }} + {{- with $serviceAccount.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +--- +{{- end }} diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 827d4985e..4ce14acf5 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -9,6 +9,7 @@ is_invitation: false # Images not part of the tezos-k8s repo go here images: octez: tezos/tezos:v16.1 + tacoinfraRemoteSigner: ghcr.io/oxheadalpha/tacoinfra-remote-signer:0.1.0 # Images that are part of the tezos-k8s repo go here with 'dev' tag tezos_k8s_images: utils: ghcr.io/oxheadalpha/tezos-k8s-utils:master @@ -20,8 +21,8 @@ tzkt_indexer_statefulset: name: tzkt-indexer bcd_indexer_statefulset: name: bcd-indexer -signer_statefulset: - name: tezos-signer +octez_signer_statefulset: + name: octez-signer pod_type: signing chain_initiator_job: name: chain-initiator @@ -76,21 +77,32 @@ accounts: {} # vote: # liquidity_baking_toggle_vote: "on" # ``` -# -# A public key account can contain a url to a remote signer that signs with the -# corresponding secret key. You shouldn't need to set this if you're deploying -# a tezos-k8s chart's signer into the same namespace. See the `signers` values -# field below in the file to define remote signers. +# A public key account can contain a `signer_url` to a remote signer +# that signs with the corresponding secret key. You don't need to +# set this if you're deploying a tezos-k8s signer into the same +# namespace of its baker. See `octezSigners` and `tacoinfraSigners` +# fields in values.yaml to define remote signers. (You shouldn't add things +# to the URL path such as the public key hash. It will be added automatically.) # ``` -# baker2: +# accounts: +# externalSignerAccount: # key: edpk... -# is_bootstrap_baker_account: false +# is_bootstrap_baker_account: true # bootstrap_balance: "4000000000000" # signer_url: http://[POD-NAME].[SERVICE-NAME].[NAMESPACE]:6732 # ``` # -# NOTE - signer_url must be URL to external remote signer without anything extra -# in the path, such as the public key hash. +# An account being signed for by a Tacoinfra AWS KMS signer requires a +# `key_id` field. This should be a valid id of the AWS KMS key. +# The key's corresponding public key must be provided here as well. +# ``` +# accounts: +# tacoinfraSigner: +# key: sppk... +# key_id: "cloud-id-of-key" +# is_bootstrap_baker_account: true +# bootstrap_balance: "4000000000000" +# ``` # # When running bakers for a public net, you must provide your own secret keys. # For non public networks you can change the @@ -272,15 +284,35 @@ serviceMonitor: # # Define remote signers. Bakers automatically use signers in their namespace # that are configured to sign for the accounts they are baking for. +# By default no signer is configured. # -# By default no signer is configured: -signers: {} -# Here is an example of octez signer config. When set, the +# https://tezos.gitlab.io/user/key-management.html#signer +octezSigners: {} +# Example: # ``` -# signers: +# octezSigners: # tezos-signer-0: -# sign_for_accounts: -# - baker0 +# accounts: +# - baker0 +# ``` +# +# Deploys a signer using AWS KMS to sign operations. +# The `AWS_REGION` env var must be set. +# https://github.com/oxheadalpha/tacoinfra-remote-signer +tacoinfraSigners: {} +# Example: +# ``` +# tacoinfraSigners +# tacoinfra-signer: +# accounts: +# - tacoinfraSigner +# env: +# AWS_REGION: us-east-2 +# serviceAccount: +# create: true +# ## EKS example for setting the role-arn +# annotations: +# eks.amazonaws.com/role-arn: # ``` # End Signers diff --git a/test/charts/mainnet.expect.yaml b/test/charts/mainnet.expect.yaml index 34474f34b..17ff4fe00 100644 --- a/test/charts/mainnet.expect.yaml +++ b/test/charts/mainnet.expect.yaml @@ -1,6 +1,7 @@ --- # Source: tezos-chain/templates/configs.yaml --- + apiVersion: v1 data: ACCOUNTS: | @@ -12,6 +13,10 @@ metadata: --- # Source: tezos-chain/templates/configs.yaml apiVersion: v1 +kind: ConfigMap +metadata: + name: tezos-config + namespace: testing data: CHAIN_NAME: "mainnet" CHAIN_PARAMS: | @@ -62,12 +67,10 @@ data: } } - SIGNERS: | + OCTEZ_SIGNERS: | + {} + TACOINFRA_SIGNERS: | {} -kind: ConfigMap -metadata: - name: tezos-config - namespace: testing --- # Source: tezos-chain/templates/static.yaml apiVersion: v1 @@ -131,9 +134,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client diff --git a/test/charts/mainnet2.expect.yaml b/test/charts/mainnet2.expect.yaml index e17b7813a..2a405cd39 100644 --- a/test/charts/mainnet2.expect.yaml +++ b/test/charts/mainnet2.expect.yaml @@ -1,6 +1,7 @@ --- # Source: tezos-chain/templates/configs.yaml --- + apiVersion: v1 data: ACCOUNTS: | @@ -12,6 +13,10 @@ metadata: --- # Source: tezos-chain/templates/configs.yaml apiVersion: v1 +kind: ConfigMap +metadata: + name: tezos-config + namespace: testing data: CHAIN_NAME: "mainnet" CHAIN_PARAMS: | @@ -109,12 +114,10 @@ data: } } - SIGNERS: | + OCTEZ_SIGNERS: | + {} + TACOINFRA_SIGNERS: | {} -kind: ConfigMap -metadata: - name: tezos-config - namespace: testing --- # Source: tezos-chain/templates/static.yaml apiVersion: v1 @@ -198,9 +201,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client @@ -571,9 +574,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client diff --git a/test/charts/private-chain.expect.yaml b/test/charts/private-chain.expect.yaml index 4d9816196..4dd1214ae 100644 --- a/test/charts/private-chain.expect.yaml +++ b/test/charts/private-chain.expect.yaml @@ -1,17 +1,43 @@ --- +# Source: tezos-chain/templates/tacoinfra-remote-signer/main.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tacoinfra-signer + namespace: testing +--- # Source: tezos-chain/templates/configs.yaml --- + apiVersion: v1 data: ACCOUNTS: | - e30= + eyJ0YWNvaW5mcmFTaWduZXIiOnsiYWNjb3VudF9uYW1lIjoidGFjb2luZnJhU2lnbmVyIiwia2V5Ijoic3Bway4uLiIsImtleV9pZCI6ImFsaWFzLy4uLiJ9fQ== kind: Secret metadata: name: tezos-secret namespace: testing --- +# Source: tezos-chain/templates/tacoinfra-remote-signer/main.yaml +apiVersion: v1 +kind: Secret +metadata: + name: tacoinfra-signer + namespace: testing + labels: + helm.sh/chart: tezos-chain-0.0.0 + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm +data: + accounts.json: | + W3siYWNjb3VudF9uYW1lIjoidGFjb2luZnJhU2lnbmVyIiwia2V5Ijoic3Bway4uLiIsImtleV9pZCI6ImFsaWFzLy4uLiJ9XQ== +--- # Source: tezos-chain/templates/configs.yaml apiVersion: v1 +kind: ConfigMap +metadata: + name: tezos-config + namespace: testing data: CHAIN_NAME: "elric" CHAIN_PARAMS: | @@ -142,6 +168,7 @@ data: "is_bootstrap_node": true }, { + "bake_using_account": "tacoinfraSigner", "is_bootstrap_node": true }, {} @@ -170,32 +197,55 @@ data: } } - SIGNERS: | + OCTEZ_SIGNERS: | { - "tezos-signer-0": { - "sign_for_accounts": [ + "octez-signer-0": { + "accounts": [ "tezos-baking-node-0" + ], + "name": "tezos-signer-0" + } + } + TACOINFRA_SIGNERS: | + { + "tacoinfra-signer": { + "accounts": [ + "tacoinfraSigner" ] } } -kind: ConfigMap -metadata: - name: tezos-config - namespace: testing --- # Source: tezos-chain/templates/configs.yaml apiVersion: v1 data: + tacoinfraSigner-013-PtJakart-per-block-votes.json: "{\"liquidity_baking_toggle_vote\":\"pass\"}" kind: ConfigMap metadata: name: per-block-votes namespace: testing --- -# Source: tezos-chain/templates/signer.yaml +# Source: tezos-chain/templates/tacoinfra-remote-signer/main.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: file-ratchet + namespace: testing + labels: + helm.sh/chart: tezos-chain-0.0.0 + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm +spec: + resources: + requests: + storage: 8Ki + accessModes: + - ReadWriteOnce +--- +# Source: tezos-chain/templates/octez-signer.yaml apiVersion: v1 kind: Service metadata: - name: tezos-signer + name: octez-signer namespace: testing spec: clusterIP: None @@ -203,7 +253,7 @@ spec: - port: 6732 name: signer selector: - app: tezos-signer + app: octez-signer --- # Source: tezos-chain/templates/static.yaml apiVersion: v1 @@ -279,6 +329,179 @@ spec: selector: node_class: us --- +# Source: tezos-chain/templates/tacoinfra-remote-signer/main.yaml +apiVersion: v1 +kind: Service +metadata: + name: tacoinfra-signer + namespace: testing + labels: + helm.sh/chart: tezos-chain-0.0.0 + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: http + port: 5000 + selector: + appType: tacoinfra-remote-signer + signerName: tacoinfra-signer +--- +# Source: tezos-chain/templates/tacoinfra-remote-signer/main.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tacoinfra-signer + namespace: testing + labels: + helm.sh/chart: tezos-chain-0.0.0 + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + appType: tacoinfra-remote-signer + signerName: tacoinfra-signer + app.kubernetes.io/instance: release-name + template: + metadata: + labels: + appType: tacoinfra-remote-signer + signerName: tacoinfra-signer + app.kubernetes.io/instance: release-name + spec: + serviceAccountName: tacoinfra-signer + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + initContainers: + - name: copy-accounts + image: ghcr.io/oxheadalpha/tacoinfra-remote-signer:0.1.0 + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: false + runAsUser: 0 + capabilities: + drop: + - ALL + add: + - CHOWN # chown + - FOWNER # chmod + - DAC_OVERRIDE # cp + command: ["/bin/sh", "-c"] + args: + - | + set -ex + cp /etc/signer-config/* /app/signer-config + chown -R 999:999 /app/signer-config /etc/file_ratchets + chmod 770 /app/signer-config + chmod 770 /etc/file_ratchets + volumeMounts: + - name: signer-secret + mountPath: /etc/signer-config + - name: signer-config + mountPath: /app/signer-config + - name: file-ratchet + mountPath: /etc/file_ratchets + - name: create-keys-json + image: ghcr.io/oxheadalpha/tacoinfra-remote-signer:0.1.0 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: ["python"] + args: + - "-c" + - | + """Get the public key hashes of the accounts provided via the signer's + ConfigMap. Create json objects with the hashes as the keys and write them to + keys.json. The signer will read the file to determine keys it is signing for.""" + + import json + import logging + import sys + from os import path + + from pytezos import Key + + config_path = "./signer-config" + accounts_json_path = f"{config_path}/accounts.json" + + if not path.isfile(accounts_json_path): + logging.warning("accounts.json file not found. Exiting.") + sys.exit(0) + + keys = {} + + with open(accounts_json_path, "r") as accounts_file: + accounts = json.load(accounts_file) + for account in accounts: + key = Key.from_encoded_key(account["key"]) + if key.is_secret: + raise ValueError( + f"'{account['account_name']}' account's key is not a public key." + ) + keys[key.public_key_hash()] = { + "account_name": account["account_name"], + "public_key": account["key"], + "key_id": account["key_id"], + } + + logging.info(f"Writing keys to {config_path}/keys.json...") + with open(f"{config_path}/keys.json", "w") as keys_file: + keys_json = json.dumps(keys, indent=2) + print(keys_json, file=keys_file) + logging.info(f"Wrote keys.") + logging.debug(f"Keys: {keys_json}") + + volumeMounts: + - name: signer-config + mountPath: /app/signer-config + containers: + - name: remote-signer + image: ghcr.io/oxheadalpha/tacoinfra-remote-signer:0.1.0 + imagePullPolicy: IfNotPresent + args: ["kms"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + ports: + - name: http + containerPort: 5000 + volumeMounts: + - name: file-ratchet + mountPath: /etc/file_ratchets + - name: signer-config + mountPath: /app/signer-config + readOnly: true + env: + + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + volumes: + - name: signer-secret + secret: + secretName: tacoinfra-signer + - name: signer-config + emptyDir: {} + - name: file-ratchet + persistentVolumeClaim: + claimName: file-ratchet +--- # Source: tezos-chain/templates/nodes.yaml apiVersion: apps/v1 kind: StatefulSet @@ -307,9 +530,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client @@ -511,9 +734,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client @@ -1083,9 +1306,9 @@ spec: args: - "-c" - | - set -x + #!/bin/sh - set + set -xe # ensure we can run octez-client commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-client @@ -1258,26 +1481,26 @@ spec: requests: storage: 15Gi --- -# Source: tezos-chain/templates/signer.yaml +# Source: tezos-chain/templates/octez-signer.yaml apiVersion: apps/v1 kind: StatefulSet metadata: - name: tezos-signer + name: octez-signer namespace: testing spec: podManagementPolicy: Parallel replicas: 1 - serviceName: tezos-signer + serviceName: octez-signer selector: matchLabels: - app: tezos-signer + app: octez-signer template: metadata: labels: - app: tezos-signer + app: octez-signer spec: containers: - - name: tezos-signer + - name: octez-signer image: "tezos/tezos:v15-release" imagePullPolicy: IfNotPresent ports: diff --git a/test/charts/private-chain.in.yaml b/test/charts/private-chain.in.yaml index db1f4812c..69fad7ed5 100644 --- a/test/charts/private-chain.in.yaml +++ b/test/charts/private-chain.in.yaml @@ -95,6 +95,7 @@ nodes: shell: {history_mode: archive} is_bootstrap_node: true - is_bootstrap_node: true + bake_using_account: tacoinfraSigner - {} runs: [octez_node, baker, logger, metrics] storage_size: 15Gi @@ -110,11 +111,22 @@ nodes: - {} rolling-node: null should_generate_unsafe_deterministic_data: true -signers: - tezos-signer-0: - sign_for_accounts: [tezos-baking-node-0] zerotier_config: {zerotier_network: null, zerotier_token: null} open_acls: true + +accounts: + tacoinfraSigner: + key: sppk... + key_id: alias/... +octezSigners: + tezos-signer-0: + accounts: + - tezos-baking-node-0 +tacoinfraSigners: + tacoinfra-signer: + accounts: + - tacoinfraSigner + protocols: - command: 013-PtJakart vote: diff --git a/utils/config-generator.py b/utils/config-generator.py index d74d7890a..35f1c832c 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -11,10 +11,11 @@ from pathlib import Path from re import sub from shutil import chown +from typing import Union - -from pytezos import pytezos +import requests from base58 import b58encode_check +from pytezos import Key with open("/etc/secret-volume/ACCOUNTS", "r") as secret_file: ACCOUNTS = json.loads(secret_file.read()) @@ -23,7 +24,8 @@ NODE_GLOBALS = json.loads(os.environ["NODE_GLOBALS"]) or {} NODES = json.loads(os.environ["NODES"]) NODE_IDENTITIES = json.loads(os.getenv("NODE_IDENTITIES", "{}")) -SIGNERS = json.loads(os.environ["SIGNERS"]) +OCTEZ_SIGNERS = json.loads(os.getenv("OCTEZ_SIGNERS", "{}")) +TACOINFRA_SIGNERS = json.loads(os.getenv("TACOINFRA_SIGNERS", "{}")) MY_POD_NAME = os.environ["MY_POD_NAME"] MY_POD_TYPE = os.environ["MY_POD_TYPE"] @@ -53,7 +55,7 @@ MY_POD_CLASS = NODES[my_node_class] if MY_POD_TYPE == "signing": - MY_POD_CONFIG = SIGNERS[MY_POD_NAME] + MY_POD_CONFIG = OCTEZ_SIGNERS[MY_POD_NAME] NETWORK_CONFIG = CHAIN_PARAMS["network"] @@ -251,8 +253,11 @@ def fill_in_missing_accounts(): return {**new_accounts, **ACCOUNTS} -# Verify that the current baker has a baker account with secret key def verify_this_bakers_account(accounts): + """ + Verify the current baker pod has an account with a secret key, unless the + account is signed for via an external remote signer (e.g. Tacoinfra). + """ accts = get_baking_accounts(MY_POD_CONFIG) if not accts or len(accts) < 1: @@ -261,15 +266,18 @@ def verify_this_bakers_account(accounts): for acct in accts: if not accounts.get(acct): raise Exception(f"ERROR: No account named {acct} found.") - signer = accounts[acct].get("signer_url") + signer_url = accounts[acct].get("signer_url") + tacoinfra_signer = get_accounts_signer(TACOINFRA_SIGNERS, acct) # We can count on accounts[acct]["type"] because import_keys will # fill it in when it is missing. - if not (accounts[acct]["type"] == "secret" or signer): + if not (accounts[acct]["type"] == "secret" or signer_url or tacoinfra_signer): raise Exception( - f"ERROR: Either a secret key or a signer_url should be provided for {acct}" + f"ERROR: Neither a secret key, signer url, or cloud remote signer is provided for baking account{acct}." ) + return True + # # import_keys() creates three files in /var/tezos/client which specify @@ -318,17 +326,16 @@ def fill_in_missing_keys(all_accounts): account_values["type"] = "secret" -# -# expose_secret_key() decides if an account needs to have its secret -# key exposed on the current pod. It returns the obvious Boolean. - - def expose_secret_key(account_name): + """ + Decides if an account needs to have its secret key exposed on the current + pod. It returns the obvious Boolean. + """ if MY_POD_TYPE == "activating": return NETWORK_CONFIG["activation_account_name"] == account_name if MY_POD_TYPE == "signing": - return account_name in MY_POD_CONFIG.get("sign_for_accounts") + return account_name in MY_POD_CONFIG.get("accounts") if MY_POD_TYPE == "node": if MY_POD_CONFIG.get("bake_using_account", "") == account_name: @@ -338,36 +345,72 @@ def expose_secret_key(account_name): return False -# -# pod_requires_secret_key() decides if a pod requires the secret key, -# regardless of a remote_signer being present. E.g. the remote signer -# needs to have the keys not a URL to itself. +def get_accounts_signer(signers, account_name): + """ + Determine if there is a signer for the account. Error if the account is + specified in more than one signer. + """ + found_signer = found_account = None + for signer in signers.items(): + signer_name, signer_config = signer + if account_name in signer_config["accounts"]: + if account_name == found_account: + raise Exception( + f"ERORR: Account '{account_name}' can't be specified in more than one signer." + ) + found_account = account_name + found_signer = {"name": signer_name, "config": signer_config} + return found_signer + + +def get_remote_signer_url(account: tuple[str, dict], key: Key) -> Union[str, None]: + """ + Return the url of a remote signer, if any, that claims to sign for the + account. Error if more than one signs for the account. + """ + account_name, account_values = account + signer_url = account_values.get("signer_url") + octez_signer = get_accounts_signer(OCTEZ_SIGNERS, account_name) + tacoinfra_signer = get_accounts_signer(TACOINFRA_SIGNERS, account_name) -def pod_requires_secret_key(account_values): - return ( - MY_POD_TYPE in ["activating", "signing"] and "signer_url" not in account_values - ) + signers = (signer_url, octez_signer, tacoinfra_signer) + if tuple(map(bool, (signers))).count(True) > 1: + raise Exception( + f"ERROR: Account '{account_name}' may only have a signer_url field or be signed for by a single signer." + ) + if octez_signer: + signer_url = f"http://{octez_signer['name']}.octez-signer:6732" -# -# remote_signer() returns a reference to a signer that -# tezos-client understands, either: -# * picks the first signer, if any, that claims to sign -# for account_name and returns a URL to locate it, -# * returns the external signer url if passed. + if tacoinfra_signer: + signer_url = f"http://{tacoinfra_signer['name']}:5000" + + return signer_url and f"{signer_url}/{key.public_key_hash()}" -def remote_signer(account_name, external_signer_url, key): - signer_url_no_path = None - if external_signer_url: - signer_url_no_path = external_signer_url - for k, v in SIGNERS.items(): - if account_name in v["sign_for_accounts"]: - signer_url_no_path = f"http://{k}.tezos-signer:6732" - if signer_url_no_path: - return f"{signer_url_no_path}/{key.public_key_hash()}" - return None +def get_secret_key(account, key: Key): + """ + For nodes and activation job, check if there is a remote signer for the + account. If found, use its url as the sk. If there is no signer and for all + other pod types (e.g. octez signer), use an actual sk. + """ + account_name, _ = account + + sk = (key.is_secret or None) and f"unencrypted:{key.secret_key()}" + if MY_POD_TYPE in ("node", "activating"): + signer_url = get_remote_signer_url(account, key) + octez_signer = get_accounts_signer(OCTEZ_SIGNERS, account_name) + if (sk and signer_url) and not octez_signer: + raise Exception( + f"ERROR: Account {account_name} can't have both a secret key and cloud signer." + ) + elif signer_url: + # Use signer for this account even if there's a sk + sk = signer_url + print(f" Using remote signer url: {sk}") + + return sk def import_keys(all_accounts): @@ -384,29 +427,15 @@ def import_keys(all_accounts): if account_key == None: raise Exception(f"{account_name} defined w/o a key") - key = pytezos.key.from_encoded_key(account_key) - try: - key.secret_key() - except ValueError: - account_values["type"] = "public" - else: - account_values["type"] = "secret" + key = Key.from_encoded_key(account_key) + account_values["type"] = "secret" if key.is_secret else "public" # restrict which private key is exposed to which pod if expose_secret_key(account_name): - signer = account_values.get("signer_url") - if signer: - print("\n Using signer outside of chart: " + signer) - sk = remote_signer(account_name, signer, key) - if sk == None or pod_requires_secret_key(account_values): - try: - sk = "unencrypted:" + key.secret_key() - except ValueError: - raise ("Secret key required but not provided.") - - print(" Appending secret key") - else: - print(" Using remote signer: " + sk) + sk = get_secret_key((account_name, account_values), key) + if not sk: + raise Exception("Secret key required but not provided.") + print(" Appending secret key") secret_keys.append({"name": account_name, "value": sk}) pk_b58 = key.public_key() @@ -420,28 +449,31 @@ def import_keys(all_accounts): account_values["pk"] = pk_b58 pkh_b58 = key.public_key_hash() - print(f" Appending public key hash: {pkh_b58}") + print(f" Appending public key hash: {pkh_b58}") public_key_hashs.append({"name": account_name, "value": pkh_b58}) account_values["pkh"] = pkh_b58 - # XXXrcd: fix this print! - - print(f" Account key type: {account_values.get('type')}") + print(f" Account key type: {account_values.get('type')}") print( - f" Account bootstrap balance: " + f" Account bootstrap balance: " + f"{account_values.get('bootstrap_balance')}" ) print( - f" Is account a bootstrap baker: " + f" Is account a bootstrap baker: " + f"{account_values.get('is_bootstrap_baker_account', False)}" ) - print("\n Writing " + tezdir + "/secret_keys") - json.dump(secret_keys, open(tezdir + "/secret_keys", "w"), indent=4) - print(" Writing " + tezdir + "/public_keys") - json.dump(public_keys, open(tezdir + "/public_keys", "w"), indent=4) - print(" Writing " + tezdir + "/public_key_hashs") - json.dump(public_key_hashs, open(tezdir + "/public_key_hashs", "w"), indent=4) + sk_path, pk_path, pkh_path = ( + f"{tezdir}/secret_keys", + f"{tezdir}/public_keys", + f"{tezdir}/public_key_hashs", + ) + print(f"\n Writing {sk_path}") + json.dump(secret_keys, open(sk_path, "w"), indent=4) + print(f" Writing {pk_path}") + json.dump(public_keys, open(pk_path, "w"), indent=4) + print(f" Writing {pkh_path}") + json.dump(public_key_hashs, open(pkh_path, "w"), indent=4) def create_node_identity_json():