sql-sink-backfill: add database scripts and rework dsn variable

pinax-network · Jul 30, 2024 · 6dcdd16 · 6dcdd16
1 parent e343b79
commit 6dcdd16
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 24 deletions.
diff --git a/charts/substreams-sink-sql-backfill/Chart.yaml b/charts/substreams-sink-sql-backfill/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.6
+version: 0.0.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/substreams-sink-sql-backfill/create-table.sql b/charts/substreams-sink-sql-backfill/create-table.sql
diff --git a/charts/substreams-sink-sql-backfill/drop-table.sql b/charts/substreams-sink-sql-backfill/drop-table.sql
diff --git a/charts/substreams-sink-sql-backfill/templates/job.yaml b/charts/substreams-sink-sql-backfill/templates/job.yaml
@@ -25,15 +25,47 @@ spec:
       labels:
         {{- include "substreams-sink-sql-backfill.labels" $ | nindent 8 }}
     spec:
-      containers:
-      - name: {{ include "substreams-sink-sql-backfill.fullname" $ }}
+      initContainers:
+      # Container to create cursor tables
+      - name: {{ include "substreams-sink-sql-backfill.fullname" $ }}-create-tables
+        image: curlimages/curl:latest
+        command:
+        - "/bin/sh"
+        - "-c"
+        - |
+          cat <<EOF > /tmp/create-tables.sql
+          CREATE TABLE IF NOT EXISTS {{ $values.database }}.{{ $values.tablePrefix }}_{{ $currentBlock }}
+          (
+              id        String,
+              cursor    String,
+              block_num Int64,
+              block_id  String
+          )
+          ENGINE = ReplacingMergeTree()
+              PRIMARY KEY (id)
+              ORDER BY (id);
+          EOF
+          curl -X POST 'http://{{ $values.host }}:8123/?user=$(USERNAME)&password=$(PASSWORD)' --data-binary @/tmp/create-tables.sql
+        env:
+        - name: USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyUsername }}
+        - name: PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyPassword }}
+      # Container to backfill data
+      - name: {{ include "substreams-sink-sql-backfill.fullname" $ }}-backfill
         image: "{{ $values.image.repository }}:{{ $values.image.tag }}"
         {{- if $values.command }}
         command:
         {{- range $values.command }}
         - {{ . | quote }}
         {{- end }}
-        - "$(DSN)"
+        - "clickhouse://$(USERNAME):$(PASSWORD)@{{ $values.host }}:9000/{{ $values.database }}"
         - "{{ $values.manifest }}"
         - "{{ $currentBlock }}:{{ add $currentBlock $blockIncrement }}"
         {{- end }}
@@ -45,15 +77,43 @@ spec:
         {{- end }}
         env:
         - name: SINK_SQL_RUN_CURSORS_TABLE
-          value: backprocess_{{ $currentBlock }}
-        - name: DSN
+          value: {{ $values.tablePrefix }}_{{ $currentBlock }}
+        - name: USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyUsername }}
+        - name: PASSWORD
           valueFrom:
             secretKeyRef:
-              key: {{ $values.dsn.key }}
-              name: {{ $values.dsn.secretName }}
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyPassword }}
         envFrom:
         - configMapRef:
             name: env-{{ $.Release.Name }}
+      # Container to drop cursor tables
+      containers:
+      - name: {{ include "substreams-sink-sql-backfill.fullname" $ }}-drop-tables
+        image: curlimages/curl:latest
+        command:
+        - "/bin/sh"
+        - "-c"
+        - |
+          cat <<EOF > /tmp/drop-tables.sql
+          DROP TABLE {{ $values.database }}.{{ $values.tablePrefix }}_{{ $currentBlock }}
+          EOF
+          curl -X POST 'http://{{ $values.host }}:8123/?user=$(USERNAME)&password=$(PASSWORD)' --data-binary @/tmp/drop-tables.sql
+        env:
+        - name: USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyUsername }}
+        - name: PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ $values.clickhouseAuth.secretName }}
+              key: {{ $values.clickhouseAuth.keyPassword }}
       restartPolicy: OnFailure
       imagePullSecrets:
         - name: {{ $values.imagePullSecret }}

diff --git a/charts/substreams-sink-sql-backfill/values.yaml b/charts/substreams-sink-sql-backfill/values.yaml
@@ -15,13 +15,23 @@ jobAmount: 1
 startBlock: 0
 endBlock: -1
 # spkg file to use
-manifest: https://github.com/file.spkg
-dsn:
-  secretName: sql-sink-dsn
-  key: dsn
+manifest: myFile.spkg
+
+# Clickhouse host name
+host: myClickhouseDB
+# Clickhouse database to use
+database:
+# Prefix to add to cursor tables being created and deleted by the backfill jobs
+# By default: backprocess-<startBlock>
+tablePrefix: backprocess
+# Username/Password to connect to database from an existing secret
+clickhouseAuth:
+  secretName: ch-auth
+  keyUsername: username
+  keyPassword: password
 # environment variables to be use by the applications <key>: <value>
 env:
-  substreams_endpoints_config_mainnet: substreams:9000
+  substreams_endpoints_config_mainnet: mySubstreams:9000
   sink_sql_run_undo_buffer_size: 100
   sink_sql_run_batch_row_flush_interval: 100000
   sink_sql_run_batch_block_flush_interval: 20000