refactor(schema): use json arrays for list columns

Signed-off-by: Nico Braun <rainbowstack@gmail.com>

.envrc

@@ -2,3 +2,4 @@
 dotenv .env
 dotenv_if_exists .local/.env
 PATH_add .local/bin
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/.local/lib"

build/.sqliterc

@@ -0,0 +1,2 @@
+.load uuid
+.load sha1

build/Dockerfile

@@ -26,7 +26,7 @@ RUN apk add --no-cache --update ca-certificates git openssh-client
 COPY --from=sqlite /usr/local/bin /usr/local/bin
 COPY --from=sqlite /usr/local/lib /usr/local/lib
 RUN ldconfig /usr/local/lib
-RUN printf '.load sha1\n.load uuid\n' > /root/.sqliterc
+COPY .sqliterc /root/
 
 FROM runtime as nonroot
 ARG NONROOT_HOME=/etc/kobold

build/Dockerfile.dockerignore

@@ -1,5 +1,6 @@
 .*
 build/
+!build/.sqliterc
 e2e/
 vendor/
 *.md

build/sqlc.yaml

@@ -1,8 +1,13 @@
 version: "2"
 sql:
 - engine: sqlite
-  queries: ../store/schema/*.query.sql
-  schema: ../store/schema/*.schema.sql
+  queries:
+  - ../store/schema/task.query.sql
+  - ../store/schema/read.query.sql
+  - ../store/schema/config.query.sql
+  schema:
+  - ../store/schema/task.schema.sql
+  - ../store/schema/read.schema.sql
   gen:
     go:
       package: model
@@ -26,22 +31,22 @@ sql:
         go_type:
           import: github.com/bluebrown/kobold/store
           package: store
-          type: SliceText
+          type: FlatList
       - column: "*.warnings"
         go_type:
           import: github.com/bluebrown/kobold/store
           package: store
-          type: SliceText
+          type: FlatList
       - column: "*.task_ids"
         go_type:
           import: github.com/bluebrown/kobold/store
           package: store
-          type: SliceText
+          type: FlatList
       - column: "*.fingerprint"
         go_type:
           type: string
       - column: "*.channels"
         go_type:
           import: github.com/bluebrown/kobold/store
           package: store
-          type: JsonArray
+          type: FlatList

build/sqlite3.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -o nounset -o errexit -o errtrace -o pipefail
+
+prefix="${1:-/usr/local}"
+
+workdir="$(mktemp -d)"
+trap 'rm -rf "$workdir"' EXIT
+cd "$workdir"
+
+curl -fsSL "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" |
+  tar --strip-components 1 -xzf -
+
+autoreconf -fi
+CFLAGS="-Os -DSQLITE_ENABLE_LOAD_EXTENSION" ./configure --prefix "$prefix"
+make -j"$(nproc)"
+make install
+
+mkdir -p "$prefix/lib"
+
+curl -fsSL https://www.sqlite.org/src/raw/4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d?at=sha1.c -o sha1.c
+gcc -shared -fPIC -o "$prefix/lib/sha1.so" sha1.c
+
+curl -fsSL https://www.sqlite.org/src/raw/5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf?at=uuid.c -o uuid.c
+gcc -shared -fPIC -o "$prefix/lib/uuid.so" uuid.c
+
+cat <<EOF >"$prefix/bin/sqlite"
+#!/usr/bin/env bash
+set -o nounset -o errexit -o errtrace -o pipefail
+exec sqlite3 -cmd '.load uuid' -cmd '.load sha1' "\$@"
+EOF
+
+chmod +x "$prefix/bin/sqlite"

cmd/cli/main.go

@@ -60,7 +60,7 @@ func run(ctx context.Context, args []string, env []string, input io.Reader) erro
 		return fmt.Errorf("parse args: %w", err)
 	}
 
-	query, err := config.Configure(ctx, *opts, schema.TaskSchema)
+	query, err := config.Configure(ctx, *opts, schema.TaskSchema, schema.ReadSchema)
 	if err != nil {
 		return fmt.Errorf("configure: %w", err)
 	}

cmd/server/main.go

@@ -64,7 +64,7 @@ func run(ctx context.Context, args []string, env []string) error {
 		return fmt.Errorf("parse args: %w", err)
 	}
 
-	query, err := config.Configure(ctx, *opts, schema.TaskSchema)
+	query, err := config.Configure(ctx, *opts, schema.TaskSchema, schema.ReadSchema)
 	if err != nil {
 		return fmt.Errorf("configure: %w", err)
 	}

store/flatlist.go

@@ -0,0 +1,61 @@
+package store
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+)
+
+// this is a special json array. When scanning, it will flatten the input up to
+// 1 level deep. however, when calling value, it will only marshal a flat list.
+// so the converion is lossy, but it fits our use case perfectly. We use it to
+// either store flat lists on actual tables or to retrieve potentially nested
+// lists from accumulated views
+type FlatList []string
+
+func (s FlatList) Value() (driver.Value, error) {
+	if len(s) == 0 {
+		return "", nil
+	}
+	return json.Marshal(s)
+}
+
+func (s *FlatList) Scan(value interface{}) error {
+	if value == nil {
+		return nil
+	}
+	var b []byte
+
+	switch v := value.(type) {
+	case string:
+		b = []byte(v)
+	case []byte:
+		b = v
+	default:
+		return fmt.Errorf("store: cannot convert %T to FlatList", value)
+	}
+
+	if len(b) == 0 {
+		return nil
+	}
+
+	var iterm []any
+	if err := json.Unmarshal(b, &iterm); err != nil {
+		return fmt.Errorf("unmarshal intermeditate: %w", err)
+	}
+
+	for _, item := range iterm {
+		switch v := item.(type) {
+		case string:
+			*s = append(*s, v)
+		case []interface{}:
+			for _, vv := range v {
+				*s = append(*s, vv.(string))
+			}
+		default:
+			return fmt.Errorf("store: cannot convert %T to FlatList", value)
+		}
+	}
+
+	return nil
+}

store/schema/embed.go

@@ -5,5 +5,8 @@ import _ "embed"
 //go:embed task.schema.sql
 var TaskSchema []byte
 
+//go:embed read.schema.sql
+var ReadSchema []byte
+
 // go:embed clean.sql
 var CleanConfig []byte

store/schema/read.schema.sql

@@ -0,0 +1,34 @@
+-- this is for the web api. So that it can display the channels embedded in the
+-- pipeline json object
+create view if not exists pipeline_list_item as
+select
+  pipeline.*,
+  json_group_array(subscription.channel_name) as channels
+from pipeline
+left join subscription on subscription.pipeline_name = pipeline.name
+group by pipeline.name;
+
+-- the run view is like a counter part to the task group view. Its primarly
+-- use case is to show the actual run information of the task groups returned by
+-- the task group view. runs with a fingerprint have been executed and will
+-- never change. However, pending runs, have not been executed yet, and they can
+-- still change until they are executed
+create view if not exists run as
+select
+  ifnull(task_group_fingerprint, '') as fingerprint,
+  repo_uri,
+  dest_branch,
+  post_hook_name as post_hook,
+  status,
+  max(timestamp) as timestamp,
+  max(warnings) as warnings,
+  max(failure_reason) as error,
+  json_group_array(json(msgs)) as msgs
+from task
+group by
+  task_group_fingerprint,
+  repo_uri,
+  dest_branch,
+  post_hook_name,
+  status;
+