feat: 🔥 remove dbt integration, quarto portal, and update dependencies

Refactored the project by eliminating dbt integration and related configurations, including assets, definitions, files, and dependencies. Removed Quarto portal setup and files, migrating to a simpler static HTML structure for the website. Updated dependencies and tooling to align with these changes and improve compatibility. Additionally, CI workflow was split into ETL and Web workflows, enhancing organization and modularity.

.devcontainer/devcontainer.json

@@ -1,48 +1,20 @@
 {
-    "name": "Datadex",
-    "build": {
-        "dockerfile": "Dockerfile",
-        "context": ".."
-    },
-    "customizations": {
-        "vscode": {
-            "settings": {
-                "quarto.render.previewType": "external",
-                "files.associations": {
-                    "*.sql": "jinja-sql"
-                },
-                "[jinja-sql]": {
-                    "editor.formatOnSave": false
-                },
-                "[python]": {
-                    "editor.defaultFormatter": "charliermarsh.ruff",
-                    "editor.formatOnSave": true,
-                    "editor.codeActionsOnSave": {
-                        "source.fixAll": "explicit",
-                        "source.organizeImports": "explicit"
-                    }
-                },
-                "notebook.formatOnSave.enabled": true,
-                "notebook.codeActionsOnSave": {
-                    "source.fixAll.ruff": "explicit",
-                    "source.organizeImports.ruff": "explicit"
-                },
-                "python.analysis.typeCheckingMode": "basic",
-                "python.analysis.autoImportCompletions": true,
-            },
-            "extensions": [
-                "charliermarsh.ruff",
-                "innoverio.vscode-dbt-power-user",
-                "ms-python.python",
-                "ms-python.vscode-pylance",
-                "ms-toolsai.datawrangler",
-                "ms-toolsai.jupyter",
-                "quarto.quarto",
-                "redhat.vscode-yaml",
-                "tamasfe.even-better-toml",
-                "visualstudioexptteam.vscodeintellicode"
-            ]
-        }
-    },
-    "postCreateCommand": "make setup"
+	"name": "Datadex",
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+	"features": {
+		"ghcr.io/jsburckhardt/devcontainer-features/uv:1": {},
+		"ghcr.io/devcontainers/features/node:1": {}
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"tamasfe.even-better-toml",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"ms-toolsai.jupyter",
+				"ms-python.vscode-pylance",
+				"charliermarsh.ruff"
+			]
+		}
+	}
 }

.github/workflows/etl.yml

@@ -0,0 +1,34 @@
+name: ETL
+
+on:
+  push:
+    branches:
+      - main
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run:
+    name: Run ETL
+    runs-on: ubuntu-latest
+    env:
+      HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: Set up Python
+        run: uv python install
+      - name: Install dependencies
+        run: uv sync
+      - name: Run
+        run: |
+          make run

.github/workflows/web.yml

@@ -0,0 +1,34 @@
+name: Web
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'web/**'
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Publish to Cloudflare Pages
+        uses: cloudflare/pages-action@v1
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: fbe814d4a37d3d3d7fcfeec0ab929ff6
+          projectName: datadex
+          directory: web
+          gitHubToken: ${{ secrets.GITHUB_TOKEN }}
+          branch: main

Makefile

@@ -1,31 +1,14 @@
 .DEFAULT_GOAL := run
+.PHONY: web run dev clean
 
 run:
-	uv run dagster-dbt project prepare-and-package --file datadex/dbt/resources.py
 	uv run dagster asset materialize --select \* -m datadex.definitions
 
 dev:
 	uv run dagster dev
 
-preview:
-	quarto preview portal
-
-setup:
-	command -v uv >/dev/null 2>&1 || pip install -U uv
-	uv sync
-	. .venv/bin/activate
-
-dbt-docs:
-	. .venv/bin/activate; cd dbt; dbt docs generate --profiles-dir .
-	mkdir -p dbt/target/docs
-	cp dbt/target/*.json dbt/target/index.html dbt/target/graph.gpickle dbt/target/docs/
-
-render: dbt-docs
-	quarto render portal
-	cp -r dbt/target/docs/ portal/.quarto/output/dbt
+web:
+	python -m http.server -d web
 
 clean:
 	rm -rf data/*.parquet data/*.duckdb
-	rm -rf dbt/target dbt/dbt_packages dbt/logs
-	rm -rf portal/.quarto
-	rm -rf .venv

datadex/definitions.py

@@ -1,6 +1,5 @@
 import dagster as dg
 
-import datadex.dbt.definitions as dbt_definitions
 import datadex.huggingface.definitions as huggingface_definitions
 import datadex.indicators.definitions as indicators_definitions
 from datadex.resources import io_manager
@@ -9,7 +8,6 @@
 
 definitions = dg.Definitions.merge(
     dg.Definitions(resources=common_resources),
-    dbt_definitions.definitions,
     indicators_definitions.definitions,
     huggingface_definitions.definitions,
 )

datadex/huggingface/assets.py

@@ -39,7 +39,7 @@ def hf_asset(data: pl.DataFrame, dp: HuggingFaceDatasetPublisher) -> None:
     return hf_asset
 
 
-datasets = ["world_development_indicators"]
+datasets = []
 
 assets = []
 for dataset in datasets: