feat: new unit tests for dbt models and macros

.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Run all tests & checks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  run_tests:
+    name: tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install requirements
+        run: pip3 install -r requirements.txt
+
+      - name: Run tests
+        run: dbt test --profiles-dir $DBT_PROFILES_DIR

dbt_project.yml

@@ -14,7 +14,7 @@ profile: "aspects"
 model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
-seed-paths: ["seeds"]
+seed-paths: ["seeds","seeds-test"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 

macros/drop_stale_tables.sql

@@ -0,0 +1,53 @@
+{% macro drop_stale_tables(dryrun=False) %}
+
+    {% set current_models = [] %}
+
+    {% for node in graph.nodes.values() | selectattr(
+        "resource_type", "in", ["model", "snapshot"]
+    ) %}
+        {% do current_models.append(node.name) %}
+
+    {% endfor %}
+
+    {% set cleanup_query %}
+    WITH MODELS_TO_DROP AS (
+        SELECT
+            CASE 
+            WHEN TABLE_TYPE = 'BASE TABLE' THEN 'TABLE'
+            WHEN TABLE_TYPE = 'VIEW' THEN 'VIEW'
+            END AS RELATION_TYPE,
+            CONCAT_WS('.', TABLE_SCHEMA, TABLE_NAME) AS RELATION_NAME
+        FROM 
+            INFORMATION_SCHEMA.TABLES
+        WHERE TABLE_SCHEMA = '{{ target.schema }}'
+          AND upper(TABLE_NAME) NOT IN
+            ({%- for model in current_models -%}
+                '{{ model.upper() }}'
+                {%- if not loop.last -%}
+                    ,
+                {% endif %}
+            {%- endfor -%})
+    )
+    SELECT 
+    concat('DROP ',RELATION_TYPE,' ',RELATION_NAME,';') as DROP_COMMANDS
+    FROM MODELS_TO_DROP
+
+    {% endset %}
+    {% do log(cleanup_query, info=True) %}
+    {% set drop_commands = run_query(cleanup_query).columns[0].values() %}
+
+    {% if drop_commands %}
+        {% if dryrun | as_bool == False %}
+            {% do log("Executing DROP commands...", True) %}
+        {% else %} {% do log("Printing DROP commands...", True) %}
+        {% endif %}
+        {% for drop_command in drop_commands %}
+            {% do log(drop_command, True) %}
+            {% if dryrun | as_bool == False %}
+                {% do run_query(drop_command) %}
+            {% endif %}
+        {% endfor %}
+    {% else %} {% do log("No relations to clean.", True) %}
+    {% endif %}
+
+{%- endmacro -%}

models/completion/fact_completions.sql

@@ -28,27 +28,7 @@ select
     ) as entity_name_with_location,
     completions.actor_id as actor_id,
     cast(completions.scaled_progress as Float) as scaled_progress,
-    case
-        when scaled_progress >= 0.9
-        then '90-100%'
-        when scaled_progress >= 0.8 and scaled_progress < 0.9
-        then '80-89%'
-        when scaled_progress >= 0.7 and scaled_progress < 0.8
-        then '70-79%'
-        when scaled_progress >= 0.6 and scaled_progress < 0.7
-        then '60-69%'
-        when scaled_progress >= 0.5 and scaled_progress < 0.6
-        then '50-59%'
-        when scaled_progress >= 0.4 and scaled_progress < 0.5
-        then '40-49%'
-        when scaled_progress >= 0.3 and scaled_progress < 0.4
-        then '30-39%'
-        when scaled_progress >= 0.2 and scaled_progress < 0.3
-        then '20-29%'
-        when scaled_progress >= 0.1 and scaled_progress < 0.2
-        then '10-19%'
-        else '0-9%'
-    end as completion_bucket,
+    {{ get_bucket("scaled_progress") }} as completion_bucket,
     users.username as username,
     users.name as name,
     users.email as email

models/courses/course_names.sql

@@ -24,7 +24,11 @@ with
         from {{ source("event_sink", "course_overviews") }}
         group by org, course_key
     )
-select course_key, display_name, splitByString('+', course_key)[-1] as course_run, org
+select
+    course_key,
+    display_name as course_name,
+    splitByString('+', course_key)[-1] as course_run,
+    org
 from {{ source("event_sink", "course_overviews") }} co
 inner join
     most_recent_overviews mro

models/enrollment/unit_tests.yaml

@@ -0,0 +1,18 @@
+unit_tests:
+  - name: test_fact_enrollments
+    model: fact_enrollments
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('course_names')
+        format: sql
+        fixture: course_names
+      - input: ref('enrollment_events')
+        format: sql
+        fixture: enrollment_events
+      - input: ref('dim_user_pii')
+        format: sql
+        fixture: dim_user_pii
+    expect:
+      format: sql
+      fixture: fact_enrollments_expected

models/grading/unit_tests.yaml

@@ -0,0 +1,21 @@
+unit_tests:
+  - name: test_fact_grades
+    model: fact_grades
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('grading_events')
+        format: sql
+        fixture: grading_events
+      - input: ref('course_block_names')
+        format: sql
+        fixture: course_block_names
+      - input: ref('dim_user_pii')
+        format: sql
+        fixture: dim_user_pii
+      - input: ref('course_names')
+        format: sql
+        fixture: course_names
+    expect:
+      format: sql
+      fixture: fact_grades_expected

models/navigation/unit_tests.yaml

@@ -0,0 +1,31 @@
+unit_tests:
+  - name: test_macro_items_per_subsection
+    model: int_pages_per_subsection
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('dim_course_blocks')
+        format: sql
+        fixture: dim_course_blocks
+    expect:
+      format: sql
+      fixture: items_per_subsection_expected
+
+
+  - name: test_fact_navigation_dropoff
+    model: fact_navigation_dropoff
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('dim_course_blocks')
+        format: sql
+        fixture: dim_course_blocks
+      - input: ref('fact_navigation')
+        format: sql
+        fixture: fact_navigation
+      - input: ref('dim_user_pii')
+        format: sql
+        fixture: dim_user_pii
+    expect:
+      format: sql
+      fixture: fact_navigation_dropoff_expected

models/video/unit_tests.yaml

@@ -0,0 +1,18 @@
+unit_tests:
+  - name: test_fact_watched_video_segments
+    model: fact_watched_video_segments
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('video_playback_events')
+        format: sql
+        fixture: video_playback_events
+      - input: ref('dim_course_blocks_extended')
+        format: sql
+        fixture: dim_course_blocks_extended
+      - input: ref('dim_user_pii')
+        format: sql
+        fixture: dim_user_pii
+    expect:
+      format: sql
+      fixture: fact_watched_video_segments_expected

requirements.txt

@@ -1,5 +1,5 @@
-dbt-clickhouse==1.7.6
-dbt-core==1.7.15
+dbt-clickhouse==1.8.0
+dbt-core==1.8
 shandy-sqlfmt[jinjafmt]==0.21.2
 dbt-coverage
 clickhouse-cityhash