feat: new unit tests for dbt models and macros

.github/workflows/coverage.yml

@@ -46,6 +46,7 @@ jobs:
           tutor local do load-xapi-test-data
       - name: Check dbt tests
         run: |
+          dbt seed
           dbt run
           dbt test
       - name: Check docs coverage

dbt_project.yml

@@ -14,7 +14,7 @@ profile: "aspects"
 model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
-seed-paths: ["seeds"]
+seed-paths: ["seeds","seeds-test"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
@@ -34,3 +34,13 @@ models:
     # Config indicated by + and applies to all files under models/example/
     enrollment:
       +materialized: view
+
+seeds:
+  aspects:
+    expected:
+      items_per_subsection_expected:
+        +column_types:
+          item_count: UInt64
+      fact_navigation_dropoff_expected:
+        +column_types:
+          total_views: UInt64

macros/format_float_value.sql

@@ -0,0 +1,9 @@
+{% macro format_float_value(value) %}
+
+    {% if target.name != 'prod' %}
+        CAST({{value}} as Float32)
+    {% else %}
+        {{ value }}
+    {% endif %}
+
+{% endmacro %}

macros/source_for_test.sql

@@ -0,0 +1,9 @@
+{% macro source_for_test(schema,table) %}
+
+    {% if target.name != 'prod' %}
+        {{ source(target.schema,table)}}
+    {% else %}
+        {{ source(schema,table) }}
+    {% endif %}
+
+{% endmacro %}

models/base/sources.yml

@@ -12,3 +12,11 @@ sources:
           - name: event_id
           - name: emission_time
           - name: event
+  - name: reporting
+    database: "reporting"
+    tables:
+      - name: xapi_events_all
+        columns:
+          - name: event_id
+          - name: emission_time
+          - name: event

models/base/xapi_events_all_parsed.sql

@@ -52,4 +52,4 @@ select
     ) as org,
     emission_time as emission_time,
     event::String as event
-from {{ source("xapi", "xapi_events_all") }}
+from {{ source_for_test("xapi", "xapi_events_all") }}

models/completion/fact_completions.sql

@@ -28,27 +28,7 @@ select
     ) as entity_name_with_location,
     completions.actor_id as actor_id,
     cast(completions.scaled_progress as Float) as scaled_progress,
-    case
-        when scaled_progress >= 0.9
-        then '90-100%'
-        when scaled_progress >= 0.8 and scaled_progress < 0.9
-        then '80-89%'
-        when scaled_progress >= 0.7 and scaled_progress < 0.8
-        then '70-79%'
-        when scaled_progress >= 0.6 and scaled_progress < 0.7
-        then '60-69%'
-        when scaled_progress >= 0.5 and scaled_progress < 0.6
-        then '50-59%'
-        when scaled_progress >= 0.4 and scaled_progress < 0.5
-        then '40-49%'
-        when scaled_progress >= 0.3 and scaled_progress < 0.4
-        then '30-39%'
-        when scaled_progress >= 0.2 and scaled_progress < 0.3
-        then '20-29%'
-        when scaled_progress >= 0.1 and scaled_progress < 0.2
-        then '10-19%'
-        else '0-9%'
-    end as completion_bucket,
+    {{ get_bucket("scaled_progress") }} as completion_bucket,
     users.username as username,
     users.name as name,
     users.email as email

models/courses/course_block_names.sql

@@ -43,7 +43,7 @@ with
             row_number() over (
                 partition by location order by time_last_dumped desc
             ) as rn
-        from {{ source("event_sink", "course_blocks") }}
+        from {{ source_for_test("event_sink", "course_blocks") }}
     )
 select
     location,

models/courses/course_names.sql

@@ -21,11 +21,15 @@
 with
     most_recent_overviews as (
         select org, course_key, max(modified) as last_modified
-        from {{ source("event_sink", "course_overviews") }}
+        from {{ source_for_test("event_sink", "course_overviews") }}
         group by org, course_key
     )
-select course_key, display_name, splitByString('+', course_key)[-1] as course_run, org
-from {{ source("event_sink", "course_overviews") }} co
+select
+    course_key,
+    display_name as course_name,
+    splitByString('+', course_key)[-1] as course_run,
+    org
+from {{ source_for_test("event_sink", "course_overviews") }} co
 inner join
     most_recent_overviews mro
     on co.org = mro.org

models/courses/sources.yml

@@ -32,3 +32,35 @@ sources:
           - name: modified
           - name: dump_id
           - name: time_last_dumped
+
+  - name: reporting
+    database: "reporting"
+    tables:
+
+      - name: course_blocks
+        columns:
+          - name: org
+          - name: course_key
+          - name: location
+          - name: display_name
+          - name: xblock_data_json
+          - name: order
+          - name: edited_on
+          - name: dump_id
+          - name: time_last_dumped
+
+      - name: course_overviews
+        columns:
+          - name: org
+          - name: course_key
+          - name: display_name
+          - name: course_start
+          - name: course_end
+          - name: enrollment_start
+          - name: enrollment_end
+          - name: self_paced
+          - name: course_data_json
+          - name: created
+          - name: modified
+          - name: dump_id
+          - name: time_last_dumped

models/enrollment/unit_tests.yaml

@@ -0,0 +1,22 @@
+unit_tests:
+  - name: test_fact_enrollments
+    model: fact_enrollments
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('course_names')
+        format: sql
+        rows: |
+          select * from course_names
+      - input: ref('enrollment_events')
+        format: sql
+        rows: |
+          select * from enrollment_events
+      - input: ref('dim_user_pii')
+        format: sql
+        rows: |
+          select * from dim_user_pii
+    expect:
+      format: sql
+      rows: |
+        select * from fact_enrollments_expected

models/grading/fact_grades.sql

@@ -40,7 +40,7 @@ select
     ) as entity_name_with_location,
     grades.grade_type as grade_type,
     grades.actor_id as actor_id,
-    grades.scaled_score as scaled_score,
+    {{ format_float_value("grades.scaled_score") }} as scaled_score,
     {{ get_bucket("scaled_score") }} as grade_bucket,
     users.username as username,
     users.name as name,

models/grading/unit_tests.yaml

@@ -0,0 +1,25 @@
+unit_tests:
+  - name: test_fact_grades
+    model: fact_grades
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('grading_events')
+        format: sql
+        rows: |
+          select * from grading_events
+      - input: ref('course_block_names')
+        format: sql
+        rows: |
+          select * from course_block_names
+      - input: ref('dim_user_pii')
+        format: sql
+        rows: |
+          select * from dim_user_pii
+      - input: ref('course_names')
+        format: sql
+        rows: |
+          select * from course_names
+    expect:
+      format: sql
+      fixture: fact_grades_expected

models/navigation/unit_tests.yaml

@@ -0,0 +1,37 @@
+unit_tests:
+  - name: test_macro_items_per_subsection
+    model: int_pages_per_subsection
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('dim_course_blocks')
+        format: sql
+        rows: |
+          select * from dim_course_blocks
+    expect:
+      format: sql
+      rows: |
+        select * from items_per_subsection_expected
+
+      
+  - name: test_fact_navigation_dropoff
+    model: fact_navigation_dropoff
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('dim_course_blocks')
+        format: sql
+        rows: |
+          select * from dim_course_blocks
+      - input: ref('fact_navigation')
+        format: sql
+        rows: |
+          select * from fact_navigation
+      - input: ref('dim_user_pii')
+        format: sql
+        rows: |
+          select * from dim_user_pii
+    expect:
+      format: sql
+      rows: |
+        select * from fact_navigation_dropoff_expected

models/users/sources.yml

@@ -37,3 +37,40 @@ sources:
           - name: user_id
           - name: dump_id
           - name: time_last_dumped
+
+  - name: reporting
+    database: reporting
+    tables:
+
+      - name: user_profile
+        columns:
+          - name: id
+          - name: user_id
+          - name: name
+          - name: email
+          - name: meta
+          - name: courseware
+          - name: language
+          - name: location
+          - name: year_of_birth
+          - name: gender
+          - name: level_of_education
+          - name: mailing_address
+          - name: city
+          - name: country
+          - name: state
+          - name: goals
+          - name: bio
+          - name: profile_image_uploaded_at
+          - name: phone_number
+          - name: dump_id
+          - name: time_last_dumped
+
+      - name: external_id
+        columns:
+          - name: external_user_id
+          - name: external_id_type
+          - name: username
+          - name: user_id
+          - name: dump_id
+          - name: time_last_dumped

models/users/user_pii.sql

@@ -28,9 +28,9 @@ with
             ROW_NUMBER() over (
                 partition by user_id order by (id, time_last_dumped) DESC
             ) as rn
-        from {{ source("event_sink", "user_profile") }}
+        from {{ source_for_test("event_sink", "user_profile") }}
     )
 select mrup.user_id as user_id, external_user_id, username, name, email
-from {{ source("event_sink", "external_id") }} ex
+from {{ source_for_test("event_sink", "external_id") }} ex
 left outer join most_recent_user_profile mrup on mrup.user_id = ex.user_id
 where mrup.rn = 1

models/video/schema.yml

@@ -257,55 +257,55 @@ models:
     description: "A dataset with one record per learner per video segment watched in a course"
     columns:
       - name: org
-        data_type: string
+        data_type: String
         description: "The organization that the course belongs to"
       - name: course_key
-        data_type: string
+        data_type: String
         description: "The course key for the course"
       - name: course_name
-        data_type: string
+        data_type: String
         description: "The name of the course"
       - name: course_run
-        data_type: string
+        data_type: String
         description: "The course run for the course"
       - name: section_with_name
-        data_type: string
+        data_type: String
         description: "The name of the section this subsection belongs to, with section_number prepended"
       - name: subsection_with_name
-        data_type: string
+        data_type: String
         description: "The name of the subsection, with section_number prepended"
       - name: video_name
-        data_type: string
+        data_type: String
         description: "The name of the video"
       - name: video_name_with_location
-        data_type: string
+        data_type: String
         description: "The name of the video with the section and subsection"
       - name: actor_id
-        data_type: string
+        data_type: String
         description: "The xAPI actor identifier"
       - name: started_at
-        data_type: datetime
+        data_type: DateTime
         description: "The time the video segment was started"
       - name: segment_start
-        data_type: int64
+        data_type: Int64
         description: "The start position of the segment"
-      - name: segment_range
-        data_type: string
-        description: "An string representing the interval of the watched segment"
       - name: video_duration
-        data_type: int64
+        data_type: Int64
         description: "The duration of the video in seconds"
+      - name: segment_range
+        data_type: String
+        description: "An string representing the interval of the watched segment"
       - name: start_position
-        data_type: int64
+        data_type: Int64
         description: "The start position of the segment"
       - name: username
-        data_type: string
+        data_type: String
         description: "The username of the learner"
       - name: name
-        data_type: string
+        data_type: String
         description: "The full name of the learner"
       - name: email
-        data_type: string
+        data_type: String
         description: "The email address of the learner"
 
   - name: section_video_engagement