From 2de25d9ae337dd67c4d948c43f4c1f636451876d Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 30 Apr 2024 16:46:29 -0400 Subject: [PATCH 1/3] feat: Add course block sort order to models --- macros/items_per_subsection.sql | 5 ++++- models/courses/course_block_names.sql | 9 ++++++++- models/courses/dim_course_blocks.sql | 1 + models/courses/dim_course_blocks_extended.sql | 1 + models/navigation/fact_navigation.sql | 1 + models/navigation/fact_navigation_completion.sql | 1 + models/navigation/fact_navigation_dropoff.sql | 8 +++++++- models/navigation/int_pages_per_subsection.sql | 10 ++++++++-- models/problems/fact_learner_problem_summary.sql | 4 ++++ models/problems/fact_problem_engagement.sql | 2 ++ models/problems/fact_problem_responses.sql | 2 ++ models/problems/fact_problem_responses_extended.sql | 1 + models/problems/int_problem_hints.sql | 1 + models/problems/int_problem_results.sql | 1 + models/video/fact_transcript_usage.sql | 1 + models/video/fact_video_engagement.sql | 1 + models/video/fact_video_plays.sql | 3 ++- 17 files changed, 46 insertions(+), 6 deletions(-) diff --git a/macros/items_per_subsection.sql b/macros/items_per_subsection.sql index 4228daac..7fe8d28a 100644 --- a/macros/items_per_subsection.sql +++ b/macros/items_per_subsection.sql @@ -6,11 +6,13 @@ course_key, section_number, subsection_number, + course_order, graded, count(*) as item_count from {{ ref("dim_course_blocks") }} where block_id like '{{ block_pattern }}' - group by org, course_key, section_number, subsection_number, graded + group by + org, course_key, section_number, subsection_number, course_order, graded ) select @@ -20,6 +22,7 @@ section_blocks.display_name_with_location as section_with_name, ips.subsection_number as subsection_number, subsection_blocks.display_name_with_location as subsection_with_name, + ips.course_order as course_order, ips.graded as graded, ips.item_count as item_count from items_per_subsection ips diff --git a/models/courses/course_block_names.sql b/models/courses/course_block_names.sql index a8bfabb5..b60d1861 100644 --- a/models/courses/course_block_names.sql +++ b/models/courses/course_block_names.sql @@ -7,6 +7,7 @@ ("block_name", "String"), ("course_key", "String"), ("graded", "Bool"), + ("course_order", "Int32"), ("display_name_with_location", "String"), ], primary_key="location", @@ -35,6 +36,7 @@ with JSONExtractInt(xblock_data_json, 'subsection') as subsection, JSONExtractInt(xblock_data_json, 'unit') as unit, JSONExtractBool(xblock_data_json, 'graded') as graded, + `order` as course_order, course_key, dump_id, time_last_dumped, @@ -44,6 +46,11 @@ with from {{ source("event_sink", "course_blocks") }} ) select - location, display_name as block_name, course_key, graded, display_name_with_location + location, + display_name as block_name, + course_key, + graded, + course_order, + display_name_with_location from most_recent_course_blocks where rn = 1 diff --git a/models/courses/dim_course_blocks.sql b/models/courses/dim_course_blocks.sql index fc98c66e..a2c29fdd 100644 --- a/models/courses/dim_course_blocks.sql +++ b/models/courses/dim_course_blocks.sql @@ -10,6 +10,7 @@ select as subsection_number, splitByString(' - ', blocks.display_name_with_location)[1] as hierarchy_location, blocks.display_name_with_location as display_name_with_location, + course_order, graded, case when block_id like '%@chapter+block@%' diff --git a/models/courses/dim_course_blocks_extended.sql b/models/courses/dim_course_blocks_extended.sql index e06b550b..6bdc183c 100644 --- a/models/courses/dim_course_blocks_extended.sql +++ b/models/courses/dim_course_blocks_extended.sql @@ -10,6 +10,7 @@ select blocks.hierarchy_location as hierarchy_location, blocks.display_name_with_location as display_name_with_location, blocks.graded as graded, + blocks.course_order as course_order, blocks.block_type as block_type, section_blocks.display_name_with_location as section_with_name, subsection_blocks.display_name_with_location as subsection_with_name diff --git a/models/navigation/fact_navigation.sql b/models/navigation/fact_navigation.sql index a0c04893..12ece559 100644 --- a/models/navigation/fact_navigation.sql +++ b/models/navigation/fact_navigation.sql @@ -8,6 +8,7 @@ select navigation.block_id as block_id, blocks.block_name as block_name, blocks.display_name_with_location as block_name_with_location, + blocks.course_order as course_order, navigation.object_type as object_type, navigation.starting_position as starting_position, navigation.ending_point as ending_point, diff --git a/models/navigation/fact_navigation_completion.sql b/models/navigation/fact_navigation_completion.sql index b5060b79..50350613 100644 --- a/models/navigation/fact_navigation_completion.sql +++ b/models/navigation/fact_navigation_completion.sql @@ -23,6 +23,7 @@ select visits.course_run as course_run, pages.section_with_name as section_with_name, pages.subsection_with_name as subsection_with_name, + pages.course_order as course_order, pages.page_count as page_count, visits.actor_id as actor_id, visits.block_id as block_id, diff --git a/models/navigation/fact_navigation_dropoff.sql b/models/navigation/fact_navigation_dropoff.sql index 12979d9a..9d0c6949 100644 --- a/models/navigation/fact_navigation_dropoff.sql +++ b/models/navigation/fact_navigation_dropoff.sql @@ -1,6 +1,11 @@ with blocks as ( - select org, course_key, display_name_with_location, hierarchy_location + select + org, + course_key, + display_name_with_location, + hierarchy_location, + course_order from {{ ref("dim_course_blocks") }} where block_id like '%@chapter+block@%' or block_id like '%@sequential+block@%' ), @@ -63,6 +68,7 @@ select page_views.course_key as course_key, page_views.rollup_name as rollup_name, blocks.display_name_with_location as block_name, + blocks.course_order as course_order, page_views.actor_id as actor_id, page_views.total_views as total_views, users.username as username, diff --git a/models/navigation/int_pages_per_subsection.sql b/models/navigation/int_pages_per_subsection.sql index 557a9fae..cebba74c 100644 --- a/models/navigation/int_pages_per_subsection.sql +++ b/models/navigation/int_pages_per_subsection.sql @@ -1,10 +1,15 @@ with pages_per_subsection as ( select - org, course_key, section_number, subsection_number, count(*) as page_count + org, + course_key, + section_number, + subsection_number, + course_order, + count(*) as page_count from {{ ref("dim_course_blocks") }} where block_id like '%@vertical+block@%' - group by org, course_key, section_number, subsection_number + group by org, course_key, section_number, subsection_number, course_order ) select @@ -14,6 +19,7 @@ select section_blocks.display_name_with_location as section_with_name, pps.subsection_number as subsection_number, subsection_blocks.display_name_with_location as subsection_with_name, + pps.course_order as course_order, pps.page_count as page_count from pages_per_subsection pps left join diff --git a/models/problems/fact_learner_problem_summary.sql b/models/problems/fact_learner_problem_summary.sql index 001fa7f1..5f823a7e 100644 --- a/models/problems/fact_learner_problem_summary.sql +++ b/models/problems/fact_learner_problem_summary.sql @@ -10,6 +10,7 @@ with problem_id, problem_name, problem_name_with_location, + course_order, actor_id, success, attempts, @@ -25,6 +26,7 @@ with problem_id, problem_name, problem_name_with_location, + course_order, actor_id, null as success, null as attempts, @@ -45,6 +47,7 @@ select problem_id, problem_name, problem_name_with_location, + course_order, actor_id, coalesce(any(success), false) as success, coalesce(any(attempts), 0) as attempts, @@ -64,6 +67,7 @@ group by problem_id, problem_name, problem_name_with_location, + course_order, actor_id, username, name, diff --git a/models/problems/fact_problem_engagement.sql b/models/problems/fact_problem_engagement.sql index 1c5ed3b4..84859d19 100644 --- a/models/problems/fact_problem_engagement.sql +++ b/models/problems/fact_problem_engagement.sql @@ -8,6 +8,7 @@ with {{ section_from_display("problem_name_with_location") }} as section_number, {{ subsection_from_display("problem_name_with_location") }} as subsection_number, + course_order as course_order, graded, actor_id, problem_id @@ -24,6 +25,7 @@ select problems.item_count as item_count, attempts.actor_id as actor_id, attempts.problem_id as problem_id, + attempts.course_order as course_order, attempts.graded as graded, users.username as username, users.name as name, diff --git a/models/problems/fact_problem_responses.sql b/models/problems/fact_problem_responses.sql index da125fba..5b65d660 100644 --- a/models/problems/fact_problem_responses.sql +++ b/models/problems/fact_problem_responses.sql @@ -26,6 +26,7 @@ select blocks.display_name_with_location as problem_name_with_location, {{ a_tag("responses.object_id", "blocks.block_name") }} as problem_link, blocks.graded as graded, + course_order as course_order, responses.actor_id as actor_id, responses.responses as responses, responses.success as success, @@ -59,6 +60,7 @@ group by responses, success, attempts, + course_order, graded, interaction_type, username, diff --git a/models/problems/fact_problem_responses_extended.sql b/models/problems/fact_problem_responses_extended.sql index 15aad4fb..6a62023d 100644 --- a/models/problems/fact_problem_responses_extended.sql +++ b/models/problems/fact_problem_responses_extended.sql @@ -23,6 +23,7 @@ select results.responses as responses, results.success as success, results.attempts as attempts, + results.course_order as course_order, results.graded as graded, results.interaction_type as interaction_type, users.username as username, diff --git a/models/problems/int_problem_hints.sql b/models/problems/int_problem_hints.sql index 5fb689b2..e9d8c11a 100644 --- a/models/problems/int_problem_hints.sql +++ b/models/problems/int_problem_hints.sql @@ -26,6 +26,7 @@ select hints.problem_id as problem_id, blocks.block_name as problem_name, blocks.display_name_with_location as problem_name_with_location, + blocks.course_order as course_order, hints.actor_id as actor_id, hints.help_type as help_type from hints diff --git a/models/problems/int_problem_results.sql b/models/problems/int_problem_results.sql index c49c2474..bf7d89d8 100644 --- a/models/problems/int_problem_results.sql +++ b/models/problems/int_problem_results.sql @@ -48,6 +48,7 @@ select problem_id, problem_name, problem_name_with_location, + course_order, problem_link, actor_id, responses, diff --git a/models/video/fact_transcript_usage.sql b/models/video/fact_transcript_usage.sql index 97cf894f..dee79230 100644 --- a/models/video/fact_transcript_usage.sql +++ b/models/video/fact_transcript_usage.sql @@ -7,6 +7,7 @@ select transcripts.video_id as video_id, blocks.block_name as video_name, blocks.display_name_with_location as video_name_with_location, + blocks.course_order as course_order, transcripts.actor_id as actor_id, users.username as username, users.name as name, diff --git a/models/video/fact_video_engagement.sql b/models/video/fact_video_engagement.sql index d145ad57..32e050aa 100644 --- a/models/video/fact_video_engagement.sql +++ b/models/video/fact_video_engagement.sql @@ -24,6 +24,7 @@ select views.course_run, videos.section_with_name, videos.subsection_with_name, + videos.course_order, videos.item_count, views.actor_id, views.video_id, diff --git a/models/video/fact_video_plays.sql b/models/video/fact_video_plays.sql index 51427561..8ad09dc8 100644 --- a/models/video/fact_video_plays.sql +++ b/models/video/fact_video_plays.sql @@ -35,7 +35,8 @@ select users.name as name, users.email as email, blocks.section_with_name as section_with_name, - blocks.subsection_with_name as subsection_with_name + blocks.subsection_with_name as subsection_with_name, + blocks.course_order as course_order from plays join {{ ref("dim_course_blocks_extended") }} blocks From 0766e973b9ba90dd492ec3d263326e0dceefaa8e Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 30 Apr 2024 16:47:08 -0400 Subject: [PATCH 2/3] docs: Add documentation for course block sort order --- models/courses/schema.yml | 9 +++++++++ models/navigation/schema.yml | 12 ++++++++++++ models/problems/schema.yml | 21 +++++++++++++++++++++ models/video/schema.yml | 9 +++++++++ 4 files changed, 51 insertions(+) diff --git a/models/courses/schema.yml b/models/courses/schema.yml index bf006ac1..1b7f9889 100644 --- a/models/courses/schema.yml +++ b/models/courses/schema.yml @@ -40,6 +40,9 @@ models: - name: block_type data_type: String description: "The type of block. This can be a section, subsection, unit, or the block type" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: course_block_names description: "A table of course blocks with their names" @@ -59,6 +62,9 @@ models: - name: display_name_with_location data_type: String description: "The block's display name with section, subsection, and unit prepended to the name. This provides additional context when looking at block names and can help data consumers understand which block they are analyzing" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: course_names description: "A table of courses with their names" @@ -121,3 +127,6 @@ models: - name: subsection_with_name data_type: string description: "The name of the section this subsection belongs to, with subsection_number prepended" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" diff --git a/models/navigation/schema.yml b/models/navigation/schema.yml index 8527bd0f..698d772b 100644 --- a/models/navigation/schema.yml +++ b/models/navigation/schema.yml @@ -83,6 +83,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_navigation_dropoff description: "A view for analyzing the number of page visits per learner per section and subsection" @@ -120,6 +123,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_navigation_completion description: "A view for analyzing how many pages a learner has visited in a section or subsection" @@ -163,6 +169,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: int_pages_per_subsection description: "A view for analyzing the number of pages in each subsection" @@ -188,3 +197,6 @@ models: - name: page_count data_type: uint64 description: The number of pages in the associated subsection + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" diff --git a/models/problems/schema.yml b/models/problems/schema.yml index 740e291f..d065895d 100644 --- a/models/problems/schema.yml +++ b/models/problems/schema.yml @@ -53,6 +53,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_problem_responses description: "One record for each submitted response to a problem" @@ -114,6 +117,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: int_problem_hints description: "Internal table for problem hints" @@ -148,6 +154,9 @@ models: - name: help_type data_type: string description: "The type of help requested" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: int_problem_results description: "Internal table for problem results" @@ -197,6 +206,9 @@ models: - name: interaction_type data_type: string description: "The type of interaction - e.g. multiple choice" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: problem_events description: "Problem events" @@ -265,6 +277,9 @@ models: - name: item_count data_type: uint64 description: "The number of problems in this subsection" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_problem_engagement description: "A dataset with one record representing a problem attempted by a learner and the section and subsection that problem belongs to" @@ -308,6 +323,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_problem_responses_extended @@ -373,3 +391,6 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" diff --git a/models/video/schema.yml b/models/video/schema.yml index 38b5f24a..c5f2c1f9 100644 --- a/models/video/schema.yml +++ b/models/video/schema.yml @@ -61,6 +61,9 @@ models: - name: subsection_with_name data_type: string description: "The name of the subsection this video belongs to, with subsection_number prepended" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_transcript_usage description: "One record for each time a transcript or closed caption was enabled" @@ -101,6 +104,9 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: video_playback_events description: "Events related to video playback" @@ -228,3 +234,6 @@ models: - name: email data_type: String description: "The email address of the learner" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" From c579e19535907b7d10582f5b77f71557fc809af7 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 30 Apr 2024 20:51:11 -0400 Subject: [PATCH 3/3] docs: Add documentation for course block sort order --- models/video/schema.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/video/schema.yml b/models/video/schema.yml index c5f2c1f9..aa42767f 100644 --- a/models/video/schema.yml +++ b/models/video/schema.yml @@ -191,6 +191,9 @@ models: - name: item_count data_type: uint64 description: "The number of videos in this subsection" + - name: course_order + data_type: Int32 + description: "The sort order of this block in the course across all course blocks" - name: fact_video_engagement description: "A dataset with one record representing a video viewed by a learner and the section and subsection that video belongs to"