Merge pull request #119 from saraburns1/coursecompare

feat: MV for video watched data
openedx · Sep 6, 2024 · ce6ff25 · ce6ff25
2 parents 022a039 + 25374ef
commit ce6ff25
Show file tree

Hide file tree

Showing 8 changed files with 2,277 additions and 0 deletions.
diff --git a/models/video/schema.yml b/models/video/schema.yml
@@ -237,3 +237,28 @@ models:
                 - 'All videos viewed'
                 - 'At least one video viewed'
               table: 'xapi.subsection_video_engagement'
+
+  - name: watched_video_duration
+    description: "Amount of time in seconds watched by each actor for every video"
+    columns:
+      - name: org
+        data_type: string
+        description: "The organization that the course belongs to"
+      - name: course_key
+        data_type: string
+        description: "The course key for the course"
+      - name: actor_id
+        data_type: string
+        description: "The xAPI actor identifier"
+      - name: video_id
+        data_type: String
+        description: "The xAPI object identifier"
+      - name: video_duration
+        data_type: Int32
+        description: "Total duration of the video"
+      - name: watched_time
+        data_type: Int32
+        description: "Seconds of the video watched once"
+      - name: rewatched_time
+        data_type: Int32
+        description: "Seconds of the video rewatched"
diff --git a/models/video/unit_tests.yaml b/models/video/unit_tests.yaml
@@ -97,3 +97,17 @@ unit_tests:
       format: sql
       rows: |
         select * from fact_video_engagement_expected
+
+  - name: test_watched_video_duration
+    model: watched_video_duration
+    config:
+      tags: 'ci'
+    given:
+      - input: ref('video_playback_events')
+        format: sql
+        rows: |
+          select * from video_playback_events
+    expect:
+      format: sql
+      rows: |
+        select * from watched_video_duration_expected
diff --git a/models/video/watched_video_duration.sql b/models/video/watched_video_duration.sql
@@ -0,0 +1,83 @@
+{{
+    config(
+        materialized="materialized_view",
+        schema=env_var("ASPECTS_EVENT_SINK_DATABASE", "event_sink"),
+        engine=get_engine("ReplacingMergeTree()"),
+        order_by="(org,course_key,actor_id,video_id)",
+        post_hook="OPTIMIZE TABLE {{ this }} {{ on_cluster() }} FINAL",
+    )
+}}
+
+with
+    starts as (
+        select
+            org,
+            course_key,
+            actor_id,
+            emission_time,
+            cast(video_position as Int32) as start_position,
+            splitByString('/xblock/', object_id)[-1] as video_id,
+            video_duration
+        from {{ ref("video_playback_events") }}
+        where verb_id = 'https://w3id.org/xapi/video/verbs/played'
+    ),
+    rewatches as (
+        select org, course_key, video_id, actor_id, start_position
+        from starts
+        group by org, course_key, video_id, actor_id, start_position
+        having count(1) > 1
+    ),
+    ends as (
+        select
+            org,
+            course_key,
+            actor_id,
+            emission_time,
+            cast(video_position as Int32) as end_position,
+            splitByString('/xblock/', object_id)[-1] as video_id
+        from {{ ref("video_playback_events") }}
+        where
+            verb_id in (
+                'http://adlnet.gov/expapi/verbs/completed',
+                'https://w3id.org/xapi/video/verbs/seeked',
+                'https://w3id.org/xapi/video/verbs/paused',
+                'http://adlnet.gov/expapi/verbs/terminated'
+            )
+    ),
+    subtotals as (
+        select
+            starts.org as org,
+            starts.course_key as course_key,
+            starts.actor_id as actor_id,
+            starts.video_id as video_id,
+            starts.video_duration,
+            sum(starts.start_position - end_position) as duration,
+            case when rewatches.org = '' then duration else 0 end as watched_time,
+            case when rewatches.org <> '' then duration else 0 end as rewatched_time
+        from starts left
+        asof join
+            ends
+            on starts.org = ends.org
+            and starts.course_key = ends.course_key
+            and starts.video_id = ends.video_id
+            and starts.actor_id = ends.actor_id
+            and starts.emission_time < ends.emission_time
+        left join
+            rewatches
+            on rewatches.org = starts.org
+            and rewatches.course_key = starts.course_key
+            and rewatches.actor_id = starts.actor_id
+            and rewatches.start_position = starts.start_position
+            and rewatches.video_id = starts.video_id
+        group by org, course_key, actor_id, video_id, video_duration, rewatches.org
+    )
+select
+    org,
+    course_key,
+    actor_id,
+    video_id,
+    video_duration,
+    sum(watched_time) as watched_time,
+    sum(rewatched_time) as rewatched_time
+from subtotals
+group by org, course_key, actor_id, video_id, video_duration
diff --git a/unit-test-seeds/tags/object_tag.csv b/unit-test-seeds/tags/object_tag.csv
@@ -0,0 +1,45 @@
+id,object_id,taxonomy,tag,_value,_export_id,lineage,dump_id,time_last_dumped
+1,course-v1:OpenedX+DemoX+DemoCourse,1,11,Brass,2-taxonomy-template,"[""Wind instruments"", ""Brass""]",f032375f-c962-4944-a11b-656dee91a8b4,2024-08-29 19:30:50.300345+00:00
+1,course-v1:OpenedX+DemoX+DemoCourse,1,11,Brass,2-taxonomy-template,"[""Wind instruments"", ""Brass""]",7f112bc9-214c-4d10-a78b-40ad381952fa,2024-08-30 13:40:36.684866+00:00
+1,course-v1:OpenedX+DemoX+DemoCourse,1,11,Brass,2-taxonomy-template,"[""Wind instruments"", ""Brass""]",a496ee7e-d813-4920-82d9-9e2ed8f2e478,2024-08-30 13:53:39.550293+00:00
+2,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",65fdb449-14e5-42d6-9bf9-19111615e08f,2024-08-29 19:30:50.301607+00:00
+2,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",1ad8fcc5-1521-4662-830f-d154bc3c3efc,2024-08-30 13:40:36.686375+00:00
+2,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",124fdee7-7f3a-4d0f-8d34-8747d12c4972,2024-08-30 13:53:39.551775+00:00
+3,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,5,Idiophone,2-taxonomy-template,"[""Percussion instruments"", ""Idiophone""]",26089844-9035-483b-8f02-fb9088d3e376,2024-08-29 19:30:50.302867+00:00
+3,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,5,Idiophone,2-taxonomy-template,"[""Percussion instruments"", ""Idiophone""]",788c5158-466d-497f-922a-80a6c59bc1f0,2024-08-30 13:40:36.687661+00:00
+3,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,5,Idiophone,2-taxonomy-template,"[""Percussion instruments"", ""Idiophone""]",598dbfa1-c792-4402-b604-b9de3539fcb6,2024-08-30 13:53:39.553186+00:00
+4,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,4,String instruments,2-taxonomy-template,"[""String instruments""]",88c7e6ac-a118-40b9-8606-777236943c7d,2024-08-29 19:30:50.303416+00:00
+4,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,4,String instruments,2-taxonomy-template,"[""String instruments""]",6b56b578-0318-4a64-9333-7c2e6287a807,2024-08-30 13:40:36.688319+00:00
+4,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,4,String instruments,2-taxonomy-template,"[""String instruments""]",3e72c1a0-9e55-4ba6-9914-e005ac1b92b8,2024-08-30 13:53:39.553831+00:00
+5,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,1,Wind instruments,2-taxonomy-template,"[""Wind instruments""]",fdd390fa-b5d9-413f-bec0-2779179b1b5f,2024-08-29 19:30:50.303935+00:00
+5,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,1,Wind instruments,2-taxonomy-template,"[""Wind instruments""]",90ed5b3b-5d3e-4e1e-8e5b-bf5112464eb9,2024-08-30 13:40:36.688933+00:00
+5,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,1,Wind instruments,2-taxonomy-template,"[""Wind instruments""]",880c74b9-159e-4607-bc6d-8ec0ff7cee75,2024-08-30 13:53:39.554426+00:00
+6,course-v1:OpenedX+DemoX+DemoCourse,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",6130cddf-52b5-4c3f-927b-409dbe965b45,2024-08-29 19:30:50.304995+00:00
+6,course-v1:OpenedX+DemoX+DemoCourse,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",0a41187a-893f-4788-8fc7-3726b7838661,2024-08-30 13:40:36.690305+00:00
+6,course-v1:OpenedX+DemoX+DemoCourse,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",a3a7fb32-37af-4797-bb11-1b1a922ea4e3,2024-08-30 13:53:39.555720+00:00
+7,course-v1:OpenedX+DemoX+DemoCourse,1,10,Chordophone,2-taxonomy-template,"[""Percussion instruments"", ""Chordophone""]",c5924f0e-6a0e-46bb-9041-15288b80005a,2024-08-29 19:30:50.306114+00:00
+7,course-v1:OpenedX+DemoX+DemoCourse,1,10,Chordophone,2-taxonomy-template,"[""Percussion instruments"", ""Chordophone""]",2269c11a-5076-4019-9174-9ec16cdb1e57,2024-08-30 13:40:36.691633+00:00
+7,course-v1:OpenedX+DemoX+DemoCourse,1,10,Chordophone,2-taxonomy-template,"[""Percussion instruments"", ""Chordophone""]",665e6e49-9701-4d42-9691-a5a632f5be2f,2024-08-30 13:53:39.556989+00:00
+8,course-v1:OpenedX+DemoX+DemoCourseV2,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",03f6677a-2965-4153-b6f9-8fe531e1c3a6,2024-08-29 19:30:50.307152+00:00
+8,course-v1:OpenedX+DemoX+DemoCourseV2,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",7aeec4db-f75d-4b7e-aeb5-f88b1c644ef4,2024-08-30 13:40:36.692940+00:00
+8,course-v1:OpenedX+DemoX+DemoCourseV2,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",048ddc04-364f-4872-b454-de084e71df5c,2024-08-30 13:53:39.558386+00:00
+9,course-v1:OpenedX+DemoX+DemoCourseV2,1,2,Percussion instruments,2-taxonomy-template,"[""Percussion instruments""]",11cb83c2-1bb4-40dc-877c-fef2d46acf2f,2024-08-29 19:30:50.307669+00:00
+9,course-v1:OpenedX+DemoX+DemoCourseV2,1,2,Percussion instruments,2-taxonomy-template,"[""Percussion instruments""]",ab2a1a65-72de-4109-be41-88269b786295,2024-08-30 13:40:36.693523+00:00
+9,course-v1:OpenedX+DemoX+DemoCourseV2,1,2,Percussion instruments,2-taxonomy-template,"[""Percussion instruments""]",0ef96d5d-1389-4a6a-8c45-cee01643dc48,2024-08-30 13:53:39.558952+00:00
+10,course-v1:OpenedX+DemoX+DemoCourseV2,1,18,Bowed strings,2-taxonomy-template,"[""String instruments"", ""Bowed strings""]",b2065167-7387-410b-9e7c-c6924c434d9a,2024-08-29 19:30:50.309000+00:00
+10,course-v1:OpenedX+DemoX+DemoCourseV2,1,18,Bowed strings,2-taxonomy-template,"[""String instruments"", ""Bowed strings""]",8b3821b3-ce7f-4c14-8176-41e592a95395,2024-08-30 13:40:36.694824+00:00
+10,course-v1:OpenedX+DemoX+DemoCourseV2,1,18,Bowed strings,2-taxonomy-template,"[""String instruments"", ""Bowed strings""]",d547e53c-ad37-4421-87fe-30217c343367,2024-08-30 13:53:39.560199+00:00
+11,course-v1:OpenedX+DemoX+DemoCourse,1,20,Cello,2-taxonomy-template,"[""String instruments"", ""Bowed strings"", ""Cello""]",6ac922ff-33b2-4f47-9ddd-3414ff633813,2024-08-29 19:30:50.310486+00:00
+11,course-v1:OpenedX+DemoX+DemoCourse,1,20,Cello,2-taxonomy-template,"[""String instruments"", ""Bowed strings"", ""Cello""]",aafb5055-35f1-4263-bbc1-762ad2c40758,2024-08-30 13:40:36.696120+00:00
+11,course-v1:OpenedX+DemoX+DemoCourse,1,20,Cello,2-taxonomy-template,"[""String instruments"", ""Bowed strings"", ""Cello""]",f3a79132-f25b-4a82-b11e-5cce24f55afb,2024-08-30 13:53:39.561489+00:00
+1,course-v1:OpenedX+DemoX+DemoCourse,1,11,Brass,2-taxonomy-template,"[""Wind instruments"", ""Brass""]",e3b02341-7c30-4a46-ae80-6ea39e54c814,2024-08-30 14:30:19.841864+00:00
+2,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",aec2d28f-5bfe-48f9-b17d-02369f8d4bf9,2024-08-30 14:30:19.843134+00:00
+3,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,5,Idiophone,2-taxonomy-template,"[""Percussion instruments"", ""Idiophone""]",a4c3aa23-8309-4f95-be2f-64e5d1a9294c,2024-08-30 14:30:19.844339+00:00
+4,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,4,String instruments,2-taxonomy-template,"[""String instruments""]",50e0df35-cdbe-434c-af10-314ba46ac9ab,2024-08-30 14:30:19.844836+00:00
+5,block-v1:OpenedX+DemoX+DemoCourse+type@sequential+block@4e1de5e13fc3422997fe246b40a43aa1,1,1,Wind instruments,2-taxonomy-template,"[""Wind instruments""]",d23f8357-8861-4087-b360-3cf9dbed254a,2024-08-30 14:30:19.845307+00:00
+6,course-v1:OpenedX+DemoX+DemoCourse,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",b42a60bd-ae77-47bd-8958-e5d1b07cb6c7,2024-08-30 14:30:19.846428+00:00
+7,course-v1:OpenedX+DemoX+DemoCourse,1,10,Chordophone,2-taxonomy-template,"[""Percussion instruments"", ""Chordophone""]",fade0cde-03fa-4795-959f-d82911bd3820,2024-08-30 14:30:19.847496+00:00
+8,course-v1:OpenedX+DemoX+DemoCourseV2,1,25,Synthesizer,2-taxonomy-template,"[""Electronic instruments"", ""Synthesizer""]",8da9ac29-d5db-4ba0-8225-8260f4c538dd,2024-08-30 14:30:19.848562+00:00
+9,course-v1:OpenedX+DemoX+DemoCourseV2,1,2,Percussion instruments,2-taxonomy-template,"[""Percussion instruments""]",b712953c-0d80-453d-b88c-48b63937a1dc,2024-08-30 14:30:19.849021+00:00
+10,course-v1:OpenedX+DemoX+DemoCourseV2,1,18,Bowed strings,2-taxonomy-template,"[""String instruments"", ""Bowed strings""]",c07fcc7e-401a-4bf5-a857-252512c0d5a2,2024-08-30 14:30:19.850113+00:00
+11,course-v1:OpenedX+DemoX+DemoCourse,1,20,Cello,2-taxonomy-template,"[""String instruments"", ""Bowed strings"", ""Cello""]",d96c4a04-5102-45ed-966e-c768100cdfab,2024-08-30 14:30:19.851294+00:00
diff --git a/unit-test-seeds/tags/seeds.yaml b/unit-test-seeds/tags/seeds.yaml
@@ -0,0 +1,12 @@
+version: 2
+
+seeds:
+  - name: object_tag
+    config:
+      schema: event_sink
+  - name: tag
+    config:
+      schema: event_sink
+  - name: taxonomy
+    config:
+      schema: event_sink