4. emotion_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# regression analysis of what plays when it comes to emotions and engagement metrics and how they correlate with each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import pandas as pd\n",
    "\n",
    "from spark_session_manager import SparkSessionManager\n",
    "from pyspark.sql import functions as F\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark =  SparkSessionManager().get_spark_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----------+------------------------------------------------------------+--------------------+------------------------+-------------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+------------------+-----------------+------------------+----------------------------------------------+-----------------+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|categoryId|video_id   |title                                                       |publishedAt         |channelId               |channelTitle |trending_date       |tags                                                                                                                                                                                                                                                                                                                                                                                                                                                            |view_count        |likes             |dislikes         |comment_count     |thumbnail_link                                |comments_disabled|ratings_disabled|description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |categoryTitle |emotions                                                                                                                                                          |\n",
      "+----------+-----------+------------------------------------------------------------+--------------------+------------------------+-------------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+------------------+-----------------+------------------+----------------------------------------------+-----------------+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|22        |3C66w5Z0ixs|I ASKED HER TO BE MY GIRLFRIEND...                          |2020-08-11T19:20:14Z|UCvtRTOMP2TqYqu51xNrqAzg|Brawadis     |2020-08-12T00:00:00Z|brawadis|prank|basketball|skits|ghost|funny videos|vlog|vlogging|NBA|browadis|challenges|bmw i8|faze rug|faze rug brother|mama rug and papa rug                                                                                                                                                                                                                                                                                                                 |14.230671839217731|11.963421298452696|8.675222055641148|10.472034764862743|https://i.ytimg.com/vi/3C66w5Z0ixs/default.jpg|False            |False           |SUBSCRIBE to BRAWADIS ▶ http://bit.ly/SubscribeToBrawadis\\r\\rFOLLOW ME ON SOCIAL\\r▶ Twitter: https://twitter.com/Brawadis\\r▶ Instagram: https://www.instagram.com/brawadis/\\r▶ Snapchat: brawadis\\r\\rHi! I’m Brandon Awadis and I like to make dope vlogs, pranks, reactions, challenges and basketball videos. Don’t forget to subscribe and come be a part of the BrawadSquad!                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |People & Blogs|{0.502659022808075, 0.08973177522420883, 0.01276390627026558, 0.007058965973556042, 0.16834966838359833, 0.18087247014045715, 0.03856417536735535, anger}         |\n",
      "|20        |M9Pmf9AB4Mo|Apex Legends | Stories from the Outlands – “The Endorsement”|2020-08-11T17:00:10Z|UC0ZV6M2THA81QT9hrVWJG3A|Apex Legends |2020-08-12T00:00:00Z|Apex Legends|Apex Legends characters|new Apex Legend|Apex Legends Rampart|Apex Legends Season 6|Apex Legends Boosted|Battle Pass|Season 6 Battle Pass|Apex Legends new season|Apex Legends game|Respawn Apex Legends|Battle Royale game|Battle Royale|Battle Royale shooter|Apex Games|squad play|multiplayer shooter|Apex Legends PS4|Apex Legends Xbox|Apex Legends PC|Apex Legends Origin|Respawn Entertainment|Electronic Arts|Titanfall 2|fun battle royale|14.683320457820873|11.896417592267985|7.9355873855892  |9.71414138080521  |https://i.ytimg.com/vi/M9Pmf9AB4Mo/default.jpg|False            |False           |While running her own modding shop, Ramya Parekh (aka Rampart) built her rep in underground gauntlet circuits. But success has a way of making enemies. Good thing she knows how to use all those fancy weapons she’s upgraded…  Apex Legends™ is a free-to-play battle royale game where legendary characters battle for glory, fame, and fortune on the fringes of the Frontier. Play for free now on Xbox One, PS4, and Origin for PC. Learn more about Apex Legends Season 6 – Boosted: http://x.ea.com/64060 Check out our YouTube channel: http://x.ea.com/56710.Follow us on Twitter: https://twitter.com/playapex.Follow us on Instagram: https://www.instagram.com/playapex/.#MeetRampart #ApexSeason6 #ApexSeasonSix #ApexLegends                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |Gaming        |{0.027806859463453293, 0.517167329788208, 0.002150328131392598, 0.07300199568271637, 0.362636536359787, 0.008639412932097912, 0.008597424253821373, disgust}      |\n",
      "|24        |J78aPJ3VyNs|I left youtube for a month and THIS is what happened.       |2020-08-11T16:34:06Z|UCYzPXprvl5Y-Sf0g4vX-m6g|jacksepticeye|2020-08-12T00:00:00Z|jacksepticeye|funny|funny meme|memes|jacksepticeye memes|reddit|subreddit|community|community memes|community subreddit|jacksepticeye subreddit|reddit memes|fan submitted|spicy memes|funny pics|reaction|react|green screen|funny memes|funny green screen|dank memes|memes compilation|try not to laugh|meme|fresh memes|meme review|funny moments|bell memes|bell of meme|jacksepticeye bell                                                                |14.527898443265606|12.776453142665115|7.874358824729881|10.602169388594548|https://i.ytimg.com/vi/J78aPJ3VyNs/default.jpg|False            |False           |I left youtube for a month and this is what happenedMY COFFEE COMPANY: https://twitter.com/TOTMCoffeeJoin the subreddit: https://www.reddit.com/r/jacksepticeye/MORE MEMES ► https://www.youtube.com/watch?v=wGdn6ldQTTg&list=PLMBYlcH3smRxmCZzsUyrxB0IyKSQAU0pPJacksepticeye's Funniest Home Videos: https://www.youtube.com/watch?v=VqfLcdpBasY&list=PLMBYlcH3smRxOk7Cp_V2ar3QDfvljWvSyEdited by: https://twitter.com/DaveDelirious►Twitter : https://twitter.com/Jack_Septic_Eye►Instagram: http://instagram.com/jacksepticeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |Entertainment |{0.019757458940148354, 0.0056221061386168, 0.0036118978168815374, 0.0033497284166514874, 0.061444465070962906, 0.013827801682054996, 0.8923864960670471, surprise}|\n",
      "|10        |kXLn3HkpjaA|XXL 2020 Freshman Class Revealed - Official Announcement    |2020-08-11T16:38:55Z|UCbg_UMjlHJg_19SZckaKajg|XXL          |2020-08-12T00:00:00Z|xxl freshman|xxl freshmen|2020 xxl freshman|2020 freshman|2020 freshmen|xxl freshman class|2020 xxl freshman class|NLE Choppa|Polo G|Chika|Baby Keem|Mulatto|Jack Harlow|Rod Wave|Lil Tjay|Calboy|Fivio Foreign|Lil Keed|24kGoldn|rapper|rap|hip-hop|music                                                                                                                                                                                                      |13.115886447304547|10.054146428821241|7.526717561352706|8.942199454731238 |https://i.ytimg.com/vi/kXLn3HkpjaA/default.jpg|False            |False           |Subscribe to XXL → http://bit.ly/subscribe-xxl \\rXXL presents the 2020 XXL Freshman Class featuring NLE Choppa, Polo G, Chika, Baby Keem, Mulatto, Jack Harlow, Rod Wave, Lil Tjay, Calboy, Fivio Foreign, Lil Keed and 10th Spot winner 24kGoldn. Visit the 2020 XXL Freshman Class website → https://www.xxlmag.com/freshman/Credits:Directed by @travissattenProduced by @ritemediagroupMagazine cover shot by Travis ShinnCover collage and lettering created by @rich_tuBeat for video produced by Jetsonmade, Neeko Baby and 1st Class#XXLFreshman #XXLFreshman #HipHop #XXLFreshmanClass #XXLFreshmanCover\\rGo here → http://www.xxlmag.com/\\rXXL on Facebook → https://www.facebook.com/xxlmag\\rXXL on Twitter → https://twitter.com/XXL\\rXXL on Instagram → https://instagram.com/xxl/                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |Music         |{0.008223666809499264, 0.002281598746776581, 0.004607288632541895, 0.3518415093421936, 0.4248347580432892, 0.008033731952309608, 0.2001773864030838, neutral}     |\n",
      "|26        |VIUo6yapDbc|Ultimate DIY Home Movie Theater for The LaBrant Family!     |2020-08-11T15:10:05Z|UCDVPcEbVLQgLZX0Rt6jo34A|Mr. Kate     |2020-08-12T00:00:00Z|The LaBrant Family|DIY|Interior Design|Makeover|Decorating|DIY Movie Theater|Home Movie Theater|Movie Theater|Stay Home|Cole LaBrant|Savannah LaBrant|Kate Albrecht|Joey Zehr|mr. Kate|mr kate|luxurious|red|beautiful|LifeLock Norton 360|Norton|LifeLock|Real estate scams|Home title scams|Identity Theft|Cyber Security|Cyber Safety|lux|collab|omg we're coming over|youtubers|home makeover|nursery makeover|home theater                                 |13.93230643987802 |10.732104870140509|6.872128101338986|7.69484807238461  |https://i.ytimg.com/vi/VIUo6yapDbc/default.jpg|False            |False           |Transforming The LaBrant Family's empty white room into a dramatic custom home movie theater. A portion of this video was sponsored by NortonLifeLock. Time to help protect your home title with Norton 360 with LifeLock Ultimate Plus. #NoPryingEyes Go to https://bit.ly/3iio4um to learn more or visit http://www.facebook.com/LifeLock & http://www.youtube.com/LifeLock    + SUBSCRIBE to our channel!: http://bit.ly/mrkateyoutube+ FOLLOW us on Instagram!: http://www.instagram.com/mrkate and http://www.instagram.com/joeyzehr+ SUBSCRIBE to the LaBrant Fam's channel: https://www.youtube.com/channel/UC4-CH0epzZpD_ARhxCx6LaQWATCH The LaBrant Fam's Nursery Makeover Here: https://www.youtube.com/watch?v=ZhoRLt42AmsWATCH how this collab all started here: https://youtu.be/Xi_F4TKJE5Y#INTERIORDESIGN #HOMETHEATER #MAKEOVER #MRKATE #THELABRANTFAMPIN PICS FROM OUR BLOG!: https://mrkate.com/2020/08/11/ultimate-diy-home-movie-theater-for-the-labrant-family/_________________________Instagram: http://www.instagram.com/mrkate and http://www.instagram.com/joeyzehrTik Tok: @mrkate Snapchat: MrKateTwitter: http://www.twitter.com/mrkateFacebook: http://www.facebook.com/mrkatedotcomPinterest: http://www.pinterest.com/mrkate/Become a member for exclusive VLOGS, LIVE CHATS, EMOJIS, BADGES and DISCOUNT CODES: https://www.youtube.com/channel/UCDVPcEbVLQgLZX0Rt6jo34A/join  _________________________Shop the entire video!: Chaise Lounge: https://bit.ly/3kzL1vlSofa: https://bit.ly/3fNAMQrSideboard: https://bit.ly/33R36yT Wallpaper: https://bit.ly/2DWAdpWWoven Poufs: https://bit.ly/39muMwe Curtains: https://amzn.to/3fIUf4B Nesting Black Tables: https://amzn.to/3fNje70 Sconces (similar): https://amzn.to/3fFPTeE Popcorn Machine (similar): https://bit.ly/2PGUS47 Drink Table: https://bit.ly/3gXUj1G Throw Blanket: https://bit.ly/3gOgV4N Terracotta Pillow: https://bit.ly/3eSXiGZ Jam Velvet Pillows: https://amzn.to/3ktNlnD Burgundy Velvet Pillows: https://amzn.to/3aekTRE Taupe Velvet Pillows: https://amzn.to/3kzIW2b Movie Clap Board: https://amzn.to/33Q1mpA Ticket Shadow Box: https://amzn.to/2XPHijs Popcorn Bags: https://amzn.to/3iymKDN _________________________Creative Credit: Audio Tracks - Epidemic Sound: http://share.epidemicsound.com/rGjgCA Mr. Kate ProductionExecutive Producers: Kate Albrecht and Joey ZehrSupervising Producer: AJ TeslerProducer: Annie O'RourkeCamera: Marco BottiglieriArt Department: Leo Martinez, Gabriel Gonzalez, Kenny NogueiraPA: Amy PattersonEdited by: Vianne Robitaille|Howto & Style |{0.009643401019275188, 0.0041331034153699875, 0.0028734798543155193, 0.6555830240249634, 0.1650134027004242, 0.009372318163514137, 0.15338127315044403, joy}      |\n",
      "+----------+-----------+------------------------------------------------------------+--------------------+------------------------+-------------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+------------------+-----------------+------------------+----------------------------------------------+-----------------+----------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 5 rows\n",
      "\n",
      "root\n",
      " |-- categoryId: string (nullable = true)\n",
      " |-- video_id: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- publishedAt: string (nullable = true)\n",
      " |-- channelId: string (nullable = true)\n",
      " |-- channelTitle: string (nullable = true)\n",
      " |-- trending_date: string (nullable = true)\n",
      " |-- tags: string (nullable = true)\n",
      " |-- view_count: double (nullable = true)\n",
      " |-- likes: double (nullable = true)\n",
      " |-- dislikes: double (nullable = true)\n",
      " |-- comment_count: double (nullable = true)\n",
      " |-- thumbnail_link: string (nullable = true)\n",
      " |-- comments_disabled: string (nullable = true)\n",
      " |-- ratings_disabled: string (nullable = true)\n",
      " |-- description: string (nullable = true)\n",
      " |-- categoryTitle: string (nullable = true)\n",
      " |-- emotions: struct (nullable = true)\n",
      " |    |-- anger: double (nullable = true)\n",
      " |    |-- disgust: double (nullable = true)\n",
      " |    |-- fear: double (nullable = true)\n",
      " |    |-- joy: double (nullable = true)\n",
      " |    |-- neutral: double (nullable = true)\n",
      " |    |-- sadness: double (nullable = true)\n",
      " |    |-- surprise: double (nullable = true)\n",
      " |    |-- highest_confidence_emotion: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# load emotion data -> data/processed/emotion_analysis_results.parquet\n",
    "df = spark.read.parquet(\"data/results/emotion-english-distilroberta-base.parquet\")\n",
    "\n",
    "# show head\n",
    "df.show(5, truncate=False)\n",
    "\n",
    "# show schema\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split column emotion into multiple columns\n",
    "#  |-- emotions: struct (nullable = true)\n",
    "#  |    |-- anger: double (nullable = true)\n",
    "#  |    |-- disgust: double (nullable = true)\n",
    "#  |    |-- fear: double (nullable = true)\n",
    "#  |    |-- joy: double (nullable = true)\n",
    "#  |    |-- neutral: double (nullable = true)\n",
    "#  |    |-- sadness: double (nullable = true)\n",
    "#  |    |-- surprise: double (nullable = true)\n",
    "#  |    |-- highest_confidence_emotion: string (nullable = true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+---------------------+---------------------+---------------------+---------------------+--------------------+---------------------+--------------------+--------------------------+\n",
      "|video_id   |anger                |disgust              |fear                 |joy                  |neutral             |sadness              |surprise            |highest_confidence_emotion|\n",
      "+-----------+---------------------+---------------------+---------------------+---------------------+--------------------+---------------------+--------------------+--------------------------+\n",
      "|3C66w5Z0ixs|0.502659022808075    |0.08973177522420883  |0.01276390627026558  |0.007058965973556042 |0.16834966838359833 |0.18087247014045715  |0.03856417536735535 |anger                     |\n",
      "|M9Pmf9AB4Mo|0.027806859463453293 |0.517167329788208    |0.002150328131392598 |0.07300199568271637  |0.362636536359787   |0.008639412932097912 |0.008597424253821373|disgust                   |\n",
      "|J78aPJ3VyNs|0.019757458940148354 |0.0056221061386168   |0.0036118978168815374|0.0033497284166514874|0.061444465070962906|0.013827801682054996 |0.8923864960670471  |surprise                  |\n",
      "|kXLn3HkpjaA|0.008223666809499264 |0.002281598746776581 |0.004607288632541895 |0.3518415093421936   |0.4248347580432892  |0.008033731952309608 |0.2001773864030838  |neutral                   |\n",
      "|VIUo6yapDbc|0.009643401019275188 |0.0041331034153699875|0.0028734798543155193|0.6555830240249634   |0.1650134027004242  |0.009372318163514137 |0.15338127315044403 |joy                       |\n",
      "|w-aidBdvZo8|0.03113676980137825  |0.03746941313147545  |0.017222099006175995 |0.008369309827685356 |0.6986414790153503  |0.12418948858976364  |0.08297140151262283 |neutral                   |\n",
      "|uet14uf9NsE|0.028064265847206116 |0.005761170294135809 |0.02652762457728386  |0.13668252527713776  |0.1168602705001831  |0.0019821105524897575|0.6841220259666443  |surprise                  |\n",
      "|ua4QMFQATco|0.09544301778078079  |0.04972346872091293  |0.03360289707779884  |0.0017893320182338357|0.3825795352458954  |0.4090122878551483   |0.027849474921822548|sadness                   |\n",
      "|SnsPZj91R7E|0.35638511180877686  |0.14808732271194458  |0.02722185291349888  |0.009189533069729805 |0.2436685711145401  |0.11404307931661606  |0.10140455514192581 |anger                     |\n",
      "|SsWHMAhshPQ|0.006590342614799738 |0.003871083026751876 |0.003730718744918704 |0.2510296106338501   |0.5705435872077942  |0.027602721005678177 |0.13663193583488464 |neutral                   |\n",
      "|49Z6Mv4_WCA|0.015745170414447784 |0.0020526389125734568|0.10015356540679932  |0.012569636106491089 |0.007295012474060059|0.6778152585029602   |0.18436875939369202 |sadness                   |\n",
      "|nt3VVyv5pxQ|0.0426550917327404   |0.011099148541688919 |0.009420313872396946 |0.11888450384140015  |0.6203137040138245  |0.10273852199316025  |0.09488870948553085 |neutral                   |\n",
      "|I6hswz4rIrU|0.005457071587443352 |0.002839075168594718 |0.004984239116311073 |0.09081445634365082  |0.8380826115608215  |0.007579706609249115 |0.050242893397808075|neutral                   |\n",
      "|W7VK4DUHvKU|0.018722908571362495 |0.006180803757160902 |0.0016876996960490942|0.3559630513191223   |0.5577959418296814  |0.023825753480196    |0.03582380339503288 |neutral                   |\n",
      "|W9Aen8hG20Y|0.0036680512130260468|0.047764964401721954 |0.004543094430118799 |0.0129720913246274   |0.045674700289964676|0.8735657334327698   |0.011811308562755585|sadness                   |\n",
      "|BNeDH6UTmXw|0.009630213491618633 |0.009518390521407127 |0.012965098023414612 |0.2674886882305145   |0.5286633372306824  |0.026481829583644867 |0.14525245130062103 |neutral                   |\n",
      "|6TIsR_7nrNc|0.5957915782928467   |0.016748659312725067 |0.018198970705270767 |0.004715076647698879 |0.25173795223236084 |0.08047379553318024  |0.032334040850400925|anger                     |\n",
      "|gPdUslndvVI|0.6720927357673645   |0.04468672722578049  |0.02751859277486801  |0.001378105254843831 |0.01982218399643898 |0.22463546693325043  |0.009866191074252129|anger                     |\n",
      "|GTp-0S82guE|0.006002509035170078 |0.012143818661570549 |0.00792758259922266  |0.104609914124012    |0.7700181603431702  |0.06062386557459831  |0.03867412731051445 |neutral                   |\n",
      "|jbGRowa5tIk|0.01637415774166584  |0.034219857305288315 |0.1880592703819275   |0.010713775642216206 |0.7046470046043396  |0.027694595977663994 |0.0182912889868021  |neutral                   |\n",
      "+-----------+---------------------+---------------------+---------------------+---------------------+--------------------+---------------------+--------------------+--------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import col\n",
    "\n",
    "# Assuming your DataFrame is named 'df'\n",
    "# Extract each field from the struct to a separate column\n",
    "df = df.withColumn(\"anger\", col(\"emotions.anger\")) \\\n",
    "       .withColumn(\"disgust\", col(\"emotions.disgust\")) \\\n",
    "       .withColumn(\"fear\", col(\"emotions.fear\")) \\\n",
    "       .withColumn(\"joy\", col(\"emotions.joy\")) \\\n",
    "       .withColumn(\"neutral\", col(\"emotions.neutral\")) \\\n",
    "       .withColumn(\"sadness\", col(\"emotions.sadness\")) \\\n",
    "       .withColumn(\"surprise\", col(\"emotions.surprise\")) \\\n",
    "       .withColumn(\"highest_confidence_emotion\", col(\"emotions.highest_confidence_emotion\"))\n",
    "\n",
    "\n",
    "# select 'video_id' and emotions columns\n",
    "df = df.select('video_id', 'anger', 'disgust', 'fear', 'joy', 'neutral', 'sadness', 'surprise', 'highest_confidence_emotion')\n",
    "\n",
    "# Show the modified DataFrame to verify the changes\n",
    "df.show(truncate=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# save df as parquet\n",
    "df.write.mode(\"overwrite\").parquet(\"data/results/emotion-english-distilroberta-base-split_min.parquet\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To predict view counts based on the levels of detected emotions in video content, you can build a regression model using the PySpark MLlib library. This involves a few key steps:\n",
    "\n",
    "* Preparing the Data: Format your data into a features vector and a label (target variable).\n",
    "* Splitting the Data: Divide your data into training and test sets.\n",
    "* Building the Model: Choose a regression model and train it on your data.\n",
    "* Evaluating the Model: Assess the model's performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load engagement data -> data/processed/engage_metrics_data.parquet\n",
    "df_engage = spark.read.parquet(\"data/results/engage_metrics_data.parquet\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparing the Data\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml import Pipeline\n",
    "\n",
    "# Define the assembler\n",
    "assembler = VectorAssembler(inputCols=[\"anger\", \"disgust\", \"fear\", \"joy\", \"neutral\", \"sadness\", \"surprise\"], outputCol=\"features\")\n",
    "\n",
    "# Define the label column\n",
    "df = df.withColumnRenamed(\"view_count\", \"label\")\n",
    "\n",
    "# Assemble the features\n",
    "data = assembler.transform(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data = data.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the LinearRegression model\n",
    "lr = LinearRegression(featuresCol='features', labelCol='label')\n",
    "\n",
    "# Train the model\n",
    "lr_model = lr.fit(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions\n",
    "predictions = lr_model.transform(test_data)\n",
    "\n",
    "# Evaluate the model\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "evaluator = RegressionEvaluator(labelCol=\"label\", predictionCol=\"prediction\", metricName=\"rmse\")\n",
    "\n",
    "# Calculate RMSE\n",
    "rmse = evaluator.evaluate(predictions)\n",
    "\n",
    "print(f\"Root Mean Squared Error (RMSE) on test data = {rmse}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "## evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build imporved model (maybe with interactions?)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation of eng. metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### basic model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare data\n",
    "\n",
    "## Removing Rows with Null Values\n",
    "df = df.na.drop(subset=[\"comment_count\", \"view_count\", \"likes\", \"dislikes\"]) # This approach will help ensure that the VectorAssembler does not encounter null values, which should prevent the Py4JJavaError you're seeing.\n",
    "\n",
    "## Filling Null Values\n",
    "# df_subset = df_subset.na.fill(value=0, subset=[\"comment_count\", \"view_count\", \"likes\", \"dislikes\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "# Step 3: Assemble features\n",
    "feature_columns = [\"comment_rate\", \"like_rate\", \"dislike_rate\", \"dislike_ratio\", \"controversy_index\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Using handleInvalid Parameter in VectorAssembler\n",
    "assembler = VectorAssembler(\n",
    "    inputCols=[\"comment_rate\", \"like_rate\", \"dislike_rate\", \"dislike_ratio\", \"controversy_index\"],\n",
    "    outputCol=\"features\",\n",
    "    handleInvalid=\"skip\"\n",
    ")\n",
    "\n",
    "# Transform the DataFrame to include a features vector column\n",
    "df_features = assembler.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4: Create and train the model\n",
    "lr = LinearRegression(featuresCol=\"features\", labelCol=\"view_count\")\n",
    "\n",
    "# Split data into training and test sets\n",
    "train_data, test_data = df_features.randomSplit([0.7, 0.3], seed=42)\n",
    "\n",
    "# Fit the model on the training data\n",
    "lr_model = lr.fit(train_data)\n",
    "\n",
    "# Make predictions on the test data\n",
    "predictions = lr_model.transform(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Model Effectiveness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "# Create an evaluator for RMSE\n",
    "evaluator_rmse = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"view_count\", metricName=\"rmse\")\n",
    "\n",
    "# Evaluate RMSE\n",
    "rmse = evaluator_rmse.evaluate(predictions)\n",
    "print(\"Root Mean Squared Error (RMSE) on test data =\", rmse)\n",
    "\n",
    "# If you want to calculate R-squared\n",
    "evaluator_r2 = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"view_count\", metricName=\"r2\")\n",
    "r2 = evaluator_r2.evaluate(predictions)\n",
    "print(\"R-squared (R2) on test data =\", r2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checks\n",
    "\n",
    "## coefficients with their names\n",
    "print(\"## Coefficients\")\n",
    "for i, feature in enumerate(feature_columns):\n",
    "    print(feature, \":\", lr_model.coefficients[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model with interaction terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, OneHotEncoder, VectorAssembler, RFormula\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.sql.functions import col\n",
    "\n",
    "# Handling null values (assuming 'df_subset' already has engagement metrics calculated)\n",
    "df = df.na.drop()  # Dropping rows with any nulls\n",
    "\n",
    "# RFormula to automatically include interactions (example formula, adjust accordingly)\n",
    "r_formula = RFormula(formula=\"view_count ~ categoryId:emotion + comment_rate + dislike_rate + dislike_ratio + controversy_index + categoryId + emotion\")\n",
    "\n",
    "# Linear Regression model\n",
    "lr = LinearRegression(featuresCol=\"features\", labelCol=\"view_count\")\n",
    "\n",
    "# Define Pipeline\n",
    "pipeline = Pipeline(stages=[r_formula, lr])\n",
    "\n",
    "# Split the data into training and test sets\n",
    "train_data, test_data = df.randomSplit([0.7, 0.3], seed=42)\n",
    "\n",
    "# Fit the pipeline to the training data\n",
    "model = pipeline.fit(train_data)\n",
    "\n",
    "# Make predictions on the test data\n",
    "predictions = model.transform(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "# Create an evaluator for RMSE\n",
    "evaluator_rmse = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"view_count\", metricName=\"rmse\")\n",
    "\n",
    "# Evaluate RMSE\n",
    "rmse = evaluator_rmse.evaluate(predictions)\n",
    "print(\"Root Mean Squared Error (RMSE) on test data =\", rmse)\n",
    "\n",
    "# If you want to calculate R-squared\n",
    "evaluator_r2 = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"view_count\", metricName=\"r2\")\n",
    "r2 = evaluator_r2.evaluate(predictions)\n",
    "print(\"R-squared (R2) on test data =\", r2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'model' is the fitted pipeline model\n",
    "lr_model = model.stages[-1]  # The last stage in the pipeline is the LinearRegression model\n",
    "\n",
    "# Extracting coefficients and intercept\n",
    "coefficients = lr_model.coefficients\n",
    "intercept = lr_model.intercept\n",
    "\n",
    "print(\"Intercept: \", intercept)\n",
    "print(\"Coefficients: \")\n",
    "    \n",
    "for i, feature in enumerate(feature_columns):\n",
    "    print(feature, \":\", coefficients[i])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}