Vine_Review_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Vine_Review_Analysis.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C-02LMoz9sWK",
        "outputId": "084c28f1-2a54-4ea7-b235-21e4e2370a27"
      },
      "source": [
        "import os\n",
        "# Find the latest version of spark 2.0  from http://www-us.apache.org/dist/spark/ and enter as the spark version\n",
        "# For example:\n",
        "# spark_version = 'spark-3.0.0'\n",
        "spark_version = 'spark-3.1.1'\n",
        "os.environ['SPARK_VERSION']=spark_version\n",
        "\n",
        "# Install Spark and Java\n",
        "!apt-get update\n",
        "!apt-get install openjdk-11-jdk-headless -qq > /dev/null\n",
        "!wget -q http://www-us.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz\n",
        "!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "\n",
        "# Set Environment Variables\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = f\"/content/{spark_version}-bin-hadoop2.7\"\n",
        "\n",
        "# Start a SparkSession\n",
        "import findspark\n",
        "findspark.init()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Get:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release [697 B]\n",
            "Hit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:6 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release.gpg [836 B]\n",
            "Get:9 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:12 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,116 kB]\n",
            "Ign:15 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages\n",
            "Get:15 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages [741 kB]\n",
            "Hit:16 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/multiverse amd64 Packages [31.6 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,181 kB]\n",
            "Hit:19 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:20 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,756 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [426 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,546 kB]\n",
            "Get:23 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [899 kB]\n",
            "Fetched 11.0 MB in 6s (1,756 kB/s)\n",
            "Reading package lists... Done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22acXNax913H"
      },
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.appName('Vine_Review_Analysis').getOrCreate()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sAqdSvW9-XFh",
        "outputId": "1c6be7f4-bf73-4bcd-f1cb-69e856c577a8"
      },
      "source": [
        "from pyspark import SparkFiles\n",
        "url = \"https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Video_Games_v1_00.tsv.gz\"\n",
        "spark.sparkContext.addFile(url)\n",
        "df = spark.read.option(\"encoding\", \"UTF-8\").csv(SparkFiles.get(\"amazon_reviews_us_Video_Games_v1_00.tsv.gz\"), sep=\"\\t\", header=True, inferSchema=True)\n",
        "df.show()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|         US|   12039526| RTIS3L2M1F5SM|B001CXYMFS|     737716809|Thrustmaster T-Fl...|     Video Games|          5|            0|          0|   N|                Y|an amazing joysti...|Used this for Eli...| 2015-08-31|\n",
            "|         US|    9636577| R1ZV7R40OLHKD|B00M920ND6|     569686175|Tonsee 6 buttons ...|     Video Games|          5|            0|          0|   N|                Y|Definitely a sile...|Loved it,  I didn...| 2015-08-31|\n",
            "|         US|    2331478|R3BH071QLH8QMC|B0029CSOD2|      98937668|Hidden Mysteries:...|     Video Games|          1|            0|          1|   N|                Y|            One Star|poor quality work...| 2015-08-31|\n",
            "|         US|   52495923|R127K9NTSXA2YH|B00GOOSV98|      23143350|GelTabz Performan...|     Video Games|          3|            0|          0|   N|                Y|good, but could b...|nice, but tend to...| 2015-08-31|\n",
            "|         US|   14533949|R32ZWUXDJPW27Q|B00Y074JOM|     821342511|Zero Suit Samus a...|     Video Games|          4|            0|          0|   N|                Y|   Great but flawed.|Great amiibo, gre...| 2015-08-31|\n",
            "|         US|    2377552|R3AQQ4YUKJWBA6|B002UBI6W6|     328764615|Psyclone Recharge...|     Video Games|          1|            0|          0|   N|                Y|            One Star|The remote consta...| 2015-08-31|\n",
            "|         US|   17521011|R2F0POU5K6F73F|B008XHCLFO|      24234603|Protection for yo...|     Video Games|          5|            0|          0|   N|                Y|              A Must|I have a 2012-201...| 2015-08-31|\n",
            "|         US|   19676307|R3VNR804HYSMR6|B00BRA9R6A|     682267517|   Nerf 3DS XL Armor|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Perfect, kids lov...| 2015-08-31|\n",
            "|         US|     224068| R3GZTM72WA2QH|B009EPWJLA|     435241890|One Piece: Pirate...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|            Excelent| 2015-08-31|\n",
            "|         US|   48467989| RNQOY62705W1K|B0000AV7GB|     256572651|Playstation 2 Dan...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|Slippery but expe...| 2015-08-31|\n",
            "|         US|     106569|R1VTIA3JTYBY02|B00008KTNN|     384411423|Metal Arms: Glitc...|     Video Games|          5|            0|          0|   N|                N|          Five Stars|Love the game. Se...| 2015-08-31|\n",
            "|         US|   48269642|R29DOU8791QZL8|B000A3IA0Y|     472622859|72 Pin Connector ...|     Video Games|          1|            0|          0|   N|                Y| Game will get stuck|Does not fit prop...| 2015-08-31|\n",
            "|         US|   52738710|R15DUT1VIJ9RJZ|B0053BQN34|     577628462|uDraw Gametablet ...|     Video Games|          2|            0|          0|   N|                Y|We have tried it ...|This was way too ...| 2015-08-31|\n",
            "|         US|   10556786|R3IMF2MQ3OU9ZM|B002I0HIMI|     988218515|NBA 2K12(Covers M...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|Works great good ...| 2015-08-31|\n",
            "|         US|    2963837|R23H79DHOZTYAU|B0081EH12M|     770100932|New Trigger Grips...|     Video Games|          1|            1|          1|   N|                Y|Now i have to buy...|It did not fit th...| 2015-08-31|\n",
            "|         US|   23092109| RIV24EQAIXA4O|B005FMLZQQ|      24647669|Xbox 360 Media Re...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|perfect lightweig...| 2015-08-31|\n",
            "|         US|   23091728|R3UCNGYDVN24YB|B002BSA388|      33706205|Super Mario Galaxy 2|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|               great| 2015-08-31|\n",
            "|         US|   10712640| RUL4H4XTTN2DY|B00BUSLSAC|     829667834|Nintendo 3DS XL -...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Works beautifully...| 2015-08-31|\n",
            "|         US|   17455376|R20JF7Z4DHTNX5|B00KWF38AW|     110680188|Captain Toad:  Tr...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Kids loved the ga...| 2015-08-31|\n",
            "|         US|   14754850|R2T1AJ5MFI2260|B00BRQJYA8|     616463426|Lego Batman 2: DC...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|           Goodngame| 2015-08-31|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6iy1NCB-eqE",
        "outputId": "6bee0397-a39c-41c6-de8f-d8443337a9ff"
      },
      "source": [
        "# Create the vine_table. DataFrame\n",
        "vine_df = df.select(['review_id', 'star_rating', 'helpful_votes', 'total_votes', 'vine', 'verified_purchase'])\n",
        "vine_df.show()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| RTIS3L2M1F5SM|          5|            0|          0|   N|                Y|\n",
            "| R1ZV7R40OLHKD|          5|            0|          0|   N|                Y|\n",
            "|R3BH071QLH8QMC|          1|            0|          1|   N|                Y|\n",
            "|R127K9NTSXA2YH|          3|            0|          0|   N|                Y|\n",
            "|R32ZWUXDJPW27Q|          4|            0|          0|   N|                Y|\n",
            "|R3AQQ4YUKJWBA6|          1|            0|          0|   N|                Y|\n",
            "|R2F0POU5K6F73F|          5|            0|          0|   N|                Y|\n",
            "|R3VNR804HYSMR6|          5|            0|          0|   N|                Y|\n",
            "| R3GZTM72WA2QH|          5|            0|          0|   N|                Y|\n",
            "| RNQOY62705W1K|          4|            0|          0|   N|                Y|\n",
            "|R1VTIA3JTYBY02|          5|            0|          0|   N|                N|\n",
            "|R29DOU8791QZL8|          1|            0|          0|   N|                Y|\n",
            "|R15DUT1VIJ9RJZ|          2|            0|          0|   N|                Y|\n",
            "|R3IMF2MQ3OU9ZM|          4|            0|          0|   N|                Y|\n",
            "|R23H79DHOZTYAU|          1|            1|          1|   N|                Y|\n",
            "| RIV24EQAIXA4O|          5|            0|          0|   N|                Y|\n",
            "|R3UCNGYDVN24YB|          5|            0|          0|   N|                Y|\n",
            "| RUL4H4XTTN2DY|          5|            0|          0|   N|                Y|\n",
            "|R20JF7Z4DHTNX5|          5|            0|          0|   N|                Y|\n",
            "|R2T1AJ5MFI2260|          4|            0|          0|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "08tuSAP6-z7q",
        "outputId": "d800c20b-b3b5-4c6d-de72-f09a661e367f"
      },
      "source": [
        "# Filter the data and create a new DataFrame or table to retrieve all the rows where the total_votes count is equal to or greater than 20 \n",
        "filtered_df = vine_df.filter(vine_df['total_votes'] > 20)\n",
        "filtered_df.show()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R3EZ0EPYLDA34S|          1|           14|         31|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "|R1U3AR67RE273L|          1|           51|         65|   N|                Y|\n",
            "|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|\n",
            "| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|\n",
            "|R36O341WWXXKNP|          5|           28|         31|   N|                N|\n",
            "|R3GSK9MM8DNOYI|          1|            4|         32|   N|                N|\n",
            "|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|\n",
            "|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|\n",
            "|R3GUUC6NT4QMCL|          1|            5|         36|   N|                N|\n",
            "|R1ZSD020RT18I5|          1|            7|        139|   N|                N|\n",
            "|R113X48B575ASY|          1|           15|         45|   N|                N|\n",
            "| RJLYLGRYH7SFQ|          1|           34|        116|   N|                N|\n",
            "|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|\n",
            "|R3CICXKAHN6NUD|          1|            0|         23|   N|                Y|\n",
            "|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|\n",
            "| R8WD0136CETA7|          2|           99|        119|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PDGVdDIO_kJz",
        "outputId": "a7e124da-661b-4962-e38b-edb29520a528"
      },
      "source": [
        "# Filter the new DataFrame or table created in Step 1 and create a new DataFrame or table to retrieve all the rows where the number of helpful_votes divided by total_votes is equal to or greater than 50%\n",
        "helpful_df = filtered_df.filter((filtered_df['helpful_votes']/filtered_df['total_votes']) >= 0.5)\n",
        "helpful_df.show()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "|R1U3AR67RE273L|          1|           51|         65|   N|                Y|\n",
            "|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|\n",
            "| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|\n",
            "|R36O341WWXXKNP|          5|           28|         31|   N|                N|\n",
            "|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|\n",
            "|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|\n",
            "|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|\n",
            "|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|\n",
            "| R8WD0136CETA7|          2|           99|        119|   N|                N|\n",
            "|R16621F16PQN5A|          3|           24|         31|   N|                Y|\n",
            "|R1LXBPG1M9DDOC|          1|          102|        106|   N|                N|\n",
            "| RNEMGSUKJ4535|          2|           21|         25|   N|                Y|\n",
            "|R3R2UKBFKP1IQ4|          1|           19|         23|   N|                Y|\n",
            "|R2T90XM5G2DBZA|          3|           15|         22|   N|                Y|\n",
            "| R5ZMK0Y0KRBRZ|          1|           79|         91|   N|                N|\n",
            "|R12E2W0XOV3I2S|          1|           24|         30|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yYGS5uv1ANRo",
        "outputId": "83a04cac-05a1-41da-e16e-9967879a0d4f"
      },
      "source": [
        "# Filter the DataFrame or table created in Step 2, and create a new DataFrame or table that retrieves all the rows where a review was written as part of the Vine program (paid), vine == 'Y'.\n",
        "paid_review_df = helpful_df.filter(helpful_df['vine'] == 'Y')\n",
        "paid_review_df.show()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|R3KKUSGFZWSUIY|          5|           56|         63|   Y|                N|\n",
            "|R10FO5UKKVZBK2|          3|           23|         23|   Y|                N|\n",
            "| RM4KSGEOR7MU1|          5|           19|         24|   Y|                N|\n",
            "| RG7VRMYLEXD23|          4|           22|         26|   Y|                N|\n",
            "|R11O4YSCPSNL6L|          3|           20|         26|   Y|                N|\n",
            "|R286MFBAJ8NPD6|          5|           46|         51|   Y|                N|\n",
            "|R1JRR530H4COA2|          5|           22|         28|   Y|                N|\n",
            "| RQ5WD90PUNBU9|          5|           21|         24|   Y|                N|\n",
            "|R12648VHCQWUV9|          4|           21|         28|   Y|                N|\n",
            "|R1OWK33OPI45KT|          5|           55|         58|   Y|                N|\n",
            "|R2UP7VTED8O425|          5|           35|         41|   Y|                N|\n",
            "|R3EPHK0JGQGE91|          5|           21|         28|   Y|                N|\n",
            "| RCP3KRO47S0GG|          4|           51|         58|   Y|                N|\n",
            "|R3JT907UUA1WUI|          4|           85|        103|   Y|                N|\n",
            "| RY0555ILXIYXW|          5|           38|         50|   Y|                N|\n",
            "|R24PS9N276OXF4|          5|          241|        262|   Y|                N|\n",
            "| ROXZOMHBY8Y9S|          4|           21|         23|   Y|                N|\n",
            "|R1L8N6VFY5KDSU|          5|           23|         26|   Y|                N|\n",
            "|R3T4LTJ189G0LT|          4|           37|         45|   Y|                N|\n",
            "| R4LQC6M9AVXAY|          5|           69|         75|   Y|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MKx1JtQ-BN4M",
        "outputId": "26acdec1-ac5b-47a8-8809-a68ece2f5929"
      },
      "source": [
        "# Repeat Step 3, but this time retrieve all the rows where the review was not part of the Vine program (unpaid), vine == 'N'.\n",
        "unpaid_review_df = helpful_df.filter(helpful_df['vine'] == 'N')\n",
        "unpaid_review_df.show()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| R4PKAZRQJJX14|          1|           21|         34|   N|                N|\n",
            "|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|\n",
            "|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|\n",
            "|R2FJ94555FZH32|          2|           55|         60|   N|                N|\n",
            "|R1U3AR67RE273L|          1|           51|         65|   N|                Y|\n",
            "|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|\n",
            "| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|\n",
            "|R36O341WWXXKNP|          5|           28|         31|   N|                N|\n",
            "|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|\n",
            "|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|\n",
            "|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|\n",
            "|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|\n",
            "| R8WD0136CETA7|          2|           99|        119|   N|                N|\n",
            "|R16621F16PQN5A|          3|           24|         31|   N|                Y|\n",
            "|R1LXBPG1M9DDOC|          1|          102|        106|   N|                N|\n",
            "| RNEMGSUKJ4535|          2|           21|         25|   N|                Y|\n",
            "|R3R2UKBFKP1IQ4|          1|           19|         23|   N|                Y|\n",
            "|R2T90XM5G2DBZA|          3|           15|         22|   N|                Y|\n",
            "| R5ZMK0Y0KRBRZ|          1|           79|         91|   N|                N|\n",
            "|R12E2W0XOV3I2S|          1|           24|         30|   N|                Y|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SMXNpixoBYms",
        "outputId": "b6365887-0d9f-4b20-862a-f28f3f235202"
      },
      "source": [
        "# Determine the total number of reviews, the number of 5-star reviews, and the percentage of 5-star reviews for the two types of review (paid vs unpaid).\n",
        "total_reviews = paid_review_df.count() + unpaid_review_df.count()\n",
        "print(f'There are {total_reviews} total reviews')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 37921 total reviews\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QaWy3ir3C1zg",
        "outputId": "069eb8c0-8dc0-4957-af40-0279ff43df35"
      },
      "source": [
        "paid_review_count = paid_review_df.count()\n",
        "unpaid_review_count = unpaid_review_df.count()\n",
        "print(f'There are {paid_review_count} paid reviews and {unpaid_review_count} unpaid reviews.')"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 90 paid reviews and 37831 unpaid reviews.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I3zFh9fJDadw",
        "outputId": "5380ea24-3754-4c68-cb8f-9034a7e60f0d"
      },
      "source": [
        "five_paid_review_count = paid_review_df.filter(paid_review_df['star_rating'] == 5).count()\n",
        "five_unpaid_review_count = unpaid_review_df.filter(unpaid_review_df['star_rating'] == 5).count()\n",
        "print(f'There are {five_paid_review_count} 5 Star paid reviews and {five_unpaid_review_count} 5 Star unpaid reviews.')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 44 5 Star paid reviews and 14704 5 Star unpaid reviews.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qOnBPzJHEj21",
        "outputId": "3d8a6159-7a9c-4bc1-a46b-89da9ed988f8"
      },
      "source": [
        "paid_percent_five_review = (five_paid_review_count / paid_review_count) * 100\n",
        "unpaid_percent_five_review = (five_unpaid_review_count / unpaid_review_count) * 100\n",
        "print(f'Of of the paid reviews, {paid_percent_five_review: .2f}% are 5 Star. Of the unpaid reviews, {unpaid_percent_five_review: .2f}% are 5 Star.')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Of of the paid reviews,  48.89% are 5 Star. Of the unpaid reviews,  38.87% are 5 Star.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}