Playing_with_pandas.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Playing with pandas.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1hAa9ciyIG_ziPxIkfa_UCHGWg-XsZuci",
      "authorship_tag": "ABX9TyO+lcXwU3mN3YPWYJN7GR/S",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dishankkalra23/Playing-with-csv-and-pandas/blob/main/Playing_with_pandas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v4sy5AVAaSPs"
      },
      "source": [
        "# Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlSWfUtRb_cQ"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wI0ND7TSaetk"
      },
      "source": [
        "## Setting up Kaggle API"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcixHlTcaGD-"
      },
      "source": [
        " ! pip install -q kaggle"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yx4f4WTjat8m"
      },
      "source": [
        "## Uplading kaggle.json file which have API token"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LfaurCePakhl"
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0SAvwb6Haza4"
      },
      "source": [
        "# Move the downloaded file to a location ~/.kaggle/kaggle.json.\n",
        "! mkdir ~/.kaggle\n",
        "! cp kaggle.json ~/.kaggle/\n",
        "\n",
        "# You need to give proper permissions to the file (since this is a hidden folder\n",
        "! chmod 600 ~/.kaggle/kaggle.json"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YsOHDYZ_bX4l"
      },
      "source": [
        "## Copy API command from respective Kaggle dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KUE4d3WrbN5T",
        "outputId": "841d8390-b5a8-47c0-d9a6-5dad3b8034b2"
      },
      "source": [
        "! kaggle datasets download -d spscientist/students-performance-in-exams"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading students-performance-in-exams.zip to /content\n",
            "\r  0% 0.00/8.70k [00:00<?, ?B/s]\n",
            "\r100% 8.70k/8.70k [00:00<00:00, 17.1MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B44dRMGXbhVy"
      },
      "source": [
        "## Unzipping dataset file and removing zip file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6UP1YQ9ibeJw",
        "outputId": "7b3a6f41-cc64-402c-926e-71a1c51890f4"
      },
      "source": [
        "!unzip \\*.zip"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  students-performance-in-exams.zip\n",
            "  inflating: StudentsPerformance.csv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vW053Z9Abxk5"
      },
      "source": [
        "! rm *.zip"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K2-ptLBpb1vU"
      },
      "source": [
        "# Loading dataset (Gathering Data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L7t3iEX3BV8J"
      },
      "source": [
        "## Reading CSV file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "uPz1daPWb1WD",
        "outputId": "c92e8678-4920-4f14-bc32-7b8cef9d8535"
      },
      "source": [
        "dataset = '/content/StudentsPerformance.csv'\n",
        "data = pd.read_csv(dataset)\n",
        "data"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "      <th>lunch</th>\n",
              "      <th>test preparation course</th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "      <th>writing score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>female</td>\n",
              "      <td>group E</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>88</td>\n",
              "      <td>99</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>62</td>\n",
              "      <td>55</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>completed</td>\n",
              "      <td>59</td>\n",
              "      <td>71</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>68</td>\n",
              "      <td>78</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>some college</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>77</td>\n",
              "      <td>86</td>\n",
              "      <td>86</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     gender race/ethnicity  ... reading score writing score\n",
              "0    female        group B  ...            72            74\n",
              "1    female        group C  ...            90            88\n",
              "2    female        group B  ...            95            93\n",
              "3      male        group A  ...            57            44\n",
              "4      male        group C  ...            78            75\n",
              "..      ...            ...  ...           ...           ...\n",
              "995  female        group E  ...            99            95\n",
              "996    male        group C  ...            55            55\n",
              "997  female        group C  ...            71            65\n",
              "998  female        group D  ...            78            77\n",
              "999  female        group D  ...            86            86\n",
              "\n",
              "[1000 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "it-JlOJ2Bpv9"
      },
      "source": [
        "### CSV stands for comma separated values - but they can actually be separated by different characters, tabs, white space, etc. If your file is separated by a colon, let's say, you can still use read_csv() with the sep parameter."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "k45V2xgf_AuX",
        "outputId": "f9f8e646-52a4-4818-c5cb-0036cd5e19ee"
      },
      "source": [
        "data_1 = pd.read_csv(dataset,sep=':')\n",
        "\n",
        "'''\n",
        "Note: This obviously didn't work because CSV file is separated by commas and seperators we used colon\n",
        "Because there are no colons, nothing was separated and everything was read into one column\n",
        "'''\n",
        "data_1"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender,\"race/ethnicity\",\"parental level of education\",\"lunch\",\"test preparation course\",\"math score\",\"reading score\",\"writing score\"</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female,\"group B\",\"bachelor's degree\",\"standard...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female,\"group C\",\"some college\",\"standard\",\"co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female,\"group B\",\"master's degree\",\"standard\",...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male,\"group A\",\"associate's degree\",\"free/redu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male,\"group C\",\"some college\",\"standard\",\"none...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>female,\"group E\",\"master's degree\",\"standard\",...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>male,\"group C\",\"high school\",\"free/reduced\",\"n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>female,\"group C\",\"high school\",\"free/reduced\",...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>female,\"group D\",\"some college\",\"standard\",\"co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>female,\"group D\",\"some college\",\"free/reduced\"...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "    gender,\"race/ethnicity\",\"parental level of education\",\"lunch\",\"test preparation course\",\"math score\",\"reading score\",\"writing score\"\n",
              "0    female,\"group B\",\"bachelor's degree\",\"standard...                                                                                  \n",
              "1    female,\"group C\",\"some college\",\"standard\",\"co...                                                                                  \n",
              "2    female,\"group B\",\"master's degree\",\"standard\",...                                                                                  \n",
              "3    male,\"group A\",\"associate's degree\",\"free/redu...                                                                                  \n",
              "4    male,\"group C\",\"some college\",\"standard\",\"none...                                                                                  \n",
              "..                                                 ...                                                                                  \n",
              "995  female,\"group E\",\"master's degree\",\"standard\",...                                                                                  \n",
              "996  male,\"group C\",\"high school\",\"free/reduced\",\"n...                                                                                  \n",
              "997  female,\"group C\",\"high school\",\"free/reduced\",...                                                                                  \n",
              "998  female,\"group D\",\"some college\",\"standard\",\"co...                                                                                  \n",
              "999  female,\"group D\",\"some college\",\"free/reduced\"...                                                                                  \n",
              "\n",
              "[1000 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NVlQUfi0Cq8l"
      },
      "source": [
        "### We can specify which line of the file is the header, which specifies the column labels. It's usually the first line, but sometimes we'll want to specify a later line if there is extra meta information at the top of the file. We can do that like this."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "snEFkf8LCdcT",
        "outputId": "06de1032-9f0d-498b-cd89-8fd0eb386c7c"
      },
      "source": [
        "df = pd.read_csv(dataset, header=1)\n",
        "'''\n",
        "Here, row 2 was used as the the header and everything above that was cut off. \n",
        "By default, read_csv uses header=0, which uses the first line for column labels.\n",
        "'''\n",
        "df.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>female</th>\n",
              "      <th>group B</th>\n",
              "      <th>bachelor's degree</th>\n",
              "      <th>standard</th>\n",
              "      <th>none</th>\n",
              "      <th>72</th>\n",
              "      <th>72.1</th>\n",
              "      <th>74</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>71</td>\n",
              "      <td>83</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   female  group B   bachelor's degree      standard       none  72  72.1  74\n",
              "0  female  group C        some college      standard  completed  69    90  88\n",
              "1  female  group B     master's degree      standard       none  90    95  93\n",
              "2    male  group A  associate's degree  free/reduced       none  47    57  44\n",
              "3    male  group C        some college      standard       none  76    78  75\n",
              "4  female  group B  associate's degree      standard       none  71    83  78"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Us51V483DnNe"
      },
      "source": [
        "### If columns labels are not included in your file, you can use header=None to prevent your first line of data from being misinterpreted as column labels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "xLzIz0r3C7hs",
        "outputId": "c5dd8e7d-bc7e-48ff-9fb1-e371d8bee37d"
      },
      "source": [
        "df = pd.read_csv(dataset, header=None)\n",
        "df.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>gender</td>\n",
              "      <td>race/ethnicity</td>\n",
              "      <td>parental level of education</td>\n",
              "      <td>lunch</td>\n",
              "      <td>test preparation course</td>\n",
              "      <td>math score</td>\n",
              "      <td>reading score</td>\n",
              "      <td>writing score</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        0               1  ...              6              7\n",
              "0  gender  race/ethnicity  ...  reading score  writing score\n",
              "1  female         group B  ...             72             74\n",
              "2  female         group C  ...             90             88\n",
              "3  female         group B  ...             95             93\n",
              "4    male         group A  ...             57             44\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "eInMyLQADzQN",
        "outputId": "e5002a90-ad48-4bd9-ec38-6fbebc1c463c"
      },
      "source": [
        "labels = ['gen', 'ethn', 'LOE', 'lun', 'test', 'math_Score', 'reading_Score', 'writing_Score']\n",
        "df = pd.read_csv(dataset, names=labels)\n",
        "df.head()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gen</th>\n",
              "      <th>ethn</th>\n",
              "      <th>LOE</th>\n",
              "      <th>lun</th>\n",
              "      <th>test</th>\n",
              "      <th>math_Score</th>\n",
              "      <th>reading_Score</th>\n",
              "      <th>writing_Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>gender</td>\n",
              "      <td>race/ethnicity</td>\n",
              "      <td>parental level of education</td>\n",
              "      <td>lunch</td>\n",
              "      <td>test preparation course</td>\n",
              "      <td>math score</td>\n",
              "      <td>reading score</td>\n",
              "      <td>writing score</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      gen            ethn  ...  reading_Score  writing_Score\n",
              "0  gender  race/ethnicity  ...  reading score  writing score\n",
              "1  female         group B  ...             72             74\n",
              "2  female         group C  ...             90             88\n",
              "3  female         group B  ...             95             93\n",
              "4    male         group A  ...             57             44\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7ABDRX0FGFB"
      },
      "source": [
        "### In above if you want to tell pandas that there was a header line that you are replacing, you can specify the row of that line like this"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "Fyp9h4L9FD9B",
        "outputId": "8f1f60a8-6cb4-4c36-c24d-f464c982b9f5"
      },
      "source": [
        "labels = ['gen', 'ethn', 'LOE', 'lun', 'test', 'math_Score', 'reading_Score', 'writing_Score']\n",
        "df = pd.read_csv(dataset, names=labels,header=0)\n",
        "df.head()\n",
        "\n",
        "# Note: Replacing 0th index with given labels"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gen</th>\n",
              "      <th>ethn</th>\n",
              "      <th>LOE</th>\n",
              "      <th>lun</th>\n",
              "      <th>test</th>\n",
              "      <th>math_Score</th>\n",
              "      <th>reading_Score</th>\n",
              "      <th>writing_Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      gen     ethn  ... reading_Score writing_Score\n",
              "0  female  group B  ...            72            74\n",
              "1  female  group C  ...            90            88\n",
              "2  female  group B  ...            95            93\n",
              "3    male  group A  ...            57            44\n",
              "4    male  group C  ...            78            75\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2YAkaYMrXofm"
      },
      "source": [
        "# Assessing and Building Intuition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "-trx6u--XX0m",
        "outputId": "93270a70-c7a8-422d-8c9e-873aa8530466"
      },
      "source": [
        "df = pd.read_csv(dataset)\n",
        "df.head()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "      <th>lunch</th>\n",
              "      <th>test preparation course</th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "      <th>writing score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity  ... reading score writing score\n",
              "0  female        group B  ...            72            74\n",
              "1  female        group C  ...            90            88\n",
              "2  female        group B  ...            95            93\n",
              "3    male        group A  ...            57            44\n",
              "4    male        group C  ...            78            75\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bkH19CFeX3xr",
        "outputId": "06d138b8-3a0e-4551-c41d-d146cfe3bd6e"
      },
      "source": [
        "# this returns a tuple of the dimensions of the dataframe\n",
        "df.shape"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfXVHnmpYUU6",
        "outputId": "4b5c63a1-0ccb-4d80-b3b4-cb994901c8c1"
      },
      "source": [
        "# this returns the datatypes of the columns\n",
        "df.dtypes"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "gender                         object\n",
              "race/ethnicity                 object\n",
              "parental level of education    object\n",
              "lunch                          object\n",
              "test preparation course        object\n",
              "math score                      int64\n",
              "reading score                   int64\n",
              "writing score                   int64\n",
              "dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QJDZWcfQYcUj",
        "outputId": "b527fae3-8f2a-4625-9d10-a674f584b05d"
      },
      "source": [
        "# although the datatype for 'parental level of education'  appears to be object, further\n",
        "# investigation shows it's a string\n",
        "type(df['parental level of education'][0])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "str"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2V3RTMxYzAo"
      },
      "source": [
        "## Note: Pandas actually stores [pointers](https://en.wikipedia.org/wiki/Pointer_(computer_programming) to strings in dataframes and series, which is why object instead of str appears as the datatype."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g2Qb8iniYmAp",
        "outputId": "0e08bc78-1e78-42aa-b81e-463e5e0aff8b"
      },
      "source": [
        "# this displays a concise summary of the dataframe,\n",
        "# including the number of non-null values in each column\n",
        "df.info()"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1000 entries, 0 to 999\n",
            "Data columns (total 8 columns):\n",
            " #   Column                       Non-Null Count  Dtype \n",
            "---  ------                       --------------  ----- \n",
            " 0   gender                       1000 non-null   object\n",
            " 1   race/ethnicity               1000 non-null   object\n",
            " 2   parental level of education  1000 non-null   object\n",
            " 3   lunch                        1000 non-null   object\n",
            " 4   test preparation course      1000 non-null   object\n",
            " 5   math score                   1000 non-null   int64 \n",
            " 6   reading score                1000 non-null   int64 \n",
            " 7   writing score                1000 non-null   int64 \n",
            "dtypes: int64(3), object(5)\n",
            "memory usage: 62.6+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KgFvNTV8Zzrr",
        "outputId": "e16c3e73-d1bf-4084-f65a-ee2cae063fa1"
      },
      "source": [
        "# this returns the number of unique values in each column\n",
        "df.nunique()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "gender                          2\n",
              "race/ethnicity                  5\n",
              "parental level of education     6\n",
              "lunch                           2\n",
              "test preparation course         2\n",
              "math score                     81\n",
              "reading score                  72\n",
              "writing score                  77\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "YsbXj37HZ9Ge",
        "outputId": "fa852b34-1728-456d-8faf-17f981372db4"
      },
      "source": [
        "# this returns useful descriptive statistics for each column of data\n",
        "df.describe()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "      <th>writing score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1000.00000</td>\n",
              "      <td>1000.000000</td>\n",
              "      <td>1000.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>66.08900</td>\n",
              "      <td>69.169000</td>\n",
              "      <td>68.054000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>15.16308</td>\n",
              "      <td>14.600192</td>\n",
              "      <td>15.195657</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.00000</td>\n",
              "      <td>17.000000</td>\n",
              "      <td>10.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>57.00000</td>\n",
              "      <td>59.000000</td>\n",
              "      <td>57.750000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>66.00000</td>\n",
              "      <td>70.000000</td>\n",
              "      <td>69.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>77.00000</td>\n",
              "      <td>79.000000</td>\n",
              "      <td>79.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>100.00000</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       math score  reading score  writing score\n",
              "count  1000.00000    1000.000000    1000.000000\n",
              "mean     66.08900      69.169000      68.054000\n",
              "std      15.16308      14.600192      15.195657\n",
              "min       0.00000      17.000000      10.000000\n",
              "25%      57.00000      59.000000      57.750000\n",
              "50%      66.00000      70.000000      69.000000\n",
              "75%      77.00000      79.000000      79.000000\n",
              "max     100.00000     100.000000     100.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "OLRTRDVRavsB",
        "outputId": "03da8f57-8053-4c24-de47-07265e609521"
      },
      "source": [
        "# `.tail()` returns the first few rows\n",
        "df.head(10)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "      <th>lunch</th>\n",
              "      <th>test preparation course</th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "      <th>writing score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "      <td>88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "      <td>93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "      <td>44</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>71</td>\n",
              "      <td>83</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>88</td>\n",
              "      <td>95</td>\n",
              "      <td>92</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>male</td>\n",
              "      <td>group B</td>\n",
              "      <td>some college</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>40</td>\n",
              "      <td>43</td>\n",
              "      <td>39</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>male</td>\n",
              "      <td>group D</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>completed</td>\n",
              "      <td>64</td>\n",
              "      <td>64</td>\n",
              "      <td>67</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>38</td>\n",
              "      <td>60</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity  ... reading score writing score\n",
              "0  female        group B  ...            72            74\n",
              "1  female        group C  ...            90            88\n",
              "2  female        group B  ...            95            93\n",
              "3    male        group A  ...            57            44\n",
              "4    male        group C  ...            78            75\n",
              "5  female        group B  ...            83            78\n",
              "6  female        group B  ...            95            92\n",
              "7    male        group B  ...            43            39\n",
              "8    male        group D  ...            64            67\n",
              "9  female        group B  ...            60            50\n",
              "\n",
              "[10 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "AQK7bhZ2eIIg",
        "outputId": "74d6c6d7-f322-4fdd-a4c7-03a031c1c968"
      },
      "source": [
        "# `.tail()` returns the last few rows\n",
        "df.tail(10)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "      <th>lunch</th>\n",
              "      <th>test preparation course</th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "      <th>writing score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>990</th>\n",
              "      <td>male</td>\n",
              "      <td>group E</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>completed</td>\n",
              "      <td>86</td>\n",
              "      <td>81</td>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>991</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>some high school</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>65</td>\n",
              "      <td>82</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>992</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>associate's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>55</td>\n",
              "      <td>76</td>\n",
              "      <td>76</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>993</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>bachelor's degree</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>62</td>\n",
              "      <td>72</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>994</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>high school</td>\n",
              "      <td>standard</td>\n",
              "      <td>none</td>\n",
              "      <td>63</td>\n",
              "      <td>63</td>\n",
              "      <td>62</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>female</td>\n",
              "      <td>group E</td>\n",
              "      <td>master's degree</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>88</td>\n",
              "      <td>99</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>62</td>\n",
              "      <td>55</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>high school</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>completed</td>\n",
              "      <td>59</td>\n",
              "      <td>71</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>some college</td>\n",
              "      <td>standard</td>\n",
              "      <td>completed</td>\n",
              "      <td>68</td>\n",
              "      <td>78</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>female</td>\n",
              "      <td>group D</td>\n",
              "      <td>some college</td>\n",
              "      <td>free/reduced</td>\n",
              "      <td>none</td>\n",
              "      <td>77</td>\n",
              "      <td>86</td>\n",
              "      <td>86</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     gender race/ethnicity  ... reading score writing score\n",
              "990    male        group E  ...            81            75\n",
              "991  female        group B  ...            82            78\n",
              "992  female        group D  ...            76            76\n",
              "993  female        group D  ...            72            74\n",
              "994    male        group A  ...            63            62\n",
              "995  female        group E  ...            99            95\n",
              "996    male        group C  ...            55            55\n",
              "997  female        group C  ...            71            65\n",
              "998  female        group D  ...            78            77\n",
              "999  female        group D  ...            86            86\n",
              "\n",
              "[10 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XEUi_WE4iumR",
        "outputId": "97c0d01f-6b54-49b2-f74c-0409b0df2a19"
      },
      "source": [
        "for count,colm in enumerate(df.columns):\n",
        "  print(count,colm)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 gender\n",
            "1 race/ethnicity\n",
            "2 parental level of education\n",
            "3 lunch\n",
            "4 test preparation course\n",
            "5 math score\n",
            "6 reading score\n",
            "7 writing score\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "U1X5PNsSeOUu",
        "outputId": "0171e7de-7e47-4aac-c07a-aedbe3be7d87"
      },
      "source": [
        "df.loc[:,'gender':'parental level of education'].head()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity parental level of education\n",
              "0  female        group B           bachelor's degree\n",
              "1  female        group C                some college\n",
              "2  female        group B             master's degree\n",
              "3    male        group A          associate's degree\n",
              "4    male        group C                some college"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "IqLr4zVRicZy",
        "outputId": "1a7cdc0d-be19-4db3-df34-44b335e7d6c1"
      },
      "source": [
        "df.iloc[:,:3].head()"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity parental level of education\n",
              "0  female        group B           bachelor's degree\n",
              "1  female        group C                some college\n",
              "2  female        group B             master's degree\n",
              "3    male        group A          associate's degree\n",
              "4    male        group C                some college"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vhdM9DqBkRKn"
      },
      "source": [
        "# Indexing and Selecting Data in Pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QoLxrs4CkQh1"
      },
      "source": [
        "# Let's separate this dataframe into two new dataframes - \n",
        "# parental level of education and scores which have gender and race in common"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "X8OQOGsNkln4",
        "outputId": "1c44ad58-ea1b-4295-f88b-32bf27a2fd27"
      },
      "source": [
        "df_PLOE_ = df.iloc[:,:3]\n",
        "df_PLOE_.head()"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>parental level of education</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>bachelor's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>master's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>associate's degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>some college</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity parental level of education\n",
              "0  female        group B           bachelor's degree\n",
              "1  female        group C                some college\n",
              "2  female        group B             master's degree\n",
              "3    male        group A          associate's degree\n",
              "4    male        group C                some college"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "grLYrdBctbaG"
      },
      "source": [
        "## Selecting Multiple Ranges in Pandas"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "THJ11N8bupyL"
      },
      "source": [
        "### Selecting columns gender,race with 3 score columns. Use this [stackoverflow link](https://stackoverflow.com/questions/41256648/select-multiple-ranges-of-columns-in-pandas-dataframe) to learn how to select multiple ranges in Pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "7ngHYhUFta_1",
        "outputId": "41507b7e-005b-452e-d228-0aaa23a1db25"
      },
      "source": [
        "# Using np.r_ to slice the columns of dataframe\n",
        "\n",
        "df_scores = df.iloc[:,np.r_[:2,5:7]]\n",
        "df_scores.head()"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>math score</th>\n",
              "      <th>reading score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>72</td>\n",
              "      <td>72</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>group C</td>\n",
              "      <td>69</td>\n",
              "      <td>90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>group B</td>\n",
              "      <td>90</td>\n",
              "      <td>95</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>male</td>\n",
              "      <td>group A</td>\n",
              "      <td>47</td>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>male</td>\n",
              "      <td>group C</td>\n",
              "      <td>76</td>\n",
              "      <td>78</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender race/ethnicity  math score  reading score\n",
              "0  female        group B          72             72\n",
              "1  female        group C          69             90\n",
              "2  female        group B          90             95\n",
              "3    male        group A          47             57\n",
              "4    male        group C          76             78"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    }
  ]
}