Part 1 (Output Notebook).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LTA Mobility Sensing Project Part 1.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LwQzKAnBuv6n",
        "colab_type": "text"
      },
      "source": [
        "**LTA Mobility Sensing Project Part 1**\n",
        "\n",
        "- Retrieve data from S3 bucket\n",
        "\n",
        "- Save to csv file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8HxDiDExn8yi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import boto3\n",
        "\n",
        "_BUCKET_NAME = ''\n",
        "_PREFIX = ''\n",
        "\n",
        "client = boto3.client('s3', aws_access_key_id='',aws_secret_access_key='')\n",
        "\n",
        "def ListFiles(client):\n",
        "    \"\"\"List files in specific S3 URL\"\"\"\n",
        "    response = client.list_objects(Bucket=_BUCKET_NAME, Prefix=_PREFIX)\n",
        "    for content in response.get('Contents', []):\n",
        "        yield content.get('Key')\n",
        "\n",
        "file_list = ListFiles(client)\n",
        "for file in file_list:\n",
        "    print ('File found: %s' % file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VVm9zs_FtM62",
        "colab_type": "text"
      },
      "source": [
        "Accessing s3 bucket data \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9PrQMFXZoNCw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "s3 = boto3.resource('s3', aws_access_key_id='',aws_secret_access_key='')\n",
        "import json\n",
        "import pandas as pd\n",
        "import boto3\n",
        "bucket = s3.Bucket('')\n",
        "import io\n",
        "prefix_objs = bucket.objects.filter(Prefix=\"\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "osjFxhECtXRj",
        "colab_type": "text"
      },
      "source": [
        "Read in accelerometer sensor data\n",
        "\n",
        "Check the shape\n",
        "\n",
        "Save as csv"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HVbd2RPIoNUU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_accel = pd.DataFrame(columns=['Altitude', 'JourneyID', 'lat', 'lng', 'Mode', 'Timestamp'])\n",
        "for obj in prefix_objs:\n",
        "    key = obj.key\n",
        "    obje = client.get_object(Bucket='', Key=key)\n",
        "    data = obje['Body'].read().decode()\n",
        "    json_content = json.loads(data)\n",
        "    try:\n",
        "        json_contents = json_content['accelerometer']\n",
        "        dff = pd.DataFrame(json_contents)\n",
        "    except KeyError:\n",
        "         pass\n",
        "    df_accel = df_accel.append(dff)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "69wW8k5XoNX4",
        "colab_type": "code",
        "outputId": "809f84d8-d50e-4146-d3e5-ed3710ac78bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_accel.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(196208, 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liA34YFTDvvx",
        "colab_type": "code",
        "outputId": "4a0de3fb-20bc-4f67-f519-f0b20bebd20f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df_accel.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Altitude</th>\n",
              "      <th>JourneyID</th>\n",
              "      <th>Mode</th>\n",
              "      <th>Timestamp</th>\n",
              "      <th>X</th>\n",
              "      <th>Y</th>\n",
              "      <th>Z</th>\n",
              "      <th>lat</th>\n",
              "      <th>lng</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122668022</td>\n",
              "      <td>-0.10137049853801727</td>\n",
              "      <td>4.078713893890381</td>\n",
              "      <td>9.32475471496582</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122669031</td>\n",
              "      <td>0.16867630183696747</td>\n",
              "      <td>0.638102650642395</td>\n",
              "      <td>10.021468162536621</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122670034</td>\n",
              "      <td>3.0051350593566895</td>\n",
              "      <td>2.4519498348236084</td>\n",
              "      <td>13.4412202835083</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122671036</td>\n",
              "      <td>4.065823554992676</td>\n",
              "      <td>3.711960792541504</td>\n",
              "      <td>12.759917259216309</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122672110</td>\n",
              "      <td>4.5745720863342285</td>\n",
              "      <td>2.1662163734436035</td>\n",
              "      <td>16.789329528808594</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Altitude   JourneyID  Mode  ...                   Z  lat  lng\n",
              "0      NaN  _or9hxk29h  Idle  ...    9.32475471496582  NaN  NaN\n",
              "1      NaN  _or9hxk29h  Idle  ...  10.021468162536621  NaN  NaN\n",
              "2      NaN  _or9hxk29h  Idle  ...    13.4412202835083  NaN  NaN\n",
              "3      NaN  _or9hxk29h  Idle  ...  12.759917259216309  NaN  NaN\n",
              "4      NaN  _or9hxk29h  Idle  ...  16.789329528808594  NaN  NaN\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ogJ-hpKxoNbl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_accel.to_csv('accelerometer.csv', index = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VAzquBMAteE7",
        "colab_type": "text"
      },
      "source": [
        "Read in gyroscope sensor data\n",
        "\n",
        "Check the shape\n",
        "\n",
        "Save as csv"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PS_0WjzyoNeu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_gyro = pd.DataFrame(columns=['Altitude', 'JourneyID', 'lat', 'lng', 'Mode', 'Timestamp'])\n",
        "for obj in prefix_objs:\n",
        "    key = obj.key\n",
        "    obje = client.get_object(Bucket='', Key=key)\n",
        "    data = obje['Body'].read().decode()\n",
        "    json_content = json.loads(data)\n",
        "    try:\n",
        "        json_contents = json_content['gyroscope']\n",
        "        dff = pd.DataFrame(json_contents)\n",
        "    except KeyError:\n",
        "         pass\n",
        "    df_gyro = df_gyro.append(dff)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GylEjoIjoNhy",
        "colab_type": "code",
        "outputId": "81f4bb7d-945a-4e1d-edd9-d7bdd13749fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_gyro.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(196306, 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHXa8rsaDpnx",
        "colab_type": "code",
        "outputId": "6ccb4db2-3d88-4b4c-b6c4-8f37b237d75e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df_gyro.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Altitude</th>\n",
              "      <th>JourneyID</th>\n",
              "      <th>Mode</th>\n",
              "      <th>Timestamp</th>\n",
              "      <th>X</th>\n",
              "      <th>Y</th>\n",
              "      <th>Z</th>\n",
              "      <th>lat</th>\n",
              "      <th>lng</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122668123</td>\n",
              "      <td>-0.07452555745840073</td>\n",
              "      <td>0.357722669839859</td>\n",
              "      <td>0.560006320476532</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122669131</td>\n",
              "      <td>0.5025151968002319</td>\n",
              "      <td>-0.5344547033309937</td>\n",
              "      <td>0.23954643309116364</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122670134</td>\n",
              "      <td>0.05110324174165726</td>\n",
              "      <td>0.03300417587161064</td>\n",
              "      <td>0.281067818403244</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122671138</td>\n",
              "      <td>1.0497456789016724</td>\n",
              "      <td>0.48973938822746277</td>\n",
              "      <td>0.33110642433166504</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122672138</td>\n",
              "      <td>0.9326341152191162</td>\n",
              "      <td>0.2906496822834015</td>\n",
              "      <td>-0.1714087873697281</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Altitude   JourneyID  Mode  ...                    Z  lat  lng\n",
              "0      NaN  _or9hxk29h  Idle  ...    0.560006320476532  NaN  NaN\n",
              "1      NaN  _or9hxk29h  Idle  ...  0.23954643309116364  NaN  NaN\n",
              "2      NaN  _or9hxk29h  Idle  ...    0.281067818403244  NaN  NaN\n",
              "3      NaN  _or9hxk29h  Idle  ...  0.33110642433166504  NaN  NaN\n",
              "4      NaN  _or9hxk29h  Idle  ...  -0.1714087873697281  NaN  NaN\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jMV2S89OoNk3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_gyro.to_csv('gyroscope.csv', index = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tv_NvG4YtiU3",
        "colab_type": "text"
      },
      "source": [
        "Read in magnetometer sensor data\n",
        "\n",
        "Check the shape\n",
        "\n",
        "Save as csv"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wl9I2aPgoNp7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_magneto = pd.DataFrame(columns=['Altitude', 'JourneyID', 'lat', 'lng', 'Mode', 'Timestamp'])\n",
        "for obj in prefix_objs:\n",
        "    key = obj.key\n",
        "    obje = client.get_object(Bucket='', Key=key)\n",
        "    data = obje['Body'].read().decode()\n",
        "    json_content = json.loads(data)\n",
        "    try:\n",
        "        json_contents = json_content['magnetometer']\n",
        "        dff = pd.DataFrame(json_contents)\n",
        "    except KeyError:\n",
        "         pass\n",
        "    df_magneto = df_magneto.append(dff)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_mAsJh5oNo_",
        "colab_type": "code",
        "outputId": "e094a31b-b70b-4f2a-84d9-eea3d974162c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_magneto.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(195913, 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uozW0ppIDstC",
        "colab_type": "code",
        "outputId": "bd8b8f17-e62c-4f97-c061-5c95a9befaa3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df_magneto.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Altitude</th>\n",
              "      <th>JourneyID</th>\n",
              "      <th>Mode</th>\n",
              "      <th>Timestamp</th>\n",
              "      <th>X</th>\n",
              "      <th>Y</th>\n",
              "      <th>Z</th>\n",
              "      <th>lat</th>\n",
              "      <th>lng</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122668038</td>\n",
              "      <td>-45.3125</td>\n",
              "      <td>18.3125</td>\n",
              "      <td>2.3125</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122669039</td>\n",
              "      <td>-31.125</td>\n",
              "      <td>27.75</td>\n",
              "      <td>-2.125</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122670043</td>\n",
              "      <td>-18.25</td>\n",
              "      <td>24.3125</td>\n",
              "      <td>-6.25</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122671047</td>\n",
              "      <td>-15.125</td>\n",
              "      <td>17.875</td>\n",
              "      <td>-4.3125</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>_or9hxk29h</td>\n",
              "      <td>Idle</td>\n",
              "      <td>1574122672116</td>\n",
              "      <td>-6.9375</td>\n",
              "      <td>20.5625</td>\n",
              "      <td>-7.6875</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Altitude   JourneyID  Mode      Timestamp  ...        Y        Z  lat  lng\n",
              "0      NaN  _or9hxk29h  Idle  1574122668038  ...  18.3125   2.3125  NaN  NaN\n",
              "1      NaN  _or9hxk29h  Idle  1574122669039  ...    27.75   -2.125  NaN  NaN\n",
              "2      NaN  _or9hxk29h  Idle  1574122670043  ...  24.3125    -6.25  NaN  NaN\n",
              "3      NaN  _or9hxk29h  Idle  1574122671047  ...   17.875  -4.3125  NaN  NaN\n",
              "4      NaN  _or9hxk29h  Idle  1574122672116  ...  20.5625  -7.6875  NaN  NaN\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tyyNOqyloNoM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_magneto.to_csv('magnetometer.csv', index = False)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}