lastfm_to_csv.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "lastfm_to_csv.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "d7d9lSzxp0qq",
        "dsWt3CfUpsVJ"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6W-2Qut7qJBo",
        "colab_type": "text"
      },
      "source": [
        "# Downloading LastFM Listening History\n",
        "\n",
        "My last.fm page: https://www.last.fm/user/reuben_francis\n",
        "\n",
        "API documentation: http://www.last.fm/api\n",
        "\n",
        "This tool separately downloads your all-time most played tracks, and albums. Then it downloads all of your scrobbles in order of recency. Each of these 4 data sets are saved to separate CSV files."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "98nUP4ObWVZw"
      },
      "source": [
        "https://github.com/gboeing/data-visualization/blob/master/lastfm-listening-history/lastfm_downloader.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XAYOV_iCOKJP",
        "colab": {}
      },
      "source": [
        "import requests, json, time, pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dEbQA3YhNloA",
        "colab": {}
      },
      "source": [
        "#need to find a way to add these variables through .config\n",
        "import config\n",
        "\n",
        "key = config.key\t\n",
        "LASTFM_SHARED_SECRET = config.LASTFM_SHARED_SECRET\n",
        "LASTFM_PASSWORD = config.LASTFM_PASSWORD\n",
        "username = \"reuben_francis\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "L8cKtDv1OPMb",
        "colab": {}
      },
      "source": [
        "pause_duration = 0.2\n",
        "\n",
        "url = 'https://ws.audioscrobbler.com/2.0/?method=user.get{}&user={}&api_key={}&limit={}&extended={}&page={}&format=json'\n",
        "limit = 200 #api lets you retrieve up to 200 records per call\n",
        "extended = 0 #api lets you retrieve extended data for each track, 0=no, 1=yes\n",
        "page = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d7d9lSzxp0qq",
        "colab_type": "text"
      },
      "source": [
        "## LastFM Top Tracks CSV (limit=200)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UeW203N8OTLI",
        "colab": {}
      },
      "source": [
        "method = 'toptracks'\n",
        "request_url = url.format(method, username, key, limit, extended, page)\n",
        "artist_names = []\n",
        "track_names = []\n",
        "play_counts = []\n",
        "response = requests.get(request_url).json()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-d00QT_uPYBf",
        "colab": {}
      },
      "source": [
        "for item in response[method]['track']:\n",
        "    artist_names.append(item['artist']['name'])\n",
        "    track_names.append(item['name'])\n",
        "    play_counts.append(item['playcount'])\n",
        "\n",
        "top_tracks = pd.DataFrame()\n",
        "top_tracks['artist'] = artist_names\n",
        "top_tracks['track'] = track_names\n",
        "top_tracks['play_count'] = play_counts\n",
        "top_tracks.to_csv('lastfm_top_tracks.csv', index=None, encoding='utf-8')\n",
        "top_tracks.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dsWt3CfUpsVJ",
        "colab_type": "text"
      },
      "source": [
        "## LastFM Top Albums CSV (limit=200)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vUB8kG2spqy8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "method = 'topalbums'\n",
        "request_url = url.format(method, username, key, limit, extended, page)\n",
        "artist_names = []\n",
        "album_names = []\n",
        "play_counts = []\n",
        "response = requests.get(request_url).json()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4WpvdFupQnmu",
        "outputId": "7553f43d-feb5-4b18-e574-0adad3128930",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "\n",
        "for item in response[method]['album']:\n",
        "    artist_names.append(item['artist']['name'])\n",
        "    album_names.append(item['name'])\n",
        "    play_counts.append(item['playcount'])\n",
        "\n",
        "top_albums = pd.DataFrame()\n",
        "top_albums['artist'] = artist_names\n",
        "top_albums['album'] = album_names\n",
        "top_albums['play_count'] = play_counts\n",
        "top_albums.to_csv('lastfm_top_albums.csv', index=None, encoding='utf-8')\n",
        "top_albums.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>artist</th>\n",
              "      <th>album</th>\n",
              "      <th>play_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Richie Hawtin</td>\n",
              "      <td>Decks, EFX &amp; 909</td>\n",
              "      <td>213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ScHoolboy Q</td>\n",
              "      <td>Blank Face LP</td>\n",
              "      <td>182</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Injury Reserve</td>\n",
              "      <td>Floss</td>\n",
              "      <td>153</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Yves Tumor</td>\n",
              "      <td>Safe In The Hands Of Love</td>\n",
              "      <td>153</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sufjan Stevens</td>\n",
              "      <td>Illinois</td>\n",
              "      <td>141</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           artist                      album play_count\n",
              "0   Richie Hawtin           Decks, EFX & 909        213\n",
              "1     ScHoolboy Q              Blank Face LP        182\n",
              "2  Injury Reserve                      Floss        153\n",
              "3      Yves Tumor  Safe In The Hands Of Love        153\n",
              "4  Sufjan Stevens                   Illinois        141"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "df9ophdpp5jR",
        "colab_type": "text"
      },
      "source": [
        "## LastFM Scrobbles CSV \n",
        "\n",
        "Last.fm provides this 'recenttracks' API method to get 'all' scrobbles, the code below retrieves time series data of all scrobbles, like a log file."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MylU6jzDRM4P",
        "colab": {}
      },
      "source": [
        "def get_scrobbles(method='recenttracks', username=username, key=key, limit=200, extended=0, page=1, pages=0):\n",
        "    '''\n",
        "    method: api method\n",
        "    username/key: api credentials\n",
        "    limit: api lets you retrieve up to 200 records per call\n",
        "    extended: api lets you retrieve extended data for each track, 0=no, 1=yes\n",
        "    page: page of results to start retrieving at\n",
        "    pages: how many pages of results to retrieve. if 0, get as many as api can return.\n",
        "    '''\n",
        "    # initialize url and lists to contain response fields\n",
        "    url = 'https://ws.audioscrobbler.com/2.0/?method=user.get{}&user={}&api_key={}&limit={}&extended={}&page={}&format=json'\n",
        "    responses = []\n",
        "    artist_names = []\n",
        "    album_names = []\n",
        "    track_names = []\n",
        "    timestamps = []\n",
        "    \n",
        "    # make first request, just to get the total number of pages\n",
        "    request_url = url.format(method, username, key, limit, extended, page)\n",
        "    response = requests.get(request_url).json()\n",
        "    total_pages = int(response[method]['@attr']['totalPages'])\n",
        "    if pages > 0:\n",
        "        total_pages = min([total_pages, pages])\n",
        "        \n",
        "    print('{} total pages to retrieve'.format(total_pages))\n",
        "    \n",
        "    # request each page of data one at a time\n",
        "    for page in range(1, int(total_pages) + 1, 1):\n",
        "        if page % 10 == 0: print(page, end=' ')\n",
        "        time.sleep(pause_duration)\n",
        "        request_url = url.format(method, username, key, limit, extended, page)\n",
        "        responses.append(requests.get(request_url))\n",
        "    \n",
        "    # parse the fields out of each scrobble in each page (aka response) of scrobbles\n",
        "    for response in responses:\n",
        "        scrobbles = response.json()\n",
        "        for scrobble in scrobbles[method]['track']:\n",
        "            # only retain completed scrobbles (aka, with timestamp and not 'now playing')\n",
        "            if 'date' in scrobble.keys():\n",
        "                artist_names.append(scrobble['artist']['#text'])\n",
        "                album_names.append(scrobble['album']['#text'])\n",
        "                track_names.append(scrobble['name'])\n",
        "                timestamps.append(scrobble['date']['uts'])\n",
        "                \n",
        "    # create and populate a dataframe to contain the data\n",
        "    df = pd.DataFrame()\n",
        "    df['artist'] = artist_names\n",
        "    df['album'] = album_names\n",
        "    df['track'] = track_names\n",
        "    df['timestamp'] = timestamps\n",
        "    df['datetime'] = pd.to_datetime(df['timestamp'].astype(int), unit='s')\n",
        "    \n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "s1Qb6xtmRTfm",
        "outputId": "91a58875-0c23-43bd-8ab1-1625a0aa2bc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "# get all scrobbled tracks ever, in order of recency (pages=0 to get all)\n",
        "scrobbles = get_scrobbles(pages=0)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "275 total pages to retrieve\n",
            "10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 250 260 270 "
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ohRGdysCRZcR",
        "outputId": "742b3072-22d1-4018-881a-cdb19521cc53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "source": [
        "# save the dataset\n",
        "scrobbles.to_csv('lastfm_scrobbles.csv', index=None, encoding='utf-8')\n",
        "print('{:,} total rows'.format(len(scrobbles)))\n",
        "scrobbles.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "54,806 total rows\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>artist</th>\n",
              "      <th>album</th>\n",
              "      <th>track</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>datetime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Xiu Xiu</td>\n",
              "      <td>Knife Play</td>\n",
              "      <td>Over Over</td>\n",
              "      <td>1591526477</td>\n",
              "      <td>2020-06-07 10:41:17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Xiu Xiu</td>\n",
              "      <td>Knife Play</td>\n",
              "      <td>Dr. Troll</td>\n",
              "      <td>1591526241</td>\n",
              "      <td>2020-06-07 10:37:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Xiu Xiu</td>\n",
              "      <td>Knife Play</td>\n",
              "      <td>Hives Hives</td>\n",
              "      <td>1591526016</td>\n",
              "      <td>2020-06-07 10:33:36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Xiu Xiu</td>\n",
              "      <td>Knife Play</td>\n",
              "      <td>Luber</td>\n",
              "      <td>1591525772</td>\n",
              "      <td>2020-06-07 10:29:32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Xiu Xiu</td>\n",
              "      <td>Knife Play</td>\n",
              "      <td>I Broke Up (SJ)</td>\n",
              "      <td>1591525630</td>\n",
              "      <td>2020-06-07 10:27:10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    artist       album            track   timestamp            datetime\n",
              "0  Xiu Xiu  Knife Play        Over Over  1591526477 2020-06-07 10:41:17\n",
              "1  Xiu Xiu  Knife Play        Dr. Troll  1591526241 2020-06-07 10:37:21\n",
              "2  Xiu Xiu  Knife Play      Hives Hives  1591526016 2020-06-07 10:33:36\n",
              "3  Xiu Xiu  Knife Play            Luber  1591525772 2020-06-07 10:29:32\n",
              "4  Xiu Xiu  Knife Play  I Broke Up (SJ)  1591525630 2020-06-07 10:27:10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    }
  ]
}