scraping_song_list.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('data', exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_URL = 'https://www.zonareferensi.com/lagu-daerah-indonesia/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(BASE_URL)\n",
    "soup = BeautifulSoup(res.text, 'html.parser')\n",
    "\n",
    "table = soup.find_all('table')[0]\n",
    "rows = table.find_all('tr')\n",
    "\n",
    "data = []\n",
    "\n",
    "for i, row in enumerate(rows):\n",
    "    if i == 0:\n",
    "        continue\n",
    "    cols = row.find_all('td')\n",
    "    \n",
    "    cols = [ele.text.strip() for ele in cols]\n",
    "    data.append([ele for ele in cols if ele])\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "df.columns = ['no', 'nama_lagu', 'asal']\n",
    "\n",
    "df.to_csv('data/lagu_daerah.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no</th>\n",
       "      <th>nama_lagu</th>\n",
       "      <th>asal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>172</td>\n",
       "      <td>Ting Kededai</td>\n",
       "      <td>Kalimantan Barat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>203</td>\n",
       "      <td>Jangan Manangis</td>\n",
       "      <td>Kalimantan Selatan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>142</td>\n",
       "      <td>Cing Kincing Kere’</td>\n",
       "      <td>Jawa Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>Tari Bali</td>\n",
       "      <td>Bali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>385</td>\n",
       "      <td>Tari Tanggai</td>\n",
       "      <td>Sumatera Selatan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      no           nama_lagu                asal\n",
       "171  172        Ting Kededai    Kalimantan Barat\n",
       "202  203     Jangan Manangis  Kalimantan Selatan\n",
       "141  142  Cing Kincing Kere’          Jawa Timur\n",
       "12    13           Tari Bali                Bali\n",
       "384  385        Tari Tanggai    Sumatera Selatan"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/lagu_daerah.csv')\n",
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no</th>\n",
       "      <th>nama_lagu</th>\n",
       "      <th>asal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>288</td>\n",
       "      <td>Bolelebo</td>\n",
       "      <td>Nusa Tenggara Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>277</td>\n",
       "      <td>Bolelebo</td>\n",
       "      <td>Nusa Tenggara Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>Dayung Sampan</td>\n",
       "      <td>Banten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>53</td>\n",
       "      <td>Dayung Sampan</td>\n",
       "      <td>Jakarta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>144</td>\n",
       "      <td>Gai Bintang</td>\n",
       "      <td>Jawa Timur Madura</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>118</td>\n",
       "      <td>Gai Bintang</td>\n",
       "      <td>Jawa Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>105</td>\n",
       "      <td>Gek Kepriye</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>98</td>\n",
       "      <td>Gek Kepriye</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>101</td>\n",
       "      <td>Jamuran</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>108</td>\n",
       "      <td>Jamuran</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>102</td>\n",
       "      <td>Jaranan</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>107</td>\n",
       "      <td>Jaranan</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>114</td>\n",
       "      <td>Kembang Malathe</td>\n",
       "      <td>Jawa Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>119</td>\n",
       "      <td>Kembang Malathe</td>\n",
       "      <td>Jawa Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>373</td>\n",
       "      <td>Kutang Barendo</td>\n",
       "      <td>Sumatera Barat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>332</td>\n",
       "      <td>Kutang Barendo</td>\n",
       "      <td>Riau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>208</td>\n",
       "      <td>Lancang Kuning</td>\n",
       "      <td>Kalimantan Selatan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>322</td>\n",
       "      <td>Lancang Kuning</td>\n",
       "      <td>Riau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>279</td>\n",
       "      <td>Lerang Wutun</td>\n",
       "      <td>Nusa Tenggara Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>287</td>\n",
       "      <td>Lerang Wutun</td>\n",
       "      <td>Nusa Tenggara Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>270</td>\n",
       "      <td>Moree</td>\n",
       "      <td>Nusa Tenggara Barat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>290</td>\n",
       "      <td>Moree</td>\n",
       "      <td>Nusa Tenggara Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>34</td>\n",
       "      <td>Sungai Suci</td>\n",
       "      <td>Bengkulu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>Sungai Suci</td>\n",
       "      <td>Bengkulu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>117</td>\n",
       "      <td>Tanduk Majeng</td>\n",
       "      <td>Jawa Timur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>145</td>\n",
       "      <td>Tanduk Majeng</td>\n",
       "      <td>Jawa Timur Madura</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>42</td>\n",
       "      <td>Yamko Rambe Yamko</td>\n",
       "      <td>Irian Jaya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>321</td>\n",
       "      <td>Yamko Rambe Yamko</td>\n",
       "      <td>Papua</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      no          nama_lagu                 asal\n",
       "287  288           Bolelebo  Nusa Tenggara Timur\n",
       "276  277           Bolelebo  Nusa Tenggara Timur\n",
       "19    20      Dayung Sampan               Banten\n",
       "52    53      Dayung Sampan              Jakarta\n",
       "143  144        Gai Bintang    Jawa Timur Madura\n",
       "117  118        Gai Bintang           Jawa Timur\n",
       "104  105        Gek Kepriye          Jawa Tengah\n",
       "97    98        Gek Kepriye          Jawa Tengah\n",
       "100  101            Jamuran          Jawa Tengah\n",
       "107  108            Jamuran          Jawa Tengah\n",
       "101  102            Jaranan          Jawa Tengah\n",
       "106  107            Jaranan          Jawa Tengah\n",
       "113  114    Kembang Malathe           Jawa Timur\n",
       "118  119    Kembang Malathe           Jawa Timur\n",
       "372  373     Kutang Barendo       Sumatera Barat\n",
       "331  332     Kutang Barendo                 Riau\n",
       "207  208     Lancang Kuning   Kalimantan Selatan\n",
       "321  322     Lancang Kuning                 Riau\n",
       "278  279       Lerang Wutun  Nusa Tenggara Timur\n",
       "286  287       Lerang Wutun  Nusa Tenggara Timur\n",
       "269  270              Moree  Nusa Tenggara Barat\n",
       "289  290              Moree  Nusa Tenggara Timur\n",
       "33    34        Sungai Suci             Bengkulu\n",
       "23    24        Sungai Suci             Bengkulu\n",
       "116  117      Tanduk Majeng           Jawa Timur\n",
       "144  145      Tanduk Majeng    Jawa Timur Madura\n",
       "41    42  Yamko Rambe Yamko           Irian Jaya\n",
       "320  321  Yamko Rambe Yamko                Papua"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated(subset='nama_lagu', keep=False)].sort_values('nama_lagu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean = df.drop_duplicates(subset='nama_lagu', keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before drop duplicated data: 418\n",
      "After drop duplicated data: 404\n"
     ]
    }
   ],
   "source": [
    "print(f'Before drop duplicated data: {len(df)}')\n",
    "print(f'After drop duplicated data: {len(df_clean)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to csv\n",
    "df_clean.to_csv('data/lagu_daerah.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping Lagu Daerah Alternatif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_URL = 'https://dianisa.com/lagu-daerah-indonesia-beserta-lirik-dan-asalnya/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(BASE_URL)\n",
    "\n",
    "soup = BeautifulSoup(res.text, 'html.parser')\n",
    "\n",
    "# find table with class has-fixed-layout\n",
    "table = soup.find_all('table', class_='has-fixed-layout')[0]\n",
    "\n",
    "rows = table.find_all('tr')\n",
    "\n",
    "data = []\n",
    "\n",
    "for i, row in enumerate(rows):\n",
    "    if i == 0:\n",
    "        continue\n",
    "    cols = row.find_all('td')\n",
    "    \n",
    "    cols = [ele.text.strip() for ele in cols]\n",
    "    data.append([ele for ele in cols if ele])\n",
    "    \n",
    "df = pd.DataFrame(data)\n",
    "df.columns = ['nama_lagu', 'asal']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nama_lagu</th>\n",
       "      <th>asal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yamko Rambe Yamko</td>\n",
       "      <td>Papua</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rasa Sayange</td>\n",
       "      <td>Maluku</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Manuk Dadali</td>\n",
       "      <td>Jawa Barat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gundul Pacul</td>\n",
       "      <td>Jawa Tengah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jali – Jali</td>\n",
       "      <td>Jakarta</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           nama_lagu         asal\n",
       "0  Yamko Rambe Yamko        Papua\n",
       "1       Rasa Sayange       Maluku\n",
       "2       Manuk Dadali   Jawa Barat\n",
       "3       Gundul Pacul  Jawa Tengah\n",
       "4        Jali – Jali      Jakarta"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nama_lagu</th>\n",
       "      <th>asal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Bungong Jeumpa</td>\n",
       "      <td>Aceh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Mecepat – Cepetan</td>\n",
       "      <td>Bali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Sarinande</td>\n",
       "      <td>Bali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Putri</td>\n",
       "      <td>Bali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Ratu Anom</td>\n",
       "      <td>Bali</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nama_lagu  asal\n",
       "21      Bungong Jeumpa  Aceh\n",
       "22   Mecepat – Cepetan  Bali\n",
       "19           Sarinande  Bali\n",
       "104              Putri  Bali\n",
       "68           Ratu Anom  Bali"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sort_values('asal')\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/lagu_daerah_alternatif.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}