data_preprocess_js.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def date_range(start, end):\n",
    "    start = datetime.strptime(start, \"%Y/%m/%d\")\n",
    "    end = datetime.strptime(end, \"%Y/%m/%d\")\n",
    "    dates = [(start + timedelta(days=i)).strftime(\"%Y/%m/%d\") for i in range((end-start).days+1)]\n",
    "    return dates\n",
    "\n",
    "dates = date_range(\"2010/07/01\", \"2021/12/27\")\n",
    "\n",
    "def slash_to_comma(x):\n",
    "    x = x.replace('-','/')\n",
    "    return x\n",
    "\n",
    "def dot_to_slash(x):\n",
    "    x = x.replace('.','/')\n",
    "    return x\n",
    "\n",
    "def slash_reorder(x):\n",
    "    x = x[6:]+'/'+x[:2]+'/'+x[3:5]\n",
    "    return x\n",
    "\n",
    "def slash_reorder_yr(x):\n",
    "    x = '20'+x[6:]+'/'+x[0:2]+'/'+x[3:5]\n",
    "    return x\n",
    "\n",
    "def gold_date(x):\n",
    "    li = x.split('.')\n",
    "    x = li[0]+'/'+li[1].zfill(2)+'/'+li[2].zfill(2)\n",
    "    return x\n",
    "# gold_date('2010.1.22')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['Date']\n",
    "\n",
    "df = pd.DataFrame(columns = columns)\n",
    "df['Date'] = dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2010/07/01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2010/07/02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2010/07/03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2010/07/04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2010/07/05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4193</td>\n",
       "      <td>2021/12/23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4194</td>\n",
       "      <td>2021/12/24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4195</td>\n",
       "      <td>2021/12/25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4196</td>\n",
       "      <td>2021/12/26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4197</td>\n",
       "      <td>2021/12/27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4198 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date\n",
       "0     2010/07/01\n",
       "1     2010/07/02\n",
       "2     2010/07/03\n",
       "3     2010/07/04\n",
       "4     2010/07/05\n",
       "...          ...\n",
       "4193  2021/12/23\n",
       "4194  2021/12/24\n",
       "4195  2021/12/25\n",
       "4196  2021/12/26\n",
       "4197  2021/12/27\n",
       "\n",
       "[4198 rows x 1 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "amzn = pd.read_csv('./AMZN.csv')\n",
    "appl = pd.read_csv('./AAPL.csv')\n",
    "bse_sensex = pd.read_csv('./BSE_SENSEX.csv')\n",
    "oil = pd.read_csv('./OIL.csv')\n",
    "ftse = pd.read_csv('./FTSE_close.csv')\n",
    "gold = pd.read_csv('./GOLD.csv')\n",
    "google = pd.read_csv('./GOOG.csv')\n",
    "msft = pd.read_csv('./MSFT.csv')\n",
    "nasdaq = pd.read_csv('./NASDAQ.csv')\n",
    "nyse = pd.read_csv('./NYSE.csv')\n",
    "russell = pd.read_csv('./RUSSELL.csv')\n",
    "snp = pd.read_csv('./S&P500.csv')\n",
    "sse = pd.read_csv('./SSE.csv')\n",
    "usd = pd.read_csv('./USD_index.csv')\n",
    "vix = pd.read_csv('./VIX.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Amazon 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# amzn['Date']=amzn['Date'].map('{}.{}.{}'.format{x[0:4],x[5:7],x[8:10]})\n",
    "amzn['Date'] = amzn['Date'].map(gold_date)\n",
    "amzn.drop(['Adj Close'],axis = 1,inplace = True)\n",
    "amzn.to_csv(\"fix/AMZN.csv\", mode='w',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.concat([df,amzn],axis=1, join='inner')\n",
    "df = pd.merge(df,amzn, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NASDAQ 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0       2010/01/04\n",
      "1       2010/01/05\n",
      "2       2010/01/06\n",
      "3       2010/01/07\n",
      "4       2010/01/08\n",
      "           ...    \n",
      "3012    2021/12/20\n",
      "3013    2021/12/21\n",
      "3014    2021/12/22\n",
      "3015    2021/12/23\n",
      "3016    2021/12/27\n",
      "Name: Date, Length: 3017, dtype: object\n"
     ]
    }
   ],
   "source": [
    "nasdaq['Date'] = nasdaq['Date'].map(slash_to_comma)\n",
    "nasdaq.drop(['Open','High','Low','Volume','Adj Close'],axis = 1,inplace = True)\n",
    "nasdaq.rename(columns = {'Close' : 'NASDAQ'}, inplace = True)\n",
    "nasdaq.to_csv(\"fix/NASDAQ.csv\", mode='w',index = False)\n",
    "print(nasdaq['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df,nasdaq, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NYSE 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open_x</th>\n",
       "      <th>High_x</th>\n",
       "      <th>Low_x</th>\n",
       "      <th>Close_x</th>\n",
       "      <th>Volume_x</th>\n",
       "      <th>Open_y</th>\n",
       "      <th>High_y</th>\n",
       "      <th>Low_y</th>\n",
       "      <th>Close_y</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Volume_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2010/07/01</td>\n",
       "      <td>108.900002</td>\n",
       "      <td>111.690002</td>\n",
       "      <td>106.699997</td>\n",
       "      <td>110.959999</td>\n",
       "      <td>8529800.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2010/07/02</td>\n",
       "      <td>110.919998</td>\n",
       "      <td>111.290001</td>\n",
       "      <td>108.559998</td>\n",
       "      <td>109.139999</td>\n",
       "      <td>4477100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2010/07/03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2010/07/04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2010/07/05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7210</td>\n",
       "      <td>2021-12-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14933.000000</td>\n",
       "      <td>15007.299805</td>\n",
       "      <td>14860.040039</td>\n",
       "      <td>14980.940430</td>\n",
       "      <td>14980.940430</td>\n",
       "      <td>4.576750e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7211</td>\n",
       "      <td>2021-12-21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15140.429688</td>\n",
       "      <td>15349.059570</td>\n",
       "      <td>15015.030273</td>\n",
       "      <td>15341.089844</td>\n",
       "      <td>15341.089844</td>\n",
       "      <td>4.546230e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7212</td>\n",
       "      <td>2021-12-22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15319.160156</td>\n",
       "      <td>15525.969727</td>\n",
       "      <td>15303.059570</td>\n",
       "      <td>15521.889648</td>\n",
       "      <td>15521.889648</td>\n",
       "      <td>4.080900e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7213</td>\n",
       "      <td>2021-12-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15544.790039</td>\n",
       "      <td>15697.980469</td>\n",
       "      <td>15528.910156</td>\n",
       "      <td>15653.370117</td>\n",
       "      <td>15653.370117</td>\n",
       "      <td>3.918200e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7214</td>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15696.830078</td>\n",
       "      <td>15871.400391</td>\n",
       "      <td>15696.830078</td>\n",
       "      <td>15871.259766</td>\n",
       "      <td>15871.259766</td>\n",
       "      <td>3.730120e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7215 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date      Open_x      High_x       Low_x     Close_x   Volume_x  \\\n",
       "0     2010/07/01  108.900002  111.690002  106.699997  110.959999  8529800.0   \n",
       "1     2010/07/02  110.919998  111.290001  108.559998  109.139999  4477100.0   \n",
       "2     2010/07/03         NaN         NaN         NaN         NaN        NaN   \n",
       "3     2010/07/04         NaN         NaN         NaN         NaN        NaN   \n",
       "4     2010/07/05         NaN         NaN         NaN         NaN        NaN   \n",
       "...          ...         ...         ...         ...         ...        ...   \n",
       "7210  2021-12-20         NaN         NaN         NaN         NaN        NaN   \n",
       "7211  2021-12-21         NaN         NaN         NaN         NaN        NaN   \n",
       "7212  2021-12-22         NaN         NaN         NaN         NaN        NaN   \n",
       "7213  2021-12-23         NaN         NaN         NaN         NaN        NaN   \n",
       "7214  2021-12-27         NaN         NaN         NaN         NaN        NaN   \n",
       "\n",
       "            Open_y        High_y         Low_y       Close_y     Adj Close  \\\n",
       "0              NaN           NaN           NaN           NaN           NaN   \n",
       "1              NaN           NaN           NaN           NaN           NaN   \n",
       "2              NaN           NaN           NaN           NaN           NaN   \n",
       "3              NaN           NaN           NaN           NaN           NaN   \n",
       "4              NaN           NaN           NaN           NaN           NaN   \n",
       "...            ...           ...           ...           ...           ...   \n",
       "7210  14933.000000  15007.299805  14860.040039  14980.940430  14980.940430   \n",
       "7211  15140.429688  15349.059570  15015.030273  15341.089844  15341.089844   \n",
       "7212  15319.160156  15525.969727  15303.059570  15521.889648  15521.889648   \n",
       "7213  15544.790039  15697.980469  15528.910156  15653.370117  15653.370117   \n",
       "7214  15696.830078  15871.400391  15696.830078  15871.259766  15871.259766   \n",
       "\n",
       "          Volume_y  \n",
       "0              NaN  \n",
       "1              NaN  \n",
       "2              NaN  \n",
       "3              NaN  \n",
       "4              NaN  \n",
       "...            ...  \n",
       "7210  4.576750e+09  \n",
       "7211  4.546230e+09  \n",
       "7212  4.080900e+09  \n",
       "7213  3.918200e+09  \n",
       "7214  3.730120e+09  \n",
       "\n",
       "[7215 rows x 12 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "nyse['Date'] = nyse['Date'].map(slash_to_comma)\n",
    "nyse.drop(['Open','High','Low','Volume','Adj Close'],axis = 1,inplace = True)\n",
    "nyse.rename(columns = {'Close' : 'NYSE'}, inplace = True)\n",
    "nyse.to_csv(\"fix/NYSE.csv\", mode='w',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(df,nyse, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open_x</th>\n",
       "      <th>High_x</th>\n",
       "      <th>Low_x</th>\n",
       "      <th>Close_x</th>\n",
       "      <th>Volume_x</th>\n",
       "      <th>Open_y</th>\n",
       "      <th>High_y</th>\n",
       "      <th>Low_y</th>\n",
       "      <th>Close_y</th>\n",
       "      <th>Adj Close</th>\n",
       "      <th>Volume_y</th>\n",
       "      <th>NYSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2010/07/01</td>\n",
       "      <td>108.900002</td>\n",
       "      <td>111.690002</td>\n",
       "      <td>106.699997</td>\n",
       "      <td>110.959999</td>\n",
       "      <td>8529800.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6462.029785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2010/07/02</td>\n",
       "      <td>110.919998</td>\n",
       "      <td>111.290001</td>\n",
       "      <td>108.559998</td>\n",
       "      <td>109.139999</td>\n",
       "      <td>4477100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6434.810059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2010/07/03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2010/07/04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2010/07/05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7334</td>\n",
       "      <td>2010/06/24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6730.240234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7335</td>\n",
       "      <td>2010/06/25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6763.930176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7336</td>\n",
       "      <td>2010/06/28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6736.600098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7337</td>\n",
       "      <td>2010/06/29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6520.089844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7338</td>\n",
       "      <td>2010/06/30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6469.649902</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7339 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date      Open_x      High_x       Low_x     Close_x   Volume_x  \\\n",
       "0     2010/07/01  108.900002  111.690002  106.699997  110.959999  8529800.0   \n",
       "1     2010/07/02  110.919998  111.290001  108.559998  109.139999  4477100.0   \n",
       "2     2010/07/03         NaN         NaN         NaN         NaN        NaN   \n",
       "3     2010/07/04         NaN         NaN         NaN         NaN        NaN   \n",
       "4     2010/07/05         NaN         NaN         NaN         NaN        NaN   \n",
       "...          ...         ...         ...         ...         ...        ...   \n",
       "7334  2010/06/24         NaN         NaN         NaN         NaN        NaN   \n",
       "7335  2010/06/25         NaN         NaN         NaN         NaN        NaN   \n",
       "7336  2010/06/28         NaN         NaN         NaN         NaN        NaN   \n",
       "7337  2010/06/29         NaN         NaN         NaN         NaN        NaN   \n",
       "7338  2010/06/30         NaN         NaN         NaN         NaN        NaN   \n",
       "\n",
       "      Open_y  High_y  Low_y  Close_y  Adj Close  Volume_y         NYSE  \n",
       "0        NaN     NaN    NaN      NaN        NaN       NaN  6462.029785  \n",
       "1        NaN     NaN    NaN      NaN        NaN       NaN  6434.810059  \n",
       "2        NaN     NaN    NaN      NaN        NaN       NaN          NaN  \n",
       "3        NaN     NaN    NaN      NaN        NaN       NaN          NaN  \n",
       "4        NaN     NaN    NaN      NaN        NaN       NaN          NaN  \n",
       "...      ...     ...    ...      ...        ...       ...          ...  \n",
       "7334     NaN     NaN    NaN      NaN        NaN       NaN  6730.240234  \n",
       "7335     NaN     NaN    NaN      NaN        NaN       NaN  6763.930176  \n",
       "7336     NaN     NaN    NaN      NaN        NaN       NaN  6736.600098  \n",
       "7337     NaN     NaN    NaN      NaN        NaN       NaN  6520.089844  \n",
       "7338     NaN     NaN    NaN      NaN        NaN       NaN  6469.649902  \n",
       "\n",
       "[7339 rows x 13 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## S&P 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# snp['Date'] = snp['Date'].map(slash_reorder)\n",
    "snp.drop(['Open','High','Low','Volume'],axis = 1,inplace = True)\n",
    "snp.rename(columns = {'Close/Last' : 'S&P 500'}, inplace = True)\n",
    "snp.to_csv(\"fix/S&P500.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,snp, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FTSE 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ftse['Date'] = ftse['Date'].map(slash_reorder_yr)\n",
    "ftse.drop([' Open', ' High', ' Low'],axis = 1,inplace = True)\n",
    "ftse.rename(columns = {' Close' : 'FTSE100'}, inplace = True)\n",
    "ftse.to_csv(\"fix/FTSE.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,ftse, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BSE SENSEX (사용X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bse_sensex['Date'] = bse_sensex['Date'].map(slash_reorder_yr)\n",
    "# bse_sensex.drop([' Open', ' High', ' Low'],axis = 1,inplace = True)\n",
    "# bse_sensex.rename(columns = {' Close' : 'BSE SENSEX'}, inplace = True)\n",
    "# bse_sensex.to_csv(\"fix/BSE_SENSEX.csv\", mode='w',index = False)\n",
    "\n",
    "# df = pd.merge(df,bse_sensex, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RUSSELL2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "russell['Date'] = russell['Date'].map(slash_to_comma)\n",
    "russell.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "russell.rename(columns = {'Close' : 'RUSSELL2000'}, inplace = True)\n",
    "russell.to_csv(\"fix/RUSSELL2000.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,russell, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sse['Date'] = sse['Date'].map(slash_to_comma)\n",
    "sse.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "sse.rename(columns = {'Close' : 'SSE'}, inplace = True)\n",
    "sse.to_csv(\"fix/SSE.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,sse, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crude Oil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "oil['Date'] = oil['Date'].map(gold_date)\n",
    "oil.rename(columns = {'DCOILWTICO' : 'Crude Oil'}, inplace = True)\n",
    "oil.to_csv(\"fix/OIL.csv\", mode='w',index = False)\n",
    "df = pd.merge(df,oil, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>GOLDAMGBD228NLBM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2010.1.4</td>\n",
       "      <td>1113.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2010.1.5</td>\n",
       "      <td>1125.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2010.1.6</td>\n",
       "      <td>1125.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2010.1.7</td>\n",
       "      <td>1130.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2010.1.8</td>\n",
       "      <td>1121.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3120</td>\n",
       "      <td>2021.12.20</td>\n",
       "      <td>1797.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3121</td>\n",
       "      <td>2021.12.21</td>\n",
       "      <td>1795.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3122</td>\n",
       "      <td>2021.12.22</td>\n",
       "      <td>1789.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3123</td>\n",
       "      <td>2021.12.23</td>\n",
       "      <td>1805.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3124</td>\n",
       "      <td>2021.12.24</td>\n",
       "      <td>1786.05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3125 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Date  GOLDAMGBD228NLBM\n",
       "0       2010.1.4           1113.00\n",
       "1       2010.1.5           1125.25\n",
       "2       2010.1.6           1125.00\n",
       "3       2010.1.7           1130.75\n",
       "4       2010.1.8           1121.75\n",
       "...          ...               ...\n",
       "3120  2021.12.20           1797.40\n",
       "3121  2021.12.21           1795.85\n",
       "3122  2021.12.22           1789.90\n",
       "3123  2021.12.23           1805.55\n",
       "3124  2021.12.24           1786.05\n",
       "\n",
       "[3125 rows x 2 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold['Date'] = gold['Date'].map(gold_date)\n",
    "gold.rename(columns = {'GOLDAMGBD228NLBM' : 'Gold'}, inplace = True)\n",
    "gold.to_csv(\"fix/GOLD.csv\", mode='w',index = False)\n",
    "df = pd.merge(df,gold, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VIX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "vix['Date'] = vix['Date'].map(slash_to_comma)\n",
    "vix.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "vix.rename(columns = {'Close' : 'VIX'}, inplace = True)\n",
    "vix.to_csv(\"fix/VIX.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,vix, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## USD Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "usd['Date'] = usd['Date'].map(slash_to_comma)\n",
    "usd.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "usd.rename(columns = {'Close' : 'USD index'}, inplace = True)\n",
    "usd.to_csv(\"fix/USD.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,usd, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "appl['Date'] = appl['Date'].map(slash_to_comma)\n",
    "appl.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "appl.rename(columns = {'Close' : 'Apple'}, inplace = True)\n",
    "appl.to_csv(\"fix/APPL.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,appl, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GOOGLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "google['Date'] = google['Date'].map(slash_to_comma)\n",
    "google.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "google.rename(columns = {'Close' : 'Google'}, inplace = True)\n",
    "google.to_csv(\"fix/GOOG.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,google, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MSFT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "msft['Date'] = msft['Date'].map(slash_to_comma)\n",
    "msft.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'],axis = 1,inplace = True)\n",
    "msft.rename(columns = {'Close' : 'Microsoft'}, inplace = True)\n",
    "msft.to_csv(\"fix/MSFT.csv\", mode='w',index = False)\n",
    "\n",
    "df = pd.merge(df,msft, how='outer',on='Date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Null값 제거"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Open'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Open'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-932a01d387e8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Open'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnotna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'#N/A'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2978\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2979\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2980\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2981\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2982\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2897\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Open'"
     ]
    }
   ],
   "source": [
    "df = df[df['Open'].notna()]\n",
    "df = df.fillna('#N/A')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('./preprocessed_data.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open_x</th>\n",
       "      <th>High_x</th>\n",
       "      <th>Low_x</th>\n",
       "      <th>Close_x</th>\n",
       "      <th>Volume_x</th>\n",
       "      <th>Open_y</th>\n",
       "      <th>High_y</th>\n",
       "      <th>Low_y</th>\n",
       "      <th>Close_y</th>\n",
       "      <th>...</th>\n",
       "      <th>FTSE100</th>\n",
       "      <th>RUSSELL2000</th>\n",
       "      <th>SSE</th>\n",
       "      <th>Crude Oil</th>\n",
       "      <th>Gold</th>\n",
       "      <th>VIX</th>\n",
       "      <th>USD index</th>\n",
       "      <th>Apple</th>\n",
       "      <th>Google</th>\n",
       "      <th>Microsoft</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2010/07/01</td>\n",
       "      <td>108.900002</td>\n",
       "      <td>111.690002</td>\n",
       "      <td>106.699997</td>\n",
       "      <td>110.959999</td>\n",
       "      <td>8529800.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4805.75</td>\n",
       "      <td>604.760010</td>\n",
       "      <td>2373.791992</td>\n",
       "      <td>72.95</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>32.860001</td>\n",
       "      <td>84.720001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>218.924423</td>\n",
       "      <td>23.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2010/07/02</td>\n",
       "      <td>110.919998</td>\n",
       "      <td>111.290001</td>\n",
       "      <td>108.559998</td>\n",
       "      <td>109.139999</td>\n",
       "      <td>4477100.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4838.09</td>\n",
       "      <td>598.969971</td>\n",
       "      <td>2382.900879</td>\n",
       "      <td>72.06</td>\n",
       "      <td>1210.5</td>\n",
       "      <td>30.120001</td>\n",
       "      <td>84.430000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>217.459900</td>\n",
       "      <td>23.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2010/07/03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2010/07/04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2010/07/05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>4823.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2363.947998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1208.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10256</td>\n",
       "      <td>2021.12.20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>169.750000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10257</td>\n",
       "      <td>2021.12.21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>172.990005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10258</td>\n",
       "      <td>2021.12.22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>175.639999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10259</td>\n",
       "      <td>2021.12.23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>176.279999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10260</td>\n",
       "      <td>2021.12.27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180.330002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10261 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Date      Open_x      High_x       Low_x     Close_x   Volume_x  \\\n",
       "0      2010/07/01  108.900002  111.690002  106.699997  110.959999  8529800.0   \n",
       "1      2010/07/02  110.919998  111.290001  108.559998  109.139999  4477100.0   \n",
       "2      2010/07/03         NaN         NaN         NaN         NaN        NaN   \n",
       "3      2010/07/04         NaN         NaN         NaN         NaN        NaN   \n",
       "4      2010/07/05         NaN         NaN         NaN         NaN        NaN   \n",
       "...           ...         ...         ...         ...         ...        ...   \n",
       "10256  2021.12.20         NaN         NaN         NaN         NaN        NaN   \n",
       "10257  2021.12.21         NaN         NaN         NaN         NaN        NaN   \n",
       "10258  2021.12.22         NaN         NaN         NaN         NaN        NaN   \n",
       "10259  2021.12.23         NaN         NaN         NaN         NaN        NaN   \n",
       "10260  2021.12.27         NaN         NaN         NaN         NaN        NaN   \n",
       "\n",
       "       Open_y  High_y  Low_y  Close_y  ...  FTSE100  RUSSELL2000          SSE  \\\n",
       "0         NaN     NaN    NaN      NaN  ...  4805.75   604.760010  2373.791992   \n",
       "1         NaN     NaN    NaN      NaN  ...  4838.09   598.969971  2382.900879   \n",
       "2         NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "3         NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "4         NaN     NaN    NaN      NaN  ...  4823.53          NaN  2363.947998   \n",
       "...       ...     ...    ...      ...  ...      ...          ...          ...   \n",
       "10256     NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "10257     NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "10258     NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "10259     NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "10260     NaN     NaN    NaN      NaN  ...      NaN          NaN          NaN   \n",
       "\n",
       "       Crude Oil    Gold        VIX  USD index       Apple      Google  \\\n",
       "0          72.95  1240.0  32.860001  84.720001         NaN  218.924423   \n",
       "1          72.06  1210.5  30.120001  84.430000         NaN  217.459900   \n",
       "2            NaN     NaN        NaN        NaN         NaN         NaN   \n",
       "3            NaN     NaN        NaN        NaN         NaN         NaN   \n",
       "4            NaN  1208.5        NaN        NaN         NaN         NaN   \n",
       "...          ...     ...        ...        ...         ...         ...   \n",
       "10256        NaN     NaN        NaN        NaN  169.750000         NaN   \n",
       "10257        NaN     NaN        NaN        NaN  172.990005         NaN   \n",
       "10258        NaN     NaN        NaN        NaN  175.639999         NaN   \n",
       "10259        NaN     NaN        NaN        NaN  176.279999         NaN   \n",
       "10260        NaN     NaN        NaN        NaN  180.330002         NaN   \n",
       "\n",
       "       Microsoft  \n",
       "0          23.16  \n",
       "1          23.27  \n",
       "2            NaN  \n",
       "3            NaN  \n",
       "4            NaN  \n",
       "...          ...  \n",
       "10256        NaN  \n",
       "10257        NaN  \n",
       "10258        NaN  \n",
       "10259        NaN  \n",
       "10260        NaN  \n",
       "\n",
       "[10261 rows x 24 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dataset.iloc[:, 1:] = pd.concat([dataset.iloc[:, 1:].ffill(), dataset.iloc[:, 1:].bfill()]).groupby(level=0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('./Stock-price-prediction-using-GAN/Code/Finaldata_with_Fourier.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Open', 'High', 'Low', 'Close', 'Volume', 'NASDAQ', 'NYSE', 'S&P 500',\n",
       "       'FTSE100', 'BSE SENSEX', 'RUSSELL2000', 'SSE', 'Crude Oil', 'Gold',\n",
       "       'VIX', 'USD index', 'Apple', 'Google', 'Microsoft', 'MA7', 'MA21',\n",
       "       'MACD', '20SD', 'upper_band', 'lower_band', 'EMA', 'logmomentum',\n",
       "       'absolute of 3 comp', 'angle of 3 comp', 'absolute of 6 comp',\n",
       "       'angle of 6 comp', 'absolute of 9 comp', 'angle of 9 comp'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dataset.iloc[:, 1:].ffill().columns # data 뺀거 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Open', 'High', 'Low', 'Close', 'Volume', 'NASDAQ', 'NYSE', 'S&P 500',\n",
       "       'FTSE100', 'BSE SENSEX', 'RUSSELL2000', 'SSE', 'Crude Oil', 'Gold',\n",
       "       'VIX', 'USD index', 'Apple', 'Google', 'Microsoft', 'MA7', 'MA21',\n",
       "       'MACD', '20SD', 'upper_band', 'lower_band', 'EMA', 'logmomentum',\n",
       "       'absolute of 3 comp', 'angle of 3 comp', 'absolute of 6 comp',\n",
       "       'angle of 6 comp', 'absolute of 9 comp', 'angle of 9 comp'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.iloc[:, 1:].bfill().columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Open', 'High', 'Low', 'Close', 'Volume', 'NASDAQ', 'NYSE', 'S&P 500',\n",
       "       'FTSE100', 'BSE SENSEX', 'RUSSELL2000', 'SSE', 'Crude Oil', 'Gold',\n",
       "       'VIX', 'USD index', 'Apple', 'Google', 'Microsoft', 'MA7', 'MA21',\n",
       "       'MACD', '20SD', 'upper_band', 'lower_band', 'EMA', 'logmomentum',\n",
       "       'absolute of 3 comp', 'angle of 3 comp', 'absolute of 6 comp',\n",
       "       'angle of 6 comp', 'absolute of 9 comp', 'angle of 9 comp'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.concat([dataset.iloc[:, 1:].ffill(), dataset.iloc[:, 1:].bfill()])\n",
    "a.columns\n",
    "# a.to_csv('concatted.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Open', 'High', 'Low', 'Close', 'Volume', 'NASDAQ', 'NYSE', 'FTSE100',\n",
       "       'BSE SENSEX', 'RUSSELL2000', 'SSE', 'Crude Oil', 'Gold', 'VIX',\n",
       "       'USD index', 'Apple', 'Google', 'Microsoft', 'MA7', 'MA21', 'MACD',\n",
       "       '20SD', 'upper_band', 'lower_band', 'EMA', 'logmomentum',\n",
       "       'absolute of 3 comp', 'angle of 3 comp', 'absolute of 6 comp',\n",
       "       'angle of 6 comp', 'absolute of 9 comp', 'angle of 9 comp'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = pd.concat([dataset.iloc[:, 1:].ffill(), dataset.iloc[:, 1:].bfill()]).groupby(level=0).mean()\n",
    "m.to_csv('mean.csv')\n",
    "m.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Open', 'High', 'Low', 'Close', 'Volume', 'NASDAQ', 'NYSE', 'FTSE100',\n",
       "       'BSE SENSEX', 'RUSSELL2000', 'SSE', 'Crude Oil', 'Gold', 'VIX',\n",
       "       'USD index', 'Apple', 'Google', 'Microsoft', 'MA7', 'MA21', 'MACD',\n",
       "       '20SD', 'upper_band', 'lower_band', 'EMA', 'logmomentum',\n",
       "       'absolute of 3 comp', 'angle of 3 comp', 'absolute of 6 comp',\n",
       "       'angle of 6 comp', 'absolute of 9 comp', 'angle of 9 comp'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.iloc[:, 1:].columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "sp = pd.read_csv('S&P500_before.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def sp_pre(x):\n",
    "    if '/' in x:\n",
    "        li = x.split('/')\n",
    "#         print(li)\n",
    "        x = li[2]+'/'+li[0].zfill(2)+'/'+li[1].zfill(2)\n",
    "        return x\n",
    "    else:\n",
    "        li = x.split('.')\n",
    "        x = li[0]+'/'+li[1].zfill(2)+'/'+li[2].zfill(2)\n",
    "        return x\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp['Date'] = sp['Date'].map(sp_pre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp.to_csv('S&P500.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('../../run/Finaldata_with_Fourier.csv', parse_dates=['Date'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>NASDAQ</th>\n",
       "      <th>NYSE</th>\n",
       "      <th>S&amp;P 500</th>\n",
       "      <th>FTSE100</th>\n",
       "      <th>...</th>\n",
       "      <th>upper_band</th>\n",
       "      <th>lower_band</th>\n",
       "      <th>EMA</th>\n",
       "      <th>logmomentum</th>\n",
       "      <th>absolute of 3 comp</th>\n",
       "      <th>angle of 3 comp</th>\n",
       "      <th>absolute of 6 comp</th>\n",
       "      <th>angle of 6 comp</th>\n",
       "      <th>absolute of 9 comp</th>\n",
       "      <th>angle of 9 comp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2011-12-19</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>179.000000</td>\n",
       "      <td>179.330002</td>\n",
       "      <td>4533600</td>\n",
       "      <td>2523.139893</td>\n",
       "      <td>7142.450195</td>\n",
       "      <td>1205.349976</td>\n",
       "      <td>5364.99</td>\n",
       "      <td>...</td>\n",
       "      <td>201.451825</td>\n",
       "      <td>177.334841</td>\n",
       "      <td>179.979513</td>\n",
       "      <td>5.183636</td>\n",
       "      <td>1887.195100</td>\n",
       "      <td>-0.117106</td>\n",
       "      <td>1773.441010</td>\n",
       "      <td>-0.035068</td>\n",
       "      <td>1844.920543</td>\n",
       "      <td>-0.047563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2011-12-20</td>\n",
       "      <td>182.690002</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>180.539993</td>\n",
       "      <td>182.520004</td>\n",
       "      <td>4600100</td>\n",
       "      <td>2603.729980</td>\n",
       "      <td>7360.970215</td>\n",
       "      <td>1241.300049</td>\n",
       "      <td>5419.60</td>\n",
       "      <td>...</td>\n",
       "      <td>201.097723</td>\n",
       "      <td>176.296563</td>\n",
       "      <td>181.673174</td>\n",
       "      <td>5.201366</td>\n",
       "      <td>1880.362306</td>\n",
       "      <td>-0.117668</td>\n",
       "      <td>1758.318228</td>\n",
       "      <td>-0.035409</td>\n",
       "      <td>1822.145141</td>\n",
       "      <td>-0.048260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2011-12-21</td>\n",
       "      <td>181.919998</td>\n",
       "      <td>183.500000</td>\n",
       "      <td>172.490005</td>\n",
       "      <td>174.350006</td>\n",
       "      <td>8066700</td>\n",
       "      <td>2577.969971</td>\n",
       "      <td>7388.520020</td>\n",
       "      <td>1243.719971</td>\n",
       "      <td>5389.74</td>\n",
       "      <td>...</td>\n",
       "      <td>201.797011</td>\n",
       "      <td>174.178227</td>\n",
       "      <td>176.791062</td>\n",
       "      <td>5.155313</td>\n",
       "      <td>1873.520814</td>\n",
       "      <td>-0.118229</td>\n",
       "      <td>1743.202331</td>\n",
       "      <td>-0.035749</td>\n",
       "      <td>1799.350687</td>\n",
       "      <td>-0.048950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2011-12-22</td>\n",
       "      <td>175.089996</td>\n",
       "      <td>179.669998</td>\n",
       "      <td>174.210007</td>\n",
       "      <td>179.029999</td>\n",
       "      <td>4694900</td>\n",
       "      <td>2599.449951</td>\n",
       "      <td>7460.750000</td>\n",
       "      <td>1254.000000</td>\n",
       "      <td>5456.97</td>\n",
       "      <td>...</td>\n",
       "      <td>201.686407</td>\n",
       "      <td>173.021213</td>\n",
       "      <td>178.283687</td>\n",
       "      <td>5.181952</td>\n",
       "      <td>1866.670782</td>\n",
       "      <td>-0.118787</td>\n",
       "      <td>1728.094451</td>\n",
       "      <td>-0.036086</td>\n",
       "      <td>1776.541160</td>\n",
       "      <td>-0.049633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2011-12-23</td>\n",
       "      <td>179.220001</td>\n",
       "      <td>179.679993</td>\n",
       "      <td>175.600006</td>\n",
       "      <td>177.279999</td>\n",
       "      <td>3496500</td>\n",
       "      <td>2618.639893</td>\n",
       "      <td>7518.660156</td>\n",
       "      <td>1265.329956</td>\n",
       "      <td>5512.70</td>\n",
       "      <td>...</td>\n",
       "      <td>201.667949</td>\n",
       "      <td>171.924432</td>\n",
       "      <td>177.614562</td>\n",
       "      <td>5.172074</td>\n",
       "      <td>1859.812370</td>\n",
       "      <td>-0.119344</td>\n",
       "      <td>1712.995716</td>\n",
       "      <td>-0.036421</td>\n",
       "      <td>1753.720549</td>\n",
       "      <td>-0.050309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2517</td>\n",
       "      <td>2021-12-20</td>\n",
       "      <td>3337.000000</td>\n",
       "      <td>3357.489990</td>\n",
       "      <td>3312.000000</td>\n",
       "      <td>3341.580078</td>\n",
       "      <td>2868600</td>\n",
       "      <td>14980.940430</td>\n",
       "      <td>16442.310550</td>\n",
       "      <td>4568.020000</td>\n",
       "      <td>7198.03</td>\n",
       "      <td>...</td>\n",
       "      <td>3626.227217</td>\n",
       "      <td>3327.484278</td>\n",
       "      <td>3361.005285</td>\n",
       "      <td>8.113900</td>\n",
       "      <td>1921.223050</td>\n",
       "      <td>-0.114266</td>\n",
       "      <td>1849.118524</td>\n",
       "      <td>-0.033329</td>\n",
       "      <td>1958.373191</td>\n",
       "      <td>-0.043998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2518</td>\n",
       "      <td>2021-12-21</td>\n",
       "      <td>3357.010010</td>\n",
       "      <td>3414.330078</td>\n",
       "      <td>3312.949951</td>\n",
       "      <td>3408.340088</td>\n",
       "      <td>2797800</td>\n",
       "      <td>15341.089840</td>\n",
       "      <td>16744.269530</td>\n",
       "      <td>4649.230000</td>\n",
       "      <td>7297.41</td>\n",
       "      <td>...</td>\n",
       "      <td>3606.901070</td>\n",
       "      <td>3321.264713</td>\n",
       "      <td>3392.561820</td>\n",
       "      <td>8.133687</td>\n",
       "      <td>1914.436122</td>\n",
       "      <td>-0.114838</td>\n",
       "      <td>1833.978328</td>\n",
       "      <td>-0.033680</td>\n",
       "      <td>1935.752384</td>\n",
       "      <td>-0.044721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2519</td>\n",
       "      <td>2021-12-22</td>\n",
       "      <td>3385.399902</td>\n",
       "      <td>3441.000000</td>\n",
       "      <td>3370.010010</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>2751800</td>\n",
       "      <td>15521.889650</td>\n",
       "      <td>16874.880860</td>\n",
       "      <td>4696.560000</td>\n",
       "      <td>7341.66</td>\n",
       "      <td>...</td>\n",
       "      <td>3588.499483</td>\n",
       "      <td>3325.206293</td>\n",
       "      <td>3411.347267</td>\n",
       "      <td>8.137320</td>\n",
       "      <td>1907.639705</td>\n",
       "      <td>-0.115407</td>\n",
       "      <td>1818.839341</td>\n",
       "      <td>-0.034030</td>\n",
       "      <td>1913.092786</td>\n",
       "      <td>-0.045439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2520</td>\n",
       "      <td>2021-12-23</td>\n",
       "      <td>3408.560059</td>\n",
       "      <td>3439.500000</td>\n",
       "      <td>3403.000000</td>\n",
       "      <td>3421.370117</td>\n",
       "      <td>1837500</td>\n",
       "      <td>15653.370120</td>\n",
       "      <td>16963.439450</td>\n",
       "      <td>4725.790000</td>\n",
       "      <td>7373.34</td>\n",
       "      <td>...</td>\n",
       "      <td>3566.357903</td>\n",
       "      <td>3332.236452</td>\n",
       "      <td>3418.029167</td>\n",
       "      <td>8.137504</td>\n",
       "      <td>1900.833957</td>\n",
       "      <td>-0.115975</td>\n",
       "      <td>1803.702702</td>\n",
       "      <td>-0.034378</td>\n",
       "      <td>1890.398321</td>\n",
       "      <td>-0.046153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2521</td>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>3458.860107</td>\n",
       "      <td>3384.310059</td>\n",
       "      <td>3393.389893</td>\n",
       "      <td>2934400</td>\n",
       "      <td>15871.259770</td>\n",
       "      <td>17141.089840</td>\n",
       "      <td>4791.190000</td>\n",
       "      <td>7372.10</td>\n",
       "      <td>...</td>\n",
       "      <td>3555.636171</td>\n",
       "      <td>3325.146753</td>\n",
       "      <td>3401.602984</td>\n",
       "      <td>8.129290</td>\n",
       "      <td>1894.019036</td>\n",
       "      <td>-0.116542</td>\n",
       "      <td>1788.569546</td>\n",
       "      <td>-0.034724</td>\n",
       "      <td>1867.672923</td>\n",
       "      <td>-0.046861</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2522 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date         Open         High          Low        Close   Volume  \\\n",
       "0    2011-12-19   182.000000   183.169998   179.000000   179.330002  4533600   \n",
       "1    2011-12-20   182.690002   183.169998   180.539993   182.520004  4600100   \n",
       "2    2011-12-21   181.919998   183.500000   172.490005   174.350006  8066700   \n",
       "3    2011-12-22   175.089996   179.669998   174.210007   179.029999  4694900   \n",
       "4    2011-12-23   179.220001   179.679993   175.600006   177.279999  3496500   \n",
       "...         ...          ...          ...          ...          ...      ...   \n",
       "2517 2021-12-20  3337.000000  3357.489990  3312.000000  3341.580078  2868600   \n",
       "2518 2021-12-21  3357.010010  3414.330078  3312.949951  3408.340088  2797800   \n",
       "2519 2021-12-22  3385.399902  3441.000000  3370.010010  3420.739990  2751800   \n",
       "2520 2021-12-23  3408.560059  3439.500000  3403.000000  3421.370117  1837500   \n",
       "2521 2021-12-27  3420.739990  3458.860107  3384.310059  3393.389893  2934400   \n",
       "\n",
       "            NASDAQ          NYSE      S&P 500  FTSE100  ...   upper_band  \\\n",
       "0      2523.139893   7142.450195  1205.349976  5364.99  ...   201.451825   \n",
       "1      2603.729980   7360.970215  1241.300049  5419.60  ...   201.097723   \n",
       "2      2577.969971   7388.520020  1243.719971  5389.74  ...   201.797011   \n",
       "3      2599.449951   7460.750000  1254.000000  5456.97  ...   201.686407   \n",
       "4      2618.639893   7518.660156  1265.329956  5512.70  ...   201.667949   \n",
       "...            ...           ...          ...      ...  ...          ...   \n",
       "2517  14980.940430  16442.310550  4568.020000  7198.03  ...  3626.227217   \n",
       "2518  15341.089840  16744.269530  4649.230000  7297.41  ...  3606.901070   \n",
       "2519  15521.889650  16874.880860  4696.560000  7341.66  ...  3588.499483   \n",
       "2520  15653.370120  16963.439450  4725.790000  7373.34  ...  3566.357903   \n",
       "2521  15871.259770  17141.089840  4791.190000  7372.10  ...  3555.636171   \n",
       "\n",
       "       lower_band          EMA  logmomentum  absolute of 3 comp  \\\n",
       "0      177.334841   179.979513     5.183636         1887.195100   \n",
       "1      176.296563   181.673174     5.201366         1880.362306   \n",
       "2      174.178227   176.791062     5.155313         1873.520814   \n",
       "3      173.021213   178.283687     5.181952         1866.670782   \n",
       "4      171.924432   177.614562     5.172074         1859.812370   \n",
       "...           ...          ...          ...                 ...   \n",
       "2517  3327.484278  3361.005285     8.113900         1921.223050   \n",
       "2518  3321.264713  3392.561820     8.133687         1914.436122   \n",
       "2519  3325.206293  3411.347267     8.137320         1907.639705   \n",
       "2520  3332.236452  3418.029167     8.137504         1900.833957   \n",
       "2521  3325.146753  3401.602984     8.129290         1894.019036   \n",
       "\n",
       "      angle of 3 comp  absolute of 6 comp  angle of 6 comp  \\\n",
       "0           -0.117106         1773.441010        -0.035068   \n",
       "1           -0.117668         1758.318228        -0.035409   \n",
       "2           -0.118229         1743.202331        -0.035749   \n",
       "3           -0.118787         1728.094451        -0.036086   \n",
       "4           -0.119344         1712.995716        -0.036421   \n",
       "...               ...                 ...              ...   \n",
       "2517        -0.114266         1849.118524        -0.033329   \n",
       "2518        -0.114838         1833.978328        -0.033680   \n",
       "2519        -0.115407         1818.839341        -0.034030   \n",
       "2520        -0.115975         1803.702702        -0.034378   \n",
       "2521        -0.116542         1788.569546        -0.034724   \n",
       "\n",
       "      absolute of 9 comp  angle of 9 comp  \n",
       "0            1844.920543        -0.047563  \n",
       "1            1822.145141        -0.048260  \n",
       "2            1799.350687        -0.048950  \n",
       "3            1776.541160        -0.049633  \n",
       "4            1753.720549        -0.050309  \n",
       "...                  ...              ...  \n",
       "2517         1958.373191        -0.043998  \n",
       "2518         1935.752384        -0.044721  \n",
       "2519         1913.092786        -0.045439  \n",
       "2520         1890.398321        -0.046153  \n",
       "2521         1867.672923        -0.046861  \n",
       "\n",
       "[2522 rows x 34 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "news = pd.read_csv(\"../Code/News.csv\", parse_dates=[\"Date\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2011-11-18</td>\n",
       "      <td>0.422875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2011-11-19</td>\n",
       "      <td>0.548850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2011-11-20</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2011-11-21</td>\n",
       "      <td>-0.325300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2011-11-22</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3644</td>\n",
       "      <td>2021-11-09</td>\n",
       "      <td>0.355160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3645</td>\n",
       "      <td>2021-11-10</td>\n",
       "      <td>0.208318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3646</td>\n",
       "      <td>2021-11-11</td>\n",
       "      <td>0.302594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3647</td>\n",
       "      <td>2021-11-12</td>\n",
       "      <td>0.242031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3648</td>\n",
       "      <td>2021-11-13</td>\n",
       "      <td>0.247312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3649 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date     Score\n",
       "0    2011-11-18  0.422875\n",
       "1    2011-11-19  0.548850\n",
       "2    2011-11-20  0.100000\n",
       "3    2011-11-21 -0.325300\n",
       "4    2011-11-22  0.100000\n",
       "...         ...       ...\n",
       "3644 2021-11-09  0.355160\n",
       "3645 2021-11-10  0.208318\n",
       "3646 2021-11-11  0.302594\n",
       "3647 2021-11-12  0.242031\n",
       "3648 2021-11-13  0.247312\n",
       "\n",
       "[3649 rows x 2 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>NASDAQ</th>\n",
       "      <th>NYSE</th>\n",
       "      <th>S&amp;P 500</th>\n",
       "      <th>FTSE100</th>\n",
       "      <th>BSE SENSEX</th>\n",
       "      <th>...</th>\n",
       "      <th>lower_band</th>\n",
       "      <th>EMA</th>\n",
       "      <th>logmomentum</th>\n",
       "      <th>absolute of 3 comp</th>\n",
       "      <th>angle of 3 comp</th>\n",
       "      <th>absolute of 6 comp</th>\n",
       "      <th>angle of 6 comp</th>\n",
       "      <th>absolute of 9 comp</th>\n",
       "      <th>angle of 9 comp</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2011-12-19</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>179.000000</td>\n",
       "      <td>179.330002</td>\n",
       "      <td>4533600</td>\n",
       "      <td>2523.139893</td>\n",
       "      <td>7142.450195</td>\n",
       "      <td>1205.349976</td>\n",
       "      <td>5364.99</td>\n",
       "      <td>15379.34</td>\n",
       "      <td>...</td>\n",
       "      <td>177.334841</td>\n",
       "      <td>179.979513</td>\n",
       "      <td>5.183636</td>\n",
       "      <td>1887.195100</td>\n",
       "      <td>-0.117106</td>\n",
       "      <td>1773.441010</td>\n",
       "      <td>-0.035068</td>\n",
       "      <td>1844.920543</td>\n",
       "      <td>-0.047563</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2011-12-20</td>\n",
       "      <td>182.690002</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>180.539993</td>\n",
       "      <td>182.520004</td>\n",
       "      <td>4600100</td>\n",
       "      <td>2603.729980</td>\n",
       "      <td>7360.970215</td>\n",
       "      <td>1241.300049</td>\n",
       "      <td>5419.60</td>\n",
       "      <td>15175.08</td>\n",
       "      <td>...</td>\n",
       "      <td>176.296563</td>\n",
       "      <td>181.673174</td>\n",
       "      <td>5.201366</td>\n",
       "      <td>1880.362306</td>\n",
       "      <td>-0.117668</td>\n",
       "      <td>1758.318228</td>\n",
       "      <td>-0.035409</td>\n",
       "      <td>1822.145141</td>\n",
       "      <td>-0.048260</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2011-12-21</td>\n",
       "      <td>181.919998</td>\n",
       "      <td>183.500000</td>\n",
       "      <td>172.490005</td>\n",
       "      <td>174.350006</td>\n",
       "      <td>8066700</td>\n",
       "      <td>2577.969971</td>\n",
       "      <td>7388.520020</td>\n",
       "      <td>1243.719971</td>\n",
       "      <td>5389.74</td>\n",
       "      <td>15685.21</td>\n",
       "      <td>...</td>\n",
       "      <td>174.178227</td>\n",
       "      <td>176.791062</td>\n",
       "      <td>5.155313</td>\n",
       "      <td>1873.520814</td>\n",
       "      <td>-0.118229</td>\n",
       "      <td>1743.202331</td>\n",
       "      <td>-0.035749</td>\n",
       "      <td>1799.350687</td>\n",
       "      <td>-0.048950</td>\n",
       "      <td>0.45100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2011-12-22</td>\n",
       "      <td>175.089996</td>\n",
       "      <td>179.669998</td>\n",
       "      <td>174.210007</td>\n",
       "      <td>179.029999</td>\n",
       "      <td>4694900</td>\n",
       "      <td>2599.449951</td>\n",
       "      <td>7460.750000</td>\n",
       "      <td>1254.000000</td>\n",
       "      <td>5456.97</td>\n",
       "      <td>15813.36</td>\n",
       "      <td>...</td>\n",
       "      <td>173.021213</td>\n",
       "      <td>178.283687</td>\n",
       "      <td>5.181952</td>\n",
       "      <td>1866.670782</td>\n",
       "      <td>-0.118787</td>\n",
       "      <td>1728.094451</td>\n",
       "      <td>-0.036086</td>\n",
       "      <td>1776.541160</td>\n",
       "      <td>-0.049633</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2011-12-23</td>\n",
       "      <td>179.220001</td>\n",
       "      <td>179.679993</td>\n",
       "      <td>175.600006</td>\n",
       "      <td>177.279999</td>\n",
       "      <td>3496500</td>\n",
       "      <td>2618.639893</td>\n",
       "      <td>7518.660156</td>\n",
       "      <td>1265.329956</td>\n",
       "      <td>5512.70</td>\n",
       "      <td>15738.70</td>\n",
       "      <td>...</td>\n",
       "      <td>171.924432</td>\n",
       "      <td>177.614562</td>\n",
       "      <td>5.172074</td>\n",
       "      <td>1859.812370</td>\n",
       "      <td>-0.119344</td>\n",
       "      <td>1712.995716</td>\n",
       "      <td>-0.036421</td>\n",
       "      <td>1753.720549</td>\n",
       "      <td>-0.050309</td>\n",
       "      <td>0.35515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-12-20</td>\n",
       "      <td>3337.000000</td>\n",
       "      <td>3357.489990</td>\n",
       "      <td>3312.000000</td>\n",
       "      <td>3341.580078</td>\n",
       "      <td>2868600</td>\n",
       "      <td>14980.940430</td>\n",
       "      <td>16442.310550</td>\n",
       "      <td>4568.020000</td>\n",
       "      <td>7198.03</td>\n",
       "      <td>55822.01</td>\n",
       "      <td>...</td>\n",
       "      <td>3327.484278</td>\n",
       "      <td>3361.005285</td>\n",
       "      <td>8.113900</td>\n",
       "      <td>1921.223050</td>\n",
       "      <td>-0.114266</td>\n",
       "      <td>1849.118524</td>\n",
       "      <td>-0.033329</td>\n",
       "      <td>1958.373191</td>\n",
       "      <td>-0.043998</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-12-21</td>\n",
       "      <td>3357.010010</td>\n",
       "      <td>3414.330078</td>\n",
       "      <td>3312.949951</td>\n",
       "      <td>3408.340088</td>\n",
       "      <td>2797800</td>\n",
       "      <td>15341.089840</td>\n",
       "      <td>16744.269530</td>\n",
       "      <td>4649.230000</td>\n",
       "      <td>7297.41</td>\n",
       "      <td>56319.01</td>\n",
       "      <td>...</td>\n",
       "      <td>3321.264713</td>\n",
       "      <td>3392.561820</td>\n",
       "      <td>8.133687</td>\n",
       "      <td>1914.436122</td>\n",
       "      <td>-0.114838</td>\n",
       "      <td>1833.978328</td>\n",
       "      <td>-0.033680</td>\n",
       "      <td>1935.752384</td>\n",
       "      <td>-0.044721</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-12-22</td>\n",
       "      <td>3385.399902</td>\n",
       "      <td>3441.000000</td>\n",
       "      <td>3370.010010</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>2751800</td>\n",
       "      <td>15521.889650</td>\n",
       "      <td>16874.880860</td>\n",
       "      <td>4696.560000</td>\n",
       "      <td>7341.66</td>\n",
       "      <td>56930.56</td>\n",
       "      <td>...</td>\n",
       "      <td>3325.206293</td>\n",
       "      <td>3411.347267</td>\n",
       "      <td>8.137320</td>\n",
       "      <td>1907.639705</td>\n",
       "      <td>-0.115407</td>\n",
       "      <td>1818.839341</td>\n",
       "      <td>-0.034030</td>\n",
       "      <td>1913.092786</td>\n",
       "      <td>-0.045439</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-12-23</td>\n",
       "      <td>3408.560059</td>\n",
       "      <td>3439.500000</td>\n",
       "      <td>3403.000000</td>\n",
       "      <td>3421.370117</td>\n",
       "      <td>1837500</td>\n",
       "      <td>15653.370120</td>\n",
       "      <td>16963.439450</td>\n",
       "      <td>4725.790000</td>\n",
       "      <td>7373.34</td>\n",
       "      <td>57315.28</td>\n",
       "      <td>...</td>\n",
       "      <td>3332.236452</td>\n",
       "      <td>3418.029167</td>\n",
       "      <td>8.137504</td>\n",
       "      <td>1900.833957</td>\n",
       "      <td>-0.115975</td>\n",
       "      <td>1803.702702</td>\n",
       "      <td>-0.034378</td>\n",
       "      <td>1890.398321</td>\n",
       "      <td>-0.046153</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>3458.860107</td>\n",
       "      <td>3384.310059</td>\n",
       "      <td>3393.389893</td>\n",
       "      <td>2934400</td>\n",
       "      <td>15871.259770</td>\n",
       "      <td>17141.089840</td>\n",
       "      <td>4791.190000</td>\n",
       "      <td>7372.10</td>\n",
       "      <td>57420.24</td>\n",
       "      <td>...</td>\n",
       "      <td>3325.146753</td>\n",
       "      <td>3401.602984</td>\n",
       "      <td>8.129290</td>\n",
       "      <td>1894.019036</td>\n",
       "      <td>-0.116542</td>\n",
       "      <td>1788.569546</td>\n",
       "      <td>-0.034724</td>\n",
       "      <td>1867.672923</td>\n",
       "      <td>-0.046861</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2522 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Open         High          Low        Close   Volume  \\\n",
       "Date                                                                      \n",
       "2011-12-19   182.000000   183.169998   179.000000   179.330002  4533600   \n",
       "2011-12-20   182.690002   183.169998   180.539993   182.520004  4600100   \n",
       "2011-12-21   181.919998   183.500000   172.490005   174.350006  8066700   \n",
       "2011-12-22   175.089996   179.669998   174.210007   179.029999  4694900   \n",
       "2011-12-23   179.220001   179.679993   175.600006   177.279999  3496500   \n",
       "...                 ...          ...          ...          ...      ...   \n",
       "2021-12-20  3337.000000  3357.489990  3312.000000  3341.580078  2868600   \n",
       "2021-12-21  3357.010010  3414.330078  3312.949951  3408.340088  2797800   \n",
       "2021-12-22  3385.399902  3441.000000  3370.010010  3420.739990  2751800   \n",
       "2021-12-23  3408.560059  3439.500000  3403.000000  3421.370117  1837500   \n",
       "2021-12-27  3420.739990  3458.860107  3384.310059  3393.389893  2934400   \n",
       "\n",
       "                  NASDAQ          NYSE      S&P 500  FTSE100  BSE SENSEX  ...  \\\n",
       "Date                                                                      ...   \n",
       "2011-12-19   2523.139893   7142.450195  1205.349976  5364.99    15379.34  ...   \n",
       "2011-12-20   2603.729980   7360.970215  1241.300049  5419.60    15175.08  ...   \n",
       "2011-12-21   2577.969971   7388.520020  1243.719971  5389.74    15685.21  ...   \n",
       "2011-12-22   2599.449951   7460.750000  1254.000000  5456.97    15813.36  ...   \n",
       "2011-12-23   2618.639893   7518.660156  1265.329956  5512.70    15738.70  ...   \n",
       "...                  ...           ...          ...      ...         ...  ...   \n",
       "2021-12-20  14980.940430  16442.310550  4568.020000  7198.03    55822.01  ...   \n",
       "2021-12-21  15341.089840  16744.269530  4649.230000  7297.41    56319.01  ...   \n",
       "2021-12-22  15521.889650  16874.880860  4696.560000  7341.66    56930.56  ...   \n",
       "2021-12-23  15653.370120  16963.439450  4725.790000  7373.34    57315.28  ...   \n",
       "2021-12-27  15871.259770  17141.089840  4791.190000  7372.10    57420.24  ...   \n",
       "\n",
       "             lower_band          EMA  logmomentum  absolute of 3 comp  \\\n",
       "Date                                                                    \n",
       "2011-12-19   177.334841   179.979513     5.183636         1887.195100   \n",
       "2011-12-20   176.296563   181.673174     5.201366         1880.362306   \n",
       "2011-12-21   174.178227   176.791062     5.155313         1873.520814   \n",
       "2011-12-22   173.021213   178.283687     5.181952         1866.670782   \n",
       "2011-12-23   171.924432   177.614562     5.172074         1859.812370   \n",
       "...                 ...          ...          ...                 ...   \n",
       "2021-12-20  3327.484278  3361.005285     8.113900         1921.223050   \n",
       "2021-12-21  3321.264713  3392.561820     8.133687         1914.436122   \n",
       "2021-12-22  3325.206293  3411.347267     8.137320         1907.639705   \n",
       "2021-12-23  3332.236452  3418.029167     8.137504         1900.833957   \n",
       "2021-12-27  3325.146753  3401.602984     8.129290         1894.019036   \n",
       "\n",
       "            angle of 3 comp  absolute of 6 comp  angle of 6 comp  \\\n",
       "Date                                                               \n",
       "2011-12-19        -0.117106         1773.441010        -0.035068   \n",
       "2011-12-20        -0.117668         1758.318228        -0.035409   \n",
       "2011-12-21        -0.118229         1743.202331        -0.035749   \n",
       "2011-12-22        -0.118787         1728.094451        -0.036086   \n",
       "2011-12-23        -0.119344         1712.995716        -0.036421   \n",
       "...                     ...                 ...              ...   \n",
       "2021-12-20        -0.114266         1849.118524        -0.033329   \n",
       "2021-12-21        -0.114838         1833.978328        -0.033680   \n",
       "2021-12-22        -0.115407         1818.839341        -0.034030   \n",
       "2021-12-23        -0.115975         1803.702702        -0.034378   \n",
       "2021-12-27        -0.116542         1788.569546        -0.034724   \n",
       "\n",
       "            absolute of 9 comp  angle of 9 comp    Score  \n",
       "Date                                                      \n",
       "2011-12-19         1844.920543        -0.047563  0.10000  \n",
       "2011-12-20         1822.145141        -0.048260  0.10000  \n",
       "2011-12-21         1799.350687        -0.048950  0.45100  \n",
       "2011-12-22         1776.541160        -0.049633  0.10000  \n",
       "2011-12-23         1753.720549        -0.050309  0.35515  \n",
       "...                        ...              ...      ...  \n",
       "2021-12-20         1958.373191        -0.043998      NaN  \n",
       "2021-12-21         1935.752384        -0.044721      NaN  \n",
       "2021-12-22         1913.092786        -0.045439      NaN  \n",
       "2021-12-23         1890.398321        -0.046153      NaN  \n",
       "2021-12-27         1867.672923        -0.046861      NaN  \n",
       "\n",
       "[2522 rows x 34 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.set_index('Date', inplace=True)\n",
    "news.set_index('Date', inplace=True)\n",
    "dataset.join(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.join(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>NASDAQ</th>\n",
       "      <th>NYSE</th>\n",
       "      <th>S&amp;P 500</th>\n",
       "      <th>FTSE100</th>\n",
       "      <th>...</th>\n",
       "      <th>lower_band</th>\n",
       "      <th>EMA</th>\n",
       "      <th>logmomentum</th>\n",
       "      <th>absolute of 3 comp</th>\n",
       "      <th>angle of 3 comp</th>\n",
       "      <th>absolute of 6 comp</th>\n",
       "      <th>angle of 6 comp</th>\n",
       "      <th>absolute of 9 comp</th>\n",
       "      <th>angle of 9 comp</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2011-12-19</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>179.000000</td>\n",
       "      <td>179.330002</td>\n",
       "      <td>4533600</td>\n",
       "      <td>2523.139893</td>\n",
       "      <td>7142.450195</td>\n",
       "      <td>1205.349976</td>\n",
       "      <td>5364.99</td>\n",
       "      <td>...</td>\n",
       "      <td>177.334841</td>\n",
       "      <td>179.979513</td>\n",
       "      <td>5.183636</td>\n",
       "      <td>1887.195100</td>\n",
       "      <td>-0.117106</td>\n",
       "      <td>1773.441010</td>\n",
       "      <td>-0.035068</td>\n",
       "      <td>1844.920543</td>\n",
       "      <td>-0.047563</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2011-12-20</td>\n",
       "      <td>182.690002</td>\n",
       "      <td>183.169998</td>\n",
       "      <td>180.539993</td>\n",
       "      <td>182.520004</td>\n",
       "      <td>4600100</td>\n",
       "      <td>2603.729980</td>\n",
       "      <td>7360.970215</td>\n",
       "      <td>1241.300049</td>\n",
       "      <td>5419.60</td>\n",
       "      <td>...</td>\n",
       "      <td>176.296563</td>\n",
       "      <td>181.673174</td>\n",
       "      <td>5.201366</td>\n",
       "      <td>1880.362306</td>\n",
       "      <td>-0.117668</td>\n",
       "      <td>1758.318228</td>\n",
       "      <td>-0.035409</td>\n",
       "      <td>1822.145141</td>\n",
       "      <td>-0.048260</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2011-12-21</td>\n",
       "      <td>181.919998</td>\n",
       "      <td>183.500000</td>\n",
       "      <td>172.490005</td>\n",
       "      <td>174.350006</td>\n",
       "      <td>8066700</td>\n",
       "      <td>2577.969971</td>\n",
       "      <td>7388.520020</td>\n",
       "      <td>1243.719971</td>\n",
       "      <td>5389.74</td>\n",
       "      <td>...</td>\n",
       "      <td>174.178227</td>\n",
       "      <td>176.791062</td>\n",
       "      <td>5.155313</td>\n",
       "      <td>1873.520814</td>\n",
       "      <td>-0.118229</td>\n",
       "      <td>1743.202331</td>\n",
       "      <td>-0.035749</td>\n",
       "      <td>1799.350687</td>\n",
       "      <td>-0.048950</td>\n",
       "      <td>0.45100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2011-12-22</td>\n",
       "      <td>175.089996</td>\n",
       "      <td>179.669998</td>\n",
       "      <td>174.210007</td>\n",
       "      <td>179.029999</td>\n",
       "      <td>4694900</td>\n",
       "      <td>2599.449951</td>\n",
       "      <td>7460.750000</td>\n",
       "      <td>1254.000000</td>\n",
       "      <td>5456.97</td>\n",
       "      <td>...</td>\n",
       "      <td>173.021213</td>\n",
       "      <td>178.283687</td>\n",
       "      <td>5.181952</td>\n",
       "      <td>1866.670782</td>\n",
       "      <td>-0.118787</td>\n",
       "      <td>1728.094451</td>\n",
       "      <td>-0.036086</td>\n",
       "      <td>1776.541160</td>\n",
       "      <td>-0.049633</td>\n",
       "      <td>0.10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2011-12-23</td>\n",
       "      <td>179.220001</td>\n",
       "      <td>179.679993</td>\n",
       "      <td>175.600006</td>\n",
       "      <td>177.279999</td>\n",
       "      <td>3496500</td>\n",
       "      <td>2618.639893</td>\n",
       "      <td>7518.660156</td>\n",
       "      <td>1265.329956</td>\n",
       "      <td>5512.70</td>\n",
       "      <td>...</td>\n",
       "      <td>171.924432</td>\n",
       "      <td>177.614562</td>\n",
       "      <td>5.172074</td>\n",
       "      <td>1859.812370</td>\n",
       "      <td>-0.119344</td>\n",
       "      <td>1712.995716</td>\n",
       "      <td>-0.036421</td>\n",
       "      <td>1753.720549</td>\n",
       "      <td>-0.050309</td>\n",
       "      <td>0.35515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2517</td>\n",
       "      <td>2021-12-20</td>\n",
       "      <td>3337.000000</td>\n",
       "      <td>3357.489990</td>\n",
       "      <td>3312.000000</td>\n",
       "      <td>3341.580078</td>\n",
       "      <td>2868600</td>\n",
       "      <td>14980.940430</td>\n",
       "      <td>16442.310550</td>\n",
       "      <td>4568.020000</td>\n",
       "      <td>7198.03</td>\n",
       "      <td>...</td>\n",
       "      <td>3327.484278</td>\n",
       "      <td>3361.005285</td>\n",
       "      <td>8.113900</td>\n",
       "      <td>1921.223050</td>\n",
       "      <td>-0.114266</td>\n",
       "      <td>1849.118524</td>\n",
       "      <td>-0.033329</td>\n",
       "      <td>1958.373191</td>\n",
       "      <td>-0.043998</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2518</td>\n",
       "      <td>2021-12-21</td>\n",
       "      <td>3357.010010</td>\n",
       "      <td>3414.330078</td>\n",
       "      <td>3312.949951</td>\n",
       "      <td>3408.340088</td>\n",
       "      <td>2797800</td>\n",
       "      <td>15341.089840</td>\n",
       "      <td>16744.269530</td>\n",
       "      <td>4649.230000</td>\n",
       "      <td>7297.41</td>\n",
       "      <td>...</td>\n",
       "      <td>3321.264713</td>\n",
       "      <td>3392.561820</td>\n",
       "      <td>8.133687</td>\n",
       "      <td>1914.436122</td>\n",
       "      <td>-0.114838</td>\n",
       "      <td>1833.978328</td>\n",
       "      <td>-0.033680</td>\n",
       "      <td>1935.752384</td>\n",
       "      <td>-0.044721</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2519</td>\n",
       "      <td>2021-12-22</td>\n",
       "      <td>3385.399902</td>\n",
       "      <td>3441.000000</td>\n",
       "      <td>3370.010010</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>2751800</td>\n",
       "      <td>15521.889650</td>\n",
       "      <td>16874.880860</td>\n",
       "      <td>4696.560000</td>\n",
       "      <td>7341.66</td>\n",
       "      <td>...</td>\n",
       "      <td>3325.206293</td>\n",
       "      <td>3411.347267</td>\n",
       "      <td>8.137320</td>\n",
       "      <td>1907.639705</td>\n",
       "      <td>-0.115407</td>\n",
       "      <td>1818.839341</td>\n",
       "      <td>-0.034030</td>\n",
       "      <td>1913.092786</td>\n",
       "      <td>-0.045439</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2520</td>\n",
       "      <td>2021-12-23</td>\n",
       "      <td>3408.560059</td>\n",
       "      <td>3439.500000</td>\n",
       "      <td>3403.000000</td>\n",
       "      <td>3421.370117</td>\n",
       "      <td>1837500</td>\n",
       "      <td>15653.370120</td>\n",
       "      <td>16963.439450</td>\n",
       "      <td>4725.790000</td>\n",
       "      <td>7373.34</td>\n",
       "      <td>...</td>\n",
       "      <td>3332.236452</td>\n",
       "      <td>3418.029167</td>\n",
       "      <td>8.137504</td>\n",
       "      <td>1900.833957</td>\n",
       "      <td>-0.115975</td>\n",
       "      <td>1803.702702</td>\n",
       "      <td>-0.034378</td>\n",
       "      <td>1890.398321</td>\n",
       "      <td>-0.046153</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2521</td>\n",
       "      <td>2021-12-27</td>\n",
       "      <td>3420.739990</td>\n",
       "      <td>3458.860107</td>\n",
       "      <td>3384.310059</td>\n",
       "      <td>3393.389893</td>\n",
       "      <td>2934400</td>\n",
       "      <td>15871.259770</td>\n",
       "      <td>17141.089840</td>\n",
       "      <td>4791.190000</td>\n",
       "      <td>7372.10</td>\n",
       "      <td>...</td>\n",
       "      <td>3325.146753</td>\n",
       "      <td>3401.602984</td>\n",
       "      <td>8.129290</td>\n",
       "      <td>1894.019036</td>\n",
       "      <td>-0.116542</td>\n",
       "      <td>1788.569546</td>\n",
       "      <td>-0.034724</td>\n",
       "      <td>1867.672923</td>\n",
       "      <td>-0.046861</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2522 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date         Open         High          Low        Close   Volume  \\\n",
       "0    2011-12-19   182.000000   183.169998   179.000000   179.330002  4533600   \n",
       "1    2011-12-20   182.690002   183.169998   180.539993   182.520004  4600100   \n",
       "2    2011-12-21   181.919998   183.500000   172.490005   174.350006  8066700   \n",
       "3    2011-12-22   175.089996   179.669998   174.210007   179.029999  4694900   \n",
       "4    2011-12-23   179.220001   179.679993   175.600006   177.279999  3496500   \n",
       "...         ...          ...          ...          ...          ...      ...   \n",
       "2517 2021-12-20  3337.000000  3357.489990  3312.000000  3341.580078  2868600   \n",
       "2518 2021-12-21  3357.010010  3414.330078  3312.949951  3408.340088  2797800   \n",
       "2519 2021-12-22  3385.399902  3441.000000  3370.010010  3420.739990  2751800   \n",
       "2520 2021-12-23  3408.560059  3439.500000  3403.000000  3421.370117  1837500   \n",
       "2521 2021-12-27  3420.739990  3458.860107  3384.310059  3393.389893  2934400   \n",
       "\n",
       "            NASDAQ          NYSE      S&P 500  FTSE100  ...   lower_band  \\\n",
       "0      2523.139893   7142.450195  1205.349976  5364.99  ...   177.334841   \n",
       "1      2603.729980   7360.970215  1241.300049  5419.60  ...   176.296563   \n",
       "2      2577.969971   7388.520020  1243.719971  5389.74  ...   174.178227   \n",
       "3      2599.449951   7460.750000  1254.000000  5456.97  ...   173.021213   \n",
       "4      2618.639893   7518.660156  1265.329956  5512.70  ...   171.924432   \n",
       "...            ...           ...          ...      ...  ...          ...   \n",
       "2517  14980.940430  16442.310550  4568.020000  7198.03  ...  3327.484278   \n",
       "2518  15341.089840  16744.269530  4649.230000  7297.41  ...  3321.264713   \n",
       "2519  15521.889650  16874.880860  4696.560000  7341.66  ...  3325.206293   \n",
       "2520  15653.370120  16963.439450  4725.790000  7373.34  ...  3332.236452   \n",
       "2521  15871.259770  17141.089840  4791.190000  7372.10  ...  3325.146753   \n",
       "\n",
       "              EMA  logmomentum  absolute of 3 comp  angle of 3 comp  \\\n",
       "0      179.979513     5.183636         1887.195100        -0.117106   \n",
       "1      181.673174     5.201366         1880.362306        -0.117668   \n",
       "2      176.791062     5.155313         1873.520814        -0.118229   \n",
       "3      178.283687     5.181952         1866.670782        -0.118787   \n",
       "4      177.614562     5.172074         1859.812370        -0.119344   \n",
       "...           ...          ...                 ...              ...   \n",
       "2517  3361.005285     8.113900         1921.223050        -0.114266   \n",
       "2518  3392.561820     8.133687         1914.436122        -0.114838   \n",
       "2519  3411.347267     8.137320         1907.639705        -0.115407   \n",
       "2520  3418.029167     8.137504         1900.833957        -0.115975   \n",
       "2521  3401.602984     8.129290         1894.019036        -0.116542   \n",
       "\n",
       "      absolute of 6 comp  angle of 6 comp  absolute of 9 comp  \\\n",
       "0            1773.441010        -0.035068         1844.920543   \n",
       "1            1758.318228        -0.035409         1822.145141   \n",
       "2            1743.202331        -0.035749         1799.350687   \n",
       "3            1728.094451        -0.036086         1776.541160   \n",
       "4            1712.995716        -0.036421         1753.720549   \n",
       "...                  ...              ...                 ...   \n",
       "2517         1849.118524        -0.033329         1958.373191   \n",
       "2518         1833.978328        -0.033680         1935.752384   \n",
       "2519         1818.839341        -0.034030         1913.092786   \n",
       "2520         1803.702702        -0.034378         1890.398321   \n",
       "2521         1788.569546        -0.034724         1867.672923   \n",
       "\n",
       "      angle of 9 comp    Score  \n",
       "0           -0.047563  0.10000  \n",
       "1           -0.048260  0.10000  \n",
       "2           -0.048950  0.45100  \n",
       "3           -0.049633  0.10000  \n",
       "4           -0.050309  0.35515  \n",
       "...               ...      ...  \n",
       "2517        -0.043998      NaN  \n",
       "2518        -0.044721      NaN  \n",
       "2519        -0.045439      NaN  \n",
       "2520        -0.046153      NaN  \n",
       "2521        -0.046861      NaN  \n",
       "\n",
       "[2522 rows x 35 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Date\n",
       "2011-12-19    0.10000\n",
       "2011-12-20    0.10000\n",
       "2011-12-21    0.45100\n",
       "2011-12-22    0.10000\n",
       "2011-12-23    0.35515\n",
       "               ...   \n",
       "2021-12-20    0.10000\n",
       "2021-12-21    0.10000\n",
       "2021-12-22    0.10000\n",
       "2021-12-23    0.10000\n",
       "2021-12-27    0.10000\n",
       "Name: Score, Length: 2522, dtype: float64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"Score\"].fillna(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}