final code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Jupyter file can be run in Google colab environment. The cells includes are sorted so just run one follows by the one above. The code requires files: \"test.csv.zip\", \"train.csv.zip\" and \"suffled_images.csv\", the three files should be put into your working directory. The models I used are CNN and LSTM, I uploaded two saved models in Learn as well. For RNN the model is named \"char_rnn_lstm_classification_model\", for CNN it is named \"best_model\". And I use ensembled learning to concatenate their output probability using simple neural network, this model is also saved as \"ensembled_model.pt\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf                                          # 1)\n",
    "import numpy as np                                               # 2)\n",
    "import keras                                                     # 3)\n",
    "import pandas as pd                                              # 4)\n",
    "from keras.models import Sequential                              # 5)\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten     # 6)\n",
    "from keras.datasets import cifar10                               # 7)\n",
    "from keras.layers import Conv2D, MaxPooling2D                    # 8)\n",
    "from keras.preprocessing.image import ImageDataGenerator         # 9)\n",
    "import matplotlib.pyplot as plt                                  # 10)\n",
    "from keras.utils import np_utils                                 # 11)\n",
    "from keras.layers.normalization import BatchNormalization        # 12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile(\"train.csv.zip\", 'r') as zip_ref:\n",
    "    zip_ref.extractall(\"./\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile(\"test.csv.zip\", 'r') as zip_ref:\n",
    "    zip_ref.extractall(\"./\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile(\"suffled-images.zip\", 'r') as zip_ref:\n",
    "    zip_ref.extractall(\"./\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1)use CNN to classiy images**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('1163.jpg', '1164.jpg', '1165.jpg', '1525.jpg', '1526.jpg', '1528.jpg', '1529.jpg', '1530.jpg', '1531.jpg', '1532.jpg')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "43255"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "dir=os.listdir('shuffled-images/')\n",
    "from tkinter import Tcl\n",
    "dir_sort=Tcl().call('lsort', '-dict', dir)\n",
    "print(dir_sort[:10])\n",
    "len(dir_sort)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'category', 'gender', 'baseColour', 'season', 'usage',\n",
      "       'noisyTextDescription'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of           id    category  gender baseColour  season   usage  \\\n",
       "0      36274     Scarves   Women       Grey  Summer  Casual   \n",
       "1      15129  Flip Flops  Unisex      Green  Summer  Casual   \n",
       "2      58976     Topwear   Women        Red  Summer  Ethnic   \n",
       "3      32922      Sandal     Men      Brown  Summer  Casual   \n",
       "4      29561     Topwear   Women       Pink    Fall  Ethnic   \n",
       "...      ...         ...     ...        ...     ...     ...   \n",
       "21622   5435   Innerwear     Men      Black  Summer  Casual   \n",
       "21623  27880       Belts     Men      Brown  Summer  Casual   \n",
       "21624  38385       Shoes   Women      Beige  Winter  Casual   \n",
       "21625  34853     Watches     Men      Steel  Winter  Casual   \n",
       "21626  41407     Topwear     Men      White  Summer  Casual   \n",
       "\n",
       "                                    noisyTextDescription  \n",
       "0      Femella Women Ankle-Length Grey AQ-S800WD-1EVD...  \n",
       "1                  Converse Unisex Casual Skirts Slipper  \n",
       "2                              Velia Women Acetone Kurta  \n",
       "3                      Enroute Men Leather Brown Sandals  \n",
       "4                  Aneri Exclusive Anu Pink Inspirartion  \n",
       "...                                                  ...  \n",
       "21622                    SHE-3802D-1ADR Men Black Briefs  \n",
       "21623                 Adipower & Taylor Men Brown Quince  \n",
       "21624                             Rocia Women R348 Flats  \n",
       "21625                  Fastrack Men Black Amethyst Watch  \n",
       "21626             Arrow Ghicha Men White Slim Surf Shirt  \n",
       "\n",
       "[21627 rows x 7 columns]>"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train=pd.read_csv('train.csv',header=0)\n",
    "print(train.columns)\n",
    "train.head#21627"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'gender', 'baseColour', 'season', 'usage',\n",
      "       'noisyTextDescription'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of           id  gender baseColour  season   usage  \\\n",
       "0      26266     Men      Black  Summer  Casual   \n",
       "1      22134   Women      Green  Summer  Casual   \n",
       "2      28358   Women      Black  Winter  Casual   \n",
       "3      15554     Men      Black    Fall  Casual   \n",
       "4      53408   Women     Purple  Summer  Casual   \n",
       "...      ...     ...        ...     ...     ...   \n",
       "21623  39737     Men      Brown  Winter  Casual   \n",
       "21624  57477   Women      Black  Summer  Casual   \n",
       "21625  22312     Men      White    Fall  Sports   \n",
       "21626  54105   Women        Red  Spring  Casual   \n",
       "21627  14080  Unisex        Red  Summer  Casual   \n",
       "\n",
       "                                    noisyTextDescription  \n",
       "0                      Chromozome Men Black Fashion Vest  \n",
       "1                       Elle Women Green Color Clash Top  \n",
       "2                   Baggit Women Chotu Mayur Black Palms  \n",
       "3      Greensboro Colors Of Ap Men Latnam Black Casua...  \n",
       "4                        Fish White Smooth daddy (SH100)  \n",
       "...                                                  ...  \n",
       "21623                Park ClubLife 602 Helium Sunglasses  \n",
       "21624              Vero Moda Women Black Rey Shift Dress  \n",
       "21625                   Nike Men Hell With White Tshirts  \n",
       "21626       Lakme Holi to Five Day Apricot Nectar Lip K9  \n",
       "21627                  Percorso Unisex Red Grit Backpack  \n",
       "\n",
       "[21628 rows x 6 columns]>"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test=pd.read_csv('test.csv',header=0)\n",
    "print(test.columns)\n",
    "test.head#21628"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from matplotlib import image\n",
    "# load all images in a directory\n",
    "loaded_images_train = list()\n",
    "for filename in train['id']:\n",
    "    #print(filename)\n",
    "  # load image\n",
    "    img_data = image.imread('shuffled-images/' + str(filename) + \".jpg\")\n",
    "    # store loaded image\n",
    "    loaded_images_train.append(img_data)\n",
    "   # print('> loaded %s %s' % (filename, img_data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from matplotlib import image\n",
    "# load all images in a directory\n",
    "loaded_images_test = list()\n",
    "for filename in test['id']:\n",
    "    #print(filename)\n",
    "  # load image\n",
    "    img_data = image.imread('shuffled-images/' + str(filename) + \".jpg\")\n",
    "    # store loaded image\n",
    "    loaded_images_test.append(img_data)\n",
    "   # print('> loaded %s %s' % (filename, img_data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 256      #batch size\n",
    "num_classes = 27      #number of classes\n",
    "epochs = 100          #epoch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=train['category'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20  8 24 ... 21 26 24]\n",
      "(21627,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "label_list=train['category'].tolist()\n",
    "le.fit(label_list)\n",
    "transfomed_label =le.transform(label_list)\n",
    "print(transfomed_label)\n",
    "print(transfomed_label.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "all categories are sorted alphabetically:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_categories=le.inverse_transform(range(27)).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Accessories',\n",
       " 'Apparel Set',\n",
       " 'Bags',\n",
       " 'Belts',\n",
       " 'Bottomwear',\n",
       " 'Cufflinks',\n",
       " 'Dress',\n",
       " 'Eyewear',\n",
       " 'Flip Flops',\n",
       " 'Fragrance',\n",
       " 'Free Gifts',\n",
       " 'Headwear',\n",
       " 'Innerwear',\n",
       " 'Jewellery',\n",
       " 'Lips',\n",
       " 'Loungewear and Nightwear',\n",
       " 'Makeup',\n",
       " 'Nails',\n",
       " 'Sandal',\n",
       " 'Saree',\n",
       " 'Scarves',\n",
       " 'Shoes',\n",
       " 'Socks',\n",
       " 'Ties',\n",
       " 'Topwear',\n",
       " 'Wallets',\n",
       " 'Watches']"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Scarves',\n",
       " 'Flip Flops',\n",
       " 'Topwear',\n",
       " 'Sandal',\n",
       " 'Topwear',\n",
       " 'Bags',\n",
       " 'Socks',\n",
       " 'Topwear',\n",
       " 'Topwear',\n",
       " 'Shoes']"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(le.inverse_transform(transfomed_label))[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split last 4325 obs into validation set, remaining set as training set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train is  17302\n",
      "x_val is  4325\n",
      "x_test is  21628\n"
     ]
    }
   ],
   "source": [
    "x_train=np.array(loaded_images_train)\n",
    "y_train=np.array(transfomed_label)\n",
    "x_test=np.array(loaded_images_test)\n",
    "x_val=x_train[-4325:]\n",
    "y_val=y_train[-4325:]\n",
    "x_train=x_train[:-4325]\n",
    "y_train=y_train[:-4325]\n",
    "print(\"x_train is \",x_train.shape[0])\n",
    "print(\"x_val is \",x_val.shape[0])\n",
    "print(\"x_test is \",x_test.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np_utils.to_categorical(y_train, num_classes)#convert train labels to one hot encoding\n",
    "y_val = np_utils.to_categorical(y_val, num_classes)  #convert test labels to one hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Typecast data samples to flot32. It is usefull when using GPU.\n",
    "x_train = x_train.astype('float32')   \n",
    "x_test = x_test.astype('float32')\n",
    "x_val=x_val.astype('float32')\n",
    "# Scale pixles of data samples between 0 and 1.\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "x_val /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function to train with or without data augmentation:\n",
    "def data_aug (save_best_model):\n",
    "  if not data_augmentation:\n",
    "      print('Not using data augmentation.')\n",
    "      history = model.fit(x_train, y_train,verbose=1,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          validation_data=(x_val,y_val),\n",
    "                          shuffle=True,\n",
    "                          callbacks=[save_best_model])\n",
    "\n",
    "  # train with data augmentation\n",
    "  else:\n",
    "      print('Using real-time data augmentation.')\n",
    "      # This will do preprocessing and realtime data augmentation:\n",
    "      datagen = ImageDataGenerator(\n",
    "          featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "          samplewise_center=False,  # set each sample mean to 0\n",
    "          featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "          samplewise_std_normalization=False,  # divide each input by its std\n",
    "          zca_whitening=False,  # apply ZCA whitening\n",
    "          zca_epsilon=1e-06,  # epsilon for ZCA whitening\n",
    "          rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)\n",
    "          # randomly shift images horizontally (fraction of total width)\n",
    "          width_shift_range=0.1,\n",
    "          # randomly shift images vertically (fraction of total height)\n",
    "          height_shift_range=0.1,\n",
    "          shear_range=0.,  # set range for random shear\n",
    "          zoom_range=0.,  # set range for random zoom\n",
    "          channel_shift_range=0.,  # set range for random channel shifts\n",
    "          # set mode for filling points outside the input boundaries\n",
    "          fill_mode='nearest',\n",
    "          cval=0.,  # value used for fill_mode = \"constant\"\n",
    "          horizontal_flip=True,  # randomly flip images\n",
    "          vertical_flip=False,  # randomly flip images\n",
    "          # set rescaling factor (applied before any other transformation)\n",
    "          rescale=None,\n",
    "          # set function that will be applied on each input\n",
    "          preprocessing_function=None,\n",
    "          # image data format, either \"channels_first\" or \"channels_last\"\n",
    "          data_format=None,\n",
    "          # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "          validation_split=0.0)\n",
    "\n",
    "      # Compute quantities required for feature-wise normalization\n",
    "      # (std, mean, and principal components if ZCA whitening is applied).\n",
    "      datagen.fit(x_train)\n",
    "\n",
    "      # Fit the model on the batches generated by datagen.flow().\n",
    "      history = model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),verbose=1,\n",
    "                          steps_per_epoch=math.ceil(x_train.shape[0]/batch_size),\n",
    "                          epochs=epochs,\n",
    "                          validation_data=(x_val,y_val),\n",
    "                          callbacks=[save_best_model])\n",
    "  return(history)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import math\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_11 (Conv2D)           (None, 80, 60, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 80, 60, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 80, 60, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 80, 60, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 80, 60, 32)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 80, 60, 32)        128       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 40, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 40, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 40, 30, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 40, 30, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 40, 30, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 40, 30, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 40, 30, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_16 (Batc (None, 40, 30, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2 (None, 20, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 20, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 20, 15, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 20, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 20, 15, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 20, 15, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 20, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_18 (Batc (None, 20, 15, 64)        256       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_8 (MaxPooling2 (None, 10, 7, 64)         0         \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 10, 7, 64)         0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 4480)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 512)               2294272   \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_19 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 27)                13851     \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 27)                0         \n",
      "=================================================================\n",
      "Total params: 2,450,875\n",
      "Trainable params: 2,449,211\n",
      "Non-trainable params: 1,664\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# libraries (do not import additional libraries)\n",
    "\n",
    "\n",
    "# parameters for this script\n",
    "batch_size = 32\n",
    "num_classes = 27\n",
    "epochs = 20\n",
    "data_augmentation = True\n",
    "\n",
    "\n",
    "# Define a convolutional neural network\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same',input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(32, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "# initiate RMSprop optimizer\n",
    "opt = keras.optimizers.RMSprop(lr=0.0001, decay=1e-6)\n",
    "\n",
    "# Compile the model before using it\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/20\n",
      "541/541 [==============================] - 401s 741ms/step - loss: 0.4590 - accuracy: 0.8715 - val_loss: 0.4700 - val_accuracy: 0.8719\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.87191, saving model to best_model.h5\n",
      "Epoch 2/20\n",
      "541/541 [==============================] - 401s 742ms/step - loss: 0.4548 - accuracy: 0.8725 - val_loss: 0.4074 - val_accuracy: 0.8990\n",
      "\n",
      "Epoch 00002: val_accuracy improved from 0.87191 to 0.89896, saving model to best_model.h5\n",
      "Epoch 3/20\n",
      "541/541 [==============================] - 400s 740ms/step - loss: 0.4381 - accuracy: 0.8757 - val_loss: 0.4383 - val_accuracy: 0.8946\n",
      "\n",
      "Epoch 00003: val_accuracy did not improve from 0.89896\n",
      "Epoch 4/20\n",
      "541/541 [==============================] - 400s 739ms/step - loss: 0.4380 - accuracy: 0.8764 - val_loss: 0.3654 - val_accuracy: 0.9089\n",
      "\n",
      "Epoch 00004: val_accuracy improved from 0.89896 to 0.90890, saving model to best_model.h5\n",
      "Epoch 5/20\n",
      "541/541 [==============================] - 404s 747ms/step - loss: 0.4219 - accuracy: 0.8791 - val_loss: 0.3815 - val_accuracy: 0.9045\n",
      "\n",
      "Epoch 00005: val_accuracy did not improve from 0.90890\n",
      "Epoch 6/20\n",
      "541/541 [==============================] - 403s 744ms/step - loss: 0.4206 - accuracy: 0.8815 - val_loss: 0.3526 - val_accuracy: 0.9101\n",
      "\n",
      "Epoch 00006: val_accuracy improved from 0.90890 to 0.91006, saving model to best_model.h5\n",
      "Epoch 7/20\n",
      "541/541 [==============================] - 404s 747ms/step - loss: 0.4210 - accuracy: 0.8802 - val_loss: 0.4661 - val_accuracy: 0.8830\n",
      "\n",
      "Epoch 00007: val_accuracy did not improve from 0.91006\n",
      "Epoch 8/20\n",
      "541/541 [==============================] - 402s 743ms/step - loss: 0.4090 - accuracy: 0.8836 - val_loss: 0.3530 - val_accuracy: 0.9098\n",
      "\n",
      "Epoch 00008: val_accuracy did not improve from 0.91006\n",
      "Epoch 9/20\n",
      "541/541 [==============================] - 403s 746ms/step - loss: 0.4135 - accuracy: 0.8806 - val_loss: 0.4049 - val_accuracy: 0.8994\n",
      "\n",
      "Epoch 00009: val_accuracy did not improve from 0.91006\n",
      "Epoch 10/20\n",
      "541/541 [==============================] - 403s 745ms/step - loss: 0.3979 - accuracy: 0.8853 - val_loss: 0.3870 - val_accuracy: 0.8985\n",
      "\n",
      "Epoch 00010: val_accuracy did not improve from 0.91006\n",
      "Epoch 11/20\n",
      "541/541 [==============================] - 403s 744ms/step - loss: 0.3888 - accuracy: 0.8915 - val_loss: 0.7435 - val_accuracy: 0.8504\n",
      "\n",
      "Epoch 00011: val_accuracy did not improve from 0.91006\n",
      "Epoch 12/20\n",
      "541/541 [==============================] - 402s 744ms/step - loss: 0.3861 - accuracy: 0.8908 - val_loss: 0.3772 - val_accuracy: 0.8973\n",
      "\n",
      "Epoch 00012: val_accuracy did not improve from 0.91006\n",
      "Epoch 13/20\n",
      "541/541 [==============================] - 399s 738ms/step - loss: 0.3730 - accuracy: 0.8934 - val_loss: 0.4041 - val_accuracy: 0.8994\n",
      "\n",
      "Epoch 00013: val_accuracy did not improve from 0.91006\n",
      "Epoch 14/20\n",
      "541/541 [==============================] - 396s 732ms/step - loss: 0.3818 - accuracy: 0.8910 - val_loss: 0.3670 - val_accuracy: 0.9068\n",
      "\n",
      "Epoch 00014: val_accuracy did not improve from 0.91006\n",
      "Epoch 15/20\n",
      "541/541 [==============================] - 396s 733ms/step - loss: 0.3791 - accuracy: 0.8923 - val_loss: 0.3575 - val_accuracy: 0.9101\n",
      "\n",
      "Epoch 00015: val_accuracy did not improve from 0.91006\n",
      "Epoch 16/20\n",
      "541/541 [==============================] - 396s 731ms/step - loss: 0.3712 - accuracy: 0.8935 - val_loss: 0.4858 - val_accuracy: 0.8765\n",
      "\n",
      "Epoch 00016: val_accuracy did not improve from 0.91006\n",
      "Epoch 17/20\n",
      "541/541 [==============================] - 393s 727ms/step - loss: 0.3756 - accuracy: 0.8948 - val_loss: 0.3944 - val_accuracy: 0.9066\n",
      "\n",
      "Epoch 00017: val_accuracy did not improve from 0.91006\n",
      "Epoch 18/20\n",
      "541/541 [==============================] - 394s 728ms/step - loss: 0.3578 - accuracy: 0.8985 - val_loss: 0.3419 - val_accuracy: 0.9119\n",
      "\n",
      "Epoch 00018: val_accuracy improved from 0.91006 to 0.91191, saving model to best_model.h5\n",
      "Epoch 19/20\n",
      "541/541 [==============================] - 392s 724ms/step - loss: 0.3564 - accuracy: 0.8983 - val_loss: 0.3515 - val_accuracy: 0.9124\n",
      "\n",
      "Epoch 00019: val_accuracy improved from 0.91191 to 0.91237, saving model to best_model.h5\n",
      "Epoch 20/20\n",
      "541/541 [==============================] - 391s 722ms/step - loss: 0.3549 - accuracy: 0.9000 - val_loss: 0.3351 - val_accuracy: 0.9151\n",
      "\n",
      "Epoch 00020: val_accuracy improved from 0.91237 to 0.91514, saving model to best_model.h5\n"
     ]
    }
   ],
   "source": [
    "# create a callback that will save the best model while training\n",
    "save_best_model = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', save_best_only=True, verbose=1)\n",
    "\n",
    "# call function \"data_aug\" to train without data augmentation:\n",
    "history=data_aug(save_best_model)\n",
    "\n",
    "\n",
    "# Evaluate the best model saved (i.e., model with best validation accuracy) on the test set\n",
    "saved_model = load_model('best_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/5\n",
      "541/541 [==============================] - 394s 727ms/step - loss: 0.3536 - accuracy: 0.9026 - val_loss: 0.3455 - val_accuracy: 0.9131\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.91306, saving model to best_model.h5\n",
      "Epoch 2/5\n",
      "541/541 [==============================] - 396s 732ms/step - loss: 0.3467 - accuracy: 0.9002 - val_loss: 0.3732 - val_accuracy: 0.9077\n",
      "\n",
      "Epoch 00002: val_accuracy did not improve from 0.91306\n",
      "Epoch 3/5\n",
      "541/541 [==============================] - 394s 729ms/step - loss: 0.3403 - accuracy: 0.9041 - val_loss: 0.3671 - val_accuracy: 0.9059\n",
      "\n",
      "Epoch 00003: val_accuracy did not improve from 0.91306\n",
      "Epoch 4/5\n",
      "541/541 [==============================] - 395s 730ms/step - loss: 0.3377 - accuracy: 0.9042 - val_loss: 0.3364 - val_accuracy: 0.9156\n",
      "\n",
      "Epoch 00004: val_accuracy improved from 0.91306 to 0.91561, saving model to best_model.h5\n",
      "Epoch 5/5\n",
      "541/541 [==============================] - 395s 729ms/step - loss: 0.3375 - accuracy: 0.9042 - val_loss: 0.3773 - val_accuracy: 0.9121\n",
      "\n",
      "Epoch 00005: val_accuracy did not improve from 0.91561\n"
     ]
    }
   ],
   "source": [
    "epochs=5\n",
    "# create a callback that will save the best model while training\n",
    "save_best_model = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', save_best_only=True, verbose=1)\n",
    "\n",
    "# call function \"data_aug\" to train without data augmentation:\n",
    "history=data_aug(save_best_model)\n",
    "\n",
    "\n",
    "# Evaluate the best model saved (i.e., model with best validation accuracy) on the test set\n",
    "saved_model = load_model('best_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/5\n",
      "541/541 [==============================] - 394s 728ms/step - loss: 0.3364 - accuracy: 0.9064 - val_loss: 0.3356 - val_accuracy: 0.9119\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.91191, saving model to best_model.h5\n",
      "Epoch 2/5\n",
      "541/541 [==============================] - 392s 725ms/step - loss: 0.3349 - accuracy: 0.9058 - val_loss: 0.3558 - val_accuracy: 0.9140\n",
      "\n",
      "Epoch 00002: val_accuracy improved from 0.91191 to 0.91399, saving model to best_model.h5\n",
      "Epoch 3/5\n",
      "541/541 [==============================] - 395s 729ms/step - loss: 0.3254 - accuracy: 0.9058 - val_loss: 0.3868 - val_accuracy: 0.9073\n",
      "\n",
      "Epoch 00003: val_accuracy did not improve from 0.91399\n",
      "Epoch 4/5\n",
      "541/541 [==============================] - 394s 728ms/step - loss: 0.3202 - accuracy: 0.9072 - val_loss: 0.3141 - val_accuracy: 0.9214\n",
      "\n",
      "Epoch 00004: val_accuracy improved from 0.91399 to 0.92139, saving model to best_model.h5\n",
      "Epoch 5/5\n",
      "541/541 [==============================] - 395s 731ms/step - loss: 0.3223 - accuracy: 0.9072 - val_loss: 0.4183 - val_accuracy: 0.9031\n",
      "\n",
      "Epoch 00005: val_accuracy did not improve from 0.92139\n"
     ]
    }
   ],
   "source": [
    "epochs=5\n",
    "# create a callback that will save the best model while training\n",
    "save_best_model = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', save_best_only=True, verbose=1)\n",
    "\n",
    "# call function \"data_aug\" to train without data augmentation:\n",
    "history=data_aug(save_best_model)\n",
    "\n",
    "\n",
    "# Evaluate the best model saved (i.e., model with best validation accuracy) on the test set\n",
    "saved_model = load_model('best_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/5\n",
      "541/541 [==============================] - 396s 732ms/step - loss: 0.3185 - accuracy: 0.9075 - val_loss: 0.3290 - val_accuracy: 0.9177\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.91769, saving model to best_model.h5\n",
      "Epoch 2/5\n",
      "541/541 [==============================] - 397s 734ms/step - loss: 0.3193 - accuracy: 0.9102 - val_loss: 0.3706 - val_accuracy: 0.9064\n",
      "\n",
      "Epoch 00002: val_accuracy did not improve from 0.91769\n",
      "Epoch 3/5\n",
      "541/541 [==============================] - 395s 730ms/step - loss: 0.3139 - accuracy: 0.9108 - val_loss: 0.3660 - val_accuracy: 0.9128\n",
      "\n",
      "Epoch 00003: val_accuracy did not improve from 0.91769\n",
      "Epoch 4/5\n",
      "541/541 [==============================] - 393s 727ms/step - loss: 0.3083 - accuracy: 0.9129 - val_loss: 0.3477 - val_accuracy: 0.9145\n",
      "\n",
      "Epoch 00004: val_accuracy did not improve from 0.91769\n",
      "Epoch 5/5\n",
      "541/541 [==============================] - 395s 730ms/step - loss: 0.3150 - accuracy: 0.9114 - val_loss: 0.3159 - val_accuracy: 0.9193\n",
      "\n",
      "Epoch 00005: val_accuracy improved from 0.91769 to 0.91931, saving model to best_model.h5\n"
     ]
    }
   ],
   "source": [
    "epochs=5\n",
    "# create a callback that will save the best model while training\n",
    "save_best_model = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', save_best_only=True, verbose=1)\n",
    "\n",
    "# call function \"data_aug\" to train without data augmentation:\n",
    "history=data_aug(save_best_model)\n",
    "\n",
    "\n",
    "# Evaluate the best model saved (i.e., model with best validation accuracy) on the test set\n",
    "saved_model = load_model('best_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /srv/jupyter_python3-extras/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From /srv/jupyter_python3-extras/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_CNN = load_model('best_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict on training set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "prd_train = model_CNN.predict(x_train) \n",
    "prd_y_train = np.argmax(prd_train, axis=1)\n",
    "label_train = list(le.inverse_transform(prd_y_train))\n",
    "\n",
    "prd_val = model_CNN.predict(x_val) \n",
    "prd_y_val = np.argmax(prd_val, axis=1)\n",
    "label_val= list(le.inverse_transform(prd_y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These lines are used for converting one hot coding back to the original label form.\n",
    "# This function of trained model predicts the label of its input data\n",
    "prd_test = model_CNN.predict(x_test) \n",
    "prd_y_test = np.argmax(prd_test, axis=1)\n",
    "label_test = list(le.inverse_transform(prd_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2)Use LSTM to classify noist text based on sentence to tensor  function (break down into letters, digits and punctuation):**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4325\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17302"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train['combine_text']=train[\"noisyTextDescription\"].astype(str)+' '+train[\"gender\"].astype(str)+' '+train['baseColour'].astype(str)+' '+train['season'].astype(str)+' '+train['usage'].astype(str)\n",
    "train['combine_text']=train[\"noisyTextDescription\"].astype(str)\n",
    "train['combine_text'][:10]\n",
    "validate_data=np.array(train['combine_text'][-4325:]).tolist()\n",
    "validate_y=np.array(train['category'][-4325:]).tolist()\n",
    "training_data=np.array(train['combine_text'][:-4325]).tolist()\n",
    "training_y=np.array(train['category'][:-4325]).tolist()\n",
    "print(len(validate_data))\n",
    "len(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "train_data_dict={}\n",
    "for i in range(len(training_data)):\n",
    "    if training_y[i] not in train_data_dict:\n",
    "        train_data_dict[training_y[i]]=[training_data[i]]\n",
    "    else:\n",
    "        train_data_dict[training_y[i]].append(training_data[i])\n",
    "print(len(train_data_dict))\n",
    "train_data=train_data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    }
   ],
   "source": [
    "val_data_dict={}\n",
    "for i in range(len(validate_data)):\n",
    "    if validate_y[i] not in val_data_dict:\n",
    "        val_data_dict[validate_y[i]]=[validate_data[i]]\n",
    "    else:\n",
    "        val_data_dict[validate_y[i]].append(validate_data[i])\n",
    "print(len(val_data_dict))\n",
    "val_data=val_data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unicodedata\n",
    "import string\n",
    "import math\n",
    "\n",
    "all_letters = string.ascii_letters + string.digits + string.punctuation\n",
    "#all_letters = string.ascii_letters + string.digits\n",
    "n_letters = len(all_letters) \n",
    "n_categories=len(all_categories)\n",
    "n_letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Find letter index from all_letters, e.g. \"a\" = 0\n",
    "def letterToIndex(letter):\n",
    "    return all_letters.find(letter)\n",
    "\n",
    "# Just for demonstration, turn a letter into a <1 x n_letters> Tensor\n",
    "def letterToTensor(letter):\n",
    "    tensor = torch.zeros(1, n_letters)\n",
    "    tensor[0][letterToIndex(letter)] = 1\n",
    "    return tensor\n",
    "\n",
    "# Turn a line into a <line_length x 1 x n_letters>,\n",
    "# or an array of one-hot letter vectors\n",
    "def lineToTensor(line):\n",
    "    tensor = torch.zeros(len(line), 1, n_letters)\n",
    "    for li, letter in enumerate(line):\n",
    "        tensor[li][0][letterToIndex(letter)] = 1\n",
    "    return tensor\n",
    "\n",
    "def sentenceToTensor(sentence):\n",
    "    tensor = torch.zeros(len(sentence), 1, n_letters)\n",
    "    for li, letter in enumerate(sentence):\n",
    "        tensor[li][0][letterToIndex(letter)] = 1\n",
    "    return tensor\n",
    "         \n",
    "\n",
    "#print(letterToTensor('J')[0])\n",
    "\n",
    "#print(lineToTensor('Jones 12&'))\n",
    "\n",
    "#print(sentenceToTensor('Joes Harr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN_LSTM(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(RNN_LSTM, self).__init__()\n",
    "\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.LSTM=nn.LSTMCell(input_size,hidden_size)\n",
    "        self.i2o = nn.Linear(input_size + hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, input, hidden, cell):\n",
    "        hidden, cell = self.LSTM(input, (hidden,cell))\n",
    "        combined = torch.cat((input, hidden), 1)\n",
    "        output = self.i2o(combined)\n",
    "        output = self.softmax(output)\n",
    "        return output,hidden,cell\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, self.hidden_size)\n",
    "\n",
    "n_hidden = 128\n",
    "rnn_lstm = RNN_LSTM(n_letters, n_hidden, n_categories)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-3.2939, -3.2215, -3.2752, -3.3243, -3.3977, -3.2587, -3.2623, -3.3840,\n",
      "         -3.3781, -3.2166, -3.3842, -3.2898, -3.2512, -3.3715, -3.3812, -3.3230,\n",
      "         -3.2490, -3.2936, -3.3170, -3.2395, -3.2764, -3.2814, -3.2662, -3.2901,\n",
      "         -3.2734, -3.2821, -3.2428]], grad_fn=<LogSoftmaxBackward>)\n"
     ]
    }
   ],
   "source": [
    "input = sentenceToTensor('Annetta Harry')\n",
    "hidden = torch.zeros(1, n_hidden)\n",
    "cell=torch.zeros(1, n_hidden)\n",
    "output,hidden,cell= rnn_lstm(input[0], hidden,cell)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Fragrance', 9)\n"
     ]
    }
   ],
   "source": [
    "def categoryFromOutput(output):\n",
    "    top_n, top_i = output.topk(1)\n",
    "    category_i = top_i[0].item()\n",
    "    return all_categories[category_i], category_i\n",
    "\n",
    "print(categoryFromOutput(output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category = Bags / line = Ivory Roshe Cheeky G260 Furry Foldover Sling Bag\n",
      "category = Makeup / line = Streetwear PowerCat Divine Whitening INSP Compact\n",
      "category = Topwear / line = Puma Men Graphic White Tshirts\n",
      "category = Saree / line = Golden Green & Purple Wedding Collection Sari\n",
      "category = Watches / line = Q&Q Men Black Dial Watch\n",
      "category = Lips / line = Lotus Herbals Moistpetals Cocoa SGW-300H-1AVDR Matka 163\n",
      "category = Wallets / line = Squid Women Black Figio\n",
      "category = Nails / line = Piper NE3039SM01 Fast Metal-Eyes Aston A633 Crush Magenta Nail Polish 25\n",
      "category = Innerwear / line = N.Masti Women Peach-Coloured Briefs PCLR02\n",
      "category = Wallets / line = United Colors of Benetton Women Basice Wallet\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "def randomChoice(l):\n",
    "    return l[random.randint(0, len(l) - 1)]\n",
    "\n",
    "def randomTrainingExample():\n",
    "    category = randomChoice(all_categories)\n",
    "    line = randomChoice(train_data[category])\n",
    "    category_tensor = torch.tensor([all_categories.index(category)], dtype=torch.long)\n",
    "    line_tensor = sentenceToTensor(line)\n",
    "    return category, line, category_tensor, line_tensor\n",
    "\n",
    "for i in range(10):\n",
    "    category, line, category_tensor, line_tensor = randomTrainingExample()\n",
    "    print('category =', category, '/ line =', line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_LSTM(category_tensor, line_tensor):\n",
    "    hidden = rnn_lstm.initHidden()\n",
    "    cell= rnn_lstm.initHidden()\n",
    "    # reset gradient\n",
    "    rnn_lstm.zero_grad()\n",
    "    \n",
    "    for i in range(line_tensor.size()[0]):\n",
    "        output,hidden,cell = rnn_lstm(line_tensor[i], hidden, cell)\n",
    "    loss = criterion_lstm(output, category_tensor)\n",
    "\n",
    "    # compute gradient by backpropagation\n",
    "    loss.backward()\n",
    "\n",
    "    # update parameters\n",
    "    optimizer_lstm.step()\n",
    "\n",
    "    return output, loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Iter \tTrain% \tTime \t\tTrain_loss_lstm \tExample\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/jupyter_python3-extras/lib/python3.7/site-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type RNN_LSTM. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000 \t5% \t(2m 52s) \t1.3251 \t\tReid & Taylor Men Edward Wallet / Wallets ✓\n",
      "10000 \t10% \t(6m 10s) \t1.0532 \t\tUnited Colors of Benetton Men Check Brown Wht/purple / Headwear ✗ (Ties)\n",
      "15000 \t15% \t(9m 29s) \t0.8467 \t\tPieces Elements Mauve Ethnic Belt / Belts ✓\n",
      "20000 \t20% \t(12m 46s) \t0.7109 \t\tColorbar Color Intense Mignight Sky Liquid Eye Liner 001 / Makeup ✓\n",
      "25000 \t25% \t(16m 3s) \t0.6503 \t\tAllen Solly Men Leather Black Wallet / Wallets ✓\n",
      "30000 \t30% \t(19m 22s) \t0.5749 \t\tPuma Men Diet Cap / Headwear ✓\n",
      "35000 \t35% \t(22m 39s) \t0.5586 \t\tADIDAS Unisex Pack of Generation Crew Socks / Socks ✓\n",
      "40000 \t40% \t(25m 59s) \t0.4961 \t\tShree Women Navy & Burberry Salwar Suit with Dupatta / Apparel Set ✓\n",
      "45000 \t45% \t(29m 16s) \t0.4236 \t\tWills AH218-C1 Reespark Magenta Leggings / Bottomwear ✓\n",
      "50000 \t50% \t(32m 33s) \t0.4604 \t\tLakme Tshirts On Aqua Marine Eye Color / Makeup ✓\n",
      "55000 \t55% \t(35m 50s) \t0.4532 \t\tNumero Uno Men's Navy Blue Shoe / Shoes ✓\n",
      "60000 \t60% \t(39m 7s) \t0.4122 \t\tParx Men Blue Tie / Ties ✓\n",
      "65000 \t65% \t(42m 24s) \t0.3540 \t\tPalm Tree Kids Boy Chrono Blue Jeans / Bottomwear ✓\n",
      "70000 \t70% \t(45m 42s) \t0.3833 \t\tThrill PL12900JS-02 Delight N9333PP02 Polish 52 / Nails ✓\n",
      "75000 \t75% \t(48m 57s) \t0.3604 \t\tMaxima Men Black Dial Watch / Free Gifts ✓\n",
      "80000 \t80% \t(52m 16s) \t0.2932 \t\tEnamor White Classique Bra / Innerwear ✓\n",
      "85000 \t85% \t(55m 33s) \t0.2859 \t\tInsincts Men St Lounge Shorts / Loungewear and Nightwear ✓\n",
      "90000 \t90% \t(58m 51s) \t0.3313 \t\tQuechua Ran Lunarfly+ / Belts ✗ (Bags)\n",
      "95000 \t95% \t(62m 8s) \t0.2506 \t\tFNF Green & Mode Wedding Collection Digi / Saree ✓\n",
      "100000 \t100% \t(65m 25s) \t0.2721 \t\tUnited Colors of Benetton Women Washed Blue Capris / Bottomwear ✓\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import math\n",
    "from torch.autograd import Variable\n",
    "\n",
    "criterion_lstm = nn.NLLLoss()\n",
    "rnn_lstm = RNN_LSTM(n_letters, n_hidden, n_categories)  \n",
    "optimizer_lstm = torch.optim.Adam(rnn_lstm.parameters())\n",
    "\n",
    "n_iters = 100000\n",
    "print_every = 5000\n",
    "plot_every = 1000\n",
    "\n",
    "# Keep track of losses for plotting\n",
    "train_loss_lstm = 0\n",
    "all_train_losses_lstm = []\n",
    "all_validation_losses_lstm = []\n",
    "all_validation_losses2_lstm = []\n",
    "\n",
    "def timeSince(since):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "# Just return an output given a line\n",
    "def evaluate_lstm(line_tensor):\n",
    "    hidden = rnn_lstm.initHidden()\n",
    "    cell= rnn_lstm.initHidden()\n",
    "    for i in range(line_tensor.size()[0]):\n",
    "        output,hidden,cell = rnn_lstm(line_tensor[i], hidden,cell)\n",
    "    return output\n",
    "\n",
    "def eval_dataset_lstm(dataset):\n",
    "    loss = 0\n",
    "    n_instances = 0\n",
    "    confusion = torch.zeros(n_categories, n_categories)\n",
    "    for category in all_categories:\n",
    "        category_tensor = Variable(torch.LongTensor([all_categories.index(category)]))\n",
    "        n_instances += len(dataset[category])\n",
    "        for line in dataset[category]:\n",
    "            line_tensor = Variable(sentenceToTensor(line))\n",
    "            output = evaluate_lstm(line_tensor)\n",
    "            loss += criterion_lstm(output, category_tensor)\n",
    "            guess, guess_i = categoryFromOutput(output)\n",
    "            category_i = all_categories.index(category)\n",
    "            confusion[category_i][guess_i] += 1\n",
    "\n",
    "    # Normalize by dividing every row by its sum\n",
    "    for i in range(n_categories):\n",
    "        confusion[i] = confusion[i] / confusion[i].sum()\n",
    "\n",
    "    return loss.item() / n_instances, confusion\n",
    "  \n",
    "print('\\nIter \\tTrain% \\tTime \\t\\tTrain_loss_lstm \\tExample')\n",
    "start = time.time()\n",
    "for iter in range(1, n_iters + 1):\n",
    "    category, line, category_tensor, line_tensor = randomTrainingExample()\n",
    "    output, loss = train_LSTM(category_tensor, line_tensor)\n",
    "    train_loss_lstm += loss\n",
    "\n",
    "    # Print iter number, train loss average, name and guess\n",
    "    if iter % print_every == 0:\n",
    "        guess, guess_i = categoryFromOutput(output)\n",
    "        correct = '✓' if guess == category else '✗ (%s)' % category\n",
    "        print('%d \\t%d%% \\t(%s) \\t%.4f \\t\\t%s / %s %s' % (iter, iter / n_iters * 100, timeSince(start), train_loss_lstm / plot_every, line, guess, correct))\n",
    "\n",
    "    # Add current train loss average to list of losses\n",
    "    if iter % plot_every == 0:\n",
    "        all_train_losses_lstm.append(train_loss_lstm / plot_every)\n",
    "        train_loss_lstm = 0\n",
    "        \n",
    "    # Compute loss based on validation data\n",
    "    if iter % plot_every == 0:\n",
    "        average_validation_loss_lstm, _ = eval_dataset_lstm(val_data)\n",
    "\n",
    "        # save model with best validation loss\n",
    "        if len(all_validation_losses_lstm) == 0 or average_validation_loss_lstm < min(all_validation_losses_lstm):\n",
    "            torch.save(rnn_lstm, 'char_rnn_lstm_classification_model.pt')\n",
    "        all_validation_losses_lstm.append(average_validation_loss_lstm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_LSTM = torch.load('char_rnn_lstm_classification_model.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[4.6963e-06, 8.7542e-05, 7.7517e-04, 3.8517e-04, 2.2609e-03, 2.9723e-09,\n",
       "          4.7479e-06, 9.7614e-05, 6.0423e-05, 1.3140e-04, 3.7311e-04, 3.4706e-08,\n",
       "          8.8298e-01, 3.8908e-05, 1.7938e-04, 4.4881e-02, 1.2076e-04, 1.0829e-08,\n",
       "          2.0813e-02, 6.6573e-06, 7.7208e-07, 1.6457e-02, 4.3971e-05, 8.7415e-07,\n",
       "          2.8190e-02, 5.7435e-04, 1.5330e-03]]),\n",
       " ['Innerwear'])"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def predict(input_line, n_predictions=1):\n",
    "    #print('\\n> %s' % input_line)\n",
    "    with torch.no_grad():\n",
    "        output = torch.exp(evaluate_lstm(sentenceToTensor(input_line)))\n",
    "\n",
    "        # Get top N categories\n",
    "        topv, topi = output.topk(n_predictions, 1, True)\n",
    "        predictions = []\n",
    "\n",
    "        for i in range(n_predictions):\n",
    "            value = topv[0][i].item()\n",
    "            category_index = topi[0][i].item()\n",
    "            #print(all_categories[category_index])\n",
    "            predictions.append(all_categories[category_index])\n",
    "    return(output, predictions)\n",
    "predict('women white winter casual')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                        Chromozome Men Black Fashion Vest\n",
       "1                         Elle Women Green Color Clash Top\n",
       "2                     Baggit Women Chotu Mayur Black Palms\n",
       "3        Greensboro Colors Of Ap Men Latnam Black Casua...\n",
       "4                          Fish White Smooth daddy (SH100)\n",
       "                               ...                        \n",
       "21623                  Park ClubLife 602 Helium Sunglasses\n",
       "21624                Vero Moda Women Black Rey Shift Dress\n",
       "21625                     Nike Men Hell With White Tshirts\n",
       "21626         Lakme Holi to Five Day Apricot Nectar Lip K9\n",
       "21627                    Percorso Unisex Red Grit Backpack\n",
       "Name: combine_text, Length: 21628, dtype: object"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['combine_text']=test['noisyTextDescription'].astype(str)\n",
    "test['combine_text']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is training accuracy? validation accuracy ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8131429892497977\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Scarves',\n",
       " 'Flip Flops',\n",
       " 'Topwear',\n",
       " 'Sandal',\n",
       " 'Belts',\n",
       " 'Innerwear',\n",
       " 'Socks',\n",
       " 'Topwear',\n",
       " 'Topwear',\n",
       " 'Topwear']"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#training accuracy:\n",
    "pred_train_lstm=[]\n",
    "label_train_lstm=[]\n",
    "\n",
    "accu=0\n",
    "\n",
    "for i in range(len(training_data)):\n",
    "        output,pred=predict(training_data[i],n_predictions=1)\n",
    "        #print(pred[0])\n",
    "        if pred[0]==training_y[i]:\n",
    "            accu+=1\n",
    "        pred_train_lstm.append(np.array(output[0]))\n",
    "        label_train_lstm.append(pred[0])\n",
    "\n",
    "print(accu/len(training_data))\n",
    "label_train_lstm[:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7567630057803468\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Saree',\n",
       " 'Eyewear',\n",
       " 'Belts',\n",
       " 'Topwear',\n",
       " 'Headwear',\n",
       " 'Bottomwear',\n",
       " 'Topwear',\n",
       " 'Bags',\n",
       " 'Flip Flops',\n",
       " 'Dress']"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#validation accuracy:\n",
    "pred_val_lstm=[]\n",
    "label_val_lstm=[]\n",
    "accu=0\n",
    "\n",
    "for i in range(len(validate_data)):\n",
    "        output,pred=predict(validate_data[i],n_predictions=1)\n",
    "        if pred[0]==validate_y[i]:\n",
    "            accu+=1\n",
    "        pred_val_lstm.append(np.array(output[0]))\n",
    "        label_val_lstm.append(pred[0])\n",
    "\n",
    "print(accu/len(validate_data))\n",
    "label_val_lstm[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_test_lstm=[]\n",
    "for line in test['combine_text']:\n",
    "    output,pred=predict(line,n_predictions=1)\n",
    "    pred_test_lstm.append(np.array(output[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ensembled learning:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3)Use simple neural network to concatenate results from RNN and CNN, and output into single probability as final result:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_train_lstm=np.array(pred_train_lstm)\n",
    "pred_val_lstm=np.array(pred_val_lstm)\n",
    "pred_test_lstm=np.array(pred_test_lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4325, 27)"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            (None, 27)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_8 (InputLayer)            (None, 27)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 54)           0           input_7[0][0]                    \n",
      "                                                                 input_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 27)           1485        concatenate_2[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 1,485\n",
      "Trainable params: 1,485\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.layers import Input, concatenate,  Dense\n",
    "from keras.models import Model\n",
    "\n",
    "\n",
    "# # Define two input layers\n",
    "LSTM_input = Input((27,))\n",
    "CNN_input = Input((27,))\n",
    " # create the input to our final set of layers as the *output* of both\n",
    " # the MLP and CNN\n",
    "combinedInput = concatenate([LSTM_input, CNN_input])\n",
    "## our final FC layer head will have two dense layers, the final one\n",
    " # being our regression head\n",
    "x = Dense(27, activation=\"softmax\")(combinedInput)\n",
    "# our final model will accept categorical/numerical data on the MLP\n",
    "# input and images on the CNN input, outputting a single value (the\n",
    "# predicted price of the house)\n",
    "model = Model(inputs=[LSTM_input, CNN_input], outputs=x)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] training model...\n",
      "Train on 17302 samples, validate on 4325 samples\n",
      "Epoch 1/1\n",
      "17302/17302 [==============================] - 0s 24us/step - loss: 3.2824 - accuracy: 0.0575 - val_loss: 3.1888 - val_accuracy: 0.1427\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7fabe8383a10>"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras import losses\n",
    "\n",
    "#opt = keras.optimizers.Adam(lr=1e-3, decay=1e-3 / 200)\n",
    "opt = keras.optimizers.RMSprop(lr=0.0001, decay=1e-6)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "# train the model\n",
    "print(\"[INFO] training model...\")\n",
    "model.fit(x=[pred_train_lstm, prd_train], y=y_train,\n",
    "         validation_data=([pred_val_lstm, prd_val], y_val),\n",
    "           epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, 'ensembled_model.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ensemble=torch.load('ensembled_model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write a csv file out which is corpus containing line and category separated by tab:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] predicting house prices...\n"
     ]
    }
   ],
   "source": [
    "# make predictions on the testing data\n",
    "print(\"[INFO] predicting house prices...\")\n",
    "preds = model.predict([pred_test_lstm, prd_test])\n",
    "prd_y = np.argmax(preds, axis=1)\n",
    "label = list(le.inverse_transform(prd_y))\n",
    "\n",
    "predict=np.stack((test['id'],np.array(label)),axis=1)\n",
    "import csv\n",
    "with open('submission_finalresult.csv', mode='w') as csv_file:\n",
    "    writer = csv.writer(csv_file)\n",
    "    writer.writerow(['id','category'])\n",
    "    writer.writerows(predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}