05-Preprocessing and Evaluation from scratch (Dev Notebook).ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('ics5110': conda)",
   "metadata": {
    "interpreter": {
     "hash": "a3d8fb96efc099e25dc4d6a873b44c7302ca5a21722eb7d7c39b09db89d1143d"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "from collections import defaultdict\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm\n",
    "from scipy.sparse import csr_matrix, diags\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using default token pattern from sklearn\n",
    "token_pattern = re.compile(r\"(?u)\\b\\w\\w+\\b\")\n",
    "ngram_range = (1, 3)\n",
    "min_freq = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"yelp_review_sentiment_2classes.tsv\", delimiter=\"\\t\")\n",
    "df = df.dropna()"
   ]
  },
  {
   "source": [
    "## CountVectorizer Implementation\n",
    "\n",
    "Something weird happen with sklearn's CountVectorizer\n",
    "\n",
    "Result from CountVectorizer. \n",
    "'beer', 'beer copyright', 'burger', 'copyright'  \n",
    "6 4 5 5  \n",
    "\n",
    "The correct one should be  \n",
    "'beer', 'pizza', 'beer copyright', 'burger', 'copyright'  \n",
    "6 4 4 5 5  \n",
    "Pizza is missing from the transformation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_stopwords = stopwords.words('english') + [\"-PRON-\", \"-pron-\", \"PRON\", \"pron\"]\n",
    "\n",
    "def preprocessing(text, ngram_range):\n",
    "    tokens = token_pattern.findall(text)\n",
    "    tokens = list(filter(lambda x: x not in en_stopwords, tokens))\n",
    "    tokens = build_ngrams(tokens, ngram_range)\n",
    "    return tokens\n",
    "\n",
    "\n",
    "def build_ngrams(text, ngram_range):\n",
    "    vocabs = []\n",
    "    join_space = \" \".join\n",
    "    for i in range(ngram_range[0], ngram_range[1] + 1):\n",
    "        for j in range(len(text)):\n",
    "            if j + i <= len(text):\n",
    "                vocabs.append(join_space(text[j:j+i]))\n",
    "    return vocabs\n",
    "\n",
    "\n",
    "def build_vocab(doc, ngram_range, min_freq):\n",
    "    vocabs = {}\n",
    "    vocab_counts = defaultdict(int)\n",
    "    for text in tqdm(doc):\n",
    "        tokens = preprocessing(text, ngram_range)\n",
    "        for word in tokens:\n",
    "            vocab_counts[word] += 1\n",
    "\n",
    "    vocab_counts = dict(filter(lambda x: x[1] >= min_freq, vocab_counts.items()))\n",
    "    for idx, key in enumerate(vocab_counts.keys()):\n",
    "        vocabs[key] = idx\n",
    "    return vocabs, vocab_counts\n",
    "\n",
    "\n",
    "def build_count_vector(doc, vocabs):\n",
    "    data = []\n",
    "    indices = []\n",
    "    indptr = [0]\n",
    "    for text in tqdm(doc):\n",
    "        tokens = preprocessing(text, ngram_range)\n",
    "        feature_counts = defaultdict(int)\n",
    "        for word in tokens:\n",
    "            if word in vocabs:\n",
    "                feature_counts[vocabs[word]] += 1\n",
    "        data.extend(feature_counts.values())\n",
    "        indices.extend(feature_counts.keys())\n",
    "        indptr.append(len(data))\n",
    "    \n",
    "    return csr_matrix((data, indices, indptr), shape=(len(doc), len(vocabs)), dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "100%|██████████| 249978/249978 [03:22<00:00, 1234.34it/s]\n"
     ]
    }
   ],
   "source": [
    "vocabs, vocab_counts = build_vocab(df[\"text\"], ngram_range, min_freq)\n",
    "# vocabs, vocab_counts = build_vocab((\n",
    "#     \"the pizza pizza beer copyright\",\n",
    "#     \"the pizza burger beer copyright\",\n",
    "#     \"the the pizza beer beer copyright\",\n",
    "#     \"the burger beer beer copyright\",\n",
    "#     \"the coke burger coke copyright\",\n",
    "#     \"the coke burger burger\",\n",
    "# ), (1, 3), 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "234784"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "len(vocabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "100%|██████████| 249978/249978 [02:18<00:00, 1810.44it/s]\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<249978x234784 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 18601591 stored elements in Compressed Sparse Row format>"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "build_count_vector(df[\"text\"], vocabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "234784"
      ]
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "len(vocabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "vocab_counts[\"fat boy\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "CountVectorizer(min_df=4, ngram_range=(1, 3),\n",
       "                stop_words=['i', 'me', 'my', 'myself', 'we', 'our', 'ours',\n",
       "                            'ourselves', 'you', \"you're\", \"you've\", \"you'll\",\n",
       "                            \"you'd\", 'your', 'yours', 'yourself', 'yourselves',\n",
       "                            'he', 'him', 'his', 'himself', 'she', \"she's\",\n",
       "                            'her', 'hers', 'herself', 'it', \"it's\", 'its',\n",
       "                            'itself', ...])"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "# tfidf_vec = TfidfVectorizer(ngram_range=(1, 3), min_df=10, stop_words=en_stopwords)\n",
    "cnt_vec = CountVectorizer(ngram_range=(1, 3), min_df=4, stop_words=en_stopwords)\n",
    "cnt_vec.fit((\n",
    "    \"the pizza pizza beer copyright\",\n",
    "    \"the pizza burger beer copyright\",\n",
    "    \"the the pizza beer beer copyright\",\n",
    "    \"the burger beer beer copyright\",\n",
    "    \"the coke burger coke copyright\",\n",
    "    \"the coke burger burger\",\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['beer', 'beer copyright', 'burger', 'copyright']\n[6 4 5 5]\n"
     ]
    }
   ],
   "source": [
    "print(cnt_vec.get_feature_names())\n",
    "print(cnt_vec.transform((\n",
    "    \"the pizza pizza beer copyright\",\n",
    "    \"the pizza burger beer copyright\",\n",
    "    \"the the pizza beer beer copyright\",\n",
    "    \"the burger beer beer copyright\",\n",
    "    \"the coke burger coke copyright\",\n",
    "    \"the coke burger burger\",\n",
    ")).toarray().sum(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<6x4 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 17 stored elements in Compressed Sparse Row format>"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "res = cnt_vec.transform((\n",
    "    \"the pizza pizza beer copyright\",\n",
    "    \"the pizza burger beer copyright\",\n",
    "    \"the the pizza beer beer copyright\",\n",
    "    \"the burger beer beer copyright\",\n",
    "    \"the coke burger coke copyright\",\n",
    "    \"the coke burger burger\",\n",
    "))\n",
    "res"
   ]
  },
  {
   "source": [
    "## TfIdfVectorizer Implementation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_tfidf_vector(doc, vocabs):\n",
    "    term_vectors = build_count_vector(doc, vocabs)\n",
    "    document_vectors = np.bincount(term_vectors.indices, minlength=term_vectors.shape[1])\n",
    "    # Add smoothing for both numerator and denumerator to prevent division by 0 or having 0\n",
    "    # as the result\n",
    "    idf = (term_vectors.shape[0] + 1) / (document_vectors + 1)\n",
    "    idf = np.log(idf)\n",
    "    # The default action for csr_matrix multiplied by numpy vector is dot product, therefore\n",
    "    # we need to create a diagonal matrix first to simulate element by element multiplication\n",
    "    # without blowing up the memory\n",
    "    diag_idf = diags(idf, offsets=0, format=\"csr\")\n",
    "    return term_vectors * diag_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "100%|██████████| 249978/249978 [02:17<00:00, 1811.88it/s]\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<249978x234784 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 18601591 stored elements in Compressed Sparse Row format>"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "build_tfidf_vector(df[\"text\"], vocabs)"
   ]
  },
  {
   "source": [
    "## Normalizer Implementation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(X):\n",
    "    new_data = []\n",
    "    for x in X:\n",
    "        square_sum = 0\n",
    "        for i in x.data:\n",
    "            square_sum += i * i\n",
    "        norm = np.sqrt(square_sum)\n",
    "        new_data.extend((x.data / norm).tolist())\n",
    "    new_mat = csr_matrix((np.array(new_data), X.indices, X.indptr), dtype=np.float64)\n",
    "    return new_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<6x4 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 17 stored elements in Compressed Sparse Row format>"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "normalize(res)"
   ]
  },
  {
   "source": [
    "## train_test_split Implementation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_data(X, y, test_size=0.1):\n",
    "    total_data = len(X)\n",
    "    total_test = int(total_data * test_size)\n",
    "    sample_idx = np.arange(total_data)\n",
    "    np.random.shuffle(sample_idx)\n",
    "\n",
    "    X_train = X.iloc[sample_idx[total_test:]]\n",
    "    y_train = y.iloc[sample_idx[total_test:]]\n",
    "\n",
    "    X_test = X.iloc[sample_idx[:total_test]]\n",
    "    y_test = y.iloc[sample_idx[:total_test]]\n",
    "    return (X_train, y_train), (X_test, y_test)"
   ]
  },
  {
   "source": [
    "## Cross validation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_division(a, b):\n",
    "    if b == 0:\n",
    "        return 0.\n",
    "    return a / b\n",
    "\n",
    "\n",
    "def confusion_matrix(true, pred):\n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    tn = 0\n",
    "    fn = 0\n",
    "    for y, y_hat in zip(true, pred):\n",
    "        if y == 1 and y_hat == 1:\n",
    "            tp += 1\n",
    "        elif y == 0 and y_hat == 1:\n",
    "            fp += 1\n",
    "        elif y == 1 and y_hat == 0:\n",
    "            fn += 1\n",
    "        elif y == 0 and y_hat == 0:\n",
    "            tn += 1\n",
    "    return tp, fp, tn, fn\n",
    "\n",
    "\n",
    "def precision_micro(true, pred):\n",
    "    tp, fp, tn, fn = confusion_matrix(true, pred)\n",
    "    return save_division(tp, (tp + fp))\n",
    "\n",
    "\n",
    "def recall_micro(true, pred):\n",
    "    tp, fp, tn, fn = confusion_matrix(true, pred)\n",
    "    return save_division(tp, (tp + fn))\n",
    "\n",
    "\n",
    "def f1_micro(true, pred):\n",
    "    precision = precision_micro(true, pred)\n",
    "    recall = recall_micro(true, pred)\n",
    "    return save_division((2 * precision * recall), (precision + recall))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import delayed, Parallel\n",
    "\n",
    "\n",
    "def cross_validate(model, X, y, folds=5):\n",
    "    fold_size = int(X.shape[0] / folds)\n",
    "    fold_ranges = [(i, i + fold_size) for i in range(0, X.shape[0], fold_size)]\n",
    "    scores = {\n",
    "        \"fit_time\": [],\n",
    "        \"score_time\": [],\n",
    "        \"test_f1_micro\": [],\n",
    "        \"train_f1_micro\": [],\n",
    "        \"test_precision_micro\": [],\n",
    "        \"train_precision_micro\": [],\n",
    "        \"test_recall_micro\": [],\n",
    "        \"train_recall_micro\": []\n",
    "    }\n",
    "\n",
    "    def _fit(start, end):\n",
    "        X_train = np.concatenate([X[0:start], X[end:]])\n",
    "        y_train = np.concatenate([y[0:start], y[end:]])\n",
    "\n",
    "        X_test = X[start:end]\n",
    "        y_test = y[start:end]\n",
    "\n",
    "        start_time = time.time()\n",
    "        model.fit(X_train, y_train)\n",
    "        fit_time = time.time() - start_time\n",
    "\n",
    "        start_time = time.time()\n",
    "        preds_train = model.predict(X_train)\n",
    "        preds_test = model.predict(X_test)\n",
    "        score_time = time.time() - start_time\n",
    "\n",
    "        metrics = [precision_micro, recall_micro, f1_micro]\n",
    "        scores = []\n",
    "        for metric in metrics:\n",
    "            name = metric.__name__\n",
    "            scores.append({\n",
    "                f\"train_{name}\": metric(y_train, preds_train),\n",
    "                f\"test_{name}\": metric(y_test, preds_test)})\n",
    "        return fit_time, score_time, scores\n",
    "\n",
    "    fit_results = Parallel(n_jobs=-1)(delayed(_fit)(start, end) for start, end in fold_ranges)\n",
    "    \n",
    "    for res in fit_results:\n",
    "        scores[\"fit_time\"].append(res[0])\n",
    "        scores[\"score_time\"].append(res[1])\n",
    "        for score in res[2]:\n",
    "            for k, v in score.items():\n",
    "                scores[k].append(v)\n",
    "\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'fit_time': [0.30196571350097656, 0.3023810386657715, 0.20588183403015137],\n",
       " 'score_time': [0.0013513565063476562,\n",
       "  0.0010638236999511719,\n",
       "  0.001065969467163086],\n",
       " 'test_f1_micro': [1.0, 1.0, 1.0],\n",
       " 'train_f1_micro': [1.0, 1.0, 1.0],\n",
       " 'test_precision_micro': [1.0, 1.0, 1.0],\n",
       " 'train_precision_micro': [1.0, 1.0, 1.0],\n",
       " 'test_recall_micro': [1.0, 1.0, 1.0],\n",
       " 'train_recall_micro': [1.0, 1.0, 1.0]}"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "digits = load_digits()\n",
    "X, y = digits.data, digits.target\n",
    "clf = LogisticRegression(random_state=42)\n",
    "\n",
    "cross_validate(clf, X, y, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([ True, False, False,  True])"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "x = np.array([1, 0, 0, 1])\n",
    "x == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}