Next_Word_Prediction_using_Universal_Sentence_Encoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Next-Word Prediction using Universal Sentence Encoder.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zavljtkc2Xsv"
      },
      "source": [
        "# **Google drive for local storage**\r\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k_Go7GAGSvkw"
      },
      "source": [
        "_NB: All comments are written to facilitate smooth evaluation of the model, that the **Current User** may be less fatigued and see beauty in the good work._\r\n",
        "\r\n",
        "Uncomment text under **PREVIEW OUTPUT** to further scrutinize."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qj-H_oca3bNa",
        "outputId": "4483bca6-15fb-48bc-ad72-017d971f257f"
      },
      "source": [
        "# This cell will prompt an external url to accept permissions for Colab to access Google Drive\r\n",
        "\r\n",
        "from google.colab import drive\r\n",
        "drive.mount(\"/gdrive\")\r\n",
        "\r\n",
        "%ls"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n",
            "corpus.txt    \u001b[0m\u001b[01;34mNWP-USE\u001b[0m/        puntuated_.txt  tokenizer1.pkl    vocabulary.npy\n",
            "nextword1.h5  pred_model4.h5  \u001b[01;34msample_data\u001b[0m/    tokenizer.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t_qUksc3r3KX"
      },
      "source": [
        "# **Import ***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4d9_NB3vpwG8"
      },
      "source": [
        "# Getting all required libraries\r\n",
        "\r\n",
        "import os\r\n",
        "import re\r\n",
        "import gdown\r\n",
        "import numpy\r\n",
        "import string\r\n",
        "import numpy as np\r\n",
        "import pandas as pd\r\n",
        "import seaborn as sns\r\n",
        "import tensorflow as tf\r\n",
        "from absl import logging\r\n",
        "import tensorflow_hub as hub\r\n",
        "from tensorflow import keras\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "from keras.models import Sequential\r\n",
        "import tensorflow.keras.backend as K\r\n",
        "from keras.layers.recurrent import LSTM\r\n",
        "from keras.layers import Dense, Activation\r\n",
        "from keras.callbacks import LambdaCallback\r\n",
        "from keras.utils.data_utils import get_file\r\n",
        "from keras.layers.embeddings import Embedding\r\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CHNTF6AHsQUL"
      },
      "source": [
        "## **Data preparation - _Generating Corpus_**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fb2U0kS38MRf",
        "outputId": "190eabff-3a0b-4273-fed7-2f03de19baf9"
      },
      "source": [
        "# Download data from Google drive\r\n",
        "\r\n",
        "'''\r\n",
        "\r\n",
        "ORIGINAL DATASET URL:\r\n",
        "    https://raw.githubusercontent.com/maxim5/stanford-tensorflow-tutorials/master/data/arxiv_abstracts.txt\r\n",
        "\r\n",
        "'''\r\n",
        "\r\n",
        "url = ' https://drive.google.com/uc?id=1YTBR7FiXssaKXHhOZbUbwoWw6jzQxxKW'\r\n",
        "output = 'corpus.txt'\r\n",
        "gdown.download(url, output, quiet=False)\r\n",
        "\r\n",
        "# sentence_length = 40\r\n",
        "\r\n",
        "# Read local file from directory\r\n",
        "with open('corpus.txt') as subject:\r\n",
        "  cache = subject.readlines()\r\n",
        "translator = str.maketrans('', '', string.punctuation) # Remove punctuation\r\n",
        "lines = [doc.lower().translate(translator) for doc in cache] # Switch to lower case"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From:  https://drive.google.com/uc?id=1YTBR7FiXssaKXHhOZbUbwoWw6jzQxxKW\n",
            "To: /content/corpus.txt\n",
            "7.55MB [00:00, 171MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yKMvgmMBEF7o"
      },
      "source": [
        "# PREVIEW OUTPUT ::\r\n",
        "\r\n",
        "# print(lines[0][:100])\r\n",
        "# len(lines)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AKFp-u8951-g"
      },
      "source": [
        "# Generate an list of single/independent words\r\n",
        "\r\n",
        "vocabulary = list(set(' '.join(lines).replace('\\n','').split(' ')))\r\n",
        "primary_store = {}\r\n",
        "for strings, texts in enumerate(vocabulary):\r\n",
        "  primary_store[texts] = strings"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s7pOLvIhEZZE"
      },
      "source": [
        "# PREVIEW OUTPUT ::\r\n",
        "\r\n",
        "# print(vocabulary[:50])\r\n",
        "# len(vocabulary)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "18Aw7bxIHLrW"
      },
      "source": [
        "# Splitting data into Train sets and test sets\r\n",
        "\r\n",
        "X = [] \r\n",
        "y = []\r\n",
        "\r\n",
        "for c in lines:\r\n",
        "  xxxx = c.replace('\\n','').split(' ')\r\n",
        "  X.append(' '.join(xxxx[:-1])) # X from the corpus\r\n",
        "\r\n",
        "  yyyy = [0 for i in range(len(vocabulary))] # Generate Y from the Vocabulary\r\n",
        "  # yyyy[primary_store[xxxx[-1]]] = 1\r\n",
        "  yyyy[primary_store[xxxx[-1]]] = 1\r\n",
        "  y.append(yyyy)\r\n",
        "\r\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)\r\n",
        "y_test = numpy.array(y_test)\r\n",
        "y_train = numpy.array(y_train)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVzeMJVLK-L3"
      },
      "source": [
        "# PREVIEW OUTPUT ::\r\n",
        "\r\n",
        "# print(X_train[:10])\r\n",
        "# print(y_train[:10])\r\n",
        "# print(X_test[:10])\r\n",
        "# print(y_test[:10])"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E-ZoLdTpMBtg"
      },
      "source": [
        "## **Embeddings!**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PLwEZ5NzMA20"
      },
      "source": [
        "# Import the Universal Sentence Encoder's TF Hub module (Here we're making use of version 4)\r\n",
        "# This will take a while but won't be long :)\r\n",
        "\r\n",
        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\"  \r\n",
        "appreciate = hub.load(module_url)\r\n",
        "\r\n",
        "# Making it easier - Function for embedding\r\n",
        "def embed(goodness):\r\n",
        "  return appreciate(goodness)\r\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1MVzWpcONq7p"
      },
      "source": [
        "# REVIEW OUTPUT ::\r\n",
        "\r\n",
        "# appreciate.variables"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFVpJJ-kOgJK"
      },
      "source": [
        "# Wrapping up with the U-S-E\r\n",
        "\r\n",
        "X_train = embed(X_train)\r\n",
        "X_test = embed(X_test)\r\n",
        "X_train = X_train.numpy()\r\n",
        "X_test = X_test.numpy()"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDz7WwWLOzwh"
      },
      "source": [
        "# PREVIEW OUTPUT ::\r\n",
        "\r\n",
        "# print(X_train[:10])\r\n",
        "# print(y_train[:10])\r\n",
        "# print(X_test[:10])\r\n",
        "# print(y_test[:10])\r\n",
        "# print(X_train.shape, X_test.shape, y_test.shape, y_train.shape)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rk1zrMu1Q_QW"
      },
      "source": [
        "# **Building the model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NE2cmc3eQ9qk",
        "outputId": "f4eb968e-0843-4a1a-c8e3-a81587620d7f"
      },
      "source": [
        "model = Sequential()\r\n",
        "# model.add(Embedding(input_dim=len(vocabulary), output_dim=100))\r\n",
        "model = Sequential()\r\n",
        "# model.add(LSTM(units=100, input_shape=[512]))\r\n",
        "model.add(Dense(512, input_shape=[512], activation = 'relu'))\r\n",
        "model.add(Dense(units=len(vocabulary), activation = 'softmax'))\r\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])\r\n",
        "model.summary()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense (Dense)                (None, 512)               262656    \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 2694)              1382022   \n",
            "=================================================================\n",
            "Total params: 1,644,678\n",
            "Trainable params: 1,644,678\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "12j8kuJWUAKJ",
        "outputId": "315baab4-59dc-438f-b6fb-356ccd458339"
      },
      "source": [
        "# Training the model. \r\n",
        "\r\n",
        "model.fit(X_train, y_train, batch_size=512, shuffle=True, epochs=20, validation_data=(X_test, y_test), callbacks=[LambdaCallback()])"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "11/11 [==============================] - 1s 51ms/step - loss: 7.7687 - acc: 0.0569 - val_loss: 6.9287 - val_acc: 0.0511\n",
            "Epoch 2/20\n",
            "11/11 [==============================] - 0s 19ms/step - loss: 6.3200 - acc: 0.0485 - val_loss: 4.4571 - val_acc: 0.0572\n",
            "Epoch 3/20\n",
            "11/11 [==============================] - 0s 17ms/step - loss: 4.2940 - acc: 0.0604 - val_loss: 4.0937 - val_acc: 0.0900\n",
            "Epoch 4/20\n",
            "11/11 [==============================] - 0s 16ms/step - loss: 4.0394 - acc: 0.1304 - val_loss: 3.8953 - val_acc: 0.2056\n",
            "Epoch 5/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 3.8269 - acc: 0.1694 - val_loss: 3.6589 - val_acc: 0.2600\n",
            "Epoch 6/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 3.5807 - acc: 0.2989 - val_loss: 3.3492 - val_acc: 0.3550\n",
            "Epoch 7/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 3.2411 - acc: 0.3852 - val_loss: 2.9764 - val_acc: 0.5217\n",
            "Epoch 8/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 2.8520 - acc: 0.6499 - val_loss: 2.5491 - val_acc: 0.8022\n",
            "Epoch 9/20\n",
            "11/11 [==============================] - 0s 14ms/step - loss: 2.4114 - acc: 0.8419 - val_loss: 2.1011 - val_acc: 0.8767\n",
            "Epoch 10/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 1.9488 - acc: 0.9303 - val_loss: 1.6541 - val_acc: 0.9506\n",
            "Epoch 11/20\n",
            "11/11 [==============================] - 0s 14ms/step - loss: 1.5076 - acc: 0.9761 - val_loss: 1.2461 - val_acc: 0.9889\n",
            "Epoch 12/20\n",
            "11/11 [==============================] - 0s 14ms/step - loss: 1.1225 - acc: 0.9861 - val_loss: 0.9127 - val_acc: 1.0000\n",
            "Epoch 13/20\n",
            "11/11 [==============================] - 0s 16ms/step - loss: 0.8083 - acc: 0.9930 - val_loss: 0.6505 - val_acc: 0.9889\n",
            "Epoch 14/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.5718 - acc: 0.9949 - val_loss: 0.4673 - val_acc: 1.0000\n",
            "Epoch 15/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.4098 - acc: 1.0000 - val_loss: 0.3413 - val_acc: 1.0000\n",
            "Epoch 16/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.3049 - acc: 1.0000 - val_loss: 0.2576 - val_acc: 1.0000\n",
            "Epoch 17/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.2295 - acc: 1.0000 - val_loss: 0.2001 - val_acc: 1.0000\n",
            "Epoch 18/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.1799 - acc: 1.0000 - val_loss: 0.1602 - val_acc: 1.0000\n",
            "Epoch 19/20\n",
            "11/11 [==============================] - 0s 15ms/step - loss: 0.1455 - acc: 1.0000 - val_loss: 0.1314 - val_acc: 1.0000\n",
            "Epoch 20/20\n",
            "11/11 [==============================] - 0s 13ms/step - loss: 0.1201 - acc: 1.0000 - val_loss: 0.1102 - val_acc: 1.0000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f23ad763518>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Vzq-1bH0oOv"
      },
      "source": [
        "#**Unto the tests!**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4Jy3Hbi0nlK"
      },
      "source": [
        "\r\n",
        "# Create function to predict and show detailed output\r\n",
        "\r\n",
        "def next_word(collection=[], extent=1):\r\n",
        "\r\n",
        "  for item in collection:\r\n",
        "    text = item\r\n",
        "    for i in range(extent):\r\n",
        "      prediction = model.predict(x=embed([item]).numpy())\r\n",
        "      idx = np.argmax(prediction[-1])\r\n",
        "      item += ' ' + vocabulary[idx]\r\n",
        "      \r\n",
        "      print(text + ' --> ' + item + '\\nNEXT WORD: ' + item.split(' ')[-1] + '\\n')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W6_rGHmFYMKD",
        "outputId": "04a875dd-ca35-4cec-a8ad-d0038be14f00"
      },
      "source": [
        "# Tests - please feel free to explore\r\n",
        "\r\n",
        "single_text = ['and some other essential']\r\n",
        "\r\n",
        "next_word(single_text)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "and some other essential --> and some other essential networks\n",
            "NEXT WORD: networks\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4tZjEA0MX5pr",
        "outputId": "bd39eb2c-1dea-45cc-9c2c-5fa9ba81c3ec"
      },
      "source": [
        "# Testing on a collection of words\r\n",
        "\r\n",
        "text_collection = ['deep convolutional', 'simple and effective', 'a nonconvex', 'a']\r\n",
        "\r\n",
        "next_word(text_collection)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "deep convolutional --> deep convolutional networks\n",
            "NEXT WORD: networks\n",
            "\n",
            "simple and effective --> simple and effective acceleration\n",
            "NEXT WORD: acceleration\n",
            "\n",
            "a nonconvex --> a nonconvex dataset\n",
            "NEXT WORD: dataset\n",
            "\n",
            "a --> a improvement\n",
            "NEXT WORD: improvement\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vvhuISuHZfzZ"
      },
      "source": [
        "## **For the record**\r\n",
        "\r\n",
        "The Dataset is based on a Tensorflow tutorial from Stanford, so all predicted words will be based on Deep learning and Machine learning _common terms_.\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6SxZt78bhEv"
      },
      "source": [
        "\r\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dJTRmQh6aHSV",
        "outputId": "167ba863-38cc-470d-f560-17e53745cc62"
      },
      "source": [
        "# Storing data\r\n",
        "\r\n",
        "vocabulary = numpy.array(vocabulary)\r\n",
        "numpy.save('./vocabulary.npy', vocabulary)\r\n",
        "model.save('./NWP-USE')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: ./NWP-USE/assets\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: ./NWP-USE/assets\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gxCntzMTaXQ3"
      },
      "source": [
        "##                                                                  END OF NOTEBOOK"
      ],
      "execution_count": 19,
      "outputs": []
    }
  ]
}