pooled GRU + fastText.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.kaggle.com/yekenot/pooled-gru-fasttext/code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate\n",
    "from keras.layers import GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.callbacks import Callback\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os\n",
    "os.environ['OMP_NUM_THREADS'] = '4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../input/train.csv')\n",
    "test = pd.read_csv('../input/test.csv')\n",
    "submission = pd.read_csv('../input/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train[\"comment_text\"].fillna(\"fillna\").values\n",
    "y_train = train[[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]].values\n",
    "X_test = test[\"comment_text\"].fillna(\"fillna\").values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 30000\n",
    "maxlen = 100\n",
    "embed_size = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = text.Tokenizer(num_words=max_features)\n",
    "tokenizer.fit_on_texts(list(X_train) + list(X_test))\n",
    "X_train = tokenizer.texts_to_sequences(X_train)\n",
    "X_test = tokenizer.texts_to_sequences(X_test)\n",
    "x_train = sequence.pad_sequences(X_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(X_test, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = '../input/crawl-300d-2M.vec'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = min(max_features, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, embed_size))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            print(\"\\n ROC-AUC - epoch: %d - score: %.6f \\n\" % (epoch+1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "#from keras import initializations\n",
    "from keras import initializers, regularizers, constraints\n",
    "\n",
    "\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        #self.init = initializations.get('glorot_uniform')\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    inp = Input(shape=(maxlen, ))\n",
    "    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "    x = SpatialDropout1D(0.2)(x)\n",
    "    x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "#     x = SpatialDropout1D(0.5)(x)\n",
    "#     x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "#     x = Attention(maxlen)(x)\n",
    "#     outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "#     avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "#     conc = concatenate([avg_pool, max_pool])\n",
    "    outp = Dense(6, activation=\"sigmoid\")(max_pool)\n",
    "#     outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "    \n",
    "    model = Model(inputs=inp, outputs=outp)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model\n",
    "# model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 300)     9000000     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 100, 300)     0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 100, 160)     182880      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_1 (Glo (None, 160)          0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 160)          0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 320)          0           global_average_pooling1d_1[0][0] \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 6)            1926        concatenate_1[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 9,184,806\n",
      "Trainable params: 9,184,806\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "m = get_model()\n",
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "127656/127656 [==============================] - 1584s 12ms/step - loss: 0.0916 - acc: 0.9720\n",
      "Epoch 2/2\n",
      "127656/127656 [==============================] - 1533s 12ms/step - loss: 0.0459 - acc: 0.9832\n",
      "acc: 98.2777166264%\n",
      "Epoch 1/2\n",
      "127657/127657 [==============================] - 1523s 12ms/step - loss: 0.1039 - acc: 0.9695\n",
      "Epoch 2/2\n",
      "127657/127657 [==============================] - 1518s 12ms/step - loss: 0.0481 - acc: 0.9826\n",
      "acc: 98.2155166107%\n",
      "Epoch 1/2\n",
      "127657/127657 [==============================] - 1525s 12ms/step - loss: 0.0912 - acc: 0.9731\n",
      "Epoch 2/2\n",
      "127657/127657 [==============================] - 1526s 12ms/step - loss: 0.0454 - acc: 0.9831\n",
      "acc: 98.3554760832%\n",
      "Epoch 1/2\n",
      "127657/127657 [==============================] - 1516s 12ms/step - loss: 0.0999 - acc: 0.9684\n",
      "Epoch 2/2\n",
      "127657/127657 [==============================] - 1522s 12ms/step - loss: 0.0475 - acc: 0.9826\n",
      "acc: 98.2536397494%\n",
      "Epoch 1/2\n",
      "127657/127657 [==============================] - 1529s 12ms/step - loss: 0.1047 - acc: 0.9675\n",
      "Epoch 2/2\n",
      "127657/127657 [==============================] - 1522s 12ms/step - loss: 0.0478 - acc: 0.9826\n",
      "acc: 98.2975076414%\n",
      "98.2799713422% (+/- 0.0466075887%)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import KFold\n",
    "\n",
    "kf = KFold(len(x_train), n_folds=5)\n",
    "cvscores = []\n",
    "\n",
    "for train,test in kf:\n",
    "    model=get_model()\n",
    "    hist = model.fit(x_train[train], y_train[train], batch_size=512, epochs=2, verbose=1)\n",
    "    \n",
    "    # evaluate the model\n",
    "    scores = model.evaluate(x_train[test], y_train[test], verbose=0)\n",
    "    print(\"%s: %.10f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "    cvscores.append(scores[1] * 100)\n",
    "    \n",
    "print(\"%.10f%% (+/- %.10f%%)\" % (np.mean(cvscores), np.std(cvscores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch_size = 512\n",
    "# epochs = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95, random_state=233)\n",
    "# RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n",
    "#                  callbacks=[RocAuc], verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_pred = model.predict(x_test, batch_size=32)\n",
    "# submission[[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]] = y_pred\n",
    "# submission.to_csv('submission_2grufasttext_attention_5fold.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### log results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission_grufasttext.csv\n",
    "\n",
    "inp = Input(shape=(maxlen, ))\n",
    "\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "avg_pool = GlobalAveragePooling1D()(x)\n",
    "\n",
    "max_pool = GlobalMaxPooling1D()(x)\n",
    "\n",
    "conc = concatenate([avg_pool, max_pool])\n",
    "\n",
    "outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "\n",
    "- result:\n",
    "\n",
    "Epoch 1/2\n",
    "151592/151592 [==============================] - 2463s 16ms/step - loss: 0.0457 - acc: 0.9828 - val_loss: 0.0454 - val_acc: 0.9829\n",
    "\n",
    " ROC-AUC - epoch: 1 - score: 0.987610 \n",
    "\n",
    "Epoch 2/2\n",
    "151592/151592 [==============================] - 2059s 14ms/step - loss: 0.0374 - acc: 0.9854 - val_loss: 0.0456 - val_acc: 0.9826\n",
    "\n",
    " ROC-AUC - epoch: 2 - score: 0.987506\n",
    "\n",
    "#### LB: 0.9812"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "x = SpatialDropout1D(0.5)(x)\n",
    "\n",
    "avg_pool = GlobalAveragePooling1D()(x)\n",
    "\n",
    "max_pool = GlobalMaxPooling1D()(x)\n",
    "\n",
    "conc = concatenate([avg_pool, max_pool])\n",
    "\n",
    "- result\n",
    "\n",
    "Epoch 1/2\n",
    "151592/151592 [==============================] - 2598s 17ms/step - loss: 0.0544 - acc: 0.9806 - val_loss: 0.0454 - val_acc: 0.9827\n",
    "\n",
    " ROC-AUC - epoch: 1 - score: 0.986717 \n",
    "\n",
    "Epoch 2/2\n",
    "151592/151592 [==============================] - 2195s 14ms/step - loss: 0.0412 - acc: 0.9842 - val_loss: 0.0460 - val_acc: 0.9824\n",
    "\n",
    " ROC-AUC - epoch: 2 - score: 0.986632 \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission_grufasttext_attention.csv\n",
    "\n",
    "inp = Input(shape=(maxlen, ))\n",
    "\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "x = Attention(maxlen)(x)\n",
    "\n",
    "outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "\n",
    "- result\n",
    "\n",
    "Epoch 1/2\n",
    "151592/151592 [==============================] - 2307s 15ms/step - loss: 0.0511 - acc: 0.9818 - val_loss: 0.0470 - val_acc: 0.9821\n",
    "\n",
    " ROC-AUC - epoch: 1 - score: 0.984153 \n",
    "\n",
    "Epoch 2/2\n",
    "151592/151592 [==============================] - 2148s 14ms/step - loss: 0.0388 - acc: 0.9850 - val_loss: 0.0455 - val_acc: 0.9829\n",
    "\n",
    " ROC-AUC - epoch: 2 - score: 0.986293 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission_grufasttext_attention_2.csv\n",
    "\n",
    "inp = Input(shape=(maxlen, ))\n",
    "\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "x = SpatialDropout1D(0.5)(x)\n",
    "\n",
    "x = Attention(maxlen)(x)\n",
    "\n",
    "outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "\n",
    "- result\n",
    "\n",
    "Epoch 1/2\n",
    "151592/151592 [==============================] - 2337s 15ms/step - loss: 0.0550 - acc: 0.9808 - val_loss: 0.0480 - val_acc: 0.9823\n",
    "\n",
    " ROC-AUC - epoch: 1 - score: 0.980985 \n",
    "\n",
    "Epoch 2/2\n",
    "151592/151592 [==============================] - 2280s 15ms/step - loss: 0.0419 - acc: 0.9841 - val_loss: 0.0461 - val_acc: 0.9827\n",
    "\n",
    " ROC-AUC - epoch: 2 - score: 0.984969 \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission_2grufasttext_attention.csv\n",
    "\n",
    "inp = Input(shape=(maxlen, ))\n",
    "\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "x = SpatialDropout1D(0.5)(x)\n",
    "\n",
    "x = Bidirectional(GRU(80, return_sequences=True))(x)\n",
    "\n",
    "x = Attention(maxlen)(x)\n",
    "\n",
    "outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "\n",
    "- result\n",
    "\n",
    "Train on 151592 samples, validate on 7979 samples\n",
    "Epoch 1/2\n",
    "151592/151592 [==============================] - 3107s 20ms/step - loss: 0.0522 - acc: 0.9813 - val_loss: 0.0474 - val_acc: 0.9826\n",
    "\n",
    " ROC-AUC - epoch: 1 - score: 0.984434 \n",
    "\n",
    "Epoch 2/2\n",
    "151592/151592 [==============================] - 3054s 20ms/step - loss: 0.0406 - acc: 0.9842 - val_loss: 0.0464 - val_acc: 0.9827\n",
    "\n",
    " ROC-AUC - epoch: 2 - score: 0.986317 \n",
    " \n",
    " #### LB: 0.9813"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission_2grufasttext_attention_10fold.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}