jupyter_best_f1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sklearn.metrics import f1_score\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torchtext\n",
    "from torchtext.data import get_tokenizer\n",
    "from collections import Counter\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection for training, Validation and testing sets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Capture training and testing set as in the example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(list_of_list):\n",
    "    return [item for sublist in list_of_list for item in sublist]\n",
    "\n",
    "path_to_training = Path(\"training\")\n",
    "path_to_test = Path(\"test\")\n",
    "\n",
    "#####\n",
    "# training and test sets of transcription ids\n",
    "#####\n",
    "training_set = ['ES2002', 'ES2005', 'ES2006', 'ES2007', 'ES2008', 'ES2009', 'ES2010', 'ES2012', 'ES2013', 'ES2015', 'ES2016', 'IS1000', 'IS1001', 'IS1002', 'IS1003', 'IS1004', 'IS1005', 'IS1006', 'IS1007', 'TS3005', 'TS3008', 'TS3009', 'TS3010', 'TS3011', 'TS3012']\n",
    "training_set = flatten([[m_id+s_id for s_id in 'abcd'] for m_id in training_set])\n",
    "training_set.remove('IS1002a')\n",
    "training_set.remove('IS1005d')\n",
    "training_set.remove('TS3012c')\n",
    "\n",
    "test_set = ['ES2003', 'ES2004', 'ES2011', 'ES2014', 'IS1008', 'IS1009', 'TS3003', 'TS3004', 'TS3006', 'TS3007']\n",
    "test_set = flatten([[m_id+s_id for s_id in 'abcd'] for m_id in test_set])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cutting the training set into train_set and val_set.   (for validation)\n",
    "\n",
    "One-hot encoding of speakers and capturing of texts for train, test, and val sets. Capturing of y for train and val sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_text = []  #this tab will contain every text of the datas\n",
    "\n",
    "list_speaker = ['PM','ME','ID','UI']\n",
    "\n",
    "with open(\"training_labels.json\", \"r\") as file:\n",
    "    training_labels = json.load(file)\n",
    "\n",
    "X_train_text = {}   #dic of the text for every discussion for the train\n",
    "X_train_speaker = {}\n",
    "y_train = {}\n",
    "\n",
    "X_val_text = {}\n",
    "X_val_speaker = {}\n",
    "y_val = {}\n",
    "\n",
    "#all the val set which represent 1/5 of training_set\n",
    "val_set = ['ES2002a', 'ES2005b', 'ES2006c', 'ES2007d', 'ES2009a', 'ES2010b', 'ES2012c', 'ES2013d', 'ES2016a', 'IS1000b', 'IS1001c', 'IS1002d', 'IS1004a', 'IS1005b', 'IS1006c', 'IS1007d', 'TS3008a', 'TS3009b', 'TS3010c', 'TS3011d']\n",
    "train_set = []  #the other 4/5 of the training_set\n",
    "\n",
    "for transcription_id in training_set:\n",
    "    with open(path_to_training / f\"{transcription_id}.json\", \"r\") as file:\n",
    "        transcription = json.load(file)\n",
    "    X_train_text_transcription = []   #tab of the text for this discussion\n",
    "    X_train_speaker_transcription = []   #tab of one_hot_encending of speaker for this discussion\n",
    "    for utterance in transcription: \n",
    "        X_train_text_transcription.append(utterance[\"text\"])\n",
    "        total_text.append(utterance[\"text\"])\n",
    "        speaker_one_hot = [0,0,0,0]\n",
    "        speaker_one_hot[list_speaker.index(utterance[\"speaker\"])] = 1\n",
    "        X_train_speaker_transcription.append(speaker_one_hot)\n",
    "    if transcription_id in val_set:\n",
    "        y_val[transcription_id] = training_labels[transcription_id]\n",
    "        X_val_text[transcription_id] = X_train_text_transcription\n",
    "        X_val_speaker[transcription_id] = X_train_speaker_transcription\n",
    "    else:\n",
    "        train_set.append(transcription_id)\n",
    "        y_train[transcription_id] = training_labels[transcription_id]\n",
    "        X_train_text[transcription_id] = X_train_text_transcription\n",
    "        X_train_speaker[transcription_id] = X_train_speaker_transcription\n",
    "\n",
    "X_test_text = {}\n",
    "X_test_speaker = {}\n",
    "for transcription_id in test_set:\n",
    "    with open(path_to_test / f\"{transcription_id}.json\", \"r\") as file:\n",
    "        transcription = json.load(file)\n",
    "    X_test_text_transcription = []\n",
    "    X_test_speaker_transcription = []\n",
    "    for utterance in transcription:\n",
    "        X_test_text_transcription.append(utterance[\"text\"])\n",
    "        total_text.append(utterance[\"text\"])\n",
    "        speaker_one_hot = [0,0,0,0]\n",
    "        speaker_one_hot[list_speaker.index(utterance[\"speaker\"])] = 1\n",
    "        X_test_speaker_transcription.append(speaker_one_hot)\n",
    "    X_test_text[transcription_id] = X_test_text_transcription\n",
    "    X_test_speaker[transcription_id] = X_test_speaker_transcription\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now retrieve the previous text and the following texts. The previous text for the first text of each discussion is \"BEGINNING\". The following texts are concatenated, and this is where we see the reason for creating a dictionary of discussions rather than a large array of text previously.\n",
    "\n",
    "We will also retrieve a one-hot encoding of the description preceding the sentence. There are 16 possible descriptions, but we add one extra for the first text, resulting in a vector of size 17. Additionally, we obtain a sum of the one-hot encodings of the descriptions following this text, resulting in a vector of size 16.\n",
    "\n",
    "Sometimes in the conversation, there are no sentences following another (leaf of the conversation tree), so the text afterward is simply \"\" (empty), and the description is encoded by [0] * 16.\n",
    "\n",
    "We give an example at the end of this next sell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Can I close this ?\n",
      "[1, 0, 0, 0]\n",
      "BEGGINING\n",
      "Uh we don't have any changes ,\n",
      "[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "X_train_text_before = {}   #sentence before \n",
    "X_train_text_after = {}   #all the sentence after (sometimes empty)\n",
    "\n",
    "list_description = [\"Parallel\", \"Correction\", \"Q-Elab\", \"Conditional\", \"Alternation\", \"Narration\", \"Background\",\"Continuation\", \"Explanation\", \"Elaboration\" , \"Acknowledgement\", \"Comment\", \"Result\", \"Question-answer_pair\", \"Contrast\", \"Clarification_question\"]\n",
    "\n",
    "X_train_description_before = {}    #the one_hot_encoding of the description before\n",
    "X_train_description_after = {}    #the sum of one_hot_encoding of the description after  (sometimes [0]*16)\n",
    "for transcription_id in train_set:\n",
    "    with open(path_to_training / f\"{transcription_id}.txt\", \"r\") as file:\n",
    "        transcription = file.readlines()\n",
    "\n",
    "    X_train_text_before_transcription = [\"BEGGINING\"]   #the sentence before the first sentence\n",
    "    total_text.append(\"BEGGINING\") \n",
    "\n",
    "    X_train_text_after_transcription = [\"\"] * (len(transcription) + 1)   \n",
    "    \n",
    "    vector = [0] * 16\n",
    "    X_train_description_after_transcription = [vector.copy() for i in range(len(transcription) + 1)] \n",
    "\n",
    "    description_one_hot = [0] * (len(list_description) + 1)\n",
    "    description_one_hot[0] = 1\n",
    "    X_train_description_before_transcription = [description_one_hot]   #the description before the first description is always [1,0,0,...]\n",
    "\n",
    "    for line in transcription:\n",
    "        tab = line.split()\n",
    "\n",
    "        description_one_hot = [0] * (len(list_description) + 1)\n",
    "        description_one_hot[list_description.index(tab[1]) + 1] = 1\n",
    "        X_train_description_before_transcription.append(description_one_hot)\n",
    "\n",
    "        X_train_description_after_transcription[int(tab[0])][list_description.index(tab[1])] += 1\n",
    "        \n",
    "        X_train_text_before_transcription.append(X_train_text[transcription_id][int(tab[0])])\n",
    "\n",
    "        X_train_text_after_transcription[int(tab[0])] += X_train_text[transcription_id][int(tab[2])]\n",
    "\n",
    "    X_train_text_before[transcription_id] = X_train_text_before_transcription\n",
    "    X_train_text_after[transcription_id] = X_train_text_after_transcription\n",
    "    X_train_description_before[transcription_id] = X_train_description_before_transcription\n",
    "    X_train_description_after[transcription_id] = X_train_description_after_transcription\n",
    "\n",
    "#we did exactement the same for the validation\n",
    "X_val_text_before = {}\n",
    "X_val_text_after = {}\n",
    "X_val_description_before = {}\n",
    "X_val_description_after = {}\n",
    "for transcription_id in val_set:\n",
    "    with open(path_to_training / f\"{transcription_id}.txt\", \"r\") as file:\n",
    "        transcription = file.readlines()\n",
    "\n",
    "    X_val_text_before_transcription = [\"BEGGINING\"]\n",
    "    total_text.append(\"BEGGINING\")\n",
    "\n",
    "    X_val_text_after_transcription = [\"\"] * (len(transcription) + 1)\n",
    "\n",
    "    vector = [0] * 16\n",
    "    X_val_description_after_transcription = [vector.copy() for i in range(len(transcription) + 1)]\n",
    "\n",
    "    description_one_hot = [0] * (len(list_description) + 1)\n",
    "    description_one_hot[0] = 1\n",
    "    X_val_description_before_transcription = [description_one_hot]\n",
    "\n",
    "    for line in transcription:\n",
    "        tab = line.split()\n",
    "\n",
    "        description_one_hot = [0] * (len(list_description) + 1)\n",
    "        description_one_hot[list_description.index(tab[1]) + 1] = 1\n",
    "        X_val_description_before_transcription.append(description_one_hot)\n",
    "\n",
    "        X_val_description_after_transcription[int(tab[0])][list_description.index(tab[1])] += 1\n",
    "        \n",
    "        X_val_text_before_transcription.append(X_val_text[transcription_id][int(tab[0])])\n",
    "\n",
    "        X_val_text_after_transcription[int(tab[0])] += X_val_text[transcription_id][int(tab[2])]\n",
    "\n",
    "    X_val_text_before[transcription_id] = X_val_text_before_transcription\n",
    "    X_val_text_after[transcription_id] = X_val_text_after_transcription\n",
    "    X_val_description_before[transcription_id] = X_val_description_before_transcription\n",
    "    X_val_description_after[transcription_id] = X_val_description_after_transcription\n",
    "\n",
    "#we did exactement the same for the test\n",
    "X_test_text_before = {}\n",
    "X_test_text_after = {}\n",
    "X_test_description_before = {}\n",
    "X_test_description_after = {}\n",
    "for transcription_id in test_set:\n",
    "    with open(path_to_test / f\"{transcription_id}.txt\", \"r\") as file:\n",
    "        transcription = file.readlines()\n",
    "\n",
    "    X_test_text_before_transcription = [\"BEGGINING\"]\n",
    "    total_text.append(\"BEGGINING\")\n",
    "\n",
    "    X_test_text_after_transcription = [\"\"] * (len(transcription) + 1)\n",
    "\n",
    "    t = [0] * 16\n",
    "    X_test_description_after_transcription = [t.copy() for i in range(len(transcription) + 1)]\n",
    "\n",
    "    description_one_hot = [0] * (len(list_description) + 1)\n",
    "    description_one_hot[0] = 1\n",
    "    X_test_description_before_transcription = [description_one_hot]\n",
    "\n",
    "    for line in transcription:\n",
    "        tab = line.split()\n",
    "\n",
    "        description_one_hot = [0] * (len(list_description) + 1)\n",
    "        description_one_hot[list_description.index(tab[1]) + 1] = 1\n",
    "        X_test_description_before_transcription.append(description_one_hot)\n",
    "\n",
    "        X_test_description_after_transcription[int(tab[0])][list_description.index(tab[1])] += 1\n",
    "\n",
    "        X_test_text_before_transcription.append(X_test_text[transcription_id][int(tab[0])])\n",
    "\n",
    "        X_test_text_after_transcription[int(tab[0])] += X_test_text[transcription_id][int(tab[2])]\n",
    "\n",
    "    X_test_text_before[transcription_id] = X_test_text_before_transcription\n",
    "    X_test_text_after[transcription_id] = X_test_text_after_transcription\n",
    "    X_test_description_before[transcription_id] = X_test_description_before_transcription\n",
    "    X_test_description_after[transcription_id] = X_test_description_after_transcription\n",
    "\n",
    "\n",
    "#example\n",
    "\n",
    "print(X_train_text[\"TS3012d\"][0])  #the first sentence\n",
    "print(X_train_speaker[\"TS3012d\"][0])    #one_hot_encoding of the speaker\n",
    "print(X_train_text_before[\"TS3012d\"][0])    #the sentence before (here BEGGINING)\n",
    "print(X_train_text_after[\"TS3012d\"][0])    #all the sentence after   (here there is just one sentence)\n",
    "print(X_train_description_before[\"TS3012d\"][0])   #one_hot_encoding of the description before (here the particular case of the first sentence)\n",
    "print(X_train_description_after[\"TS3012d\"][0])    #sum of one_hot_encoding of the descrption after"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we no longer need the conversation segmentation (it was simpler for retrieving sentences afterward). Therefore, we concatenate all conversations into large vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_text_tab = []\n",
    "X_train_speaker_tab = []\n",
    "X_train_text_before_tab = []\n",
    "X_train_text_after_tab = []\n",
    "X_train_description_before_tab = []\n",
    "X_train_description_after_tab = []\n",
    "y_train_tab = []\n",
    "\n",
    "for transcription_id in train_set:\n",
    "    X_train_text_tab.extend(X_train_text[transcription_id])\n",
    "    X_train_speaker_tab.extend(X_train_speaker[transcription_id])\n",
    "    X_train_text_before_tab.extend(X_train_text_before[transcription_id])\n",
    "    X_train_text_after_tab.extend(X_train_text_after[transcription_id])\n",
    "    X_train_description_before_tab.extend(X_train_description_before[transcription_id])\n",
    "    X_train_description_after_tab.extend(X_train_description_after[transcription_id])\n",
    "    y_train_tab.extend(y_train[transcription_id])\n",
    "\n",
    "X_val_text_tab = []\n",
    "X_val_speaker_tab = []\n",
    "X_val_text_before_tab = []\n",
    "X_val_text_after_tab = []\n",
    "X_val_description_before_tab = []\n",
    "X_val_description_after_tab = []\n",
    "y_val_tab = []\n",
    "\n",
    "for transcription_id in val_set:\n",
    "    X_val_text_tab.extend(X_val_text[transcription_id])\n",
    "    X_val_speaker_tab.extend(X_val_speaker[transcription_id])\n",
    "    X_val_text_before_tab.extend(X_val_text_before[transcription_id])\n",
    "    X_val_text_after_tab.extend(X_val_text_after[transcription_id])\n",
    "    X_val_description_before_tab.extend(X_val_description_before[transcription_id])\n",
    "    X_val_description_after_tab.extend(X_val_description_after[transcription_id])\n",
    "    y_val_tab.extend(y_val[transcription_id])\n",
    "\n",
    "X_test_text_tab = []\n",
    "X_test_speaker_tab = []\n",
    "X_test_text_before_tab = []\n",
    "X_test_text_after_tab = []\n",
    "X_test_description_before_tab = []\n",
    "X_test_description_after_tab = []\n",
    "\n",
    "for transcription_id in test_set:\n",
    "    X_test_text_tab.extend(X_test_text[transcription_id])\n",
    "    X_test_speaker_tab.extend(X_test_speaker[transcription_id])\n",
    "    X_test_text_before_tab.extend(X_test_text_before[transcription_id])\n",
    "    X_test_text_after_tab.extend(X_test_text_after[transcription_id])\n",
    "    X_test_description_before_tab.extend(X_test_description_before[transcription_id])\n",
    "    X_test_description_after_tab.extend(X_test_description_after[transcription_id])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the same tokenizer and vectorizer than the TD, for us it has the best result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = get_tokenizer(\"basic_english\")\n",
    "\n",
    "words=[]\n",
    "num_words = 1500\n",
    "\n",
    "total_text.append(\"First\")\n",
    "\n",
    "for text in total_text:\n",
    "    tokens=tokenizer(text)\n",
    "    words.extend(tokens)\n",
    "\n",
    "top = dict(Counter(words).most_common(1500))\n",
    "vocab = torchtext.vocab.vocab(top, specials = ['<unk>', '<pad>'])\n",
    "\n",
    "vocab.set_default_index(vocab['<unk>'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len=60   #our first hyperparameter \n",
    "\n",
    "def vectorize_sentences(reviews, max_len):\n",
    "    vectors=[]\n",
    "    for text in reviews:\n",
    "        tokens=tokenizer(text)\n",
    "        v=vocab.forward(tokens)\n",
    "        if len(v) > max_len : v = v[:max_len]\n",
    "        if len(v) < max_len : #padding\n",
    "            tmp = np.full(max_len, vocab['<pad>'])\n",
    "            tmp[0:len(v)]=v \n",
    "            v = tmp\n",
    "        vectors.append(np.array(v))\n",
    "    return np.array(vectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We vectorize our data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tr_text_vector = vectorize_sentences(X_train_text_tab, max_len)\n",
    "X_tr_speaker_vector = np.array(X_train_speaker_tab)\n",
    "X_tr_text_before_vector = vectorize_sentences(X_train_text_before_tab, max_len)\n",
    "X_tr_text_after_vector = vectorize_sentences(X_train_text_after_tab, max_len)\n",
    "X_tr_description_before_vector = np.array(X_train_description_before_tab)\n",
    "X_tr_description_after_vector = np.array(X_train_description_after_tab)\n",
    "y_tr_vector = np.array(y_train_tab).reshape(-1,1)\n",
    "\n",
    "X_va_text_vector = vectorize_sentences(X_val_text_tab, max_len)\n",
    "X_va_speaker_vector = np.array(X_val_speaker_tab)\n",
    "X_va_text_before_vector = vectorize_sentences(X_val_text_before_tab, max_len)\n",
    "X_va_text_after_vector = vectorize_sentences(X_val_text_after_tab, max_len)\n",
    "X_va_description_before_vector = np.array(X_val_description_before_tab)\n",
    "X_va_description_after_vector = np.array(X_val_description_after_tab)\n",
    "y_va_vector = np.array(y_val_tab).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We transform our data into tensor and cut by batches, we can see that we shuffle all our data and we didn't keep the structure by discussions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64   #our second hyperparameter\n",
    "\n",
    "X_tr_text_tensor = torch.tensor(X_tr_text_vector).to(device)\n",
    "X_tr_text_before_tensor = torch.tensor(X_tr_text_before_vector).to(device)\n",
    "X_tr_text_after_tensor = torch.tensor(X_tr_text_after_vector).to(device)\n",
    "X_tr_description_before_tensor = torch.tensor(X_tr_description_before_vector, dtype=torch.float32).to(device)\n",
    "X_tr_description_after_tensor = torch.tensor(X_tr_description_after_vector, dtype=torch.float32).to(device)\n",
    "X_tr_speaker_tensor = torch.tensor(X_tr_speaker_vector, dtype=torch.float32).to(device)\n",
    "y_tr_tensor = torch.tensor(y_tr_vector, dtype=torch.float32).to(device)\n",
    "\n",
    "X_va_text_tensor = torch.tensor(X_va_text_vector).to(device)\n",
    "X_va_text_before_tensor = torch.tensor(X_va_text_before_vector).to(device)\n",
    "X_va_text_after_tensor = torch.tensor(X_va_text_after_vector).to(device)\n",
    "X_va_description_before_tensor = torch.tensor(X_va_description_before_vector, dtype=torch.float32).to(device)\n",
    "X_va_description_after_tensor = torch.tensor(X_va_description_after_vector, dtype=torch.float32).to(device)\n",
    "X_va_speaker_tensor = torch.tensor(X_va_speaker_vector, dtype=torch.float32).to(device)\n",
    "y_va_tensor = torch.tensor(y_va_vector, dtype=torch.float32).to(device)\n",
    "\n",
    "trainset = TensorDataset(X_tr_text_tensor, X_tr_text_before_tensor, X_tr_text_after_tensor, X_tr_description_before_tensor, X_tr_description_after_tensor, X_tr_speaker_tensor, y_tr_tensor)\n",
    "valset = TensorDataset(X_va_text_tensor, X_va_text_before_tensor, X_va_text_after_tensor, X_va_description_before_tensor, X_va_description_after_tensor, X_va_speaker_tensor, y_va_tensor)\n",
    "\n",
    "train_loader = DataLoader(trainset, shuffle=True, batch_size=batch_size)\n",
    "val_loader = DataLoader(valset, shuffle=True, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here there are some of our models, we try to show that we complexified our models step by step. Our best model is GRU_LSTM_final. Some models use just the sentence, other use different informations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearModel(nn.Module):\n",
    "    def __init__(self, input_dim, embedding_dim, hidden_dim, max_len):\n",
    "        super(LinearModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
    "        \n",
    "        self.fc1 = nn.Linear(max_len*embedding_dim, hidden_dim) \n",
    "        \n",
    "        self.fc2 = nn.Linear(hidden_dim, 1) \n",
    "        \n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded = self.embedding(text)\n",
    "\n",
    "        embedded = embedded.view(-1, max_len * embedding_dim)\n",
    "\n",
    "        out = self.fc1(embedded)\n",
    "        \n",
    "        out = torch.relu(out)\n",
    "\n",
    "        out = self.fc2(out)\n",
    "        \n",
    "        out = torch.sigmoid(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvModel(nn.Module):\n",
    "    def __init__(self, input_dim, embedding_dim, num_filter, max_len):\n",
    "        super(ConvModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
    "\n",
    "        self.conv_0 = nn.Conv2d(1, num_filter, (3, embedding_dim))\n",
    "        self.conv_1 = nn.Conv2d(1, num_filter, (4, embedding_dim))\n",
    "        self.conv_2 = nn.Conv2d(1, num_filter, (5, embedding_dim))\n",
    "\n",
    "        self.maxpool_0 = nn.MaxPool2d((max_len - 3 + 1, 1))\n",
    "        self.maxpool_1 = nn.MaxPool2d((max_len - 4 + 1, 1))\n",
    "        self.maxpool_2 = nn.MaxPool2d((max_len - 5 + 1, 1))\n",
    "\n",
    "        self.fc = nn.Linear(num_filter * 3, 1)\n",
    "\n",
    "        self.dropout = nn.Dropout(0.5)\n",
    "        \n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker): \n",
    "        embedded = self.embedding(text) \n",
    "        embedded = embedded.unsqueeze(1)\n",
    "\n",
    "        x0 = torch.relu(self.conv_0(embedded))\n",
    "        x1 = torch.relu(self.conv_1(embedded))\n",
    "        x2 = torch.relu(self.conv_2(embedded))\n",
    "\n",
    "        x0 = self.maxpool_0(x0)\n",
    "        x1 = self.maxpool_1(x1)\n",
    "        x2 = self.maxpool_2(x2)\n",
    "        \n",
    "        x = torch.cat((x0, x1, x2), dim=1)\n",
    "\n",
    "        x = x.view(x.size(0), -1)\n",
    "        \n",
    "        x = torch.relu(self.fc(x))\n",
    "\n",
    "        x = self.dropout(x)\n",
    "\n",
    "        x = torch.sigmoid(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMModel(nn.Module):\n",
    "    def __init__(self, input_dim, embed_dim, hidden_dim):\n",
    "        super(LSTMModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(input_dim, embed_dim)\n",
    "        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers=2, dropout=0.4, bidirectional=True, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim * 2, 1)\n",
    "        self.dropout = nn.Dropout(0.4)\n",
    "\n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded = self.dropout(self.embedding(text))\n",
    "        output, (hidden, cell) = self.lstm(embedded)\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "        out = torch.sigmoid(self.fc(hidden))\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRU(nn.Module):\n",
    "    def __init__(self, input_dim, embed_dim, hidden_dim):\n",
    "        super(GRU, self).__init__()\n",
    "        self.embeding = nn.Embedding(num_embeddings=input_dim, embedding_dim=embed_dim)\n",
    "        self.gru = nn.GRU(input_size=embed_dim, hidden_size=hidden_dim, num_layers=3, dropout=0.5, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded = self.embeding(text)\n",
    "        output, hidden = self.gru(embedded)\n",
    "        truncated = output[:, -1 , :]\n",
    "        out = self.linear(truncated)\n",
    "        out = torch.sigmoid(out)\n",
    "        return(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiInputModel(nn.Module):\n",
    "    def __init__(self, input_dim, embed_dim, hidden_dim, hidden_dim_2):\n",
    "        super(MultiInputModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(input_dim, embed_dim)\n",
    "        self.lstm = nn.LSTM(embed_dim + 17 + 16, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, hidden_dim_2)\n",
    "        self.dropout = nn.Dropout(p=0.5)\n",
    "        self.fc2 = nn.Linear(hidden_dim_2, 1)\n",
    "        \n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded = self.embedding(text)\n",
    "        embedded = torch.cat((embedded, description_before.unsqueeze(1).expand(-1, text.size(1), -1)), dim=2)\n",
    "        embedded = torch.cat((embedded, description_after.unsqueeze(1).expand(-1, text.size(1), -1)), dim=2)\n",
    "        embedded = torch.relu(embedded)\n",
    "        lstm_out, _ = self.lstm(embedded)\n",
    "        lstm_out = lstm_out[:, -1, :]\n",
    "        lstm_out = self.dropout(lstm_out)\n",
    "        out = torch.relu(self.fc(lstm_out))\n",
    "        out = self.fc2(out)\n",
    "        out = torch.sigmoid(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRU_LSTM(nn.Module):\n",
    "    def __init__(self, input_dim, embed_dim, hidden_dim, hidden_dim_2):\n",
    "        super(GRU_LSTM, self).__init__()\n",
    "        self.embeding = nn.Embedding(num_embeddings=input_dim, embedding_dim=embed_dim)\n",
    "        self.gru = nn.GRU(input_size=embed_dim + 16, hidden_size=hidden_dim, num_layers=3, dropout=0.5, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim + hidden_dim_2, 1)\n",
    "        self.lstm = nn.LSTM(embed_dim + 16, hidden_dim_2, batch_first=True)\n",
    "        self.dropout = nn.Dropout(p=0.5)\n",
    "        self.dropout2 = nn.Dropout(p=0.5)\n",
    "\n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded = self.embeding(text)\n",
    "        embedded = torch.cat((embedded, description_after.unsqueeze(1).expand(-1, text.size(1), -1)), dim=2)\n",
    "        embedded = torch.relu(embedded)\n",
    "\n",
    "        output, hidden = self.gru(embedded)\n",
    "        truncated = output[:, -1 , :]\n",
    "        truncated = self.dropout(truncated)\n",
    "\n",
    "        lstm_out, _ = self.lstm(embedded)\n",
    "        lstm_out = lstm_out[:, -1, :]\n",
    "        lstm_out = self.dropout2(lstm_out)\n",
    "\n",
    "        x = torch.cat([truncated, lstm_out], dim=1)\n",
    "\n",
    "        out = self.linear(x)\n",
    "\n",
    "        out = torch.sigmoid(out)\n",
    "        \n",
    "        return(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRU_LSTM_final(nn.Module):\n",
    "    def __init__(self, input_dim, embed_dim, hidden_dim, hidden_dim_2, num_layers):\n",
    "        super(GRU_LSTM_final, self).__init__()\n",
    "        self.embedding = nn.Embedding(input_dim, embed_dim)\n",
    "        \n",
    "        self.gru = nn.GRU(embed_dim, hidden_dim, num_layers=num_layers, dropout=0.5, batch_first=True)\n",
    "        self.gru2 = nn.GRU(embed_dim, hidden_dim, num_layers=num_layers, dropout=0.5, batch_first=True)\n",
    "        self.gru3 = nn.GRU(embed_dim, hidden_dim, num_layers=num_layers, dropout=0.5, batch_first=True)\n",
    "\n",
    "        self.lstm = nn.LSTM(embed_dim, hidden_dim_2, num_layers=num_layers, dropout=0.5, bidirectional=True, batch_first=True)\n",
    "        self.lstm2 = nn.LSTM(embed_dim, hidden_dim_2, num_layers=num_layers, dropout=0.5, bidirectional=True, batch_first=True)\n",
    "        self.lstm3 = nn.LSTM(embed_dim, hidden_dim_2, num_layers=num_layers, dropout=0.5, bidirectional=True, batch_first=True)\n",
    "\n",
    "        self.linear_gru = nn.Linear(hidden_dim * 3, 64)\n",
    "\n",
    "        self.linear_lstm = nn.Linear(hidden_dim_2 * 6, 64)\n",
    "\n",
    "        self.linear2 = nn.Linear(64 * 2 + 17 + 16 + 4, 1)\n",
    "\n",
    "        self.dropout = nn.Dropout(0.3)\n",
    "        \n",
    "    def forward(self, text, text_before, text_after, description_before, description_after, speaker):\n",
    "        embedded_text = self.dropout(self.embedding(text))\n",
    "        embedded_before = self.dropout(self.embedding(text_before))\n",
    "        embedded_after = self.dropout(self.embedding(text_after))\n",
    "        \n",
    "        output_before, _ = self.gru(embedded_before)\n",
    "        output_text, _ = self.gru2(embedded_text)\n",
    "        output_after, _ = self.gru3(embedded_after)\n",
    "\n",
    "        t_text = output_text[:, -1, :] #we keep the last word because it collected information from the other words\n",
    "        t_before = output_before[:, -1, :]\n",
    "        t_after = output_after[:, -1, :]\n",
    "\n",
    "        out1 = torch.cat([t_text, t_before, t_after], dim=1)\n",
    "\n",
    "        out1 = self.linear_gru(out1)\n",
    "        \n",
    "        output_before, _ = self.lstm(embedded_before)\n",
    "        output_text, _ = self.lstm2(embedded_text)\n",
    "        output_after, _ = self.lstm3(embedded_after)\n",
    "\n",
    "        t_text = output_text[:, -1, :]  #we keep the last word because it collected information from the other words\n",
    "        t_before = output_before[:, -1, :]\n",
    "        t_after = output_after[:, -1, :]\n",
    "\n",
    "        out2 = torch.cat([t_text, t_before, t_after], dim=1)\n",
    "\n",
    "        out2 = self.linear_lstm(out2)\n",
    "\n",
    "        x = torch.cat([out1, out2, description_before, description_after, speaker], dim=1)\n",
    "\n",
    "        out = self.linear2(x)\n",
    "\n",
    "        out = torch.sigmoid(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now create our model, we can see that we have a lot of hyperparameters but some model just take a few parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GRU_LSTM_final(\n",
       "  (embedding): Embedding(1502, 50)\n",
       "  (gru): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (gru2): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (gru3): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (lstm): LSTM(50, 16, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (lstm2): LSTM(50, 16, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (lstm3): LSTM(50, 16, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (linear_gru): Linear(in_features=96, out_features=64, bias=True)\n",
       "  (linear_lstm): Linear(in_features=96, out_features=64, bias=True)\n",
       "  (linear2): Linear(in_features=165, out_features=1, bias=True)\n",
       "  (dropout): Dropout(p=0.3, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#our differents hyperparameters\n",
    "input_dim = num_words + 2 #add 2 for <unk> and <pad> symbols\n",
    "embedding_dim = 50\n",
    "num_filter = 25\n",
    "hidden_dim = 32\n",
    "hidden_dim_2 = 16\n",
    "num_layers = 2\n",
    "\n",
    "model = GRU_LSTM_final(input_dim, embedding_dim, hidden_dim, hidden_dim_2, num_layers)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is our train fonction, at each epochs we print the f1 score on the validation set, for differents thresholds. We keep the best model in terms of f1 score on the validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_f1 = 0\n",
    "def train_model(model, lr, best_f1):\n",
    "    loss_criterion = nn.BCELoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "    num_epochs = 5\n",
    "    history_val_f1 = []\n",
    "    for epoch in range(num_epochs):\n",
    "        loss_tot = 0\n",
    "        iter = 0\n",
    "        for text, text_before, text_after, description_before, description_after, speaker, y in train_loader:\n",
    "            model.train()\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(text, text_before, text_after, description_before, description_after, speaker)\n",
    "            loss = loss_criterion(outputs, y)\n",
    "            loss_tot += loss\n",
    "            iter += 1\n",
    "            loss.backward()\n",
    "            optimizer.step()              \n",
    "        \n",
    "        model.eval()\n",
    "\n",
    "        true_labels = []\n",
    "        predicted_labels = []\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for text, text_before, text_after, description_before, description_after, speaker, y in val_loader:\n",
    "                outputs = model(text, text_before, text_after, description_before, description_after, speaker)\n",
    "                predicted_labels.extend(outputs.cpu().numpy())\n",
    "                true_labels.extend(y.cpu().numpy())\n",
    "\n",
    "        predicted_labels1 = [1 if pred > 0.3 else 0 for pred in predicted_labels]\n",
    "        predicted_labels2 = [1 if pred > 0.20 else 0 for pred in predicted_labels]\n",
    "        predicted_labels3 = [1 if pred > 0.4 else 0 for pred in predicted_labels]\n",
    "        loss_tot = loss_tot / iter\n",
    "\n",
    "        f11 = f1_score(true_labels, predicted_labels1)\n",
    "        f12 = f1_score(true_labels, predicted_labels2)\n",
    "        f13 = f1_score(true_labels, predicted_labels3)\n",
    "\n",
    "        print(f'{epoch} val Set Evaluation - F1 Score: {f11}' + f'     loss : {loss_tot}')\n",
    "        print(f'{epoch} val Set Evaluation - F1 Score: {f12}' + f'     loss : {loss_tot}')\n",
    "        print(f'{epoch} val Set Evaluation - F1 Score: {f13}' + f'     loss : {loss_tot}')\n",
    "        print(\"\")\n",
    "\n",
    "        history_val_f1.append(f11)\n",
    "\n",
    "        if f11 > best_f1:\n",
    "            best_f1 = f11\n",
    "            torch.save(model.state_dict(), 'best_model.pth')\n",
    "\n",
    "    print(best_f1)\n",
    "    return (history_val_f1, best_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 30\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m history_val_f1, best_f1 \u001b[39m=\u001b[39m train_model(model, \u001b[39m0.003\u001b[39;49m, best_f1)\n",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 30\u001b[0m line \u001b[0;36m1\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m     loss_tot \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m loss\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m     \u001b[39miter\u001b[39m \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m     loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m     optimizer\u001b[39m.\u001b[39mstep()              \n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X41sZmlsZQ%3D%3D?line=19'>20</a>\u001b[0m model\u001b[39m.\u001b[39meval()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\_tensor.py:492\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    482\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[0;32m    483\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    484\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[0;32m    485\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    490\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[0;32m    491\u001b[0m     )\n\u001b[1;32m--> 492\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[0;32m    493\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[0;32m    494\u001b[0m )\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\autograd\\__init__.py:251\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    246\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[0;32m    248\u001b[0m \u001b[39m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[0;32m    249\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    250\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 251\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    252\u001b[0m     tensors,\n\u001b[0;32m    253\u001b[0m     grad_tensors_,\n\u001b[0;32m    254\u001b[0m     retain_graph,\n\u001b[0;32m    255\u001b[0m     create_graph,\n\u001b[0;32m    256\u001b[0m     inputs,\n\u001b[0;32m    257\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    258\u001b[0m     accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    259\u001b[0m )\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history_val_f1, best_f1 = train_model(model, 0.003, best_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_losses(history_val_f1):\n",
    "    plt.style.use('bmh')\n",
    "    plt.rc('axes', facecolor='none')\n",
    "    plt.rc('figure', figsize=(16, 4))\n",
    "\n",
    "    plt.plot(history_val_f1, label='Validation')\n",
    "    plt.title('F1 Graph')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have already trained some good model that you can test on the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GRU_LSTM_final(\n",
       "  (embedding): Embedding(1502, 50)\n",
       "  (gru): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (gru2): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (gru3): GRU(50, 32, num_layers=2, batch_first=True, dropout=0.5)\n",
       "  (lstm): LSTM(50, 32, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (lstm2): LSTM(50, 32, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (lstm3): LSTM(50, 32, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  (linear_gru): Linear(in_features=96, out_features=64, bias=True)\n",
       "  (linear_lstm): Linear(in_features=192, out_features=64, bias=True)\n",
       "  (linear2): Linear(in_features=165, out_features=1, bias=True)\n",
       "  (dropout): Dropout(p=0.3, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_dim = 70\n",
    "hidden_dim = 32\n",
    "hidden_dim_2 = 16\n",
    "\n",
    "model1 = GRU_LSTM(input_dim, embedding_dim, hidden_dim, hidden_dim_2)\n",
    "model1.load_state_dict(torch.load('best_modelGL.pth'))\n",
    "model1.eval()\n",
    "\n",
    "\n",
    "embedding_dim = 30\n",
    "hidden_dim = 32\n",
    "\n",
    "model2 = GRU(input_dim, embedding_dim, hidden_dim)\n",
    "model2.load_state_dict(torch.load('best_modelG1.pth'))\n",
    "model2.eval()\n",
    "\n",
    "\n",
    "embedding_dim = 50\n",
    "hidden_dim = 16\n",
    "hidden_dim_2 = 32\n",
    "num_layers = 2\n",
    "\n",
    "model3 = GRU_LSTM_final(input_dim, embedding_dim, hidden_dim, hidden_dim_2, num_layers)\n",
    "model3.load_state_dict(torch.load('best_modelGL2.pth'))\n",
    "model3.eval()\n",
    "\n",
    "\n",
    "embedding_dim = 50\n",
    "hidden_dim = 32\n",
    "hidden_dim_2 = 32\n",
    "num_layers = 2\n",
    "\n",
    "model4 = GRU_LSTM_final(input_dim, embedding_dim, hidden_dim, hidden_dim_2, num_layers)\n",
    "model4.load_state_dict(torch.load('best_modelGL2_2.pth'))\n",
    "model4.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have implemented a fonction that can make the sum of result of our different model (the result are regression so it can be sum) with different weights. It improved our results a lot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_fusion_model(models, alphas):\n",
    "    true_labels = []\n",
    "    predicted_labels = []\n",
    "    s = sum(alphas)\n",
    "    with torch.no_grad():\n",
    "        for text, text_before, text_after, description_before, description_after, speaker, y in val_loader:\n",
    "            outputs = np.array([model(text, text_before, text_after, description_before, description_after, speaker) for model in models])\n",
    "            outputs_final = np.zeros(len(text), dtype=np.float64)\n",
    "            for i in range(len(models)):\n",
    "                weighted_output = (alphas[i]/s) * outputs[i] \n",
    "                outputs_final += weighted_output.reshape(len(text))\n",
    "            predicted_labels.extend(outputs_final.tolist())\n",
    "            true_labels.extend(y.cpu().numpy())\n",
    "    \n",
    "    predicted_labels2 = [1 if pred > 0.3 else 0 for pred in predicted_labels]\n",
    "    f1 = f1_score(true_labels, predicted_labels2)\n",
    "    print(f1)\n",
    "\n",
    "    return alphas, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 36\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m alphas, f1 \u001b[39m=\u001b[39m train_fusion_model([model1, model2, model3, model4], [\u001b[39m2.\u001b[39;49m,\u001b[39m1.\u001b[39;49m,\u001b[39m1.\u001b[39;49m,\u001b[39m1.\u001b[39;49m])\n",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 36\u001b[0m line \u001b[0;36m7\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     \u001b[39mfor\u001b[39;00m text, text_before, text_after, description_before, description_after, speaker, y \u001b[39min\u001b[39;00m val_loader:\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m         outputs \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray([model(text, text_before, text_after, description_before, description_after, speaker) \u001b[39mfor\u001b[39;49;00m model \u001b[39min\u001b[39;49;00m models])\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m         outputs_final \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mzeros(\u001b[39mlen\u001b[39m(text), dtype\u001b[39m=\u001b[39mnp\u001b[39m.\u001b[39mfloat64)\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m         \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mlen\u001b[39m(models)):\n",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 36\u001b[0m line \u001b[0;36m7\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     \u001b[39mfor\u001b[39;00m text, text_before, text_after, description_before, description_after, speaker, y \u001b[39min\u001b[39;00m val_loader:\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m         outputs \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray([model(text, text_before, text_after, description_before, description_after, speaker) \u001b[39mfor\u001b[39;00m model \u001b[39min\u001b[39;00m models])\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m         outputs_final \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mzeros(\u001b[39mlen\u001b[39m(text), dtype\u001b[39m=\u001b[39mnp\u001b[39m.\u001b[39mfloat64)\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m         \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mlen\u001b[39m(models)):\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "\u001b[1;32mc:\\Users\\augus\\Downloads\\INF_554_MARTINEZ_GADONNEIX_CRABEIL\\jupyter_best_f1.ipynb Cell 36\u001b[0m line \u001b[0;36m2\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m truncated \u001b[39m=\u001b[39m output[:, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m , :]\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m truncated \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(truncated)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=19'>20</a>\u001b[0m lstm_out, _ \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlstm(embedded)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=20'>21</a>\u001b[0m lstm_out \u001b[39m=\u001b[39m lstm_out[:, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, :]\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/augus/Downloads/INF_554_MARTINEZ_GADONNEIX_CRABEIL/jupyter_best_f1.ipynb#X50sZmlsZQ%3D%3D?line=21'>22</a>\u001b[0m lstm_out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout2(lstm_out)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compiled_call_impl(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_impl(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\torch\\nn\\modules\\rnn.py:879\u001b[0m, in \u001b[0;36mLSTM.forward\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    876\u001b[0m         hx \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpermute_hidden(hx, sorted_indices)\n\u001b[0;32m    878\u001b[0m \u001b[39mif\u001b[39;00m batch_sizes \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m--> 879\u001b[0m     result \u001b[39m=\u001b[39m _VF\u001b[39m.\u001b[39;49mlstm(\u001b[39minput\u001b[39;49m, hx, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_flat_weights, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnum_layers,\n\u001b[0;32m    880\u001b[0m                       \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdropout, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbidirectional, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbatch_first)\n\u001b[0;32m    881\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    882\u001b[0m     result \u001b[39m=\u001b[39m _VF\u001b[39m.\u001b[39mlstm(\u001b[39minput\u001b[39m, batch_sizes, hx, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_flat_weights, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbias,\n\u001b[0;32m    883\u001b[0m                       \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_layers, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbidirectional)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "alphas, f1 = train_fusion_model([model1, model2, model3, model4], [2.,1.,1.,1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we just have to transform our testing data as the training data and apply the model that give the best result for the validation set and make the submission. We keep the same threshold for the testing set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_labels = {}\n",
    "model1.eval()\n",
    "model2.eval()\n",
    "model3.eval()\n",
    "model4.eval()\n",
    "\n",
    "for transcription_id in test_set:\n",
    "    text = X_test_text[transcription_id]\n",
    "    text = np.array(vectorize_sentences(text, max_len))\n",
    "\n",
    "    before = X_test_text_before[transcription_id]\n",
    "    before = np.array(vectorize_sentences(before, max_len))\n",
    "\n",
    "    after = X_test_text_after[transcription_id]\n",
    "    after = np.array(vectorize_sentences(after, max_len))\n",
    "\n",
    "    speaker = X_test_speaker[transcription_id]\n",
    "    speaker = np.array(speaker)\n",
    "\n",
    "    description_before = X_test_description_before[transcription_id]\n",
    "    description_before = np.array(description_before)\n",
    "\n",
    "    description_after = X_test_description_after[transcription_id]\n",
    "    description_after = np.array(description_after)\n",
    "\n",
    "    text = torch.tensor(text).to(device)\n",
    "    before = torch.tensor(before).to(device)\n",
    "    after = torch.tensor(after).to(device)\n",
    "    description_before = torch.tensor(description_before, dtype=torch.float32).to(device)\n",
    "    description_after = torch.tensor(description_after, dtype=torch.float32).to(device)\n",
    "    speaker = torch.tensor(speaker, dtype=torch.float32).to(device)\n",
    "\n",
    "    testset = TensorDataset(text, before, after, description_before, description_after, speaker)\n",
    "    \n",
    "    test_loader = DataLoader(testset, shuffle=False, batch_size=len(text))\n",
    "\n",
    "    predicted_labels = []\n",
    "    with torch.no_grad():\n",
    "        for text, text_before, text_after, description_before, description_after, speaker in test_loader:\n",
    "            outputs1 = model2(text, text_before, text_after, description_before, description_after, speaker)\n",
    "            outputs2 = model1(text, text_before, text_after, description_before, description_after, speaker)\n",
    "            outputs3 = model3(text, text_before, text_after, description_before, description_after, speaker)\n",
    "            outputs4 = model4(text, text_before, text_after, description_before, description_after, speaker)\n",
    "            outputs = (2 * outputs1.cpu().numpy() + outputs3.cpu().numpy() + outputs2.cpu().numpy() + 2 * outputs4.cpu().numpy()) / 6\n",
    "            predicted_labels.extend(outputs)\n",
    "\n",
    "    # Convert predicted probabilities to binary predictions\n",
    "    predicted_labels = [1 if pred > 0.3 else 0 for pred in predicted_labels]\n",
    "\n",
    "    test_labels[transcription_id] = predicted_labels\n",
    "\n",
    "with open(\"test_labels.json\", \"w\") as file:\n",
    "    json.dump(test_labels, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_submission(json_path: Path = Path(\"test_labels.json\")):\n",
    "    with open(json_path, \"r\") as file:\n",
    "        test_labels = json.load(file)\n",
    "\n",
    "    file = open(\"submission.csv\", \"w\")\n",
    "    file.write(\"id,target_feature\\n\")\n",
    "    for key, value in test_labels.items():\n",
    "        u_id = [key + \"_\" + str(i) for i in range(len(value))]\n",
    "        target = map(str, value) \n",
    "        for row in zip(u_id, target):\n",
    "            file.write(\",\".join(row))\n",
    "            file.write(\"\\n\")\n",
    "    file.close()\n",
    "\n",
    "make_submission(Path(\"test_labels.json\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}