Semantic_Similarity_TFIDF_BERT(pooled).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "25caedfd-b8de-471c-9afa-6aaff7207949",
   "metadata": {},
   "source": [
    "## Semantic Similarity using TF-IDF + Bert(Pooled) technique\n",
    "#### I used SNLI (Stanford Natural Language Inference) Corpus to predict sentence semantic similarity with Transformers.\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd615539-7c54-406b-a853-92a9212a1d44",
   "metadata": {},
   "source": [
    "### Installing and importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9aa20c44-1f0c-4d63-b62f-c2641bf95f60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandas\n",
      "  Using cached pandas-2.0.3-cp38-cp38-win_amd64.whl (10.8 MB)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from pandas) (2023.3.post1)\n",
      "Collecting numpy>=1.20.3\n",
      "  Using cached numpy-1.24.4-cp38-cp38-win_amd64.whl (14.9 MB)\n",
      "Collecting tzdata>=2022.1\n",
      "  Using cached tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Installing collected packages: tzdata, numpy, pandas\n",
      "Successfully installed numpy-1.24.4 pandas-2.0.3 tzdata-2023.3\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.1.1; however, version 23.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\real\\desktop\\myenv\\scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b757b4ae-944d-4728-b7da-b135f1d46b85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting scikit-learn\n",
      "  Downloading scikit_learn-1.3.2-cp38-cp38-win_amd64.whl (9.3 MB)\n",
      "Collecting scipy>=1.5.0\n",
      "  Using cached scipy-1.10.1-cp38-cp38-win_amd64.whl (42.2 MB)\n",
      "Collecting threadpoolctl>=2.0.0\n",
      "  Using cached threadpoolctl-3.2.0-py3-none-any.whl (15 kB)\n",
      "Collecting joblib>=1.1.1\n",
      "  Using cached joblib-1.3.2-py3-none-any.whl (302 kB)\n",
      "Requirement already satisfied: numpy<2.0,>=1.17.3 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from scikit-learn) (1.24.4)\n",
      "Installing collected packages: threadpoolctl, scipy, joblib, scikit-learn\n",
      "Successfully installed joblib-1.3.2 scikit-learn-1.3.2 scipy-1.10.1 threadpoolctl-3.2.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.1.1; however, version 23.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\real\\desktop\\myenv\\scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b79603ec-3aeb-4a4f-95d8-194a4b684c86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "  Downloading transformers-4.34.1-py3-none-any.whl (7.7 MB)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from transformers) (23.2)\n",
      "Collecting regex!=2019.12.17\n",
      "  Using cached regex-2023.10.3-cp38-cp38-win_amd64.whl (269 kB)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from transformers) (6.0.1)\n",
      "Collecting huggingface-hub<1.0,>=0.16.4\n",
      "  Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from transformers) (1.24.4)\n",
      "Collecting filelock\n",
      "  Using cached filelock-3.12.4-py3-none-any.whl (11 kB)\n",
      "Collecting tqdm>=4.27\n",
      "  Using cached tqdm-4.66.1-py3-none-any.whl (78 kB)\n",
      "Collecting safetensors>=0.3.1\n",
      "  Downloading safetensors-0.4.0-cp38-none-win_amd64.whl (277 kB)\n",
      "Collecting tokenizers<0.15,>=0.14\n",
      "  Downloading tokenizers-0.14.1-cp38-none-win_amd64.whl (2.2 MB)\n",
      "Requirement already satisfied: requests in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.8.0)\n",
      "Collecting fsspec>=2023.5.0\n",
      "  Downloading fsspec-2023.10.0-py3-none-any.whl (166 kB)\n",
      "Collecting huggingface-hub<1.0,>=0.16.4\n",
      "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
      "Requirement already satisfied: colorama in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from requests->transformers) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from requests->transformers) (2023.7.22)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from requests->transformers) (3.3.1)\n",
      "Installing collected packages: tqdm, fsspec, filelock, huggingface-hub, tokenizers, safetensors, regex, transformers\n",
      "Successfully installed filelock-3.12.4 fsspec-2023.10.0 huggingface-hub-0.17.3 regex-2023.10.3 safetensors-0.4.0 tokenizers-0.14.1 tqdm-4.66.1 transformers-4.34.1\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.1.1; however, version 23.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\real\\desktop\\myenv\\scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f2f21b9a-602d-44e5-b564-0ee2b86afb68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting torch\n",
      "  Using cached torch-2.1.0-cp38-cp38-win_amd64.whl (192.3 MB)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from torch) (3.1.2)\n",
      "Requirement already satisfied: filelock in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from torch) (3.12.4)\n",
      "Collecting networkx\n",
      "  Using cached networkx-3.1-py3-none-any.whl (2.1 MB)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from torch) (4.8.0)\n",
      "Collecting sympy\n",
      "  Using cached sympy-1.12-py3-none-any.whl (5.7 MB)\n",
      "Requirement already satisfied: fsspec in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from torch) (2023.10.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\real\\desktop\\myenv\\lib\\site-packages (from jinja2->torch) (2.1.3)\n",
      "Collecting mpmath>=0.19\n",
      "  Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "Installing collected packages: mpmath, sympy, networkx, torch\n",
      "Successfully installed mpmath-1.3.0 networkx-3.1 sympy-1.12 torch-2.1.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.1.1; however, version 23.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\real\\desktop\\myenv\\scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "pip install torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a365126b-a115-44ab-96d4-ba8e3f430ca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from transformers import BertTokenizer, BertModel\n",
    "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b58e6f7-53c8-4bf0-a475-d41f1feac9fa",
   "metadata": {},
   "source": [
    "### Upload the SNLI data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "72be03cb-29ad-4528-9c8e-148ce591268c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
      "  0 11.1M    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0\n",
      " 48 11.1M   48 5505k    0     0  5505k      0  0:00:02  0:00:01  0:00:01 2985k\n",
      " 98 11.1M   98 11.0M    0     0  5656k      0  0:00:02  0:00:02 --:--:-- 3977k\n",
      "100 11.1M  100 11.1M    0     0  5728k      0  0:00:02  0:00:02 --:--:-- 3984k\n",
      "x SNLI_Corpus/\n",
      "x SNLI_Corpus/snli_1.0_dev.csv\n",
      "x SNLI_Corpus/snli_1.0_train.csv\n",
      "x SNLI_Corpus/snli_1.0_test.csv\n"
     ]
    }
   ],
   "source": [
    "!curl -LO https://raw.githubusercontent.com/MohamadMerchant/SNLI/master/data.tar.gz\n",
    "!tar -xvzf data.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65cb8183-beff-4e60-afd7-d6503f171cb4",
   "metadata": {},
   "source": [
    "#### Reading the SNLI dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7c7a2323-f7c3-468c-82bb-2c503ce7398a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>similarity</th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>neutral</td>\n",
       "      <td>A person on a horse jumps over a broken down a...</td>\n",
       "      <td>A person is training his horse for a competition.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>contradiction</td>\n",
       "      <td>A person on a horse jumps over a broken down a...</td>\n",
       "      <td>A person is at a diner, ordering an omelette.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entailment</td>\n",
       "      <td>A person on a horse jumps over a broken down a...</td>\n",
       "      <td>A person is outdoors, on a horse.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>neutral</td>\n",
       "      <td>Children smiling and waving at camera</td>\n",
       "      <td>They are smiling at their parents</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>entailment</td>\n",
       "      <td>Children smiling and waving at camera</td>\n",
       "      <td>There are children present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>entailment</td>\n",
       "      <td>A man on a street in a bright t-shirt holds so...</td>\n",
       "      <td>A man is showing a woman something</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>neutral</td>\n",
       "      <td>A child with a yellow cup and milk all over hi...</td>\n",
       "      <td>The child spilled his milk.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>contradiction</td>\n",
       "      <td>A child with a yellow cup and milk all over hi...</td>\n",
       "      <td>The child has a clean face.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>entailment</td>\n",
       "      <td>A child with a yellow cup and milk all over hi...</td>\n",
       "      <td>The child had milk all over his face.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>entailment</td>\n",
       "      <td>Two barefoot men are playing on a green lawn o...</td>\n",
       "      <td>Two men are barefoot on the lawn.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        similarity                                          sentence1  \\\n",
       "0          neutral  A person on a horse jumps over a broken down a...   \n",
       "1    contradiction  A person on a horse jumps over a broken down a...   \n",
       "2       entailment  A person on a horse jumps over a broken down a...   \n",
       "3          neutral              Children smiling and waving at camera   \n",
       "4       entailment              Children smiling and waving at camera   \n",
       "..             ...                                                ...   \n",
       "995     entailment  A man on a street in a bright t-shirt holds so...   \n",
       "996        neutral  A child with a yellow cup and milk all over hi...   \n",
       "997  contradiction  A child with a yellow cup and milk all over hi...   \n",
       "998     entailment  A child with a yellow cup and milk all over hi...   \n",
       "999     entailment  Two barefoot men are playing on a green lawn o...   \n",
       "\n",
       "                                             sentence2  \n",
       "0    A person is training his horse for a competition.  \n",
       "1        A person is at a diner, ordering an omelette.  \n",
       "2                    A person is outdoors, on a horse.  \n",
       "3                    They are smiling at their parents  \n",
       "4                           There are children present  \n",
       "..                                                 ...  \n",
       "995                 A man is showing a woman something  \n",
       "996                        The child spilled his milk.  \n",
       "997                        The child has a clean face.  \n",
       "998              The child had milk all over his face.  \n",
       "999                  Two men are barefoot on the lawn.  \n",
       "\n",
       "[1000 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snli_df = pd.read_csv('snli_1.0_train.csv', nrows=1000)\n",
    "snli_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d99d0849-9bd6-4d99-8579-edd30d8e0fbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total train samples : 1000\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total train samples : {snli_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17eca3f6-0869-494d-bd9d-8336abd87a6a",
   "metadata": {},
   "source": [
    "### Dataset overview:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b158d43b-38c2-4b5f-8aeb-749604ff396f",
   "metadata": {},
   "source": [
    "##### * sentence1: The premise caption that was supplied to the author of the pair.\n",
    "##### * sentence2: The hypothesis caption that was written by the author of the pair.\n",
    "##### * similarity: This is the label chosen by the majority of annotators.\n",
    "##### Note: Where no majority exists, the label \"-\" is used (we will skip such samples here).\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcc56469-b5c5-4315-adc2-5fad00a0c5eb",
   "metadata": {},
   "source": [
    "##### Here are the \"similarity\" label values in our dataset:\r",
    "##### * Contradiction: The sentences share no similarity.\r",
    "##### * Entailment: The sentences have similar meaning.\r",
    "##### * Neutral: The sentences are neutral.\n",
    "\n",
    "\n",
    "\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c06b58f7-135d-4eaa-b841-4f7ead7deb50",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence1: A person on a horse jumps over a broken down airplane.\n",
      "Sentence2: A person is at a diner, ordering an omelette.\n",
      "Similarity: contradiction\n"
     ]
    }
   ],
   "source": [
    "print(f\"Sentence1: {snli_df.loc[1, 'sentence1']}\")\n",
    "print(f\"Sentence2: {snli_df.loc[1, 'sentence2']}\")\n",
    "print(f\"Similarity: {snli_df.loc[1, 'similarity']}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96b28362-fb34-45db-beaf-e514383772e6",
   "metadata": {},
   "source": [
    "#### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "14942be5-7fae-4e50-a591-0e8b7e1eab76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values\n",
      "similarity    0\n",
      "sentence1     0\n",
      "sentence2     0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# We have some NaN entries in our train data, we will simply drop them.\n",
    "print(\"Number of missing values\")\n",
    "print(snli_df.isnull().sum())\n",
    "snli_df.dropna(axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22d3391d-1db5-4d37-81d1-92571eccb71b",
   "metadata": {},
   "source": [
    "Distribution of our training targets.\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "133cba93-0e9d-4885-9bcc-f0bd56cc90a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Target Distribution\n",
      "similarity\n",
      "entailment       334\n",
      "neutral          332\n",
      "contradiction    332\n",
      "-                  2\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Train Target Distribution\")\n",
    "print(snli_df.similarity.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65311d11-965c-4cbc-a32e-7e5db079e22d",
   "metadata": {},
   "source": [
    "The value \"-\" appears as part of our training and validation targets. We will skip these samples.\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd5953ba-7499-4be8-8211-0bc09da26243",
   "metadata": {},
   "outputs": [],
   "source": [
    "snli_df = (\n",
    "    snli_df[snli_df.similarity != \"-\"]\n",
    "    .sample(frac=1.0, random_state=42)\n",
    "    .reset_index(drop=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "11f51877-e31b-4e79-b57f-ae7bf3dc1e06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>similarity</th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>neutral</td>\n",
       "      <td>A woman in capri jeans crouches on the edge of...</td>\n",
       "      <td>A woman is very eager to touch the water</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>entailment</td>\n",
       "      <td>Woman at Walmart check-out having her grocerie...</td>\n",
       "      <td>A woman is in Walmart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>neutral</td>\n",
       "      <td>People on bicycles waiting at an intersection.</td>\n",
       "      <td>People are on mountain bikes.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contradiction</td>\n",
       "      <td>A man and a woman are standing next to sculptu...</td>\n",
       "      <td>Three people are looking at painting at a scho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>entailment</td>\n",
       "      <td>People in orange vests and blue pants with a y...</td>\n",
       "      <td>the runners waited to start the race</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>entailment</td>\n",
       "      <td>A skier slides along a metal rail.</td>\n",
       "      <td>A skier is near the rail.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>entailment</td>\n",
       "      <td>An Asian woman in a blue top and green headsca...</td>\n",
       "      <td>An Asian woman is smiling at while another lad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>contradiction</td>\n",
       "      <td>Two people wearing blue clothing are making ha...</td>\n",
       "      <td>A man is sitting with his hands in his pockets.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>neutral</td>\n",
       "      <td>A man in a gold foils skirt, sitting at a comp...</td>\n",
       "      <td>he is covering up his face</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>contradiction</td>\n",
       "      <td>A woman is walking across the street eating a ...</td>\n",
       "      <td>the woman is having coffee at the cafe</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>998 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        similarity                                          sentence1  \\\n",
       "0          neutral  A woman in capri jeans crouches on the edge of...   \n",
       "1       entailment  Woman at Walmart check-out having her grocerie...   \n",
       "2          neutral     People on bicycles waiting at an intersection.   \n",
       "3    contradiction  A man and a woman are standing next to sculptu...   \n",
       "4       entailment  People in orange vests and blue pants with a y...   \n",
       "..             ...                                                ...   \n",
       "993     entailment                 A skier slides along a metal rail.   \n",
       "994     entailment  An Asian woman in a blue top and green headsca...   \n",
       "995  contradiction  Two people wearing blue clothing are making ha...   \n",
       "996        neutral  A man in a gold foils skirt, sitting at a comp...   \n",
       "997  contradiction  A woman is walking across the street eating a ...   \n",
       "\n",
       "                                             sentence2  \n",
       "0             A woman is very eager to touch the water  \n",
       "1                                A woman is in Walmart  \n",
       "2                        People are on mountain bikes.  \n",
       "3    Three people are looking at painting at a scho...  \n",
       "4                 the runners waited to start the race  \n",
       "..                                                 ...  \n",
       "993                          A skier is near the rail.  \n",
       "994  An Asian woman is smiling at while another lad...  \n",
       "995    A man is sitting with his hands in his pockets.  \n",
       "996                         he is covering up his face  \n",
       "997             the woman is having coffee at the cafe  \n",
       "\n",
       "[998 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snli_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49c21765-f75c-4510-b191-2f0be8d67a5f",
   "metadata": {},
   "source": [
    "#### Sample search query\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4178cec9-313d-40a3-a962-fa7025bb93f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_query = \"A person is walking on a beach.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f950dc8a-e150-4e22-bb62-19ac1b649215",
   "metadata": {},
   "source": [
    "#### Define the chunk size \r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2345c13c-9889-4a5d-8898-fcacc75c977f",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3391eeca-24fb-41ae-ad58-dc84cbc7a08a",
   "metadata": {},
   "source": [
    "#### Initialize TF-IDF vectorizer\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7ced745b-0fe9-4196-983b-aa42fcf8ce8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8c6652e-190d-432b-b9c7-df84d4f70307",
   "metadata": {},
   "source": [
    "#### Function to get BERT (pooled) embeddings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d2e21606-f411-450c-a4f0-a801f736588d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bert_pooled_embedding(text):\n",
    "    inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True)\n",
    "    outputs = model(**inputs)\n",
    "    pooled_embedding = outputs.pooler_output\n",
    "    return pooled_embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cf026ca-f77d-4e01-a919-5d2178483155",
   "metadata": {},
   "source": [
    "#### Initialize BERT tokenizer and model\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "40503172-15c2-473c-aa32-9747c50bbdd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "model = BertModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79ea05ba-2e16-4550-92d0-308d7d120ad4",
   "metadata": {},
   "source": [
    "#### Calculate the number of chunks needed\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "90340425-33b3-486c-98b6-00ea447514b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_chunks = len(snli_df) // chunk_size + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c2616c1-a8a2-4fd7-84b7-49b5181d17c8",
   "metadata": {},
   "source": [
    "#### Initialize an empty list to store similarity scores\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8591eeda-e9a1-4bbc-afcb-32a3a8609506",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity_scores = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fbe0a820-f0af-4dda-b1ee-2046f0f99af0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scipy.sparse._csr.csr_matrix'>\n",
      "<class 'torch.Tensor'>\n"
     ]
    }
   ],
   "source": [
    "print(type(query_tfidf))\n",
    "print(type(query_bert_embedding))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9ac2cfc1-b415-494c-a073-a0925a206c86",
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_average_similarity(cosine_sim, euclidean_dist, weight=0.5):\n",
    "    return weight * cosine_sim + (1 - weight) * (1 / (1 + euclidean_dist))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31336936-fa0a-48c0-aa56-8da7c52fdc82",
   "metadata": {},
   "source": [
    "#### Process data in chunks\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3e906a21-9452-48b5-892d-81c30655a8ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(num_chunks):\n",
    "    chunk_start = i * chunk_size\n",
    "    chunk_end = (i + 1) * chunk_size\n",
    "    \n",
    "    # Extract sentence pairs and labels from the dataset chunk\n",
    "    chunk_data = snli_df[chunk_start:chunk_end]\n",
    "    chunk_sentence_pairs = chunk_data[['sentence1', 'sentence2']].values.tolist()\n",
    "    chunk_labels = chunk_data['similarity'].values\n",
    "\n",
    "    # Compute TF-IDF vectors for the chunk\n",
    "    tfidf_vectors = tfidf_vectorizer.fit_transform([' '.join(pair) for pair in chunk_sentence_pairs])\n",
    "\n",
    "    # Compute BERT (pooled) embeddings for the chunk\n",
    "    bert_embeddings = [get_bert_pooled_embedding(' '.join(pair)) for pair in chunk_sentence_pairs]\n",
    "\n",
    "    # Preprocess the search query\n",
    "    query_tfidf = tfidf_vectorizer.transform([search_query])\n",
    "    query_bert_embedding = get_bert_pooled_embedding(search_query)\n",
    "\n",
    "    # Calculate similarity scores for the chunk\n",
    "    for j, pair in enumerate(chunk_sentence_pairs):\n",
    "        cosine_sim = cosine_similarity(query_tfidf.toarray(), tfidf_vectors[j])\n",
    "        euclidean_dist = euclidean_distances(query_bert_embedding.detach().numpy(), bert_embeddings[j].detach().numpy())\n",
    "        weighted_sim = weighted_average_similarity(cosine_sim[0][0], euclidean_dist[0][0])\n",
    "        similarity_scores.append((j + chunk_start, cosine_sim[0][0], euclidean_dist[0][0], weighted_sim, chunk_labels[j]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d6c4dd26-f192-4cf1-8c09-4daab708b91f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.04006722746106942, 8.892938, 0.07057471862505582, 'neutral'),\n",
       " (1, 0.021418652141788494, 5.7490416, 0.08479391952821155, 'entailment'),\n",
       " (2, 0.09605325854230122, 2.427765, 0.19389427646090143, 'neutral'),\n",
       " (3, 0.0, 5.0432715, 0.08273664298526574, 'contradiction'),\n",
       " (4, 0.0, 10.756921, 0.04252814218011401, 'entailment'),\n",
       " (5, 0.0, 3.4979722, 0.11116120158404627, 'neutral'),\n",
       " (6, 0.0, 4.8742256, 0.08511760232691493, 'entailment'),\n",
       " (7, 0.021865280925434716, 4.093399, 0.10909891318209207, 'neutral'),\n",
       " (8, 0.01792310684760326, 2.9625814, 0.13514192677631798, 'contradiction'),\n",
       " (9, 0.062309996940060494, 2.936513, 0.15817096583032222, 'entailment'),\n",
       " (10, 0.04324958708445728, 3.5824518, 0.1307366881347708, 'neutral'),\n",
       " (11, 0.03203408693466361, 4.5985446, 0.10532596854261714, 'entailment'),\n",
       " (12, 0.0, 3.9483624, 0.10104353007882076, 'entailment'),\n",
       " (13, 0.09232822939146387, 3.2084446, 0.16497285475946366, 'entailment'),\n",
       " (14, 0.02343009991348863, 4.5757327, 0.10138936294607423, 'entailment'),\n",
       " (15, 0.07263859062496766, 3.4669003, 0.14825373776084053, 'entailment'),\n",
       " (16, 0.0410000563840086, 4.054836, 0.11941520964269099, 'neutral'),\n",
       " (17, 0.0, 3.1443696, 0.12064561030806065, 'neutral'),\n",
       " (18, 0.05557375897459146, 5.768526, 0.1016582060201221, 'contradiction'),\n",
       " (19, 0.1942927920488553, 2.758456, 0.23017974831009788, 'entailment'),\n",
       " (20, 0.04027039499889159, 3.1880531, 0.13952241259627393, 'neutral'),\n",
       " (21, 0.0, 5.09767, 0.08199853281357512, 'contradiction'),\n",
       " (22, 0.019917415133598418, 5.7906947, 0.08358887665674222, 'contradiction'),\n",
       " (23, 0.0, 4.48262, 0.09119727824773369, 'entailment'),\n",
       " (24, 0.025568550507805092, 3.5924227, 0.12165927001691206, 'entailment'),\n",
       " (25, 0.07351116659347832, 4.05168, 0.13573255556848823, 'entailment'),\n",
       " (26, 0.0, 2.8375003, 0.13029314827621363, 'entailment'),\n",
       " (27, 0.04922905087376392, 2.5115535, 0.167001647349921, 'neutral'),\n",
       " (28, 0.036942651279370964, 2.5512497, 0.15926683039258244, 'entailment'),\n",
       " (29, 0.0, 4.669286, 0.08819453100647857, 'contradiction'),\n",
       " (30, 0.03726592703826918, 7.17534, 0.07979249807777469, 'entailment'),\n",
       " (31, 0.0, 4.160483, 0.09689015762521094, 'entailment'),\n",
       " (32, 0.04258492731548251, 4.5974827, 0.11061833181498973, 'neutral'),\n",
       " (33, 0.03993515624124109, 2.416476, 0.1663172080754997, 'contradiction'),\n",
       " (34, 0.05265560951905415, 2.5447237, 0.1673825202696928, 'entailment'),\n",
       " (35, 0.0, 3.0935314, 0.12214392776965453, 'contradiction'),\n",
       " (36, 0.09947176598459508, 3.3422713, 0.16488299431605513, 'entailment'),\n",
       " (37, 0.043183236155645253, 2.6819458, 0.15738938019966264, 'entailment'),\n",
       " (38, 0.0, 5.0651393, 0.08243833748744744, 'contradiction'),\n",
       " (39, 0.07283601125553832, 9.0549135, 0.08614493753673426, 'contradiction'),\n",
       " (40, 0.033964932256749444, 2.6350818, 0.15453095386711424, 'entailment'),\n",
       " (41, 0.04940889717065583, 3.3715022, 0.139081608102195, 'neutral'),\n",
       " (42, 0.043786310857372245, 3.4307723, 0.13474029936525178, 'entailment'),\n",
       " (43, 0.0, 5.4678044, 0.07730598617558032, 'entailment'),\n",
       " (44, 0.06408133406331128, 2.592217, 0.171230477775109, 'neutral'),\n",
       " (45, 0.02950919703014937, 2.6615462, 0.15130893063307355, 'neutral'),\n",
       " (46, 0.031364499547257806, 4.661304, 0.10400112476452181, 'contradiction'),\n",
       " (47, 0.0, 5.687772, 0.0747633165669348, 'entailment'),\n",
       " (48, 0.1321340455131835, 3.232447, 0.18420199544640786, 'contradiction'),\n",
       " (49, 0.05418767950029699, 2.8564498, 0.1567467641823151, 'contradiction'),\n",
       " (50, 0.04358945810481527, 4.292665, 0.11626509503119789, 'neutral'),\n",
       " (51, 0.054297422738782186, 3.4964805, 0.13834679260184102, 'contradiction'),\n",
       " (52, 0.08650699749412731, 7.0305977, 0.1055153651114907, 'entailment'),\n",
       " (53, 0.0, 3.1103663, 0.12164365852082519, 'entailment'),\n",
       " (54, 0.016895835210008116, 2.812998, 0.13957832800718756, 'neutral'),\n",
       " (55, 0.0, 6.537179, 0.06633781690065106, 'contradiction'),\n",
       " (56, 0.0, 3.7158937, 0.10602444138724317, 'contradiction'),\n",
       " (57, 0.0, 6.0345664, 0.07107758622150535, 'contradiction'),\n",
       " (58, 0.02166263889562339, 4.944807, 0.09493833849400375, 'neutral'),\n",
       " (59, 0.061012618095916855, 4.0783863, 0.1289627812021482, 'neutral'),\n",
       " (60, 0.02392345575261047, 3.7141316, 0.11802580140954214, 'contradiction'),\n",
       " (61, 0.03107158345314023, 4.2486663, 0.11079808746235374, 'neutral'),\n",
       " (62, 0.02346443648580148, 2.358375, 0.16061374255426533, 'entailment'),\n",
       " (63, 0.027696619701332508, 4.976241, 0.09751293961543447, 'contradiction'),\n",
       " (64, 0.0, 5.6787906, 0.07486385368900689, 'entailment'),\n",
       " (65, 0.02962986611487789, 2.5653226, 0.1550546955779125, 'entailment'),\n",
       " (66, 0.05068410412151815, 10.594721, 0.06846512564572153, 'contradiction'),\n",
       " (67, 0.0, 6.003582, 0.07139203909481047, 'neutral'),\n",
       " (68, 0.05975156500381326, 3.443659, 0.142395665878162, 'contradiction'),\n",
       " (69, 0.0, 4.8341618, 0.08570211466594045, 'neutral'),\n",
       " (70, 0.24696016510029165, 2.4317195, 0.26917963461033895, 'entailment'),\n",
       " (71, 0.021813022877974507, 3.4864385, 0.12235348626617858, 'contradiction'),\n",
       " (72, 0.0685287759859251, 4.137991, 0.1315786893835328, 'contradiction'),\n",
       " (73, 0.035129373434974014, 2.9728105, 0.1434201734419558, 'contradiction'),\n",
       " (74, 0.0, 3.3275766, 0.11553810409597794, 'neutral'),\n",
       " (75, 0.0, 4.794254, 0.08629238811461794, 'contradiction'),\n",
       " (76, 0.3628474964235841, 4.093794, 0.27958241204016115, 'neutral'),\n",
       " (77, 0.14421430827055817, 2.1835942, 0.22916234533555357, 'neutral'),\n",
       " (78, 0.0, 2.4384491, 0.14541439439918497, 'entailment'),\n",
       " (79, 0.05530504437912765, 2.914942, 0.15536833424034902, 'contradiction'),\n",
       " (80, 0.05331537788472556, 3.7569673, 0.13176667284371044, 'contradiction'),\n",
       " (81, 0.0, 7.2056932, 0.06093330387517022, 'contradiction'),\n",
       " (82, 0.13542703588659036, 3.3813896, 0.18183256317839494, 'entailment'),\n",
       " (83, 0.0, 4.6044893, 0.08921419434913923, 'neutral'),\n",
       " (84, 0.02599891196008382, 7.906174, 0.06914028477277605, 'contradiction'),\n",
       " (85, 0.0, 4.169699, 0.09671742618394165, 'neutral'),\n",
       " (86, 0.0, 4.66327, 0.08828821516480367, 'neutral'),\n",
       " (87, 0.0, 4.5942993, 0.08937669790632467, 'neutral'),\n",
       " (88, 0.027991917634232182, 2.4586253, 0.15856206651236995, 'contradiction'),\n",
       " (89, 0.0, 2.952066, 0.12651610752450235, 'contradiction'),\n",
       " (90, 0.0, 2.8191786, 0.1309182038400389, 'neutral'),\n",
       " (91, 0.11445336388069582, 4.350567, 0.150674724495026, 'entailment'),\n",
       " (92, 0.20424054477134385, 6.0693684, 0.1728479490841261, 'neutral'),\n",
       " (93, 0.0235467459207871, 5.9721346, 0.08348742172624216, 'entailment'),\n",
       " (94, 0.06659743226959661, 4.493044, 0.12432292951168815, 'contradiction'),\n",
       " (95, 0.02481485110986557, 4.1297746, 0.10987759566314238, 'entailment'),\n",
       " (96, 0.09621124007181593, 3.2046437, 0.16702175955724083, 'entailment'),\n",
       " (97, 0.04503592130359416, 2.9738111, 0.1483417557887869, 'entailment'),\n",
       " (98, 0.04557899496439307, 3.5838835, 0.13186731269904567, 'neutral'),\n",
       " (99, 0.024195616971992396, 2.5064793, 0.15469098028437578, 'neutral'),\n",
       " (100, 0.03869107390856421, 3.1528788, 0.1397439470067116, 'contradiction'),\n",
       " (101, 0.0, 4.5216036, 0.09055340398261016, 'entailment'),\n",
       " (102, 0.08285517295042895, 8.780164, 0.0925514747879754, 'neutral'),\n",
       " (103, 0.062055161928667016, 4.0672827, 0.12969979483141864, 'contradiction'),\n",
       " (104, 0.0, 3.5930228, 0.10886076974402807, 'entailment'),\n",
       " (105, 0.024929081340273763, 3.06432, 0.135486347330009, 'entailment'),\n",
       " (106, 0.0, 2.8233314, 0.13077600486, 'contradiction'),\n",
       " (107, 0.04102601578140939, 7.7164154, 0.07787603807596866, 'neutral'),\n",
       " (108, 0.01836156527849506, 4.3603883, 0.10245760774582315, 'contradiction'),\n",
       " (109, 0.0164966248937915, 5.644419, 0.08349943454747061, 'entailment'),\n",
       " (110, 0.03566334576003568, 3.723363, 0.12368845504689095, 'entailment'),\n",
       " (111, 0.043113070350970464, 2.5446706, 0.16261336639832097, 'contradiction'),\n",
       " (112, 0.17483064959884734, 7.33811, 0.14738095271671015, 'neutral'),\n",
       " (113, 0.11671706711628992, 4.2721305, 0.1531968538558099, 'contradiction'),\n",
       " (114, 0.0, 3.9376976, 0.10126176925819903, 'entailment'),\n",
       " (115, 0.0, 3.373228, 0.11433202010963739, 'neutral'),\n",
       " (116, 0.0, 3.524489, 0.11050971903899662, 'contradiction'),\n",
       " (117, 0.015122444854024223, 3.193963, 0.12678020312346427, 'contradiction'),\n",
       " (118, 0.0, 5.991207, 0.07151840751065502, 'contradiction'),\n",
       " (119, 0.0, 3.0096545, 0.12469902263744588, 'contradiction'),\n",
       " (120, 0.03894152628464915, 12.61126, 0.05620505407405797, 'neutral'),\n",
       " (121, 0.06470843697955009, 4.2114687, 0.1282964622428185, 'contradiction'),\n",
       " (122, 0.0, 5.2145076, 0.08045689760278842, 'contradiction'),\n",
       " (123, 0.054939000492865556, 3.159027, 0.1476899239236339, 'contradiction'),\n",
       " (124, 0.04627052821494699, 2.944478, 0.14989474801556532, 'neutral'),\n",
       " (125, 0.029424068184199813, 3.710713, 0.12085308101741614, 'contradiction'),\n",
       " (126, 0.04038473725146162, 2.6084816, 0.15875480266550956, 'contradiction'),\n",
       " (127, 0.08056866056928838, 3.9638584, 0.14101242587727608, 'neutral'),\n",
       " (128, 0.06248242195683827, 2.7256463, 0.16544611543138885, 'neutral'),\n",
       " (129, 0.0, 2.2797837, 0.15244907646689798, 'neutral'),\n",
       " (130, 0.0, 5.5259514, 0.07661718122986616, 'entailment'),\n",
       " (131, 0.05659181794586696, 3.9842443, 0.12861201814185969, 'entailment'),\n",
       " (132, 0.0, 4.848211, 0.085496233994671, 'contradiction'),\n",
       " (133, 0.024127819357864778, 3.9397757, 0.11328308031045321, 'neutral'),\n",
       " (134, 0.023094072693713413, 3.6751425, 0.11849564746455495, 'contradiction'),\n",
       " (135, 0.0, 4.0912375, 0.09820794955633548, 'contradiction'),\n",
       " (136, 0.0, 3.2811058, 0.11679225611526321, 'entailment'),\n",
       " (137, 0.1422311307547251, 5.1492553, 0.15242622456786795, 'neutral'),\n",
       " (138, 0.06428741011828798, 2.5547745, 0.17279960230260225, 'entailment'),\n",
       " (139, 0.0, 2.1865735, 0.15690835286003668, 'neutral'),\n",
       " (140, 0.025681379833458796, 3.201505, 0.1318456670512144, 'contradiction'),\n",
       " (141, 0.03408886555656737, 3.0105562, 0.1417154191568998, 'neutral'),\n",
       " (142, 0.11916971209515209, 2.8298779, 0.1901373225072996, 'entailment'),\n",
       " (143, 0.08421731916835892, 7.364946, 0.10188190936529753, 'entailment'),\n",
       " (144, 0.07008964872782375, 3.6019003, 0.14369559103716345, 'neutral'),\n",
       " (145, 0.0, 3.7492156, 0.10528054352751308, 'entailment'),\n",
       " (146, 0.038031600670177676, 8.9146385, 0.06944628239727818, 'entailment'),\n",
       " (147, 0.0490843376279683, 2.852767, 0.1543190282015115, 'entailment'),\n",
       " (148, 0.10667958646854596, 5.0209346, 0.1363833794434635, 'contradiction'),\n",
       " (149, 0.027497919685916316, 4.2973113, 0.10813646533724504, 'entailment'),\n",
       " (150, 0.09224976229635995, 3.0675535, 0.16904889370673903, 'entailment'),\n",
       " (151, 0.0, 2.9812095, 0.12558997408315034, 'contradiction'),\n",
       " (152, 0.0, 3.6095738, 0.10846989705261531, 'entailment'),\n",
       " (153, 0.018224273667157422, 3.7980137, 0.1133219270934075, 'contradiction'),\n",
       " (154, 0.09468497956219182, 2.8625953, 0.17678913345191938, 'contradiction'),\n",
       " (155, 0.08843572628140416, 3.162616, 0.1643346354446182, 'entailment'),\n",
       " (156, 0.0, 3.327045, 0.11555230051426689, 'entailment'),\n",
       " (157, 0.019584459907448764, 3.933077, 0.11114884574840632, 'neutral'),\n",
       " (158, 0.029171808337871075, 2.6168916, 0.15282615358188553, 'entailment'),\n",
       " (159, 0.020795509374193905, 4.0104966, 0.11018826215238003, 'neutral'),\n",
       " (160, 0.04039949191406672, 3.8029103, 0.1243032927366446, 'contradiction'),\n",
       " (161, 0.11583081849074377, 3.650506, 0.16543059101129842, 'neutral'),\n",
       " (162, 0.0, 3.6634974, 0.10721566926511619, 'contradiction'),\n",
       " (163, 0.1054219708552647, 3.2253518, 0.17104432722482585, 'neutral'),\n",
       " (164, 0.03202463355753964, 4.1614056, 0.11288515381175834, 'contradiction'),\n",
       " (165, 0.024739560941724616, 4.8447247, 0.09791700972473567, 'contradiction'),\n",
       " (166, 0.01859207579228964, 7.3499455, 0.06917666793062865, 'contradiction'),\n",
       " (167, 0.0, 3.6184018, 0.1082625603731685, 'contradiction'),\n",
       " (168, 0.0879451061410996, 2.446201, 0.18905984992760005, 'entailment'),\n",
       " (169, 0.21516497713161653, 3.8506007, 0.2106625046130779, 'neutral'),\n",
       " (170, 0.12673157557642836, 3.1272507, 0.1845118096342787, 'neutral'),\n",
       " (171, 0.022310462997515036, 2.8009295, 0.14270200023045632, 'contradiction'),\n",
       " (172, 0.06712348247026519, 3.0319922, 0.15756991773097703, 'contradiction'),\n",
       " (173, 0.03674104379958417, 3.5763197, 0.12762862303263883, 'entailment'),\n",
       " (174, 0.01670970275750192, 5.241168, 0.08846806067307389, 'contradiction'),\n",
       " (175, 0.05179201382840774, 2.8143566, 0.15697971432782226, 'contradiction'),\n",
       " (176, 0.0, 4.1743417, 0.09663065005274443, 'contradiction'),\n",
       " (177, 0.12426430255144813, 3.636046, 0.1699826789130286, 'contradiction'),\n",
       " (178, 0.0, 5.102793, 0.08192969714774835, 'neutral'),\n",
       " (179, 0.06772478486341237, 4.839077, 0.11949236448985814, 'contradiction'),\n",
       " (180, 0.08055276933602291, 3.9201808, 0.14189866656591255, 'contradiction'),\n",
       " (181, 0.08500774424459367, 2.9108071, 0.17035471800436533, 'neutral'),\n",
       " (182, 0.03927076703342072, 2.5165977, 0.16181826477243932, 'entailment'),\n",
       " (183, 0.0, 7.6860723, 0.057563416453237246, 'entailment'),\n",
       " (184, 0.08735033425052478, 4.368502, 0.13681101525510841, 'contradiction'),\n",
       " (185, 0.0, 4.2731223, 0.09482048216315013, 'entailment'),\n",
       " (186, 0.03086287771832875, 2.845529, 0.14545255940499277, 'contradiction'),\n",
       " (187, 0.0, 6.0165534, 0.07126005766039707, 'contradiction'),\n",
       " (188, 0.021633665316439166, 2.695066, 0.14613241385684678, 'neutral'),\n",
       " (189, 0.08258544900733591, 3.7801907, 0.14589106183484205, 'contradiction'),\n",
       " (190, 0.02344274565768124, 3.461152, 0.12380004475062018, 'entailment'),\n",
       " (191, 0.2646510323502834, 3.404345, 0.24584977368990826, 'entailment'),\n",
       " (192, 0.03322995308918496, 4.652242, 0.10507544652289699, 'neutral'),\n",
       " (193, 0.0, 3.198716, 0.1190840268561864, 'entailment'),\n",
       " (194, 0.13759037231079946, 7.9320297, 0.12477350412379405, 'entailment'),\n",
       " (195, 0.13251684735644387, 4.0711517, 0.16485535516165922, 'contradiction'),\n",
       " (196, 0.0, 3.4650207, 0.11198156480251077, 'entailment'),\n",
       " (197, 0.04072850153283757, 3.2401793, 0.13828379250930073, 'entailment'),\n",
       " (198, 0.08719842182235774, 2.4219792, 0.18971348349912515, 'entailment'),\n",
       " (199, 0.0, 3.2522614, 0.1175844928004202, 'neutral'),\n",
       " (200, 0.041442837188629573, 5.0926585, 0.10278739986991355, 'neutral'),\n",
       " (201, 0.0484940407450031, 2.7347252, 0.1581256776458278, 'contradiction'),\n",
       " (202, 0.0, 3.5733588, 0.10932883788219376, 'entailment'),\n",
       " (203, 0.05958086258261907, 3.995167, 0.12988718447294562, 'neutral'),\n",
       " (204, 0.0, 4.01028, 0.09979481921125646, 'entailment'),\n",
       " (205, 0.03263925585744804, 3.2727609, 0.13333998442378703, 'contradiction'),\n",
       " (206, 0.07454492592755879, 6.862391, 0.10086634939302709, 'contradiction'),\n",
       " (207, 0.04446396934256457, 2.5667682, 0.1624149111337214, 'neutral'),\n",
       " (208, 0.03901185966225239, 3.2187707, 0.13802386563003036, 'contradiction'),\n",
       " (209, 0.02246880682642393, 7.173889, 0.07240479491267807, 'contradiction'),\n",
       " (210, 0.02408801920576986, 5.5105195, 0.08884279649704703, 'contradiction'),\n",
       " (211, 0.023456164775256215, 4.5377994, 0.10201665509376497, 'contradiction'),\n",
       " (212, 0.04010988715892364, 5.3117104, 0.09927277387687318, 'contradiction'),\n",
       " (213, 0.0, 3.2734823, 0.11700060097240146, 'neutral'),\n",
       " (214, 0.025132931355381007, 2.9748254, 0.13835815510258523, 'entailment'),\n",
       " (215, 0.034647962968438216, 6.091722, 0.08782871914659938, 'contradiction'),\n",
       " (216, 0.05115478615319222, 5.178636, 0.10650140187178476, 'neutral'),\n",
       " (217, 0.07986271199103868, 5.5825887, 0.11588931491802748, 'entailment'),\n",
       " (218, 0.0, 5.015749, 0.08311517017360577, 'contradiction'),\n",
       " (219, 0.02883199059390225, 3.9383643, 0.1156640954636049, 'entailment'),\n",
       " (220, 0.02332090368002329, 4.1290956, 0.10914352555186178, 'neutral'),\n",
       " (221, 0.030415650199900036, 3.895455, 0.11734337979120692, 'neutral'),\n",
       " (222, 0.04236895763333952, 2.9183922, 0.14878783673828647, 'entailment'),\n",
       " (223, 0.0, 3.9465148, 0.10108126947523055, 'neutral'),\n",
       " (224, 0.0619339529236993, 4.928669, 0.11530293821623223, 'contradiction'),\n",
       " (225, 0.0, 4.262394, 0.0950137911787199, 'entailment'),\n",
       " (226, 0.05249677892093406, 2.267857, 0.1792538571307924, 'entailment'),\n",
       " (227, 0.030789729726902335, 7.921963, 0.0714363422780211, 'contradiction'),\n",
       " (228, 0.0, 3.9413052, 0.10118784081473998, 'neutral'),\n",
       " (229, 0.4363744143304071, 4.857994, 0.3035406563504933, 'entailment'),\n",
       " (230, 0.04001706960763533, 3.905422, 0.12193656522117566, 'contradiction'),\n",
       " (231, 0.016538710017153263, 2.4973342, 0.15123538725520375, 'entailment'),\n",
       " (232, 0.0, 5.0887723, 0.08211836074968903, 'entailment'),\n",
       " (233, 0.03925169825785173, 4.5066037, 0.11042591857706857, 'contradiction'),\n",
       " (234, 0.04084676730802281, 3.1622777, 0.14054991922326623, 'contradiction'),\n",
       " (235, 0.0, 5.314198, 0.07918662016051706, 'neutral'),\n",
       " (236, 0.0, 5.1975145, 0.08067750341806229, 'neutral'),\n",
       " (237, 0.09106236674085934, 2.2588644, 0.19895886191233295, 'contradiction'),\n",
       " (238, 0.06316413716665542, 4.2681236, 0.1264925216078236, 'entailment'),\n",
       " (239, 0.023487391614756246, 7.6694603, 0.06941741284306382, 'neutral'),\n",
       " (240, 0.0, 4.1298747, 0.09746826747815861, 'entailment'),\n",
       " (241, 0.3579002523158624, 3.715878, 0.28497492132043667, 'entailment'),\n",
       " (242, 0.025897781541474662, 4.7520776, 0.09987400488639414, 'contradiction'),\n",
       " (243, 0.12629001823898575, 4.6912885, 0.1509985773454327, 'entailment'),\n",
       " (244, 0.17803885374165346, 3.6909995, 0.19560651969774562, 'entailment'),\n",
       " (245, 0.06886149478207157, 2.8617802, 0.16390471494648268, 'entailment'),\n",
       " (246, 0.13862797915554498, 4.5756087, 0.15899029653980132, 'contradiction'),\n",
       " (247, 0.022484372948219783, 2.8860006, 0.1399091752859855, 'contradiction'),\n",
       " (248, 0.07755088723675524, 3.7601535, 0.1438140724432258, 'contradiction'),\n",
       " (249, 0.0, 4.716096, 0.08747228993615194, 'neutral'),\n",
       " (250, 0.03623395682664417, 2.5834641, 0.15764676849242848, 'entailment'),\n",
       " (251, 0.0, 6.8783, 0.0634654669082598, 'entailment'),\n",
       " (252, 0.16579978999279343, 3.6951127, 0.18939361120537335, 'neutral'),\n",
       " (253, 0.09799608568820539, 2.4632561, 0.1933708476777146, 'neutral'),\n",
       " (254, 0.30380256764024505, 2.8388135, 0.28214986045914664, 'entailment'),\n",
       " (255, 0.12949885515031756, 2.7496872, 0.19809388379811033, 'contradiction'),\n",
       " (256, 0.0, 5.16523, 0.08109997784897456, 'entailment'),\n",
       " (257, 0.036850293675967837, 3.5618675, 0.128029382972472, 'neutral'),\n",
       " (258, 0.0, 2.8599243, 0.12953621859236888, 'contradiction'),\n",
       " (259, 0.0, 5.887324, 0.07259713793460362, 'contradiction'),\n",
       " (260, 0.06950717650887642, 5.1111627, 0.11657107988621174, 'entailment'),\n",
       " (261, 0.26253838456496426, 3.2472854, 0.24899144471121873, 'neutral'),\n",
       " (262, 0.029105625236489425, 3.074302, 0.13727322095909317, 'entailment'),\n",
       " (263, 0.15971347653753704, 4.236251, 0.17534490560618676, 'entailment'),\n",
       " (264, 0.0, 4.1802096, 0.0965211902736152, 'neutral'),\n",
       " (265, 0.1317258737733699, 3.1526234, 0.18626875028038714, 'entailment'),\n",
       " (266, 0.04446209699629817, 3.8684058, 0.1249340725671135, 'contradiction'),\n",
       " (267, 0.028135202790163783, 2.5994196, 0.15297888610879437, 'neutral'),\n",
       " (268, 0.08470700411131236, 3.9094362, 0.14419818991678526, 'contradiction'),\n",
       " (269, 0.023502460284417832, 2.5352705, 0.15318312530216946, 'neutral'),\n",
       " (270, 0.05608632749981759, 3.8625915, 0.13086899221620332, 'neutral'),\n",
       " (271, 0.025081705202287155, 3.5226383, 0.12309579078553817, 'neutral'),\n",
       " (272, 0.06855881891509504, 2.8354619, 0.16464180619386276, 'entailment'),\n",
       " (273, 0.03567501359172876, 3.7602777, 0.12287339471201325, 'contradiction'),\n",
       " (274, 0.0, 4.0355697, 0.09929363170082824, 'neutral'),\n",
       " (275, 0.06914255243563673, 3.9787586, 0.13499791723995627, 'neutral'),\n",
       " (276, 0.05616518519573978, 2.883346, 0.15683753475321016, 'contradiction'),\n",
       " (277, 0.13254786045933603, 3.2812498, 0.1830622579013777, 'neutral'),\n",
       " (278, 0.08445435590851136, 3.6374435, 0.15004520213500092, 'contradiction'),\n",
       " (279, 0.09722064129906449, 5.7053366, 0.12317779309854594, 'entailment'),\n",
       " (280, 0.016123844812818467, 3.1365595, 0.12893532012864398, 'neutral'),\n",
       " (281, 0.02916377686082726, 4.6936193, 0.10239949230164558, 'contradiction'),\n",
       " (282, 0.04879512862034383, 2.5634706, 0.1647102133648296, 'neutral'),\n",
       " (283, 0.0684664682447653, 2.239153, 0.18859457924903955, 'contradiction'),\n",
       " (284, 0.16467497449757554, 3.4004965, 0.19596102994419579, 'contradiction'),\n",
       " (285, 0.0, 5.3728557, 0.0784577631154326, 'entailment'),\n",
       " (286, 0.2159863418513403, 4.818616, 0.19392425959733983, 'contradiction'),\n",
       " (287, 0.04475246429374097, 4.424179, 0.1145560796202796, 'entailment'),\n",
       " (288, 0.03991538581696983, 4.9466667, 0.1040384096781206, 'neutral'),\n",
       " (289, 0.06618505074283165, 4.9468136, 0.11717116563649653, 'contradiction'),\n",
       " (290, 0.04704835536928809, 3.3589401, 0.1382309609203989, 'contradiction'),\n",
       " (291, 0.0, 3.4454253, 0.11247517828061676, 'entailment'),\n",
       " (292, 0.0, 3.3174164, 0.11581000076372734, 'entailment'),\n",
       " (293, 0.0, 3.4533813, 0.11227423980135806, 'contradiction'),\n",
       " (294, 0.056428449815718526, 2.7299984, 0.16226254157287148, 'neutral'),\n",
       " (295, 0.02331419625856509, 3.9587252, 0.11248946500210127, 'contradiction'),\n",
       " (296, 0.0, 2.4877086, 0.14336060200252795, 'neutral'),\n",
       " (297, 0.058080746663789144, 3.1760457, 0.14877086508482004, 'entailment'),\n",
       " (298, 0.07081370142880038, 2.2851875, 0.18760516591335222, 'neutral'),\n",
       " (299, 0.04240761858958978, 4.1548204, 0.11820039833342993, 'neutral'),\n",
       " (300, 0.1483792977008334, 4.316655, 0.16823373955167736, 'entailment'),\n",
       " (301, 0.25056857715673625, 2.0602481, 0.2886697324033641, 'neutral'),\n",
       " (302, 0.1023326490369966, 2.1793935, 0.20842902109855627, 'neutral'),\n",
       " (303, 0.02064017881068756, 4.445865, 0.10213286585800602, 'neutral'),\n",
       " (304, 0.07106581821607148, 11.515531, 0.07548327290240896, 'entailment'),\n",
       " (305, 0.015518353813291566, 4.974128, 0.09145340295838286, 'entailment'),\n",
       " (306, 0.030202653756389444, 4.530386, 0.10551092986352158, 'entailment'),\n",
       " (307, 0.0, 4.052538, 0.09896016776236936, 'neutral'),\n",
       " (308, 0.0, 4.940717, 0.08416492175236245, 'entailment'),\n",
       " (309, 0.1068227525849986, 2.758638, 0.1864382899302267, 'contradiction'),\n",
       " (310, 0.16448537681757935, 4.463546, 0.17375834845288507, 'contradiction'),\n",
       " (311, 0.0, 3.8281927, 0.10355841822006204, 'entailment'),\n",
       " (312, 0.0, 2.4960833, 0.14301718891549137, 'contradiction'),\n",
       " (313, 0.0619765598926417, 4.0670514, 0.12966499732610137, 'contradiction'),\n",
       " (314, 0.0, 3.0969708, 0.12204138736277086, 'entailment'),\n",
       " (315, 0.015576133268207596, 4.109467, 0.10564563125017858, 'neutral'),\n",
       " (316, 0.0428488386589118, 3.5993552, 0.13013530944279006, 'neutral'),\n",
       " (317, 0.136559191615883, 4.4976134, 0.15922815127919998, 'contradiction'),\n",
       " (318, 0.0928102941397579, 4.134999, 0.14377615330237886, 'contradiction'),\n",
       " (319, 0.12517770939199785, 5.9524546, 0.13450590142888347, 'entailment'),\n",
       " (320, 0.17078237880802016, 6.820443, 0.14932618518283797, 'entailment'),\n",
       " (321, 0.12167140346189864, 3.4401464, 0.17344460012094673, 'entailment'),\n",
       " (322, 0.0, 7.1900473, 0.06104970873510562, 'entailment'),\n",
       " (323, 0.06359121983264952, 2.9734547, 0.15763069198251306, 'neutral'),\n",
       " (324, 0.0, 4.14423, 0.09719627831511224, 'contradiction'),\n",
       " (325, 0.0574710406357444, 2.4646826, 0.173048884845039, 'contradiction'),\n",
       " (326, 0.02414854412293, 3.9180284, 0.11374103035247418, 'entailment'),\n",
       " (327, 0.02891455609826517, 5.236665, 0.09462833393818543, 'contradiction'),\n",
       " (328, 0.01717099432912429, 4.09664, 0.10668934385709292, 'entailment'),\n",
       " (329, 0.050859953523429806, 5.228823, 0.1057019615576111, 'contradiction'),\n",
       " (330, 0.0, 6.960262, 0.06281200432805341, 'neutral'),\n",
       " (331, 0.0, 3.5720596, 0.1093599034824094, 'contradiction'),\n",
       " (332, 0.0662504900789906, 5.9886904, 0.10466940754766751, 'contradiction'),\n",
       " (333, 0.28609491196080844, 2.4251068, 0.28902830693626586, 'neutral'),\n",
       " (334, 0.0, 4.4649, 0.09149298220723538, 'neutral'),\n",
       " (335, 0.0, 3.7816148, 0.10456718555555655, 'entailment'),\n",
       " (336, 0.0, 4.3435307, 0.09357109227787913, 'entailment'),\n",
       " (337, 0.048906170486201155, 8.175559, 0.07894567774984942, 'neutral'),\n",
       " (338, 0.0, 4.4248643, 0.09216820423028867, 'entailment'),\n",
       " (339, 0.06608752987477304, 5.30928, 0.1122921112534822, 'entailment'),\n",
       " (340, 0.0, 3.553467, 0.10980643894521203, 'contradiction'),\n",
       " (341, 0.27470626464289183, 3.4109695, 0.2507068974016875, 'entailment'),\n",
       " (342, 0.0, 3.3655865, 0.11453214769567589, 'entailment'),\n",
       " (343, 0.0, 4.0991874, 0.09805483958839216, 'contradiction'),\n",
       " (344, 0.0, 6.015638, 0.07126935696182503, 'contradiction'),\n",
       " (345, 0.06859049575086115, 2.4664705, 0.17853417992167092, 'entailment'),\n",
       " (346, 0.13262485197896345, 2.80586, 0.19768877572568938, 'contradiction'),\n",
       " (347, 0.0, 2.0488992, 0.16399361589488903, 'entailment'),\n",
       " (348, 0.03840584900164832, 7.2345076, 0.07992300961975451, 'neutral'),\n",
       " (349, 0.0, 4.404385, 0.09251746344589823, 'neutral'),\n",
       " (350, 0.14575344416683203, 4.499439, 0.16379509058939085, 'entailment'),\n",
       " (351, 0.058992027318614304, 5.1619864, 0.11063867959815454, 'contradiction'),\n",
       " (352, 0.16711056299774982, 4.0188932, 0.18317883910467012, 'neutral'),\n",
       " (353, 0.0, 2.6631262, 0.13649543274605067, 'entailment'),\n",
       " (354, 0.08097327075804991, 4.027584, 0.1399379806870068, 'entailment'),\n",
       " (355, 0.0, 3.5788686, 0.10919728008613515, 'entailment'),\n",
       " (356, 0.02943678533808005, 6.71934, 0.07949077086116488, 'contradiction'),\n",
       " (357, 0.0, 4.268771, 0.09489878829773235, 'entailment'),\n",
       " (358, 0.051092626185668275, 4.2625012, 0.12055816719383661, 'contradiction'),\n",
       " (359, 0.0, 4.0888724, 0.09825359283645012, 'contradiction'),\n",
       " (360, 0.08472254441013612, 3.3013444, 0.15860399869262046, 'entailment'),\n",
       " (361, 0.0, 2.9597745, 0.12626981681305557, 'entailment'),\n",
       " (362, 0.0, 4.277929, 0.0947341307890352, 'contradiction'),\n",
       " (363, 0.0, 4.77447, 0.08658803540000731, 'contradiction'),\n",
       " (364, 0.0, 6.6230826, 0.06559026364499179, 'entailment'),\n",
       " (365, 0.05441390124050703, 3.7049959, 0.13347696950069057, 'contradiction'),\n",
       " (366, 0.0, 5.046105, 0.0826978703825144, 'contradiction'),\n",
       " (367, 0.032373926725849766, 5.5153165, 0.09292920620667136, 'contradiction'),\n",
       " (368, 0.0, 5.5021267, 0.07689791718184903, 'neutral'),\n",
       " (369, 0.0, 4.093473, 0.09816484825994003, 'entailment'),\n",
       " (370, 0.019097310191961276, 3.2303367, 0.12774255772456453, 'neutral'),\n",
       " (371, 0.0, 5.217312, 0.08042060802623094, 'entailment'),\n",
       " (372, 0.06067157934378644, 3.0499403, 0.1537943982134912, 'contradiction'),\n",
       " (373, 0.0207216402651622, 3.4810026, 0.12194299226452457, 'entailment'),\n",
       " (374, 0.0, 5.0306487, 0.08290982018372939, 'entailment'),\n",
       " (375, 0.17366299889240422, 6.556711, 0.15299784972370833, 'contradiction'),\n",
       " (376, 0.03624916508481212, 2.0141468, 0.1840090043635885, 'neutral'),\n",
       " (377, 0.03437550536128583, 3.4552562, 0.12941474365427608, 'contradiction'),\n",
       " (378, 0.0, 3.2996144, 0.11628949716339625, 'neutral'),\n",
       " (379, 0.0, 5.11997, 0.08169974896581536, 'neutral'),\n",
       " (380, 0.0, 3.8063784, 0.10402843098796731, 'neutral'),\n",
       " (381, 0.0, 5.528198, 0.07659081694090912, 'neutral'),\n",
       " (382, 0.05728351998507123, 7.3197455, 0.08873975187929295, 'neutral'),\n",
       " (383, 0.04639373314282359, 4.297828, 0.11757516299066983, 'entailment'),\n",
       " (384, 0.02207826952219373, 4.621828, 0.09997817611097914, 'contradiction'),\n",
       " (385, 0.016329848505821365, 3.5968623, 0.1169347689022423, 'neutral'),\n",
       " (386, 0.07591050819062531, 3.3125954, 0.153894719173803, 'entailment'),\n",
       " (387, 0.0, 5.4413915, 0.07762297982977778, 'entailment'),\n",
       " (388, 0.05156028079923651, 3.6525092, 0.13324903023164583, 'contradiction'),\n",
       " (389, 0.2562464471502298, 2.8401675, 0.25832587673529606, 'contradiction'),\n",
       " (390, 0.13938656868555913, 5.0043163, 0.15296671174457327, 'entailment'),\n",
       " (391, 0.24620688424551987, 2.1461678, 0.28202694490966845, 'contradiction'),\n",
       " (392, 0.07559155092454464, 2.8710353, 0.16696018663750384, 'entailment'),\n",
       " (393, 0.02214626740314333, 2.9546516, 0.13750652185088924, 'entailment'),\n",
       " (394, 0.13324529096686227, 3.10106, 0.18854234686052976, 'neutral'),\n",
       " (395, 0.0, 3.4603927, 0.11209775284616519, 'neutral'),\n",
       " (396, 0.0, 2.9694648, 0.12596156606748865, 'neutral'),\n",
       " (397, 0.026772883032249034, 4.172327, 0.11005472949870723, 'neutral'),\n",
       " (398, 0.0, 6.417471, 0.06740842054971456, 'contradiction'),\n",
       " (399, 0.0, 5.590808, 0.07586323353199836, 'neutral'),\n",
       " (400, 0.0, 4.685395, 0.08794464074345536, 'neutral'),\n",
       " (401, 0.0, 3.8812575, 0.10243262038819198, 'neutral'),\n",
       " (402, 0.0, 2.644731, 0.13718433372952768, 'entailment'),\n",
       " (403, 0.0836122133345933, 4.6300306, 0.13061557018182995, 'neutral'),\n",
       " (404, 0.03793309466484864, 3.0348866, 0.14288576731126917, 'neutral'),\n",
       " (405, 0.0, 7.5141764, 0.05872558640402672, 'entailment'),\n",
       " (406, 0.0387792875564478, 3.8251674, 0.12301299126744714, 'contradiction'),\n",
       " (407, 0.03417573710475889, 7.334306, 0.0770808678739384, 'neutral'),\n",
       " (408, 0.0179947398366644, 3.160562, 0.12917344117568622, 'entailment'),\n",
       " (409, 0.0, 3.023303, 0.12427599811223211, 'contradiction'),\n",
       " (410, 0.30726884434619123, 2.342668, 0.3032155326963234, 'entailment'),\n",
       " (411, 0.052082946072864886, 2.7028913, 0.16107108983251298, 'neutral'),\n",
       " (412, 0.06050635610767831, 5.3452225, 0.10905262159048362, 'neutral'),\n",
       " (413, 0.13163486373865146, 3.993424, 0.16594912630043035, 'entailment'),\n",
       " (414, 0.10515452917587419, 3.243715, 0.17039855898774123, 'entailment'),\n",
       " (415, 0.023744244430798193, 6.1201963, 0.08209490483977147, 'entailment'),\n",
       " (416, 0.07660861026100389, 2.081084, 0.20058485246148997, 'entailment'),\n",
       " (417, 0.0, 7.2806726, 0.06038156888450301, 'neutral'),\n",
       " (418, 0.050578432847406894, 1.9871554, 0.19267253827171762, 'entailment'),\n",
       " (419, 0.1711871533970734, 1.9928412, 0.25265890334931645, 'entailment'),\n",
       " (420, 0.0, 4.949419, 0.08404181957669396, 'contradiction'),\n",
       " (421, 0.08132163541804431, 3.9427214, 0.14181966581851624, 'contradiction'),\n",
       " (422, 0.01995203112641513, 6.9400744, 0.07294771734483825, 'neutral'),\n",
       " (423, 0.0, 3.0330417, 0.12397590584378407, 'entailment'),\n",
       " (424, 0.13228620689051324, 4.715957, 0.15361751684398064, 'entailment'),\n",
       " (425, 0.0919523307129918, 3.1813955, 0.16555346827579523, 'contradiction'),\n",
       " (426, 0.0, 5.162537, 0.08113541420592447, 'entailment'),\n",
       " (427, 0.02251511072281732, 3.076187, 0.13392121451989167, 'entailment'),\n",
       " (428, 0.0232181956637862, 2.919666, 0.13917098534579114, 'neutral'),\n",
       " (429, 0.07448196691929394, 3.2458143, 0.15500402298809887, 'entailment'),\n",
       " (430, 0.10589517673655555, 2.9814248, 0.17853077128617867, 'neutral'),\n",
       " (431, 0.027210778941967915, 2.7018454, 0.14867315837048686, 'contradiction'),\n",
       " (432, 0.0, 4.1951365, 0.09624386105504815, 'contradiction'),\n",
       " (433, 0.035782849969989115, 4.5894594, 0.1073455139221045, 'neutral'),\n",
       " (434, 0.09560883969302401, 4.3961387, 0.14046326921216964, 'contradiction'),\n",
       " (435, 0.0, 4.0042825, 0.09991442380521826, 'entailment'),\n",
       " (436, 0.0, 3.793, 0.10431879862106215, 'entailment'),\n",
       " (437, 0.03164949109445568, 3.2792, 0.1326690134001781, 'neutral'),\n",
       " (438, 0.07870233055999964, 4.2827077, 0.1339995973394916, 'entailment'),\n",
       " (439, 0.0, 3.7824833, 0.10454819484476666, 'contradiction'),\n",
       " (440, 0.08965949917286038, 3.6726358, 0.15183573551746404, 'entailment'),\n",
       " (441, 0.022911804555086252, 7.9994664, 0.06701475173690224, 'contradiction'),\n",
       " (442, 0.0398041717011435, 3.0466177, 0.14346206392129632, 'neutral'),\n",
       " (443, 0.04679332670186925, 4.1290784, 0.12088006332176145, 'contradiction'),\n",
       " (444, 0.05886950717668789, 3.346432, 0.14447163979351374, 'entailment'),\n",
       " (445, 0.0, 4.432818, 0.09203327000744727, 'contradiction'),\n",
       " (446, 0.0, 10.651665, 0.04291232295294613, 'entailment'),\n",
       " (447, 0.017383017610969245, 2.3369234, 0.1585301313732038, 'neutral'),\n",
       " (448, 0.020483881153589893, 3.3753304, 0.12451902339949696, 'contradiction'),\n",
       " (449, 0.02110354608679083, 3.9353416, 0.11186188303539892, 'entailment'),\n",
       " (450, 0.07643280375223323, 3.0081754, 0.16296144256432146, 'entailment'),\n",
       " (451, 0.08165779077606118, 3.4127467, 0.1541370088225731, 'neutral'),\n",
       " (452, 0.031450667854901505, 2.849733, 0.14560446714585218, 'entailment'),\n",
       " (453, 0.045379307768052914, 2.8697145, 0.15189815238985727, 'entailment'),\n",
       " (454, 0.0, 3.7888477, 0.10440925101475498, 'entailment'),\n",
       " (455, 0.0, 3.396839, 0.11371806222199182, 'contradiction'),\n",
       " (456, 0.04071614444984672, 2.0556083, 0.18399161271875342, 'neutral'),\n",
       " (457, 0.0, 5.0244126, 0.08299564298519191, 'neutral'),\n",
       " (458, 0.12754529017372782, 3.6230597, 0.17192612505062765, 'entailment'),\n",
       " (459, 0.031696313200797764, 6.1762486, 0.08552244344921245, 'contradiction'),\n",
       " (460, 0.054263968276162936, 9.837571, 0.07326778277337037, 'neutral'),\n",
       " (461, 0.030119710046077528, 7.5722556, 0.07338756043279304, 'neutral'),\n",
       " (462, 0.10020193493824647, 4.495764, 0.14108013253787716, 'contradiction'),\n",
       " (463, 0.16810947891599173, 3.3792038, 0.19823074570199622, 'entailment'),\n",
       " (464, 0.0445517137502789, 3.363199, 0.13687067606751163, 'contradiction'),\n",
       " (465, 0.0, 2.6529915, 0.13687411959346438, 'entailment'),\n",
       " (466, 0.025276096698552485, 4.0215216, 0.11220946175444393, 'neutral'),\n",
       " (467, 0.05716906990822213, 10.399023, 0.07244794303235788, 'neutral'),\n",
       " (468, 0.03016908557438393, 5.1103997, 0.09691225008326748, 'neutral'),\n",
       " (469, 0.12183465901194561, 2.5377443, 0.2022503258751844, 'contradiction'),\n",
       " (470, 0.0, 4.5973463, 0.08932804451954025, 'contradiction'),\n",
       " (471, 0.09040437981261175, 2.987626, 0.17059007495433726, 'neutral'),\n",
       " (472, 0.01437835120787661, 6.168184, 0.07694185127497727, 'neutral'),\n",
       " (473, 0.13508558520657168, 3.3848228, 0.18157248493573905, 'contradiction'),\n",
       " (474, 0.023341299883749958, 3.290239, 0.1282142711359832, 'entailment'),\n",
       " (475, 0.0, 9.257873, 0.04874305037699796, 'contradiction'),\n",
       " (476, 0.07642646522920088, 2.7633774, 0.17107261479419672, 'neutral'),\n",
       " (477, 0.02587856747849295, 3.2475045, 0.1306554634889844, 'neutral'),\n",
       " (478, 0.09059669701297973, 3.378569, 0.15949091069350063, 'entailment'),\n",
       " (479, 0.161786390419487, 2.3517401, 0.2300694385413125, 'contradiction'),\n",
       " (480, 0.0, 5.151496, 0.08128104210789217, 'neutral'),\n",
       " (481, 0.06148862376203047, 4.2700014, 0.12562094681223218, 'neutral'),\n",
       " (482, 0.0, 3.7641122, 0.10495134779141468, 'contradiction'),\n",
       " (483, 0.056403448809131757, 3.0008366, 0.15317558578421425, 'entailment'),\n",
       " (484, 0.06483077310124324, 2.646064, 0.16954956579077024, 'neutral'),\n",
       " (485, 0.030778872309372957, 6.4264016, 0.08271679412849627, 'entailment'),\n",
       " (486, 0.026892451021812962, 2.7660058, 0.14621288403404428, 'contradiction'),\n",
       " (487, 0.0, 4.059928, 0.09881563648583726, 'entailment'),\n",
       " (488, 0.05178307782884201, 1.850979, 0.2012698932161582, 'contradiction'),\n",
       " (489, 0.055586063592352984, 7.0915413, 0.08958595629134397, 'neutral'),\n",
       " (490, 0.02156661737491693, 3.3370142, 0.12606999605768102, 'neutral'),\n",
       " (491, 0.15927809455668102, 3.8374252, 0.18299981794189912, 'neutral'),\n",
       " (492, 0.0, 4.191067, 0.09631930755800533, 'neutral'),\n",
       " (493, 0.0, 2.4597306, 0.14451992197454622, 'neutral'),\n",
       " (494, 0.021452179047705116, 5.2317276, 0.09096066203503943, 'contradiction'),\n",
       " (495, 0.0, 3.1146717, 0.1215163773894171, 'contradiction'),\n",
       " (496, 0.025320199233497948, 3.300412, 0.12892803092661992, 'neutral'),\n",
       " (497, 0.0, 5.41414, 0.07795277036406621, 'contradiction'),\n",
       " (498, 0.09908868445900759, 2.7304156, 0.1835776661102421, 'contradiction'),\n",
       " (499, 0.0, 2.4782066, 0.14375224146761703, 'contradiction'),\n",
       " (500, 0.06028429630332921, 3.7048075, 0.13641642141019647, 'contradiction'),\n",
       " (501, 0.0, 4.247948, 0.09527533120202222, 'entailment'),\n",
       " (502, 0.0, 6.958124, 0.0628288764913961, 'neutral'),\n",
       " (503, 0.02753008602820843, 3.5109172, 0.12460724634031323, 'contradiction'),\n",
       " (504, 0.09001567043592106, 5.0866194, 0.12715524234484601, 'neutral'),\n",
       " (505, 0.0, 4.7559147, 0.08686716657437958, 'neutral'),\n",
       " (506, 0.0, 2.4845839, 0.14348915705648643, 'neutral'),\n",
       " (507, 0.023061569503106158, 3.1117754, 0.13313275751793255, 'contradiction'),\n",
       " (508, 0.0770761220518334, 2.6031275, 0.17730639571220388, 'entailment'),\n",
       " (509, 0.018059326726801047, 3.2183099, 0.127560547605271, 'neutral'),\n",
       " (510, 0.017256426377986037, 3.8582537, 0.11154585172378567, 'entailment'),\n",
       " (511, 0.021783285632387846, 2.6140292, 0.14924138395204545, 'entailment'),\n",
       " (512, 0.02508351748960834, 5.5060477, 0.08939333147868445, 'neutral'),\n",
       " (513, 0.0656665401131197, 2.5102148, 0.1752746953510737, 'contradiction'),\n",
       " (514, 0.0, 3.9660554, 0.10068353258458124, 'entailment'),\n",
       " (515, 0.10630465225136912, 5.1624594, 0.13428876365778664, 'entailment'),\n",
       " (516, 0.0, 5.0593314, 0.08251735473492916, 'neutral'),\n",
       " (517, 0.08388763168576047, 3.6750357, 0.14889487043797184, 'contradiction'),\n",
       " (518, 0.0451023977997128, 3.4166737, 0.1357585668104147, 'contradiction'),\n",
       " (519, 0.0, 3.077848, 0.12261369359462676, 'neutral'),\n",
       " (520, 0.054878647813181206, 5.396671, 0.10560498448229394, 'entailment'),\n",
       " (521, 0.0, 4.3003197, 0.09433393285242259, 'neutral'),\n",
       " (522, 0.0, 7.453292, 0.05914855494505308, 'neutral'),\n",
       " (523, 0.02525335157600196, 3.4705367, 0.12447007011965314, 'contradiction'),\n",
       " (524, 0.13714730865974037, 2.9731126, 0.19441957224299322, 'neutral'),\n",
       " (525, 0.025084711284899253, 2.8288362, 0.14313033944224468, 'entailment'),\n",
       " (526, 0.07915291743058726, 2.181033, 0.1967581089340466, 'neutral'),\n",
       " (527, 0.14197961817906707, 2.8846662, 0.19970099653387857, 'contradiction'),\n",
       " (528, 0.15466568078659984, 2.3780036, 0.2253492606261368, 'neutral'),\n",
       " (529, 0.0, 8.199717, 0.05434950047695551, 'entailment'),\n",
       " (530, 0.08444904688445884, 4.2181215, 0.1380444458524508, 'neutral'),\n",
       " (531, 0.09134881618297708, 3.1961448, 0.16483140136423707, 'contradiction'),\n",
       " (532, 0.023203776450687837, 4.382913, 0.10448839587226824, 'entailment'),\n",
       " (533, 0.030301658695855476, 2.8070078, 0.14648757138569038, 'contradiction'),\n",
       " (534, 0.017665236454954183, 3.201108, 0.1278488402623946, 'entailment'),\n",
       " (535, 0.03610278376478412, 2.471154, 0.16209570788461058, 'neutral'),\n",
       " (536, 0.13176578178955503, 4.483392, 0.15706732960023145, 'neutral'),\n",
       " (537, 0.0, 6.378497, 0.06776447718502789, 'neutral'),\n",
       " (538, 0.013510620763563677, 4.1901164, 0.10309226333940594, 'neutral'),\n",
       " (539, 0.04069872587254409, 3.1313307, 0.14137574229317282, 'neutral'),\n",
       " (540, 0.017598244574842214, 3.2781076, 0.1256732268469976, 'contradiction'),\n",
       " (541, 0.0, 3.2624733, 0.11730278632935956, 'contradiction'),\n",
       " (542, 0.0, 5.3339386, 0.07893982428372853, 'neutral'),\n",
       " (543, 0.05627660635588878, 2.161191, 0.1863065385405616, 'entailment'),\n",
       " (544, 0.027881535605223818, 2.6620817, 0.15047513225484346, 'entailment'),\n",
       " (545, 0.0, 5.783117, 0.07371242652226218, 'entailment'),\n",
       " (546, 0.042360873007383754, 3.39807, 0.13486666454824128, 'neutral'),\n",
       " (547, 0.16297879688568934, 2.9616685, 0.207698848095183, 'entailment'),\n",
       " (548, 0.05146075661542661, 2.6302788, 0.16346084585529633, 'neutral'),\n",
       " (549, 0.04404821680999386, 2.8723147, 0.15114584561605257, 'entailment'),\n",
       " (550, 0.017201988914548348, 3.6449816, 0.11624404681230215, 'entailment'),\n",
       " (551, 0.02898141869183887, 6.091397, 0.08499868027090285, 'neutral'),\n",
       " (552, 0.08134608078399241, 3.4075134, 0.1541156909039847, 'neutral'),\n",
       " (553, 0.022279662404680843, 4.8029633, 0.0973027065938046, 'contradiction'),\n",
       " (554, 0.0, 8.42481, 0.05305146504523593, 'neutral'),\n",
       " (555, 0.0, 3.2428546, 0.11784518860662913, 'neutral'),\n",
       " (556, 0.03133570944185814, 4.654414, 0.104094344955591, 'neutral'),\n",
       " (557, 0.3233171611331862, 2.9095926, 0.28954914296376055, 'neutral'),\n",
       " (558, 0.047414275504554494, 5.6729646, 0.09863635317627262, 'entailment'),\n",
       " (559, 0.07793494599465924, 3.2683685, 0.15610824986311653, 'contradiction'),\n",
       " (560, 0.057748124074536084, 2.474536, 0.17277817134431395, 'entailment'),\n",
       " (561, 0.05422114456398833, 3.3645458, 0.14167002941112922, 'entailment'),\n",
       " (562, 0.0, 3.9776273, 0.10044946560638293, 'contradiction'),\n",
       " (563, 0.09302717932019836, 5.5007157, 0.1234281973077691, 'contradiction'),\n",
       " (564, 0.02478158769198818, 3.7133155, 0.11847323228849505, 'entailment'),\n",
       " (565, 0.07784222833364549, 3.6158237, 0.14724414111576245, 'neutral'),\n",
       " (566, 0.09029927317194014, 3.3935812, 0.15895201844858522, 'neutral'),\n",
       " (567, 0.049010070228507764, 4.846154, 0.11003134553765877, 'contradiction'),\n",
       " (568, 0.05360878990951323, 3.1674485, 0.14678188178422003, 'contradiction'),\n",
       " (569, 0.10298040310783446, 2.3699286, 0.19986129767649102, 'neutral'),\n",
       " (570, 0.0, 3.2468078, 0.11773549026484523, 'neutral'),\n",
       " (571, 0.04610892881729203, 3.6623542, 0.13029642313591072, 'neutral'),\n",
       " (572, 0.07050521172961156, 3.31831, 0.1510386419325509, 'neutral'),\n",
       " (573, 0.11620138707606831, 4.4882574, 0.14920429215172643, 'contradiction'),\n",
       " (574, 0.019827424621787878, 4.3328867, 0.10367156401773542, 'neutral'),\n",
       " (575, 0.13778341782951053, 2.7540185, 0.2020823135924742, 'entailment'),\n",
       " (576, 0.13265973449410282, 3.4465854, 0.17877569997784853, 'contradiction'),\n",
       " (577, 0.01925019688389365, 2.7978141, 0.14127977736561065, 'contradiction'),\n",
       " (578, 0.0440090303370847, 4.254728, 0.11715692176010713, 'neutral'),\n",
       " (579, 0.03679778157242528, 4.173492, 0.11504541200787254, 'neutral'),\n",
       " (580, 0.022541910765330374, 2.065079, 0.17439888920715146, 'contradiction'),\n",
       " (581, 0.027018905730162617, 3.4083323, 0.12693102831449188, 'contradiction'),\n",
       " (582, 0.035297588875029755, 3.4320517, 0.13046336396681366, 'contradiction'),\n",
       " (583, 0.024122471053353398, 2.8050208, 0.14346656156183063, 'entailment'),\n",
       " (584, 0.0, 4.6162543, 0.08902730728513222, 'contradiction'),\n",
       " (585, 0.06985157508783701, 3.4385035, 0.14757636888783116, 'entailment'),\n",
       " (586, 0.023096094535335836, 3.5322862, 0.12186764791966895, 'contradiction'),\n",
       " (587, 0.0, 3.2256467, 0.11832508286810271, 'neutral'),\n",
       " (588, 0.0, 4.1948543, 0.09624909093195504, 'entailment'),\n",
       " (589, 0.06056550334723565, 3.5264492, 0.1407446120683737, 'contradiction'),\n",
       " (590, 0.04264292782823686, 3.0735087, 0.14406576912576707, 'neutral'),\n",
       " (591, 0.055296324438216554, 7.996954, 0.08322252683047632, 'neutral'),\n",
       " (592, 0.023404267811209807, 2.9313517, 0.13888485451372834, 'contradiction'),\n",
       " (593, 0.08207592439570521, 3.9647927, 0.14174710099783436, 'entailment'),\n",
       " (594, 0.04198623246169888, 2.2115333, 0.1766819888170209, 'entailment'),\n",
       " (595, 0.025615882833133846, 4.5639296, 0.10267248675377923, 'contradiction'),\n",
       " (596, 0.0, 2.2711527, 0.1528513158946777, 'entailment'),\n",
       " (597, 0.051897189828360786, 2.9333963, 0.15306520276513802, 'contradiction'),\n",
       " (598, 0.08260090926486231, 3.8469403, 0.14445831737020767, 'contradiction'),\n",
       " (599, 0.023375167459134243, 6.0869823, 0.08223947483736953, 'entailment'),\n",
       " (600, 0.05476480921377944, 2.1803641, 0.18459710684193545, 'entailment'),\n",
       " (601, 0.01976431599387946, 5.0912027, 0.09196775285869913, 'entailment'),\n",
       " (602, 0.04770814684951633, 2.1284294, 0.18367867996616355, 'neutral'),\n",
       " (603, 0.10148623039403543, 3.4438117, 0.16325913495955796, 'entailment'),\n",
       " (604, 0.024886584263784265, 2.950149, 0.13902079401713854, 'neutral'),\n",
       " (605, 0.07559107665918335, 3.2181525, 0.15633084431221206, 'entailment'),\n",
       " (606, 0.04496452597922966, 3.5126886, 0.13328095535337509, 'neutral'),\n",
       " (607, 0.020141681892709287, 2.0863056, 0.1720768318752947, 'contradiction'),\n",
       " (608, 0.0, 5.239224, 0.08013817158047082, 'entailment'),\n",
       " (609, 0.0, 3.5403142, 0.1101245372557862, 'contradiction'),\n",
       " (610, 0.09968165633407208, 3.265612, 0.16705730572008665, 'contradiction'),\n",
       " (611, 0.0, 4.564067, 0.08986232735214413, 'neutral'),\n",
       " (612, 0.0608228040346902, 2.6075401, 0.16900999881652454, 'contradiction'),\n",
       " (613, 0.08218091623202921, 4.1244917, 0.1386611109908416, 'contradiction'),\n",
       " (614, 0.02656219922403117, 2.87701, 0.14224645853435414, 'neutral'),\n",
       " (615, 0.05416601714618329, 2.7565975, 0.1601821754339662, 'contradiction'),\n",
       " (616, 0.03272470574748926, 7.58038, 0.0746348305682846, 'contradiction'),\n",
       " (617, 0.02889723366802872, 3.7049518, 0.12071963196232237, 'entailment'),\n",
       " (618, 0.18419338888998016, 2.6407819, 0.22942983236946246, 'entailment'),\n",
       " (619, 0.0, 4.759969, 0.08680601920720267, 'entailment'),\n",
       " (620, 0.0986984569601935, 2.6791048, 0.18525185330379906, 'entailment'),\n",
       " (621, 0.1446127874657096, 3.7565615, 0.17742434461770887, 'contradiction'),\n",
       " (622, 0.04415201536900835, 3.7524292, 0.12728535937046342, 'entailment'),\n",
       " (623, 0.02353684827038029, 6.5347505, 0.07812762243241693, 'contradiction'),\n",
       " (624, 0.0, 4.0606084, 0.09880234978961855, 'neutral'),\n",
       " (625, 0.02755457847151382, 3.3846474, 0.12781154510116252, 'contradiction'),\n",
       " (626, 0.06007724457280689, 4.0571237, 0.1289090540798491, 'contradiction'),\n",
       " (627, 0.03871776627289533, 2.9487123, 0.14598243932799343, 'neutral'),\n",
       " (628, 0.0, 4.426167, 0.09214607640390432, 'neutral'),\n",
       " (629, 0.01793395835281531, 3.0082114, 0.13371089942174694, 'neutral'),\n",
       " (630, 0.0, 4.5774255, 0.08964709646106439, 'entailment'),\n",
       " (631, 0.028245473590440127, 2.946871, 0.14080536580626649, 'neutral'),\n",
       " (632, 0.05073062067013769, 4.4257274, 0.11751885328750358, 'neutral'),\n",
       " (633, 0.0, 5.404379, 0.0780715832886356, 'entailment'),\n",
       " (634, 0.07188189977694716, 4.1300755, 0.13340540327010092, 'contradiction'),\n",
       " (635, 0.0, 6.7405725, 0.06459470576179359, 'neutral'),\n",
       " (636, 0.0954848126800773, 4.348364, 0.14122894008386863, 'contradiction'),\n",
       " (637, 0.04615423277551821, 4.0924516, 0.1212616534205912, 'entailment'),\n",
       " (638, 0.0, 2.5633183, 0.14031864811409883, 'neutral'),\n",
       " (639, 0.0, 4.4761443, 0.0913051174964236, 'contradiction'),\n",
       " (640, 0.04538372084347223, 4.2868714, 0.11726575091861036, 'contradiction'),\n",
       " (641, 0.0, 2.948485, 0.12663085030468302, 'entailment'),\n",
       " (642, 0.0, 3.182269, 0.119552326375516, 'contradiction'),\n",
       " (643, 0.051254851822279575, 1.7269223, 0.2089843205443481, 'entailment'),\n",
       " (644, 0.0, 2.1177666, 0.16037120835038318, 'entailment'),\n",
       " (645, 0.04888583583838117, 4.201071, 0.12057696812127291, 'neutral'),\n",
       " (646, 0.0, 1.8060818, 0.1781844011160995, 'entailment'),\n",
       " (647, 0.0, 3.6687763, 0.10709444417237289, 'neutral'),\n",
       " (648, 0.0, 4.3720164, 0.09307492010205988, 'entailment'),\n",
       " (649, 0.07508651776075112, 3.732537, 0.14319483576322906, 'entailment'),\n",
       " (650, 0.02371692494714161, 4.7617745, 0.09863728322287334, 'contradiction'),\n",
       " (651, 0.023214262884440947, 2.4204943, 0.1577848337022623, 'contradiction'),\n",
       " (652, 0.022178377392738285, 2.5970178, 0.1500932283368911, 'neutral'),\n",
       " (653, 0.0271983375673842, 4.74054, 0.1006989882422172, 'entailment'),\n",
       " (654, 0.06727497063223521, 3.1567247, 0.1539244990400604, 'entailment'),\n",
       " (655, 0.0, 3.4675071, 0.11191923955081211, 'contradiction'),\n",
       " (656, 0.027642527388683797, 3.2524803, 0.13139970461181336, 'entailment'),\n",
       " (657, 0.0, 2.861103, 0.12949667297328005, 'entailment'),\n",
       " (658, 0.02652376174561455, 2.6150806, 0.15157138376169674, 'entailment'),\n",
       " (659, 0.03580190354513404, 2.3655841, 0.16646357268346826, 'contradiction'),\n",
       " (660, 0.05893260407721872, 2.7603233, 0.16243359287076387, 'neutral'),\n",
       " (661, 0.0, 4.770601, 0.08664609071010518, 'entailment'),\n",
       " (662, 0.020042024826767656, 3.204528, 0.128940422368627, 'entailment'),\n",
       " (663, 0.04274237756448167, 3.5120726, 0.1321850094820938, 'neutral'),\n",
       " (664, 0.10154001962705136, 5.4053364, 0.1288299227262678, 'neutral'),\n",
       " (665, 0.0, 2.7584317, 0.1330342130695848, 'neutral'),\n",
       " (666, 0.14045842897685368, 2.3696876, 0.2186109239227677, 'entailment'),\n",
       " (667, 0.043864212727006115, 2.990598, 0.14722661224812422, 'contradiction'),\n",
       " (668, 0.07401302596679404, 2.6515014, 0.17393648859569638, 'neutral'),\n",
       " (669, 0.3511326437154331, 3.0322223, 0.29956742259075414, 'neutral'),\n",
       " (670, 0.019818606119988483, 2.9203134, 0.13745012806586604, 'entailment'),\n",
       " (671, 0.0673221698041297, 3.3897057, 0.1475639382957654, 'neutral'),\n",
       " (672, 0.05179779389647379, 2.1826785, 0.182999278314171, 'contradiction'),\n",
       " (673, 0.08642163147929516, 3.6656404, 0.1503772413632093, 'neutral'),\n",
       " (674, 0.15072250771503531, 2.424442, 0.2213704408388423, 'contradiction'),\n",
       " (675, 0.06973731050678803, 5.548977, 0.1112164585473609, 'neutral'),\n",
       " (676, 0.024731934837167825, 2.4707325, 0.1564277777520474, 'contradiction'),\n",
       " (677, 0.0337799847410511, 3.6825786, 0.12366876675716745, 'contradiction'),\n",
       " (678, 0.11892657488015626, 5.387671, 0.13773907848894773, 'neutral'),\n",
       " (679, 0.0, 4.1669755, 0.09676840932848145, 'contradiction'),\n",
       " (680, 0.025433563121021546, 3.6771886, 0.11961860636015065, 'entailment'),\n",
       " (681, 0.08040791672621557, 2.4015703, 0.18719489282686455, 'neutral'),\n",
       " (682, 0.05876510648386661, 3.0141256, 0.15394268205283618, 'entailment'),\n",
       " (683, 0.0, 3.5616217, 0.10961014231665073, 'contradiction'),\n",
       " (684, 0.11376381926375351, 4.6932154, 0.14470574335140593, 'contradiction'),\n",
       " (685, 0.21111526651344686, 6.3411326, 0.17366701425584974, 'neutral'),\n",
       " (686, 0.04578619123392974, 2.768975, 0.15555515822525104, 'contradiction'),\n",
       " (687, 0.08763540771651639, 5.045433, 0.1265247649281647, 'neutral'),\n",
       " (688, 0.07711774061980063, 2.6046085, 0.1772701880684773, 'contradiction'),\n",
       " (689, 0.048727588958727856, 2.4219224, 0.1704804899898515, 'entailment'),\n",
       " (690, 0.05176715238623955, 3.3603466, 0.14055336071867636, 'contradiction'),\n",
       " (691, 0.0874418889857709, 3.5507832, 0.15359214307825353, 'neutral'),\n",
       " (692, 0.029945808786679642, 3.338799, 0.13021216761447954, 'contradiction'),\n",
       " (693, 0.048081673238637115, 2.8537004, 0.15378626262026104, 'entailment'),\n",
       " (694, 0.0, 3.9717448, 0.10056831606412293, 'entailment'),\n",
       " (695, 0.10544696067837389, 4.742576, 0.13979241758143984, 'neutral'),\n",
       " (696, 0.0, 3.3614318, 0.11464125055191361, 'entailment'),\n",
       " (697, 0.1700472238143363, 1.904856, 0.257149185689312, 'entailment'),\n",
       " (698, 0.25972018282750897, 5.729397, 0.20416095572137022, 'entailment'),\n",
       " (699, 0.0235734235866471, 3.920517, 0.11340205084706792, 'entailment'),\n",
       " (700, 0.02211138881557196, 6.39895, 0.07863284971306532, 'entailment'),\n",
       " (701, 0.0435927357387854, 9.367658, 0.07002326890411015, 'entailment'),\n",
       " (702, 0.026952461316292642, 6.8236756, 0.07738481065147496, 'neutral'),\n",
       " (703, 0.0, 2.9383538, 0.1269565987008747, 'neutral'),\n",
       " (704, 0.1485619182950359, 3.9562373, 0.17516394115739808, 'neutral'),\n",
       " (705, 0.3415913720003025, 3.4774323, 0.28246683409869505, 'contradiction'),\n",
       " (706, 0.07873643271595385, 3.8603964, 0.14224048445595405, 'entailment'),\n",
       " (707, 0.0, 5.2338095, 0.08020777701268683, 'entailment'),\n",
       " (708, 0.0, 3.5620847, 0.1095990179134633, 'contradiction'),\n",
       " (709, 0.04285475435518553, 2.0151184, 0.18725834638500588, 'entailment'),\n",
       " (710, 0.038141790039322505, 7.2859545, 0.07941397335593124, 'neutral'),\n",
       " (711, 0.0708055330950035, 3.5325947, 0.145714858232982, 'contradiction'),\n",
       " (712, 0.05137899892744529, 5.288893, 0.10519474502689706, 'neutral'),\n",
       " (713, 0.04220336763445603, 3.277851, 0.1379827971982928, 'entailment'),\n",
       " (714, 0.0, 3.4036646, 0.11354179908640398, 'contradiction'),\n",
       " (715, 0.0, 6.6027665, 0.06576553404485418, 'neutral'),\n",
       " (716, 0.028998846142313357, 6.493514, 0.08122379242998355, 'neutral'),\n",
       " (717, 0.050920025337876665, 2.3434107, 0.17500789664844354, 'contradiction'),\n",
       " (718, 0.030048559349526983, 3.6742985, 0.12199220167228775, 'contradiction'),\n",
       " (719, 0.08128069752547731, 2.4560213, 0.1853153826557727, 'contradiction'),\n",
       " (720, 0.0, 3.1888392, 0.11936481122004819, 'contradiction'),\n",
       " (721, 0.02966237248831755, 3.2828884, 0.13157483021665983, 'entailment'),\n",
       " (722, 0.08321068180505997, 5.519445, 0.11829898651375159, 'entailment'),\n",
       " (723, 0.02111909527928875, 4.3131933, 0.10466491324008823, 'entailment'),\n",
       " (724, 0.03119666776192165, 3.5500958, 0.12548613017203183, 'entailment'),\n",
       " (725, 0.03931584526361033, 3.7779777, 0.12430470645345726, 'entailment'),\n",
       " (726, 0.07751302335131563, 2.9135468, 0.16651785591756924, 'entailment'),\n",
       " (727, 0.04375880758061775, 8.677853, 0.0735437573493085, 'neutral'),\n",
       " (728, 0.033724080189313166, 4.6431994, 0.10546426033831911, 'entailment'),\n",
       " (729, 0.023873280314004038, 3.8977475, 0.1140243852484075, 'contradiction'),\n",
       " (730, 0.0, 5.5370164, 0.0764874936875949, 'entailment'),\n",
       " (731, 0.0, 3.0996993, 0.1219601654746317, 'entailment'),\n",
       " (732, 0.0, 4.0514374, 0.09898172789086877, 'entailment'),\n",
       " (733, 0.033519047727562715, 3.0948763, 0.13886333474037543, 'neutral'),\n",
       " (734, 0.0, 6.350908, 0.06801880984337606, 'contradiction'),\n",
       " (735, 0.028407646288387316, 3.0889409, 0.1364848776424313, 'contradiction'),\n",
       " (736, 0.030210233724357802, 3.9719527, 0.11566922768644226, 'neutral'),\n",
       " (737, 0.04888428240078124, 7.290559, 0.08475170671426277, 'contradiction'),\n",
       " (738, 0.0, 6.477586, 0.06686650128424117, 'contradiction'),\n",
       " (739, 0.06239110257854412, 5.2603564, 0.11106320832942898, 'entailment'),\n",
       " (740, 0.0, 6.4441113, 0.06716718446026063, 'entailment'),\n",
       " (741, 0.0, 4.137628, 0.09732117474525465, 'entailment'),\n",
       " (742, 0.09196154103981445, 4.722356, 0.13335737095405037, 'contradiction'),\n",
       " (743, 0.056005685074994246, 2.895698, 0.15634954470696835, 'neutral'),\n",
       " (744, 0.0, 2.8870835, 0.12863114365468753, 'neutral'),\n",
       " (745, 0.0, 5.4030933, 0.07808725776831846, 'entailment'),\n",
       " (746, 0.0, 4.5354495, 0.09032690110562967, 'neutral'),\n",
       " (747, 0.018806304232849024, 7.631679, 0.06732930954604409, 'neutral'),\n",
       " (748, 0.19003068425224548, 3.359484, 0.2097078160286399, 'entailment'),\n",
       " (749, 0.03624775577129066, 2.9286003, 0.1453956694609969, 'entailment'),\n",
       " (750, 0.04490869297779724, 4.5939803, 0.11183614121855938, 'contradiction'),\n",
       " (751, 0.0, 2.8675892, 0.12927949934178265, 'neutral'),\n",
       " (752, 0.053650662253241034, 3.565773, 0.1363358124574028, 'neutral'),\n",
       " (753, 0.0, 2.7369962, 0.13379730048899782, 'contradiction'),\n",
       " (754, 0.02539530337294086, 3.8273633, 0.11627386375764252, 'entailment'),\n",
       " (755, 0.23794899815218706, 3.5017838, 0.2300415820155301, 'entailment'),\n",
       " (756, 0.0, 4.8702974, 0.08517455985773914, 'contradiction'),\n",
       " (757, 0.0, 4.1415286, 0.0972473437911399, 'contradiction'),\n",
       " (758, 0.022789007294103644, 7.748061, 0.0685500252769819, 'entailment'),\n",
       " (759, 0.0, 3.817179, 0.10379518877600488, 'entailment'),\n",
       " (760, 0.0, 3.0513074, 0.1234169480912513, 'entailment'),\n",
       " (761, 0.026463316452092732, 4.2143354, 0.10912115471145037, 'contradiction'),\n",
       " (762, 0.0, 6.1439705, 0.06998909090326573, 'neutral'),\n",
       " (763, 0.0, 3.6124513, 0.10840222819956329, 'entailment'),\n",
       " (764, 0.0, 2.5759344, 0.13982359368461872, 'contradiction'),\n",
       " (765, 0.06670553756373791, 3.4375863, 0.146026633660183, 'neutral'),\n",
       " (766, 0.020636734823033725, 3.1436856, 0.13098389341034875, 'neutral'),\n",
       " (767, 0.0, 4.5205927, 0.09056998552885497, 'neutral'),\n",
       " (768, 0.04228491327584552, 2.9280827, 0.1484310189234348, 'contradiction'),\n",
       " (769, 0.09665248188816478, 4.5479794, 0.13844914299889033, 'contradiction'),\n",
       " (770, 0.0, 5.7913957, 0.07362256960444481, 'neutral'),\n",
       " (771, 0.0, 4.039367, 0.09921880669953283, 'entailment'),\n",
       " (772, 0.0, 7.799442, 0.05682178603425523, 'contradiction'),\n",
       " (773, 0.17712122310579312, 3.457397, 0.20073370310913752, 'entailment'),\n",
       " (774, 0.10673689228176397, 4.6144137, 0.14242493953635868, 'contradiction'),\n",
       " (775, 0.0, 2.9466436, 0.1266899299317126, 'contradiction'),\n",
       " (776, 0.050246461061424774, 4.2419667, 0.12050727745207769, 'neutral'),\n",
       " (777, 0.040077897643101885, 4.6601214, 0.10837627607183074, 'entailment'),\n",
       " (778, 0.0, 2.983277, 0.1255247851583835, 'contradiction'),\n",
       " (779, 0.045031282551892544, 3.9053833, 0.12444447425012437, 'contradiction'),\n",
       " (780, 0.06575957251166158, 3.220503, 0.15134907513516146, 'entailment'),\n",
       " (781, 0.05544995435488148, 1.6947364, 0.21327188526941138, 'contradiction'),\n",
       " (782, 0.0, 3.9298694, 0.10142256479317414, 'contradiction'),\n",
       " (783, 0.0763202316802962, 5.047072, 0.12084476147991567, 'neutral'),\n",
       " (784, 0.0, 2.1484597, 0.158807814595584, 'entailment'),\n",
       " (785, 0.0, 2.1622298, 0.15811627723396943, 'contradiction'),\n",
       " (786, 0.1951972013796503, 4.078693, 0.19604912891819803, 'neutral'),\n",
       " (787, 0.021328030422656304, 3.1596978, 0.13086505565160464, 'contradiction'),\n",
       " (788, 0.026067575088153647, 3.217729, 0.13158099366752046, 'contradiction'),\n",
       " (789, 0.02813936523640987, 3.2105691, 0.13281847476099076, 'contradiction'),\n",
       " (790, 0.04797508878726875, 3.065946, 0.14696015333612666, 'contradiction'),\n",
       " (791, 0.020683882394884424, 3.185566, 0.12980009958121358, 'entailment'),\n",
       " (792, 0.06946468252832941, 3.3552308, 0.14953681937630933, 'neutral'),\n",
       " (793, 0.01756707612578935, 2.4585583, 0.15335244614351787, 'contradiction'),\n",
       " (794, 0.057192882664639136, 3.2212832, 0.1470438367374924, 'entailment'),\n",
       " (795, 0.040722430017377236, 3.2430315, 0.13820149024384074, 'entailment'),\n",
       " (796, 0.0, 2.9803176, 0.1256181167369546, 'neutral'),\n",
       " (797, 0.0, 3.1787646, 0.1196525887560165, 'entailment'),\n",
       " (798, 0.07504319415023361, 5.4498777, 0.11504244634336859, 'neutral'),\n",
       " (799, 0.029227205147824026, 3.030086, 0.13868043265537608, 'entailment'),\n",
       " (800, 0.03272624377629319, 3.0905619, 0.13859571884823182, 'contradiction'),\n",
       " (801, 0.0, 11.19537, 0.04099916701674068, 'neutral'),\n",
       " (802, 0.159656364248439, 4.3601556, 0.173109056589717, 'contradiction'),\n",
       " (803, 0.0, 4.346435, 0.09352026040717652, 'entailment'),\n",
       " (804, 0.03709526157146993, 4.7827377, 0.10501186678919164, 'neutral'),\n",
       " (805, 0.0, 3.5181952, 0.11066365731178886, 'contradiction'),\n",
       " (806, 0.022012695097179756, 4.832084, 0.09673899211617915, 'neutral'),\n",
       " (807, 0.05320627782038242, 5.505696, 0.10345886869608895, 'entailment'),\n",
       " (808, 0.0, 3.33571, 0.11532136475610007, 'entailment'),\n",
       " (809, 0.28765695174222206, 3.8694649, 0.24650916315611904, 'neutral'),\n",
       " (810, 0.0, 6.147833, 0.06995127181340849, 'neutral'),\n",
       " (811, 0.021726734662771686, 7.240901, 0.07153634476475716, 'neutral'),\n",
       " (812, 0.0627866789470611, 4.2468867, 0.12668794477207185, 'contradiction'),\n",
       " (813, 0.0, 3.0577004, 0.12322250320505118, 'neutral'),\n",
       " (814, 0.0, 2.53804, 0.1413211865671717, 'contradiction'),\n",
       " (815, 0.0, 3.2556162, 0.11749179857998175, 'contradiction'),\n",
       " (816, 0.14536607740526625, 2.7781053, 0.2050245077013194, 'neutral'),\n",
       " (817, 0.0, 11.217946, 0.04092340871775198, 'entailment'),\n",
       " (818, 0.0, 2.971345, 0.12590193160508587, 'neutral'),\n",
       " (819, 0.045077867687079345, 4.0834427, 0.1208974736538645, 'entailment'),\n",
       " (820, 0.02286454579233559, 5.6252437, 0.08690119530469634, 'contradiction'),\n",
       " (821, 0.0, 2.9753842, 0.12577400583064502, 'neutral'),\n",
       " (822, 0.018907562390043195, 2.3975961, 0.15661665230084762, 'contradiction'),\n",
       " (823, 0.02752315535042801, 6.6614985, 0.07902296184944535, 'entailment'),\n",
       " (824, 0.0742041176387337, 2.0681171, 0.20006845707074059, 'entailment'),\n",
       " (825, 0.04404389138308493, 2.6124072, 0.16043380550718883, 'contradiction'),\n",
       " (826, 0.059348511335818255, 4.7749586, 0.1162549627725282, 'contradiction'),\n",
       " (827, 0.018489159244294544, 7.4876575, 0.06815364814092838, 'entailment'),\n",
       " (828, 0.0286041653180897, 6.724789, 0.079028768497249, 'entailment'),\n",
       " (829, 0.04008979382802679, 3.1113455, 0.14165958398069825, 'neutral'),\n",
       " (830, 0.048396163197084924, 4.42823, 0.11630914130142084, 'contradiction'),\n",
       " (831, 0.07867419961412578, 3.023829, 0.1635968539055304, 'neutral'),\n",
       " (832, 0.0, 3.4768126, 0.11168660485773292, 'entailment'),\n",
       " (833, 0.0, 4.8864074, 0.08494145377960453, 'neutral'),\n",
       " (834, 0.05652373190831794, 2.3418186, 0.17788099977709737, 'entailment'),\n",
       " (835, 0.16276395543108307, 3.0457003, 0.2049699752147308, 'contradiction'),\n",
       " (836, 0.023458726549475006, 3.738006, 0.11725898646819577, 'neutral'),\n",
       " (837, 0.0, 4.453561, 0.09168321683078683, 'entailment'),\n",
       " (838, 0.03697260842135351, 4.346377, 0.11200758221541535, 'entailment'),\n",
       " (839, 0.0, 4.8652315, 0.0852481268315637, 'neutral'),\n",
       " (840, 0.022603753765723895, 4.057354, 0.11016780610809587, 'entailment'),\n",
       " (841, 0.11611407450411147, 2.0911748, 0.2198078361327936, 'neutral'),\n",
       " (842, 0.03230087713129819, 3.2262547, 0.1344584997927696, 'entailment'),\n",
       " (843, 0.0, 4.2114797, 0.09594204184819381, 'contradiction'),\n",
       " (844, 0.030909290127789972, 3.2130375, 0.13413386429385454, 'entailment'),\n",
       " (845, 0.0, 3.3630676, 0.11459826955493849, 'contradiction'),\n",
       " (846, 0.024120442661276185, 7.2017307, 0.07302296402151119, 'neutral'),\n",
       " (847, 0.29372610026002616, 5.1835566, 0.22772266472480585, 'contradiction'),\n",
       " (848, 0.04285330476895015, 3.2087545, 0.1402266430488435, 'contradiction'),\n",
       " (849, 0.05880479195980756, 4.004225, 0.12931797177016585, 'neutral'),\n",
       " (850, 0.033838870644276556, 3.3763125, 0.13117087432558056, 'neutral'),\n",
       " (851, 0.0561071522557966, 3.014069, 0.15261545834544515, 'neutral'),\n",
       " (852, 0.020082232043187875, 3.811364, 0.11396175197970025, 'contradiction'),\n",
       " (853, 0.0, 6.501844, 0.06665028021281033, 'neutral'),\n",
       " (854, 0.025365301714166347, 4.6137896, 0.10174904616769723, 'contradiction'),\n",
       " (855, 0.0, 4.2050223, 0.09606106714140694, 'neutral'),\n",
       " (856, 0.0, 3.2929103, 0.11647110251485482, 'contradiction'),\n",
       " (857, 0.02533076130326893, 7.843254, 0.06920565360408132, 'contradiction'),\n",
       " (858, 0.023954268645096307, 3.4686358, 0.1238681056743999, 'entailment'),\n",
       " (859, 0.07733388182637042, 2.2479737, 0.19260907505129543, 'entailment'),\n",
       " (860, 0.0, 3.123371, 0.12126001124855165, 'neutral'),\n",
       " (861, 0.08942479102458842, 2.3916376, 0.19213380791513895, 'entailment'),\n",
       " (862, 0.10928536689768531, 4.184102, 0.1510914019282183, 'entailment'),\n",
       " (863, 0.07387165591819145, 3.4213238, 0.1500241303101697, 'contradiction'),\n",
       " (864, 0.0, 3.6480515, 0.10757195783341261, 'neutral'),\n",
       " (865, 0.02112878342367466, 3.5953138, 0.11937088961942263, 'entailment'),\n",
       " (866, 0.07166717583324228, 3.7845523, 0.1403365727131618, 'neutral'),\n",
       " (867, 0.22454536696085545, 6.2659297, 0.18108700191137284, 'neutral'),\n",
       " (868, 0.10475240387680478, 4.163683, 0.14920631448405283, 'contradiction'),\n",
       " (869, 0.05169793160662773, 3.4418843, 0.13841380733590328, 'contradiction'),\n",
       " (870, 0.04737919901265562, 2.9439888, 0.15046480738567317, 'entailment'),\n",
       " (871, 0.165850661452058, 7.8834763, 0.13920960340450458, 'entailment'),\n",
       " (872, 0.03044970639123835, 3.6106005, 0.12367059753869863, 'neutral'),\n",
       " (873, 0.05559517142095038, 3.2412488, 0.1456873908477484, 'entailment'),\n",
       " (874, 0.06476647274220816, 3.2059834, 0.1512614991714292, 'neutral'),\n",
       " (875, 0.1825442881016764, 3.2336686, 0.20937302800877028, 'contradiction'),\n",
       " (876, 0.0, 4.43193, 0.0920483132151168, 'entailment'),\n",
       " (877, 0.035318845702661965, 2.6379828, 0.15509822376406296, 'contradiction'),\n",
       " (878, 0.37232723081343044, 2.745886, 0.31964338183087193, 'entailment'),\n",
       " (879, 0.0, 4.8751464, 0.08510426241216777, 'neutral'),\n",
       " (880, 0.0, 2.5149078, 0.14225124048742582, 'neutral'),\n",
       " (881, 0.0719919713637036, 2.742131, 0.16960969420654673, 'entailment'),\n",
       " (882, 0.024517626212669896, 3.6281843, 0.12029253985243422, 'contradiction'),\n",
       " (883, 0.06701836485820858, 4.535145, 0.12384105585476443, 'entailment'),\n",
       " (884, 0.0239293380304113, 3.6124113, 0.12036783858305365, 'contradiction'),\n",
       " (885, 0.0, 6.0216613, 0.07120821981419839, 'entailment'),\n",
       " (886, 0.0, 2.773034, 0.13251934313589445, 'contradiction'),\n",
       " (887, 0.0, 5.1245446, 0.08163872271013752, 'contradiction'),\n",
       " (888, 0.0, 3.3819203, 0.11410522361642618, 'neutral'),\n",
       " (889, 0.10698479943297172, 3.3588915, 0.16820046287365745, 'neutral'),\n",
       " (890, 0.0, 4.9767375, 0.0836576811452445, 'neutral'),\n",
       " (891, 0.14745592417693715, 3.4625468, 0.18577160433174775, 'contradiction'),\n",
       " (892, 0.03984871155594891, 3.1985316, 0.139013609913059, 'entailment'),\n",
       " (893, 0.04944580772368207, 6.7143054, 0.08953755327214465, 'neutral'),\n",
       " (894, 0.0, 9.216553, 0.048940186871221356, 'neutral'),\n",
       " (895, 0.08196142595207703, 3.8212645, 0.14468794568260257, 'entailment'),\n",
       " (896, 0.04670809080909102, 6.7572794, 0.08780963279181173, 'entailment'),\n",
       " (897, 0.0, 6.1318345, 0.07010818878509409, 'neutral'),\n",
       " (898, 0.02681600368799676, 2.215904, 0.16888527989736526, 'neutral'),\n",
       " (899, 0.027220873886064204, 3.157682, 0.13386975619272798, 'entailment'),\n",
       " (900, 0.05133694252078158, 3.9471438, 0.12673688990970472, 'neutral'),\n",
       " (901, 0.04704320137388667, 2.763352, 0.1563818834858817, 'neutral'),\n",
       " (902, 0.0, 3.565654, 0.10951333493231057, 'contradiction'),\n",
       " (903, 0.07073270070817396, 5.7764254, 0.10915156502359327, 'entailment'),\n",
       " (904, 0.0, 4.8309684, 0.0857490501435712, 'contradiction'),\n",
       " (905, 0.10018953499809441, 4.4724073, 0.14146223502172622, 'entailment'),\n",
       " (906, 0.03459067652285915, 2.258014, 0.1707630660155091, 'contradiction'),\n",
       " (907, 0.0, 5.9535084, 0.07190614764317167, 'neutral'),\n",
       " (908, 0.2813309697860038, 4.2323127, 0.23622552318715329, 'contradiction'),\n",
       " (909, 0.03074404952366103, 3.991045, 0.11555144614220957, 'neutral'),\n",
       " (910, 0.025817596536058158, 2.650829, 0.1498639910437973, 'neutral'),\n",
       " (911, 0.0, 8.989457, 0.05005276998254366, 'contradiction'),\n",
       " (912, 0.07625589684080458, 3.9992151, 0.13814364836414575, 'neutral'),\n",
       " (913, 0.3852066660262905, 1.9953, 0.35953151746680234, 'contradiction'),\n",
       " (914, 0.04862921693729962, 2.629995, 0.1620558409419871, 'contradiction'),\n",
       " (915, 0.10365575240854058, 3.1644433, 0.1718919446636531, 'neutral'),\n",
       " (916, 0.108303204124922, 4.3976865, 0.14678388104271228, 'entailment'),\n",
       " (917, 0.16717450249672022, 4.843079, 0.169158572856103, 'neutral'),\n",
       " (918, 0.05495604591662567, 3.251596, 0.1450809191557359, 'entailment'),\n",
       " (919, 0.07062541389374184, 6.3907948, 0.1029644300213721, 'neutral'),\n",
       " (920, 0.021689925392861666, 3.8188088, 0.11460504560744374, 'neutral'),\n",
       " (921, 0.07497819329311188, 5.215743, 0.11793000206702847, 'neutral'),\n",
       " (922, 0.0, 4.054423, 0.09892326271595384, 'contradiction'),\n",
       " (923, 0.0491177188166244, 8.476268, 0.07732224788365086, 'entailment'),\n",
       " (924, 0.028424824476740256, 3.8014178, 0.11834831916261304, 'contradiction'),\n",
       " (925, 0.11093797228163996, 4.952731, 0.13946404468958348, 'entailment'),\n",
       " (926, 0.0, 4.6521983, 0.08846115655612481, 'neutral'),\n",
       " (927, 0.059341571354429225, 3.3396652, 0.14488704773812866, 'neutral'),\n",
       " (928, 0.027693867676361818, 10.698078, 0.056588997393651974, 'neutral'),\n",
       " (929, 0.0, 4.6644535, 0.08826976855382761, 'neutral'),\n",
       " (930, 0.08528918079683037, 2.4552417, 0.18735226826452012, 'entailment'),\n",
       " (931, 0.0, 5.1474757, 0.08133419680176938, 'entailment'),\n",
       " (932, 0.0, 4.4357915, 0.09198292478535144, 'contradiction'),\n",
       " (933, 0.0, 2.8570352, 0.1296332491798216, 'contradiction'),\n",
       " (934, 0.16859834022468964, 2.1605198, 0.24250099309590245, 'neutral'),\n",
       " (935, 0.0, 3.8299112, 0.10352157130505127, 'contradiction'),\n",
       " (936, 0.026361966841688452, 2.2809162, 0.16557743837086297, 'contradiction'),\n",
       " (937, 0.0, 4.2796392, 0.0947034403081012, 'neutral'),\n",
       " (938, 0.06210582997656568, 3.6476958, 0.13863310605265397, 'contradiction'),\n",
       " (939, 0.06980657622327818, 2.982851, 0.16044150092185822, 'neutral'),\n",
       " (940, 0.0, 3.0223558, 0.12430526425164437, 'contradiction'),\n",
       " (941, 0.31813167581850477, 3.0775175, 0.28168946830199143, 'neutral'),\n",
       " (942, 0.0370012875087683, 2.6519325, 0.155414456651749, 'neutral'),\n",
       " (943, 0.07699491861659664, 4.165741, 0.13528899477509418, 'neutral'),\n",
       " (944, 0.07228312270899509, 2.6168325, 0.17438407071254702, 'entailment'),\n",
       " (945, 0.04308416582834797, 2.7586086, 0.15457003445694215, 'neutral'),\n",
       " (946, 0.0, 6.296092, 0.06852983730359308, 'contradiction'),\n",
       " (947, 0.09330567502019192, 7.2198296, 0.10748134938996856, 'entailment'),\n",
       " (948, 0.0675411386901665, 1.4267747, 0.23980534946433504, 'neutral'),\n",
       " (949, 0.10291473559449464, 4.144576, 0.14864710567554845, 'contradiction'),\n",
       " (950, 0.020057253394831817, 6.314953, 0.07838176720694592, 'neutral'),\n",
       " (951, 0.06431805196328183, 3.376866, 0.14639601389099244, 'neutral'),\n",
       " (952, 0.04099735459193963, 3.753391, 0.12568674137749153, 'neutral'),\n",
       " (953, 0.028567505094616964, 3.5389264, 0.12444196178304505, 'contradiction'),\n",
       " (954, 0.01888739762480105, 4.775864, 0.09601083218670957, 'neutral'),\n",
       " (955, 0.08095109869950615, 3.6690347, 0.14756406550819806, 'contradiction'),\n",
       " (956, 0.0, 3.939049, 0.10123406336773738, 'entailment'),\n",
       " (957, 0.05714128194934031, 4.157123, 0.12552392120222985, 'contradiction'),\n",
       " (958, 0.02339857745820038, 3.5891833, 0.12065113571090598, 'contradiction'),\n",
       " (959, 0.18861248080075255, 3.9540606, 0.19523354932181924, 'neutral'),\n",
       " (960, 0.0, 7.616532, 0.05802798721894091, 'entailment'),\n",
       " (961, 0.0, 6.8527765, 0.06367174696173887, 'contradiction'),\n",
       " (962, 0.03986516195737243, 3.1180973, 0.14134787618231334, 'contradiction'),\n",
       " (963, 0.0, 2.8945353, 0.1283850218535629, 'neutral'),\n",
       " (964, 0.0, 5.463336, 0.07735943183265191, 'neutral'),\n",
       " (965, 0.024474855471597043, 3.0392416, 0.13602304281215963, 'neutral'),\n",
       " (966, 0.0, 4.299221, 0.09435349013323889, 'neutral'),\n",
       " (967, 0.05443742734204632, 2.668993, 0.16349588850229085, 'contradiction'),\n",
       " (968, 0.032234206865410266, 5.2466736, 0.09615970423098155, 'neutral'),\n",
       " (969, 0.08773846283276626, 3.0348377, 0.16778995248461698, 'neutral'),\n",
       " (970, 0.0, 4.630949, 0.08879497899729663, 'entailment'),\n",
       " (971, 0.028691316414548047, 2.9572022, 0.1406975541225476, 'neutral'),\n",
       " (972, 0.0, 2.4726732, 0.1439813004022465, 'neutral'),\n",
       " (973, 0.0, 3.944244, 0.10112769703739988, 'contradiction'),\n",
       " (974, 0.05654090822410323, 6.433384, 0.09553457014922054, 'entailment'),\n",
       " (975, 0.0, 5.1395893, 0.0814386719989015, 'entailment'),\n",
       " (976, 0.0, 4.750376, 0.08695083251564566, 'entailment'),\n",
       " (977, 0.05860713875728374, 2.277565, 0.18185584511613537, 'neutral'),\n",
       " (978, 0.029151837657863663, 3.6840734, 0.12132061562572359, 'neutral'),\n",
       " (979, 0.03597741291433592, 9.022187, 0.06787801588406532, 'neutral'),\n",
       " (980, 0.07105915873215644, 3.2381651, 0.15350516159694674, 'contradiction'),\n",
       " (981, 0.10571407200203123, 2.9842532, 0.17835106936277506, 'entailment'),\n",
       " (982, 0.0, 2.4721935, 0.14400119202646902, 'neutral'),\n",
       " (983, 0.030716206486084457, 2.779728, 0.14764275666092397, 'contradiction'),\n",
       " (984, 0.0246534024780361, 2.6726658, 0.14846759249928299, 'entailment'),\n",
       " (985, 0.0, 4.888836, 0.0849064242743031, 'contradiction'),\n",
       " (986, 0.09553807650422803, 7.4935822, 0.10663701472661627, 'entailment'),\n",
       " (987, 0.05989997490739087, 2.998701, 0.15499059140194943, 'contradiction'),\n",
       " (988, 0.029552409167610726, 3.6766744, 0.12168978480554346, 'neutral'),\n",
       " (989, 0.044368378182128534, 2.502663, 0.16493272490150304, 'neutral'),\n",
       " (990, 0.04711620600929175, 3.1247401, 0.14477786109829613, 'neutral'),\n",
       " (991, 0.033306523747428686, 3.6517594, 0.12413947483177833, 'entailment'),\n",
       " (992, 0.0572390250398569, 3.5607746, 0.13825001341449975, 'neutral'),\n",
       " (993, 0.027443951706443877, 3.1107593, 0.13535400745275455, 'entailment'),\n",
       " (994, 0.03681193488604347, 3.421833, 0.1314812454252105, 'entailment'),\n",
       " (995, 0.025418980713924354, 4.6351476, 0.10143831132397953, 'contradiction'),\n",
       " (996, 0.019272760705014002, 5.049312, 0.09229040621830724, 'neutral'),\n",
       " (997, 0.1428150700455241, 3.2696464, 0.18851325114881495, 'contradiction')]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a05cd8a8-5c1c-4302-bbcc-9e148e11735e",
   "metadata": {},
   "source": [
    "#### Rank and retrieve the top 10 related reviews\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "55387caa-9667-4a87-9181-3ecb41f6f8de",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_10_reviews = sorted(similarity_scores, key=lambda x: x[3], reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5f6016f2-9d48-40c8-9951-79c80a9d7e6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_pairs = snli_df[['sentence1', 'sentence2']].values.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81586e2b-255e-433a-a580-32894cd3fe25",
   "metadata": {},
   "source": [
    "#### Display the similarity report with labels\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a21f794c-3f89-4b83-ab65-0e3f01743f50",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search query: A person is walking on a beach.\n",
      "\n",
      "Sentence 1: A person on a horse jumps over a broken down airplane.\n",
      "Sentence 2: A person is at a diner, ordering an omelette.\n",
      "Label: contradiction\n",
      "Cosine Similarity: 0.3852066660262905\n",
      "Euclidean Distance: 1.995300054550171\n",
      "Weighted Similarity: 0.35953151746680234\n",
      "\n",
      "Sentence 1: A man, woman, and child enjoying themselves on a beach.\n",
      "Sentence 2: A family of three is at the beach.\n",
      "Label: entailment\n",
      "Cosine Similarity: 0.37232723081343044\n",
      "Euclidean Distance: 2.7458860874176025\n",
      "Weighted Similarity: 0.31964338183087193\n",
      "\n",
      "Sentence 1: A small white dog running on a pebble covered beach.\n",
      "Sentence 2: A dog on the beach.\n",
      "Label: entailment\n",
      "Cosine Similarity: 0.4363744143304071\n",
      "Euclidean Distance: 4.857994079589844\n",
      "Weighted Similarity: 0.3035406563504933\n",
      "\n",
      "Sentence 1: A woman on the side of a street is making food on her cart.\n",
      "Sentence 2: A person is cooking.\n",
      "Label: entailment\n",
      "Cosine Similarity: 0.30726884434619123\n",
      "Euclidean Distance: 2.342668056488037\n",
      "Weighted Similarity: 0.3032155326963234\n",
      "\n",
      "Sentence 1: A dog zips along the beach.\n",
      "Sentence 2: A dog is chasing a ball on the beach.\n",
      "Label: neutral\n",
      "Cosine Similarity: 0.3511326437154331\n",
      "Euclidean Distance: 3.032222270965576\n",
      "Weighted Similarity: 0.29956742259075414\n",
      "\n",
      "Sentence 1: A couple playing with a little boy on the beach.\n",
      "Sentence 2: A couple are playing frisbee with a young child at the beach.\n",
      "Label: neutral\n",
      "Cosine Similarity: 0.3233171611331862\n",
      "Euclidean Distance: 2.909592628479004\n",
      "Weighted Similarity: 0.28954914296376055\n",
      "\n",
      "Sentence 1: A person on a horse jumps over a broken down airplane.\n",
      "Sentence 2: A person is training his horse for a competition.\n",
      "Label: neutral\n",
      "Cosine Similarity: 0.28609491196080844\n",
      "Euclidean Distance: 2.4251067638397217\n",
      "Weighted Similarity: 0.28902830693626586\n",
      "\n",
      "Sentence 1: A woman on the side of a street is making food on her cart.\n",
      "Sentence 2: A person is cooking mexican food.\n",
      "Label: neutral\n",
      "Cosine Similarity: 0.25056857715673625\n",
      "Euclidean Distance: 2.0602481365203857\n",
      "Weighted Similarity: 0.2886697324033641\n",
      "\n",
      "Sentence 1: A person on skis on a rail at night.\n",
      "Sentence 2: The person skiis\n",
      "Label: entailment\n",
      "Cosine Similarity: 0.3579002523158624\n",
      "Euclidean Distance: 3.7158780097961426\n",
      "Weighted Similarity: 0.28497492132043667\n",
      "\n",
      "Sentence 1: A dog drops a red disc on a beach.\n",
      "Sentence 2: a dog catch the ball on a beach\n",
      "Label: contradiction\n",
      "Cosine Similarity: 0.3415913720003025\n",
      "Euclidean Distance: 3.4774322509765625\n",
      "Weighted Similarity: 0.28246683409869505\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"Search query: {search_query}\")\n",
    "print()\n",
    "for index, cosine_sim, euclidean_dist, weighted_sim, label in top_10_reviews:\n",
    "    print(f\"Sentence 1: {sentence_pairs[index][0]}\")\n",
    "    print(f\"Sentence 2: {sentence_pairs[index][1]}\")\n",
    "    print(f\"Label: {label}\")\n",
    "    print(f\"Cosine Similarity: {cosine_sim}\")\n",
    "    print(f\"Euclidean Distance: {euclidean_dist}\")\n",
    "    print(f\"Weighted Similarity: {weighted_sim}\")\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca0918da-8467-4f2c-8f12-4c91ae261aa6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}