model_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d73d1c17",
   "metadata": {},
   "source": [
    "# Summary source prediction: Model selection\n",
    "\n",
    "Sébastien Meyer\n",
    "\n",
    "This notebook gathers the main preprocessing functions for data preparation which can be found in our package. Then, you can try out some models or feature selection. Enjoy!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4bc6fd8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import nltk\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split\n",
    "\n",
    "from catboost import CatBoostClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier, cv\n",
    "\n",
    "from mlxtend.classifier import StackingCVClassifier\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "from src.preprocessing.reader import check_features_names\n",
    "\n",
    "from src.preprocessing.features.gltr import create_gltr_feat\n",
    "from src.preprocessing.features.polynomial import create_poly_feat\n",
    "from src.preprocessing.features.manual_regex import create_regex_feat\n",
    "from src.preprocessing.features.pos_tagging import ALL_NLTK_TAGS, create_tagging_feat\n",
    "from src.preprocessing.features.tfidf import create_idf_feat\n",
    "from src.preprocessing.reduction.droping import drop_corr_feat, drop_list_feat\n",
    "from src.preprocessing.reduction.pca import perform_pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b8ccb9f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/sebastien/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/sebastien/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# Seed to use everywhere for reproducibility\n",
    "seed = 8005\n",
    "\n",
    "# Package options and global variables\n",
    "warnings.simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n",
    "\n",
    "np.random.seed(seed)\n",
    "\n",
    "nltk.download(\"averaged_perceptron_tagger\")\n",
    "nltk.download(\"stopwords\")\n",
    "\n",
    "# Display options\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "\n",
    "font = {\"size\": 18}\n",
    "mpl.rc(\"font\", **font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1359bb47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_rep(df, feature, save=False):\n",
    "    \n",
    "    plt.figure(figsize=(10, 8))\n",
    "\n",
    "    color = iter(plt.cm.brg(np.linspace(0, 1, 2)))\n",
    "\n",
    "    sns.kdeplot(df.loc[df[\"label\"] == 0, feature], label=\"label = 0\", color=next(color))\n",
    "    sns.kdeplot(df.loc[df[\"label\"] == 1, feature], label=\"label = 1\", color=next(color))\n",
    "\n",
    "    plt.legend(loc=\"best\")\n",
    "    \n",
    "    if save:\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"report/figures/{feature}_repart.png\", facecolor=\"white\")\n",
    "    else:\n",
    "        plt.title(f\"Repartition of {feature} among label\")\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7d8a8a6",
   "metadata": {},
   "source": [
    "## Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5cbe7fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read files\n",
    "train_df = pd.read_json(\"data/train_set.json\")\n",
    "test_df = pd.read_json(\"data/test_set.json\")\n",
    "documents = pd.read_json(\"data/documents.json\")\n",
    "\n",
    "# Extract target\n",
    "label_var = [\"label\"]\n",
    "y_train = train_df[label_var]\n",
    "train_df.drop(columns=label_var, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49a87e32",
   "metadata": {},
   "source": [
    "### Regex features\n",
    "\n",
    "List of all regex features to compute. The name of a regex feature should be of the form \"A_B\". A is the name of the regex expression, for example \"upper_word\". B is the type of feature we want, it can be \"count\" for the number of instances, \"avg\" for the average length of instances, \"overlap\" for the number of instances both found in the summary and the original document or \"ratio\" for the ratio between the number of instances found in the summary and the number of instances found in the document. Example: --regex-feat char_count group_overlap."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "89e83eb6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing regex features...\n",
      "Number of regex features: 11.\n"
     ]
    }
   ],
   "source": [
    "regex_feat = [\n",
    "    \"char_count\", \"char_ratio\",\n",
    "    \"word_count\", \"word_overlap\", \"word_ratio\",\n",
    "    \"sent_avg\",\n",
    "    \"upper_word_ratio\",\n",
    "    \"group_overlap\",\n",
    "    \"is_last_ponct_count\",\n",
    "    \"space_before_ponct_count\",\n",
    "    \"word_3_count\"\n",
    "]\n",
    "\n",
    "create_regex_feat(train_df, test_df=test_df, regex_feat=regex_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "377cc260",
   "metadata": {},
   "source": [
    "### Tf-idf features\n",
    "\n",
    "List of all tf-idf features names to compute. The name of an idf feature is expected to be of the form \"A_B\". A is a sequence of characters for the composition performed in `src.preprocessing.features.tfidf.idf_composition()`. B is the type of feature we want, it can be \"count\" for the term frequencies, \"lda\" for the latent dirichlet allocation, \"idf\" for idf features after a PCA transformation that keeps **svd_components** components from the initial counter features or \"cos\" for the cosine similarity between the summary and the original document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "51860253",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing idf features...\n",
      "Number of idf features: 5.\n"
     ]
    }
   ],
   "source": [
    "idf_feat = [\n",
    "    \"n_count\", \"n_idf\", \"n_lda\"\n",
    "]\n",
    "\n",
    "create_idf_feat(seed, train_df, test_df=test_df, idf_feat=idf_feat, more_docs=documents, svd_components=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca882c80",
   "metadata": {},
   "source": [
    "### Tagging features\n",
    "\n",
    "List of all tagging features to compute. The name of a tagging feature must be of the form \"A_B\". A is a tag from all available tags in `nltk` or \"tags\". B is the type of feature we want, it can be \"count\" for the number of instances, \"avg\" for the average length of instances, \"overlap\" for the number of instances both found in the summary and the original document, \"ratio\" for the ratio between the number of instances found in the summary and the number of instances found in the document or \"C_D\" where C and D are parameters for idf features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2fbec7b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing tagging features...\n",
      "Number of tagging features: 7.\n"
     ]
    }
   ],
   "source": [
    "tag_feat = [\n",
    "    \"DT_count\", \"DT_overlap\",\n",
    "    \"MD_count\",\n",
    "    \"NN_overlap\",\n",
    "    \"RB_count\",\n",
    "    \"$_overlap\",\n",
    "    \":_overlap\"\n",
    "]\n",
    "\n",
    "create_tagging_feat(seed, train_df, test_df=test_df, tag_feat=tag_feat, more_docs=documents, svd_components=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed9ae203",
   "metadata": {},
   "source": [
    "### GLTR features\n",
    "\n",
    "List of all GLTR features to compute. The name of a GLTR feature should be of the form \"A_B\". A is the name of a topk value computed by GLTR, it can be \"count\" or \"frac\". B is the number of bins to compute the feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eaf680ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing GLTR features...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5689957e0714a82b050a5397a10ffbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/0.99M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a93c2fd89034426b97fd13bec592725e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b0e65d86bab4b01915918ed66cd7586",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/665 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4dabbf0bccd44d1893c5b936bc442dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/523M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded GPT-2 model!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8000/8000 [05:13<00:00, 25.52it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3200/3200 [02:01<00:00, 26.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of GLTR features: 14.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gltr_feat = [\n",
    "    \"count_4\", \"frac_10\"\n",
    "]\n",
    "\n",
    "train_df, test_df = create_gltr_feat(train_df, test_df=test_df, gltr_feat=gltr_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72367464",
   "metadata": {},
   "source": [
    "### Polynomial features\n",
    "\n",
    "- List of specific polynomial features. A fixed polynomial feature must be of the form \"A B C\". A, B and C can be features or powers of features, and their product will be computed. Example: --fixed-poly-feat \"char_count^2 group_overlap\".\n",
    "- List of polynomial features to compute of which interaction terms will be computed.\n",
    "- Define the degree until which products and powers of features are computed. If 1 or less, there will be no polynomial features. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dc554625",
   "metadata": {},
   "outputs": [],
   "source": [
    "fixed_poly_feat = []\n",
    "poly_feat = []\n",
    "poly_degree = 2\n",
    "\n",
    "create_poly_feat(\n",
    "    train_df, test_df=test_df, fixed_poly_feat=fixed_poly_feat, \n",
    "    poly_feat=poly_feat, poly_degree=poly_degree\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d35f077",
   "metadata": {},
   "source": [
    "### Drop excluded features\n",
    "\n",
    "List of features names to drop after computation. Example: --excl-feat \"char_count^2 group_overlap\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8615f15a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Dropping specific features...\n",
      "Number of features after dropping excluded ones: 27\n"
     ]
    }
   ],
   "source": [
    "excl_feat = [\n",
    "    \"document\", \"summary\",\n",
    "    \"n_lda_2\", \"n_lda_3\"\n",
    "] + [\n",
    "    f\"topk_frac_{i}\" for i in range(1, 10)\n",
    "]\n",
    "\n",
    "drop_list_feat(train_df, test_df=test_df, excl_feat=excl_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d68810c4",
   "metadata": {},
   "source": [
    "### Drop correlated features\n",
    "\n",
    "Correlation threshold to select features. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ecdf4179",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_threshold = 1.\n",
    "\n",
    "drop_corr_feat(train_df, test_df=test_df, corr_threshold=corr_threshold)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d57e780",
   "metadata": {},
   "source": [
    "### Rescale the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "565311a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rescale the data\n",
    "sc = StandardScaler()\n",
    "\n",
    "train_df[train_df.columns] = sc.fit_transform(train_df[train_df.columns])\n",
    "test_df[test_df.columns] = sc.transform(test_df[test_df.columns])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ae74a78",
   "metadata": {},
   "source": [
    "### PCA\n",
    "\n",
    "Variance ratio parameter for the Principal Component Analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "be2cdb54",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_ratio = 1.\n",
    "\n",
    "train_df, test_df = perform_pca(seed, train_df, test_df=test_df, pca_ratio=pca_ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b825ad10",
   "metadata": {},
   "source": [
    "### Final df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "12368f3d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final training shape: (8000, 27)\n",
      "Features names: ['char_count', 'char_ratio', 'word_count', 'word_overlap', 'word_ratio', 'sent_avg', 'upper_word_ratio', 'group_overlap', 'is_last_ponct_count', 'space_before_ponct_count', 'word_3_count', 'n_count_PCA_1', 'n_idf_PCA_1', 'n_lda_0', 'n_lda_1', 'DT_count', 'DT_overlap', 'MD_count', 'NN_overlap', 'RB_count', '$_overlap', ':_overlap', 'topk_count_0', 'topk_count_1', 'topk_count_2', 'topk_count_3', 'topk_frac_0']\n"
     ]
    }
   ],
   "source": [
    "# Print some information\n",
    "print(f\"Final training shape: {train_df.shape}\")\n",
    "print(f\"Features names: {list(train_df.columns)}\")\n",
    "\n",
    "# Re-create the label variable\n",
    "train_df = train_df.merge(y_train, left_index=True, right_index=True, how=\"left\")\n",
    "\n",
    "# Train test split\n",
    "X = train_df.drop(columns=label_var).to_numpy()\n",
    "y = train_df[label_var].to_numpy().flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebcbaa81",
   "metadata": {},
   "source": [
    "## Show some distribution of labels wrt a feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9737cd14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoIAAAIGCAYAAADA0e7bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABxMklEQVR4nO3deZwcVbn/8c8zM5mZTPaNJCxJkCCbrAZuAGUTlAsXBUFFBQQFFAEFrz+uG4gLgnBZBBEEkSjbBSQEcEHZgooCSdiXBEIIJCGQBEL2yWzn98epytT0VHdX93R3zUx/37zm1UzVqarTVdU9T56zlDnnEBEREZHqU5N2BUREREQkHQoERURERKqUAkERERGRKqVAUERERKRKKRAUERERqVIKBEVERESqVL8NBM1soZm5jJ/1Zvaqmf3GzLZPu45pMbNJwflYWMS204JtTyx9zdJjZqPN7LdmtsTM2oL3eH7a9eovKn3f6HqKSCmY2czg++OAEuzr/Ep9F5nZicGxpuUr228DwYi/Ar8Lfh4FRgJfAZ42s/3TrFi5FPtHtycBYj/wG+AkYD1wB/5+eSbNCkmP6Hr2cmZ2QPB9MzPtuvRGOj9SKXVpV6ACLnLOzQx/MbMRwD3AR4FfA9WYGVwC7AC0FrHtd4GLgKUlrVGKzKweOBzYAOzunFubcpWkB3Q9RUSSq4aMYBfOuZXAOcGv25nZB9KsTxqcc63OubnOudeK2HZpsO2qctQtJePw/yhapqChX9D1FBFJqOoCwcALkf8fm7nSzAab2ffM7CkzWxP0LXzGzL4dZBsyy29qijWzPczsPjN718zWmdnjZvbZuEqY2X+Y2aVmNsfMlpnZRjNbZGY3m9mHsmyTeawZwbYdZnakmTngS0HxGzP6SJ4Y7KNbE3DQZ+H14NeJGdstjDt+TN3qzewsM5sdOW/Pmdm5ZjY4pvymPgxmNszMfhG8/41m9pqZ/dDMCs5am9mQYNvngzqsMbNZZvYNMxuQUdYBb8S974THmmxmvzazecH1Xh3U/XYz+1hG2U19TczsY2b2sJmtCur3iJkdlOUYh5jZr4Jz+Z6ZNZvZAjO71swm5qhbjZl90cweMLMVkfvrz2b2xSzljwvq9V5QfkFwXbp9TopVyGckKF9vZmeY2b/M7P3g/b9sZj8xsyEZZRNdTzPbLPjsvRLs730z+7uZnWBmFlOHnJ+7SLkxZnaRmb0YufceN7OT4/Zb4HkL34uZ2Wlm9mxwjOVmdpuZbZNj249G6txivu9kru+ZTefMzI43/5leH9wXf8hzrElmdlXwmVgfnNvng/M9MSgzDXgk2GR/6/p9M7MH52iImX3X/Od9VXD8+Wb2ezPbJ6b81mZ2nfk+5RuDe/KvZvZfWfYf9j2flGV97HdHoeezFOfHev73ZZfgnnnX/PfaQ2Y2JVL2pGDf64L9/9rMhmXZb02wz39Y52d4npldYmajY8pvahY3swYz+1FwHTea2WIzu8LMBmU5VoP5vznhZ3ux+e/PUVbCvspmtpP576B/m9nS4HP1tpndbWb7Jtj+A8HndllQz2fN7Gtm8d8TVs7vZ+dcv/wBFgIOOCBm3ebBOgdsk7FuK2BusG4p8Cfgj8CKYNkjQH3GNtOCddcCzcArwG34PontwbrvxdTjQXzz7DP45urpwLyg/Hpgv5htwmP9BtgYlL8NeADfHDYNmB+U+Wfwe/jzkWAfk4L1CyP7PRL4Q7B8bcZ2/xtz/BMz6jUQ+HuwbnXwfv4QOW/PAaMztjkxWDcDeAl4G7gzeC/NwbrrCrzumwEvBtsuD+pwD7Amcv0aM95P7PtOcKxdIvt9MdjPXcCTQAtwbUb5mUHZK4P74hngVuCJYHkHcFzMcebjmzlnB/u/Fx/sOOBdYLuYbRrw964L6jIzONZM4L3otQ/KDwiugwve0yPBsV4Lli0GPtCDz2N43xT6GRkO/CvyXv8W1HNJsOwFYGQh1xP4YGT7RcDtwF8i99wtgBXyuQvK7Ir/znD4758ZQX1Xh/vt4Xda+J11BdAGPBzUIbxG7wEfitnuzODecsG5vDW491zwnj+Z41g/C+6fB4PzujhY/hYwKma7/6TzM/FGsM3dwPNEvjeAk4H7g2Vv0/X75jtFnp+tgVeDfb6P/96+HXg8uGbTMsrvA6wKyof34yPBuXXAhTHHWBism5TrGvX0fJbi/NCzvy9XB2WeAf4P/zkLvxu2By4L7p37g+v7brD+oZh9WnAdwvvtL8Hv4Xt/E5icsc0BdN6vM4GV+M/Tn+j8PP015lh1+M9k+Pm/NzjPy/HfozOI+fuV5zzOJCaWwH8XdATn5k/4v13PBmXbgGNj9nV+sP73+M/rouD8/jW4L2L/5lHE9zOdf1+n5X2PPfli6s0/5A4ETw7WvUzkCz+4YR8P1v0v0BBZN5zOD+aPs3x4HHApUBNZ97Hg5m/H91eKbncosFnS+sUc67zM9RllYm92YgLBXMuT7Ds4Xw7/xbFZZPlQ/B8sB9ye5UZ1+C+paID2H8GHqYMsX7pZ6hcGAX8FhkSWj6fzy+znhb7vLMe6Mdjuf2LWjQT2yFg2M/J+v5mx7jg6v7w2z1j3KWBYxrJa4EfBNvfHHP8qOgPwrTPWNQD/mbHs4qD8A8C4yPIa4IJg3d978HmM3reFfEbuoDM4GxpZ3hjZ5+8LuZ7ArGD9NCL/qAO2ozNAPK2Qzx3QhM+oO+DsjPe3BTAnWPflHpzD8PhrgX0y7oVrg3VPZWyzG/5z1AL8V8a6M4JtVgFjsxxrGZHgEhhM53fkeRnbTKQzCPxW9BwE67cHdoj8fkBQdmax5yTjPn0m2N+tRD77wfrRBP8Qjtw/i4LyF9D178A+kfeR+TlZSM8CwULOZ4/ODz3/+/KNyHIDbqLzH19LgQ9G1m+JD7YcsH+W+6xLwIf/HrolWPdElvfu8MHgiMi6bfCBviMjmA3uO4cPeLeILB+CD57CfZ5YwHmcSXwguD8wMab8YfjP23tAU8a68yN1+D+6xhi70BlQfzJju4K/n1Eg2OUDe0Bk2Wb4ZtOV+H9V7BtzAV1w4eMCrPH4f1muoOsXR/jhWURGtjBYf02w/oYC6v9YsM1OGcvDY71ExhdtTJnYm50SB4L4bODaYPm+MdtMxv8xagcmxNyoq4ExMdv9MVj/pYTnbCI+cGwh5ouazi+XNXQNOvO+7yzHCzNuuyUsP5OYL73I+r8E688toA6Lg/MaDXrHBueglQRZPGAUPuP4HvFZnugf2V0KOUc9+YwAO9GZrWmI2aYJny1ppWtWMOv1BPYL1r1LRrCQcU/Oz1L/2M8d8PVg/e+yvP89iAnUCjyH4R+Qi2PWDaIz+/7RyPLfBsuuz3NP/iDLsb4Ws80xwbpHMpb/Ilj+24TvJ/w8ziz2nET2dVSwr7nAgATlT4iUj7ue5wfrH8xYvpCeBYKFnM+SnZ+YY+b7+/LPmG12jbyPk2PWXx6s+2HG8gXB8i/GbDOczqAuGqiH770d2DFmu19mOdbrwfKjYrbZkc7M+IkFnKvwM3JAAduEAe7hWe6rdWS0kAXr/ydY/1BkWVHfzxQQCFZDH8FHIv0z3sHf6Ovx2ZrHMsr+Z/D6BxecySjn3FJ808MoYNuYY/3BOdcSs/zm4HW/zBXm+yp9JejL8ZugD8M0fId38M1Yce51znVkWVdpH8b/IXot5pzinJuPbzauwY/WzjTHObc8Zvm84HXzhPX4KP5frn93zi2MqcdM/BfF4KDOPTU7eP2V+T5/3fqPZnFrluW57pOJZvb1oG/MDZH7ZAD+vE6OFD8oWD7TObcgQX0OwGdIHnbOvZu5MrjP/hn8OjXB/nIp5DNyaPB6r3NuY0y91uOvQR0wJXN9FuH+73bOrclSj1ZgGzPbImZ9ts9d+N1xZ5bjPo3/x9KuZtaYsK7Z3JK5wDm3Dt9EB13PYfj/v8uyr98Gr/tnWf+XmGXZPpfh9bohy77KKTz2Tc65JLMhhOfl5izXMzwv+5pZbY9r16mQ89ljPfj78reYZa8lXL/pfZjZlvgm+xZ8BqwL59z7+NYgiL8H33TOvRSzvNs5M7Ot8P8IbMY3hWce6yV8023JmO/b/kUzu9jMro+c37APZtbz65xbEbM8/B7cxzr7xx9Amb+fq2H6mL/iswY1+P5/H8XfPLea2X7OueZI2XAE8VVmdlWe/Y7BZyqiFmYpGy7fMrrQzL6ObybL9YdhaJblb2RZnobwD+brOcosAA6MlI1alGWb8A91Q4nrsXWWehTqYmAv/B+hB4GNZjYH3xT+e+fcq1m2W5hneeZ98lPgO/gmwGyi98mE4HVeXMEY4X1/tOUfJDMm4T6zWZhnefS9h/X6bzP77zz7TVqvnPeIc67NzN7ENz9tgW8qjsr2uQvrel+Wvt5Ro2L2W4iFeZZHz2G+z8SCjHKZ4j6b2T6Xhd53pVTosfOdl8X44KURf72WFV+1Lgo5nz3Sw78vizMXOOfWRu7tbuvx/9CBru8jPM9vOufasxwr1z1YyN+GcPvFOZIkb+C7S/SYmR2F/wfD8BzFsp3fhVmWv0XX++4dKvD9XA2BYOY8gjvg+wrsie+4+61I2fAP7cNkvwFD3SLzQpjZnvj0dltQhz/ib+ANwfpbgc/jM1xxNvTk+L1Mb8lsFiTIwvxnMJLucPy/aKfi+xh918xOc85d35NjmNkxwPfxzedn4e/dpWGGzMz+BexN1/sk35dFpvC+fwnffy6XFwvcd0+E9XoS358pl0r9wyjb5y6s6734rie5dMtu9lYFtjoUet+VUprHBvyoznxlKtWKU4K/LznrWcHWqGKOk+teKEm9g+zjrfiA7QL8QKOFwHrnnDOzn+Hn3O3RTAGBsn8/V0Mg2IVz7mUzOwX/hX2Gmf0y0nwWBn+3OueKad6YmGX5pOA1mgU4Gn+TXOmcuzxmm8kxy3qr8H3lmpMxXNeTTEivrIdzbjZBM3HQ7HcqfmTnlWZ2h+s+52Ih98kxwev3nXM3xmwTd5+8Gbxma5bIFN73TznnTky4TbEKee9hvf7mnDu3RMfPeY8EzTETMsomsQg/2ORK59xDxVcvkYn4QUCZJgWv0XovwWc3P0D8+ynl5+FN/Dn4IL6/YiUVes/n+67YEqjHNzO+F1kedmvoNh0WvsWpt+gtf1/C8zzBzGqzZAVLdQ++FbxuZWYW172Lzs9ITx2ODwLvcs79IGZ9vvOb7Xtwc/x9t5HOZFPZv5+roY9gN865+/CZlQFA9CLeH7we022jZI6xjHnqAl8IXv8eWTYyeO2WeTT/HOTdi6wDdH5ZFRroF7vdHHzn1w/EzZ9kfo6sj+L/NfaPAvddiH8QjCSzmHm+zD9ScGt8E8acclTAOdfsnLsSP1VBI/F/mD6fZfNC75OPEd8U8Ai+n9uBZrZ1gmo/FJQ/1GLmeyyxQj4j4efxqCTZloTC/R9pGXMQBr6I/154zTlXyB+mnn53FOILmQvMrAk/uhy6nsPw/0/Isq+TgtdHS1CvsN/YlxOWL/b7Jtexj89yf2UKz8sXs9xb4Xl5zDnXFlkeBhvbxWzz8QTHLURPzk85/74k5pxbjG9+rweOjanLMPxAH+jhPeicexP/D4JG4JMxx9oeP+ClFHKd39HAIXm2/7iZjYpZHn62/xW578r+/VyVgWAgzDAcH/ljeTe+U/ehZna5mXVr3zc/WepxWfa5FXBBdEJI8w+q/jI+CLo6UnZu8HpC9OIGN9GN9OzLMfwDtkOB2y3Hf/mMNf8ovkSC5oZfB7/+0sw2BSfBH9tf49/PH4IPa1k4597AX8M64NqM8zoWP6UKwK8y+oYWJRi80W3QkJntTOcI5ri+NFPN7IyMbT6PH7W+ns6O6tB5n5wS/QMXBLrXxNXLOfcOcB3+PEy3jEmnzU+4+p+R8m8H+xoN3G0xT9sxs+Fm9lUrYoLvDIk/I865OfjM/U7ALXGTpprZ2CDDn4hz7u/4fwSMxGdso+d0W3wzD/i+VYW4Dn+tv2pm3zGzbv29zGxHM/t0gfuNc7qZbeoUHgxmuAT/j4Jn6fqPrXDOyi+Z2WEZ9TkN3xF9NX5OtJ66DP8Pwi+bn7y9y98XM9s++GMcCr+nJpfgvroHnyXdHvht5h9MMxttZh+JLLozOP52wI8y7sf/AMI+qZdlHOeR4PXbFpnQ2Mw+DPykh+8hU0/OTzn/vhQqzEheaJGJs80Prvslvo/dk865f8ZsW6hfBq8Xmdn4yLEG479bShXzhOf36Oj3UnBP/Ibc/QbBD6680iIDDM1P8v0/wa+bxihU5Ps56XDovvZDjnkEI2XC6Tp+E1k2gc4Jid/H/yvlFvwXzSvB8scz9jMtWH4tnZPN3or/0ggnyz03Y5sRdM5j9Q6dEwWvxt9kdxMzzJ08U8MEZXYLjtuOHyxzA/7m3CdYP4ns02tMD9cF7/s3+H6WOY9P1wmlV+Env7yTzrmlnif7hNKxw9vpHGp/fgHXfTN8XwqH7+B9Z1CXcBLSR4hMHZPvfOQ51jPBdq8G1+sW/FQDrcRM80HnNARX4YOep4P7JJxHrIOMqXLwTQzhpLcL8fPq3Y/vq/YondNAHJCxXWNw7R3+nnyYznsybkLp+uAedPh/DDyBn/T1TnzgFL6nxkLOUU8+I8F2w+nM9K4L3u+t+Pv0heCcvV3I9aTrhNJv4kcz/pnOCaVvJfv8ark+d7vS+Zlehh9AdDO+f1Y4Afj/9eA7zQU/v8D3/XoI3zcpnEB+JTHT+9B1QunHgvv06eD3ZuBT2Y6VpR5Zzy/wX8F1Cu/XO4Nr9Vzc+QOeonNanpvw3zf/r8jzsw2dU5WsBO4Lrm22CaX3pfOzNTe47g+Re0Lp0fisoMMH/ncF57QV3+c89rz14HwWdX4o09+XPO/jRGK+z/HBVzgf6Ab8Z+3/IvVbRPYJpWcWeKy64Bo6/ICSe4J7cFlwb9wbrPtCAffVTDK+Y/GtBs/Q+ffunuAcLw/Odzht0/kZ+zo/WB5OKB1+/9wf3KOOmGnmKOL7Ods5in2PxXzg+sIPyQLBD0dO7NaR5QOBb+KHZK8M1i8B/o3/V98uGfvZ9OHBT2Px52C79fiO7t1mGA+2G4cP0hbiv5Bfx//raTjZA67Y5TH7Pgb/BRhOjLppG3J/8YzCf9ksitxcC5McP7hZzw5uzLX4D/0L+Al4Byf9MMd8aM7P9V5jthsSbPtCUIe1+D583yR+Drus5yPPcf4Ln+18Bt8nqjm4lveRMRFtUH5meE/im5Fm4r+Y1wb/f0iW40ymM4OxAf9Ffj5+xNymfcZsV4tv4poZ3I8b8V88f8xxTx4V1P9t/H2/HJ9lugb4RA8+j5vuGwr4jATb1gXbPRic59agfrPxmbt9MsrnvZ74fzBchg/iNwbX4R/4eUYLnqQ9Um4EvrvJrGCfzfgg8FF85/Ftcm2fZ9/h59jwk/Q+H9wPK/B/TCbn2HY//B+rMOv/Fj4g3DnXsbKsy3l+g/v1Wvwf3o3BdX4On7WcELOv24PrGQZgM3twjoYBPwzu2XXBz6vB9ZsaU/4DwPXBNQonAf4bMU9biWyzDf7z+F5w/p8GTsp13oo9nz05P5Th70ue93EiWb7P8cHgSfi/qauD++JV/IMI4uaQPSDXe81zrMbgHpgfHGcJPmM/Bv8d4oCPF3BPzST+H9tD6fwOacb/w+AG/Ojl88kdCJ6P/5zcgf9MNuM/I6eTZX7gYPvE38+5zlHmjwUbSA+YnzfoS/gvg2np1kZ6K/PPCN0fONBFRrKLJBFOHeGcK8VIRJGqEnT1WoDvFjLOOVeqKYH6vGruIygiIiL9iJntntlXLujzfgO+xesvCgK7qrrpY0RERKTfuh7Y2syexffXG4cfJT0M3yXijBzbViUFgiJSEDM7GfhI3oLeP51zpRiR2q8E3UmSusg5Nzd/sf5D50d64Ff4Kbp2wk/u34HvJ3k98L/Oz6ogEeojKCIFifSJTeJ3rvyTVPc5lv9RUVFV16dU50ekchQIioiIiFQpNQ0Do0ePdpMmTUq7GiIiIiJ5zZkzZ4VzLu7JUgVTIAhMmjSJ2bNnp10NERERkbzM7I1S7UvTx4iIiIhUKQWCIiIiIlVKgaCIiIhIlVIgKCIiIlKlFAiKiIiIVCkFgiIiIiJVSoGgiIiISJXSPIIiIiIl0NzczPLly2lubqatrS3t6kgfUldXR2NjI2PGjKGxsbGyx67o0URERPqhVatW8c477zBmzBjGjRtHXV0dZpZ2taQPcM7R1tbG2rVrefPNNxk7dizDhg2r2PEVCIqIiPTQihUr2HLLLWlqakq7KtLHmBkDBgxgxIgRNDQ08Pbbb1c0EFQfQRERkR5qaWlh4MCBaVdD+riBAweycePGih5TgaCIiEgJqClYeiqNe0iBoIiIiEiVUiAoIiIiUqUUCIqIiEhiM2fOxMyYNm1aUdsvXLgQM+P8888vab0Apk2bhpkxc+bMku+7v1IgKCIiIlIi8+bN48gjj2TEiBEMGjSIj370ozz88MNpVysrBYIiIiIiJfDaa6+xzz778O9//5tzzjmHSy65hLVr1/KJT3yCBx98MO3qxdI8gn2Yw/EAD3AwB1OjmF5ERCRV3/3ud3n//feZM2cOu+22GwAnnHACO+20E6effjpz587tdaPLFT30YXdxF5/gEzzCI2lXRUREqlhHRwcXXHAB++23H+PGjaO+vp4JEyZw2mmn8e6772bd7rbbbmOXXXahsbGRCRMmcP7558c+nm/p0qWcdtppTJgwgfr6ejbffHNOPfVUli1bVs63VZB169Zx7733csABB2wKAgEGDx7MySefzCuvvMKsWbPSq2AWygj2Yb/ltwC8wzsp10RERKpZS0sLl1xyCUcffTSf+tSnGDRoELNmzeKGG27gn//8J3PmzKG+vr7LNvfeey8LFizg9NNPZ9y4cdx777386Ec/4o033uDGG2/cVO7NN99k7733pqWlha985Stss802zJ8/n2uuuYZHHnmE2bNnF/Ukjo0bN7JmzZpEZWtraxkxYkTOMs899xwbN25k77337rZu6tSpAMyaNYu99tqr4LqWkwLBPuot3uKv/BWA93gv5dqIiEg1a2hoYOnSpV2ervK1r32NffbZh5NPPpkZM2bw2c9+tss2zz77LLNmzWKPPfYA4IwzzuDTn/4006ZN46tf/eqm4OnMM8+ktbWVp59+mi233HLT9p/5zGeYOnUql19+eVEjkG+77TZOOumkRGUnTpzIwoULc5Z56623ANhiiy26rQuXLVmypLBKVoACwT7qZm6mgw5AgaCISG911lnwzDNp16Kr3XaDK64o7T7NbFMQ2N7ezpo1a2hra+Oggw4C4IknnugWCB5yyCGbgsBwH+eccw4zZszg7rvvZurUqaxatYo//vGPnHTSSTQ2NrJixYpN5SdNmsTkyZP529/+VlQg+IlPfIIHHnggUdkkjw9cv3494IPiTI2NjV3K9CYKBPsgh+NGbmRf9uV5nlcgKCIiqbvjjju49NJLefrpp2ltbe2ybuXKld3K77DDDt2W7bjjjgAsWLAA8FOxdHR0cMMNN3DDDTfEHvcDH/hAUfUdP34848ePL2rbOE1NTQCxzwpubm7uUqY3USDYBz3Jk8xlLtdzPRdwgQJBEZFeqtSZt95q+vTpfO5zn2OvvfbiF7/4BVtttRWNjY20t7dz6KGH0tHRUdR+nXMAHHfccXzpS1+KLZMkWxdnw4YNrFq1KlHZ2tpaxowZk7PM5ptvDsQ3/4bL4pqN06ZAsA+axjQGMpDP8lmu4RoFgiIikqqbbrqJxsZGHnnkkS5Zr7lz52bd5uWXX+627KWXXgI6s3yTJ0/GzGhpaeHggw8uaZ1vv/32kvYR3HnnnWloaODf//53t3WPP/44AFOmTCm4nuWmQLAPepqn2Yd9GMpQRjJSgaCIiKSqtrYWM+uS+XPO8dOf/jTrNg888ABPPfXUpn6CzjkuvvhiAI488kgARo0axWGHHcb06dN5/PHHNw0giR5jxYoVebN1cUrdR3Dw4MEcccQRTJ8+nWeffZZdd90VgLVr1/Kb3/yGbbfdtteNGAYFgn1SM82MYxwAIxnJIhalXCMREalmxxxzDHfddRcHHXQQJ5xwAq2trcyYMSPn4Ihdd92Vgw46iNNPP53x48dzzz338OCDD3L88cd3mYLlmmuu4SMf+Qj77bcfJ5xwArvvvjsdHR0sWLCAe+65hxNOOKGowSKl7iMIcOGFF/LQQw/x8Y9/nLPPPpuhQ4dy/fXXs2TJEv70pz/1usmkQYFgn9RMM434EUgjGclKunfCFRERqZRjjz2WNWvWcPnll/Ptb3+bESNGcMQRR3DRRRcxatSo2G0++clPst1223HhhRcyb948NttsM84991zOPffcLuW22mor5syZw89//nPuuecebr75ZhobG9lqq6044ogjuo1GTtPkyZN57LHH+M53vsNFF11ES0sLe+yxB/fff3/Jm7ZLxcKOmNVsypQpbvbs2WlXI7FJTOIADmAa0/g+3+diLqaFFoze9y8NEZFq8PLLL8eOghUpVJJ7yczmOOdK0uEw1UfMmVmNmZ1tZnPNrNnMFpnZpWY2qIB9jDSz/zWz+cE+lpvZI2b20XLWPU2ZGcE22ljL2pRrJSIiIn1N2k3DlwPfAO4GLgV2CH7f3cwOds7lHG9uZhOBmcBg4AbgFWAYsAvQ+8Zol0hmIAh+UukhDEmzWiIiItLHpBYImtlOwJnAdOfc0ZHlrwNXAscCt+bZzc3497CLc25puera22xgQ2wgOJGJaVZLRERE+pg0m4Y/DxhwRcby64H1wHG5Njaz/YCPABc755aa2QAz631TdpdYBx200BIbCIqIiIgUIs1AcE+gA3gyutA51ww8E6zP5bDg9U0zuw/YAKwzs1fMLGcQ2ZdtxD+6RoGgiIiI9FSageDmwArnXPeH8sESYLSZ1efYfrvg9XpgJPAl4MtAC3CTmSWbLryPacY/r1CBoIiIiPRUmoNFmoC4IBAIoh1fpiVLmXBkxBrgQOdcC4CZzQAWAD8zs99lG3BiZqcCpwJMmDCh4MqnJTMQHMEIQIGgiIiIFC7NjOB6oCHLusZImWw2BK+3hUEggHNuJXAvMI7OrGE3zrnrnHNTnHNTink0TVrCQHAg/nE3jTQykIEKBEVERKRgaQaCb+Gbf+OCwS3wzcbZsoEAi4PXt2PWhSOIR/Sgfr1SZkYQ0POGRUREpChpBoKzguN3eQKzmTUCuwH5HvURDjLZMmZduGxZD+rXKykQFBERkVJJMxC8HXDAWRnLT8H3DbwlXGBm25jZ9hnlZuD7Bx5nZoMjZccDRwKvOOfml7zWKVMgKCIiIqWS2mAR59zzZnY1cIaZTQf+TOeTRR6l62TSDwETofNhus65lWb2beDXwONm9lugHjgteD2zIm+kwrIFgq/yalpVEhERkT4q1WcN47OB3wZ2Aq7GP03kKuC/8j1eDvyAD+BoYC3wE+D7wDz8KOK/lanOqdoQjJFRRlBERNIwc+ZMzIxp06YVtf3ChQsxM84///yS1gtg2rRpmBkzZ84s+b77q1QDQedcu3PuUufcds65BufcFs65bznn1maUm+Scsyz7mO6cm+qcG+ScG+Kc+7hz7rHKvIPKU9OwiIhI7/Tkk0/yjW98g3333ZfBgwf3KGCulLQzglKgbIFgM82bsoUiIiJSeX/+85+5+uqref/999l1113Trk4iCgT7mGyBIGhSaRERkTSddtpprF69mhdffJGzzz477eokokCwj8mcUBoUCIqISLo6Ojq44IIL2G+//Rg3bhz19fVMmDCB0047jXfffTfrdrfddhu77LILjY2NTJgwgfPPP5+2trZu5ZYuXcppp53GhAkTqK+vZ/PNN+fUU09l2bLeNUvc2LFjGTRoUNrVKEiaj5iTIigjKCIivU1LSwuXXHIJRx99NJ/61KcYNGgQs2bN4oYbbuCf//wnc+bMob6+vss29957LwsWLOD0009n3Lhx3HvvvfzoRz/ijTfe4MYbb9xU7s0332TvvfempaWFr3zlK2yzzTbMnz+fa665hkceeYTZs2czbNiwguu8ceNG1qxZk6hsbW0tI0b0u2dUAAoE+5xcgeBKVqZSJxERqW4NDQ0sXbqUgQM7W6u+9rWvsc8++3DyySczY8YMPvvZz3bZ5tlnn2XWrFnsscceAJxxxhl8+tOfZtq0aXz1q19l6tSpAJx55pm0trby9NNPs+WWnc+Q+MxnPsPUqVO5/PLLixqBfNttt3HSSSclKjtx4kQWLlxY8DH6AgWCfUwzzRjGAAZsWqaMoIhI73QWZ/EMz6RdjS52Yzeu4IqS7tPMNgWB7e3trFmzhra2Ng466CAAnnjiiW6B4CGHHLIpCAz3cc455zBjxgzuvvtupk6dyqpVq/jjH//ISSedRGNjIytWrNhUftKkSUyePJm//e1vRQWCn/jEJ3jggQcSlY0GuP2NAsE+pplmGmnEOufWViAoIiKpu+OOO7j00kt5+umnaW1t7bJu5cruLVY77LBDt2U77rgjAAsWLABg3rx5dHR0cMMNN3DDDTfEHvcDH/hAUfUdP34848ePL2rb/kSBYB+zgQ1dmoUBBjGIAQxQICgi0suUOvPWW02fPp3Pfe5z7LXXXvziF79gq622orGxkfb2dg499FA6OvI+IyKWcw6A4447ji996UuxZYrN1m3YsIFVq1YlKltbW8uYMWOKOk5vp0CwjwkzglGGaVJpERFJzU033URjYyOPPPIITU1Nm5bPnTs36zYvv/xyt2UvvfQS0Jnlmzx5MmZGS0sLBx98cEnrfPvtt6uPIAoE+5y4QBD0dBEREUlPbW0tZtYl8+ec46c//WnWbR544AGeeuqpTf0EnXNcfPHFABx55JEAjBo1isMOO4zp06fz+OOPbxpAEj3GihUrisrWqY+gp0Cwj1EgKCIivc0xxxzDXXfdxUEHHcQJJ5xAa2srM2bMYP369Vm32XXXXTnooIM4/fTTGT9+PPfccw8PPvggxx9/PHvvvfemctdccw0f+chH2G+//TjhhBPYfffd6ejoYMGCBdxzzz2ccMIJRQ0WKUcfwTfeeIObbroJgBdffBGA++67j8WLFwNw/PHHM3HixJIes6cUCPYx2QLBEYxgCUtSqJGIiFS7Y489ljVr1nD55Zfz7W9/mxEjRnDEEUdw0UUXMWrUqNhtPvnJT7Lddttx4YUXMm/ePDbbbDPOPfdczj333C7lttpqK+bMmcPPf/5z7rnnHm6++WYaGxvZaqutOOKII7qNRk7T66+/3q3+06dPZ/r06QB85CMf6XWBoIUdMavZlClT3OzZs9OuRiKHcAjrWc9jPNZl+Rf4ArOZzSu8klLNRESq18svvxw7ClakUEnuJTOb45ybUorj6RFzfUy2jOBgBrOWtSnUSERERPoqBYJ9TK5AcA3JHpUjIiIiAgoE+5xcgeA61tFBcXM1iYiISPVRINjHZAsEhzAEh2MDG1KolYiIiPRFCgT7mLgni4DPCALqJygiIiKJKRDsY3I1DYMCQREREUlOgWAfo0BQRKR30nRs0lNp3EMKBPsYBYIiIr1PbW0tra2taVdD+rjW1lZqa2srekwFgn1IG220085Auj/zUIGgiEh6hgwZwurVq9OuhvRxq1evZsiQIRU9pgLBPqSZZgBlBEVEepmRI0eycuVKVqxYQUtLi5qJJTHnHC0tLaxYsYKVK1cycuTIih5fzxruQxQIioj0Tg0NDUyYMIH33nuPhQsX0t7ennaVpA+pra1lyJAhTJgwgYaGhooeW4FgH6JAUESk92poaGD8+PGMHz8+7aqIJKam4T5EgaCIiIiUkgLBPiRXINhIIzXU6HnDIiIikpgCwT4kfHxcXCBoGEMYooygiIiIJKZAsA/JlREE3zysQFBERESSUiDYh9z9FwWCIiIiUjoKBPuIp5+Gy3+lQFBERERKR4FgH/Hqq0CjDwTr27s/WQQUCIqIiEhhFAj2EUuXsikQXL5IGUERERHpOQWCfcRbb7EpEHx3iQJBERER6TkFgn1ENCO4YrECQREREek5BYJ9xPvvw5itfCC47E0FgiIiItJzCgT7iLVrYeAoP6H0skXxD6QOA0GHq2TVREREpI9SINhHrFsHdYOaobWO95bVxZYZzGDaaWcjGytcOxEREemLFAj2EWvXQs2gZmpaG1m+PL7MEIYA6HnDIiIikogCwT5i7VqobWqmLkcgOJjBvqz6CYqIiEgCCgT7iLVrwZqaGdA2UIGgiIiIlIQCwT5i3Tqwgc3Uu0ZWrAAXMx5EgaCIiIgUQoFgH9DaChs3Ao3NNLpG2tr8dDKZFAiKiIhIIRQI9gHr1vlX19DMQPNzCMY1DysQFBERkUIoEOwD1q/3rx31zTTVKBAUERGR0lAgmLK2tvxlNgbTArYPaGZQnQJBERERKQ0Fgim6914YMABefjl3uTAQbBuwgSFBILhiRfdyCgRFRESkEAoEU/Sb3/jXuXNzl2v2jximra6ZIQN8IBg3WGQgAzFMgaCIiIgkokAwRW+/7V8HDsxdblNGsLaZwXWN1NbCypXdy9VQwyAGKRAUERGRRBQIpmjZMv8ajgrOJgwEW2ubabRGhg+PzwiCf8ycHjEnIiIiSSgQTFFHh39dmyeBFzYNt9Q2M5CBOQPBwQxWRlBEREQSUSCYogED/Gu+QHBTRrCmmUYaGTEivmkYFAiKiIhIcqkGgmZWY2Znm9lcM2s2s0VmdqmZDUq4vcvy0ycioTAQTNY07NgYBILKCIqIiEgp1KV8/MuBbwB3A5cCOwS/725mBzvnOhLs4x/AdRnLWktayzKpCcLwRBnB+haATYHgkiXxZQczmBXEzC0jIiIikiG1QNDMdgLOBKY7546OLH8duBI4Frg1wa4WOOduLk8ty6vFx3bJ+gg2+o6CYdNwrozgQhaWqooiIiLSj6XZNPx5wIArMpZfD6wHjku6IzOrN7PBpataZYR9/xI1DQ/cAHRmBNVHUERERHoqzUBwT6ADeDK60DnXDDwTrE/iGHzguMbMlpnZVWY2rJQVLZcwEAyfJZyzXCQjOHy4zxKGo4mjFAiKiIhIUmn2EdwcWOGc2xizbgmwj5nVO+dacuzjSeBOYD4wFDgMOAPY38z2cc716oho02jgPD0a45qGAVatgsbGrmUVCIqIiEhSaQaCTUBcEAjQHCmTNRB0zv1HxqLfm9lzwAXAN4PXWGZ2KnAqwIQJExJWubSSBoJxGUHwzcNjx3YtO5jBtNJKCy3UU1/S+oqIiEj/kmbT8HqgIcu6xkiZQl2CDx4Pz1XIOXedc26Kc27KmDFjijhMzzjXOVikJVfOM1hfOzg+EMw0GN9VUllBERERySfNQPAtYLSZxQWDW+CbjfOESN0551rDffewfmXV1uaDQcifEWxpgbogEBzIQIYFPSBXr+5edghDAAWCIiIikl+ageCs4Ph7RReaWSOwGzC7mJ0G228JvNPD+pXVxkijeL6MYGsr1A7qzAjmCgTDjKCeNywiIiL5pBkI3g444KyM5afg+wbeEi4ws23MbPtoITMblWW/P8H3fbyvZDUtg2ggmC8jGA0EG2hg6FC/fNWq7mXVNCwiIiJJpTZYxDn3vJldDZxhZtOBP9P5ZJFH6TqZ9EPARPy8g6EfmNlU4BHgTWAwftTwgcATwFVlfxM9UEgg2NICdQN9oQEM2BQI5soIKhAUERGRfNJ+xNxZwEL86N3DgRX4AO68BI+XmwnsCHwJGAW0A68C3wcuC+Yj7LUKbhpuagOgjjqG+G6ACgRFRESkR1INBJ1z7fhnDF+ap9ykmGX3APeUp2blFw3+kjQN1zR0ZgRramDIkPim4UEMAhQIioiISH5p9hGsam0+wUdNTcKMYGNnIAgwdGh8RjAMBNeR57l1IiIiUvUUCKYkDAQHDkzWRzCaEQQYNix3ILi+qCkYRUREpJooEExJGAg2NSXLCGYGgkOHxjcNN9EEKCMoIiIi+SkQTEkhGcHWVqht6BwsAtmbhuupp446ZQRFREQkLwWCKSkkI9jSAlafrI8g+OZhZQRFREQkHwWCKSk0I5i0jyD45mEFgiIiIpKPAsGURDOCSQLBuIxgXB9B8BlBNQ2LiIhIPgoEUxLNCLa0gHPZy7a0QM2ANmqpxYKHqwwdCmvXQnt79/JqGhYREZEkFAimJJoRhPiALtTaCtS3bhooAr5pGHwwmElNwyIiIpKEAsGURDOCkHvASNg0HDYLA5ueN5zt6SJqGhYREZF8FAimJDMQzNVPMBw1HBcIZptUWhlBERERyUeBYErCQLCx0b/mbRoe0DUQDJuG4wJBNQ2LiIhIEgoEUxIGgg0NXX+P09oK1LWpaVhERERKSoFgSorJCEYHi+RqGlZGUERERJJQIJiSzEAwV0awrQ2oUx9BERERKS0FgikptGnY1cX3EczWNNxGG63kmalaREREqpoCwZQU0jTc1tY9EBw0CMyyNw0DygqKiIhITgoEU5K0abijw/+4urYufQRramDIkOxNw6BAUERERHJTIJiSpE3DYabQ1XbNCIJvHs7WNAxo5LCIiIjkpEAwJUkzguHyjrrugeDQoWoaFhERkeIpEExJ0j6CYbm4jGC2QFAZQREREUlCgWBKkjYNb8oI1rYV3DSsjKCIiIjkokAwJW1tfsDHgAGdv2crB9BR23VCaVDTsIiIiPSMAsGUtLVBXZ3/gfxNwx1qGhYREZESUyCYksxAMG9GsEaDRURERKS0FAimJAwEa2s7f4/TGjwcJC4jOGwYrFvXfVv1ERQREZEkFAimpK3NB4GJm4Zrug8WCZ83vGZN123UNCwiIiJJKBBMSaFNw+018YNFoHvz8AAGUEutMoIiIiKSkwLBlHR0+FHD+ZqGo4FgXNMwdJ9CxjAGMUiBoIiIiOSkQDAl7e2FNQ3HBYLZMoLgm4fVNCwiIiK5KBBMSWYgmDcjaNn7CGYbOayMoIiIiOSiQDAlHR0+ECxH0zCgpmERERHJS4FgStrbfR/BpBnBNks+WATUNCwiIiL5KRBMSWF9BB0dNe1qGhYREZGSUiCYkjAQTNQ0PMDPKp0ZCA4a5LOK2ZqGlREUERGRXBQIpiTsI5ivabi1FajzKzMDQbPcj5lTRlBERERyUSCYksw+gjmbhoOMYGYfQcgeCGqwiIiIiOSjQDAlBU0fk6VpGHwgqKZhERERKYYCwZSUoo8g+Clk1DQsIiIixVAgmJLwEXOJmoaz9BGE3E3DrcF/IiIiInEUCKakEk3DgJqHRUREJCsFgikJA8Ga4AokCQTjBovkahoG1DwsIiIiWSkQTEk4fYyZf00yarjQpmFQICgiIiLZKRBMSTh9DPjm4Z40DW/YEMw3GKGmYREREclHgWBKwqZh8K85A8Ecg0WGDfOvmVlBNQ2LiIhIPgoEUxINBOvqsjcNt7aSNyMI3QNBNQ2LiIhIPgoEUxL2EYTkTcPZniwC2QNBNQ2LiIhINgoEUxLtI5i3aThBRjBzChk1DYuIiEg+CgRTktk03JMni4CahkVERKRwCgRTkrSPYFsb1DTkfrIIZB8soqZhERERyUaBYEoK6SNY01B407AygiIiIpKPAsGUFNJHsLYx95NFoHtGsJ56aqlVRlBERESyUiCYksKahrNnBAcO9PvJDAQNo4kmZQRFREQkq1QDQTOrMbOzzWyumTWb2SIzu9TMBhWxryYzW2Bmzsx+WY76llJBTcP12fsImvnm4cymYfDNwwoERUREJJu0M4KXA5cBLwFnAncC3wDuM7NC6/ZjYExpq1c+hTQN58oIgm8ezva8YTUNi4iISDbdO51ViJnthA/+pjvnjo4sfx24EjgWuDXhvvYAzgLOAS4teWXLoJAni4SBYFwfQfAZwbhAUE3DIiIikkuaGcHPAwZckbH8emA9cFySnZhZbbDN/cD0EtavrAqZRzBfRlBNwyIiIlKMNAPBPYEO4MnoQudcM/BMsD6Js4HtgTNKWblyi/YRVNOwiIiIpCHNQHBzYIVzbmPMuiXAaDOrz7UDM9sa+BHwY+fcwkIObmanmtlsM5u9fPnyQjYtiWgfwbyjhgdkHywCahoWERGR4qQZCDYBcUEgQHOkTC7XAgvwA04K4py7zjk3xTk3ZcyYyo8xKaRp2NQ0LCIiImWQ2mARfD/AzbKsa4yUiWVmxwGHAPs551pLXLeyiwaC+ZqGrd6/vZoscXuujKCahkVERCSbNDOCb+Gbfxti1m2BbzZuidsw2OYy4M/A22Y22cwmAxODIsOCZcPLUO+SyJxHMFfTsNW3MoABGBZbZtgw2LjR/0QpIygiIiK5pBkIzgqOv1d0oZk1ArsBs3NsOxA/Z+DhwKuRn5nB+uOC308uZYVLKbOPYM6M4IC2rM3C0Pm84cysoAJBERERySXNpuHbge/h5//7R2T5Kfi+gbeEC8xsG2CAc25usGgd8JmYfY4BfoWfSuYG4LmS17pECmoaHtCaMxAcPNi/rl0L0e6OTTTRGvyXa3sRERGpTqkFgs65583sauAMM5uOb+bdAf9kkUfpOpn0Q/hmXwu2bQX+kLlPM5sU/O9rzrlu63uTpINFWluB+tyB3JAh/nXNmq7LB+Gf1Lee9QxjWA9rLCIiIv1NmhlB8NnAhcCp+GbeFcBVwHnOuY70qlV+HR3Jp4+xAa1ZnyoCnYHg2rVdl4eB4DrWKRAUERGRblINBJ1z7fhHwuV8LJxzblLC/S2ELCMqehHn/E/S6WNI2DScmRFsCmbf0chhERERiZPmYJGqFWb/kvYRJM9gkSQZQREREZFMCgRT0BE0eiedPqbYjGC0j6CIiIhIJgWCKQiDvmgfwdYsU2InCQSzDRYJm4aVERQREZE4CgRTENc0nDMjWNfzwSIiIiIimRQIpiAzEMzXNOzqcmcEGxr8vjRYRERERAqhQDAFcX0Ecw4Wqcs9WMTMZwWVERQREZFCpD2PYFXK7COYb9Rwvowg+AEj2QaLKBAUERGROMoIpqCYpuFcfQQhPiOopmERERHJRYFgCuICwXJkBBtooIYaZQRFREQkVsGBoJk9aGafM7P6clSoGoR9BKNNw851Lo9qawNXm7uPIPiMYGYgaBiDGKRAUERERGIVkxHcHbgVeMvMrjCznUtcp34vLiMI8VlBHwjmzwjGNQ2Dbx5W07CIiIjEKSYQHA98EXgaOBN4xsyeMLNTzGxwSWvXT2ULBOP6Cba1QUeCQDCuaRhQRlBERESyKjgQdM61OOf+zzl3CPAB4KfAWODXwFIzu8HM9i1xPfuVuOljIHtGsKPIwSKgQFBERESy69FgEefcG865HwJbA4cCjwAnAn83s5fM7CxlCbuLmz4GugeCHR1B38EeZATVNCwiIiLZlGrU8G7AJ4GPAga8BnQAlwGvmtk+JTpOv5C0aTgMDF1NssEizc3dg0llBEVERCSbogNBMxtuZqeb2VPAbOBk4K/Awc65DzrnPgQcDKwHri5JbfuJpINFwnJJMoLZnjesjKCIiIhkU8z0MR8zs1uAt4CrgCbgHGAL59yxzrmHw7LB/18E7FSi+vYLmX0EszUNh7+31yRrGob4p4soIygiIiJxinnE3APARmA6cJ1z7tE85ecDjxVxnH4rs49gvqbhjtpkg0Ug/nnDCgRFREQkTjGB4LeA3zvn3ktS2Dn3CH4QiQSSNg1vCgQT9BHMlhFU07CIiIhkU0wfwaHA5tlWmtlOZnZe8VXq/zIDwVI0DYcZQTUNi4iISFLFBII/BHbJsf5DQRnJIts8grFNwzXtOHNFDxYZxCBaaKGNLA8zFhERkapVTCBoedY3gqKOXLL1EYzNCA5oBehR0zCg5mERERHpJlEfQTMbCgyPLBplZhNiio7EP35uUc+r1n8V1DQcBII9GSwCsI51DGVosVUWERGRfijpYJGzgbDfnwOuCH7iGH46Gcki6SPm2tqAOr+wJ9PHAOonKCIiIt0kDQRnBq+GDwjvBp7LKOOAtcDjzrl/laR2/VRB08ckbBoe5OM9NQ2LiIhIYokCwWCuwEcBzGwicK1z7olyVqw/K2j6mISBYE2NzwrmahoWERERiSp4HkHn3EnlqEg1KUcfQfCBYLaMoAJBERERyZQ3uggHhTjn3oz+nk9YXroraPqYhH0EwQ8YyZYRVNOwiIiIZEqSEVwIdJhZk3OuJfjdJdiutgf16tfKMX0MxGcE1TQsIiIi2SQJBH+MD/zaMn6XIhXTNJw0I6imYREREUkqbyDonDs/1+9SuGyDRXoyahhg6FBYvLjrMjUNi4iISDbFPFlEeqigeQQLGCwybBisWtV1mZqGRUREJJuCA0Ezm2xmh2Ys+w8zu8/MHjOzU0tXvf4ps49gzqbhAgaLDB8O77/fdVkDDRimjKCIiIh0U/D0McDP8Y+Sux/AzEYDfwEGAxuAa8xsmXNuRqkq2d+Uq2k4zAg6BxY8EdowBjFIGUERERHpppim4SnAg5HfPw8MBfYAxgBPAN/sedX6r3JMKA0+I9jRET+FjAJBERERyVRMIDgGeCvy+6HAY865F4LpZf4P2LEUleuvwj6CiZqGC8wIQvd+gk00qWlYREREuikmEFwHDAcws1rgI8DfI+s34DOEkkUxGcEkg0WGD/evmf0ElREUERGROMUEgi8CJ5jZKOAUfN/AByLrJwLLS1C3fqugPoIFDBbJlRFUICgiIiKZihkscglwD7As+P1p4B+R9R8Hnuphvfq1zOljStU0nCsjqKZhERERyVRwIOic+5OZHQR8ClgF/NI55wCCLOFi4PclrWU/U65HzIWBYNxcgktYUlxlRUREpN8qJiOIc+7vdO0XGC5/F/h0TyvV35Vz+hjonhFU07CIiIjE0ZNFUlDQYJGgj2Ahg0XiMoJqGhYREZFMRWUEzWxv4AxgW2AUYBlFnHNumx7Wrd8qVx/Bxkaor9eoYREREUmm4EDQzE4AbgRagVeAN0tdqf4u2yPmeto0DD4rqFHDIiIikkQxGcHvA/OAg51zb+UrLN1lNg3X1PifnmYEwfcTjMsIttBCG22JmphFRESkOhTTR3AicI2CwOJlBoLh//d0QmmIzwgOYhCA+gmKiIhIF8UEgouBhlJXpJqEfQQt0rOyri6+abimPvmE0hCfEWyiCVAgKCIiIl0VEwheC3wxeLycFKG93TcFZwaCcRlBa2illlqs23iceLkyguonKCIiIlHFdBibAxwNPGlmVwOvA+2ZhYK5BiVGe3vXZmHI3jRc09BKbcJsIOTOCCoQFBERkahiAsGHIv//G8BlrLdgmTKGWXR0dA8EszcNtyZuFgb1ERQREZHkigkETyp5LapM2DQclbVpuL6wkb7DhsH69dDaCgOC+FFNwyIiIhKnmGcN/64cFakmhTQNWxEZQfBZwdGj/f+raVhERETipPqIOTOrMbOzzWyumTWb2SIzu9TMBiXYdjszu8XMXjazVWa2PtjPZWY2vhL1L1ZcIJg9I1hYIBj3vGE1DYuIiEicogJBM9vKzH5rZovNrMXMDgqWjwmW75lwV5cDlwEvAWcCdwLfAO4zs3x12xIYD9wNfBc4C3gAOBWYY2abFfq+KqWQPoI9yQiG1DQsIiIicYp5xNzWwONAY/C6KfvmnFtuZlOAk4FZefazEz74m+6cOzqy/HXgSuBY4NZs2zvnHqLrwJVw+78DdwAnAhcnfV+VVFgfwdJlBBUIioiISFQxGcELgA7gQ8AXodsEd38GPpJgP58Ptr0iY/n1wHrguCLqBvBG8DqiyO3LrqA+gnWFDRaJywgOZjAAa1lbRG1FRESkvypm1PDBwFXOuUVmNipm/Rv4Ztt89sQHlE9GFzrnms3smWB9XmbWCAzGZyh3BH4erPpzku3TkK2PYFzTMEU2DUczgg00MIABrGFNMdUVERGRfqqYjOBQYGmO9fUkCzA3B1Y45zbGrFsCjDaz+gT7ORlYDiwC/goMB45zzv0jwbapyNZHMDYjOKC4puHMuQQHM1iBoIiIiHRRTEZwEbBTjvVTgfkJ9tMExAWBAM2RMi159jMDmIvPCu4OfBIYne/gZnYqfmAJEyZMyF/bEorrI5itaZgCA8GhQ/2j6957r+vyIQxR07CIiIh0UUwgOB34mpndQGdm0AGY2dHAZ4AfJtjPeiDbyN7GSJmcnHOLgcXBrzPM7C5glpk1OecuzLHddcB1AFOmTMl8OkpZFdQ0PKCtoECwpgZGjICVK7suV0ZQREREMhU7WGQx8ARwMz4I/I6Z/Rs/WvdZ4NIE+3kL3/zbELNuC3yzcb5sYDfOueeAp4GvF7ptpRQyjyB1rQUNFgEYORLefbfrMmUERUREJFPBgaBzbjWwN/45w1PwI38PAbYDfgUc6Jxrzr6HTWYFx98rujAY/LEbMLvQukUMBEb2YPuy6ugoX9MwwKhR8U3DygiKiIhIVFETSjvnVjvnvumcGwOMBcYBo5xzZwaBYhK347OJZ2UsPwXfN/CWcIGZbWNm20cLmdm4uJ2a2YH4qW0eT1iPiiukadgVEQiOHNk9EFTTsIiIiGQqZkLpfYDDgQ/iRxCvxg/W+BMFBF/OuefN7GrgDDObjp/uZQf8k0Uepetk0g8BE+k6Z+E1waPkHsZPWdMIfBg/EfUa4L8LfW+VUmjTcDGB4Ny5XZepaVhEREQyJQ4EzWwocBtwKN0nkQb4npn9Cfiicy5p6uksYCF+9O7hwArgKuA851xHnm1vA04AjgfG4LOLbwC/Bi5xzr2ZsA4VFzd9TLamYVfghNIQ3zSsjKCIiIhkKiTC+AN+Mul/AjcAz+GzgUOBXfDz+f0Xvsn3sCQ7dM614weW5Bxc4pybFLPsDvzglD6nkEfMuSIzgqtW+e3rgiusjKCIiIhkShQImtkn8EHgpc65/xdT5Gngd2b2v8DZZnaIc+6BEtazXymoj2BdC/UkmVe708hgmMzKlTBmjP//IQxhIxtppfDAUkRERPqnpINFPo9vdj0nT7lzgDeBL/SkUv1dIX0Ei8kIjgoe/BdtHg6fN6zmYREREQklDQQ/DMxwzuWceDno1zcDP62MZFFIH8GOHmQEo4HgEIYAqHlYRERENkkaCG4BzEtYdh6wZXHVqQ7Z+gjGNQ13FNlHELpOKq2MoIiIiGRKGggOhcQRxBoIog6JVUjTcEdt4RnBuKZhZQRFREQkU9JAsIbgecIl3m9VigsEs08fU5qMYBgIKiMoIiIioUKmjzks29M8Mny42MpUi7hHzMU1Dbe2OTpq2wrOCA4b5vevwSIiIiKSSyGB4BdIPhq4kOxh1UncNGytAAVnBGtqYMQINQ2LiIhIbkkDwQPLWosqk7hpuMhAEHzzsAaLiIiISC6JAkHn3KPlrkg1iZs+JnbUcE0LQMFNw+AHjKiPoIiIiOSiQR0pSPyIuR5kBDOfN9xEE4apaVhEREQ2USCYgiRNwx0d4Gp9IFhMRnD0aFi+vPN3wxjMYGUERUREZBMFginINlikowPCZ7e0twP1vmm4mIzg6NGwYkXXZUMYooygiIiIbKJAMAXZ+ghCZz/BtjZgQPFNw2PGwIYNsH595zJlBEVERCRKgWAKsvURhM7m4bY2NmUEi20ahq5ZwSEMUSAoIiIimygQTEG2PoKQEQj2ICMYBoLRfoJqGhYREZEoBYIpyNZHMFwH0NpKyTOCahoWERGRKAWCKcjVRzDMCLa00OM+gtC9aVgZQREREQkpEExBXB/BzKbh1lZK0jSsjKCIiIhko0AwBUmahlta6FHT8PDhPtjM7COoQFBERERCCgRTUImm4Zoa/3SRzKbh9aynnfbsG4qIiEjVUCCYgiRNwz3NCEL3SaUHMxiAdawran8iIiLSvygQTEHiUcM9yAiCHzCSmREENGBEREREAAWCqcgVCFYiI6h+giIiIgIKBFORq49gq08C9riPIPhAMHOwCCgQFBEREU+BYAri+gjWB0m/MBAsRdPw6NHw7rs+8AQ1DYuIiEhXCgRTENc0PCCI9VpaIq89bBoeM8Yfa9Uq/7uahkVERCRKgWCFhdm5zEAwzAh2CQRLkBGEzn6CygiKiIhIlALBCksaCPb0WcPQGQiG/QSVERQREZEoBYIVFk4Pk6+PYDkzggoERUREBBQIVlwYCCZtGjZn1JJROKHMQDDMCKppWERERECBYMUlDQTDpuFim4XBDxaBzkCwlloGMlAZQREREQEUCFZc2EcwW9NwZkaw2GZhgKYmaGzs/nQRZQRFREQEFAhWXEFNwz3MCJp1n1R6MIOVERQRERFAgWDFFdQ03MOMIHR/zNwQhigQFBEREUCBYMUVNI9gDzOC4PsJqmlYRERE4igQrLB808dEA8GahlYGWGkzgmoaFhERkZACwQorpGm4pqE0TcPRPoLKCIqIiEhIgWCFZQsE4541bA09bxoePdo/azicqFoZQREREQkpEKywbH0EzaCuLjMQLE1GEODdd/2rBouIiIhISIFghWXrIwi+ebhr03BpBotAZz/BoQxlDWvooKNH+xUREZG+T4FghWVrGoaugaAfNVy6jGAYCI5kJA7HKlb1aL8iIiLS9ykQrLBCAkGrbylZIBgOGBnBCABWsrJH+xUREZG+T4FghWV7xBz4QDAc1OGfNdza46bhUaP8a9hHUIGgiIiIhBQIVlhBTcMleLJIGAiGTcMKBEVERCSkQLDCCusj2PPBIg0NMHhw94zge7zXo/2KiIhI36dAsMKyTR8D3UcNU9fzjCB0fbqIMoIiIiISUiBYYUmnj2lpATeg54NFwDcPhxnBkYwEFAiKiIiIAsGKK7SPYE+bhqFrRnAgA6mnXoGgiIiIKBCstKSBYGsrdJQhI2gYIxihQFBEREQUCFZa0j6CLS3gakufEQQUCIqIiAigQLDiCuojWKLBIqNGwerVnXMUKhAUERERSDkQNLMaMzvbzOaaWbOZLTKzS81sUIJtP2hmPzazx81suZmtMbNnzOz7SbZPS0FNw3U9nz4GOp8uEp1CRtPHiIiISNoZwcuBy4CXgDOBO4FvAPeZWb66fRk4G3gN+DHw/4B5wE+Bf5nZwHJVuidyBYIDBnQGghtbHB0lzAhC10BQGUERERGpS+vAZrYTPvib7pw7OrL8deBK4Fjg1hy7+ANwoXNuVWTZtWb2KvB94CvAL0te8R5KmhHc2N4GULJ5BKGzn+BIRioQFBERkVQzgp8HDLgiY/n1wHrguFwbO+dmZwSBoduD1w/1tILlkDQQ3NDmO/SVq2l4Fatop73H+xYREZG+K81AcE+gA3gyutA51ww8E6wvxpbB6ztF16yM2nyij7qYXGwYCDoHLR0+ECxl03Dm00VWERdHi4iISLVIMxDcHFjhnNsYs24JMNrMCkqHmVktcC7QRu5m5dQkyQg2NwP1PjVYioxgXB9B0NNFREREql2agWATEBcEAjRHyhTiCmBv4Dzn3LxcBc3sVDObbWazly9fXuBhipcvEGxthQ0bgAGlywgOHAhNTXresIiIiHSVZiC4HmjIsq4xUiYRM/sJcAZwnXPuwnzlnXPXOeemOOemjBkzJulheixJ0/CGDWzKCJYiEATfTzAzI6gpZERERKpbmoHgW/jm37hgcAt8s3FLkh2Z2fnAD4Abga+VrIZlkC8j2N4O69axKSNYiqZh8M3D0VHDoIygiIhItUszEJwVHH+v6EIzawR2A2Yn2UkQBP4Q+B1wsnPOlbSWJRYGgnEZwYYgJH7/fSqSEVQgKCIiUt3SDARvBxxwVsbyU/B9A28JF5jZNma2feYOzOw8fBB4E/Bl51xH2WpbImHTcFxGsCnoEfnuu5Q1I6hAUERERCDFCaWdc8+b2dXAGWY2HfgzsAP+ySKP0nXU70PARPy8gwCY2enAj4A3gQeBL5hZZBPecc49UNY3UYRcTcODggfjRQPBcmQEBzKQBhoUCIqIiFS51ALBwFnAQuBU4HBgBXAVftRvvuxeOM/gBHyzcKZHgV4bCMY1DXfJCJa4aXjUKFi50mck6+r0mDkRERFJORB0zrUDlwY/ucpNill2InBiOepVTrmahuMygqVqGg4HRr/7Lowdq0BQRERE0u0jWJVyNQ2HGcEVKyh5RnCzzfzrsmX+dSQjNX2MiIhIlVMgWGG5mobDjOA771C2jGA4d7YygiIiIqJAsMKSjBp++23KnhFUICgiIiIKBCssSdPw22+jjKCIiIiUnQLBCgsDwZqYMx9tGm4YXNrpY0aO9MeMZgRXs5p22kuyfxEREel7FAhWWFubzwZ2nfLQCzOC69ZB47DSNg3X1vopZKIZQYD3eb8k+xcREZG+R4FghbW3xzcLAwwZ0pkpDDOCpWoaBt9PMDpqGPR0ERERkWqmQLDC2tvjRwyDDwJHjfL/3ziktBlB8P0EMzOCmkJGRESkeikQrLCwaTib0aP9a32ZM4J63rCIiIgoEKywXE3D0JkRHDKytINFID4jqEBQRESkeikQrLDwWb/ZDB/uX4eOLn3T8Gab+ecNt7YqEBQREREFghWXLyPY2Ohf6wb6jGBdCR8HHc4luGKF+giKiIiIAsGKyxcI/uQnsPXWMOmDLQxgAEbMPDNFij5dpJFGhjCEZSwr2f5FRESkb1EgWGH5moa33x4WLIDhY1pLOlAEuj9dZDzjeZu3S3oMERER6TsUCFZYvoxgqIWWkvYPhO7PGx7HOJaytKTHEBERkb5DgWCF5ZpHMKqV1pIHgnEZQQWCIiIi1UuBYIXlm0cw1Erpm4ZHjPDHDjOCCgRFRESqmwLBCkuzabimxk9YHc0IrmMda1hT0uOIiIhI36BAsMIKaRoudUYQuj5dZDzjAZQVFBERqVIKBCssadNwOTKC0PXpIgoERUREqpsCwQpL2jRczoygAkEREREBBYIVl7RpuJwZwcymYc0lKCIiUp0UCFZYIaOGyxEIbrYZrFoFLS3+MXMNNCgjKCIiUqUUCFZY2k3D0bkEDdOk0iIiIlVMgWCFpd00HD5dRJNKi4iIiALBCktzQmnozAhqUmkRERFRIFhhaU4oDd0zgmoaFhERqV4KBCsszWcNQ3xG8D3eYyMbS34sERER6d0UCFZY2k3Dw4f7QDRzLkFNISMiIlJ9FAhWWNpNw2bxcwmqeVhERKT6KBCssLa2dJ81DPFPF1FGUEREpPooEKywtDOCoIygiIiIeAoEKyztCaWha0ZwMzajhhoFgiIiIlVIgWCFJW0arlRGsJZaNmMzBYIiIiJVSIFghRWSESxnILhmDTQ3+981qbSIiEh1UiBYYUnmEWynHYcra9Mw6DFzIiIi1U6BYIUlmUewhRaAsmYEQYGgiIhItVMgWGFJmoZbaQUoe0YwOnL4Hd6hnfayHE9ERER6JwWCFZakabjSGcEt2IIOOjSXoIiISJVRIFhhSZqGw4xguQLBsWP969tB3DeRiQC8wRtlOZ6IiIj0TgoEKyxJ03CYESxX0/DQoTBkCCxZ4n9XICgiIlKdFAhWWJKm4XJnBAG22KJ7ILiQhWU7noiIiPQ+CgQryDno6Eh/sAj4QHDxYv//gxjEaEYrIygiIlJlFAhWUHswKDft6WOga0YQfFZQgaCIiEh1USBYQWEg2Fuahpcu9RlKUCAoIiJSjRQIVlBbm39Ne7AIwJZb+vqEcwmGgaDDle2YIiIi0rsoEKygpE3DlcoIQmc/wYlMZD3rWcGKsh1TREREehcFghWUtGm4EhnBMBAM+wlOYhKgKWRERESqiQLBCkraNFzJjKDmEhQREaleCgQrqNCm4XJmBDfbzGcmFQiKiIhULwWCFVRo03A5M4K1tTB+fGcgOJzhDGGIAkEREZEqokCwgnpT0zB0nVTaMCYyUU8XERERqSIKBCuoN00oDd0nlZ7EJGUERUREqkiqgaCZ1ZjZ2WY218yazWyRmV1qZoMSbv9dM7vTzBaYmTOzhWWuco+0+kQfA/LEdxvYAEATTWWtj54uIiIiUt3SzgheDlwGvAScCdwJfAO4z8yS1O1nwEHAa8DKclWyVMKm4XyB4FrWAjCYwWWtz5Zbwpo1/gd8IPg+77Oa1WU9roiIiPQOeYYtlI+Z7YQP/qY7546OLH8duBI4Frg1z262cc4tCLZ7AcocOfVQmBHMN1gkDATLnRHcckv/+uabsNNOXUcO78zOZT22iIiIpC/NjODnAQOuyFh+PbAeOC7fDsIgsK9I2jS8lrUMYhA1Zb48E33cxxtBa3AYCGrAiIiISHVIMxDcE+gAnowudM41A88E6/uVQgLBcjcLA0ya5F8zA0H1ExQREakOaQaCmwMrnHMbY9YtAUabWdlmVDazU81stpnNXr58ebkO00XYRzBJ03AlAsFx46C+HhYu9L+PZSyNNCojKCIiUiXSDASbgLggEKA5UqYsnHPXOeemOOemjBkzplyH6aK3ZQRramDChM6MoGFMZjKv8ErZjy0iIiLpSzMQXA80ZFnXGCnTb/S2QBB8P8EwIwiwIzvyEi9V5NgiIiKSrjQDwbfwzb9xweAW+GbjlgrXqaySTh+zjnUVCwQnTerMCIIPBBewYNNchiIiItJ/pRkIzgqOv1d0oZk1ArsBs1OoU1kVMn1MJTOCb78NzUFj/A7sgMOpeVhERKQKpBkI3g444KyM5afg+wbeEi4ws23MbPvKVa08emvTMPi5BMFnBAE1D4uIiFSB1CaUds49b2ZXA2eY2XTgz8AO+CeLPErXyaQfAibi5x3cxMyOD5YDjAHqzewHwe9vOOduKuNbKFhvDATDKWQWLoQPfhC2ZVtqqVUgKCIiUgVSCwQDZwELgVOBw4EVwFXAec65jgTbfwXYP2PZT4LXR4FeFQj2tuljoDMQfP11/9pAA5OZrEBQRESkCqQaCDrn2oFLg59c5SZlWX5A6WtVPkkygq20spGNFQsEt9gCGhpg/vzOZTuwAy/zckWOLyIiIulJs49g1UkSCK5jHUDFAsHaWpg8GV6JjA3ZkR15lVdpoV8N2hYREZEMCgQrKEkguJa1QOUCQfB9A199tfP3HdmRNtqYz/zsG4mIiEifp0CwgpL0EUwjENx2W9803N7uf9fIYRERkeqgQLCCenNGsLW1c2Lp7dgOw9RPUEREpJ9TIFhBvTkQhM7m4SaamMQkZQRFRET6OQWCFdSbm4ah+4ARBYIiIiL9mwLBCmpt9aN0zbKXSSMQHDsWhgzpPmBkHvNop71i9RAREZHKUiBYQa2tyZ4qApUNBM1883A0I/ghPsRGNqqfoIiISD+mQLCCWluTPVUEKhsIQvdAcD/2A+AhHqpoPURERKRyFAhWUGurf4pHLmEgOIhBFahRp2239aOGN270v09iEtuwjQJBERGRfkyBYAVt3Aj19bnLrGUtDTQwgDxtyCW23XbQ0dG1n+DH+BgzmUkbbRWti4iIiFSGAsEKamlJFghWulkYYEc/hzQvR7oEfoyPsYY1zGJWxesjIiIi5adAsIJaWpI1DVe6WRh8RtAMXorMGHMQBwHqJygiItJfKRCsoKRNw2lkBAcOhK237poRHM1odmM3BYIiIiL9lALBCkqaEUwjEATfPPxSxhzSH+Nj/It/sZ71qdRJREREykeBYAX15owg+EBw3rzOJ6AAHMzBtNDCP/lnKnUSERGR8lEgWEG9ebAI+ECwpQUWLOhc9lE+ygAG8CAPplInERERKR8FghXU25uGd9jBv0abhwcxiP3Zn7u4C4dLpV4iIiJSHgoEK6i3Nw3HBYIAX+ALLGABT/Jk5SslIiIiZaNAsIJ6e9PwkCGw1VZdRw4DfJpP00ADt3BLKvUSERGR8lAgWEEbN+ZuGu6gg3WsSy0QBN9P8Pnnuy4bxjCO4Ahu53Y9ZURERKQfUSBYQfkyguEULWkGgrvvDi++CM3NXZd/gS+wjGUaNCIiItKPKBCsoHyDRdayFkg3EPzwh/30MZlZwcM4jOEMV/OwiIhIP6JAsILyDRbpDYHglCn+dc6crssbaOAYjuFu7mYd6ypfMRERESk5BYIVlK9puDcEghMnwsiRMHt293Vf4AusYx33c3/lKyYiIiIlp0CwgvpCRtDMNw9nZgTBTy49ilHczd2Vr5iIiIiUnALBCmlv933vGhuzl+kNgSD4QPCFF7oPGKmjjk/ySf7IH2mhJZ3KiYiISMkoEKyQDRv8a1NT9jK9JRCcMsUHrc89133dURzFKlYxk5kVr5eIiIiUlgLBCgkDwYEDs5fpLYHghz/sX+Oahw/mYAYxSM3DIiIi/YACwQrpS4HgxIkwZgw8/nj3dQMZyKEcyj3cQwcdla+ciIiIlIwCwQrpS4GgGeyzDzz2WPz6oziKpSzlCZ6obMVERESkpBQIVkiSQHAd66ihhkZyjCipkH33hddeg3fe6b7ucA6njjo1D4uIiPRxCgQrJGlGcDCDMawylcph333967/+1X3dcIZzEAdxN3fjcJWtmIiIiJSMAsEKWe8fI5woEOwNPvxh/zi8XM3D85nPS7xU2YqJiIhIySgQrJBCMoK9QUMD7Lkn/P3v8es/xacwTM3DIiIifZgCwQrpa4EgwMEH+0fNrVjRfd14xjOVqQoERURE+jAFghXSFwPBQw8F5+CBB+LXH8VRPMVTvMEbla2YiIiIlIQCwQrpi4HglCkwahTcd1/8+iM5EoAZzKhYnURERKR0FAhWyFo/RSBDhuQo08sCwdpaOOYYmDED1qzpvn5btmUndlLzsIiISB+lQLBCVq/2r4NzxHm9LRAE+NKXfDbzjjvi1x/FUfyDf7Cc5ZWtmIiIiPSYAsEKWbPGB4E1Wc54G20sYxmjGV3ZiuUxdSrssgtccgm0t3df/1k+Swcd3MiNla+ciIiI9IgCwQpZvRqGDs2+fj7zaaGFD/GhylUqATP4wQ9g3jy4MSbW25md+Tgf5zIuYwMbKl9BERERKZoCwQpZsyZ3/8AXeAGg1wWCAEcfDfvtB9/+Nixa1H39d/ku7/AO05hW8bqJiIhI8RQIVki+jOALvEANNWzP9pWrVEI1NXDDDdDRAZ/9LLS0dF2/P/uzN3tzMRfTRls6lRQREZGCKRCskDVrcgeCz/M8k5nMQHLML5OiyZN9MPj443DOOV3XGcZ3+S4LWcj/8X/pVFBEREQKpkCwQlavzt803BubhaM+8xn45jfhF7+Ae+/tuu5wDmdnduY8zmMVq9KpoIiIiBREgWCFvPuun5w5zgY2MJ/57MzOla1UES6+GHbeGU47DVZF4r0aariWa1nEIr7CV3C49CopIiIiiSgQrADnYNky2Gyz+PVzmUsHHb0+IwhQX++biN9+G/7nf7qu24d9uIiLuIu7uJIr06mgiIiIJKZAsALefx/a2mDMmPj1vXnEcJw994SzzoJf/xoefbTrum/xLT7Fp/g23+Ze7o3dXkRERHoHBYIVsGyZf82WEXyBF6innslMrlyleujHP4att4ZTTul8jjL4gSM3ciO7siuf4lN8j+9pJLGIiEgvpUCwAsJAMFdGcAd2oI66ylWqhwYNguuug1df9UFh1AhG8E/+yamcyoVcyMf4GPOYl05FRUREJCsFghXwxhv+dcKE+PV9YcRwnIMPhpNO8o+fe/DBrusaaeTX/Jrf8Tue4zl2YRfO4zyaaU6nsiIiItJN6oGgmdWY2dlmNtfMms1skZldamaDKrF9Jcyf7x/VtvXW3detYhVv8mafDAQBLr8cdtwRjjoK/v3v7utP4ATmMpfP8Bl+wk/Ygz2YzezKV1RERES6ST0QBC4HLgNeAs4E7gS+AdxnZknq19Pty27+fJ8NbGjovu5FXgT6zkCRTMOGwV/+AmPHwoEHwjXXQHt71zJjGcvN3Mz93M9qVjOVqXyf7/M+7+fdv6ahERERKZ9UO6WZ2U744G26c+7oyPLXgSuBY4Fby7V9peyxB2y1Vfy6B/Ftqn1hDsFsttgCnngCjj0Wvv51uPZa+Na34HOfg8bGznKf4BO8wAt8k2/yM37GL/klZ3ImB3AAwxjGRjbyb/7Nv/gXr/Eai1nMBjawP/tzOIdzJEeyFVlOpIiIiBTMnEsv42JmPwW+D+znnPtHZHkj8C7wqHPusHJtH5oyZYqbPbvyzZVXcAVnczaHczj3cR+GVbwOpeQc3H47/OQn8NJLMHq0H1V8yindm8Wf4Rku4ALu4q5uWb/JTGYndmILtqCGGv7G33iFVzCMAzmQ4zmeAzmQCUzo8+dMRESkUGY2xzk3pST7SjkQ/CtwMNDknNuYse4x4IPOuSxjbXu+fahSgeBiFvMKr/AO7/AYj3E1V3M0R3MLt9BATLtxH+UcPPwwXHkl/PGP/vcDD4QPBa3f69dDbS0MHAi1W78JE9+gfswqmppg4rI9qV85lvfe809jee89aG4GN/lVFky9lad2/h0rhrwOwNj28exS+yG2Yis2Z3MMoy3jv3baaaONNaxhCUt4m7epp55hDGM0o5nMZLZlW8YwhsEMppFGWoP/VrOad3mXVayimWY2spEBDGBwxn8Ox1rWsoY1rA3+cziGZfzXRBMW/Ackfo0GyuH/J32t1DYOx3rW8y7vspKVbGQjbbRRRx1DGdrlPAxkYJcAPjOYT7ouTWl2WajWY6d9/Go9dtrHT/PYHXSwlrWsivy3mtXUUcegHP/twR7UU1/WuvWnQPB5YDPn3NiYdXcAnwEanHMt5dg+VKlA8BzO4RIu2fT7yZzMNVzTp6aNKdSiRf5JJDNmwGuv+QCwqcn3I1y/Htauzb394MG+b+X77wd9D60Ddn8apj7Oh055nIG7zWMxi3mbtwGoi/xXS+2m/x/EIDZnc8YxjjbaWMUq3uEd5jOfjWzMWYfQAAbQRlveL6aaoOttBx2J9ttf1VGnOSRFpF+pp5422nJ+v7/DO2xGlomDS6Q/BYKvAQOcc90mVjGz3wPHAyOcc++XenszOxU4Nfh1O+jzE92NBlakXQmpGF3v6qFrXV10vatLsdd7YpIWzyTSTkWth6xhc2OkTMm3d85dB1yXr4J9hZnNLtW/DqT30/WuHrrW1UXXu7r0huud9vQqbwGjzSyug9wWwIo8zbo93V5ERESkaqUdCM4K6rBXdGEw6nc3yDvzcE+3FxEREalaaQeCtwMOOCtj+SlAE3BLuMDMtjGz7Yvdvgr0m2ZuSUTXu3roWlcXXe/qkvr1TnWwCICZXQWcAdwN/BnYAf9kkMeAg5xzHUG5hfjOkVbM9iIiIiLSVW8IBGvxGb1TgUn40TO3A+c559ZGyi0kPhBMtL2IiIiIdJV6ICgiIiIi6Ui7j6AUycxqzOxsM5trZs1mtsjMLjWzQWnXTYpnZt81szvNbIGZuSATnqv8f5jZg2a2xsxWm9n9ZrZbZWorPWFmHzSzH5vZ42a2PLiGz5jZ9+M+x2a2nZnNMLOVZrbOzP5hZgelUXcpXHD9bjGzl81slZmtD76/LzOz8VnK63r3I2bWFPlu/2XM+lSuedrzCErxLsf3hbwbuJTOvpG7m9nB6hvZZ/0MeA94Chieq6CZTQVmAkuA84LFZwD/MLN9nHPPl6+aUgJfBk4H7sUPbGsFDgR+CnzWzKY65zaAHywH/AtoAy4GVuEHxf3VzP7TOfdgCvWXwmwJjMd/Zy/GX8ud8d2ajjWz3Zxzy0DXux/7MRA7CXSq19w5p58+9gPsBHQAd2UsPxM/ivoLaddRP0Vf2w9E/v8FYGGOsk8Cq4EtIsu2CJb9Le33op+813oKMCxm+U+Dz/EZkWV3AO3AbpFlg4E38E9FsrTfj36Kvg8+E1zvc3S9++8PsAc+yPtWcL1/mbE+tWuupuG+6fOAAVdkLL8e/ySV4ypdISkN59yCJOXMbDKwJ3Cnc25JZPslwJ3AwWY2rjy1lFJwzs12zq2KWXV78PohgKCZ+JPATOfcM5Ht1wK/AT6Ivxekb3ojeB0But79UTCo9XrgfmB6zPpUr7kCwb5pT3xG8MnoQudcM/AM+pKoBuE1/nfMusfx/1D4cOWqIyW0ZfD6TvC6C9BA9msN+sz3GWbWaGajzWxLM/s48Otg1Z+DV13v/udsYHt81504qV5zBYJ90+b4x+dtjFm3BP/YvfoK10kqa/PgdUnMunDZFhWqi5RIkDk4F9+EdGuwWNe6fzkZWA4sAv6K7wt8nHPuH8F6Xe9+xMy2Bn4E/Ng5tzBLsVSvuQaL9E1NQFwQCNAcKaPnLPdfTcFr3H3QnFFG+o4rgL2B7znn5gXLdK37lxnAXHz/r93xTYKjI+t1vfuXa4EFwGU5yqR6zRUI9k3rgc2yrGuMlJH+K7y+DTHrdA/0QWb2E3zT0XXOuQsjq3St+xHn3GL8qGGAGWZ2FzDLzJqC667r3U+Y2XHAIcB+zrnWHEVTveZqGu6b3sI3/8bdNFvgm42VDezf3gpe45oLwmVxzQzSC5nZ+cAPgBuBr2Ws1rXux5xzzwFPA18PFul69wPB3+fL8H0/3zazycEgv4lBkWHBsuGkfM0VCPZNs/DXbq/oQjNrBHYDZqdQJ6msWcHr3jHrpuKnJ5hTuepIsYIg8IfA74CTXTBvRMTz+CajbNca9Jnv6wYCI4P/1/XuHwbi5ww8HHg18jMzWH9c8PvJpHzNFQj2Tbfj/9CflbH8FHw/glsqXSGpLOfcfPwXw2fMLOxoTPD/nwEeds69nVb9JBkzOw8fBN4EfNnFTAQfTCFxH3CAme0a2XYw/o/Iq2TMICC9T7bpnMzsQPxUQY+Drnc/sg7/XZz5E2Z+7w9+vzfta65nDfdRZnYVvj/R3fjUc/hkkceAg+L+oEjvZ2bH09l0cCZQj39yDMAbzrmbImX3AR7B9ze6KrLNWGBf59yzFam0FMXMTgd+CbyJHymc+Zl9xzn3QFB2Mv4PQSv+qUKr8f/w2xk43Dn310rVW4pjZnfjnyzyMH7uwEb8FE/H4vt/HRDOIafr3X+Z2STgdeBq59wZkeWpXXMFgn1UMM3EWfjHE00CVuAzhecF/7qQPsjMZgL7Z1n9qHPugIzye+OfRPEf+Czxv4DvOueeKmM1pQTMbBrwpRxFulxvM9sBuAh/f9TjH0N4vtPjxvoEM/sscAKwK77J0OEDwgeAS5xzb2aU1/Xuh7IFgsG6VK65AkERERGRKqU+giIiIiJVSoGgiIiISJVSICgiIiJSpRQIioiIiFQpBYIiIiIiVUqBoIiIiEiVUiAoIiIiUqUUCIpIKszsADNzZnZimfZ/oJk9bmZrynkcEZG+TIGgiPQ7ZjYCmA4MAv4bOB74e6qV6kfMbJKZnW9mu6Vdl0oI/tFyvpkNT7suIqVWl3YFRETKYE9gOPAV59z0lOvSH00CfggsBJ5JsyIVcgD+/U4D3k+zIiKlpoygiPRH44LX90q5UzOrNbOmUu5TRCRNCgRFegEzawyanuaZ2Xoze9/MnjezSyJlnJlNM7ODg75v683sbTP7hZkNztjf5mZ2qZk9Y2YrzazZzF4ys/8xs9qY49eb2TlB+fVmtsrMZptZ5kPRh5nZz81svpltNLPlZnabmX2gh+//TDN7JajnK2Z2ZpZy25rZTWa21MxazGyhmV1iZoMiZRYCvwt+fSQ4by6yfrSZXW1mi4J9LAp+H5VxrBODbQ82s3PN7DWgGfhssN7M7DQzmxOcs7Vm9oiZHVjkOUh8fYPyk4Jz8U5wLV4zs59lBqrBfeXMbLtg/eKg/LNmdliWuhxtZjOD+3B9cF9eGdwnJwKPBEVvDM+vmc0s4j1nPU6kzCAzuzB4fxuDc/J7M5uYsa/weh0Qc5yZwX0RXbYwWL69mf3JfF/SVWb2BzMbFyk3DZ8NBHg98n7PL/T9ivRGahoW6R2uBr4M/B64DP/Z3BY4KKPcHsAxwPVB2QOBbwAfMrNDnHMdQbldgE8DdwOvAQOAQ4GLgA8AXw13GPzR/Su++etvwM34gGfnYB+/DMoNA/4FTAB+C7wIjAe+DjxhZlOcc28U8d7PxGfwfg2sAT4PXGlmI51zP4rU88PAw/imuV8DS4Bdg/e/r5nt75xrBc4C/hM4FfgZ8HJkH+F7mBy8h6eA3YHTgIPMbC/n3JqM+v0v/vxdD6wG5gXLbwrq+gfgRqAB+CLwgJl92jl3bxHnItH1DYKgJ4FhwK+AV/HX77vBufiYc64tY9+/A1qD91MfnKcZZvZB59zCyDm6APge8BJwObAU2AY4GjgP39fyZ0GZ64B/BJu+U8gbTXCcFjMbgL8398Wf50vxn4vTgI8H99ziQo6bYQtgJv5z8v/w99NXgaHAx4Myvw5+Pwo4G1gRLH+uB8cV6T2cc/rRj35S/sE3Yf45TxkX/ByZsfwXwfJjI8sGAhazj5uAdmB8ZNk5wfY/iylfk3GcDcCuGWUm4gOkaQW+5wOC464Btowsr8cHOa0Zy58F5gJDMvZzVLCfEyPLTgyWHZBR9oJg+dczlp8eLP9JzD7mAU1ZjnlqxvI6YDbwetz5L+H1vSVYdlhG2UuC5V+JLDs/WPbHaJ3w/SgdcGFk2V7BsoeBxox9W7h95NqdWMh7LOI4pwTlLs4oc3iw/KZ81zxYNxNYmLFsYVD+sxnLrw6WbxdzDicV8371o5/e/KOmYZHeYRWwk5l9KE+5ec65GRnLLgpejwoXOOc2OOf8X1XfnDfSzEbjsys1wJTI9l8EVgI/zjyY68xAWVDu78CSoHl1dLDPdcDjdGZQCnWLi2R1nHMt+AxRHXBEcPyd8VnOW4GGjOP/M6hDkuMfBSzHZ7Kifh0sP6rbFnCNc259xrLj8AHsjIy6DAfuww+m2DZBfTLlvb5mVgN8EnjaOffnjLIXAh1Z3scvwnsCwDk3C1ibUc8vBq/fdc41Rzd2gQLeSy5Jj3MU/v1cmFHmT/hBKp8Kzkex3nLO3ZGx7OHgtZjrJ9LnqGlYpHc4C5+te97MFuD7YN0H3Oc6m3sh0swZcs4tNbP38U2+AJhZHfAd4AR8M6hlbDYi8v/bAs9k/kHOMAYYhQ+2lmcp05FleT7d3hO+uRA639MOweuPgp84YxMca2tgtstoNnXOtZnZK/im2UyvxCzbARhC7ubQsVm2zSXJ9R0DDMY3zWeWfc/MlkbKRi2IWfYu/rqGtsVnvp4trNoFS3qcrfHB2sqYdS8CuwGjgWVF1iPbOYGu50Wk31IgKNILOOfuMbNJwGHA/sDBwFeAf5jZwUGWrBCX4fve3Y5vDl2Gb2rdA/g5hQ8UCwPJB4PtKy08/qXA/VnKxAULpZCZDQRfn+XAF3Js90J5qlO09izLM/+REDZRl1upj5NrX9n+1mU7J9D9vIj0SwoERXoJ59x7+IEaNwdNsRfh++99CrgzKLZD5nZmNh7fJBnNbhwP/N05d2xG2ckxh34F2N7MGpxzG7NUbzl+kMZQ59yDSd9TQt3eE7Bj8Bq+p1eD1/YeHn8BsJ2Z1UWzgkEG9YPEZ4jivBqUf9w5t7YH9cmU5PouxzdL7xRTdgR+AM8zRR7/FfxAm13x/TSz6WkAl/Q4C4BDzWy4c+79jHU74vumhoM3wqmCRsbsZ2v8P4SKVYnAWCQV6iMokjLzc9MNjy4L+kg9Hfwa/cO2nZkdmbGL/wleZ0SWtZOR0TA/xcrZMVW4Bd9U/IOYullQn46g3F5mdkyW97FZ3PIEvmhmW0b2Ux/Usx0/wAH8uXgB+JrFTFVjZnVmFhcAZJqBb1o9OWP5KcHyuxPW+ff4788L41aaWZJm6jh5r29wLe4DdjezQzPKfieoV9L3kenW4PVn0SlcQuH9gO9bCPFBVymPMwP/fr6Tsf4/8aO97410nQib4Q/OKPt5YPMi6xnq6fsV6bWUERRJ3xBgqZndiw94luEzGKfhmzvvi5R9Hp8xvB6flToQP93Io/hm4NAfgK+a2e345tyx+Olp3qW7X+AHZfzAzPbETyHTjM84bUfnH9bv46fxuMPM7sAPEGnBjxo+DJiDH7lZqFfw089ci890fQE/ovUnzrlF4ANjMzse35H/OTMLp69pwveB/DR+6pRpeY51MfAZ4Goz2wN/vnfHN8PPC9bn5Zz7g5ndCJwR7OeP+MzUlsDeQZ2KmVsx6fX9HnAIfrDKr4D5wH7A5/ADen5HEZxzT5rZz/HB51PB/fM2/n48Bj/a9318H841wNfNbH2wbJlz7uG4/fbgONOALwH/E3Sd+Dv+3H4d3z/ze5F9zjOzB/H3veGzorvhB5zMx08BVKzHg9efm9kt+M/HC8653tb8L1K4tIct60c/1f6Dny7lQnwT2bvARvzUFr8Fto2Uc/g/jAcDT+CncnkHuIruU6o04acSeQP/R+tVfFblY8RM+wE04gO9F4Py7wOz6D7NShNwLj5g2YAPBl7Gz3v3HwW+7wPCuuDnyns1eO+vAt/Mss1E4Nrg/LQE52tOcP62ipQ7kexTiYzBz723GN9cuBg/ZcjojHJZ9xEpczx+Hr3VwXlbiH/G8eeKuA8SX9+g/Nb4AUbLgnOxAD+/X+ZUN+eTZeqToL4zY5Z/HngsuL7r8NP2XAHUR8ochp+HsTnYf7f9JHjPSY4zKLi+C4L3uSx43xNj9jcO341iNT6L9xd8c/tM4qePiXvvm+7LjOXnBHVoDdafX8nvCf3op1w/4VxNItLLmX86xu+ccyemXRcpPV1fEUmD+giKiIiIVCn1ERSRkjH/HOMxCYq+5wqfEqfPMbMxQLdnO2dY60o78jg10Wf05rDKObeh7JURkUQUCIpIKW2Ff7xaPgfi+231d7Pw/Rpz+RG+H19/sDRBmZPIP6hHRCpEgaBIH+Gc6wsT3L6NH82aT7mfXNFbfBH/3OdcFkCfub75JLn23Z6IIiLp0WARERERkSqlwSIiIiIiVUqBoIiIiEiVUiAoIiIiUqUUCIqIiIhUKQWCIiIiIlXq/wMdT1LHiOvJoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature = \"space_before_ponct_count\"\n",
    "\n",
    "if feature in train_df.columns:\n",
    "    plot_rep(train_df, feature, save=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85f4bb0a",
   "metadata": {},
   "source": [
    "## Try it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a93c71a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-val accuracy: 0.8434\n"
     ]
    }
   ],
   "source": [
    "logreg = LogisticRegression(solver=\"newton-cg\")\n",
    "\n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(cross_val_score(logreg, X, y, scoring=\"accuracy\", cv=5, n_jobs=-1)), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e875440b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-val accuracy: 0.9074\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1, verbose=0)\n",
    "\n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(cross_val_score(rfc, X, y, scoring=\"accuracy\", cv=5, n_jobs=-1)), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7db21c1e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-val accuracy: 0.897\n"
     ]
    }
   ],
   "source": [
    "etc = ExtraTreesClassifier(n_estimators=100, random_state=seed, n_jobs=-1)\n",
    "\n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(cross_val_score(etc, X, y, scoring=\"accuracy\", cv=5, n_jobs=-1)), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "07bdc0c6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-val accuracy: 0.9174\n"
     ]
    }
   ],
   "source": [
    "lgbm = LGBMClassifier(random_state=seed)\n",
    "\n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(cross_val_score(lgbm, X, y, scoring=\"accuracy\", cv=5, n_jobs=-1)), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "36cb839c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-val accuracy: 0.9156\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold()\n",
    "scores = []\n",
    "\n",
    "for train_index, val_index in skf.split(X, y):\n",
    "\n",
    "    X_train, X_val = X[train_index], X[val_index]\n",
    "    y_train, y_val = y[train_index], y[val_index]\n",
    "    \n",
    "    xgb = XGBClassifier(random_state=seed, use_label_encoder=False, eval_metric=\"logloss\")\n",
    "    xgb.fit(X_train, y_train)\n",
    "    \n",
    "    scores.append(accuracy_score(y_val, xgb.predict(X_val)))    \n",
    "    \n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(scores), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "44fdb831",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Custom logger is already specified. Specify more than one logger at same time is not thread safe."
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.005203\n",
      "0:\tlearn: 0.6862881\ttotal: 2.3ms\tremaining: 11.5s\n",
      "250:\tlearn: 0.3108395\ttotal: 553ms\tremaining: 10.5s\n",
      "500:\tlearn: 0.2595875\ttotal: 1.11s\tremaining: 9.94s\n",
      "750:\tlearn: 0.2322414\ttotal: 1.62s\tremaining: 9.16s\n",
      "1000:\tlearn: 0.2142721\ttotal: 2.14s\tremaining: 8.54s\n",
      "1250:\tlearn: 0.2001637\ttotal: 2.63s\tremaining: 7.87s\n",
      "1500:\tlearn: 0.1881426\ttotal: 3.12s\tremaining: 7.27s\n",
      "1750:\tlearn: 0.1769910\ttotal: 3.6s\tremaining: 6.68s\n",
      "2000:\tlearn: 0.1664409\ttotal: 4.08s\tremaining: 6.12s\n",
      "2250:\tlearn: 0.1570737\ttotal: 4.56s\tremaining: 5.57s\n",
      "2500:\tlearn: 0.1483805\ttotal: 5.04s\tremaining: 5.04s\n",
      "2750:\tlearn: 0.1407140\ttotal: 5.53s\tremaining: 4.52s\n",
      "3000:\tlearn: 0.1339696\ttotal: 6.02s\tremaining: 4.01s\n",
      "3250:\tlearn: 0.1276292\ttotal: 6.53s\tremaining: 3.51s\n",
      "3500:\tlearn: 0.1218820\ttotal: 7.04s\tremaining: 3.01s\n",
      "3750:\tlearn: 0.1164719\ttotal: 7.54s\tremaining: 2.51s\n",
      "4000:\tlearn: 0.1114868\ttotal: 8.05s\tremaining: 2.01s\n",
      "4250:\tlearn: 0.1070862\ttotal: 8.56s\tremaining: 1.51s\n",
      "4500:\tlearn: 0.1027664\ttotal: 9.07s\tremaining: 1s\n",
      "4750:\tlearn: 0.0986708\ttotal: 9.57s\tremaining: 502ms\n",
      "4999:\tlearn: 0.0949315\ttotal: 10.1s\tremaining: 0us\n",
      "Learning rate set to 0.005203\n",
      "0:\tlearn: 0.6861949\ttotal: 2.13ms\tremaining: 10.6s\n",
      "250:\tlearn: 0.3103607\ttotal: 511ms\tremaining: 9.67s\n",
      "500:\tlearn: 0.2598310\ttotal: 1.01s\tremaining: 9.12s\n",
      "750:\tlearn: 0.2333016\ttotal: 1.51s\tremaining: 8.55s\n",
      "1000:\tlearn: 0.2151271\ttotal: 2.01s\tremaining: 8.03s\n",
      "1250:\tlearn: 0.2015276\ttotal: 2.51s\tremaining: 7.51s\n",
      "1500:\tlearn: 0.1900358\ttotal: 3s\tremaining: 7s\n",
      "1750:\tlearn: 0.1787703\ttotal: 3.5s\tremaining: 6.5s\n",
      "2000:\tlearn: 0.1673492\ttotal: 4s\tremaining: 6s\n",
      "2250:\tlearn: 0.1571723\ttotal: 4.5s\tremaining: 5.5s\n",
      "2500:\tlearn: 0.1484360\ttotal: 5s\tremaining: 5s\n",
      "2750:\tlearn: 0.1407776\ttotal: 5.51s\tremaining: 4.5s\n",
      "3000:\tlearn: 0.1336746\ttotal: 6.01s\tremaining: 4s\n",
      "3250:\tlearn: 0.1272470\ttotal: 6.51s\tremaining: 3.5s\n",
      "3500:\tlearn: 0.1210607\ttotal: 7.02s\tremaining: 3s\n",
      "3750:\tlearn: 0.1155020\ttotal: 7.54s\tremaining: 2.51s\n",
      "4000:\tlearn: 0.1105021\ttotal: 8.06s\tremaining: 2.01s\n",
      "4250:\tlearn: 0.1057278\ttotal: 8.57s\tremaining: 1.51s\n",
      "4500:\tlearn: 0.1011812\ttotal: 9.08s\tremaining: 1.01s\n",
      "4750:\tlearn: 0.0971036\ttotal: 9.6s\tremaining: 503ms\n",
      "4999:\tlearn: 0.0931280\ttotal: 10.1s\tremaining: 0us\n",
      "Learning rate set to 0.005203\n",
      "0:\tlearn: 0.6863329\ttotal: 2.09ms\tremaining: 10.5s\n",
      "250:\tlearn: 0.3111406\ttotal: 503ms\tremaining: 9.52s\n",
      "500:\tlearn: 0.2591101\ttotal: 1.01s\tremaining: 9.11s\n",
      "750:\tlearn: 0.2318850\ttotal: 1.53s\tremaining: 8.65s\n",
      "1000:\tlearn: 0.2139058\ttotal: 2.04s\tremaining: 8.15s\n",
      "1250:\tlearn: 0.2000319\ttotal: 2.54s\tremaining: 7.61s\n",
      "1500:\tlearn: 0.1882086\ttotal: 3.05s\tremaining: 7.1s\n",
      "1750:\tlearn: 0.1769250\ttotal: 3.56s\tremaining: 6.61s\n",
      "2000:\tlearn: 0.1657983\ttotal: 4.08s\tremaining: 6.11s\n",
      "2250:\tlearn: 0.1564339\ttotal: 4.59s\tremaining: 5.6s\n",
      "2500:\tlearn: 0.1484891\ttotal: 5.14s\tremaining: 5.14s\n",
      "2750:\tlearn: 0.1412399\ttotal: 5.67s\tremaining: 4.63s\n",
      "3000:\tlearn: 0.1342031\ttotal: 6.17s\tremaining: 4.11s\n",
      "3250:\tlearn: 0.1282135\ttotal: 6.7s\tremaining: 3.6s\n",
      "3500:\tlearn: 0.1222432\ttotal: 7.23s\tremaining: 3.1s\n",
      "3750:\tlearn: 0.1168586\ttotal: 7.77s\tremaining: 2.59s\n",
      "4000:\tlearn: 0.1119901\ttotal: 8.28s\tremaining: 2.07s\n",
      "4250:\tlearn: 0.1073000\ttotal: 8.85s\tremaining: 1.56s\n",
      "4500:\tlearn: 0.1028325\ttotal: 9.38s\tremaining: 1.04s\n",
      "4750:\tlearn: 0.0986227\ttotal: 9.89s\tremaining: 518ms\n",
      "4999:\tlearn: 0.0947236\ttotal: 10.5s\tremaining: 0us\n",
      "Learning rate set to 0.005203\n",
      "0:\tlearn: 0.6862614\ttotal: 3.03ms\tremaining: 15.1s\n",
      "250:\tlearn: 0.3119182\ttotal: 537ms\tremaining: 10.2s\n",
      "500:\tlearn: 0.2588386\ttotal: 1.04s\tremaining: 9.3s\n",
      "750:\tlearn: 0.2313465\ttotal: 1.55s\tremaining: 8.75s\n",
      "1000:\tlearn: 0.2129496\ttotal: 2.05s\tremaining: 8.19s\n",
      "1250:\tlearn: 0.1990974\ttotal: 2.56s\tremaining: 7.67s\n",
      "1500:\tlearn: 0.1869994\ttotal: 3.08s\tremaining: 7.18s\n",
      "1750:\tlearn: 0.1753677\ttotal: 3.59s\tremaining: 6.66s\n",
      "2000:\tlearn: 0.1645088\ttotal: 4.1s\tremaining: 6.14s\n",
      "2250:\tlearn: 0.1549010\ttotal: 4.59s\tremaining: 5.61s\n",
      "2500:\tlearn: 0.1473005\ttotal: 5.08s\tremaining: 5.08s\n",
      "2750:\tlearn: 0.1400322\ttotal: 5.61s\tremaining: 4.59s\n",
      "3000:\tlearn: 0.1331497\ttotal: 6.13s\tremaining: 4.08s\n",
      "3250:\tlearn: 0.1267269\ttotal: 6.66s\tremaining: 3.58s\n",
      "3500:\tlearn: 0.1207318\ttotal: 7.19s\tremaining: 3.08s\n",
      "3750:\tlearn: 0.1151778\ttotal: 7.7s\tremaining: 2.56s\n",
      "4000:\tlearn: 0.1098344\ttotal: 8.22s\tremaining: 2.05s\n",
      "4250:\tlearn: 0.1048785\ttotal: 8.75s\tremaining: 1.54s\n",
      "4500:\tlearn: 0.1003580\ttotal: 9.25s\tremaining: 1.02s\n",
      "4750:\tlearn: 0.0960805\ttotal: 9.79s\tremaining: 513ms\n",
      "4999:\tlearn: 0.0920034\ttotal: 10.3s\tremaining: 0us\n",
      "Learning rate set to 0.005203\n",
      "0:\tlearn: 0.6864990\ttotal: 1.94ms\tremaining: 9.71s\n",
      "250:\tlearn: 0.3147178\ttotal: 587ms\tremaining: 11.1s\n",
      "500:\tlearn: 0.2629171\ttotal: 1.1s\tremaining: 9.91s\n",
      "750:\tlearn: 0.2351549\ttotal: 1.62s\tremaining: 9.17s\n",
      "1000:\tlearn: 0.2166187\ttotal: 2.14s\tremaining: 8.56s\n",
      "1250:\tlearn: 0.2026486\ttotal: 2.66s\tremaining: 7.98s\n",
      "1500:\tlearn: 0.1909019\ttotal: 3.18s\tremaining: 7.41s\n",
      "1750:\tlearn: 0.1797378\ttotal: 3.7s\tremaining: 6.86s\n",
      "2000:\tlearn: 0.1686583\ttotal: 4.2s\tremaining: 6.3s\n",
      "2250:\tlearn: 0.1585780\ttotal: 4.71s\tremaining: 5.76s\n",
      "2500:\tlearn: 0.1494737\ttotal: 5.22s\tremaining: 5.21s\n",
      "2750:\tlearn: 0.1414280\ttotal: 5.72s\tremaining: 4.67s\n",
      "3000:\tlearn: 0.1341827\ttotal: 6.24s\tremaining: 4.16s\n",
      "3250:\tlearn: 0.1274940\ttotal: 6.77s\tremaining: 3.64s\n",
      "3500:\tlearn: 0.1215108\ttotal: 7.28s\tremaining: 3.12s\n",
      "3750:\tlearn: 0.1161299\ttotal: 7.82s\tremaining: 2.6s\n",
      "4000:\tlearn: 0.1108636\ttotal: 8.35s\tremaining: 2.08s\n",
      "4250:\tlearn: 0.1059271\ttotal: 8.86s\tremaining: 1.56s\n",
      "4500:\tlearn: 0.1013509\ttotal: 9.42s\tremaining: 1.04s\n",
      "4750:\tlearn: 0.0969817\ttotal: 9.95s\tremaining: 521ms\n",
      "4999:\tlearn: 0.0929485\ttotal: 10.5s\tremaining: 0us\n",
      "Cross-val accuracy: 0.9172\n"
     ]
    }
   ],
   "source": [
    "cat = CatBoostClassifier(n_estimators=5000, random_state=seed, verbose=250, allow_writing_files=False)\n",
    "\n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(cross_val_score(cat, X, y, scoring=\"accuracy\", cv=5, n_jobs=1)), 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ebe9b21d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.024888\n",
      "0:\tlearn: 0.8528125\ttest: 0.8593750\tbest: 0.8593750 (0)\ttotal: 2.12ms\tremaining: 10.6s\n",
      "250:\tlearn: 0.9254687\ttest: 0.9112500\tbest: 0.9125000 (241)\ttotal: 536ms\tremaining: 10.1s\n",
      "500:\tlearn: 0.9528125\ttest: 0.9243750\tbest: 0.9250000 (461)\ttotal: 1.09s\tremaining: 9.8s\n",
      "750:\tlearn: 0.9671875\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 1.66s\tremaining: 9.41s\n",
      "1000:\tlearn: 0.9778125\ttest: 0.9231250\tbest: 0.9268750 (613)\ttotal: 2.26s\tremaining: 9.02s\n",
      "1250:\tlearn: 0.9871875\ttest: 0.9225000\tbest: 0.9268750 (613)\ttotal: 2.81s\tremaining: 8.44s\n",
      "1500:\tlearn: 0.9926563\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 3.38s\tremaining: 7.87s\n",
      "1750:\tlearn: 0.9959375\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 3.92s\tremaining: 7.28s\n",
      "2000:\tlearn: 0.9981250\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 4.47s\tremaining: 6.7s\n",
      "2250:\tlearn: 0.9992188\ttest: 0.9206250\tbest: 0.9268750 (613)\ttotal: 5.01s\tremaining: 6.12s\n",
      "2500:\tlearn: 0.9993750\ttest: 0.9193750\tbest: 0.9268750 (613)\ttotal: 5.56s\tremaining: 5.55s\n",
      "2750:\tlearn: 0.9998438\ttest: 0.9193750\tbest: 0.9268750 (613)\ttotal: 6.1s\tremaining: 4.99s\n",
      "3000:\tlearn: 1.0000000\ttest: 0.9200000\tbest: 0.9268750 (613)\ttotal: 6.65s\tremaining: 4.43s\n",
      "3250:\tlearn: 1.0000000\ttest: 0.9193750\tbest: 0.9268750 (613)\ttotal: 7.2s\tremaining: 3.87s\n",
      "3500:\tlearn: 1.0000000\ttest: 0.9200000\tbest: 0.9268750 (613)\ttotal: 7.74s\tremaining: 3.31s\n",
      "3750:\tlearn: 1.0000000\ttest: 0.9206250\tbest: 0.9268750 (613)\ttotal: 8.29s\tremaining: 2.76s\n",
      "4000:\tlearn: 1.0000000\ttest: 0.9193750\tbest: 0.9268750 (613)\ttotal: 8.85s\tremaining: 2.21s\n",
      "4250:\tlearn: 1.0000000\ttest: 0.9200000\tbest: 0.9268750 (613)\ttotal: 9.4s\tremaining: 1.66s\n",
      "4500:\tlearn: 1.0000000\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 9.95s\tremaining: 1.1s\n",
      "4750:\tlearn: 1.0000000\ttest: 0.9212500\tbest: 0.9268750 (613)\ttotal: 10.5s\tremaining: 550ms\n",
      "4999:\tlearn: 1.0000000\ttest: 0.9218750\tbest: 0.9268750 (613)\ttotal: 11s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.926875\n",
      "bestIteration = 613\n",
      "\n",
      "Shrink model to first 614 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f41255a6790>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train, x_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)\n",
    "\n",
    "cat = CatBoostClassifier(n_estimators=5000, random_state=seed, verbose=250, allow_writing_files=False,\n",
    "                         eval_metric=\"Accuracy\")\n",
    "\n",
    "cat.fit(x_train, y_train, eval_set=[(x_val, y_val)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ca5f1201",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 classifiers...\n",
      "Fitting classifier1: randomforestclassifier (1/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier2: xgbclassifier (2/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier3: lgbmclassifier (3/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier4: catboostclassifier (4/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.91375\n",
      "Fitting 4 classifiers...\n",
      "Fitting classifier1: randomforestclassifier (1/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier2: xgbclassifier (2/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier3: lgbmclassifier (3/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier4: catboostclassifier (4/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.913125\n",
      "Fitting 4 classifiers...\n",
      "Fitting classifier1: randomforestclassifier (1/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier2: xgbclassifier (2/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier3: lgbmclassifier (3/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier4: catboostclassifier (4/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.920625\n",
      "Fitting 4 classifiers...\n",
      "Fitting classifier1: randomforestclassifier (1/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier2: xgbclassifier (2/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier3: lgbmclassifier (3/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier4: catboostclassifier (4/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.908125\n",
      "Fitting 4 classifiers...\n",
      "Fitting classifier1: randomforestclassifier (1/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier2: xgbclassifier (2/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier3: lgbmclassifier (3/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting classifier4: catboostclassifier (4/4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    4.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9225\n",
      "Cross-val accuracy: 0.9156\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(\n",
    "    max_samples=None, bootstrap=True, random_state=seed, n_jobs=1, n_estimators=69,\n",
    "    criterion=\"gini\", max_depth=None, min_samples_split=7, min_samples_leaf=2,\n",
    "    max_features=0.4, ccp_alpha=6.32e-6\n",
    ")\n",
    "lgbm = LGBMClassifier(\n",
    "    objective=\"binary\", verbose=-1, random_state=seed, n_jobs=1, n_estimators=122,\n",
    "    num_leaves=20, min_split_gain=2.42e-7, min_child_weight=5.45e-7, min_child_samples=2,\n",
    "    subsample=0.910, subsample_freq=4, reg_alpha=6.92e-6, reg_lambda=8.82e-4\n",
    ")\n",
    "xgb = XGBClassifier(\n",
    "    eval_metric=\"logloss\", use_label_encoder=False, random_state=seed, n_estimators=178,\n",
    "    subsample=0.649, colsample_bytree=0.83, colsample_bylevel=0.75, colsample_bynode=0.67,\n",
    "    max_delta_step=0.14, reg_alpha=3.01e-5, reg_lambda=3.02e-7, grow_policy=\"lossguide\"\n",
    ")\n",
    "cat = CatBoostClassifier(\n",
    "    verbose=False, allow_writing_files=False, random_state=seed, n_estimators=222,\n",
    "    learning_rate=0.03, grow_policy=\"Depthwise\", bootstrap_type=\"MVS\",\n",
    "    leaf_estimation_iterations=10, leaf_estimation_backtracking=\"AnyImprovement\",\n",
    "    od_type=\"IncToDec\", l2_leaf_reg=1.61e-4, min_data_in_leaf=5, subsample=0.62\n",
    ")\n",
    "\n",
    "meta_cls = LogisticRegression()\n",
    "\n",
    "skf = StratifiedKFold()\n",
    "scores = []\n",
    "\n",
    "for train_index, val_index in skf.split(X, y):\n",
    "\n",
    "    X_train, X_val = X[train_index], X[val_index]\n",
    "    y_train, y_val = y[train_index], y[val_index]\n",
    "    \n",
    "    stacking = StackingCVClassifier(\n",
    "        [rfc, xgb, lgbm, cat], meta_cls, use_probas=False, cv=5, shuffle=True, random_state=seed,\n",
    "        verbose=1, n_jobs=1\n",
    "    )\n",
    "    stacking.fit(X_train, y_train)\n",
    "    \n",
    "    scores.append(accuracy_score(y_val, stacking.predict(X_val)))    \n",
    "    print(scores[-1])\n",
    "    \n",
    "print(\"Cross-val accuracy:\", np.around(np.mean(scores), 4))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9503e442",
   "metadata": {},
   "source": [
    "## Some feature selection using SFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f04e8b14",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  27 out of  27 | elapsed:   13.9s finished\n",
      "\n",
      "[2022-04-17 12:55:33] Features: 1/5 -- score: 0.7095[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  26 out of  26 | elapsed:   16.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:    1.8s finished\n",
      "\n",
      "[2022-04-17 12:55:52] Features: 2/5 -- score: 0.760375[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  23 out of  25 | elapsed:   14.5s remaining:    1.3s\n",
      "[Parallel(n_jobs=-1)]: Done  25 out of  25 | elapsed:   15.5s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:    2.1s finished\n",
      "\n",
      "[2022-04-17 12:56:09] Features: 3/5 -- score: 0.81325[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:   15.3s remaining:    1.4s\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:   15.7s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed:    2.4s finished\n",
      "\n",
      "[2022-04-17 12:56:27] Features: 4/5 -- score: 0.8665[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  20 out of  23 | elapsed:   13.6s remaining:    2.0s\n",
      "[Parallel(n_jobs=-1)]: Done  23 out of  23 | elapsed:   14.0s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:    3.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:    3.2s finished\n",
      "\n",
      "[2022-04-17 12:56:44] Features: 5/5 -- score: 0.8821249999999999"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(estimator=RandomForestClassifier(n_jobs=1,\n",
       "                                                           random_state=8005),\n",
       "                          floating=True, k_features=5, n_jobs=-1,\n",
       "                          scoring='accuracy', verbose=2)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=1)\n",
    "\n",
    "sfs = SequentialFeatureSelector(\n",
    "    rfc, k_features=5, forward=True, floating=True, verbose=2, scoring=\"accuracy\", cv=5, n_jobs=-1\n",
    ")\n",
    "\n",
    "sfs.fit(X, y, custom_feature_names=test_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "74351e7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: {'feature_idx': (5,),\n",
       "  'cv_scores': array([0.72375 , 0.696875, 0.698125, 0.715   , 0.71375 ]),\n",
       "  'avg_score': 0.7095,\n",
       "  'feature_names': ('sent_avg',)},\n",
       " 2: {'feature_idx': (0, 5),\n",
       "  'cv_scores': array([0.756875, 0.75125 , 0.761875, 0.751875, 0.78    ]),\n",
       "  'avg_score': 0.760375,\n",
       "  'feature_names': ('char_count', 'sent_avg')},\n",
       " 3: {'feature_idx': (0, 3, 5),\n",
       "  'cv_scores': array([0.80875 , 0.814375, 0.809375, 0.813125, 0.820625]),\n",
       "  'avg_score': 0.81325,\n",
       "  'feature_names': ('char_count', 'word_overlap', 'sent_avg')},\n",
       " 4: {'feature_idx': (0, 3, 5, 9),\n",
       "  'cv_scores': array([0.865625, 0.87125 , 0.85375 , 0.86625 , 0.875625]),\n",
       "  'avg_score': 0.8665,\n",
       "  'feature_names': ('char_count',\n",
       "   'word_overlap',\n",
       "   'sent_avg',\n",
       "   'space_before_ponct_count')},\n",
       " 5: {'feature_idx': (0, 3, 5, 9, 26),\n",
       "  'cv_scores': array([0.881875, 0.885625, 0.875625, 0.88    , 0.8875  ]),\n",
       "  'avg_score': 0.8821249999999999,\n",
       "  'feature_names': ('char_count',\n",
       "   'word_overlap',\n",
       "   'sent_avg',\n",
       "   'space_before_ponct_count',\n",
       "   'topk_frac_0')}}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfs.subsets_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "33d10ee6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  27 out of  27 | elapsed:    2.1s finished\n",
      "\n",
      "[2022-04-17 12:56:59] Features: 1/5 -- score: 0.69875[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  26 out of  26 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:    0.4s finished\n",
      "\n",
      "[2022-04-17 12:57:02] Features: 2/5 -- score: 0.7769999999999999[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  23 out of  25 | elapsed:    2.1s remaining:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  25 out of  25 | elapsed:    2.4s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:    0.3s finished\n",
      "\n",
      "[2022-04-17 12:57:05] Features: 3/5 -- score: 0.834875[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:    2.3s remaining:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   3 | elapsed:    0.4s finished\n",
      "\n",
      "[2022-04-17 12:57:07] Features: 4/5 -- score: 0.8786249999999999[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  20 out of  23 | elapsed:    2.5s remaining:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done  23 out of  23 | elapsed:    2.5s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:    0.5s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:    0.5s finished\n",
      "\n",
      "[2022-04-17 12:57:10] Features: 5/5 -- score: 0.891875"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SequentialFeatureSelector(estimator=LGBMClassifier(random_state=8005),\n",
       "                          floating=True, k_features=5, n_jobs=-1,\n",
       "                          scoring='accuracy', verbose=2)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm = LGBMClassifier(random_state=seed)\n",
    "\n",
    "sfs2 = SequentialFeatureSelector(\n",
    "    lgbm, k_features=5, forward=True, floating=True, verbose=2, scoring=\"accuracy\", cv=5, n_jobs=-1\n",
    ")\n",
    "\n",
    "sfs2.fit(X, y, custom_feature_names=test_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5995cfcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: {'feature_idx': (5,),\n",
       "  'cv_scores': array([0.705   , 0.70125 , 0.679375, 0.696875, 0.71125 ]),\n",
       "  'avg_score': 0.69875,\n",
       "  'feature_names': ('sent_avg',)},\n",
       " 2: {'feature_idx': (0, 5),\n",
       "  'cv_scores': array([0.781875, 0.774375, 0.765   , 0.76625 , 0.7975  ]),\n",
       "  'avg_score': 0.7769999999999999,\n",
       "  'feature_names': ('char_count', 'sent_avg')},\n",
       " 3: {'feature_idx': (0, 3, 5),\n",
       "  'cv_scores': array([0.835625, 0.828125, 0.83125 , 0.83625 , 0.843125]),\n",
       "  'avg_score': 0.834875,\n",
       "  'feature_names': ('char_count', 'word_overlap', 'sent_avg')},\n",
       " 4: {'feature_idx': (0, 3, 5, 9),\n",
       "  'cv_scores': array([0.8775  , 0.881875, 0.878125, 0.87625 , 0.879375]),\n",
       "  'avg_score': 0.8786249999999999,\n",
       "  'feature_names': ('char_count',\n",
       "   'word_overlap',\n",
       "   'sent_avg',\n",
       "   'space_before_ponct_count')},\n",
       " 5: {'feature_idx': (0, 3, 5, 8, 9),\n",
       "  'cv_scores': array([0.885   , 0.893125, 0.893125, 0.8875  , 0.900625]),\n",
       "  'avg_score': 0.891875,\n",
       "  'feature_names': ('char_count',\n",
       "   'word_overlap',\n",
       "   'sent_avg',\n",
       "   'is_last_ponct_count',\n",
       "   'space_before_ponct_count')}}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfs2.subsets_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "itmnlp",
   "language": "python",
   "name": "itmnlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}