Asmanova HW2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Описание датасета\n",
    "База данных о детях (их семьях, родителях и т.д.) для ранжирования заявок в детские сады.\n",
    "Входные данные:\n",
    "\n",
    "Parents\n",
    "\n",
    "Has Nurse\n",
    "\n",
    "Form\n",
    "\n",
    "Children\n",
    "\n",
    "Housing\n",
    "\n",
    "Finance\n",
    "\n",
    "Social\n",
    "\n",
    "Health\n",
    "\n",
    "Выходной параметр:\n",
    "\n",
    "CLASS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Загружаем данные**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('nursery.csv', header=None, sep = ',')\n",
    "data.columns = ['Parents', 'Has Nurse', 'Form', 'Children', 'Housing', 'Finance', 'Social', 'Health', 'CLASS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Parents</th>\n",
       "      <th>Has Nurse</th>\n",
       "      <th>Form</th>\n",
       "      <th>Children</th>\n",
       "      <th>Housing</th>\n",
       "      <th>Finance</th>\n",
       "      <th>Social</th>\n",
       "      <th>Health</th>\n",
       "      <th>CLASS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>usual</td>\n",
       "      <td>proper</td>\n",
       "      <td>complete</td>\n",
       "      <td>1</td>\n",
       "      <td>convenient</td>\n",
       "      <td>convenient</td>\n",
       "      <td>nonprob</td>\n",
       "      <td>recommended</td>\n",
       "      <td>recommend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>usual</td>\n",
       "      <td>proper</td>\n",
       "      <td>complete</td>\n",
       "      <td>1</td>\n",
       "      <td>convenient</td>\n",
       "      <td>convenient</td>\n",
       "      <td>nonprob</td>\n",
       "      <td>priority</td>\n",
       "      <td>priority</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>usual</td>\n",
       "      <td>proper</td>\n",
       "      <td>complete</td>\n",
       "      <td>1</td>\n",
       "      <td>convenient</td>\n",
       "      <td>convenient</td>\n",
       "      <td>nonprob</td>\n",
       "      <td>not_recom</td>\n",
       "      <td>not_recom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>usual</td>\n",
       "      <td>proper</td>\n",
       "      <td>complete</td>\n",
       "      <td>1</td>\n",
       "      <td>convenient</td>\n",
       "      <td>convenient</td>\n",
       "      <td>slightly_prob</td>\n",
       "      <td>recommended</td>\n",
       "      <td>recommend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>usual</td>\n",
       "      <td>proper</td>\n",
       "      <td>complete</td>\n",
       "      <td>1</td>\n",
       "      <td>convenient</td>\n",
       "      <td>convenient</td>\n",
       "      <td>slightly_prob</td>\n",
       "      <td>priority</td>\n",
       "      <td>priority</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Parents Has Nurse      Form Children     Housing     Finance         Social  \\\n",
       "0   usual    proper  complete        1  convenient  convenient        nonprob   \n",
       "1   usual    proper  complete        1  convenient  convenient        nonprob   \n",
       "2   usual    proper  complete        1  convenient  convenient        nonprob   \n",
       "3   usual    proper  complete        1  convenient  convenient  slightly_prob   \n",
       "4   usual    proper  complete        1  convenient  convenient  slightly_prob   \n",
       "\n",
       "        Health      CLASS  \n",
       "0  recommended  recommend  \n",
       "1     priority   priority  \n",
       "2    not_recom  not_recom  \n",
       "3  recommended  recommend  \n",
       "4     priority   priority  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Приведем данные к надлежащему виду**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12960,)\n"
     ]
    }
   ],
   "source": [
    "X = pd.get_dummies(data.drop('CLASS', axis=1))\n",
    "\n",
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(data['CLASS'].unique())\n",
    "y = le.transform(data['CLASS'])\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['recommend' 'priority' 'not_recom' 'very_recom' 'spec_prior']\n"
     ]
    }
   ],
   "source": [
    "print(data['CLASS'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Разбиваем данные на обучающую и тестовую выборки**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.35, random_state = 7)\n",
    "N_test, _ = X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Обучаем классификатор. Используем стандартный многослойный персептрон**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='logistic', alpha=0.0001, batch_size='auto',\n",
       "       beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(100,), learning_rate='adaptive',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=42465,\n",
       "       shuffle=True, solver='lbfgs', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_model = MLPClassifier(solver = 'lbfgs', learning_rate='adaptive', activation = 'logistic', random_state = 42465)\n",
    "mlp_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Пример:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Parents_great_pret  Parents_pretentious  Parents_usual  \\\n",
      "10609                   1                    0              0   \n",
      "\n",
      "       Has Nurse_critical  Has Nurse_improper  Has Nurse_less_proper  \\\n",
      "10609                   0                   1                      0   \n",
      "\n",
      "       Has Nurse_proper  Has Nurse_very_crit  Form_complete  Form_completed  \\\n",
      "10609                 0                    0              0               1   \n",
      "\n",
      "              ...          Housing_critical  Housing_less_conv  \\\n",
      "10609         ...                         0                  1   \n",
      "\n",
      "       Finance_convenient  Finance_inconv  Social_nonprob  Social_problematic  \\\n",
      "10609                   1               0               0                   1   \n",
      "\n",
      "       Social_slightly_prob  Health_not_recom  Health_priority  \\\n",
      "10609                     0                 0                1   \n",
      "\n",
      "       Health_recommended  \n",
      "10609                   0  \n",
      "\n",
      "[1 rows x 27 columns]\n",
      "[3]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([3], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()\n",
    "No = 5\n",
    "sample = X_test[No-1:No]\n",
    "cl = y_test[No-1:No]\n",
    "print(sample)\n",
    "print(cl)\n",
    "\n",
    "mlp_model.predict(sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Проверим работу классификатора**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train_pred = mlp_model.predict(X_train)\n",
    "y_test_pred = mlp_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Результат работы на обучающей и тестовой выборке:_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.999881291548 0.998456790123\n"
     ]
    }
   ],
   "source": [
    "print(mlp_model.score(X_train, y_train), mlp_model.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00154320987654\n"
     ]
    }
   ],
   "source": [
    "print(1 - mlp_model.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Все хорошо_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Подбор гиперпараметров**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " _Однако для лучшего результата подберем параметр регуляризации alpha:_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "alpha_arr = np.logspace(-3, 2, 10)\n",
    "test_err = []\n",
    "train_err = []\n",
    "for alpha in alpha_arr:\n",
    "    mlp_model = MLPClassifier(alpha = alpha, hidden_layer_sizes = (100,), \n",
    "                              solver = 'lbfgs', activation = 'logistic', random_state = 42)\n",
    "    mlp_model.fit(X_train, y_train)\n",
    "\n",
    "    y_train_pred = mlp_model.predict(X_train)\n",
    "    y_test_pred = mlp_model.predict(X_test)\n",
    "    train_err.append(np.mean(y_train != y_train_pred))\n",
    "    test_err.append(np.mean(y_test != y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEeCAYAAAByoJkBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4FFXa9/HvnQQCyKYoKCCCgApugBLx0YGMC6vijiKL\nu4wO4Dg6Ou5xZpx51PEdAedFfMXdEcYdAQW3gCuCiiKbgLKJ4MauLCH3+0dVkiYm0CHdqXTy+1xX\nX3SdOlV9p4x955xTdY65OyIiImWRFnUAIiKSepQ8RESkzJQ8RESkzJQ8RESkzJQ8RESkzJQ8RESk\nzJQ8RCo5M2tpZvlmttv/X8tSV6Q89AsmKcXMlprZz2a2MeY1Muq4RKqbjKgDECkjB05z97d2V9HM\n0t19R7GyNHfPj/fDylpfpLpQy0OqDDO72MzeM7P/Y2Y/ADlm9qiZjTazyWa2Ccg2s3Zmlmtma83s\nCzM7PeYcjxWvX+wzzjezmcXKrjWzl8P3vc1srpltMLOVZnZdnLH3MbNPzWy9mS03szt2UTfXzP5h\nZjPC+i+Z2d7Fqg00s2Vm9r2Z3RxzbJaZfRD+7KvMbJSZ1YgnRpFYSh6SimwX+7KAJUBj4K6wbn/g\nr+5eF5gJvAK8BuwHDAOeNrNDYs4RW/+9YuefABxqZm1iyi4Eng7fjwWudPf6wOHAbltIoU3AQHdv\nAPQBrjKzM3ZRfxBwCXAAkAcU77o7ATgEOBm43cwODcvzgGuARsDx4f6r44xRpJCSh6QaA14K/3Iu\neF0Ws3+Vu//b3fPdfQtBN9dL7v5BuL8DsJe7/6+757n728BEgoRRoLC+u2+N/XB3/wV4uaC+mbUF\nDiVIKgDbgMPNrL67r3f3T+P5odx9mrvPDd/PAcYB3UqrDjzh7vPc/WfgNqCfmcUm1Tvdfau7fw58\nFv7cuPsn7v5ReH2WAQ/t4nNESqXkIanGgTPcfe+Y19iY/StKOGZlzPumJdRZFpYXnL+kc8T6D0XJ\n5kLgxTBRAZwD9AaWht1LXXZzLgDM7Dgze9vMvjOzdcAQgtZBaWJjXA7UAPaNKVsd8/5nYK/wcw4x\ns4lm9q2ZrSdone3qc0RKpOQhVU1J00THlq0CDiz2V/pBwDdl+Iw3gP3M7GjgAoJkEnyQ+yx3P5Og\nS+wl4L9xnvM/Yf3m7t4QeJBd///Zotj77cAPcXzOaGAe0CbsIrtlN58jUiL90kgq2tWYx+7qfkjw\nl/gNZlbDzLKB0wi6ieI6t7tvB54F/gnsDbwOEJ5vgJk1CO/y2gjsKP1MO6kLrHX3bWaWRdCiKW29\nBCMYEG9nZnWAvwDPenzrK9QN4/rZzA4DroozPpGdKHlIKnql2HMez4flzq+/cHcqC7/4Twd6Ad8D\nDwCD3P3LXZyjJP8hGGx+ttitvAOBr8MuoSuBAQBm1iKMtXkp57sa+IuZbSAYwxhfws8R+/5J4DHg\nW6AmMLyUusVdT5CYNhCMd4zbTX2RElkyF4Mys57A/UA68LC7311CnZEE/yP/DFxcMMBoZtcAlxP8\nlfX/3H1E0gIVSSFm9jbwpLs/EnUsUn0lreVhZukEf9X1BNoD/c2sXbE6vQn6XtsS/JU2Oiw/giBx\ndAaOBk4zs9bJilUkBZWl604k4ZLZbZUFLHb3pWFXwTig+H3rfYHHAdx9BtDQzPYH2gEz3H1L2Hc8\nDTg7ibGKpBp1NUmkkpk8mrHz7YQrw7Ld1WkKzAF+Y2b7hAOCfYDS+opFqhV3/626rCRqyZzbKt6/\njH7V/Hb3BWZ2NzAV2Ax8Cmh+IRGRSiKZyeMb4MCY7QPZ+WGtkuo0D8sI/7J6BMDM/k7wINROzExN\ndxGRPeDu5Ro3S2a31Sygbbi+QE3gfIqmcCgwARgMED6Ju87d14TbjcN/WwBnEfMgVix3x9254447\nCt/vyXZpZfHsi2d/eeuX5Xy6FroWuha6Frt6JULSWh7unmdmQ4EpBLfqjnX3+WY2JNw/xt0nh7OQ\nLibonrok5hTPmVkjgidnr3b3Dbv6vOzs7HJtl1YWz7549pe3fnnOrWux59ullcWzL5795a1fnnPr\nWuz5dmll8eyLZ39561eIRGazin4F4Yu7+x133BF1CJWGrkURXYsiuhZFwu/Ocn3/6gnzKqJS/mUS\nEV2LIroWRXQtEiupT5gnm5l59+63MHx4d/r06Rp1OCIiKcHM8HIOmKf8MrQ/TZ3JtZ9/CQ+jBCJS\nTe08SbLESlYDIeVbHg6cT2tWdzyWaZ+M2+0xIlL1hH9JRx1GpVPadUlEy6NKjHmMZwmZS2dFHYaI\nSLVRJZIHwI5123nnnaijEBGpHqpM8uhY4wcePnMil1/m/PRT1NGIiFRtVSJ5XLN/U/pedw2PHHAz\nt0/szB9av8JTTzrqAhURSY6UTx639ejBOQ8/RNe//530z2fTYvTNPLjvrXQaciy3dXiFRV8qg4hI\ndFq2bMlbb71VrnM89thj/OY3v0lQRImR8rfq/vW114o20tLg7LOpc+aZHPr8S1xzza2sap/D+Aty\nOGvsadTM1O18ItXJpEnTGTlyKlu3ZpCZmbdHz4SV9xxV9k6w8j6iHuWL3U1PsmOHr3nwef+q3lH+\nRa1O/sU/Jrjn5+/6GBFJOSV9F0ycOM1bt77ZwQtfrVvf7BMnTov7vOU9x8CBAz0tLc1r167tdevW\n9Xvvvdc/+OADP/74471hw4Z+9NFHe25ubmH9Rx991A8++GCvV6+et2rVyp9++mmfP3++Z2Zmenp6\nutetW9f33nvvuOMv7TuSBExPEnkCKFfwcc5tlZ+3w9+//nmfm3GUL23UyTc8rSQiUpWU9F3Qvfst\nO33pF7x69Lg17vMm4hwtW7b0N998093dV65c6Y0aNfJXX33V3d1ff/11b9Sokf/www++adMmr1+/\nvn/55Zfu7r569WqfO3euu7s/9thjfuKJJ8b9mQWSmTxSfswjHpaexvH3nk3z7z/l9axbWH7Rrfx4\n8LH4hFeC3wURqXK2bi25V37KlHTMiOs1dWrJ59iyJX2PYnrqqafo3bs3PXv2BOCUU07h2GOPZdKk\nSZgZaWlpzJkzh19++YUmTZrQvn17IHlPiZdHtUgeBeo3TOPyyWez+Z1P+Ru38tWAW9ly5LEwYYKS\niEgVk5mZV2J5jx47SmhLlPzq3r3kc9SqtWOPYlq2bBnPPvsse++9d+HrvffeY/Xq1dSpU4fx48fz\n4IMP0rRpU0477TQWLly4R59TEapV8iiQ1SWNexedxUu3f8qVy25lzZDbyO90jJKISBUyfHh3Wre+\nZaey1q1vZtiwUyv0HLHzbrVo0YJBgwaxdu3awtfGjRu54YYbAOjevTtTp05l9erVHHbYYVxxxRW/\nOkdlkfJ3W+2pjAy47k9pLOt3FpdffQYtP3uZf/zxNurm5EBODpx+etBuFZGUVHBH1KhRt7FlSzq1\nau1g2LCeZbpTKhHnaNKkCUuWLOGkk05iwIABZGVlMXXqVE4++WS2b9/Ohx9+SNu2balRowYffPAB\np5xyCrVr12avvfYiPT298BwrV65k+/bt1KhRowxXIYnKO2iyqxfQE1gALAJuLKXOyHD/Z0DHmPJr\ngS+AOQRL0GaWcGyZB5BKkp/v/txz7s2b7vB/n/KCbz/iaPeOHd1fflkD6yIpIFHfBcnw8ssve4sW\nLbxhw4Z+3333+YwZM7xbt26+zz77+H777eennXaar1ixwr/99lvv1q2bN2jQwBs2bOi//e1vff78\n+e7uvm3bNu/Tp0/hMfEq7bqQgAHzpM2qa2bpwELgFOAbYCbQ393nx9TpDQx1995mdhwwwt27mFkz\n4B2gnbtvNbPxwGR3f7zYZ3gi41+/Hm69FZ5/Np9nLniZrrl3YmlpaomIVHJV9lmKckrVWXWzgMXu\nvtTdtwPjgDOK1ekLPA7g7jOAhmbWJNyXAdQxswygDkECSqoGDWDUKHhpQhrX5J5Fj30/4dsrbofb\nb4djNCYiIlIgmcmjGbAiZntlWLbbOu7+DXAfsBxYBaxz9zeSGOtOsrJg1izo0SuNI287k7vO+YTt\nNymJiIgUSGbyiPfb9VdNJzPbm6BV0hJoCtQ1swGJC233MjLguuvg44/hgxlpHH3Hmbw78pMggRQk\nkZdfVhIRkWopmXdbfQMcGLN9IEHLYld1modlpwBfu/uPAGb2AvA/wNPFPyQnJ6fwfXZ2dsIXuT/o\nIHjlFXjhBbjgwjR69TqTu9/oyz7vTgjGQu68E+64A/r2ZfrkyUwdOZKMrVvJy8yk+/DhdO3TJ6Hx\niIiUVW5uLrm5uYk9aXlH3Et7ESSmJQSth5rAbIIB8Ng6vQkGwgG6AB+G748juNOqNkHL5HHg9yV8\nRtx3HSTCunXuQ4e677+/+1NPuefvyHd/8UX3Dh182sEH+83777/TM0Y3t27t0yZOrNAYRaqjiv4u\nSBWlXRcq891WAGbWC7gfSAfGuvs/zGxI+K0/JqzzAMEtvZuBS9z9k7A8BzgfyAM+AS73YOA99vye\nzPhL89FHcOWVsN9+MHo0tGnt3NqpE3+bPftXdW/r0WPnmX9FJOF0t1XJknm3VVIfEnT3V4FXi5WN\nKbY9tJRjc4CcZMVWHgUD6iNGQJcucO21Rlq9BiXWTd+ypYKjExFJvmo5PUkiFAyoz5oF778Pb3/0\nc4n1vt34SwVHJiKSfEoe5dSyJUycCOsPaMX5tN5pXz9as8BbRROYiFQJV111FX/729+iDuNXqu3c\nVolkBvsc1I7JSwfTmVEczRf8TCavMoJj6s+MOjwRiVDLli155JFHOOmkk/bo+NGjRyc4osRQ8kiQ\nzMw8NtGHWfRhC3N4lV78lx7UqvVh1KGJVFvTJ00q9+3z5T3Hrgbz8/LyyMhI0a/h8t6uFeWLSnR7\nXvHlKj8kyy/b/7wyLXkpInumpO+CaRMn+s2tW5fr9vnynqP4MrT33HOPm5mPHTvWW7Ro4d26dXN3\n93PPPdf3339/b9CggXft2rVwBUF394suushvvTVYufDtt9/2Zs2a+X333eeNGzf2Aw44wB999NEy\nXZeY8nJ9/2rMI0H69OnKiBE96NHjNho0yGHKgfvyt1aryzR1s4gkztSRI7lryZKdyu5asoTXR42q\nsHM8+eSTtGjRgokTJ7Jx40b69esHwPTp01mwYAFTpkwBoE+fPixevJjvv/+eTp06MWBA0YQaZrbT\neh5r1qxhw4YNrFq1irFjx/L73/+e9evXx/0zJUqKtpcqpz59utKnT1fGjoW3XtrA/u+0gDVroEmT\n3R8sIgmVsXVrieXpU6bEPUN2aV+Qe3oLvofdVzk5OdSuXbuw/OKLLy58f8cddzBixAg2btxIvXr1\ndjoOoEaNGtx+++2kpaXRq1cv6taty8KFC8nKytqjmPaUWh5JcO65MHF6fbb2ORueeCLqcESqpbzM\nzBLLd/ToEd8atO7kde9e8jlq1SpXbAceWDQrU35+Pn/+859p06YNDRo0oFWr4A7NH374ocRjGzVq\nRFpa0Vd3nTp12LRpU7ni2RNKHknQoAH06AGvNr0MHn5YkyeKRKD78OHc0nrn2+dvbt2aU4cNq9Bz\nlLSEbGzZ008/zYQJE3jzzTdZv349X3/9NbBza0PL0FYjAwfCvff8D2empcF778GJJ0Ydkki1UnBH\n1G2jRpG+ZQs7atWi57BhZbpTKhHniF2GtiSbNm0iMzOTffbZh82bN3PzzTfvtL9ggLqyUfJIkp49\n4bLLjB+vuIxGDz+s5CESga59+pR7ZuvynuOmm25i2LBh3Hjjjdxyyy2/akUMHjyYKVOm0KxZMxo1\nasRf/vIXxowpmsWp+IB5ZWmFJHVixGSLamLEeP3+99C63nf88cFDYNmyoD9LRBJOEyOWLFWXoa32\nBg2CMS82xk85BcaNizocEZGEUfJIouOOg/x8WPSby2Ds2KjDERFJGCWPJDILBs7/7+Lu8O238Pnn\nUYckIpIQGvNIssWL4YQTYNXlt5O+aX2wCIiIJJTGPEqWsmMeZtbTzBaY2SIzu7GUOiPD/Z+ZWcew\n7FAz+zTmtd7Mhicz1mRp0wYOPhimt7kUnn4atDiUiFQBSUseZpYOFCwx2x7ob2btitXpDbRx97bA\nlcBoAHdf6O4d3b0jcAzwM/BismJNtkGD4KGpLaFjR3gxZX8MEZFCyWx5ZAGL3X2pB2uPjwPOKFan\nL/A4gLvPABqaWfGJoE4Blrj7iiTGmlT9+sHkyfDzhZdr4FwkSQqeh9Cr6JVMyUwezYDYL/yVYdnu\n6jQvVucC4D8Jj64C7bsvZGfDc3lnwmefwVdfRR2SSJVS3unFq/IrWZL5hHm8URdPj4XHmVlN4HSg\nxPESCGanLJCdnU12dnbcAVakgQNhzJhMBg8YAI8+Cn/9a9QhiUg1kZubS25ubkLPmbS7rcysC5Dj\n7j3D7ZuAfHe/O6bOg0Cuu48LtxcA3dx9Tbh9BnBVwTlK+IxKf7dVgS1boGlTmP/fOTS5uFfwxHl6\netRhiUg1VNnvtpoFtDWzlmEL4nxgQrE6E4DBUJhs1hUkjlB/4JkkxlhhatWCs8+GJz49Epo1g3AR\nGBGRVJS05OHuecBQYAowDxjv7vPNbIiZDQnrTAa+MrPFwBjg6oLjzWwvgsHyF5IVY0UbOBCeegq4\nLJyqXUQkRekhwQqUnw8tW8LkcRs4oncLWLhQqwyKSIWr7N1WUkxaGgwYAE+8VD/sw9IqgyKSmtTy\nqGDz5sGpp8LyZ94j/crLYP78uNdTFhFJBLU8UlD79kFPVe62/wmSxnvvRR2SiEiZKXlEYOBAeOpp\nCwbO9cS5iKQgdVtF4NtvgxbIN59+R50Oh8Dy5VC/ftRhiUg1oW6rFHXAAZCVBRM+bAwnn6xVBkUk\n5Sh5RGTQIHjySeDyy/XMh4ikHHVbRWTTJmjeHL6cv4PGWS1h0iQ46qiowxKRakDdVimsbl04/XQY\n/1w6XHKJBs5FJKWo5RGhKVPgttvgo/FfQ+fOsHJlMAmWiEgSqeWR4k4+GVasgIXbWgWrDL70UtQh\niYjERckjQhkZ0L9/zGSJ6roSkRShbquIffIJnHMOfDVvC9biQPjoI2jVKuqwRKQKU7dVFdCxI9Sp\nA+99XAsuvBAeeSTqkEREdkvJI2Jmxdb5ePRR2LEj6rBERHZJyaMSGDAAnnsOth56lFYZFJGUkNTk\nYWY9zWyBmS0ysxtLqTMy3P+ZmXWMKW9oZs+Z2XwzmxcuU1sltWgBRxwBkyejgXMRSQlJSx5mlg48\nAPQE2gP9zaxdsTq9gTbu3ha4Ehgds3sEMNnd2wFHAfOTFWtlUNh1dcEF8NZbsGbNbo8REYlKMlse\nWcBid1/q7tuBccAZxer0BR4HcPcZQEMza2JmDYDfuPsj4b48d1+fxFgjd+658MYbsHZHfTjzzHDi\nKxGRyimZyaMZsCJme2VYtrs6zYFWwPdm9qiZfWJm/8/M6iQx1sg1bAjdu8Ozz1I0WWKK34YsIlVX\nRhLPHe83X/F7jZ0grk7AUHefaWb3A38Gbi9+cE5OTuH77OxssrOz9yTWSmHQIPjnP+HKaf8TFLz/\nPpxwQrRBiUjKy83NJTc3N6HnTNpDguEAd4679wy3bwLy3f3umDoPArnuPi7cXgB0I0goH7h7q7D8\nRODP7n5asc9I+YcEY23bFtxsNXMmtHzunzB3bnDrrohIAlX2hwRnAW3NrKWZ1QTOByYUqzMBGAyF\nyWadu69x99XACjM7JKx3CjA3ibFWCjVrQr9+8PTTwODB8OKLsGFD1GGJiPxK0pKHu+cBQ4EpwDxg\nvLvPN7MhZjYkrDMZ+MrMFgNjgKtjTjEMeNrMPiO42+rvyYq1Mhk4MBgr9/20yqCIVF6a26qScYe2\nbYOccex3k+HOO2HGjKjDEpEqpLJ3W8keKJiu5MkngR49YNUqmDMn6rBERHai5FEJDRgQtDy256fD\nxRfriXMRqXSUPCqhtm3h4IODhwa59NJgBH3r1qjDEhEppORRSRV2XbVqBUcfrVUGRaRS0YB5JfXD\nD9CmTbBMbb1J44Kuq9dfjzosEakCNGBehe27L3TtCi+8QDDX1ezZ8PXXUYclIgIoeVRqhTPt1gpX\nGdTT5iJSSajbqhL75ZdgupI5c6DZj59Dnz6wdCmkp0cdmoikMHVbVXG1a8NZZ8EzzwBHHQUHHABT\np0YdloiIkkdlN2hQ2HUFRVO1i4hETN1WlVx+PrRsCZMmwZEHbQjWrP3yS2jcOOrQRCRFqduqGkhL\nC544f+opoH79oB/riSeiDktEqjm1PFLA3LnBNFfLlkH6B+/CFVfAvHnBRFgiImWklkc1cfjhsN9+\nMG0awcqC7sEqgyIiEVHySBGFA+dmcNllmixRRCKV1G4rM+sJ3A+kAw/HLkEbU2ck0Av4GbjY3T8N\ny5cCG4AdwHZ3zyrh2GrRbQXw7bfQvj188w3U2bgGDjss6MeqXz/q0EQkxVTqbiszSwceAHoC7YH+\nZtauWJ3eQBt3bwtcCYyO2e1Atrt3LClxVDcHHABZWfDKK0CTJvDb38L48VGHJSLVVDK7rbKAxe6+\n1N23A+OAM4rV6Qs8DuDuM4CGZtYkZr9GhGMUzrQLeuZDRCKVzOTRDFgRs70yLIu3jgNvmNksM7si\naVGmkLPOgnffhe+/R6sMikikkpk84h2MKK11caK7dyQYD/m9mf0mMWGlrrp14bTTglUGSdcqgyIS\nnYwknvsb4MCY7QMJWha7qtM8LMPdV4X/fm9mLxJ0g71T/ENycnIK32dnZ5OdnV3+yCuxgQPhjjtg\n2DCCVQazsuDuuyEzM+rQRKSSys3NJTc3N6HnTNrdVmaWASwETgZWAR8B/d19fkyd3sBQd+9tZl2A\n+929i5nVAdLdfaOZ7QVMBe5096nFPqPa3G1VIC8PmjeH6dPhkEOAU04JHho8//yoQxORFJH0u60s\ncOCu6pTG3fOAocAUYB4w3t3nm9kQMxsS1pkMfGVmi4ExwNXh4fsD75jZbGAGMLF44qiuMjKgf/9i\nkyWq60pEKtguWx5mZsAcdz+i4kKKX3VseQB88gmcey4sWQK2dUvQFJk1K5hBUURkN5Le8gi/mT82\ns2r/nEVl0rFjsLjg++8TvBkwQKsMikiF2u2Yh5ktBNoAy4DNYbG7+1FJjm23qmvLA+Dvf4cVK2D0\naODzz4PbsL7+WqsMishuJaLlEU/yaBm+LahoAO6+tDwfnAjVOXksWwbHHBM86lGzJsFdV3/5C/Ts\nGXVoIlLJVcj0JGGSaEjwNPjpQIPKkDiqu4MOCmbbnTw5LLjsMj1xLiIVZrfJw8yuAZ4C9gOaAE+Z\n2fBkBya7t9MStf37w5tvwnffRRqTiFQP8XRbzQG6uPvmcHsv4EN3P7IC4tul6txtBbBuXdACWboU\n9t6b4InzI4+E666LODIRqcwqclbd/FLeS4QaNoTu3eG558KCgskSq3FCFZGKEU/yeBSYYWY5ZnYn\n8CHwSHLDknjtNNNuwSqDH3wQaUwiUvXt7iHBNOB4YAtwIsEdV+8ULNgUterebQWwbRs0bRrzjOC9\n98L8+fCI8ruIlKyibtWd7e4dyvMhyaLkEbj66uAh85tvBtZolUER2bWKGvN4w8zODacqkUqooOvK\nHa0yKCIVIp7k8Tvgv8A2M9sYvjYkOS4pg+OPD7qvPvkkLNBkiSKSZLubVTcN6OHuae5ew93rhS/1\nh1QiZsUGznv0gJUr4YsvIo1LRKqu3U2MmA/8u4JikXIYODBYYTAvj2B+q0suUetDRJJGYx5VRNu2\nwd1Wr78eFlxySfD4+datUYYlIlWUxjyqkIEDY6YrOfhgOPpoePnlSGMSkaopnuTRALgY+Ju71wOO\nAE6N5+Rm1tPMFpjZIjO7sZQ6I8P9n5lZx2L70s3sUzN7JZ7Pq+7OPx8mTYKNG8MCTZYoIkkSz3Me\nDwI7gJPcvZ2Z7QNMcffOuzkunWAN81OAb4CZ7HoN8+OAEe7eJWb/H4FjgHru3reEz9BzHsWcfjqc\ndx4MHgxs2cL0/fZjaseOZKSlkZeZSffhw+nap0/UYYpIhBLxnEdGHHWOc/eOZvYpgLv/ZGY14zgu\nC1hcMH27mY0DzgDmx9TpCzwenneGmTU0sybuvsbMmgO9gbuAP8b9E1VzgwYFjY3Bg2H6m28yJS2N\nu955p3D/LUuWACiBiEi5xNNttS1sRQBgZvsR3+SIzYAVMdsrw7J46/wL+FOcnyWh008PpipZtQqm\njhzJXRt2Hp66a8kSXh81KqLoRKSqiCd5jAJeBBqb2d+B94B/xHFcvP1JxZtOZmanAd+Fc2jpLq8y\nqF0bzjoLnnkGMkq50yp9y5YKjkpEqprddlu5+1Nm9jFwclh0Ruy4xS58AxwYs30gQctiV3Wah2Xn\nAH3DMZFaQH0ze8LdBxf/kJycnML32dnZZGdnxxFa1TZwIPzxj9CncWaJ+3fUqlXBEYlIlHJzc8nN\nzU3oOXc7YL7HJzbLIBgwPxlYBXzErgfMuwD3xw6Yh3W6Ade7++klfIYGzEuQnx8sEnXXnyex8F/X\ncFc4zgFwc+vW9BwxQmMeItVYRQ2Y7xF3zzOzocAUIB0Y6+7zzWxIuH+Mu082s95mthjYDFxS2umS\nFWdVlJYGAwbAvBV96D0Cbhs1ivQ5c9hRq5YSh4gkRNJaHhVBLY/SffEF9OoVzMyelkawxsdJJwUF\nNeO5WU5EqqqKXIZWUswRR8C++8K0aWFBu3bBOh8vvRRpXCJSNSh5VGE7zbQLcNVVMHp0ZPGISNWh\nbqsqbNUqOPzw4N/atQkW/TjoIHjrraAlIiLVUoUsQ1uZKXnsXseO08nPn8ree2eQmZnHA3svp22T\nvWHEiKhDE5GIKHkoeezSpEnTufTSKXz33V2FZSe2GMqba5+k5rerYK+9IoxORKKiAXPZpZEjp+6U\nOADeXf4An9RqEqwcJSKyh5Q8qrCtW0t+jOeFJp01cC4i5aLkUYVlZuaVWD6naSv46SeYObOCIxKR\nqkLJowobPrw7rVvfslNZ69Y3M3R4dxgyBB58MKLIRCTVacC8ips0aTqjRr3O6tXpfPnlDsaNO5W+\nfbvCd98jV7jMAAAVDElEQVTBoYfCV1/B3ntHHaaIVCDdbaXkETd3OP54uOEGOPvssPDCC+G44+Ca\nayKNTUQqlu62kriZBYnj7ruDRAIET5w/+GBMgYhIfJQ8qpEzzoC1a6FwVdoTT4T0dEjwPP8iUvUp\neVQj6elw3XVw771hgZnmuxKRPaIxj2rml1+gVSt4881g3is2bAjmu5o3Dw44IOrwRKQCaMxDyqx2\nbRg2DP75z7Cgfn3o1w/Gjo00LhFJLUlteZhZT+B+gpUEH3b3u0uoMxLoBfwMXOzun5pZLWAakEmw\n2uFz7p5TwrFqeeyBn36CNm1gzhxo1gyYPRv69oWvvw76tkSkSqvULQ8zSwceAHoC7YH+ZtauWJ3e\nQBt3bwtcCYwGcPctwG/dvQPQAehpZsclK9bqZp99YPDgmIl1O3QIssikSZHGJSKpI5ndVlnAYndf\n6u7bgXHAGcXq9AUeB3D3GUBDM2sSbv8c1qkJ1ADykxhrtXPttUFP1fr1YYEGzkWkDJKZPJoBK2K2\nV4Zlu6vTHIKWi5nNBtYAU91dEzEl0EEHQc+e8NBDYUG/fjBrVvDEuYjIbiQzecQ7GFG8380B3H1H\n2G3VHDjOzA5PZHACf/oT3H9/sMAgtWrBRRfBmDFRhyUiKaDkObsT4xvgwJjtAwlaFruq0zwsK+Tu\n683sbYKxk7nFPyQnJ6fwfXZ2NtnZ2eWJuVrp0CG4Xfc//4GLLyaYLPGEE+Avf4HMzKjDE5EEyc3N\nJTfBDwMn7W4rM8sAFgInA6uAj4D+7j4/pk5vYKi79zazLsD97t7FzPYF8tx9nZnVBqYA/+vuk4t9\nhu62Kqc33gimtpozB9LSgFNPhUsuCea9EpEqqVLfbeXuecBQgi/+ecB4d59vZkPMbEhYZzLwlZkt\nBsYAV4eHHwC8ZWafESSdqcUThyTGyScHjYzJBVdXA+ciEgc9YS4880wwP+K0aUBeXjCa/tprcOSR\nUYcmIklQqVsekjrOOw+WLYMPPwQyMuCKK7RQlIjskloeAsDIkUHL4/nngW++CVody5dD3bpRhyYi\nCaaWhyTMZZfB9OmwaBHB0+bZ2fD001GHJSKVlJKHALDXXsFY+X33hQUFA+dq2YlICdRtJYUKljVf\nsACa7JcfbDzxRLB+rYhUGeq2koRq3BguuAAeeIDgoY/f/U637YpIidTykJ0sXhw0NL7+Gupu/TGY\nu33xYmjUKOrQRCRB1PKQhGvTBrp1g0ceIUgYffvCo49GHZaIVDJqecivzJgB558f3HlV4+MPYeBA\n+PLLcP4SEUl1anlIUhx3XPCQ+bPPhhv16gWTYImIhJQ8pEQ33AD33guOaeBcRH5FyUNK1KsXbN8e\nNjgGDAgeP19ZfEZ9EamulDykRGlpcP31QeuDunWhf394+OGowxKRSkID5lKqbdvg4IPhlVegY40v\noEcPWLoUatSIOjQRKQcNmEtS1awJf/hD2Po44oiiTCIi1Z5aHrJLGzZAq1bw8cfQ8v3/BM98vP56\n1GGJSDmkRMvDzHqa2QIzW2RmN5ZSZ2S4/zMz6xiWHWhmb5vZXDP7wsyGJztW+bX69eHyy+Ff/wLO\nOQc+/zyceldEqrOktjzMLJ1gHfNTgG+Amex6HfPjgBHhOub7A/u7+2wzqwt8DJxZ7Fi1PCrAqlVB\nr9WiRdDo3j8Ht2EVTr8rIqkmFVoeWcBid1/q7tuBccAZxer0BR4HcPcZQEMza+Luq919dli+CZgP\nNE1yvFKCpk3hzDPDRz2GDAlm2v3ll6jDEpEIJTt5NANWxGyvDMt2V6d5bAUzawl0BGYkPEKJy/XX\nB7Pt/rJ/K+jcGf7736hDEpEIZST5/PH2KRVvPhUeF3ZZPQdcE7ZAdpKTk1P4Pjs7m+zs7DIHKbvX\nvn2QMx5/HH531VVw111w0UVRhyUiccjNzSU3Nzeh50z2mEcXIMfde4bbNwH57n53TJ0HgVx3Hxdu\nLwC6ufsaM6sBTARedff7Szi/xjwq0DvvwKWXwoK5O0hvezC89BJ07Bh1WCJSRqkw5jELaGtmLc2s\nJnA+MKFYnQnAYChMNuvCxGHAWGBeSYlDKt6JJwaztL/0SjpceaXmuxKpxpL+nIeZ9QLuB9KBse7+\nDzMbAuDuY8I6DwA9gc3AJe7+iZmdCEwHPqeoG+smd38t5txqeVSwF16Au++GD19ajbVvFzxx3qBB\n1GGJSBkkouWhhwSlTHbsgHbtgmmuuj7QD7p2haFDow5LRMogFbqtpIpJT4frroN77gGuuiroulIC\nF6l2lDykzAYPhlmzYO5+2ZCfD+++G3VIIlLB1G0le+Suu2DxYni0w4hg3dr//CfqkEQkThrzUPKI\nzE8/QZs28MW762h6QitYuBAaN446LBGJg5KHkkek/vCHYNr2e368DNq2hT//OeqQRCQOSh5KHpFa\ntgw6dYJlz8+i7iXnBf1Y6elRhyUiu6G7rSRSBx0EPXvC6JnHwr77wpQpUYckIhVELQ8pl9mzoU8f\nWHr7I9SY+KJWGhRJAWp5SOQ6dIDDD4dn/AL44IOgL0tEqjwlDym3G26Au0fVwQcMhIceijocEakA\nSh5SbiefDJmZkHvY72DsWNi2LeqQRCTJlDyk3MzgT3+CO545LJj46sUXow5JRJJMyUMS4rzzYPly\n+PLkqzRVu0g1oOQhCZGRAX/8I9z28ZnB0+bz5kUdkogkkW7VlYTZvBlatoSF593KPhkbYOTIqEMS\nkRLoCXMlj0rn9tth+5Ll/OO1jkE/1l57RR2SiBSTEs95mFlPM1tgZovM7MZS6owM939mZh1jyh8x\nszVmNifZcUpiDB0KD05uwdZjT4Bx46IOR0SSJKnJw8zSgYIlZtsD/c2sXbE6vYE27t4WuBKIHW19\nNDxWUkTjxnDBBTB+Hw2ci1RlyW55ZAGL3X2pu28HxgFnFKvTF3gcwN1nAA3NbP9w+x1gbZJjlAS7\n7jq4/vUe5P/4E8ycGXU4IpIEyU4ezYAVMdsrw7Ky1pEU0qYNdM1O4/0jhqj1IVJFZST5/PGOZhcf\nuIl7FDwnJ6fwfXZ2NtnZ2fEeKkn0pz/BVedeysebDsHuuw/23jvqkESqrdzcXHJzcxN6zqTebWVm\nXYAcd+8Zbt8E5Lv73TF1HgRy3X1cuL0A6Obua8LtlsAr7n5kCefX3VaVWLdu8HjeAFqe1zlYOUpE\nKoVUuNtqFtDWzFqaWU3gfGBCsToTgMFQmGzWFSQOSW033AA5a67CH3wQlORFqpSkJg93zwOGAlOA\necB4d59vZkPMbEhYZzLwlZktBsYAVxccb2bPAO8Dh5jZCjO7JJnxSmL16gWzMk9g09Ya8PbbUYcj\nIgmkhwQlqR57DDb+778ZdmQuPPts1OGICHrCXMkjBWzbBke13MAXmw4iY+E8OOCAqEMSqfZSYcxD\nqrmaNeGya+szrXE/ePjhqMMRkQRRy0OSbv166NtiNm/udToZy78OpuAVkcio5SEpoUED6PK7Dqzw\n5jBpUtThiEgCKHlIhbjmGrh7w1VsG6knzkWqAiUPqRBNm0L+Of3Y/uHH8NVXUYcjIuWk5CEV5g9/\nrsUTXMT2f4+JOhQRKSclD6kw7dvDp1lDyHvoUdi6NepwRKQcdLeVVKh33oE5p3Tkm0N2UKPRPuRl\nZtJ9+HC69ukTdWgi1UYi7rbSPZNSofLXT2L2juU89MVPhWV/+PwLePghJRCRFKKWh1SoIZ2yGPPp\nrxeIGtIpizEfz4ggIpHqR895SMr57uuSJ0yu8cU8ePxxmDcPduyo4KhEpKzUbSUVarOV/Cv3/fZM\nPr7rNdqu/yt7bf6OtGM6Yp07Q+fOcOyxcPDBYOX6Q0lEEkjdVlKhunU6n/0//ZjxLCks60drFrc9\nlguvHMfMmbBoxk+0+P5jTmsyky7pMzl47Swyd/xMWtax2LHHBgmlc+fg4RElFJEy06y6Sh4pZ9Kk\n6Vx7+QM0WL2BvdjCZmqxfv96/OvhYfTp07Ww3o8/wqxZMHNm8O+yD7+l/c+z6NV4FlnM5KDvZ5JR\nK4P042JaJ507w777RvjTiRSZPmkSU0eOJGPr1kp3V2GlTx5m1hO4H0gHHo5dfjamzkigF/AzcLG7\nf1qGY5U8UtCkSdMZNep1tmxJp1atHQwbdupOiaM0q1YVJZSZHzlrPlpGZ5tFj0YzOWbHTJqt+Zi0\nffchPSumddKpUzC5lkgFmj5pElOuuYa7lhS1sG9p3ZoeI0ZUigRSqZOHmaUDC4FTgG+AmUB/d58f\nU6c3MNTde5vZccAId+8Sz7Hh8UoeodzcXLKzs6MOo0K5w9KlRa2TWR/ls27WIg7OeIJLG2/m6G0z\n2X/NZ3Bg86CFUtA66dAB6tTZ6VyV8a/ERMSUjN+LVL1WFfL/yNatsH49t559Nn97771f7b7tt7/l\nr1OmQI0ayY2jFAXX6a6pUyv1cx5ZwGJ3XwpgZuOAM4DYBNAXeBzA3WeYWUMz2x9oFcexEqM6Jg8z\naNUqePXrB5BGfv6hDB9egx+Pu5+7Z8KnM/PYPnsup2yaxUkzZnLEL0/Q6Pv52CFtScsKWifTf/mF\nF+6+h/tXf1t47qifPZk+aRIvXH4l969eVa6YEv17kai4EinemHZ7LcIv/l+91q0rubykV34+NGhA\nxqZNJX5E+vTpULt2sNBNw4ZBq3hPX2VMQLHX6a4yHVmyZCaPZsCKmO2VwHFx1GkGNI3j2J0U/8Uo\n63ZpZfHsi2d/eeuXRXW+FmlpwbDHoEHBKzf3XU44IZu5c49m1qzL+NfLuXy/ogs1F3xOr59mUnfK\niyxfPY3787YEsQDZwP2rV/GnQRfRtVcPctesIbtJk5Jj38W+ePaXVn/Cq69x/9qfdtoXG1PcPv8c\nFi0qMZaybgP8e8IrjN+08VdxXX/xJeSfeALZLVsG67Wkpwf/FrzS08ldtozsQw/dqax4ndjt3Llz\nye7UqfT6YdkzN97E6JjEURDT8GF/oOuaNUVf7K+9BsuXw/r15C5dSnZaWuG+3LVryYbCL+bctDSy\nmzcv2t64kezDD4fmzeHww4OypUvJ7tq16Mu8YUOoVQvM+LjtYbB44a/+c3x79DEw60NyX3uN7KOO\n2nUiWrSoKL6lS8k2K9q3YUOQgMqQbP57w008UOw6lUcyk0e8/UkJuV2mOn9h7u7cuhbZdOgQ9Fat\nXJlLzivZbNmSxWefZXHnnd+Tt2Y+sDyoT5A8AFashevf7s37G5/nf+r1LvH8u9oXz/7S6q9cO6XE\n/SvWwc3v9sYsaHlZGqQVvLcgecbue/e7X7h7TlD/jTXP8/FPvQvrvbbqeT7fWLQ9ecXzzN/Su/Ac\nryx7nsX5vXc6/7zNL5YY1/dr8xg9ewvr17YgLT+PNM8jLX9H+H4L6fl5PL7yc+yA2qTn55GWn4f5\njsL3Ox3jwfZzPy2gaYNWpOXnlXxMWL/BupJnafal3/D+Pe/wS40G/FyjAV+sacAj6SewuUYDXl77\nMm+2Hcbmxg3YlN6ADxaN4pB2fyPfjfx8mD8/h7a1c8jfAvk/w6JFObRal0N+PoWvpUtzaPb8cTuV\nFbwWf53O+bT+1V2Fb33RimntjB9+mMG++/Yi+Ft59374IYd9982BTKAxsJ9TxzdTd8d66m1dT73V\n66m7aj31dqynXv566uYXvF8cvM9fT8amr+P6rHglc8yjC5Dj7j3D7ZuA/NiBbzN7EMh193Hh9gKg\nG0G31S6PDcs14CEisgcq85jHLKCtmbUEVgHnA/2L1ZkADAXGhclmnbuvMbMf4zi23D+8iIjsmaQl\nD3fPM7OhwBSC223Huvt8MxsS7h/j7pPNrLeZLQY2A5fs6thkxSoiImWT0g8JiohINDQxooiIlJmS\nh4iIlFmVSh5mdoaZPWRm48zs1KjjiZKZtTKzh83s2ahjiZqZ7WVmj4e/GxdGHU9loN+PIvreKGJm\nh5nZaDN71sx+t8u6VXHMw8waAv9098ujjiVqZvasu58XdRxRMrNBwE/uPsnMxrn7BVHHVFno96OI\nvjeKmFka8Li7DyqtTqVveZjZI2a2xszmFCvvaWYLzGyRmd1Y7LBbgQcqLsqKsYfXokoq47WIncmg\nSq00pd+JIgm4FlXme6M818LMTgcmApN3+SHuXqlfwG+AjsCcmLJ0YDHQEqgBzAbaETytfjdwctRx\nR30tYvY/G3XcUV8LYCDQJ6zzTNSxR3gdBgH/AppWxd+PPb0WVfF7o7y/F2H9ibv6jErf8nD3d4C1\nxYoLJ1109+1AwcSJQ4GTgXMLniepSspyLcxsn/AJ/g5V8S/PMv5evACcY2b/l+DB1CqjLNfB3Z90\n92vdfVVV/P3Y02sBDKOKfW+U4/eim5mNCH83Ju3qM1J1GdoSJ11092HAqGhCikxp1+InYJcDXlVQ\nadfiZ+DSaEKKxG4nJa1Gvx/xXIuRwMiKDCoi8VyLacC0eE5W6Vsepah6o/x7TteiiK5FQNehiK5F\nkYRei1RNHt8AB8ZsH0iQRasjXYsiuhYBXYciuhZFEnotUjV5FE66aGY1CSZOrFJ92WWga1FE1yKg\n61BE16JIQq9FpU8eZvYM8D5wiJmtMLNL3D2PYHB8CjAPGO/VYOJEXYsiuhYBXYciuhZFKuJaVMmH\nBEVEJLkqfctDREQqHyUPEREpMyUPEREpMyUPEREpMyUPEREpMyUPEREpMyUPEREpMyUPkXIws6Vm\ntk9564ikGiUPkfKJ5ylbJ1gzQqTKUPIQiZOZvWhms8zsCzO7oti+luEKbU+Z2bxwDejaMVWGmdnH\nZva5mR0aHpNlZu+b2Sdm9p6ZHVKhP5BIOSh5iMTvUnc/FugMDC+hK+oQ4N/u3h7YAFwds+97dz8G\nGA1cH5bNB37j7p2AO4C/JzV6kQRS8hCJ3zVmNhv4AGgOtC22f4W7fxC+fwo4MWbfC+G/nxAsAwrQ\nEHguXGf6/wCHJyNokWRQ8hCJg5llEyxV2sXdOxCs/1yrWLXY8Q8rtr01/HcHRSt4/hV4092PBE4v\n4XwilZaSh0h86gNr3X2LmbUDupRQp4WZFZRfCLwTxzlXhe8vSUyYIhVDyUMkPq8BGWY2j2BsoqB7\nKrZ1sRD4fVinAcH4RvE6HrN9D/APM/sESEdLpkoK0XoeIglgZi2BV8IuKJEqTy0PkcTRX2JSbajl\nISIiZaaWh4iIlJmSh4iIlJmSh4iIlJmSh4iIlJmSh4iIlJmSh4iIlNn/B1/wW9YvTUuzAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa62e9e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.semilogx(alpha_arr, train_err, 'b-o', label = 'test')\n",
    "plt.semilogx(alpha_arr, test_err, 'r-o', label = 'train')\n",
    "plt.xlim([np.max(alpha_arr), np.min(alpha_arr)])\n",
    "plt.title('Error vs. alpha')\n",
    "plt.xlabel('alpha')\n",
    "plt.ylabel('error')\n",
    "plt.legend()\n",
    "pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Минимальное значение ошибки:_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.000220458553792\n"
     ]
    }
   ],
   "source": [
    "min_train_err = np.min(train_err)\n",
    "min_test_err = np.min(test_err)\n",
    "print(min_train_err, min_test_err)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Оптимальное значение alpha:_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.001      0.0129155]\n"
     ]
    }
   ],
   "source": [
    "alpha_opt = alpha_arr[test_err == min_test_err]\n",
    "print(alpha_opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "alpha_opt = alpha_opt[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Классификатор с оптимальным значением alpha:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.000118708452042\n",
      "0.000220458553792\n"
     ]
    }
   ],
   "source": [
    "mlp_model = MLPClassifier(alpha = alpha_opt, hidden_layer_sizes = (100,),\n",
    "                          solver = 'lbfgs', activation = 'logistic', random_state = 42)\n",
    "mlp_model.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = mlp_model.predict(X_train)\n",
    "y_test_pred = mlp_model.predict(X_test)\n",
    "print(np.mean(y_train != y_train_pred))\n",
    "print(np.mean(y_test != y_test_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Матрица рассогласования:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1534    0    0    0    0]\n",
      " [   0 1513    0    0    0]\n",
      " [   0    0    0    0    0]\n",
      " [   0    0    0 1378    0]\n",
      " [   0    0    1    0  110]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "print(confusion_matrix(y_test_pred, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Aarven\\Anaconda3\\lib\\site-packages\\matplotlib\\collections.py:590: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if self._edgecolors == str('face'):\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAEKCAYAAAB62GIGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHD9JREFUeJzt3Xu0XVV59/Hv7yQkEAh3MAZiCVEuUiMgQqryEjTyBuRS\nqlaplZtoFRSKgoi2JXaMvopW1FqrFQyNytUoCEIgyRAQ5RISgiQQjCBBCElgoBCChYTkef9Y84TN\nyTn7rLP3WuestfP7jLHH2Ze1nzlzezLXXHPNRxGBmVlddQ11B8zM2uEkZma15iRmZrXmJGZmteYk\nZma15iRmZrU2fCgbl+T1HWZDKCLU6ndb+ffbTnt96eiRWEQM6HHBBRcM+DtViO2+u++tPIogKfej\nLEM6EjOzehtIcioqcfbkJGZmLStzhJWXk1iDyZMn1zJ22fHd96GJX3bfi9DVlX9Gav369aX0QWUN\n8XI1XvLE/lD+2syqThLR5sT+iBEjch+/du3attrri0diZtayKpxOlnp1UtJUSQ9J+p2k88psy8wG\nX0dfnZQ0DPhPYAqwHLhH0nURsaSsNs1scHX6SOxg4OGIWBYR64ArgeNKbM/MBllHj8SA3YDHG14/\nARxSYntmNsiqMBIrM4n50qBZhdx6663ceuuthcYcyBKLspS2xELSJGBaRExNr88HNkTEhQ3HeImF\n2RApYonFtttum/v41atXl7LEosw0Oh94g6Q9JI0APgBcV2J7ZjbI2pkTk7SlpLsl3SdpsaRpDZ99\nStKS9P6Fm3y5QWmnkxHxsqRPAjcDw4Dv+8qkWWdpZ04sIl6UdHhE/FnScOBXkmYBo4BjgYkRsU7S\nLs3ilLrYNSJmAbPKbMPMhk67E/sR8ef0dASwBdlc+seBL6VVDUTE081iDP2snJnVVrtLLCR1SboP\nWAXMjoh5wF7A/5F0l6RbJR3UrA++7cjMWlbASGwDsL+k7YBrJO1Hlpd2iIhJkt4KXA3s2VcMJzEz\na1mzJRZr165l3bp1ueJExHOSbgGmkq0p/Wl6/x5JGyTtFBHP9NqHAffazCxpdvo4cuRIttlmm42P\nXr67s6Tt0/OtgHcDS4BrgXem9/cCRvSVwMAjMTNrQ5unk68FZqT7rLuAqyLiRklbANMlLQLWAic2\n7YP3EzPbPBWx2HXMmDG5j1+5cqX3EzOzaun0eydzKXO0VOZvsEd5Zk5iZlZzTmJmVmtV2MXCSczM\nWuaRmJnVmpOYmdWak5iZ1VoVkljZJdumS1qVVt6aWYepQqGQsi8tXEp2Q6eZdaCurq7cj7KUvSni\n7ZL2KLMNMxs6VTid9JyYmbXMSczMas1JDJg2bdrG55MnT2by5MlD1hezTlZG3ckqJLHSt+JJc2LX\nR8SbevksfAO42dAoYiuefffdN/fxS5YseVV7krYEbgNGkg2oZkbENElfBY4m20vsEeCUiHiur7hl\nL7G4ArgD2EvS45JOKbM9Mxtc7SyxiIgXgcMjYn9gf2CqpEOA2cB+EfFmYClwfrM+lH118oQy45vZ\n0Gp36UQvJds2RMSchkPuBt7btA9t9cDMNmsllGy7p8chpwI3NuvDkE/sm1l9NZt3XrNmDWvWrGn6\n/d5KtkXEAyn2F4C1EXF5sxhOYmbWsmZJbPTo0YwePXrj66eeeqrPY3uUbHtA0snAUcC7+uuDTyfN\nrGXtnE72VbJN0lTgXOC4NPnflEdiZtaykkq2/Y5son9Oin9nRJzeVxAnMTNrWTtJLCIWAQf28v4b\nBhLHSczMWuY99kvmVfVm5arCbUcdncTMrFxOYmZWa05iZlZrTmJmVmtOYmZWa05iZlZrXmJhZrVW\nhZFY2ZsijpN0i6QHJC2WdGaZ7ZnZ4KpC3cmyR2LrgLMj4j5J2wALJM2JiCUlt2tmg6AKI7Gyd3Zd\nCaxMz9dIWgKMBZzEzDpAxyexRqlgyAFk282aWQfYbJJYOpWcCZwVEa/a6tEl28wGh0u2tdqAtAXw\nc2BWRHyjx2ellmwzs74VUbJtypQpuY+fO3duz5Jt44AfALsCAXwvIv5D0v7Ad8lKub0MnN7L3vsb\nlToSU5amvw882DOBmVn9tTkS6/XCH/AV4IKIuFnSken14X0FKft08u3A3wP3S1qY3js/Im4quV0z\nGwRtborY24W/3YANwHbpsO2B5c3ilH118ld4H3+zjlXUnFjDhb+7gH8Ebpb072T546+afdcr9s2s\nZc2S2DPPPMMf//jHPDFedeFP0unAP0bENZLeD0wnKyLS+/eHcmLdE/tmQ6eIif0jjzwy9/GzZs3a\npL3eLvxJejYiuqsgCXg2IrbbJGDiUz0za1lXV1fuR09NLvw9Kemw9PydwNJmffDppJm1rM05sd4u\n/H0e+CjwTUnDgf8FPtYsiJOYmbWszauTzS78HZQ3jpOYmbWsCiv2ncTMrGVOYtanbbfdttT4q1ev\nLjW+bR6cxMys1pzEzKzWvMe+mdWaR2JmVmtOYmZWa5VOYpIWNfleRMTEEvpjZjVS6SQGHNNucElb\nAreR7dA4HJgZEdPajWtm1VDpJBYRy7qfp71+Xh8RcyWNAoblCR4RL0o6PCL+nO6D+pWkWRHhYiFm\nHaDSSaybpI+R3ZC5IzAB2B34DvCuPA1ExJ/T0xHAFmS7NppZB6jCEos8PTgDeAewGiAilpJt7J+L\npC5J9wGrgNnNNvw3s3qpQgXwPEnspYh4qaHTw8kqk+QSERsiYn+yEdwhkvYbeDfNrIqqkMTyLLG4\nTdIXgFGS3g2cDlw/0IYi4jlJtwBTgQe633fdSbPBUbW6k32VbGv4/DPAV4GdI6LPfa773Z5a0jDg\nI8AR6a2bgUvy7CstaWfg5Yh4VtJW6btfjogb0+fenroPvgHcylbE9tQnnXRS7uNnzJjRs+7kGGBM\nY8k24K8jYklKcBcDewNvaZbE+h2JRcR6STOAu8my5UMDyDyvBWakRNgFXNWdwMys/koo2TYWWAJc\nBHwW+Fl/cfJcnXwPWTXe36e39pT0D3mSUUQsAg7s7zgzq6cSSrbdLek44ImIuD9P/DxzYhcBh0fE\nw6mxCcCN6WFmm7FmSyxWrFjBihUr+o3RWLKNbAnW53l1ibammSxPElvdncCS35OWW5jZ5q3ZSGns\n2LGMHTt24+uFCxduckwq2fYT4EcRca2kNwF7AL9JsXcHFkg6OCKe6q2dZvdOvjc9nS/pRuDq9Pr9\nwPw+e25mm402r05uUrItTUG9puGYR2ljYv8YXlkP9hTQXQfuaWDLlntuZh2jjJJtETGr4Zh+LyI2\nu3fy5HZ6Z2adr8SSbd3H7NlfnDxXJ7ciWyf2RmArUmaMiFNz9dTMOlYVbgDPc9vRD8nOUacCtwLj\ngDUl9snMaqIutx29PiLeJ+m4iJgh6XLgV6X1qCbWrl1bavyyV9TPnTu3tNh7770348aNKy2+VUcV\ndrHIk8S6/7U+ly5/rgR2Ka9LZlYXVTidzJPELpa0I/BPwHXANsA/l9orM6uFWiSxiLg4Pb0NGF9u\nd8ysTiqdxNI2GD0F2S0AEREXldYrM6uFSicxYDQD2PzQzDY/lU5irkpkZv2py9VJM7NeVXokVpS0\nIeJ8sv2B2q5laWbVsVkkMbI9gh4km2Mzsw5S6STW4+pk91XJ7ufkuTopaXfgKODfgE+33k0zq6Iq\nJLFms3KjyRa2vgX4BNne17sBHyf/ltNfB87FBXPNOlI7905KGifpFkkPSFos6cz0/o6S5khaKmm2\npO2b9aHfq5OSbgcOjIjn0+sLyLE1taSjgaciYqGkyX0d55JtZoOjaiXbgHXA2Y3VjiTNAU4B5kTE\nVySdB3wuPXqVZ05s19RYY8N5KoC/DThW0lFkmyhuK+kHEXFi40GNSczMytNzkPDFL36x7ZjtLLHo\no9rRbsCxvLIJ6wyy3XPaSmI/AOZJ+inZvNhfp8D9dfDzZBv+I+kw4JyeCczM6q2MakfAayJiVfpo\nFQ3bVfcmz72T/ybpJuAd6a2TI2LTHf/759X/Zh2miCSWTiV/ApwVEc83xoyIkNQ0d+RdYjEKeD4i\npkvaRdL4iHg0bycj4jayG8jNrIM0S2LLli3jscce6+/73dWOfhgR16a3V0kaExErJb2WrMZHn/Js\nTz2N7Arl3sB0YATwI7JN/s1sM9YsiY0fP57x41/Z+OaXv/xlz+9uUu0ouQ44Cbgw/byWJvKMxI4n\nO1ddABARyyV54aqZlVHt6Hzgy8DVkj4CLAP+tlmQPEnspYjY0N1ZSVu32mMz6ywlVjuakjdOniT2\nY0n/DWwv6WPAqcAleRsws85Vi10sIuKrko4Angf2Av45IuaU3jMzq7wq3HaUZ2L/wog4D5jdy3tm\nthmrRRIDjgB6JqyjennPamTkyJGlxR42bFhpsa1aKp3EJH0COB2YIGlRw0ejgV+X3bGqGzFixFB3\noS2HHnroUHfBOkClkxhwOTCL7HLnebyyFc/zEfFM2R0zs+qrdBKLiOfICuZ+E/hTRKwGkLStpEMi\n4u7B6qSZVVMVklie66PfAdY0vH4B+G453TGzOunq6sr9KEuueycjYkPD8/Vp33wz28zVZST2qKQz\nJW0haYSks4Dfl90xM6u+dnZ2LUqeJPZxsnuclgNPAJOAj5XWIzOrjSoksTwr9lcBH2i1AUnLgNXA\nemBdRBzcaiwzq5YqnE42Wyd2XkRcKOlbvXwcEXFmzjYCmBwRf2yph2ZWWZVOYmS1IiFtwdPDQHdp\nHfpfqZkVrtJJLCKuTz//p802ApgraT3w3xFxcZvxzKwi2lk6IWk68B6yqmhvanj/U2R3C60Hbujv\nPu1mp5PXN7xsLJ4L2enksTn7+vaIWCFpF2COpIci4vac3zWzCmtzJHYp8C2yYkTd8Q4nq3Y0MSLW\npbzRVLPTya+ln8cDY8i2pBZwAlkFklwiYkX6+bSka4CDgY1JzHUnzQZH1epORsTtqcpRo08AX4qI\ndemYp/vtQ0Tz6S1JCyLiLf2918d3RwHDUgWTrcm28/liRMxOn0d/7ZtZOSQRES1nIUnx7W9/O/fx\nZ5xxxibtpSR2fffpZNqm+mfAVOBFslKP85vFzbNif5SkCRHxSGpkT7LqR3m8BrgmZevhwGXdCczM\n6q+Eif3hwA4RMUnSW4GrgT37+0J/zgZukdRdom0Pci52TWXd9s9zrJnVT7MktnTpUpYuXTrQkE8A\nPwWIiHskbZC0U7Odc/Isdr1J0l5kJdsAHoqIlwbaMzPrPM2uTu6zzz7ss88+G1/fcMMNeUJeC7wT\nuC3lnRH9bf2VZ3vqrYFPA6+LiI9KeoOkvSPi53l6ZGadq53TSUlXAIcBO0l6HPgXstq209NGrGuB\nE/uLk+d08lKyBa9vS6+fBGYCTmJmm7k2r06e0MdHHx5InDwr1SZExIVkWZGIeGEgDZhZ56rFDeDA\nS5K2auj0BMBzYmZW7duOGkwDbgJ2l3Q52bY8J5fYJzOriconMUldwA7Ae8n2EQM4K88qWjPrfJVP\nYhGxQdJnI+IqPJFvZj2UuXd+XnlOJ+dIOge4iqxICADeH8zMKj8SSz5ItovFGQ3vBf3cCmBmna8W\nSSwi9hiEfphZDdUiiaXlFacD7yAbgd0OfCciXiy5b2ZWcbVIYmQblq0G/oNsP7G/A34IvL/EfplZ\nDdQlie0XEW9seP0LSQ/2ebSZbTaqkMTyXB+9V9Jfdb+QNInei4eY2Wamq6sr96MseUZiBwG/TneZ\nB/A64LfpLvOIiIl9fVHS9sAlwH7pu6dGxF3td9vMqqAKI7E8SWxqG/G/CdwYEe+TNBzYuo1YZlYx\ntUhiEbGslcCStgMOjYiTUpyXgedaiWVm1dTmfmKblGyT9FXgaLJdcx4BTomIpnmjzHsGxgNPS7pU\n0r2SLk6FQ8ysQ7S5Fc+lbHqmN5vsYuKbgaXA+f31Ic/pZKuGAwcCn0x7ZX8D+BzZ7o0buWSb2eCo\nQ8m2iJjT8PJuss0nmvehrJJpksYAd0bE+PT6HcDnIuLohmNcss1siBRRsu2aa67Jffzxxx/fb8m2\nHp9dD1wREZc3i1vaSCwiVkp6XNJeEbEUmAI8UFZ7Zjb4mi2dWLRoEYsXL24prqQvAGv7S2BQ7ukk\nwKeAyySNIE3SldyemQ2iZqeTEydOZOLEV1ZgXXnllXljngwcBbwrz/GlJrGI+A3w1jLbMLOhU/QS\nC0lTgXOBw/Len132SMzMOlhBJdt2TovpLyC7GjmCbB9DyObVT28Wx0nMzFpWQsm26QON4yRmZi2r\nxYp9M7O+OImZWa3VpVCImVmvPBIzs1pzEjOzWnMSM7NacxIzs1pzEjOzWvPVSTOrNY/EzKzWnMTM\nrNaqkMRKPaGVtLekhQ2P5ySdWWabZjZ42txjvxBl7yf2W+AAAEldwHIg/362ZlZpHT8S62EK8EhE\nPD6IbZpZidodiUk6W9JiSYskXS5p5ED7MJhJ7INAv/tlm1l9dHV15X70JGk3si3s35IKhQwjyxMD\n60Pbv4oc0h77xwA/Hoz2zGxwFDAnNhwYJWk4MIpsymlABuvq5JHAgoh4uucHrjtpNjgqWHdyuaSv\nAX8A/he4OSLmDrgPg1H3UdKVwKyImNHjfdedNBsiRdSdvOeee/r8fMGCBSxYsGDj64svvvhV7Una\nAZgJ/C3wHNmZ2syIuGxA/Sg7iUjaGngMGB8Rz/f4zEnMbIgUkcTmz5+f+/iDDjqoZxJ7P/B/I+K0\n9PrDwKSIOGMg/Sj9dDIiXgB2LrsdMxt8bS6xeAyYJGkr4EWyFQzzBhrEK/bNrGVtzonNkzQTuBd4\nOf383kDjOImZWcva3cUiIqYB09qJ4SRmZi2rwop9JzEza5mTmJnVmpOYmdWak5iZ1ZqTmJnVmpOY\nmdWaC4WYWa15JGZmteYkZma15iRmZrXmJGZmteYkZma11vFJTNLZwEeAABYBp0TES2W2aWaDpwpL\nLErrQVGVTMysuooonitpWCqufX0rfSj7dLK7ksl6WqxkYmbVVdDp5FnAg8DoVr5c2kgsIpYD3ZVM\nngSebaWSiZlVVwHFc3cHjgIuAVrKiKWNxFIlk2OBPUiVTCR9qGclE5dsMxscVSvZlnwdOBfYtuU+\nlFVtKE8lE1c7Mhs6RVQ7Wr687xmiO+64gzvvvHPj64suuqhntaOjgSMj4gxJk4HPRMQxA+5HiUns\nYGA68FaySib/A8yLiG83HOMkZjZEikhiTz75ZO7jx44d2zOJ/T/gw2RFQrYkG439JCJOHFA/ykwi\nkqYBH+CVSianRcS6hs+dxMyGSBFJbOXKlbmPHzNmTJ/tSToMOKeVkVipVyeLqGRiZtVV8GLXlkY0\nXrFvZi0rKolFxG3Aba1810nMzFrW8bcdmVlncxIzs1pzEjOzWqvCDeBOYmbWMo/EzKzWnMTMrNac\nxMys1pzEzKzWnMTMrNacxMys1rzEwsxqzSMxM6u1KiSxUseCks6StEjSYklnldmWmQ2+AvbYnyrp\nIUm/k3ReK30os2TbXwKnke3s+mbgaEkT2olZ9P7ggxnffR+a+O57udpJYpKGAf8JTAXeCJwgad+B\n9qHMkdg+wN0R8WJErCfbK+hv2gnov5BDE999H5r4nZ7EgIOBhyNiWdrx+UrguIH2ocwkthg4VNKO\nkkYB7wF2L7E9MxtkbSax3YDHG14/kd4bkNIm9iPiIUkXArOBF4CFwIay2jOzwdfmEotCCmyUWijk\nVQ1llU3+EBHfbXjPVULMhlC7hULaaU/SJGBaRExNr88HNkTEhQOJWeoSC0m7RsRTkl4HHA8c0vh5\nO7+BZja0Cvj3Ox94g6Q9gCfJKqOdMNAgZa8TmylpJ2AdcHpErC65PTOriYh4WdIngZuBYcD3I2LJ\nQOMM2umkmVkZhv7Gp5yKWBTXJPZ0SaskLSoyboo9TtItkh5Ii37PLDD2lpLulnRfij2tqNg92hkm\naaGk6wuOu0zS/Sn2vCJjp/jbS5opaYmkB9McTBFx90597n48V+Sfa2rj7PRnukjS5ZJGFhi7sxah\nR0TlH2RDzYeBPYAtgPuAfQuMfyhwALCohL6PAfZPz7cBfltw30eln8OBu4BDSvg1fBq4DLiu4LiP\nAjuW+PdmBnBqw+/PdiW00QWsAMYVGHM34PfAyPT6KuCkgmL/JbAI2DL9u5oDTCjrz2AwHnUZiRWy\nKK4vEXE78Kei4vWIvTIi7kvP1wBLgLEFxv9zejqCLMEXuoxF0u7AUcAlQBkXYkq5uCNpO+DQiJgO\n2fxLRDxXQlNTgEci4vF+jxyY4cAoScOBUcDyguIWvgh9qNUliRWyKG6opaswBwB3FxizS9J9wCpg\ndkTcU1Ts5OvAuZSzxi+AuZLmS/powbHHA09LulTSvZIuTouui/ZB4PIiA0bEcuBrwB/Irto9GxFz\nCwrfcYvQ65LEan/1QdI2wEzgrDQiK0REbIiI/cn+Ih4iab+iYks6GngqIhZSzojp7RFxAHAkcIak\nQwuMPRw4EPiviDiQbMH15wqMj6QRwDHAjwuOuwNwLNn0yVhgG0kfKiJ2RDwEdC9Cn0UHLEKvSxJb\nDoxreD2ObDRWC5K2AH4C/Cgiri2jjXSqdAvZzbRFeRtwrKRHgSuAd0r6QVHBI2JF+vk0cA3ZtEFR\nngCeaBiZziRLakU6EliQ+l+kKcCjEfFMRLwM/JTsz6IQETE9Ig6KiMOAZ8nmaWurLkls46K49L/f\nB4DrhrhPuSi7aez7wIMR8Y2CY+8safv0fCvg3WRzboWIiM9HxLiIGE922vSLiDixiNiSRkkanZ5v\nDRxBNuFciIhYCTwuaa/01hTggaLiJyeQJfeiPQZMkrRV+vszBXiwqOCSdk0/uxehF3o6PNhqsSli\nFLQori+SrgAOA3aS9DjwLxFxaUHh3w78PXC/pIXpvfMj4qYCYr8WmJG2NOkCroqIGwuI25ciT+tf\nA1yTbgweDlwWEbMLjA/wKeCy9B/fI8ApRQVOiXcKUPRcHhExT9JM4F7g5fTzewU20VGL0L3Y1cxq\nrS6nk2ZmvXISM7NacxIzs1pzEjOzWnMSM7NacxIzs1pzEutwkraT9IkS458s6Vv9HDNN0mcGGLew\nW7OsszmJdb4dgNN7+yDtkNCuPAsNW1mM6AWMlouTWOf7MjAhbd73FUmHSbpd0s+AxZL+QtLi7oMl\nnSPpgvR8gqRZaZeJX0rau1lDko6RdFfaNWJO9+0tyZsl3SFpqaTTGr5zrqR5kn5T1qaO1tlqcduR\nteU8YL+0WwSSJpNtB7RfRDyWtgdqHPVEw+vvAf8QEQ9LOgT4L+BdTdq6PSImpXZOAz4LnEO2A8ZE\nskIx2wALJd0AvAl4fUQcLKkLuE7SoWl/N7NcnMQ6X29b6MyLiMeafSfdG/g24Md6pfDpiH7aGifp\narLdbEeQ7U4KWVK8NiJeAl6SdAvZjhWHAkc03FO6NfB6wEnMcnMS2zy90PD8ZV49rbAVWdLpAv7U\nPYLL6VvAv0fEzyUdBkxrcmz3aO9LEVHkzc22mfGcWOd7Hhjd5PNVwK5pp8+RwNEAEfE88Kik90E2\nNJM0sZfvN470tiXbiRTg5B7HHCdpZNo9YTIwj2xXklPTqA9Ju0naZYC/PtvMeSTW4SLiGUm/VlbJ\n6cb0iIbP10n6V7KkspxX71v1IeA7kv6JbP/+K4D7ezbREG8a2ennn4BfAH/RcMz9ZJs27gz8a9rv\na6WkfYE70ynrmtTm0/jqpOXkrXjMrNZ8OmlmteYkZma15iRmZrXmJGZmteYkZma15iRmZrXmJGZm\nteYkZma19v8BHlzXx7YiFRYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x5a6dc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "M = confusion_matrix(y_test_pred, y_test)\n",
    "M = np.sqrt(M)\n",
    "plt.imshow(M, interpolation = 'nearest')\n",
    "plt.set_cmap('binary')\n",
    "plt.xticks(range(10))\n",
    "plt.yticks(range(10))\n",
    "plt.xlabel(\"true label\")\n",
    "plt.ylabel(\"predicted label\")\n",
    "plt.colorbar()\n",
    "pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Другие метрики:_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00      1534\n",
      "          1       1.00      1.00      1.00      1513\n",
      "          2       0.00      0.00      0.00         1\n",
      "          3       1.00      1.00      1.00      1378\n",
      "          4       0.99      1.00      1.00       110\n",
      "\n",
      "avg / total       1.00      1.00      1.00      4536\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Aarven\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Все ошибки:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element number: 3409\n",
      "Predicted class: 4\n",
      "Actual class: 2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(N_test):\n",
    "    sample = X_test[i:i+1]\n",
    "    cl = y_test[i]\n",
    "    y_pred = mlp_model.predict(sample)[0]\n",
    "    if y_pred != cl:\n",
    "        print(\"Element number: %d\" % i)\n",
    "        print(\"Predicted class: %d\" % y_pred)\n",
    "        print(\"Actual class: %d\" % cl)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}