3-random-undersampling-model.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Modelo de submuestreo","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pylab as plt\nimport os\nfrom os import listdir\nfrom os.path import isfile, join","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:02.436254Z","iopub.execute_input":"2021-12-01T15:19:02.436575Z","iopub.status.idle":"2021-12-01T15:19:02.538832Z","shell.execute_reply.started":"2021-12-01T15:19:02.436502Z","shell.execute_reply":"2021-12-01T15:19:02.538187Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"## <font color=red>1. </font>Cargar las imágenes y los datos tabulares","metadata":{}},{"cell_type":"code","source":"# Resized images directory\ndir_2019_images = \"/kaggle/input/resizedsiimisic/train_resized/\"\n\n# CSV file\ntrain_df = pd.read_csv('/kaggle/input/resizedsiimisic/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:19.255678Z","iopub.execute_input":"2021-12-01T15:19:19.256219Z","iopub.status.idle":"2021-12-01T15:19:19.306289Z","shell.execute_reply.started":"2021-12-01T15:19:19.256182Z","shell.execute_reply":"2021-12-01T15:19:19.305568Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:24.815306Z","iopub.execute_input":"2021-12-01T15:19:24.815569Z","iopub.status.idle":"2021-12-01T15:19:24.842149Z","shell.execute_reply.started":"2021-12-01T15:19:24.815541Z","shell.execute_reply":"2021-12-01T15:19:24.841402Z"},"trusted":true},"execution_count":4,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"         image_name     sex  age_approx anatom_site_general_challenge  target\n0  ISIC_0000000.jpg  female        55.0                anterior torso     0.0\n1  ISIC_0000001.jpg  female        30.0                anterior torso     0.0\n2  ISIC_0000002.jpg  female        60.0               upper extremity     1.0\n3  ISIC_0000003.jpg    male        30.0               upper extremity     0.0\n4  ISIC_0000004.jpg    male        80.0               posterior torso     1.0","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>image_name</th>\n      <th>sex</th>\n      <th>age_approx</th>\n      <th>anatom_site_general_challenge</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ISIC_0000000.jpg</td>\n      <td>female</td>\n      <td>55.0</td>\n      <td>anterior torso</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>ISIC_0000001.jpg</td>\n      <td>female</td>\n      <td>30.0</td>\n      <td>anterior torso</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>ISIC_0000002.jpg</td>\n      <td>female</td>\n      <td>60.0</td>\n      <td>upper extremity</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ISIC_0000003.jpg</td>\n      <td>male</td>\n      <td>30.0</td>\n      <td>upper extremity</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>ISIC_0000004.jpg</td>\n      <td>male</td>\n      <td>80.0</td>\n      <td>posterior torso</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"print(\"Train shape:\", train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:25.413645Z","iopub.execute_input":"2021-12-01T15:19:25.414449Z","iopub.status.idle":"2021-12-01T15:19:25.422255Z","shell.execute_reply.started":"2021-12-01T15:19:25.414401Z","shell.execute_reply":"2021-12-01T15:19:25.421388Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Train shape: (22480, 5)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## <font color=red>2. </font>Submuestreo de las imágenes mayoritarias","metadata":{}},{"cell_type":"code","source":"train_df.target.value_counts().rename_axis('Tipo').reset_index(name='Total de muestras')","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:25.725955Z","iopub.execute_input":"2021-12-01T15:19:25.726407Z","iopub.status.idle":"2021-12-01T15:19:25.743721Z","shell.execute_reply.started":"2021-12-01T15:19:25.726363Z","shell.execute_reply":"2021-12-01T15:19:25.742889Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"   Tipo  Total de muestras\n0   0.0              18134\n1   1.0               4346","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Tipo</th>\n      <th>Total de muestras</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.0</td>\n      <td>18134</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>4346</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"# Number of samples of majority class to delete\nN = train_df.target.value_counts()[0] - train_df.target.value_counts()[1]\n\ntrain_df = train_df.drop(train_df[train_df['target'].eq(0)].sample(N).index)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:25.891227Z","iopub.execute_input":"2021-12-01T15:19:25.891456Z","iopub.status.idle":"2021-12-01T15:19:25.923669Z","shell.execute_reply.started":"2021-12-01T15:19:25.891433Z","shell.execute_reply":"2021-12-01T15:19:25.923006Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"train_df.target.value_counts().rename_axis('Tipo').reset_index(name='Total de muestras')","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:26.293876Z","iopub.execute_input":"2021-12-01T15:19:26.294387Z","iopub.status.idle":"2021-12-01T15:19:26.308539Z","shell.execute_reply.started":"2021-12-01T15:19:26.294353Z","shell.execute_reply":"2021-12-01T15:19:26.307861Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"   Tipo  Total de muestras\n0   0.0               4346\n1   1.0               4346","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Tipo</th>\n      <th>Total de muestras</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.0</td>\n      <td>4346</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>4346</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:26.375001Z","iopub.execute_input":"2021-12-01T15:19:26.375472Z","iopub.status.idle":"2021-12-01T15:19:26.395114Z","shell.execute_reply.started":"2021-12-01T15:19:26.375434Z","shell.execute_reply":"2021-12-01T15:19:26.393980Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"                      image_name     sex  age_approx  \\\n2               ISIC_0000002.jpg  female        60.0   \n3               ISIC_0000003.jpg    male        30.0   \n4               ISIC_0000004.jpg    male        80.0   \n12              ISIC_0000013.jpg  female        30.0   \n19  ISIC_0000020_downsampled.jpg  female        25.0   \n\n   anatom_site_general_challenge  target  \n2                upper extremity     1.0  \n3                upper extremity     0.0  \n4                posterior torso     1.0  \n12               posterior torso     1.0  \n19                anterior torso     0.0  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>image_name</th>\n      <th>sex</th>\n      <th>age_approx</th>\n      <th>anatom_site_general_challenge</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>ISIC_0000002.jpg</td>\n      <td>female</td>\n      <td>60.0</td>\n      <td>upper extremity</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ISIC_0000003.jpg</td>\n      <td>male</td>\n      <td>30.0</td>\n      <td>upper extremity</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>ISIC_0000004.jpg</td>\n      <td>male</td>\n      <td>80.0</td>\n      <td>posterior torso</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>ISIC_0000013.jpg</td>\n      <td>female</td>\n      <td>30.0</td>\n      <td>posterior torso</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>ISIC_0000020_downsampled.jpg</td>\n      <td>female</td>\n      <td>25.0</td>\n      <td>anterior torso</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"from collections import Counter\nfrom sklearn.model_selection import train_test_split\n\nX = train_df\ny = train_df['target']","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:26.520776Z","iopub.execute_input":"2021-12-01T15:19:26.521248Z","iopub.status.idle":"2021-12-01T15:19:27.338288Z","shell.execute_reply.started":"2021-12-01T15:19:26.521210Z","shell.execute_reply":"2021-12-01T15:19:27.337483Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"# Split into train, validation test and calibration sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, \n                                                    test_size=0.2, \n                                                    stratify=y,\n                                                    random_state=42)\n\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, \n                                                  test_size=0.2,\n                                                  stratify=y_train,\n                                                  random_state=42)\n\nX_test, X_calib, y_test, y_calib = train_test_split(X_test, y_test, \n                                                    test_size=0.4,\n                                                    stratify=y_test,\n                                                    random_state=42)\n\n\nprint(\"Conjunto de train:\", X_train.shape)\nprint(\"Conjunto de validacion:\", X_val.shape)\nprint(\"Conjunto de prueba:\", X_test.shape)\nprint(\"Conjunto de calibracion:\", X_calib.shape)\nprint(\"-----------------------\")\nprint('Distribucion de train ->', Counter(y_train))\nprint('Distribucion de validacion ->', Counter(y_val))\nprint(\"Distribucion de prueba:\", Counter(y_test))\nprint(\"Distribucion de calibracion:\", Counter(y_calib))","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:27.339858Z","iopub.execute_input":"2021-12-01T15:19:27.340134Z","iopub.status.idle":"2021-12-01T15:19:27.375428Z","shell.execute_reply.started":"2021-12-01T15:19:27.340098Z","shell.execute_reply":"2021-12-01T15:19:27.374670Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"Conjunto de train: (5562, 5)\nConjunto de validacion: (1391, 5)\nConjunto de prueba: (1043, 5)\nConjunto de calibracion: (696, 5)\n-----------------------\nDistribucion de train -> Counter({1.0: 2781, 0.0: 2781})\nDistribucion de validacion -> Counter({1.0: 696, 0.0: 695})\nDistribucion de prueba: Counter({0.0: 522, 1.0: 521})\nDistribucion de calibracion: Counter({1.0: 348, 0.0: 348})\n","output_type":"stream"}]},{"cell_type":"code","source":"X_train.to_csv('/kaggle/working/train.csv', index=False)\nX_val.to_csv('/kaggle/working/val.csv', index=False)\nX_calib.to_csv('/kaggle/working/calib.csv', index=False)\nX_test.to_csv('/kaggle/working/test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:27.376519Z","iopub.execute_input":"2021-12-01T15:19:27.376787Z","iopub.status.idle":"2021-12-01T15:19:27.425172Z","shell.execute_reply.started":"2021-12-01T15:19:27.376753Z","shell.execute_reply":"2021-12-01T15:19:27.424529Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"X_train[\"target\"] = X_train['target'].astype(str)\nX_val[\"target\"] = X_val['target'].astype(str)\nX_calib[\"target\"] = X_calib['target'].astype(str)\nX_test[\"target\"] = X_test['target'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:27.558722Z","iopub.execute_input":"2021-12-01T15:19:27.558931Z","iopub.status.idle":"2021-12-01T15:19:27.572534Z","shell.execute_reply.started":"2021-12-01T15:19:27.558908Z","shell.execute_reply":"2021-12-01T15:19:27.571795Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"## <font color=red>3. </font>Crear y entrenar el modelo","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.layers import Input, Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, AUC\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:30.621231Z","iopub.execute_input":"2021-12-01T15:19:30.621907Z","iopub.status.idle":"2021-12-01T15:19:34.913183Z","shell.execute_reply.started":"2021-12-01T15:19:30.621869Z","shell.execute_reply":"2021-12-01T15:19:34.912208Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stderr","text":"2021-12-01 15:19:31.123155: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n","output_type":"stream"}]},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1./255.)\n\ntrain_generator = datagen.flow_from_dataframe(\n    dataframe=X_train,\n    directory=dir_2019_images,\n    x_col=\"image_name\",\n    y_col=\"target\",\n    batch_size=32,\n    seed=42,\n    shuffle=True,\n    class_mode=\"binary\"\n)\n\nvalid_generator = datagen.flow_from_dataframe(\n    dataframe=X_val,\n    directory=dir_2019_images,\n    x_col=\"image_name\",\n    y_col=\"target\",\n    batch_size=32,\n    seed=42,\n    shuffle=True,\n    class_mode=\"binary\"\n)\n\ncalib_generator=datagen.flow_from_dataframe(\n    dataframe=X_calib,\n    directory=dir_2019_images,\n    x_col=\"image_name\",\n    y_col=\"target\",\n    batch_size=32,\n    seed=42,\n    shuffle=False,\n    class_mode=\"binary\"\n)\n\ntest_generator = datagen.flow_from_dataframe(\n    dataframe=X_test,\n    directory=dir_2019_images,\n    x_col=\"image_name\",\n    y_col=\"target\",\n    batch_size=32,\n    seed=42,\n    shuffle=False,\n    class_mode=\"binary\"\n)\n\nSTEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size\nSTEP_SIZE_CALIB=calib_generator.n//calib_generator.batch_size\nSTEP_SIZE_TEST = test_generator.n//test_generator.batch_size","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:46.778900Z","iopub.execute_input":"2021-12-01T15:19:46.779175Z","iopub.status.idle":"2021-12-01T15:19:52.637710Z","shell.execute_reply.started":"2021-12-01T15:19:46.779146Z","shell.execute_reply":"2021-12-01T15:19:52.636935Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Found 5562 validated image filenames belonging to 2 classes.\nFound 1391 validated image filenames belonging to 2 classes.\nFound 696 validated image filenames belonging to 2 classes.\nFound 1043 validated image filenames belonging to 2 classes.\n","output_type":"stream"}]},{"cell_type":"code","source":"encoder = DenseNet121(input_shape=(None,None,3), \n                      include_top=False, \n                      weights='imagenet')","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:19:57.828180Z","iopub.execute_input":"2021-12-01T15:19:57.828430Z","iopub.status.idle":"2021-12-01T15:20:00.481299Z","shell.execute_reply.started":"2021-12-01T15:19:57.828397Z","shell.execute_reply":"2021-12-01T15:20:00.480475Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"inputs = Input(shape=(None, None, 3))\nx = encoder(inputs, training=False)\nx = GlobalAveragePooling2D()(x)\npredictions = Dense(1, activation='sigmoid')(x)\nmodel_undersampling = Model(inputs=inputs, outputs=predictions)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:20:00.483104Z","iopub.execute_input":"2021-12-01T15:20:00.483380Z","iopub.status.idle":"2021-12-01T15:20:01.225586Z","shell.execute_reply.started":"2021-12-01T15:20:00.483346Z","shell.execute_reply":"2021-12-01T15:20:01.224886Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"model_undersampling.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:20:01.226845Z","iopub.execute_input":"2021-12-01T15:20:01.227097Z","iopub.status.idle":"2021-12-01T15:20:01.261184Z","shell.execute_reply.started":"2021-12-01T15:20:01.227067Z","shell.execute_reply":"2021-12-01T15:20:01.260515Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stdout","text":"Model: \"model\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ninput_3 (InputLayer)         [(None, None, None, 3)]   0         \n_________________________________________________________________\ndensenet121 (Functional)     (None, None, None, 1024)  7037504   \n_________________________________________________________________\nglobal_average_pooling2d (Gl (None, 1024)              0         \n_________________________________________________________________\ndense (Dense)                (None, 1)                 1025      \n=================================================================\nTotal params: 7,038,529\nTrainable params: 6,954,881\nNon-trainable params: 83,648\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"METRICS = [\n      TruePositives(name='tp'),\n      FalsePositives(name='fp'),\n      TrueNegatives(name='tn'),\n      FalseNegatives(name='fn'),\n      AUC(name='auc')\n]","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:20:01.263325Z","iopub.execute_input":"2021-12-01T15:20:01.263575Z","iopub.status.idle":"2021-12-01T15:20:01.287382Z","shell.execute_reply.started":"2021-12-01T15:20:01.263544Z","shell.execute_reply":"2021-12-01T15:20:01.286768Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"checkpoint_filepath = '/kaggle/working/undersampling_model.h5'\nmodel_checkpoint_callback = ModelCheckpoint(filepath=checkpoint_filepath,\n                                            monitor='val_auc',\n                                            mode='max',\n                                            verbose=1,\n                                            save_best_only=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:20:01.288526Z","iopub.execute_input":"2021-12-01T15:20:01.288779Z","iopub.status.idle":"2021-12-01T15:20:01.293280Z","shell.execute_reply.started":"2021-12-01T15:20:01.288748Z","shell.execute_reply":"2021-12-01T15:20:01.292450Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"model_undersampling.compile(\n    optimizer=Adam(),\n    loss=BinaryCrossentropy(),\n    metrics=METRICS\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:20:05.898667Z","iopub.execute_input":"2021-12-01T15:20:05.899241Z","iopub.status.idle":"2021-12-01T15:20:05.923757Z","shell.execute_reply.started":"2021-12-01T15:20:05.899205Z","shell.execute_reply":"2021-12-01T15:20:05.923012Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"model_undersampling.fit(train_generator,  \n                        validation_data =valid_generator,\n                        steps_per_epoch=STEP_SIZE_TRAIN, \n                        validation_steps=STEP_SIZE_VALID,\n                        callbacks=[model_checkpoint_callback],\n                        epochs = 100)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T15:25:55.952337Z","iopub.execute_input":"2021-12-01T15:25:55.952613Z","iopub.status.idle":"2021-12-01T16:51:50.816636Z","shell.execute_reply.started":"2021-12-01T15:25:55.952585Z","shell.execute_reply":"2021-12-01T16:51:50.815943Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"Epoch 1/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.6940 - tp: 878.0000 - fp: 908.0000 - tn: 1858.0000 - fn: 1886.0000 - auc: 0.4941 - val_loss: 0.6932 - val_tp: 0.0000e+00 - val_fp: 0.0000e+00 - val_tn: 687.0000 - val_fn: 689.0000 - val_auc: 0.5000\n\nEpoch 00001: val_auc did not improve from 0.51527\nEpoch 2/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.6954 - tp: 1440.0000 - fp: 1382.0000 - tn: 1380.0000 - fn: 1328.0000 - auc: 0.5179 - val_loss: 2.2025 - val_tp: 691.0000 - val_fp: 685.0000 - val_tn: 0.0000e+00 - val_fn: 0.0000e+00 - val_auc: 0.3941\n\nEpoch 00002: val_auc did not improve from 0.51527\nEpoch 3/100\n173/173 [==============================] - 51s 296ms/step - loss: 1.1627 - tp: 1306.0000 - fp: 1270.0000 - tn: 1501.0000 - fn: 1453.0000 - auc: 0.5082 - val_loss: 0.6952 - val_tp: 0.0000e+00 - val_fp: 0.0000e+00 - val_tn: 685.0000 - val_fn: 691.0000 - val_auc: 0.5502\n\nEpoch 00003: val_auc improved from 0.51527 to 0.55015, saving model to /kaggle/working/undersampling_model.h5\nEpoch 4/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.8281 - tp: 1346.0000 - fp: 1363.0000 - tn: 1403.0000 - fn: 1418.0000 - auc: 0.5001 - val_loss: 0.6948 - val_tp: 687.0000 - val_fp: 689.0000 - val_tn: 0.0000e+00 - val_fn: 0.0000e+00 - val_auc: 0.5000\n\nEpoch 00004: val_auc did not improve from 0.55015\nEpoch 5/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.7045 - tp: 1938.0000 - fp: 1895.0000 - tn: 865.0000 - fn: 832.0000 - auc: 0.5084 - val_loss: 0.6998 - val_tp: 0.0000e+00 - val_fp: 0.0000e+00 - val_tn: 685.0000 - val_fn: 691.0000 - val_auc: 0.5000\n\nEpoch 00005: val_auc did not improve from 0.55015\nEpoch 6/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.6939 - tp: 1307.0000 - fp: 1254.0000 - tn: 1509.0000 - fn: 1460.0000 - auc: 0.5062 - val_loss: 0.6927 - val_tp: 685.0000 - val_fp: 691.0000 - val_tn: 0.0000e+00 - val_fn: 0.0000e+00 - val_auc: 0.6202\n\nEpoch 00006: val_auc improved from 0.55015 to 0.62021, saving model to /kaggle/working/undersampling_model.h5\nEpoch 7/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.6985 - tp: 1304.0000 - fp: 1225.0000 - tn: 1542.0000 - fn: 1459.0000 - auc: 0.5215 - val_loss: 0.6934 - val_tp: 0.0000e+00 - val_fp: 0.0000e+00 - val_tn: 689.0000 - val_fn: 687.0000 - val_auc: 0.5000\n\nEpoch 00007: val_auc did not improve from 0.62021\nEpoch 8/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.6940 - tp: 978.0000 - fp: 978.0000 - tn: 1784.0000 - fn: 1790.0000 - auc: 0.4930 - val_loss: 0.6935 - val_tp: 689.0000 - val_fp: 687.0000 - val_tn: 0.0000e+00 - val_fn: 0.0000e+00 - val_auc: 0.5558\n\nEpoch 00008: val_auc did not improve from 0.62021\nEpoch 9/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.6968 - tp: 1332.0000 - fp: 1216.0000 - tn: 1555.0000 - fn: 1427.0000 - auc: 0.5317 - val_loss: 0.6904 - val_tp: 81.0000 - val_fp: 50.0000 - val_tn: 636.0000 - val_fn: 609.0000 - val_auc: 0.6239\n\nEpoch 00009: val_auc improved from 0.62021 to 0.62392, saving model to /kaggle/working/undersampling_model.h5\nEpoch 10/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.6798 - tp: 1522.0000 - fp: 1127.0000 - tn: 1640.0000 - fn: 1241.0000 - auc: 0.5978 - val_loss: 0.6648 - val_tp: 540.0000 - val_fp: 389.0000 - val_tn: 300.0000 - val_fn: 147.0000 - val_auc: 0.6730\n\nEpoch 00010: val_auc improved from 0.62392 to 0.67298, saving model to /kaggle/working/undersampling_model.h5\nEpoch 11/100\n173/173 [==============================] - 52s 302ms/step - loss: 0.6675 - tp: 1460.0000 - fp: 889.0000 - tn: 1874.0000 - fn: 1307.0000 - auc: 0.6371 - val_loss: 0.6578 - val_tp: 199.0000 - val_fp: 76.0000 - val_tn: 615.0000 - val_fn: 486.0000 - val_auc: 0.7140\n\nEpoch 00011: val_auc improved from 0.67298 to 0.71403, saving model to /kaggle/working/undersampling_model.h5\nEpoch 12/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.6424 - tp: 1781.0000 - fp: 1019.0000 - tn: 1744.0000 - fn: 986.0000 - auc: 0.6812 - val_loss: 0.6233 - val_tp: 357.0000 - val_fp: 160.0000 - val_tn: 529.0000 - val_fn: 330.0000 - val_auc: 0.7230\n\nEpoch 00012: val_auc improved from 0.71403 to 0.72297, saving model to /kaggle/working/undersampling_model.h5\nEpoch 13/100\n173/173 [==============================] - 52s 301ms/step - loss: 0.6334 - tp: 1728.0000 - fp: 959.0000 - tn: 1808.0000 - fn: 1035.0000 - auc: 0.6941 - val_loss: 0.5832 - val_tp: 447.0000 - val_fp: 197.0000 - val_tn: 491.0000 - val_fn: 241.0000 - val_auc: 0.7554\n\nEpoch 00013: val_auc improved from 0.72297 to 0.75540, saving model to /kaggle/working/undersampling_model.h5\nEpoch 14/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.6119 - tp: 1868.0000 - fp: 988.0000 - tn: 1775.0000 - fn: 899.0000 - auc: 0.7220 - val_loss: 0.5937 - val_tp: 451.0000 - val_fp: 219.0000 - val_tn: 467.0000 - val_fn: 239.0000 - val_auc: 0.7458\n\nEpoch 00014: val_auc did not improve from 0.75540\nEpoch 15/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5924 - tp: 1945.0000 - fp: 974.0000 - tn: 1794.0000 - fn: 817.0000 - auc: 0.7458 - val_loss: 0.5697 - val_tp: 560.0000 - val_fp: 303.0000 - val_tn: 387.0000 - val_fn: 126.0000 - val_auc: 0.7790\n\nEpoch 00015: val_auc improved from 0.75540 to 0.77896, saving model to /kaggle/working/undersampling_model.h5\nEpoch 16/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5880 - tp: 1910.0000 - fp: 901.0000 - tn: 1869.0000 - fn: 850.0000 - auc: 0.7505 - val_loss: 0.5674 - val_tp: 427.0000 - val_fp: 169.0000 - val_tn: 516.0000 - val_fn: 264.0000 - val_auc: 0.7781\n\nEpoch 00016: val_auc did not improve from 0.77896\nEpoch 17/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.5833 - tp: 1888.0000 - fp: 901.0000 - tn: 1865.0000 - fn: 876.0000 - auc: 0.7550 - val_loss: 0.5639 - val_tp: 501.0000 - val_fp: 202.0000 - val_tn: 488.0000 - val_fn: 185.0000 - val_auc: 0.7827\n\nEpoch 00017: val_auc improved from 0.77896 to 0.78266, saving model to /kaggle/working/undersampling_model.h5\nEpoch 18/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5712 - tp: 1940.0000 - fp: 875.0000 - tn: 1888.0000 - fn: 827.0000 - auc: 0.7676 - val_loss: 0.5475 - val_tp: 472.0000 - val_fp: 184.0000 - val_tn: 506.0000 - val_fn: 214.0000 - val_auc: 0.7916\n\nEpoch 00018: val_auc improved from 0.78266 to 0.79159, saving model to /kaggle/working/undersampling_model.h5\nEpoch 19/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5671 - tp: 1955.0000 - fp: 873.0000 - tn: 1889.0000 - fn: 813.0000 - auc: 0.7715 - val_loss: 0.5576 - val_tp: 472.0000 - val_fp: 201.0000 - val_tn: 488.0000 - val_fn: 215.0000 - val_auc: 0.7831\n\nEpoch 00019: val_auc did not improve from 0.79159\nEpoch 20/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5678 - tp: 1938.0000 - fp: 862.0000 - tn: 1905.0000 - fn: 825.0000 - auc: 0.7722 - val_loss: 0.5520 - val_tp: 391.0000 - val_fp: 125.0000 - val_tn: 561.0000 - val_fn: 299.0000 - val_auc: 0.7903\n\nEpoch 00020: val_auc did not improve from 0.79159\nEpoch 21/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.5544 - tp: 1941.0000 - fp: 825.0000 - tn: 1936.0000 - fn: 828.0000 - auc: 0.7833 - val_loss: 0.5444 - val_tp: 466.0000 - val_fp: 177.0000 - val_tn: 509.0000 - val_fn: 224.0000 - val_auc: 0.8018\n\nEpoch 00021: val_auc improved from 0.79159 to 0.80179, saving model to /kaggle/working/undersampling_model.h5\nEpoch 22/100\n173/173 [==============================] - 52s 299ms/step - loss: 0.5499 - tp: 1956.0000 - fp: 804.0000 - tn: 1960.0000 - fn: 810.0000 - auc: 0.7864 - val_loss: 0.5522 - val_tp: 448.0000 - val_fp: 157.0000 - val_tn: 531.0000 - val_fn: 240.0000 - val_auc: 0.7892\n\nEpoch 00022: val_auc did not improve from 0.80179\nEpoch 23/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.5583 - tp: 1908.0000 - fp: 799.0000 - tn: 1967.0000 - fn: 856.0000 - auc: 0.7795 - val_loss: 0.5471 - val_tp: 527.0000 - val_fp: 252.0000 - val_tn: 439.0000 - val_fn: 158.0000 - val_auc: 0.7980\n\nEpoch 00023: val_auc did not improve from 0.80179\nEpoch 24/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5518 - tp: 1897.0000 - fp: 774.0000 - tn: 1993.0000 - fn: 866.0000 - auc: 0.7853 - val_loss: 0.5595 - val_tp: 348.0000 - val_fp: 75.0000 - val_tn: 608.0000 - val_fn: 345.0000 - val_auc: 0.8003\n\nEpoch 00024: val_auc did not improve from 0.80179\nEpoch 25/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.5470 - tp: 1936.0000 - fp: 760.0000 - tn: 2003.0000 - fn: 831.0000 - auc: 0.7896 - val_loss: 0.5476 - val_tp: 473.0000 - val_fp: 175.0000 - val_tn: 511.0000 - val_fn: 217.0000 - val_auc: 0.7976\n\nEpoch 00025: val_auc did not improve from 0.80179\nEpoch 26/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5446 - tp: 1936.0000 - fp: 769.0000 - tn: 1999.0000 - fn: 826.0000 - auc: 0.7898 - val_loss: 0.5444 - val_tp: 390.0000 - val_fp: 113.0000 - val_tn: 574.0000 - val_fn: 299.0000 - val_auc: 0.7981\n\nEpoch 00026: val_auc did not improve from 0.80179\nEpoch 27/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.5682 - tp: 1893.0000 - fp: 817.0000 - tn: 1947.0000 - fn: 873.0000 - auc: 0.7740 - val_loss: 0.5476 - val_tp: 430.0000 - val_fp: 155.0000 - val_tn: 536.0000 - val_fn: 255.0000 - val_auc: 0.7886\n\nEpoch 00027: val_auc did not improve from 0.80179\nEpoch 28/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.5465 - tp: 1948.0000 - fp: 775.0000 - tn: 1986.0000 - fn: 821.0000 - auc: 0.7895 - val_loss: 0.5359 - val_tp: 475.0000 - val_fp: 186.0000 - val_tn: 502.0000 - val_fn: 213.0000 - val_auc: 0.7956\n\nEpoch 00028: val_auc did not improve from 0.80179\nEpoch 29/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.5336 - tp: 1931.0000 - fp: 776.0000 - tn: 1988.0000 - fn: 835.0000 - auc: 0.7980 - val_loss: 0.5488 - val_tp: 369.0000 - val_fp: 92.0000 - val_tn: 595.0000 - val_fn: 320.0000 - val_auc: 0.7986\n\nEpoch 00029: val_auc did not improve from 0.80179\nEpoch 30/100\n173/173 [==============================] - 52s 299ms/step - loss: 0.5461 - tp: 1947.0000 - fp: 782.0000 - tn: 1982.0000 - fn: 819.0000 - auc: 0.7929 - val_loss: 0.5469 - val_tp: 480.0000 - val_fp: 195.0000 - val_tn: 492.0000 - val_fn: 209.0000 - val_auc: 0.7859\n\nEpoch 00030: val_auc did not improve from 0.80179\nEpoch 31/100\n173/173 [==============================] - 52s 301ms/step - loss: 0.5332 - tp: 1921.0000 - fp: 705.0000 - tn: 2059.0000 - fn: 845.0000 - auc: 0.8007 - val_loss: 0.5239 - val_tp: 504.0000 - val_fp: 211.0000 - val_tn: 477.0000 - val_fn: 184.0000 - val_auc: 0.8059\n\nEpoch 00031: val_auc improved from 0.80179 to 0.80588, saving model to /kaggle/working/undersampling_model.h5\nEpoch 32/100\n173/173 [==============================] - 52s 302ms/step - loss: 0.5349 - tp: 1976.0000 - fp: 745.0000 - tn: 2020.0000 - fn: 789.0000 - auc: 0.8012 - val_loss: 0.5449 - val_tp: 488.0000 - val_fp: 200.0000 - val_tn: 488.0000 - val_fn: 200.0000 - val_auc: 0.8012\n\nEpoch 00032: val_auc did not improve from 0.80588\nEpoch 33/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.5274 - tp: 1938.0000 - fp: 719.0000 - tn: 2046.0000 - fn: 827.0000 - auc: 0.8050 - val_loss: 0.5409 - val_tp: 396.0000 - val_fp: 108.0000 - val_tn: 579.0000 - val_fn: 293.0000 - val_auc: 0.8040\n\nEpoch 00033: val_auc did not improve from 0.80588\nEpoch 34/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5243 - tp: 1977.0000 - fp: 711.0000 - tn: 2051.0000 - fn: 791.0000 - auc: 0.8081 - val_loss: 0.5193 - val_tp: 461.0000 - val_fp: 160.0000 - val_tn: 526.0000 - val_fn: 229.0000 - val_auc: 0.8098\n\nEpoch 00034: val_auc improved from 0.80588 to 0.80981, saving model to /kaggle/working/undersampling_model.h5\nEpoch 35/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.5167 - tp: 1965.0000 - fp: 689.0000 - tn: 2076.0000 - fn: 800.0000 - auc: 0.8145 - val_loss: 0.5550 - val_tp: 600.0000 - val_fp: 302.0000 - val_tn: 384.0000 - val_fn: 90.0000 - val_auc: 0.8182\n\nEpoch 00035: val_auc improved from 0.80981 to 0.81824, saving model to /kaggle/working/undersampling_model.h5\nEpoch 36/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5184 - tp: 1970.0000 - fp: 700.0000 - tn: 2068.0000 - fn: 792.0000 - auc: 0.8139 - val_loss: 0.5204 - val_tp: 471.0000 - val_fp: 172.0000 - val_tn: 513.0000 - val_fn: 220.0000 - val_auc: 0.8124\n\nEpoch 00036: val_auc did not improve from 0.81824\nEpoch 37/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5120 - tp: 2009.0000 - fp: 713.0000 - tn: 2052.0000 - fn: 756.0000 - auc: 0.8182 - val_loss: 0.5235 - val_tp: 459.0000 - val_fp: 144.0000 - val_tn: 542.0000 - val_fn: 231.0000 - val_auc: 0.8106\n\nEpoch 00037: val_auc did not improve from 0.81824\nEpoch 38/100\n173/173 [==============================] - 52s 300ms/step - loss: 0.5141 - tp: 1978.0000 - fp: 714.0000 - tn: 2049.0000 - fn: 789.0000 - auc: 0.8168 - val_loss: 0.5194 - val_tp: 508.0000 - val_fp: 211.0000 - val_tn: 482.0000 - val_fn: 175.0000 - val_auc: 0.8130\n\nEpoch 00038: val_auc did not improve from 0.81824\nEpoch 39/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.5130 - tp: 2000.0000 - fp: 687.0000 - tn: 2076.0000 - fn: 767.0000 - auc: 0.8207 - val_loss: 0.5270 - val_tp: 454.0000 - val_fp: 153.0000 - val_tn: 536.0000 - val_fn: 233.0000 - val_auc: 0.8095\n\nEpoch 00039: val_auc did not improve from 0.81824\nEpoch 40/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.5167 - tp: 1978.0000 - fp: 690.0000 - tn: 2074.0000 - fn: 788.0000 - auc: 0.8167 - val_loss: 0.5274 - val_tp: 508.0000 - val_fp: 199.0000 - val_tn: 484.0000 - val_fn: 185.0000 - val_auc: 0.8097\n\nEpoch 00040: val_auc did not improve from 0.81824\nEpoch 41/100\n173/173 [==============================] - 53s 307ms/step - loss: 0.5124 - tp: 2012.0000 - fp: 690.0000 - tn: 2078.0000 - fn: 750.0000 - auc: 0.8199 - val_loss: 0.5162 - val_tp: 531.0000 - val_fp: 206.0000 - val_tn: 483.0000 - val_fn: 156.0000 - val_auc: 0.8176\n\nEpoch 00041: val_auc did not improve from 0.81824\nEpoch 42/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.5061 - tp: 2007.0000 - fp: 702.0000 - tn: 2063.0000 - fn: 758.0000 - auc: 0.8228 - val_loss: 0.5545 - val_tp: 565.0000 - val_fp: 279.0000 - val_tn: 409.0000 - val_fn: 123.0000 - val_auc: 0.8001\n\nEpoch 00042: val_auc did not improve from 0.81824\nEpoch 43/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.5160 - tp: 1975.0000 - fp: 693.0000 - tn: 2077.0000 - fn: 785.0000 - auc: 0.8183 - val_loss: 0.5611 - val_tp: 359.0000 - val_fp: 74.0000 - val_tn: 612.0000 - val_fn: 331.0000 - val_auc: 0.8075\n\nEpoch 00043: val_auc did not improve from 0.81824\nEpoch 44/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.4999 - tp: 2005.0000 - fp: 669.0000 - tn: 2098.0000 - fn: 758.0000 - auc: 0.8282 - val_loss: 0.5218 - val_tp: 481.0000 - val_fp: 173.0000 - val_tn: 514.0000 - val_fn: 208.0000 - val_auc: 0.8098\n\nEpoch 00044: val_auc did not improve from 0.81824\nEpoch 45/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4967 - tp: 1986.0000 - fp: 634.0000 - tn: 2134.0000 - fn: 776.0000 - auc: 0.8311 - val_loss: 0.5321 - val_tp: 469.0000 - val_fp: 156.0000 - val_tn: 529.0000 - val_fn: 222.0000 - val_auc: 0.8034\n\nEpoch 00045: val_auc did not improve from 0.81824\nEpoch 46/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4924 - tp: 2043.0000 - fp: 664.0000 - tn: 2101.0000 - fn: 722.0000 - auc: 0.8359 - val_loss: 0.5251 - val_tp: 581.0000 - val_fp: 282.0000 - val_tn: 407.0000 - val_fn: 106.0000 - val_auc: 0.8181\n\nEpoch 00046: val_auc did not improve from 0.81824\nEpoch 47/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4920 - tp: 2036.0000 - fp: 645.0000 - tn: 2121.0000 - fn: 728.0000 - auc: 0.8357 - val_loss: 0.5306 - val_tp: 484.0000 - val_fp: 184.0000 - val_tn: 503.0000 - val_fn: 205.0000 - val_auc: 0.8001\n\nEpoch 00047: val_auc did not improve from 0.81824\nEpoch 48/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.4911 - tp: 2022.0000 - fp: 669.0000 - tn: 2101.0000 - fn: 738.0000 - auc: 0.8350 - val_loss: 0.5522 - val_tp: 467.0000 - val_fp: 174.0000 - val_tn: 512.0000 - val_fn: 223.0000 - val_auc: 0.7977\n\nEpoch 00048: val_auc did not improve from 0.81824\nEpoch 49/100\n173/173 [==============================] - 52s 302ms/step - loss: 0.4990 - tp: 2020.0000 - fp: 678.0000 - tn: 2083.0000 - fn: 749.0000 - auc: 0.8314 - val_loss: 0.5433 - val_tp: 565.0000 - val_fp: 266.0000 - val_tn: 424.0000 - val_fn: 121.0000 - val_auc: 0.8102\n\nEpoch 00049: val_auc did not improve from 0.81824\nEpoch 50/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.4855 - tp: 2042.0000 - fp: 628.0000 - tn: 2133.0000 - fn: 727.0000 - auc: 0.8402 - val_loss: 0.5477 - val_tp: 577.0000 - val_fp: 278.0000 - val_tn: 410.0000 - val_fn: 111.0000 - val_auc: 0.8058\n\nEpoch 00050: val_auc did not improve from 0.81824\nEpoch 51/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4831 - tp: 2055.0000 - fp: 661.0000 - tn: 2106.0000 - fn: 708.0000 - auc: 0.8422 - val_loss: 0.5428 - val_tp: 537.0000 - val_fp: 227.0000 - val_tn: 461.0000 - val_fn: 151.0000 - val_auc: 0.8064\n\nEpoch 00051: val_auc did not improve from 0.81824\nEpoch 52/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4915 - tp: 2019.0000 - fp: 628.0000 - tn: 2142.0000 - fn: 741.0000 - auc: 0.8370 - val_loss: 0.5315 - val_tp: 482.0000 - val_fp: 188.0000 - val_tn: 502.0000 - val_fn: 204.0000 - val_auc: 0.8069\n\nEpoch 00052: val_auc did not improve from 0.81824\nEpoch 53/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.4774 - tp: 2039.0000 - fp: 625.0000 - tn: 2139.0000 - fn: 727.0000 - auc: 0.8456 - val_loss: 0.5367 - val_tp: 421.0000 - val_fp: 132.0000 - val_tn: 554.0000 - val_fn: 269.0000 - val_auc: 0.8034\n\nEpoch 00053: val_auc did not improve from 0.81824\nEpoch 54/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4819 - tp: 2032.0000 - fp: 651.0000 - tn: 2114.0000 - fn: 733.0000 - auc: 0.8423 - val_loss: 0.5287 - val_tp: 510.0000 - val_fp: 225.0000 - val_tn: 464.0000 - val_fn: 177.0000 - val_auc: 0.8067\n\nEpoch 00054: val_auc did not improve from 0.81824\nEpoch 55/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4750 - tp: 2073.0000 - fp: 648.0000 - tn: 2112.0000 - fn: 697.0000 - auc: 0.8488 - val_loss: 0.5745 - val_tp: 577.0000 - val_fp: 269.0000 - val_tn: 420.0000 - val_fn: 110.0000 - val_auc: 0.8102\n\nEpoch 00055: val_auc did not improve from 0.81824\nEpoch 56/100\n173/173 [==============================] - 51s 292ms/step - loss: 0.4846 - tp: 2002.0000 - fp: 649.0000 - tn: 2121.0000 - fn: 764.0000 - auc: 0.8400 - val_loss: 0.5337 - val_tp: 479.0000 - val_fp: 177.0000 - val_tn: 513.0000 - val_fn: 207.0000 - val_auc: 0.8092\n\nEpoch 00056: val_auc did not improve from 0.81824\nEpoch 57/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.4639 - tp: 2058.0000 - fp: 605.0000 - tn: 2166.0000 - fn: 701.0000 - auc: 0.8555 - val_loss: 0.5220 - val_tp: 469.0000 - val_fp: 159.0000 - val_tn: 527.0000 - val_fn: 221.0000 - val_auc: 0.8110\n\nEpoch 00057: val_auc did not improve from 0.81824\nEpoch 58/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4670 - tp: 2077.0000 - fp: 593.0000 - tn: 2174.0000 - fn: 686.0000 - auc: 0.8551 - val_loss: 0.5492 - val_tp: 445.0000 - val_fp: 148.0000 - val_tn: 537.0000 - val_fn: 246.0000 - val_auc: 0.8036\n\nEpoch 00058: val_auc did not improve from 0.81824\nEpoch 59/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4918 - tp: 2050.0000 - fp: 628.0000 - tn: 2135.0000 - fn: 717.0000 - auc: 0.8379 - val_loss: 0.5336 - val_tp: 535.0000 - val_fp: 228.0000 - val_tn: 457.0000 - val_fn: 156.0000 - val_auc: 0.8121\n\nEpoch 00059: val_auc did not improve from 0.81824\nEpoch 60/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4587 - tp: 2098.0000 - fp: 585.0000 - tn: 2178.0000 - fn: 669.0000 - auc: 0.8599 - val_loss: 0.5732 - val_tp: 512.0000 - val_fp: 202.0000 - val_tn: 483.0000 - val_fn: 179.0000 - val_auc: 0.8071\n\nEpoch 00060: val_auc did not improve from 0.81824\nEpoch 61/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4711 - tp: 2082.0000 - fp: 621.0000 - tn: 2146.0000 - fn: 681.0000 - auc: 0.8516 - val_loss: 0.5598 - val_tp: 504.0000 - val_fp: 232.0000 - val_tn: 458.0000 - val_fn: 182.0000 - val_auc: 0.7984\n\nEpoch 00061: val_auc did not improve from 0.81824\nEpoch 62/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.4483 - tp: 2080.0000 - fp: 566.0000 - tn: 2202.0000 - fn: 682.0000 - auc: 0.8664 - val_loss: 0.5402 - val_tp: 440.0000 - val_fp: 160.0000 - val_tn: 526.0000 - val_fn: 250.0000 - val_auc: 0.8002\n\nEpoch 00062: val_auc did not improve from 0.81824\nEpoch 63/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4500 - tp: 2125.0000 - fp: 569.0000 - tn: 2194.0000 - fn: 642.0000 - auc: 0.8667 - val_loss: 0.5392 - val_tp: 446.0000 - val_fp: 148.0000 - val_tn: 544.0000 - val_fn: 238.0000 - val_auc: 0.8127\n\nEpoch 00063: val_auc did not improve from 0.81824\nEpoch 64/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4461 - tp: 2078.0000 - fp: 565.0000 - tn: 2203.0000 - fn: 684.0000 - auc: 0.8678 - val_loss: 0.5438 - val_tp: 430.0000 - val_fp: 143.0000 - val_tn: 545.0000 - val_fn: 258.0000 - val_auc: 0.8001\n\nEpoch 00064: val_auc did not improve from 0.81824\nEpoch 65/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4499 - tp: 2104.0000 - fp: 588.0000 - tn: 2176.0000 - fn: 662.0000 - auc: 0.8664 - val_loss: 0.5301 - val_tp: 459.0000 - val_fp: 158.0000 - val_tn: 529.0000 - val_fn: 230.0000 - val_auc: 0.8109\n\nEpoch 00065: val_auc did not improve from 0.81824\nEpoch 66/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.4300 - tp: 2151.0000 - fp: 540.0000 - tn: 2223.0000 - fn: 616.0000 - auc: 0.8788 - val_loss: 0.5324 - val_tp: 421.0000 - val_fp: 107.0000 - val_tn: 580.0000 - val_fn: 268.0000 - val_auc: 0.8177\n\nEpoch 00066: val_auc did not improve from 0.81824\nEpoch 67/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.4337 - tp: 2126.0000 - fp: 555.0000 - tn: 2206.0000 - fn: 643.0000 - auc: 0.8763 - val_loss: 0.6259 - val_tp: 581.0000 - val_fp: 303.0000 - val_tn: 386.0000 - val_fn: 106.0000 - val_auc: 0.8029\n\nEpoch 00067: val_auc did not improve from 0.81824\nEpoch 68/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.4275 - tp: 2141.0000 - fp: 544.0000 - tn: 2222.0000 - fn: 623.0000 - auc: 0.8793 - val_loss: 0.5963 - val_tp: 417.0000 - val_fp: 145.0000 - val_tn: 543.0000 - val_fn: 271.0000 - val_auc: 0.7883\n\nEpoch 00068: val_auc did not improve from 0.81824\nEpoch 69/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.4347 - tp: 2133.0000 - fp: 554.0000 - tn: 2212.0000 - fn: 631.0000 - auc: 0.8761 - val_loss: 0.5388 - val_tp: 490.0000 - val_fp: 184.0000 - val_tn: 506.0000 - val_fn: 196.0000 - val_auc: 0.8159\n\nEpoch 00069: val_auc did not improve from 0.81824\nEpoch 70/100\n173/173 [==============================] - 51s 294ms/step - loss: 0.4265 - tp: 2143.0000 - fp: 570.0000 - tn: 2192.0000 - fn: 625.0000 - auc: 0.8797 - val_loss: 0.5572 - val_tp: 514.0000 - val_fp: 198.0000 - val_tn: 488.0000 - val_fn: 176.0000 - val_auc: 0.8076\n\nEpoch 00070: val_auc did not improve from 0.81824\nEpoch 71/100\n173/173 [==============================] - 52s 300ms/step - loss: 0.4318 - tp: 2134.0000 - fp: 527.0000 - tn: 2241.0000 - fn: 628.0000 - auc: 0.8780 - val_loss: 0.5631 - val_tp: 457.0000 - val_fp: 163.0000 - val_tn: 524.0000 - val_fn: 232.0000 - val_auc: 0.7865\n\nEpoch 00071: val_auc did not improve from 0.81824\nEpoch 72/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.4162 - tp: 2188.0000 - fp: 527.0000 - tn: 2233.0000 - fn: 582.0000 - auc: 0.8874 - val_loss: 0.5567 - val_tp: 477.0000 - val_fp: 187.0000 - val_tn: 500.0000 - val_fn: 212.0000 - val_auc: 0.7968\n\nEpoch 00072: val_auc did not improve from 0.81824\nEpoch 73/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.4040 - tp: 2187.0000 - fp: 466.0000 - tn: 2299.0000 - fn: 578.0000 - auc: 0.8950 - val_loss: 0.5829 - val_tp: 465.0000 - val_fp: 188.0000 - val_tn: 502.0000 - val_fn: 221.0000 - val_auc: 0.7879\n\nEpoch 00073: val_auc did not improve from 0.81824\nEpoch 74/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.3943 - tp: 2193.0000 - fp: 482.0000 - tn: 2282.0000 - fn: 573.0000 - auc: 0.8996 - val_loss: 0.5820 - val_tp: 439.0000 - val_fp: 148.0000 - val_tn: 540.0000 - val_fn: 249.0000 - val_auc: 0.7903\n\nEpoch 00074: val_auc did not improve from 0.81824\nEpoch 75/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.4053 - tp: 2184.0000 - fp: 521.0000 - tn: 2245.0000 - fn: 580.0000 - auc: 0.8931 - val_loss: 0.6159 - val_tp: 526.0000 - val_fp: 215.0000 - val_tn: 474.0000 - val_fn: 161.0000 - val_auc: 0.7986\n\nEpoch 00075: val_auc did not improve from 0.81824\nEpoch 76/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.3814 - tp: 2220.0000 - fp: 451.0000 - tn: 2309.0000 - fn: 550.0000 - auc: 0.9084 - val_loss: 0.5816 - val_tp: 498.0000 - val_fp: 180.0000 - val_tn: 506.0000 - val_fn: 192.0000 - val_auc: 0.8041\n\nEpoch 00076: val_auc did not improve from 0.81824\nEpoch 77/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.3693 - tp: 2263.0000 - fp: 440.0000 - tn: 2325.0000 - fn: 502.0000 - auc: 0.9139 - val_loss: 0.6265 - val_tp: 500.0000 - val_fp: 198.0000 - val_tn: 490.0000 - val_fn: 188.0000 - val_auc: 0.8008\n\nEpoch 00077: val_auc did not improve from 0.81824\nEpoch 78/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.3859 - tp: 2197.0000 - fp: 455.0000 - tn: 2309.0000 - fn: 569.0000 - auc: 0.9037 - val_loss: 0.6277 - val_tp: 472.0000 - val_fp: 170.0000 - val_tn: 516.0000 - val_fn: 218.0000 - val_auc: 0.7984\n\nEpoch 00078: val_auc did not improve from 0.81824\nEpoch 79/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.3750 - tp: 2212.0000 - fp: 456.0000 - tn: 2308.0000 - fn: 554.0000 - auc: 0.9095 - val_loss: 0.6609 - val_tp: 490.0000 - val_fp: 195.0000 - val_tn: 493.0000 - val_fn: 198.0000 - val_auc: 0.7856\n\nEpoch 00079: val_auc did not improve from 0.81824\nEpoch 80/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.3778 - tp: 2234.0000 - fp: 454.0000 - tn: 2311.0000 - fn: 531.0000 - auc: 0.9084 - val_loss: 0.6277 - val_tp: 454.0000 - val_fp: 177.0000 - val_tn: 512.0000 - val_fn: 233.0000 - val_auc: 0.7855\n\nEpoch 00080: val_auc did not improve from 0.81824\nEpoch 81/100\n173/173 [==============================] - 51s 297ms/step - loss: 0.3493 - tp: 2278.0000 - fp: 383.0000 - tn: 2383.0000 - fn: 486.0000 - auc: 0.9236 - val_loss: 0.7484 - val_tp: 548.0000 - val_fp: 274.0000 - val_tn: 415.0000 - val_fn: 139.0000 - val_auc: 0.7912\n\nEpoch 00081: val_auc did not improve from 0.81824\nEpoch 82/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.3526 - tp: 2247.0000 - fp: 422.0000 - tn: 2344.0000 - fn: 517.0000 - auc: 0.9206 - val_loss: 0.7080 - val_tp: 481.0000 - val_fp: 202.0000 - val_tn: 486.0000 - val_fn: 207.0000 - val_auc: 0.7843\n\nEpoch 00082: val_auc did not improve from 0.81824\nEpoch 83/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.3296 - tp: 2302.0000 - fp: 396.0000 - tn: 2369.0000 - fn: 463.0000 - auc: 0.9312 - val_loss: 0.6496 - val_tp: 425.0000 - val_fp: 150.0000 - val_tn: 539.0000 - val_fn: 262.0000 - val_auc: 0.7758\n\nEpoch 00083: val_auc did not improve from 0.81824\nEpoch 84/100\n173/173 [==============================] - 52s 299ms/step - loss: 0.3290 - tp: 2306.0000 - fp: 356.0000 - tn: 2403.0000 - fn: 465.0000 - auc: 0.9318 - val_loss: 0.7556 - val_tp: 396.0000 - val_fp: 116.0000 - val_tn: 569.0000 - val_fn: 295.0000 - val_auc: 0.7885\n\nEpoch 00084: val_auc did not improve from 0.81824\nEpoch 85/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.3411 - tp: 2305.0000 - fp: 366.0000 - tn: 2397.0000 - fn: 462.0000 - auc: 0.9278 - val_loss: 0.6928 - val_tp: 538.0000 - val_fp: 248.0000 - val_tn: 440.0000 - val_fn: 150.0000 - val_auc: 0.8033\n\nEpoch 00085: val_auc did not improve from 0.81824\nEpoch 86/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.3252 - tp: 2288.0000 - fp: 363.0000 - tn: 2402.0000 - fn: 477.0000 - auc: 0.9331 - val_loss: 0.6741 - val_tp: 492.0000 - val_fp: 186.0000 - val_tn: 499.0000 - val_fn: 199.0000 - val_auc: 0.7948\n\nEpoch 00086: val_auc did not improve from 0.81824\nEpoch 87/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.3109 - tp: 2312.0000 - fp: 346.0000 - tn: 2420.0000 - fn: 452.0000 - auc: 0.9395 - val_loss: 0.6900 - val_tp: 539.0000 - val_fp: 256.0000 - val_tn: 431.0000 - val_fn: 150.0000 - val_auc: 0.7845\n\nEpoch 00087: val_auc did not improve from 0.81824\nEpoch 88/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.2906 - tp: 2374.0000 - fp: 328.0000 - tn: 2437.0000 - fn: 391.0000 - auc: 0.9470 - val_loss: 0.8297 - val_tp: 499.0000 - val_fp: 238.0000 - val_tn: 450.0000 - val_fn: 189.0000 - val_auc: 0.7714\n\nEpoch 00088: val_auc did not improve from 0.81824\nEpoch 89/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.3053 - tp: 2356.0000 - fp: 337.0000 - tn: 2425.0000 - fn: 412.0000 - auc: 0.9421 - val_loss: 0.7190 - val_tp: 504.0000 - val_fp: 204.0000 - val_tn: 481.0000 - val_fn: 187.0000 - val_auc: 0.7933\n\nEpoch 00089: val_auc did not improve from 0.81824\nEpoch 90/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.2943 - tp: 2357.0000 - fp: 318.0000 - tn: 2453.0000 - fn: 402.0000 - auc: 0.9465 - val_loss: 0.7579 - val_tp: 433.0000 - val_fp: 153.0000 - val_tn: 537.0000 - val_fn: 253.0000 - val_auc: 0.7771\n\nEpoch 00090: val_auc did not improve from 0.81824\nEpoch 91/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.2817 - tp: 2374.0000 - fp: 311.0000 - tn: 2452.0000 - fn: 393.0000 - auc: 0.9502 - val_loss: 0.8361 - val_tp: 475.0000 - val_fp: 188.0000 - val_tn: 499.0000 - val_fn: 214.0000 - val_auc: 0.7762\n\nEpoch 00091: val_auc did not improve from 0.81824\nEpoch 92/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.2598 - tp: 2411.0000 - fp: 278.0000 - tn: 2488.0000 - fn: 353.0000 - auc: 0.9581 - val_loss: 0.8166 - val_tp: 507.0000 - val_fp: 237.0000 - val_tn: 450.0000 - val_fn: 182.0000 - val_auc: 0.7757\n\nEpoch 00092: val_auc did not improve from 0.81824\nEpoch 93/100\n173/173 [==============================] - 51s 296ms/step - loss: 0.2784 - tp: 2401.0000 - fp: 316.0000 - tn: 2448.0000 - fn: 365.0000 - auc: 0.9525 - val_loss: 0.7306 - val_tp: 470.0000 - val_fp: 207.0000 - val_tn: 484.0000 - val_fn: 215.0000 - val_auc: 0.7682\n\nEpoch 00093: val_auc did not improve from 0.81824\nEpoch 94/100\n173/173 [==============================] - 52s 299ms/step - loss: 0.2635 - tp: 2416.0000 - fp: 267.0000 - tn: 2497.0000 - fn: 350.0000 - auc: 0.9570 - val_loss: 0.8438 - val_tp: 501.0000 - val_fp: 221.0000 - val_tn: 467.0000 - val_fn: 187.0000 - val_auc: 0.7674\n\nEpoch 00094: val_auc did not improve from 0.81824\nEpoch 95/100\n173/173 [==============================] - 51s 295ms/step - loss: 0.2750 - tp: 2372.0000 - fp: 284.0000 - tn: 2480.0000 - fn: 394.0000 - auc: 0.9529 - val_loss: 0.8322 - val_tp: 469.0000 - val_fp: 193.0000 - val_tn: 492.0000 - val_fn: 222.0000 - val_auc: 0.7723\n\nEpoch 00095: val_auc did not improve from 0.81824\nEpoch 96/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.2457 - tp: 2441.0000 - fp: 263.0000 - tn: 2499.0000 - fn: 327.0000 - auc: 0.9629 - val_loss: 0.8744 - val_tp: 535.0000 - val_fp: 216.0000 - val_tn: 473.0000 - val_fn: 152.0000 - val_auc: 0.8017\n\nEpoch 00096: val_auc did not improve from 0.81824\nEpoch 97/100\n173/173 [==============================] - 52s 297ms/step - loss: 0.2425 - tp: 2445.0000 - fp: 260.0000 - tn: 2502.0000 - fn: 323.0000 - auc: 0.9636 - val_loss: 1.0257 - val_tp: 549.0000 - val_fp: 271.0000 - val_tn: 417.0000 - val_fn: 139.0000 - val_auc: 0.7943\n\nEpoch 00097: val_auc did not improve from 0.81824\nEpoch 98/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.2205 - tp: 2469.0000 - fp: 224.0000 - tn: 2545.0000 - fn: 292.0000 - auc: 0.9701 - val_loss: 0.9954 - val_tp: 394.0000 - val_fp: 123.0000 - val_tn: 564.0000 - val_fn: 295.0000 - val_auc: 0.7614\n\nEpoch 00098: val_auc did not improve from 0.81824\nEpoch 99/100\n173/173 [==============================] - 52s 301ms/step - loss: 0.2535 - tp: 2449.0000 - fp: 284.0000 - tn: 2485.0000 - fn: 312.0000 - auc: 0.9613 - val_loss: 0.8041 - val_tp: 425.0000 - val_fp: 139.0000 - val_tn: 542.0000 - val_fn: 270.0000 - val_auc: 0.7632\n\nEpoch 00099: val_auc did not improve from 0.81824\nEpoch 100/100\n173/173 [==============================] - 52s 298ms/step - loss: 0.2176 - tp: 2489.0000 - fp: 232.0000 - tn: 2533.0000 - fn: 276.0000 - auc: 0.9710 - val_loss: 0.8885 - val_tp: 496.0000 - val_fp: 208.0000 - val_tn: 481.0000 - val_fn: 191.0000 - val_auc: 0.7799\n\nEpoch 00100: val_auc did not improve from 0.81824\n","output_type":"stream"},{"execution_count":25,"output_type":"execute_result","data":{"text/plain":"<tensorflow.python.keras.callbacks.History at 0x7f1a781c4310>"},"metadata":{}}]},{"cell_type":"markdown","source":"## <font color=red>4. </font>Evaluar el modelo","metadata":{}},{"cell_type":"code","source":"eval_metrics = model_undersampling.evaluate(test_generator,\n                                            steps=STEP_SIZE_TEST,\n                                            return_dict=True,\n                                            use_multiprocessing=False,\n                                            verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:17:21.080207Z","iopub.execute_input":"2021-12-01T17:17:21.082340Z","iopub.status.idle":"2021-12-01T17:17:25.511962Z","shell.execute_reply.started":"2021-12-01T17:17:21.082305Z","shell.execute_reply":"2021-12-01T17:17:25.511255Z"},"trusted":true},"execution_count":120,"outputs":[{"name":"stdout","text":"32/32 [==============================] - 4s 132ms/step - loss: 0.8363 - tp: 353.0000 - fp: 137.0000 - tn: 375.0000 - fn: 159.0000 - auc: 0.7870\n","output_type":"stream"}]},{"cell_type":"code","source":"calib_labels = calib_generator.classes\ntest_labels = test_generator.classes\n\npredict_calib = model_undersampling.predict(calib_generator,\n                                            verbose=1)\npredict_test = model_undersampling.predict(test_generator,\n                                           verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:17:25.514054Z","iopub.execute_input":"2021-12-01T17:17:25.514326Z","iopub.status.idle":"2021-12-01T17:17:32.008653Z","shell.execute_reply.started":"2021-12-01T17:17:25.514289Z","shell.execute_reply":"2021-12-01T17:17:32.007989Z"},"trusted":true},"execution_count":121,"outputs":[{"name":"stdout","text":"22/22 [==============================] - 3s 117ms/step\n33/33 [==============================] - 4s 111ms/step\n","output_type":"stream"}]},{"cell_type":"code","source":"from sklearn import metrics\n\nfpr, tpr, tr = metrics.roc_curve(test_labels, predict_test.ravel())\nauc = metrics.roc_auc_score(test_labels, predict_test.ravel())\nplt.plot(fpr,tpr,'b',label=\"AUC=\"+str(auc))\nplt.plot([0,1],[0,1],'k--')\nplt.title('Test evaluation')\nplt.grid()\nplt.legend(loc='best')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:17:32.009831Z","iopub.execute_input":"2021-12-01T17:17:32.010103Z","iopub.status.idle":"2021-12-01T17:17:32.217188Z","shell.execute_reply.started":"2021-12-01T17:17:32.010069Z","shell.execute_reply":"2021-12-01T17:17:32.216482Z"},"trusted":true},"execution_count":122,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA60klEQVR4nO3deVxUVRvA8d8JNFvMciNXXFHApYw0NRV3LU19U9NMW3DPMk3LskzNfSk3zMxKy3Jp1cq0TdTqbdFcXtckBEUtFRVBY3/eP+6AgAuoA5eZeb6fz3zmLufeec4MPBzO3HuOERGUUkq5vuvsDkAppZRzaEJXSik3oQldKaXchCZ0pZRyE5rQlVLKTWhCV0opN6EJXalcMMZUMsaIMcY7j86/yxgTnBfnVp5DE7rKU8aY+EyPNGPMv5nWe13F+cKMMX3zItb8YoxZbIyZkHmbiASKSJhNISk3kSetDaXSicjN6cvGmEigr4h8Z19ESrkvbaErWxhjrjPGjDLG/GWMiTHGrDTGFHfsK2KMWerYftoY87sxxscYMxFoAsxztPDnXeLc9xhjfnYcuz29K8MY85AxZnO2ssOMMasdy/cbY7YaY84YYw4ZY8ZeJv5IY0yrTOtjjTFLM61/ZIz52xgTa4zZaIwJdGzvD/QCnnPU4Yvs5zPGXG+MmWWMOeJ4zDLGXO/YF2yMiTbGPGuMOWaMOWqMefwK337lpjShK7s8BXQGmgFlgVNAqGPfo0AxoAJQAhgI/Csio4FNwBARuVlEhmQ/qTGmHPAVMAEoDowAPjHGlAK+AGoYY6pnOuRh4EPH8lmgD3ArcD8wyBjT+Srr9zVQHSgN/AF8ACAiCx3L0xx16HiRY0cD9wB3AHWB+sBLmfbfjvX+lANCgFBjzG1XGadyI5rQlV0GAqNFJFpEEoGxQFfHl47JWIm8moikisgWETmTy/M+AqwRkTUikiYi3wKbgftE5BywCugJ4EjsNYHVACISJiL/cxy3A1iG9QfnionIOyISl6ludY0xxXJ5eC9gvIgcE5HjwDigd6b9yY79ySKyBogHalxNnMq9aEJXdvEFPnN0i5wG9gCpgA/wPrAOWO7ocphmjCl0Beftln5ex7nvBco49n+II6Fjtc4/dyR6jDENjDHrjTHHjTGxWH90Sl5pxYwxXsaYKY7upDNApGNXbs9VFojKtB7l2JYuRkRSMq2fA25GeTxN6Mouh4D2InJrpkcRETnsaHmOE5EAoBHQAasrBCCn4UEPAe9nO+9NIjLFsf9boJQx5g6sxP5hpmM/xGqtVxCRYsACwFzidc4CN2Zavz3T8sNAJ6AVVtdIJcf29HPlVIcjWH+Y0lV0bFPqsjShK7ssACYaY3wBjDGljDGdHMvNjTG1jTFewBmsLoY0x3H/AFUuc96lQEdjTFtHS7mI44vE8gAikgx8BEzH6mP/NtOxRYGTIpJgjKmPlZgvZRvQwxhTyBgTBHTNdp5EIAYr6U/KdmxOdVgGvOR4T0oCYxz1UuqyNKEru8zGag1/Y4yJA34BGjj23Q58jJXM9wAbsLph0o/raow5ZYyZk/2kInIIq3X8InAcq8U+kqw/6x9itZ4/ytZ1MRgY74hnDLDyMvG/DFTF+jJ3HFlb+u9hdZMcBnY76pbZ20CAo0vo84ucewJWv/8O4H9YX6pOuEg5pbIwOsGFUkq5B22hK6WUm9CErpRSbkITulJKuQlN6Eop5SZsG5yrZMmSUqlSpas69uzZs9x0003ODaiA0zp7Bq2zZ7iWOm/ZsuWEiJS62D7bEnqlSpXYvHlzzgUvIiwsjODgYOcGVMBpnT2D1tkzXEudjTFRl9qnXS5KKeUmNKErpZSb0ISulFJuokDNWJScnEx0dDQJCQmXLVesWDH27NmTT1EVDFrngq1IkSKUL1+eQoVyOyikUs5XoBJ6dHQ0RYsWpVKlShhzqUHuIC4ujqJFi+ZjZPbTOhdcIkJMTAzR0dFUrlzZ7nCUB8uxy8UY845jqqudl9hvjDFzjDHhxpgdxph6VxtMQkICJUqUuGwyV6qgMcZQokSJHP+zVCqv5aYPfTHQ7jL722NNtVUd6A+8cS0BaTJXrkh/blVBkGOXi4hsNMZUukyRTsB7Yg3b+Isx5lZjTBkROeqsIJVS6nJSU+HgwUvvP3QI1q0DL6/8i+likpLOcu7ccQICipIXl947ow+9HNaY0+miHdsuSOiOGc/7A/j4+BAWFpZlf7FixYiLi8vxBVNTU3NV7mp9+eWXPPzww2zevBk/Pz82bdrEnDlz+OijjzLKDBw4kHbt2tG5c2eSk5OZMGECq1atomjRohQuXJjnn3+eNm3a5PhaiYmJDBgwgK1bt1K8eHEWL16Mr69vljL79+/n0UcfzWgFRkZG8uKLL/Lkk0+yY8cOnnnmGRITE/H29mbmzJkEBQVx6tQpnnzySQ4cOMD111/P/PnzCQgIAGDw4MGsXbuWUqVK8euvv14Q09y5cxk9ejQHDhygRIkS/PnnnwwaNIjt27czZswYnn766SzlU1NTadasGWXKlMl4jzZs2MBLL71EUlISd9xxB6GhoXh7ezN79mxWrrSGGU9JSWHfvn1EREQQExPDY489lnHOyMhIXnjhBYYMGcLJkyd5/PHHiYqKwtfXl8WLF3Pbbbddso7R0dEMGDCAY8eOYYzhscceY/DgwQC8+uqrrFmzhuuuu46SJUuyYMECypQpQ2xsLP369SM6OpqUlBSefvppHnnkEQBefvll1q1bR1paGs2bN2fatGkXbZEnJCRc8DN9peLj46/5HK7mUnVOSrqOiRP9OXMm5zS1bVvu58g2xp4hw0V+wEp/xRg06J28+ZxFJMcH1hRaOy+x70vg3kzr3wNBOZ3zrrvukux27959wbaLOXPmTK7KXa3u3bvLvffeK2PGjBERkfXr18v999+fpcyjjz4qH330kYiIPP/889KnTx9JSEgQEZG///5bVqxYkavXCg0NlQEDBoiIyLJly6R79+4XLZde55SUFPHx8ZHIyEgREWndurWsWbNGRES++uoradasmYiIjBgxQsaOHSsiInv27JEWLVpknGvDhg2yZcsWCQwMvOB1Dh48KG3atJGKFSvK8ePHRUTkn3/+kd9++01efPFFmT59+gXHzJw5U3r27JnxHqWmpkr58uVl3759IiLy8ssvy6JFiy44bvXq1dK8efMLtqfXcefOnSIiMnLkSJk8ebKIiEyePFmee+65y9bxyJEjsmXLloz3rXr16rJr1y4REYmNjc14ndmzZ2e89xMnTsw477Fjx+S2226TxMRE+emnn6RRo0aSkpIiKSkpcs8998j69esviFkk9z+/l3Opc7uz9evXS0qKSFiYSM+eIiEhIpD10bTp5R9Nmoi0aCGyePGlHxs32lO/U6dOSd++fQWQatWqSVhY2DV9zsBmuURedUYL/TBQIdN6ecc2lxQfH8+PP/7I+vXr6dixI+PGjbts+XPnzvHWW29ltBLB+u+je/fuuXq9VatWMXbsWAC6du3KkCFDEJFL9sl+//33VK1aNaMVb4zhzJkzAMTGxlK2rDWX8O7duxk1ahQANWvWJDIykn/++QcfHx+aNm1KZGTkRc8/bNgwpk2bRqdOnTK2lS5dmtKlS/PVV19dUD46OpqvvvqK0aNH89prrwEQExND4cKF8fPzA6B169ZMnjyZkJCQLMcuW7aMnj17XnDO9DpWrFgx4z1Kb808+uijBAcHM3Xq1EvWsUyZMpQpY80JXbRoUfz9/Tl8+DABAQHccsstGa9z9uzZjPfZGENcXBwiQnx8PMWLF8fb2xtjDAkJCSQlJSEiJCcn4+Pjc9H3Tl2506dh5cryNG+edXvp0uDjAw89BEOGQLFitoR3zVJTU2nUqBH79u3jueeeY+zYsdxwww159l+YMxL6amCIMWY51hRiseKE/vNnnoFt2y6+LzX1hqvqC7vjDpg16/JlVq1aRbt27fDz86NEiRJs2bLlsuXDw8OpWLFilkSR2UMPPcS+ffsu2D58+HD69OnD4cOHqVDB+nvo7e1NsWLFiImJoWTJi08Qv3z58ixJcNasWbRt25YRI0aQlpbGzz//DEDdunX59NNPadKkCb/99htRUVFER0dfNhmtWrWKcuXKUbdu3cvWObNnnnmGadOmZekCK1myJCkpKWzevJmgoCA+/vhjDh06lOW4c+fOsXbtWubNm5djHdOTNMDtt9/OP//8k+s6RkZGsnXrVho0aJCxbfTo0bz33nsUK1aM9evXAzBkyBAeeOABypYtS1xcHCtWrOC6666jYcOGNG/enDJlyiAiDBkyBH9//1y/P8py8qSVnDNfpr99Oxw5AlANsPaFhUHDhuDq3zHHxMRQvHhxvLy8mDhxIhUqVCAoKCjPXzc3ly0uA/4L1DDGRBtjQowxA40xAx1F1gARQDjwFta8jC5r2bJl9OjRA4AePXqwbNmyS7aWc3Nlw4oVK9i2bdsFjz59+uR4bHZJSUmsXr2abt26ZWx74403eP311zl06BCvv/56Rit41KhRnD59mjvuuIO5c+dy55134nWZv4Lnzp1j0qRJjB8/PtfxfPnll5QuXZq77rory3ZjDMuXL2fYsGHUr1+fokWLXvDaX3zxBY0bN6Z48eI51jH7udPf95zqGB8fz4MPPsisWbOy/MGdOHEihw4dolevXhl/UNatW8cdd9zBkSNH2LZtG0OGDOHMmTOEh4ezZ88eoqOjOXz4MD/88AObNm3K9XvkqRISYOhQKzEXLgwlSsB338HXX8OJE9ajbFmoVAmefDKc06chKQkaNXLtZC4iLF26FD8/PxYtWgRAly5d8iWZZwRgx6Mg9qHHxMTIDTfcIBUrVhRfX18pX768VKhQQXbs2CGNGjXKUrZjx44SFhYmZ8+eleLFi2fpm82se/fuUrdu3QseS5YsERGRNm3ayM8//ywiIsnJyVKiRAlJS0u74DxnzpyRzz//XFq3bp1l+y233JJRPi0tTYoWLXrBsWlpaeLr65slxgMHDmTpQ9+xY4eUKlVKfH19xdfXV7y8vKRChQpy9OjRjDKvvPJKlj70UaNGSbly5cTX11d8fHzkhhtukF69el3w+uvWrZNu3bpl2da5c2f54IMPLiibuY7pn7Ofn58cOXJERKz+cT8/vxzrmJSUJG3atJGZM2deUDZdVFRUxntw3333ycZMnazNmzeXX3/9VaZNmybjx4/P2D5u3DiZOnXqRc/nyX3oMTEijRuL1K4tUqdO1v7vypVFnn1WZPJkkdTUC4911Tpnd/DgQbnvvvsEkHvuuSfje5uLyas+dB3LJZOPP/6Y3r17ExUVRWRkJIcOHaJy5cqcPHmSI0eOZNyGHhUVxfbt27njjju48cYbCQkJYejQoSQlJQFw/PjxjKs9cmqhP/DAAyxZsiTj9Vu0aHHJlv/F+pzLli3Lhg0bAPjhhx+oXr06AKdPn86IZ9GiRTRt2vSS3UIAtWvX5tixY0RGRhIZGUn58uX5448/uP322y95zOTJk4mOjiYyMpLly5fTokULli5dCsCxY8cA6yqeqVOnMnDgwIzjYmNj2bBhQ5Z++svVMfN7tGTJkozjLlVHESEkJAR/f3+GDx+e5Vz79+/PWF61ahU1a9YEoGLFinz//feA1cWzb98+qlSpQsWKFdmwYQMpKSkkJyezYcMGj+9y2bULvvzy/GPsWKsF/tNPsGcPVKkCnTtDnz7w778QEQEzZsCoUXCdm2acZcuWERgYSFhYGLNmzeLHH3/MuKosX10q0+f1oyC20IODg+Xrr7/Osm327NkycOBA+fHHH6VBgwZSt25dCQoKkm+++SajTGJioowcOVKqVq0qgYGBUr9+fVm7dm2uXvPff/+Vrl27StWqVeXuu++Wv/76S0REDh8+LO3bt88od/ToUSlevLicPn06y/GbNm2SevXqSZ06daR+/fqyefNmERH5+eefpXr16uLn5yddunSRkydPZhzTo0cPuf3228Xb21vKlSt30StQfH19M65yOXr0qJQrV06KFi0qxYoVk3Llyl3wH0n2K4FGjBghNWvWFD8/P3n99dezlH333XfloYceuuA14+Pjs9Qx/XM+ceKEtGjRQqpVqyYtW7aUmJiYy9Zx06ZNAkjt2rUz/iP66quvRETkP//5jwQGBkrt2rWlQ4cOEh0dnfF+t27dWmrVqiWBgYHy/vvvi4h1xU3//v2lZs2a4u/vL8OGDbsg7nTu1EJfvFjklltEbr1VpGTJrI/sV6CkP559ViQu7spfq6DU+Vp8/fXX0qpVK4mIiMhV+bxqoRtrf/4LCgqS7BNc7NmzJ1etH1cZ48OZtM4FX25/fi/H7skeEhOhSxerrxvg5putlnZmKSlw772Quaq33grVql3da9pd56uRkpLC66+/TlJSEqNHjwa47NVp2V3jBBdbROSinfIFanAupZQ99uyxrjjp3RuOOq5R++kn60tKldX27dsJCQlhy5YtdO/ePSORF4ThHzShK+WhTp6EN96AJUsg01cLgHV9uKte+51XEhMTmTBhAlOmTKF48eJ89NFHPPjggwUikacrcAn9Sv5tUaqgsKvr8mocOAATJ8Lbb2fd/u67ULWq1ZWiyfxC+/fvZ+rUqTz88MO89tprlChRwu6QLlCgEnqRIkWIiYnRIXSVSxGxxkMvUqSI3aFcVkSE1bXSocP5bd27w9KlWW/4UefFx8ezatUqevXqRa1atdi7dy9VqlSxO6xLKlAJvXz58kRHR3P8+PHLlktISCjwvzzOpnUu2NJnLCootm6FZ5+FyEgrWSclWcvpmjSBr74CF/rOOd99++239O/fn6ioKOrVq4e/v3+BTuZQwBJ6oUKFcjXjS1hYGHfeeWc+RFRwaJ3VpSQmQr9+1t2X6dKvUgFw3PjMnXdC8+bQoAHUr5+/MbqSU6dOMWLECN555x38/Pxc6t6DApXQlVKXduYM/PWX1dJet866pf7LL60+8XR33209BwXBAw/A6NHuezNPXkhNTaVx48b8+eefvPDCC4wZM8Zl/ksETehKFXiJiTB9Orz8ctbtN998fnnYMJg0CVwo9xQoJ06cyBhMa9KkSVSsWJF69a56Nk3b6N9upQqotDRYtcpK0unJvEcP+Pxz2LwZ4uKshwi89pom86shIrz33ntZBtPq3LmzSyZz0Ba6UgVSdDT07Ak//mit33svLF5sXVaonCMqKooBAwawbt06GjVqRNOmTe0O6ZppC12pAuLsWVi71ofOnaFChfPJfO1a2LhRk7kzLV26lFq1avHjjz8yd+5cNm3alDFQmyvTFrpSBcTq1TB16vmrKQYOhPnzXXt88IKqVKlSNG7cmDfffPOCOXxdmSZ0pWx27Bh88IE1DC3AL79Ys2s5ZjRUTpCcnMzMmTNJTk7m5Zdfpm3btrRp08btbmDUhK5UPvrgA5g9+/wVKrt3g2NGPQB69jzI3XdX1EsNnWjr1q2EhISwdetWevToUaAG03I2TehKOdnhw3DwIKSmWjfypKSQMQduaqr1XKsW3HYbVK9uTQ7Rr591DXlycgTXXVfRvuDdSEJCAuPHj2fatGmULFmSTz75hP/85z92h5WnNKEr5UQrV1qTIWdWurSVsMFK6J07W3drXkweTQbvkcLDw5kxYwZ9+vRh5syZ3HbbbXaHlOc0oSvlJG+/DX37Wsuvvmq1uL29rUsOtT88f8THx/PZZ5/Ru3dvatWqxb59+3I1nIi70ISu1FWKi7NuxV+61OoLTx8/ZfZsePppe2PzROvWraN///4cOnSIoKAg/P39PSqZgyZ0pXIUGWldUpjZnj2wYMGFZd9+G554Il/CUg4xMTEMHz6c9957j5o1a7Jp0yaXGUzL2TShK3URZ89aN/N8+ik47gi/qAcfhMcegxYt4MYb8y085ZA+mFZ4eDijR4/mpZdecqnBtJxNE7ryeFFRkJCQdduCBTBr1vn13r2zrgPccIP1UPnv+PHjlChRAi8vL6ZOnYqvry933HGH3WHZThO68mhr10L79hffV7QofP+9NSVb5pENlX1EhMWLFzN8+HCmTJnCgAED6NSpk91hFRia0JVHOnXKGuxq+HBrffZsKFUqa5lq1c6PL67sFxkZSf/+/fn2229p0qQJzZs3tzukAkcTuvIocXHW9GzNmp3f1qkTPPWUjplSkL3//vsMGjQIYwzz589nwIABXKe3015AE7ryGGlpULkyxMRY6y1bwltvWdtUwebj40PTpk1ZsGABFSvqnbSXogldua34eFix4vzNPl5e1p2aDz5oTRTxn//o9GwFVXJyMtOmTSM1NZUxY8bQpk0b2rRpY3dYBZ4mdOVW4uOtySHuvx8iIs5vr1MHOnSwknq/ftZ446pg+uOPP3jiiSfYvn07Dz/8cMZgWipnmtCV2zh8GKpUgaSk89sGDIBnngE3mLvA7f3777+MGzeOGTNmUKpUKT777DM6d+5sd1guJVcJ3RjTDpgNeAGLRGRKtv0VgSXArY4yo0RkjXNDVepCCQmwd691A1B4uJXM+/aFtm2ha1e7o1NXIiIigtdee43HHnuM6dOne8RgWs6WY0I3xngBoUBrIBr43RizWkR2Zyr2ErBSRN4wxgQAa4BKeRCvUgD8/jsMGQK//ZZ1e5kyMHSoNTytKvjOnDnD2rVrCQ4OJjAwkP3797vVDEL5LTdfCdUHwkUkQkSSgOVA9iv5BbjFsVwMOOK8EJXKKjbWGjPlt9+slnjPnvD559a15UeOaDJ3FWvWrKFWrVpMnz6dPXv2AGgyv0ZGRC5fwJiuQDsR6etY7w00EJEhmcqUAb4BbgNuAlqJyJaLnKs/0B/Ax8fnruXLl19V0PHx8dzsYbfueXKdU1IMH31Ung8+8KVw4TROnSoMQKFCaaxe/SNFiqTZHKnzeMLnHBsbS2hoKN9++y2+vr4MGTKEoKAgu8PKV9fyOTdv3nyLiFz8DRORyz6Arlj95unrvYF52coMB551LDcEdgPXXe68d911l1yt9evXX/WxrsqT6pyaKpKSIrJ69SaZOlUEzj8GDrQezz8vEhlpd6TO5+6fc0pKivj5+Ym3t7eMGTNGEhIS3L7OF3MtdQY2yyXyam6+FD0MZL7Iq7xjW2YhQDvHH4j/GmOKACWBY7k4v1KAdc34K6/Avn3pW+7N2BcYaHWzXGqmH1Ww/fPPP5QqVQovLy9mzJiBr68vderUsTsst5ObPvTfgerGmMrGmMJADyDb6NAcBFoCGGP8gSLAcWcGqtzX9OnWDT49eljJvEgRGDkSHn/8AK+9Zt3ZuXOnJnNXJCK8/fbb1KhRg4ULFwLQsWNHTeZ5JMcWuoikGGOGAOuwLkl8R0R2GWPGYzX9VwPPAm8ZY4ZhfUH6mONfA6Uua9iw88PSNmgAU6ZAcLC1HhYWRXCw3pfvqiIiIujXrx8//PADzZo1o1WrVnaH5PZydR26WNeUr8m2bUym5d1AY+eGptyNCDRuDP/9LxQqZK2npFj79uzRm3/cyZIlSxg8eDBeXl4sWLCAfv366WBa+UDvFFX55sUXrWQOMGKE9ZySYl12qMncvZQtW5YWLVrwxhtvUL58ebvD8Ria0FWe+/dfmDPH6k4Ba2LlKlXsjUk5V1JSElOmTCEtLY2xY8fSunVrWrdubXdYHkcTuspTO3ZAq1Zw3PEV+TvvaDJ3N7///jtPPPEEO3fupHfv3jqYlo20U0s53ZEjEBAAd94Jdetayfy226wxVx5/3O7olLOcO3eOESNGcM8993Dq1ClWr17Ne++9p8ncRtpCV04VHZ11aNqOHaFhQ3jhBftiUnnjwIEDzJ07l379+jF16lSKFStmd0geTxO6cpqEhPPJ/P774csv7Y1HOV9sbCyffvopjz/+OIGBgYSHh1NBB5cvMLTLRTnNm29azzfeCKuz33qmXN5XX31FYGAgffv2Ze/evQCazAsYbaGrq7JnD/zzT9ZtkyZZz7//rlO7uZPjx4/zzDPP8OGHH1KrVi0+/fRTaup1pgWSJnR1RQ4fhmbNrEsPL6ZzZ+sLUeUeUlNTuffeezlw4ADjxo1j1KhRFC5c2O6w1CVoQle5du4cLFpkJfOmTa0rVipVylqmXj1bQlNO9vfff1O6dGm8vLyYOXMmlSpVopYONF/gaUJXuZKYCDfddH593jyoXdu+eFTeSEtL46233mLkyJFMnTqVQYMG0aFDB7vDUrmkPZ3qspKSrOvIK1a01kuUsK4z12TufsLDw2nZsiUDBw7k7rvvpm3btnaHpK6QttBVFgkJ8N13ViIHePDBrPsPHrSuYlHu5d1332Xw4MEULlyYt956i5CQEL1ByAVpQlecOgX+/hAXZ/WTZ+fjA2PHwiOPaDJ3VxUrVqRt27aEhoZSrlw5u8NRV0kTuodKS7MGzTp+3LrcMP0SxKFDwRh47DHr2Rgr2XvrT4pbSUxMZPLkyaSlpTF+/HhatmxJy5Yt7Q5LXSP9NfVAf/0F1apl3Xb99RAZCbffbktIKh/9+uuvhISEsGvXLh599FEdTMuN6JeiHmbvXhgyxFquUgWmTYPFi61p3jSZu7ezZ88yfPhwGjZsSGxsLF9++SWLFy/WZO5GtIXuIRIT4ZNP4N13rS89K1WCXbus+TuVZ4iKimL+/PkMHDiQKVOmcMstt9gdknIyTege4ptvoFcvazkgwErmyv2dPn2ajz/+mL59+xIQEEB4eLjOIOTGtMvFA8TFwdGj1vI331hjrSj3t2rVKgICAhg4cGDGYFqazN2bttDdzLvvwocfnl//91/46afz6xUr6qWH7u7YsWM8/fTTrFixgjp16rB69WodTMtDaEJ3E5GR8MYbsHSp1SJPv5NTxBpfpVEjCA4GPz87o1R5LTU1lcaNG3Pw4EEmTJjAc889R6FChewOS+UTTehu4Nw5qFHj/N2do0bB5Mn2xqTy15EjR7j99tvx8vJi9uzZVKpUiQAd9tLjaB+6G0hIsJL5wIFw+rQmc0+SlpbGG2+8Qc2aNVmwYAEA9913nyZzD6UJ3Y34+4NO6+g5/vzzT5o3b87gwYNp0KAB7du3tzskZTNN6Eq5oLfffpu6deuyY8cO3nnnHb755hsqV65sd1jKZtqH7uIOHYJffrE7CpXfKlWqRPv27QkNDaVMmTJ2h6MKCE3oLigtDWbNsm7XT5/HE0Bv/HNfiYmJvPrqqwBMmDBBB9NSF6UJ3QX99Rc8++z5iZjbt4dXX4U77rA1LJVHfv75Z0JCQti7dy9PPPGEDqalLkkTugvZtw/mzLFmDALrmvOePe2NSeWd+Ph4Ro8ezdy5c6lQoQJr167VWYTUZeUqoRtj2gGzAS9gkYhMuUiZ7sBYQIDtIvKwE+P0aMnJMGhQPRx3bwPWdec6Z697O3jwIG+++SZPPvkkkyZNomjRonaHpAq4HBO6McYLCAVaA9HA78aY1SKyO1OZ6sALQGMROWWMKZ1XAXuiDz6AvXutDvK334aHHso6YbNyH3FxcSxcuJD+/fsTEBBAREQEZcuWtTss5SJy00KvD4SLSASAMWY50AnYnalMPyBURE4BiMgxZwfqicLD4dNP4fnnrfUtW6zb+JV7+uyzz+jbty+xsbE0a9aMGjVqaDJXVyQ3Cb0ccCjTejTQIFsZPwBjzE9Y3TJjRWRt9hMZY/oD/QF8fHwICwu7ipCtvsWrPdYVzJpVnY0bS3HqVOGMbUOHbuPMmdO4cbUv4O6fc7qTJ08yZ84cNmzYQJUqVZg8eTJHjx7laPoQmW7OUz7nzPKqzs76UtQbqA4EA+WBjcaY2iJyOnMhEVkILAQICgqS4ODgq3qxsLAwrvbYgioiArZvh3nzrNERy5SBe+6BLl2sVnlc3Gm3q3NO3PFzzi41NZWaNWty6NAhJk2axN13302rVq3sDitfecLnnF1e1Tk3Cf0wUCHTennHtsyigV9FJBk4YIz5EyvB68jbufTII/Df/55fnz4dunY9v+5hDRi3Fx0dTdmyZfHy8mLOnDlUrlyZmjVrelxLVTlXbm79/x2oboypbIwpDPQAVmcr8zlW6xxjTEmsLpgI54Xp3uLirGQeHAw7d1o3DmVO5sp9pKWlMXfuXGrWrMkbb7wBQPv27XW8cuUUOSZ0EUkBhgDrgD3AShHZZYwZb4x5wFFsHRBjjNkNrAdGikhMXgXtbj75xHouVw4CA0HvGXFPe/fupWnTpjz99NPce++9dOjQwe6QlJvJVR+6iKwB1mTbNibTsgDDHQ+Vg5MnrQfAn3/C449by9Om2ReTyluLFi1iyJAh3HjjjSxZsoTevXvr3Z7K6fRO0XyUlgZ9+ljXlWd3993WF6HKPVWtWpWOHTsyb948fHx87A5HuSlN6Plg1y7rZqBdu85vGzkS6tSxlmvUsBK6ch8JCQmMHz8egEmTJtG8eXOaN29uc1TK3WlCzyPx8bB5M3TuDLGx57c//7w1Rdytt9oVmcprP/30EyEhIezbt4++ffvqYFoq32hCzwOJiVC+/PlE7u8PL79sXVNepIi9sam8ExcXx4svvkhoaCi+vr6sW7eONm3a2B2W8iCa0PNAQoKVzLt1sx5du+qVK54gOjqaRYsW8dRTTzFx4kRuvvlmu0NSHkYTupP98gt895213LChldCV+4qJiWHlypUMGjQIf39/IiIidAYhZRtN6E721FNW3zmAr6+9sai8IyJ88sknPPnkk5w8eZIWLVpQo0YNTebKVjpJtBMNGmQl83bt4OxZ+M9/7I5I5YWjR4/y4IMP0q1bNypUqMDmzZupUaOG3WEppS10Zzl6FBYssJaHDYMbb7Q3HpU3UlNTadKkCYcPH2batGkMGzYMb2/9NVIFg/4kOsmsWdbzvHmgFza4n0OHDlGuXDm8vLwIDQ2lcuXK+Pn52R2WUllol4uTJCVZz08+aW8cyrlSU1OZM2dOlsG02rZtq8lcFUia0K/RqlVQty6sXAm33GJ3NMqZ9uzZQ5MmTRg6dCjNmjWjY8eOdoek1GVpl8tVOH4c9u61RkmcPdvaVqOGNaa5cg8LFy7kqaeeomjRorz//vv06tVL7/ZUBZ4m9Cu0ZQsEBWXdFhYGzZrZEo7KI9WrV6dLly7MmTOH0qV1znPlGjShX4Hvv4f02cF69bKGvS1XDnRuAtf377//MnbsWIwxTJkyRQfTUi5J+9BzkJoK27bB8OHnk7m3N7z7LrRsqcncHWzcuJG6desybdo0YmNjsYb3V8r1aEK/jPBwqFUL7rwTXn/d2vbhh9bgW4UK2RubunZnzpxh8ODBNGvWjNTUVL7//nveeOMN7StXLku7XC5j4kTry0+Azz+Hu+6yRlFU7uHIkSMsXryY4cOHM378eG666Sa7Q1LqmmhCvwgR+PFHWLzYSuAREdoidxcnTpxg5cqVDB48mJo1a3LgwAGdQUi5De1yuYhVq6BpU2u5UydN5u5ARFixYgUBAQE888wz/PnnnwCazJVb0YR+EWPHWs9Llpy/zly5riNHjtC5c2d69OiBr68vW7Zs0Ts9lVvSLpdsVq6E7dut5e7dwcvL3njUtUlNTaVp06YcPnyYGTNmMHToUB1MS7kt/cnOZO9eazJngB07dLo4VxYVFUX58uXx8vJi/vz5VKlShWrVqtkdllJ5SrtcgORk+Ppra+5PgCZNoHZte2NSVyc1NZXXXnsNf3//jMG02rRpo8lceQRN6MBbb8F991nL7drBxo32xqOuzs6dO2nUqBHPPvssLVu2pHPnznaHpFS+0oQOxMVZzz/+aLXUletZsGAB9erVIyIigg8//JDVq1dTXm8aUB5GE3om9erZHYG6Uum36fv7+9OtWzd2795Nz5499W5P5ZH0S1Hlks6dO8eYMWPw8vJi6tSpNGvWjGY65KXycNpCVy4nLCyMOnXqMHPmTOLj43UwLaUcPD6hp6TAqFHWsv6XXrDFxsYyYMCAjGFtf/jhB0JDQ7V7RSkHj0/o//5rPfv56XXnBd3Ro0dZunQpI0aMYMeOHTpeuVLZ5CqhG2PaGWP2GWPCjTGjLlPuQWOMGGOCLlWmIPngg/PzgA4ebG8s6uKOHz/O3LlzAahZsyaRkZFMnz6dG2+80ebIlCp4cvxS1BjjBYQCrYFo4HdjzGoR2Z2tXFFgKPBrXgTqTH/9BXXqwLlz1nqVKtCzp70xqaxEhO+++46uXbty5swZ2rZti5+fH6VKlbI7NKUKrNy00OsD4SISISJJwHKg00XKvQpMBRKcGF+eePNNK5n7+MA331gJXqeNLDgOHTpEx44dmThxItWqVWPr1q06mJZSuZCbyxbLAYcyrUcDDTIXMMbUAyqIyFfGmJGXOpExpj/QH6xhS8PCwq44YID4+PirPhYgKqoqRYqUZfnyTYA1yXNBd611dhWpqan06dOHkydP0rdvX3r06MHx48c9ou7gOZ9zZlpn57nm69CNMdcBrwGP5VRWRBYCCwGCgoIkODj4ql4zLCyMqz0WYPlyaxKLazlHfrvWOhd0kZGRVKhQAS8vL5YsWUKVKlU4ePCgW9f5Ytz9c74YrbPz5KbL5TBQIdN6ece2dEWBWkCYMSYSuAdYXVC/GD11yupySUy0OxIFkJKSwowZM/D392f+/PkAtGrViipVqtgcmVKuJzct9N+B6saYyliJvAfwcPpOEYkFSqavG2PCgBEistm5oTrH6dPW8+OP2xqGAnbs2EFISAibN2+mU6dOPPjgg3aHpJRLy7GFLiIpwBBgHbAHWCkiu4wx440xD+R1gHlF7xK31/z587nrrruIiopixYoVfPbZZ5QtW9busJRyabnqQxeRNcCabNvGXKJs8LWHpdyViGCMoVatWvTo0YPXX3+dkiVL5nygUipHHjU4V1yczhFql7Nnz/LSSy/h7e3N9OnTadq0KU3TZ+JWSjmFR936P2OGldCLFbNu9Vf54/vvv6d27drMmjWLxMREHUxLqTziMQk9MRHGj7eWt2+Hhg3tjccTnD59mr59+9KqVSu8vb3ZuHEjc+bM0cG0lMojHpHQFy48P/DWI4+Ar6+98XiKf/75h+XLl/P888+zfft2mjRpYndISrk1j+hD/+knuOkmeOghcIzzpPJIehIfOnQoNWrUIDIyUr/0VCqfeEQLHaBUKXj7bdBB+vKGiLB06VICAgJ47rnn2L9/P4Amc6Xykdsn9J9/hv/9z+4o3NvBgwe5//776d27NzVq1GDbtm1Ur17d7rCU8jhu3+UyfDhs3QqtWtkdiXtKSUkhODiYY8eOMWfOHAYPHoyXl5fdYSnlkdw2oe/eDYGB1vL998Pq1fbG424iIiLw9fXF29ubt956i6pVq1KpUiW7w1LKo7ltl8vixdazvz+88AJc57Y1zV8pKSlMnTqVgIAAQkNDAWjZsqUmc6UKALdsoaekwPTp4O1ttdSVc2zbto2QkBD++OMPunTpQrdu3ewOSSmVidu1W//88/yVLD4+9sbiTubNm8fdd9/N4cOH+fjjj/n0008pU6aM3WEppTJxu4S+fz8kJ0OvXrBli93RuL702/Tr1KlDr1692L17tw5zq1QB5XZdLgcPWs9Dh2oL/VrEx8czevRoChUqxIwZM3QwLaVcgFu10I8cgcGDreVbbrE3Flf2zTffUKtWLebOnUtycrIOpqWUi3CrhD5woPXcvTvUqGFvLK7o1KlTPP7447Rt25YiRYqwceNGZs+erYNpKeUi3CKhHzkCXbvCF19Y60uX2huPqzp27Bgff/wxL7zwAtu2bePee++1OySl1BVwiz70tm1h505r+eOPoVAhe+NxJX///TfLli1j2LBhGYNplShRwu6wlFJXwS0S+vXXQ+nScPSo3kCUWyLCe++9x7Bhwzh37hwdOnSgevXqmsyVcmFukf6MgaAgTea5FRkZSbt27XjssccICAjQwbSUchNu0UJXuZeSkkLz5s05ceIEoaGhDBw4kOv0L6FSbkETuocIDw+ncuXKeHt7884771ClShV8deompdyKNs3cXHJyMpMmTSIwMDBjMK3mzZtrMlfKDbl8C33DBjh2zPpSVGX1xx9/EBISwrZt2+jWrRsPPfSQ3SEppfKQS7fQ//0XgoOt2/11nKis5syZQ/369fn777/59NNPWblyJT46FoJSbs2lE/q0adbzc8/Bm2/aG0tBkX6b/p133kmfPn3YvXs3Xbp0sTkqpVR+cOkul/h463ncOPD0Wc/i4uJ44YUXuP7665k5cyZNmjShSZMmdoellMpHLt1CB2vs8yJF7I7CXmvXrqVWrVrMnz8fEdHBtJTyUC6f0D1ZTEwMjz76KO3bt+emm27ip59+4rXXXtPBtJTyUC6d0DdtgqQku6OwT0xMDJ999hkvv/wyW7dupWHDhnaHpJSyUa4SujGmnTFmnzEm3Bgz6iL7hxtjdhtjdhhjvjfG5PlFzn//Db/+as0f6kmOHj3KjBkzEBH8/PyIiopi/PjxXH/99XaHppSyWY4J3RjjBYQC7YEAoKcxJiBbsa1AkIjUAT4Gpjk70OzmzrWeZ8zI61cqGESEd955B39/f15++WXCw8MBuO2222yOTClVUOSmhV4fCBeRCBFJApYDnTIXEJH1InLOsfoLUN65YV5o2zbr2ROmtzxw4AAjR44kJCSEunXrsn37dh1MSyl1gdxctlgOOJRpPRpocJnyIcDXF9thjOkP9Afw8fEhLCwsd1FmEx8fT0xMDDVqFCIy8g8iI6/qNC4hNTWVRx55hNjYWIYNG0aHDh04cuQIR44csTu0PBcfH3/VPyOuSuvsGfKqzk69Dt0Y8wgQBDS72H4RWQgsBAgKCpLg4OCrep2wsDBKlChBaipc7TkKuv3791OlShW8vLxYtmwZx44do3v37naHla/CwsLc9vO9FK2zZ8irOuemy+UwUCHTennHtiyMMa2A0cADIpLonPA8T3JyMhMmTKBWrVrMmzcPsP5oldbBapRSOchNC/13oLoxpjJWIu8BPJy5gDHmTuBNoJ2IHHN6lB5i8+bNhISEsGPHDnr06EHPnj3tDkkp5UJybKGLSAowBFgH7AFWisguY8x4Y8wDjmLTgZuBj4wx24wxq/MsYjc1e/ZsGjRowIkTJ1i1ahXLli3TVrlS6orkqg9dRNYAa7JtG5NpuZWT4/IYIoIxhqCgIEJCQpg2bRq33nqr3WEppVyQSw/O5crOnDnD888/T5EiRXj99ddp3LgxjRs3tjsspZQLc+lb/13VmjVrCAwMZOHChXh7e+tgWkopp9CEno9OnDjBI488wv3330+xYsX4+eefmT59ug6mpZRyCk3o+ejUqVN88cUXvPLKK/zxxx80aHC5+7OUUurKaB96Hjt8+DAffPABI0eOpHr16kRFRemXnkqpPOGSLfTo6BtYswZSU+2O5NJEhLfeeouAgADGjh3LX3/9BaDJXCmVZ1w2oQO0b29zIJfw119/0bJlS/r370+9evXYsWMH1apVszsspZSbc+kul06dci6T31JSUmjZsiUnT57kzTffpG/fvlx3nUv+3VRKuRiXTugFyb59+6hatSre3t4sWbKEqlWrUr58no8irJRSGbTpeI2SkpIYN24ctWvXJjQ0FIBmzZppMldK5TttoV+D3377jZCQEHbu3MnDDz9Mr1697A5JKeXBtIV+lWbNmkXDhg0zri3/4IMPKFmypN1hKaU8mCb0K5R+m379+vXp168fu3btokOHDjZHpZRS2uWSa7GxsTz33HPccMMNzJo1i0aNGtGoUSO7w1JKqQzaQs+FL774goCAABYtWsT111+vg2kppQokl0zo//1vCQAKF87b1zl+/DgPP/wwDzzwACVKlOCXX35h6tSpOpiWUqpAcsmE/uuvJahVC+rWzdvXiY2NZc2aNYwbN47Nmzdz99135+0LKqXUNXDZPvS77oK8aCgfOnSIpUuXMmrUKKpVq0ZUVBTFihVz/gsppZSTuWQLPS+kpaWxYMECAgMDmTBhQsZgWprMlVKuQhM6sH//flq0aMGgQYOoX78+//vf/3QwLaWUy3HZLhdnSUlJoXXr1pw+fZq3336bxx9/XL/0VEq5JJdL6BER8M8/Ra75PHv27KF69ep4e3vz/vvvU7VqVcqWLeuECJVSyh4u1+Xy6afW89Ve4ZKYmMgrr7xCnTp1mDdvHgBNmjTRZK6Ucnku10JPv6enf/8rP/aXX34hJCSE3bt307t3b3r37u3c4JRSykYu10K/WjNnzqRRo0bExcWxZs0a3nvvPUqUKGF3WEop5TRun9DT0tIAaNiwIQMHDmTnzp20L6hz1yml1DVwuS6X3Dp9+jTPPvssN954I3PnztXBtJRSbs8tW+iff/45AQEBLFmyhKJFi+pgWkopj+BWCf3YsWN0796dLl264OPjw2+//cakSZP0unKllEdwq4R+5swZvv32WyZOnMhvv/1GvXr17A5JKaXyjcv3oR88eJD333+fF198kWrVqnHw4EGKFi1qd1hKKZXvctVCN8a0M8bsM8aEG2NGXWT/9caYFY79vxpjKjk90mzS0tKYP38+gYGBTJo0KWMwLU3mSilPlWNCN8Z4AaFAeyAA6GmMCchWLAQ4JSLVgNeBqc4ONKt9tGsXzJNPPknDhg3ZtWuXDqallPJ4uWmh1wfCRSRCRJKA5UCnbGU6AUscyx8DLU0efROZmpoCtGX37v/x7rvvsm7dOipVqpQXL6WUUi4lN33o5YBDmdajgQaXKiMiKcaYWKAEcCJzIWNMf6A/gI+PD2FhYVcccEpKCWrXnsULLwhlytzGhg0brvgcrig+Pv6q3i9XpnX2DFpn58nXL0VFZCGwECAoKEiCg4Ov+BzBwXDvvTFczbGuLCwsTOvsAbTOniGv6pybLpfDQIVM6+Ud2y5axhjjDRQDYpwRoFJKqdzJTUL/HahujKlsjCkM9ABWZyuzGnjUsdwV+EH09kyllMpXOXa5OPrEhwDrAC/gHRHZZYwZD2wWkdXA28D7xphw4CRW0ldKKZWPctWHLiJrgDXZto3JtJwAdHNuaEoppa6EW936r5RSnkwTulJKuQlN6Eop5SY0oSullJswdl1daIw5DkRd5eElyXYXqgfQOnsGrbNnuJY6+4pIqYvtsC2hXwtjzGYRCbI7jvykdfYMWmfPkFd11i4XpZRyE5rQlVLKTbhqQl9odwA20Dp7Bq2zZ8iTOrtkH7pSSqkLuWoLXSmlVDaa0JVSyk0U6IReECenzmu5qPNwY8xuY8wOY8z3xhhfO+J0ppzqnKncg8YYMca4/CVuuamzMaa747PeZYz5ML9jdLZc/GxXNMasN8Zsdfx832dHnM5ijHnHGHPMGLPzEvuNMWaO4/3YYYypd80vKiIF8oE1VO9fQBWgMLAdCMhWZjCwwLHcA1hhd9z5UOfmwI2O5UGeUGdHuaLARuAXIMjuuPPhc64ObAVuc6yXtjvufKjzQmCQYzkAiLQ77musc1OgHrDzEvvvA74GDHAP8Ou1vmZBbqEXqMmp80mOdRaR9SJyzrH6C9YMUq4sN58zwKvAVCAhP4PLI7mpcz8gVEROAYjIsXyO0dlyU2cBbnEsFwOO5GN8TiciG7Hmh7iUTsB7YvkFuNUYU+ZaXrMgJ/SLTU5d7lJlRCQFSJ+c2lXlps6ZhWD9hXdlOdbZ8a9oBRH5Kj8Dy0O5+Zz9AD9jzE/GmF+MMe3yLbq8kZs6jwUeMcZEY82/8FT+hGabK/19z1G+ThKtnMcY8wgQBDSzO5a8ZIy5DngNeMzmUPKbN1a3SzDWf2EbjTG1ReS0nUHlsZ7AYhGZaYxpiDULWi0RSbM7MFdRkFvonjg5dW7qjDGmFTAaeEBEEvMptrySU52LArWAMGNMJFZf42oX/2I0N59zNLBaRJJF5ADwJ1aCd1W5qXMIsBJARP4LFMEaxMpd5er3/UoU5ITuiZNT51hnY8ydwJtYydzV+1UhhzqLSKyIlBSRSiJSCet7gwdEZLM94TpFbn62P8dqnWOMKYnVBRORjzE6W27qfBBoCWCM8cdK6MfzNcr8tRro47ja5R4gVkSOXtMZ7f4mOIdvie/Dapn8BYx2bBuP9QsN1gf+ERAO/AZUsTvmfKjzd8A/wDbHY7XdMed1nbOVDcPFr3LJ5edssLqadgP/A3rYHXM+1DkA+AnrCphtQBu7Y77G+i4DjgLJWP9xhQADgYGZPuNQx/vxP2f8XOut/0op5SYKcpeLUkqpK6AJXSml3IQmdKWUchOa0JVSyk1oQldKKTehCV0ppdyEJnSllHIT/wd/uF6fnQC15gAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":"#### Finalmente, se observa la matriz de confusión para el conjunto de prueba y calibración.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ncm_test = confusion_matrix(test_labels, np.where(np.array(predict_test.ravel()) > 0.5, 1, 0))\ncm_test","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:20:32.455375Z","iopub.execute_input":"2021-12-01T17:20:32.456295Z","iopub.status.idle":"2021-12-01T17:20:32.470474Z","shell.execute_reply.started":"2021-12-01T17:20:32.456253Z","shell.execute_reply":"2021-12-01T17:20:32.469491Z"},"trusted":true},"execution_count":131,"outputs":[{"execution_count":131,"output_type":"execute_result","data":{"text/plain":"array([[380, 142],\n       [162, 359]])"},"metadata":{}}]},{"cell_type":"code","source":"import seaborn as sns\n\ncm = [cm_test[0][0], cm_test[0][1], cm_test[1][0], cm_test[1][1]]\n\ngroup_names = ['True Neg','False Pos','False Neg','True Pos']\ngroup_counts = [\"{0:0.0f}\".format(value) for value in cm]\ngroup_percentages = [\"{0:.2%}\".format(value) for value in cm/np.sum(cm)]\nlabels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\n\nsns.heatmap([[cm[0], cm[1]], [cm[2], cm[3]]], annot=labels, fmt='', cmap='Blues')\nprint(\"Test dataset\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:20:34.121355Z","iopub.execute_input":"2021-12-01T17:20:34.121608Z","iopub.status.idle":"2021-12-01T17:20:34.354446Z","shell.execute_reply.started":"2021-12-01T17:20:34.121581Z","shell.execute_reply":"2021-12-01T17:20:34.353081Z"},"trusted":true},"execution_count":132,"outputs":[{"name":"stdout","text":"Test dataset\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAWAAAAD4CAYAAADSIzzWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAArTUlEQVR4nO3dd3gVxRrH8e97khBIQknoVZBelNBBelEQQbogHUF6E/RSBZRuAaVIk440qYIK0nsJTTqK0ntv6cncP84hBlMIkuRwDu/nPvuQzJnd2Y3wu5PZ2VkxxqCUUirxWex9Akop9bLSAFZKKTvRAFZKKTvRAFZKKTvRAFZKKTtxTegGkhXpqtMsVBR3/CbY+xTUCyipK/K8x3iWzAk4OOG523seCR7ASimVqMRxfrHXAFZKORexa6f2mWgAK6Wci/aAlVLKTrQHrJRSdmJxsfcZxJkGsFLKuegQhFJK2YkOQSillJ1oD1gppexEe8BKKWUn2gNWSik70VkQSillJ9oDVkopO7HoGLBSStmH9oCVUspOdBaEUkrZid6EU0opO9EhCKWUshMdglBKKTvRHrBSStmJ9oCVUspOtAeslFJ2orMglFLKTrQHrJRSduJAY8CO838VSikVF2KJ+xbbYUSSisheEfldRI6JyGe28lkickZEDtk2X1u5iMg4ETktIodFpOjTTlV7wEop5xJ/PeAgoIox5qGIuAHbReRX22efGGOW/Kv+20Bu21YKmGT7M0YawEop5xJPY8DGGAM8tH3rZttMLLvUAebY9tstIqlEJKMx5kpMO+gQhFLKqYjFEvdNpL2I7Iu0tX/iWCIuInIIuA6sM8bssX003DbMMFZE3G1lmYELkXa/aCuLkfaAlVJORZ5hCMIYMxWYGsvnYYCviKQClotIIaAfcBVIYtu3D/D5fzlX7QErpZyLPMMWR8aYu8AmoIYx5oqxCgJmAiVt1S4BWSPtlsVWFiMNYKWUUxGROG9POU5aW88XEUkGvAmcFJGMtjIB6gJHbbv8BLS0zYYoDdyLbfwXdAhCKeVknmUI4ikyArNFxAVrZ3WxMWa1iGwUkbRY+9CHgI62+r8ANYHTgD/Q5mkNaAArpZyKxRJvsyAOA0WiKa8SQ30DdHmWNjSAlVLOxXEehNMAVko5l3gcgkhwGsBKKaeiAayUUnaiAayUUnaiAezgfFJ68suUbgCkT52C8PBwbtyxPhJevvmXhISGPXcba6f1wNPDnXLNvgCgaIFsjPyoHtU//Pa5j60SRpHX8pM7d56I78eOn0jmzFmirVu6eBF27zv4XO192r8v+/btJblXcsRiof/AQRT2jXJTXv2LWDSAHdrte48o3WQUAAM61OSRfxDfzN0Q8bmLi4WwsPDnbiedtxdvlS3AbzuOP/exVMJzd0/K4mUrE7XNXr3/x5vVa7Bzx3aGfjaIJctXJWr7jkh7wE5o6mfNCQwOxTdvFnb9/jf3HwY+Ecz7fuxP/e6TOX/lNk1qlqDL+xVxc3PF78hZeoxcRHh41EWUxs7ZQJ+21aMEsMUiDOtehwrFc5PEzZUpi7cyfekORISxfRtRqUQeLl67S0hoGHNW7mL5+kOJ8SNQ/+L/6BE9unXm/v37hIaG0rV7DypXqfZEnRs3rvO/3h/x6OFDQsPCGDhoCEWLFWfnju1Mmjie4OBgsmbNyufDRuLh6RljW8WKl+DC+fMAzJk1kxXLlwJQv0FDmrdsjb+/P//r3ZNrV68SFh5O+46dqfF2zYS7+BeYBrCTypwuFZVaf014uGFAh+j/cufNkZ6GbxWlcpsxhIaG802/92hSswTzV++NUnfP4TO8W/l1KhTPzUP/oIjy1nXf4N7DAMo1/5Ikbq5snNWL9btOUrRAVl7JlJoiDYaTzseLg8s+Zc7KXQl2vepJQUGBvFe/DgCZsmThqzHfMnbcRLy8vLhz5zYt3m9MpcpVnwiAX35ezRtly/Fhh06EhYURGBjAnTu3mTZlElO+n4mHhwczvp/KnNkz6di5a4xtb9m8kVy583D82FFWrljGvAWLwRiavf8exUqU5NKFC6RNm44Jk6zryjx48CBhfxgvMsfJXw3gZ7Fs/cFoe7KRVS6Zl6IFsrF93v8ASObuxo3bD2OsP+r7tfRtV4OB4/751bZamXwUyp2ZetWs430pvZKSK1ta3vDNybJ1BzHGcO3WA7b6/REPV6Xi6t9DECEhIYz7ZgwH9vthEQvXr1/j1s2bpEmbNqJOoUKvMXhgf0JDQ6lcpRr58udnn98m/v7rNK2bvx9xnNd9faNtc8zXXzBtyiS8fXwYMnQ4e3fvokrVanh4eABQtdqbHNi/j7LlyvP1l6MZ+/WXVKxUmaLFiifcD+IFpz1gJ+Uf8E8vNTQsDEukwf6kSdwA63/8eav2MGj8T3E65ha/PxjSpRYlX8seUSYi9Br9I+t3nXiibo1yBZ/j7FV8+2X1Ku7cuc2Cxctwc3Pj7TerEBQc9ESdYsVLMGPOPLZt2cKgAX1p0aoNyVOkoHSZsoz+asxT23g8BvzY3t3R/8aTPXsOFv64jG3btjBh3DeULFU61h61M3OkANbV0P6jc5dv45vfuvKcb74sZM+cGoBNe09Rr5ovab29APBO4UG2jN6xHmvU92vo1eqfscN1O0/QvlE5XF2t/3lyZUuHR9Ik7Dr0N3Wr+iIipPNJTvniuRPi0lQcPXz4AB+f1Li5ubF3z24uX4668uDly5dInToNDRq9R70GjThx/BivF/bl0MEDnD93DgB/f3/Onj0TpzaLFivOpo3rCQgIwN/fn40b1lO0WHGuX79G0mTJqFW7Dq3atOXkiZf3xq7FYonzZm/aA/6PVmw4RLNaJdm/ZAB+R87y57nrAJz8+yqfTVzNqkldsYgQEhrGR6MWc/7KnRiPtXb78YhpbgAzl+/klUw+7JrfFxG4eech7/WayvINh6hUKi8Hlw7g4rW7HDp5gXsPAhP8WlX0ataqTfcunWhQtzYFChYix6uvRqmzb+9eZs2cjqurKx4eHgwbORofHx8+Hz6Svp/0IjgkGICu3XqSPXuOp7aZv0BB3q1Tn2ZNGgHWm3D58xdgx/ZtjP36CyxiwdXVlQGDhsTrtToUx+kAI9YFfBJOsiJdE7aBl4xnsiQ8CgjGJ6Un2+Z+TJU2Y7h2y/FuuNzxm2DvU1AvoKSuzx+fmTstj3PmXJpUz65xrT1gB7NsXCdSJk9GEjcXRk5b45Dhq1RCcqQxYA1gB6NPyikVOw1gpZSyE30U+SXlnsSV9dN7kiSJK64uLixff5Bhk3+hUsk8jOhZD4tFeOQfxIeD5/L3hZskcXNl+tAWFMmfjdv3HtG8zwzOX7lt78tQ8WzQwH5s3bIZH5/ULFu5+onPZs+awZgvR7N5+y68vX34efVPzJw+DWPA09OTAZ8OIW++fHY6c8fkSD1g+8/DcCJBwaHUaD+OUo1HUarJSN56owAlX8vOuP5NaDNgFqWbjGLRr/vo2846r7N13TLceRBAoTqfMf6HTQzvUcfOV6ASQp269Zk05fso5VevXGHXjh1kzJgpoixz5izMmDWPpStW0b5jJz4f8mlinqpTiK+XciYGDeB49ijAOq3IzdUFV1cXjDEYY0jhmRSAFMmTceXGPQBqVXqdH1btAaxP2VUqmdc+J60SVLHiJUiRMmWU8i9Hj+Sj3p88EQS+RYpG1H39dV+uXbuaaOfpLBwpgJ86BCEi+YA6QGZb0SXgJ2PMiZj3enlZLMLO+X3ImTUtUxZtxe/oOTp/Pp/l4zsTGBTM/UeBVGz5NQCZ0qXk4lXr/OCwsHDuPwwgdSpPbt19ZM9LUIlg08b1pEufLtbhheXLllCufIVEPCsnYf9cjbNYe8Ai0gdYiPWS9to2ARaISN9Y9msvIvtEZF/ozWPxeb4vvPBwQ+kmo8hVfSDFC71CgZwZ6dasMvW6fUeuGp8yd+VuRveub+/TVHYUEBDA91On0Llrjxjr7N2zm+XLltCz18eJeGbOwZl6wG2BgsaYkMiFIjIGOAaMim4nY8xUYCq8vA9i3HsYwJZ9f1C9bAFey5MZv6PWx06X/HaAlRM7A3D5+j2yZPDm0vW7uLhYSOGVTHu/L4GLF85z6dLFiJXVrl27SpOG9flh4Y+kSZuWP06d5LPBA5k4eRqpUsX+GLuKyuJAsyCeNgYcDmSKpjyj7TMVSRpvL1J6JQMgqbsbVUvl4+SZa6TwSkaubOkAqFI6H6fOXAPg5y1HaFa7FAD1qxVhi65u9lLInScvm7ft4td1G/l13UbSp8/AwiXLSJM2LVcuX6ZXj24MH/lFnB5NVlE5Uw+4J7BBRP4ELtjKsgG5gJdzqaVYZEiTgmmft8DFYsFiEZauO8Cv247SZeh8FnzVjnATzt37AXQYMg+AWSt2MmNYS46uHMyd+49o0Xemna9AJYQ+H/din99e7t69w5tVKtCpSzfqN2gUbd0pkydy995dRgz9DAAXVxcWLF6WmKfr8F6AXI2zp64FISIWoCRP3oTzM8bE6cVoL+sQhIqdrgWhohMfa0Hk7bM2zplzanT1F3stCGNMOLA7Ec5FKaWemyP1gPVJOKWUU3Gmm3AK6yPG2+Z+zJ5Ffdm/ZAADO/7zPrghXWpzeMUgDi4dSOf3K8Z4jOSeSTm9Zihj+/wz9rdyQueIY44b0CTiL86w7nXYu6gf3w9tEVG3Sc0SdG1aKf4vTv1ngwb2o1L5MtSvUyuibMK4b2hYrzbv1a9Dhw8/4Pr1a9Hue+XyZTp8+AF1a79Nvdo1uXTpIgALfphHrRpvUrhgXu7c+eex9PW/raXeu+/QukVT7t61zh2/cP48n/TumXAX6KAsFonzZm8awHEQ0yPGLd4tTZYMqShcbyhFGgzjxzX7YzzG4M7vsP3AX0+UNe8zg1KNR1Gs4XDSenvR4M2ipPBKim/+rJRsPJLgkDAK5spEUnc3Wr5bmsmLtyb0papnEN0jxq0/aMeS5atYvGwlFSpWYsqkidHuO7B/H1q3acuKVb/yw8If8fGxvlHFt2hRpkyfSaZMmZ+ov2D+POYvWkLD9xrzy8/W9SQmjPuGrt17xv+FOTiRuG/2pgEcR9E9Yty+UTlGTP2VxzcyI7/VIrIi+bOSLnWKKO94e/DI+jYLV1cLbrZjhocb3FxdAPBImoSQ0DB6tqzKpIVbCA3VmX8vkugeMfby8or4OjAgINqpTn+dPk1oaChl3igLgIenJ8mSWacv5s9fgMyZs0TZR0QICQ4mMCAQV1dXDuzfR5o0aXjllezxeEXOwZGmoWkAx5HFIuxe2JfzG0axcfdJ/I6eI0eWtDR8qxjbf/gfKyZ0Ime2tFH2ExFG9apPvzHLoz3uTxO7cH7DKB76B7Fs/UEe+gexdvsxdi/sy9Wb97j/MIAShbKzavPhhL5EFU/GfzuWt6pW5OfVq6J92u3cubMkT5GCj3p05b0GdRnz1WjCwmKfVNT2ww60b9eGLZs38XbNWkyZ/B3tO3ZOqEtwaBrATii6R4zdk7gSFBxCuWZfMHPZTqYMbhZlvw7vlWft9mNcun432uO+22UiOd7sj3sSVyqVsC7GM2b2eko3GUXfMcsZ1LkWQyetpnW9Mswb/QF92lVPyMtU8aBbj4/4bcMW3qlVm4Xz50X5PCw0lIP799H74z7MX7SEixcusnJF7HN9y7xRloU/LmP8d5PZvHED5ctX4Ny5s/Tu2Z3PBg0kICAgoS7H4egQhBN7/IjxW28U4NK1O6zY8DsAKzf+TqHcmaPUL/V6Djo2rsDJnz9j5Ef1aFqrJEO7v/tEnaDgUFZtPkztSq89UV44bxZE4I+z16lfrSjN+8zg1Sxpo+1pqxdPzXdqs37db1HK02fIQN58+cmSNSuurq5UrlqVk8fj9hbjgIAAVq5YRuP3m/HdhPEMHTGKIkWL8cvqVfF9+g7LkW7C6TS0OEjj7UVISBj3HgZEPGL89az1rNp8mIolcjNn5S3KF8vN6fPXo+zbZsDsiK+b1y5FsQLZ+HTcT3gmS0Jyz6RcvXkfFxcLb5cryI6DT96kG9S5Fl2HLcDN1QUXF+tflnATjkfSJAl7weo/O3fubMS47KZNG8iRI+qbkgsWeo0H9+9z+/ZtfHx82LtnDwULForT8WfPnE7T5i1xc3MjKCjQ+qu0RQgM1B7wYy/C0EJcaQDHQUyPGO88+BczR7SiW7MqPAoIotPn8wEoWiAb7RqWo7Pt++h4JnNnyTcdSOLmisUibN33J9OWbI/4vHal1zlw/HzE2sGHT13Cb3F/jv55iSN/XErYC1ZxEt0jxtu3buXs2TNYLELGjJkZONj6SPGxo0f4cfFChnw+HBcXF3p90of2bVthDBQoUJAGDa3TE3+YN4dZM77n1s2bNKr3LuUqVGTI58MBuH79GkePHKZjZ+sqAO83a07Txg1JkTw5Y8d/Z58fwgvIgfJXX0uv7EMfRVbRiY9HkUsM3xznzPEbUCnG9kQkKbAVcMfaWV1ijBksIjmwLtObGtgPtDDGBIuIOzAHKAbcAhobY87G1r6OASulnEo83oQLAqoYYwoDvkANESkNjAbGGmNyAXewLtuL7c87tvKxtnqx0gBWSjmV+JqGZqweT+53s20GqAIssZXPBuravq5j+x7b51XlKY1oACulnMqzzIKI/PYe29Y+8rFExEVEDgHXgXXAX8BdY0yorcpF/lkpMjO2ZXttn9/DOkwRI70Jp5RyKs9yEy7y23ti+DwM8BWRVMByIOaX+P0H2gNWSjmVhHgSzhhzF9gElAFSicjjzmsWrGukY/szq+0cXIGUWG/GxUgDWCnlVOLrJpyIpLX1fBGRZMCbwAmsQdzQVq0VsNL29U+277F9vtE8ZZqZDkEopZxKPD6IkRGYLSIuWDuri40xq0XkOLBQRIYBB4HptvrTgbkichq4DTR5WgMawEoppxJfAWyMOQwUiab8b6yvaft3eSAQ/cv+YqABrJRyKi/CGg9xpQGslHIqjvQosgawUsqp6GI8SillJw6UvxrASinnYnGgBNYAVko5Fb0Jp5RSduJA+asBrJRyLnoTTiml7MSB8lcDWCnlXOT5X6qRaDSAlVJORceAlVLKTnQWhFJK2YnOA1ZKKTtxoPzVAFZKORedhqaUUnbiQPmrAayUci4uDpTAGsBKKaeiQxBKKWUnDjQLTQNYKeVctAeslFJ24kD5qwGslHIu2gNWSik7cXGgQWANYKWUU3Gc+NUAVko5GV0LQiml7MSB8lcDWCnlXPQmnFJK2YkD5a8GsFLKuegsCKWUshMdgojkzOaxCd2EckDetcbY+xTUCyhgTa/nPoYlHs4jsWgPWCnlVLQHrJRSduJAQ8AawEop56I34ZRSyk4cKH81gJVSzsWBhoA1gJVSzsWR1oJwpBkbSin1VJZn2GIjIllFZJOIHBeRYyLSw1Y+REQuicgh21Yz0j79ROS0iJwSkepPO1ftASulnEo8doBDgd7GmAMikhzYLyLrbJ+NNcZ89WS7UgBoAhQEMgHrRSSPMSYspgY0gJVSTiW+ZkEYY64AV2xfPxCRE0DmWHapAyw0xgQBZ0TkNFAS2BXTDjoEoZRyKhaJ+yYi7UVkX6StfXTHFJHsQBFgj62oq4gcFpEZIuJtK8sMXIi020ViD2wNYKWUc7GIxHkzxkw1xhSPtE399/FExAtYCvQ0xtwHJgE5AV+sPeSv/+u56hCEUsqpxOckCBFxwxq+PxhjlgEYY65F+nwasNr27SUga6Tds9jKYqQ9YKWUU3mWIYjYiHVRienACWPMmEjlGSNVqwcctX39E9BERNxFJAeQG9gbWxvaA1ZKORWJv9dylgVaAEdE5JCtrD/wvoj4AgY4C3QAMMYcE5HFwHGsMyi6xDYDAjSAlVJOxjWefq83xmwn+pcs/xLLPsOB4XFtQwNYKeVUdDlKpZSyE12MRyml7MSBOsAawEop5+JIi/FoACulnIqLA02u1QBWSjkVS/xNQ0twGsBKKafiQCMQGsBKKeeisyCUUspO9CacUkrZiQPlrwawUsq56GvplVLKThxoFpoGsFLKuehaEEopZSeOE78awEopJ6OzIJRSyk4cJ341gJVSTsaisyCUUso+dBaEUkrZic6CUEopO3Gc+NUAVko5Ge0BK6WUnbhoACullH04TvxqAEdRufTrvJozd8T3w74cR8ZMmaOtW6NiCdZs8Xuu9kZ+NoB9e3exYPkakiRJwt27d+jQqjGLVv72XMdVCcMneVJ+GdUQgPTenoSHG27c8wegfI/5hISGP3cba79oRAYfTwKDw3gUEEyHsb/x58U7z33cl4UDdYA1gP/N3d2d6T8sTdQ2LRYLv/y0jLoNmyRqu+rZ3X4QSOku8wAY0LwMjwKC+Wbp/ojPXSxCWLh57nbajP6VA39e44O3X2NEuwo0GrLyuY/5stBXEjkRf39/BnzcjQcP7hMaGkq7jt0oV7HKE3Vu3bzBkP4f4//oIWFhYXzU51MKFymG3+4dzJj6HSEhwWTKnJW+g4bh4eERpY2GTVrw44K51KrbMMpnC+bOYNP6tYSEBFO+UlU+aN8VgNnTJ7Pu19Wk8vYmbfoM5M1XgCbN2yTMD0HFamrv6gQGh+KbMx27jl/mvn/wE8G8b3JL6g9ewflr92lSJT9d6hTBzdWC36mr9JiwgfBYAnv7kYt0rVsUgBHtKvBW8ewYA6MX7GbJ1j/I4OPJ3H7vkNwjCa4uFnqM38COY5cS5bpfVNoDdmBBQUG0bdYAgAyZMvPZyDEM++JbPL28uHv3Dp0/aErZCpWfuNO6fu3PlCz9Bi0+6EBYWBhBgYHcvXuHOTOmMmbiNJIl82D+7Oksnj+b1u06RWkzfYaMvOZbhN9+XcUb5StFlPvt3sHFC+eZMmshxhj69+7K7wf2kSSpO1s3rmP6D0sJCw2lXctG5M1XIMF/NipmmdMkp1KvhYSHGwY0LxNtnbxZfWhYIQ+Vey0kNCycb7pUoUnlfMzfcCLG475TOifHzt6kbtncvP5qWkp2nkuaFMnYPq4p249eonGlfKzbf5YvFu7FYhE83PWftGgP2HH9ewgiNDSEaZO+5feD+7CIhRs3rnP71i1Sp0kTUSdf/kKMHvYpoaGhlKtUldx58nFomx/nzvxF13YtAAgJDaFgocIxttu81Yf0/6QbZcpWiCjz27OTfXt20q65tWccEODPxQvn8Pf3p2zFyri7u4O7O2+UqxTPPwX1rJZt+yPWnixAZd9sFM2dnu3jmgKQzN2VG/cCoq07s8/bBASFcv7afXp9t5Hu9YuxePNJwsMN1+/6s+3IRYrlSc++P64ypVd13FxdWLXzNIf/vhHv1+ZodBaEE1m35mfu3rnNtDmLcXV1o3GdtwgODnqiTuGixRk3ZTa7dmxl1GcDaNS0JclTpKRYqTIMHvZlnNrJku0VcuXOx6b1ayPKjIFmrdrxbv33nqj744K5z39hKl75B4ZEfB0aFv7EilxJk1j/mYnAvPXHGTRz+1OP93gM+Gl2HL3Emx8vokbJV5nauzrjlu2PtUf9MnCg/HWox6bt4tHDB6TyTo2rqxsH9u3l6pXLUepcvXIZb5/U1K7bkHfqNODPUycoWOh1jv5+kIsXzgPW3uuFc2djbatFm/Ys+mFWxPclS7/BL6uW4+9vvct+4/o17ty+RaHCvuzctoWgoCD8/f3ZtX1LvF2ven7nrt3HN1c6AHxzpSN7+hQAbDp0nnrlcpM2ZTIAvL2Ski1d8jgdc8exSzSsmBeLRUiTMhnlCmVh36mrZEuXnGt3/Zm55giz1hylSK70CXNRDkQk7pu9aQ/4KarVqEW/Xl1p/X498uYvSLbsOaLUObTfj4XzZuLi6kqyZB4MGDKCVN4+9Bs0nM8HfkJISDAA7Tp2J+sr2WNsK0fOXOTOm58/T1l7MCVKl+Xc2b/p3LYZAMmSeTDw85HkL/AaZctX4oOm9fH2Sc2ruXLj6RW3f8gq4a3Y/ifNquZn/5SW+J28yp+XrFPITp6/zWezd7BqRAMsFiEkNJyPJm7k/PUHTz3myh2nKZU/E3u/a4ExMGD6Vq7d8adZtQJ81LA4IaHhPAoMpu2XaxL68l54jjQGLMY8/5SZ2Fy9F5KwDbyk/P398fDwIDAwgO7tW/Fx/yHkcaAbcTkaj7f3KagXUMCaXs+dnhtO3oxz5lTNl8auaa09YAf11YghnDvzF8HBwVR/512HCl+lEpK+EUMluEHDvrD3KSj1QnKkIQgN4Hg0auhAdm3fire3D7MWrogoX7roB1YsWYjFYqF02Qp06t4bvz07mTrxG0JCQnBzc6NTt94ULVHKfievEoS7mwvrv2pMEjcXXF2E5dv+ZNi8XUztXZ3yr2Xh3iPrjJr2X6/l8N83SOXlzpSPqpMjU0qCgsPoMGYtx8/dsvNVOBYHeiGGBnB8evudutRv1JQRQ/pHlB3Yt5cdWzcx/YelJEmShDu3rf+YUqbyZuTXE0iTNh1///Unn3TvwNKfN9rr1FUCCQoJo0afH3kUGIKri4WNXzfmt31nAej//VaWb//zifr/a1KK3/++TuOhP5EnizffdKlKzX5L7HDmjsuResA6DS0eFS5anOQpUj5RtnLpIpq2akuSJEkA8PZJDUCevPlJk9Y6VSnHq7kICgokODg4cU9YJYpHtjnCbq4WXF0txHbjO182H7YcugDAHxfv8Er6FKRLFfXxdRUzR5qGpgGcwC6eP8vhQ/vp2OZ9undozYnjR6LU2bJxHXnyFogIaeVcLBZh98TmnF/YkY0HzuN36ioAQ1qXZe+kFnzRviJJ3FwAOPL3DeqUzQVA8TwZyJY+BZnTeNnt3B2RPMMW63FEsorIJhE5LiLHRKSHrdxHRNaJyJ+2P71t5SIi40TktIgcFpGiTzvX/xzAIhLjyi8i0l5E9onIvrmzvv+vTTiFsLAw7t+7z6QZ8+nUvTdD+n38RA/ozF+nmTJhDL37DbLjWaqEFB5uKN1lHrmaT6N43gwUeCU1g2Zup3C7WZTrPh/v5Enp3agEAF8t9iOllzu7JzanUx1ffv/rerysrvYycRGJ8/YUoUBvY0wBoDTQRUQKAH2BDcaY3MAG2/cAbwO5bVt7YNLTGnieMeDPgJnRfWCMmQpMBZ0HnDZdeipUroaIkL/ga1gswr27d0jl7cP1a1cZ+L8e9B8ygsxZstn7VFUCu/coiC2/X+Ct4tkjVkoLDgljzrpj9GxQHIAH/sF0GPPPWtAnZ7flzNV7djlfhxVPQwvGmCvAFdvXD0TkBJAZqANUslWbDWwG+tjK5xhrD2u3iKQSkYy240Qr1h6wrRsd3XYE0Gce46BcxSoc3L8XgAvnzhISEkLKVN48eHCfvh91pkPXnrxW+Km/qSgHlSZlMlJ6ugPWNSGqFs3GqQu3yeDjGVHn3TK5OH72JgApPd1xc7X+s2xT4zW2H7nEA3+9N/As5Fn+F+m3ddvWPtpjimQHigB7gPSRQvUq/2RhZuBCpN0u2spi9LQecHqgOvDv5fgF2PmUfV86nw38hEP7/bh39y4Na1WlzYedqflufUYPHUjrJnVxdXOj/+ARiAjLFy/g0sULzP5+MrO/nwzAV+OnRtykU84hg48n03rXwMVFsIiwdOsf/Lr3DL+OakialB6IwOG/b9Bt3HrAehNuWu8aGAwnzt2i41h9M8qzepaba5F/W4/5eOIFLAV6GmPuR16K1hhjROQ//5Yf66PIIjIdmGmMibJ8k4jMN8Y0fVoDL/sQhIqePoqsohMfjyL7/X0vzplT4tWUsbYnIm7AamCtMWaMrewUUMkYc0VEMgKbjTF5RWSK7esF/64X0/FjHYIwxrSNLnxtnz01fJVSKtHF0zQIsXZ1pwMnHoevzU9AK9vXrYCVkcpb2mZDlAbuxRa+oA9iKKWcTDyuBVEWaAEcEZFDtrL+wChgsYi0Bc4Bjxfs/gWoCZwG/IGnviNMA1gp5VTiK35tv/3HdLiq0dQ3QJdnaUMDOA6iW+Nh5tSJrF65lFSpvAH4sHMPSkd6ndBjjeu8RTIPT1wsFlxcXJg6ZzEA0yePZ/vWjVjEQiof69rBadKmY8vGdcyYMoHkKVIy/MtxpEyViksXzzPtu28ZMuLrRLtm9XQxrfPw2NedKtPyrYKkrTchxmNkTZucA1NbMXzeLr5Zup8sabz4/pO3SZfKA4Nhxi9HmLjyIADDPijPWyWyc/ivG7T7yrrub5Mq+UmTIikTVhxM2It1JC/AE25xpQEcB9Gt8QDQ6P0WcXoT8TeTZkQE9WNNmrehbcduACxZNI/Z30+id7/BLFv8A1NmL2TrpvWsX/szDRo34/vJ42nXsXv8XZCKFzGt87D35BWK5k5PKi/3px5jdPuKEWtDAISGG/pO28Kh09fxSubGzvHN2XDwHJdvPsQ3VzpKdprLdz3fpGD2NPx1+S4t3yzIuwOXJeBVOh5dC8LJRLfGw/Py9Prn8dLAgICIuTMiFkKCgwkMDMTV1ZXfD+4ntU8asmR7JV7bV/EjunUeLBZhRLsKDJi+LdZ9a5fJydlr959Y7ezq7UccOn0dgIcBIZy8cItMqb0INyZifrCHuxshoWH0bFiMST8dJDQsPIGuzjHpWhAvieU/LqBN03qMGjqQB/djelpJ+Lhbez5s+R4/Lf/xiU+mffctDWtVZf2an2nboSsAzVq3o1fXD9m5bTNVq9dkzozJtGzbMWEvRP1n0a3z0Km2Lz/v/ourtx/FuJ9nUjd6v1eC4ZGGLP4tW/oU+OZMh9+pqzwMCGHt3jPsnticq7cfct8/mBJ5M7Jq118JcVkOzZECWF9JFEdXLl+iX68uEWPAt2/dJGUqb0SE6ZPHc+vWDfp+OizKfjeuXyNtuvTcuX2L3l0/pMfH/SlctPgTdebNmkZwcBAftO/6RPman1fy4P59ChR6nUU/zMIreQq69+5L0qTJEuw6E4uzzQNO6enOokHvMnTuTj5vU463PllMWLjhxvKu0Y4Bj2xXgX2nrrJ02x8MaF6GRwHBEY8ngzWgf/vyPb5YuIeVO05H2f+7nm8ydfXv+OZKR7Wir3DkzE1GL9iToNeYGOJjHvDhCw/jnDmvZ/WyawxrD/g/8kmdBhcXFywWC7XqNuTksaPR1kubzvqUordPaspXqhrtamhv1qjF1o3rnygLDAxgzeqV1GvUhJlTJ9Jv8HBeL1yUdWt+jv+LUc/t8ToPFQtn5dWMqTg28wNOzm6Lh7sbR2d8EKV+iXwZGN6uPCdnt6Vr3SJ80qQUHWv7AuDqYmHBp7VZtOlEtOFbOGdaRIQ/Ltymfvk8NB/xM69mTEnOTKkS+CodgyP1gPUm3H906+YNUqdJC8C2zRvIkTNXlDoBAf6YcIOHpycBAf747dlJq3adALh4/lzEuO72LRujvG154dyZNGjcDFdXN4KCghARxCIEBQYk8JWpuEqTMhkhoeHcexQUsc7D14v9yNF0SkSdG8u7UuiDGVH2rfbx4oivH/eAJ686BMDkj97i1PnbjFt2INp2B7UsS9dv1+Hm6oKL7fUP4QY83PWfMzjUJAgN4LiIbo2Hgwf8OP3HKUQgQ8bMfNxvMAA3b1zni+GD+eKbSdy5fYuBn/QArMtSVqtek1JlygEwZeJYLpw7i1iE9Bky0bvvP8tR3rxxnRPHj9D6w84A1H+vKR1aNcEreXKGfzkuka9exSSmdR5i8k7pVymaOwND58a8jMobBTPRrFoBjpy5we6JzQEYPGsHa/2sx61dJicH/rzGFdv48uG/buA3qSVHz9zgyJmb8Xh1DsyBEljHgJVdONsYsIof8TEGfOzSozhnTsHMnvpaeqWUii/6Uk6llLIXDWCllLIPR3oSTgNYKeVUXoTpZXGlAayUcioOlL8awEopJ+NACawBrJRyKvG4IHuC0wBWSjkVx4lfDWCllLNxoATWAFZKORWdhqaUUnbiQEPAGsBKKeeiAayUUnaiQxBKKWUn2gNWSik7caD81QBWSjkX7QErpZTdOE4CawArpZyKLsiulFJ2okMQSillJzoNTSml7MVx8lcDWCnlXBwofzWAlVLORceAlVLKTsSBElgDWCnlVBwnfjWAlVJOxoE6wBrASinn4kjT0Cz2PgGllIpPInHfnn4smSEi10XkaKSyISJySUQO2baakT7rJyKnReSUiFR/2vE1gJVSTiU+AxiYBdSIpnysMcbXtv1ibVcKAE2AgrZ9vhMRl9gOrgGslHIq8gz/expjzFbgdhybrgMsNMYEGWPOAKeBkrHtoAGslHIqz9IDFpH2IrIv0tY+js10FZHDtiEKb1tZZuBCpDoXbWUx0gBWSjkVeYbNGDPVGFM80jY1Dk1MAnICvsAV4Ov/eq4awEop5/IsCfwfGGOuGWPCjDHhwDT+GWa4BGSNVDWLrSxGGsBKKacSn2PA0R5fJGOkb+sBj2dI/AQ0ERF3EckB5Ab2xnYsnQeslHIq8bkgu4gsACoBaUTkIjAYqCQivoABzgIdAIwxx0RkMXAcCAW6GGPCYju+BrBSyrnEYwAbY96Ppnh6LPWHA8PjenwNYKWUU3GkJ+E0gJVSTsWR1oIQY4y9z+GlISLt4zjNRb1E9O/Fy0tnQSSuuE7yVi8X/XvxktIAVkopO9EAVkopO9EATlw6zqeio38vXlJ6E04ppexEe8BKKWUnGsBKKWUnGsCJRERq2F5TclpE+tr7fJT9Rfe6G/Vy0QBOBLbXkkwE3gYKAO/bXl+iXm6ziP51N+oloQGcOEoCp40xfxtjgoGFWF9fol5iz/i6G+WENIATxzO/qkQp5fw0gJVSyk40gBPHM7+qRCnl/DSAE4cfkFtEcohIEqAJ1teXKKVeYhrAicAYEwp0BdYCJ4DFxphj9j0rZW+2193sAvKKyEURaWvvc1KJSx9FVkopO9EesFJK2YkGsFJK2YkGsFJK2YkGsFJK2YkGsFJK2YkGsFJK2YkGsFJK2cn/AVc238rS3gWMAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"cm_calib = confusion_matrix(calib_labels, np.where(np.array(predict_calib.ravel()) > 0.5, 1, 0))\ncm_calib","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:20:35.961942Z","iopub.execute_input":"2021-12-01T17:20:35.962498Z","iopub.status.idle":"2021-12-01T17:20:35.971624Z","shell.execute_reply.started":"2021-12-01T17:20:35.962464Z","shell.execute_reply":"2021-12-01T17:20:35.970835Z"},"trusted":true},"execution_count":133,"outputs":[{"execution_count":133,"output_type":"execute_result","data":{"text/plain":"array([[258,  90],\n       [ 99, 249]])"},"metadata":{}}]},{"cell_type":"code","source":"cm = [cm_calib[0][0], cm_calib[0][1], cm_calib[1][0], cm_calib[1][1]]\n\ngroup_names = ['True Neg','False Pos','False Neg','True Pos']\ngroup_counts = [\"{0:0.0f}\".format(value) for value in cm]\ngroup_percentages = [\"{0:.2%}\".format(value) for value in cm/np.sum(cm)]\nlabels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\n\nsns.heatmap([[cm[0], cm[1]], [cm[2], cm[3]]], annot=labels, fmt='', cmap='Reds')\nprint(\"Calibration dataset\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:20:36.159841Z","iopub.execute_input":"2021-12-01T17:20:36.160392Z","iopub.status.idle":"2021-12-01T17:20:36.460644Z","shell.execute_reply.started":"2021-12-01T17:20:36.160355Z","shell.execute_reply":"2021-12-01T17:20:36.460019Z"},"trusted":true},"execution_count":134,"outputs":[{"name":"stdout","text":"Calibration dataset\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAWAAAAD4CAYAAADSIzzWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvDElEQVR4nO3dd3hUxdfA8e/ZbAgJEHonFJGOdBABqSqIYLDQREFEQlWwIFURFUX5iVgARYogRapSRIq00HtvCi810iG0FFLm/WOXECAhCyS52eV8nuc+7M6dO3cu4slk7hQxxqCUUir12ayugFJKPaw0ACullEU0ACullEU0ACullEU0ACullEXsKX2DzuKvwyzUHX68dtzqKqi0yC+zPGgR9xJzfjSXH/h+DyLFA7BSSqUmd/q1XgOwUsqj2MTSRu090QCslPIo2gJWSimL2NynAawBWCnlWezaBaGUUtbQLgillLKIdkEopZRFtAWslFIWEe0DVkopa2gLWCmlLGJ3nwawBmCllGfRmXBKKWUR7YJQSimL6DA0pZSyiLaAlVLKIjoVWSmlLKItYKWUsoj2ASullEVsuE8EdqfWulJKJckmrh93IyIBIrJcRPaKyB4R6XHb+fdExIhIDud3EZHvROSgiOwUkUpJ1VVbwEopj5KMrcpo4D1jzFYRyQRsEZElxpi9IhIAPAMci5f/WaCY83gcGOX8MzXqqpRS1rOLuHzcjTHmpDFmq/PzFWAfkN95+hvgAyD+DsyBwETjsB7IIiJ571rX+3xGpZRKk1LiJZyIFAYqAhtEJBAIMcbsuG3ltfzA8XjfTzjTTiZWrgZgpZRHuZf4KyJBQFC8pNHGmNG35ckIzAJ64uiW6Iej++GBaQBWSnmUe2kBO4Pt6MTOi4g3juA72RgzW0QeA4oAN1q/BYCtIlINCAEC4l1ewJmWeF1dr6pSSqV9NsTl427EEWHHAvuMMcMAjDG7jDG5jDGFjTGFcXQzVDLGnALmAm2doyGqA5eMMYl2P4C2gJVSHiYZ+4BrAq8Bu0RkuzOtnzFmQSL5FwCNgYNAGNA+qRtoAFZKeRSvZCrHGLOaJLqUna3gG58N0O1e7qEBWCnlUXRBdqWUsoj7hF8NwEopD6MBWCmlLKIBWCmlLKJ9wEopZRF3mtygAVgp5VHcqAGsAVgp5VnEjXqBNQArpTyK+4RfDcBKKQ+jAdjNZciWjZ5L5wLgnyc3JiaGK2fPATCkWj1ioqIe+B7vLv8Tn4wZ+KJqXQAKVq7Iy//7jGH1nnvgslXKKFW5OsUfLRr3fcQ3QymQL1+CeSvWqMO2tSsf6H59PhrExi1byZQxIzabjY/69KJi+XIPVObDwMuNOoE1ACfg2oULDK5YC4AmA/sSefUqS77+Pu68zcuL2JiYB75Pplw5KdPoafYsXPLAZamUl97HhznTJqfqPT/o+TaNnm7A6nXr+WjwEOZNn5Kq93dH7hN+NQC7rN34UURFRBBQsTyH1qwn4vKVWwLzh7vWM7JJC84fPUa1Ni2p/3ZnvNJ5c3jDZqZ2fRcTG3tHmYuHfsez/d+/IwCLzcYLQwZRvO6T2H3SsXLEz6waPR4RodUPX1Oifm0uHj9BTFQ0a8f9ytZZc1Ll70Dd6lpYGF3feZ/Ll68QHR1Nj66deapenVvynDl7jnd69+PqtWvExMTwcb/eVKlUkdXr1vP9qNFcj4oioEB+vhj0ERn8/BK9V9VKFTl23LHZwvhfJzNrzjwAXn4hkNfbtCYsPJyeH/Tj1OnTxMbG0rVjBxo3fDrlHj4Nc6MGsAbge5GlQH6+qvEUJjaWJgP7JpgnT8niVGn5Il/VfJrY6GhajxhGtTYt2fDr1Dvy/t+6jVR4oQnF6z5JxJWrcek1O7Ql/NJlhlSriz1dOnqtWczexcsoVLkC2QsXZFDpqmTKlZOB+zaxdtyvKfa86lYRkZEEtmwDQIH8+fj2qy8Y8fVXZMyYkQsXQ2nZ7g0a1K1N/G1q5v+1iFo1qtPlzTeIiYkhPCKCCxdDGfXzOMb/NAI/X19Gj5/A+F+n0L3Tm4nee1nwKoo/+ii79+5j9tz5TP91PMYYWrzWnmqVK3H8RAi5cuZg9PffAHAl3r+nh40bxV8NwPdi64w/EmzJxleyQV0KVq5A300rAPD29eXKmbOJ5v/rs6E0HtCL2b0HxqWVfqY++cuVpdLLgQD4ZvYnV7GiFK31BFtm/I4xhsunz/DP8lUP/lDKZbd3QURFRTPsh1Fs2roNmwinz5zl3Pnz5MyRIy7PY2VK0W/QZ0RHR/NUvbqUKlGc5VtWcfDwYVq//mZcORXKlU3wnl8N/45RY8aRLWtWBg8cwLqNm3iqXl38fH0BeLp+PTZv3c6TNavz5bBvGfrt99R7shZVKlVMsb+HtC6phdbTEg3A9+D6tWtxn2OioxHbzTk33unTOz6IsH7CFP7oN8ilMg8sD+b5zz7kkepVbyaKMO2tXuxdvPSWvGUbJ8s2VCqZzPtrIRcuXmT25Il4e9up3ziQyOvXb8lTtXIlJo35iZWr19Dno0G0f/UV/P39qfn44wwb8lmS97jRB3zDuo2bEsxXpFAhZk+dyMrVaxk+4keqV6t61xa1J3Of8Otes/bSlPNHjhFQqQIAARXLk6NIIQAOLF1BpZebkSmnoxXklzUr2QoGJFYM4GgFP/NBj7jvexctpXaXDtjsjp+PuYo9Sjo/Pw6tWU/FlwIRETLlyknxuk+mwJMpV125epXsWbPi7W1n/abNhJy8c/eZkP9OkiN7Nlq82IzmLwSyZ/8BKjxWlq07dnD0mKNPNyw8nMNHj7p0zyoVK/D3ipWEh0cQFh7O38tXUKVSBU6fOYtv+vQEPvcsHdq9yt79+5P1Wd2JTVw/rKYt4Pu0bdYcqrdtzUe7N3B4w2ZO/3MQgJP7DjBnwKe8vfgPxGYjJiqK37q9z4VjxxMta/dfi+OGuQGsGTOB7IUL0n/rKhDh6tlzjGr2CttmzaFkgzoM3LuJi8dPcGzrdsIvXU7xZ1UJa/psI7r0eJemzVtTtnQpHilS+I48GzdvYezESdjtdvz8fPny04/Jli0rXwz6iHf7DuC6c0hjz66dKVKoUJL3LFOqJC82fY7mr70OOF7ClS5ZglVr1/HV8O+xiWC32/m4X+/kfFS3klwz4UQkAJgI5AYMjh2TvxWRoUBT4DpwCGhvjAl1XtMX6ADEAG8bYxbd9R6OXTRSTmfxT9kbPGR8MmQg8to1MmTLRp+Nyxla82kunz5jdbXu2Y/XEv+BpB5ifpkfOHoG5y7gcsypffpEovcTkbxAXmPMVhHJBGwBmuHY7XiZMSZaRL4EMMb0FpHSwFSgGpAP+BsoboxJdMyqtoDdTLf50/HNkhl7unQs+PRLtwy+SqWk5OpZcO5ofNL5+YqI7APyG2MWx8u2HnjZ+TkQ+M0YEwkcFpGDOILxusTuoQHYzehMOaXuLiW6dkWkMFAR2HDbqTeAac7P+XEE5BtOONMSpQFYKeVR7mVBdhEJAoLiJY02xoy+LU9GYBbQ0xhzOV56fyAauO/pkRqAk0nWAvl5feJP+OfOhTGG1aN/Ydl3o2gysC+1OraLe8k2p98n7P5rMTa7ndfG/EDBSuWx2e2snziVRUOGWfwUKqVNmPIbM2b/gTGG5i824/U2rQm9dIl3evcn5L+T5M+Xl+FffU5mf3+rq+q27mVolzPYjk7svIh44wi+k40xs+Olvw40ARqYmy/SQoD4Q54KONMSpQE4mcRERzPzvf4c37YDn4wZ6bclmH1LlgGw9JsRt6wlAVC5+QvYfXz4tNwTePv68vHejWyeOpPzR49ZUX2VCv45eIgZs/9gxq+/4O1t581uPaj3ZC2mzf6dJ6pVJeiNdoweN4HR4yfQq8dbVlfXbSVXF4Q4pjSOBfYZY4bFS28EfADUMcaExbtkLjBFRIbheAlXDNh4t3voOOBkcvnUaY5v2wFA5NWrnNp3gCz5E14pC8AYg08GP2xeXqTz9SX6ehThl6+kVnWVBQ4dPky5smXw9U2P3W6nauVKLF62nKUrgmnW1NG336zpc/y9/MFWUXvYiYjLRxJqAq8B9UVku/NoDPwAZAKWONN+BDDG7AGmA3uBhUC3u42AABdawCJSEsfbvRudySHAXGPMvqSufVhlL1SQgIrlOLxhM0VrVqdu9yAeb9uao5u3Meu9/oSFhrJ15h+UD3yOL0/+Szo/X2a805ewixetrrpKQcWLFmX4D6O4GBpKep/0BK9eQ9nSpTh//gK5nBN3cubIzvnzFyyuqXtLxlEQqxMpbsFdrhkMDHb1HndtAYtIb+A3ZyU2Og8BpopIn7tcFyQim0Vk816uJ5bNI/lkyEDQrF+Z3rMPEVeusHLUGAYULc/gCjW5fPIUL33t+G9TpFplTEwMvfMVZ0CRx3jqvbfIkcBAfuU5ij5ShDdfb0uHrm/zZre3KVmiODYvr1vyuNgyU3ch93BYLakWcAegjDHmlhXInX0ce4AhCV0Uv2P7YZqIYbPbCZo1iY2Tp7P9d8dygfEX4ln98wS6zp8OQNVXWrBn4d/ERkdz5ew5Dq1ZT6EqFTl3+IgVVVeppPkLgTR/wbHI0rDvR5I7dy6yZ8/GmbPnyJUzB2fOniNbtqwW19K9eaWFOcYuSqoPOBZHZ/Lt8jrPqXjajh3BqX0HWPrNiLg0/zy54z5XeKEp/+129NxcOHacEvVrA5DOz49Hqlfl1P5/UrfCKtWdv+DoXvjv5CkWL1tO02cbUr9Obf6Y9ycAf8z7kwZ1a1tZRbcnNnH5sNpdpyI73/b9APwL3Jg7WhB4FOhujFmY1A0elhZw0ZrV6bV6MSd27o5bsnJOv0+o0vplAio8hjGG80eOMblTDy6fOo1Phgy0HT+SvKVLIiKsHT+JJf/7zuKnSD0P61TkV97oSGjoZex2L/q+15MnHq/GxdBQevbux8mTp8mXNw/Dv/qcLJkzW11VayTDVOQdBQu7HHPKHztiaRROci0IEbHhmE4X/yXcpqTe7t3wsARgdW8e1gCskpAMAXhnIdcDcLmj1gbgJEdBGGNiuXV6nVJKpVnu9BJTJ2IopTyKG8VfDcCusPv48H7wQuw+6bDZ7WydOYf5H3/Oe8ELSZ8pI+DY4fjIxi38+MIrd1xfve0rNB7QC4AFnw1l/cQp+GTMyPurbnahZy2Qnw2TpjHjnT7U7d6JJzu15+KxE4xq1pqYqCiK1qxOpZcCmfFuwnvRqdTX9+NPWRG8muzZsjJ/5m8AfPnNdywPXoW3tzcFnZtt+mfKdMe1CU1JBhg+4keWrgzGJkL2bNn4YtBH5M6Vk0V/L+O7UT+RObM/I4YNJWuWLBw7foJhP4xk+Jefp+pzp3W2NPByzVW6HrCLbqzDa7Pb6bV6MdN79ObwhpvbwwTN/JUdcxbcsfmmX9as9N28gi+q1AVj6LtlJV9UrkNYaOgt+fpuXsmMd/pycNVaPli3lKE1nqJRv/c5sWMXu+Yv5K2FvzO29RseM1nDE/qAN23Zip+fH70//DguAK9et57qVatgt9sZ+q1j+vnt04r/OXiId/v0v2VK8qD+fShUMICrV6+SMaPjh/rEKdM4+H//xycD+vLam50Z/f1wFi9bzuXLl3mtdUve7TOAt7sEUbhQwdR98JSUDH3A+x8t6nLMKXnwkKXRWqciuyjSuR+cl7c3Xt524v/gSp8pEyXq12bHH/PvuK50wwbsW7KcsIsXCQsNZd+S5ZRu9NQteXIVe5RMuXJycNVawNGH5eXtTTo/X2Kionn81Vbs+WuJxwRfT1G1ciUyZ7510ZxaT1TH7txKqsJjZTmVwHrNiU1JBuKCL0B4eHhcf6bYhOtR14mIiMBut7N56zZy5MjmWcE3mYi4flhNA7CLxGaj/7bVDD1ziH1LlnNk4+a4c+WbNeHA0pVEXLlzLYes+fNy8fjNBZFCT/xH1vx5b8lTpdVLbJkWt9ASK34YzQfrl5KtYACH1qznifZtWDEi0QWbVBo1a848atescUd68aJF2bJtOxdDQwkPjyB49RpOnTodd/6bH0ZSp1ET5v21kB5dOgHQ6Y3Xad+5O8uDV9GkUUNG/jyOrh07pNqzuJNkXAsixWkfsItMbCyDK9bCN3NmOv8+mXxlSvHfHsekiqqtX2bNmAn3XXbVVi8x/rWbS5JumPQbGyY5fqVt/GFvln/3I2WffZrqbVtz8XgIM9/rR0p3HakHM2rMOLy8vHi+caM7zsWfkuybPv0dU5Lf6d6Vd7p35aexvzBp2gze7hJEzeqPU7P644BjskbtWjU4cvQY4yZOxt8/E/17vYevb/pUe760TNyoWelGVU0bwi9d4sDyVZRxdiNkyJ6NwtUqs+vPhPfeuxhykqwBNxfFz1IgHxdDbu6em79cWWx2O8e2br/j2sx581C4WmV2zPmTp957i59bvk5Y6CVKNqibrM+kktfsufNZEbya/w3+NNFWVvMXApk9ZSKTx40ms79/gl0JTRs3YvHSZbekhYdHMHvefNq0aM73P45myKcDqVyhPPP+SnJO1EPDZhOXD6tpAHZBxhzZ8XXOTPJOn55ST9fj1P5/Aaj0cjN2zV9IdGRkgtfuXbSU0s/Uxy9LFvyyZKH0M/XZu2hp3PmqrV9m09SZCV77/KcDmPeRY/GedL6+YAwmNpZ0fr7J+XgqGQWvWceYX35l1PCv79oiTWhKMsCReOtBL12xkkcKF77lurETf6Vt65Z4e9uJiIhEEMRmIzwiIvkfxk1pF4SHyZw3D+0m/IjNywux2dgy/Xd2/elocVRt9RILh3xzS/6ClStSu/MbTOr4FmEXL7Lg06/os2kFAH9+8uUtL9Mqt3iBHxq/zO0CKpQDiFtjeOOUGXy4az0Xj4ew+KvhKfCU6l6922cAG7ds4WJoKLUbNuGtzh0ZPX4C169fp32X7gCUf6wsnwzoy+kzZxnwyWB+/mE4AG+93ztuSvLAPr3ihqp9/d0IDh89iths5M+bh0H9by46ePrMWXbu3kv3Th0BeLV1C15+tR2ZMmVi5LChqfvwaVgaiKsu02FoyhKeMAxNpYBkGIZ29LESLsecQrsOpO2pyEop5U7cqQWsfcBKKY+SXH3AIhIgIstFZK+I7BGRHs70bCKyRET+df6Z1ZkuIvKdiBwUkZ0iUimpumoAVkp5FJvN9SMJ0cB7xpjSQHWgm4iUBvoAS40xxYClzu8Az+LYiLMYjq3uRyVZ1/t6QqWUSqOSa0F2Y8xJY8xW5+crwD4cy/IGAjcG/k8Amjk/BwITjcN6IIuI3Drr6jYagJVSHuVepiLH37/SeQQlXKYUBioCG4Dcxpgbg/lPATe2vcnPzY0rAE5wcx31BOlLOKWUR7Hdw1u4+PtXJkZEMgKzgJ7GmMvx+46NMUZE7nukl7aAlVIeJTkX4xERbxzBd7Ix5saCLadvdC04/7yx4lIIEBDv8gLOtERpAFZKeZRkHAUhwFhgnzFmWLxTc4F2zs/tgDnx0ts6R0NUBy7F66pIkHZBKKU8SjKu8VATeA3YJSLbnWn9gCHAdBHpABwFWjjPLQAaAweBMKB9UjfQAKyU8ijJNRHDGLMaSKy0BgnkN0C3e7mHBmCllEdJC4vsuEoDsFLKo7jTesAagJVSHkVbwEopZRUv92kCawBWSnkUbQErpZRV0sBWQ67SAKyU8izaAlZKKWsktcpZWqIBWCnlWbQFrJRS1hAdBaGUUhbRLgillLKGDkNTSimraAtYKaUsoi1gpZSyhnhpAFZKKUvoOGCllLKKG3VBuM+AOaWUcoVNXD+SICLjROSMiOyOl1ZBRNaLyHbnVvbVnOkiIt+JyEER2SkilZKs6gM9qFJKpTHJtSmn0y9Ao9vSvgIGGWMqAB85vwM8CxRzHkHAqKQK1wCslPIsydgCNsYEAxduTwb8nZ8zA/85PwcCE43DeiDLje3rE6N9wEopjyI219uVIhKEo7V6w2hjzOgkLusJLBKR/+FoxNZwpucHjsfLd8KZlujW9BqAlVKe5R5GQTiDbVIB93ZdgHeMMbNEpAUwFnjqHssAtAtCKeVhkrkPOCHtgNnOzzOAas7PIUBAvHwFnGmJ0gCslPIsydgHnIj/gDrOz/WBf52f5wJtnaMhqgOXjDGJdj+AdkEopTxNMo4DFpGpQF0gh4icAAYCHYFvRcQORHCzD3kB0Bg4CIQB7ZMqXwOwUsqjJOdMOGNM60ROVU4grwG63Uv5GoCVUp5FF2RXSilr6HrA8fx4bl9K30K5oUFZC1tdBZUGDYy8+OCF6GI8SillEW0BK6WURTQAK6WURby8rK6ByzQAK6U8i7aAlVLKIhqAlVLKIhqAlVLKIvewHKXVNAArpTyLtoCVUsoi2gJWSimLaABWSimLaBeEUkpZRAOwUkpZRAOwUkpZ4152Rbaa+9RUKaVcYbO5fiRBRMaJyBkR2X1b+lsisl9E9ojIV/HS+4rIQRE5ICINkypfW8BKKc+SvF0QvwA/ABNvFi/1gECgvDEmUkRyOdNLA62AMkA+4G8RKW6MiUmscG0BK6U8SzK2gI0xwcCF25K7AEOMMZHOPGec6YHAb8aYSGPMYRybc1bjLjQAK6U8i4jLh4gEicjmeEdQ0jegOPCkiGwQkZUiUtWZnh84Hi/fCWdaorQLQinlWe6hC8IYMxoYfY93sAPZgOpAVWC6iDxyj2XEFaSUUp4j5RdkPwHMdm5Dv1FEYoEcQAgQEC9fAWdaorQLQinlWe6hC+I+/QHUc9xKigPpgHPAXKCViPiISBGgGLDxbgVpC1gp5VmScRSEiEwF6gI5ROQEMBAYB4xzDk27DrRztob3iMh0YC8QDXS72wgI0ACslPI0yTgRwxjTOpFTryaSfzAw2NXyNQArpTyLTkVWSimLaABWSimL6Lb0SillEW0BK6WURTQAK6WURcR9pjdoAFZKeRabtoCVUsoa2gJWSimL6CgIpZSyiL6EU0opi2gXhFJKWURbwEopZRE32hVZA7BSyrPY9CWcUkpZQ8cBu69StZ6ieNEicd9HDPmUAnnzJJi3YoPGbFu64IHu1+ezL1mzcTNLZ04mXbp0XAi9xMtvdGbZ7KkPVK5KGb7ZstJ24RwAMubORWxMDGHnzgPwc80GxEZFPfA92i2eR8a8uYmOiOT61WvM7dSd8/8cfOByHxr6Es59pfdJx5wJP6fqPb28vJg5/y9eeTEwVe+r7l34hYv8VK02AHUG9Ob6tWus++aHuPPi5YWJuesmCC6Z3S6Ik1u3U6lDO57+4hN+e+mVBy7zoaEv4TzHtbBwuvYewOUrV4iOjqFH0Bs8VbvmLXnOnDvPOx9+wtWwMGKiY/i4V0+qVCjH6g2b+H7MBK5HXScgfz6+6N+bDH6+d9yjXYuXmDBtFi2eb3LHuTGTf+OvpSu5HhXF03Vq8fabrwMwYvyvzF20hGxZspA3V07KlCxOh1dapsjfgbq7wJ9HEB0ZSZ7yj3F83QYiL1+5JTB32bqWKS+05NLR4zzWugWPdwvCK106QjZt4c+33sPExiZa9tHVa6n+VmcAnv7iEx5t+BTGGFZ98T/2zPydjHly8/Kkcfj4Z8Jmt/PnW+9xbM26VHnuNCsZW8AiMg5oApwxxpS97dx7wP+AnMaYcyIiwLdAYyAMeN0Ys/Vu5WsAvk1E5HUC23UEoEDePHz72ceMGPIJGTNk4ELoJVp27EaDJ2sg8X7Kzl+8lFqPV6XL668SExNDeEQkF0IvMeqXSYz/bih+vr6M/nUq43+bQfc32t5xz7y5c1GpXFnmLFxMvVo14tJXb9jE0eMhzBw7EmMMXT4YwKZtO/Dx8WHximDmThhDVHQ0L7bvRJmSxVP+L0clyj9/PsbVaYiJjaXOgN4J5slRsjhlmr/AuLqNiI2OpvF3/+Ox1s3ZOXlaouWWeK4Rp3fvpVSzpuQuX5Yfq9TCL0d2Oq5ZxtHVa3ms1cscWrKMVV9+jdhsePv5pdQjuo/k7QP+BfgBmBg/UUQCgGeAY/GSn8WxEWcx4HFglPPPRGkAvs3tXRBR0dEM+3EMm7bvwmYTTp89x7kLF8mZPVtcnsdKlaDf50OJjo7mqdq1KFX8UZavWcfBI0dp3fltRzlR0VQoWzrR+3Zq+wpde39I3RrV49LWbNzMmo2bafZ6EABhYeEcORHCtbAwGjxZEx+fdPj4pKNezSeS+69B3aO9s/64a0sWoEi9OuSrWJ6Oa5cBYPdNz7UzZxPM++KE0USHRxB69Bh/vdObJ3p0Zfe0WZjYWK6dOcuRVWvIV6USIZu3ETj6e2zedvbP/ZPTO3cn+7O5nWQcBWGMCRaRwgmc+gb4AJgTLy0QmOjcoHO9iGQRkbzGmJOJla8BOAnzFv3NhdBLzB7/I952O/VfbE3k9eu35KlasTyTRg5n5dr19Bn8Je1bNcc/U0ZqVq3MsE8+dOk+hQMKUKpYUf5atiIuzRgIavsKrZo1vSXvL9NmPvBzqeR1/VpY3OfY6Ggk3lhUe3ofwNE1uWPSbyz98JMky7vRB5yUY6vXMr7BcxR/9hmajRnJum9H3LVF/VC4hxawiAQBQfGSRhtjRidxTSAQYozZIbf2N+cHjsf7fsKZlmgAdp/XhRa5cvUa2bNmwdtuZ/2WbYScOn1HnpCTp8iRLSstApvQvGlj9hz4hwplSrN11x6OnggBICw8nMPHjt9xbXyd27Vh3JTpcd9rPV6FWfP/4lpYOACnz57l/IWLVHqsLMtXryUy8jrXwsJZsXZ9Mj6xelChR4+Tt0J5APJUKEeWwoUA+L9lwZR68Xn8cuYAIH3WLGQuGOBSmUdXr6NM8xcQmw2/HNkpVKsGIZu2kLlgANdOn2HruIlsHf8reSuWT5mHcidic/kwxow2xlSJdyQVfP2AfsBHyVFVbQEnoWnDp+jSqz9NX+1A2ZLFeaRQwTvybNy2g7GTp2G32/Hz8+XLD/uQLWsWvuj/Ae9+9BnXnUOTega1p8hd/ocr9kgRSpcoxt4D/wJQ6/GqHDpyjFZB3QHw8/Vl6MC+lCtdkvq1avB82zfJni0rxYsWIVOGDCnw9Op+7Pt9LuVfbUmXbWsJ2biF8/86hpCd23+A5QMH89qfsxGbjZioKBb06MWlJH4wA+yfM5+A6tXovHk1xhj+7jeQa6fPUP7VVtR4921ioqK4fvUaf3TonNKPl/al7CiIokAR4EbrtwCwVUSqASFA/P/BCzjTEiWO7ooUdD4khW/wcLoWFk4GP1/CIyJo07Unn/Z+lzIl3OdF3KB8ZZPOpB46AyMvPnD0jJkz0uWY4xXYNcn7OfuA598+CsJ57ghQxTkK4jmgO45REI8D3xljqt2tbG0Bu6mPvvyag0eOEhl5nRcaN3Sr4KtUikrG9YBFZCpQF8ghIieAgcaYsYlkX4Aj+B7EMQytfVLlawB2U18PGmB1FZRKm5KxC8IY0zqJ84XjfTZAt3spX1/CpZAJ02bRpM0bPNemfdyohf3/HqJlx+40fbUDnXv14+q1axbXUqU0/wL5abtoLl23r6PLtrU83r3TLeef6NmNgZEX8XUOa0yfJTMtpv9K582reXP13+QsXcqKars3m831w+qqWl0BT/TPocPMmPsnM8aOZM6EMaxYs56jJ0Lo/8X/eK9rR+ZNGstTdZ5kzMM+XOghEBsdzeLeAxhZ4QnGPvkMVTu/SY6SJQBHcH7kqXqEHr35Eu7J3u9xescufqxSi987dKHRsC+sqrr7EnH9sJgG4BRw6OhRypUphW/69NjtXlStWJ7FK1Zx5PgJqlYoB0DNqpVZvGKVxTVVKe3qqdOc2r4TgOtXr3J2/z/4588LQMOhg/m778eOAd9OOUqV4LDz38X5A/+SpVBBMuTKmer1dmv3MAzNatbXwAMVf6QIW3bs4uKlS4RHRBC8dgOnzpyhWJFCLA1eA8DCZSs5eeaMxTVVqSlzoQDyli/HiY1bKNH0Wa78d5LTu26duXZ6525KNXOsCZKvSiWyFAzAP38+K6rrvh6GFrCIJPqGT0SCRGSziGwePWHS/d7CbRUtXIg3X21Fh54f8OY7vSlZvCg2m43B/T5gyuw5vNi+E9fCwkhn97a6qiqVeGfIQIvfJrLw/b7ERkdT64N3WT7ozu6F1UOH45M5M502BlOtaxAnt+8kNhlWV3uoeHm5fljsvscBi8gxY8ydsxJup+OAGfbjGHLnzEmbl24uN3n42HF6DfqcmWNHWVgz6zxM44Btdjut//iNQ0uWsf7bkeQqU5q2C/8gyjnD0b9APq78d4qfazXg2ulbfyvqcWAHo6rU4vqVK1ZUPdUlyzjgZZNdHwdcv42lzeC7DkMTkZ2JnQJyJ391PMf5CxfJni0r/506zeIVq5j+84i4tNjYWEb9MolWLzxvdTVVKnj+p+85t/8f1n87EoAze/byv4Cb47Z7HNjB6Br1CD9/AZ/M/kSFhRMbFUWlN9pydPXahyb4Jps00LXgqqTGAecGGgIXb0sXYG2K1MhDvNX/Y0IvXcZu92Lg+z3wz5SRCdNmMWW2Y/Gkp+vU4qXnGllcS5XSAmpUp/yrrTi9aw+dNgYDsPSjTzm4cEmC+XOWLEEz5/KjZ/fuZ26nt1Kzup4hDbxcc9VduyBEZCww3hizOoFzU4wxSS/Tr10QKgEPUxeEcl2ydEEET3O9C6J2y7TbBWGM6XCXc7pHilIq7XGjFrBORVZKeRbdll4ppawhbvQSzn3a6hbqO/grnmj8Ik3avHHHuXFTplOiRn0uhF6649y+fw7SsmN3nmvTnqavvcmCv5fHnXvv48E0bNWWJm3eoO/gr4iKjgZg0fJgnmvTnle69ODiJUeZx06E0NOFXRRU6vLy8eHN1X/TadMqumxbS90P+wCOTTrfPrCdThuD6bQxmNzl7uzvLlynVtz5ThuD6X/pJCWebwxAkXq1CVq/gk4bg2m/7C+yFi0CQLWuHemydS2vzJmOzdsxhjygRnUaDh2cSk/sJnQmnGd5sXFDxnwz5I70k6fPsGbjZvLlzpXgdenT+/DlR334c/J4xgwbwuffjuDylasAPP9MAxZOncC8SWOJjIxkxtw/AZg083dmjh1Fy8AmzF+8FIDho8fRM+jO4K+sFRMZyYSGgfxU9Ul+qlqbos80IH+1KgAs6fMRP1WrzU/Vaie4T9uRlavjzk9o+DxRYeEcWuL4Af3c918z+/UgfqpWm13TZlK7z/sAPNaqOaMq1+T4ug08+kwDAGr3e5+Vnw9NpSd2ExqAPUvViuXJ7O9/R/oX346kV7dOif7KU6RgAIUDCgCQO2cOsmXNwoXQUADq1KiOiCAilCtdktNnzgGOX5+uX48iIiISu93O5u07yZE9W1w5Km2Jcq5oZ/P2xsvb+5Z1HVxV+sVA/l30N9HhjokZxhh8MmUCIL2/P1dOnnJkFMHL2xtvPz9ioqIo90pLDi76m4iLocnyLB7DJq4fVlfV6gq4q7+D15ArZw5KFivqUv6de/cRFRVNwdvm9UdFRzNn4RKerF4VcOyO3L7H+yxfs5YmT9dn5Phf6dr+tWSvv0oeYrPRaWMwvU78w/8tXUHIpi0A1P9kAJ03r6bh0MF4pUt31zLKNn+R3dNnxX2f17kHr8yZzjuHdlOuTQtWDx0OwKZRY+iwagmZAwpwfO0GKrR9hU2jxqTYs7ktm5frh8X0Jdx9CI+I4KeJkxk3/CuX8p85d55en3zBlwP6YLttDdJBQ4dTpUI5qtxYJa1aFWo6f43946/F1K7xOEeOnWDclOn4Z8pI/3e645s+ffI+kLpvJjaWn6rVxiezPy2nTyJn6VIs/fATrp46jVe6dDQZOZya7/cgOJFugox5cpOrbGkOObubAKq/3YUpgS0I2bSFGu++RcOvPmNelx7snDKNnVMcS5jW7teLDSN/4tFGT1O+TUsunwhh0QcD7qsF7nHSQNeCq9ynpmnIsZD/OPHfKQLbdqT+i605dfYsL7bvxNnzF+7Ie/XaNTq935d3gjpQoWzpW879MHYCF0Iv0fftrndcFx4Rwew/F9LmpWZ8P+YXhnzYm8rlH2Peor9T7LnU/Yu8dJkjK1fxaMMGXHXunB1z/TrbJ04mf9XKiV5X5uVm7J87n1jnS1i/HNnJXa5sXEt694zfCXji1m3FMubNQ/6qlTkwdwFP9OjGzDZvEBF6iUfq10mhp3MzybgamoiME5EzIrI7XtpQEdkvIjtF5HcRyRLvXF8ROSgiB0SkYVLlawC+DyWKPsK6BbNZNnsqy2ZPJU/OnMwe/xM5nbsa3HA9KopufT4i8NlnaHTb/xwz5v7J6g2bGPbJgDtaxQBjJ0+jbfMX8bbbiYiMjOsvDo+ITNFnU67zy5Edn8yOdwP29Ol5pEE9zh34l4x5bi6TUvL55zizZ1+iZZRt8RK7p93sfgi/GEp6f3+yObu2ijaoy9n9/9xyTf2B/Vg+6HMAvH3TY4zBxMbi7eebbM/m1pL3JdwvwO1rBiwByhpjygH/AH0BRKQ00Aoo47xmpIjctZ9DuyBc8O5Hn7Jx2w4uhl6idmAL3nrzdZo3bZxg3l37DvDbH/MY3Pd9/lq6gs3bdxJ6+TK/L1gEwJD+vSlV/FEGDv2GfLlz09K55fzTdZ6k+xttATh99hw79+6ne4d2ALza/AVe7tCFTBkzMnLIp6nwxMoVGfPkodnYkdi8vBCbjT0zf+ffBYtou3AOfjlzICKc2rGL+d3fBSBvpQpU6dieeV16AI71gf0L5OeIc41oABMTw7wuPWjx20RMbCwRF0OZ06l73Pk85R8DiFvkfde0mXTZuobLJ0JY8/V3qfXoaVsyvlwzxgQ7d0WOn7Y43tf1wMvOz4HAb8aYSOCwiBwEqgHrEitft6VXltC1IFRCkmMtiNidy11fC6J8/U5AULyk0caY0fHzJLEt/TxgmjFmkoj8AKw3xkxynhsL/GWMmZnY/bUFrJTyLPcwusEZbEcnmTEBItIfiAYm38/1oAFYKeVpUmEqsoi8DjQBGpib3QghQEC8bAWcaYnSl3BKKc+SwjPhRKQR8AHwvDEmLN6puUArEfERkSJAMWDj3crSFrBSyrMkMKrofonIVKAukENETgADcYx68AGWOGfBrjfGdDbG7BGR6cBeHF0T3Ywxd93QTwOwUsqjJOdqaMaY1gkkj71L/sGAy6sjaQBWSnkWN5oJpwFYKeVZ0sAaD67SAKyU8ixutCC7BmCllGdJxpdwKU0DsFLKs2gLWCmlLKIv4ZRSyiIagJVSyiLaBaGUUhbRAKyUUlbRAKyUUtbQFrBSSlnEfeKvBmCllIfRURBKKWUR7YJQSimraABWSilraAtYKaWs4j4B2H16q5VSyhUirh9JFiXjROSMiOyOl5ZNRJaIyL/OP7M600VEvhORgyKyU0QqJVW+BmCllGdJ3k05fwEa3ZbWB1hqjCkGLHV+B3gWx0acxYAgYFRShWsAVkp5FBFx+UiKMSYYuHBbciAwwfl5AtAsXvpE47AeyCIiee9WvgZgpZRnuYcuCBEJEpHN8Y4gF+6Q2xhz0vn5FJDb+Tk/cDxevhPOtETpSzillIdx/SWcMWY0MPp+72SMMSJi7vd6bQErpTxLMr6ES8TpG10Lzj/PONNDgIB4+Qo40xKlAVgp5VmS9yVcQuYC7Zyf2wFz4qW3dY6GqA5citdVkSDtglBKeZZknIghIlOBukAOETkBDASGANNFpANwFGjhzL4AaAwcBMKA9kmVrwFYKeVZknEehjGmdSKnGiSQ1wDd7qV8DcBKKQ/jPjPhNAArpTyLrgWhlFIW0QCslFIW0QXZlVLKItoCVkopq2gAVkopa7hRC1gcQ9dUahCRIOfcc6Xi6L+Lh5f79FZ7BldWWlIPH/138ZDSAKyUUhbRAKyUUhbRAJy6tJ9PJUT/XTyk9CWcUkpZRFvASillEQ3ASillEQ3AqUREGonIARE5KCJ9kr5CeToRGSciZ0Rkt9V1UdbQAJwKRMQLGAE8C5QGWotIaWtrpdKAX4BGVldCWUcDcOqoBhw0xvyfMeY68BsQaHGdlMWMMcHABavroayjATh15AeOx/t+wpmmlHqIaQBWSimLaABOHSFAQLzvBZxpSqmHmAbg1LEJKCYiRUQkHdAKmGtxnZRSFtMAnAqMMdFAd2ARsA+YbozZY22tlNVEZCqwDighIidEpIPVdVKpS6ciK6WURbQFrJRSFtEArJRSFtEArJRSFtEArJRSFtEArJRSFtEArJRSFtEArJRSFvl/tVoC8t5NfAMAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"under_test_predictions = {'Ground Truth': test_labels,\n                          'Undersampling Classifier prediction': predict_test.ravel().astype(str)}\n\ndf_test_predictions = pd.DataFrame(under_test_predictions)\ndf_test_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:21:36.380403Z","iopub.execute_input":"2021-12-01T17:21:36.381222Z","iopub.status.idle":"2021-12-01T17:21:36.393781Z","shell.execute_reply.started":"2021-12-01T17:21:36.381171Z","shell.execute_reply":"2021-12-01T17:21:36.392829Z"},"trusted":true},"execution_count":136,"outputs":[{"execution_count":136,"output_type":"execute_result","data":{"text/plain":"   Ground Truth Undersampling Classifier prediction\n0             0                          0.22006375\n1             1                           0.9999994\n2             0                       5.0115576e-13\n3             1                           0.9999515\n4             0                          0.80792683","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Ground Truth</th>\n      <th>Undersampling Classifier prediction</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0.22006375</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0.9999994</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>5.0115576e-13</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0.9999515</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0.80792683</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"under_calib_predictions = {'Ground Truth': calib_labels,\n                           'Undersampling Classifier prediction': predict_calib.ravel().astype(str)}\n\ndf_calib_predictions = pd.DataFrame(under_calib_predictions)\ndf_calib_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:21:53.198269Z","iopub.execute_input":"2021-12-01T17:21:53.198536Z","iopub.status.idle":"2021-12-01T17:21:53.213850Z","shell.execute_reply.started":"2021-12-01T17:21:53.198510Z","shell.execute_reply":"2021-12-01T17:21:53.213098Z"},"trusted":true},"execution_count":137,"outputs":[{"execution_count":137,"output_type":"execute_result","data":{"text/plain":"   Ground Truth Undersampling Classifier prediction\n0             1                           0.9874215\n1             0                         0.008577817\n2             0                         0.004195224\n3             1                           0.9955598\n4             1                          0.07049355","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Ground Truth</th>\n      <th>Undersampling Classifier prediction</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0.9874215</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0.008577817</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0.004195224</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0.9955598</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0.07049355</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df_calib_predictions.to_csv('/kaggle/working/under_calib_predictions.csv', index=False)\ndf_test_predictions.to_csv('/kaggle/working/under_test_predictions.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T17:21:59.808618Z","iopub.execute_input":"2021-12-01T17:21:59.809173Z","iopub.status.idle":"2021-12-01T17:21:59.820788Z","shell.execute_reply.started":"2021-12-01T17:21:59.809137Z","shell.execute_reply":"2021-12-01T17:21:59.820036Z"},"trusted":true},"execution_count":138,"outputs":[]}]}