validation_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version: 2.9.1\n",
      "Num GPUs Available:  0\n",
      "Models Availabe for:\n",
      " ['model_checkpoint_base_JUN.h5', 'model_checkpoint_GC_JUN.h5', 'model_checkpoint_base_CTCF.h5', 'model_checkpoint_GC_MAX.h5', 'model_checkpoint_base_CEBPB.h5', 'model_checkpoint_GC_CEBPB.h5', 'model_checkpoint_GC_CTCF.h5', 'model_checkpoint_base_REST.h5', 'model_checkpoint_CB_MYC.h5', 'model_checkpoint_CB_JUN.h5', 'model_checkpoint_base_FOSL2.h5', 'model_checkpoint_CB_MAX.h5', 'model_checkpoint_CB_CEBPB.h5', 'model_checkpoint_GC_JUNB.h5', 'model_checkpoint_CB_JUNB.h5', 'model_checkpoint_GC_MYC.h5', 'model_checkpoint_base_MYC.h5', 'model_checkpoint_GC_REST.h5', 'model_checkpoint_base_MAX.h5', 'model_checkpoint_base_JUNB.h5', 'model_checkpoint_CB_REST.h5', 'model_checkpoint_CB_FOSL2.h5', 'model_checkpoint_CB_CTCF.h5', 'model_checkpoint_GC_FOSL2.h5']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-14 15:38:58.252488: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.252650: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.252766: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.252882: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.252995: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.253104: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.253215: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.253325: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory\n",
      "2022-06-14 15:38:58.253343: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n",
      "2022-06-14 15:38:58.255859: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-06-14 15:38:58.257644: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.layers import Layer,Input, Dense,LSTM, Dropout, Activation, Concatenate, Flatten, BatchNormalization, Conv1D, MaxPooling1D,Lambda,multiply,Permute,Reshape,RepeatVector\n",
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "\n",
    "print('Tensorflow version:', tf.__version__)\n",
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n",
    "\n",
    "config = ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)\n",
    "\n",
    "import os\n",
    "#print('Data Availabe for:\\n', os.listdir('data/15tf/15tf_fasta_1000_offset_pos'))\n",
    "print('Models Availabe for:\\n', os.listdir('checkpoints'))\n",
    "#print('Subset data Availabe for:\\n', os.listdir('data_subset'))\n",
    "\n",
    "# Recreate the exact same model, including its weights and the optimizer\n",
    "# model = tf.keras.models.load_model('train_models/checkpoints/model_checkpoint_MYC.h5')\n",
    "# Show the model architecture\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB CEBPB - roc 0.9993473209972776 -pr 0.9990500690051347\n",
      "MAX (13542, 1000, 4)\n",
      "MAX CEBPB - roc 0.42541493611907577 -pr 0.1989418208143155\n",
      "MYC (8267, 1000, 4)\n",
      "MYC CEBPB - roc 0.4677414808760112 -pr 0.23405950533210698\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB CEBPB - roc 0.5725765142573286 -pr 0.5624918166241293\n",
      "REST (10417, 1000, 4)\n",
      "REST CEBPB - roc 0.33966939392915796 -pr 0.28667175885209584\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF CEBPB - roc 0.40038168475168473 -pr 0.35926107337906893\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 CEBPB - roc 0.555328146475446 -pr 0.5342720452287762\n",
      "JUN (23887, 1000, 4)\n",
      "JUN CEBPB - roc 0.5663311410807013 -pr 0.5311706135447873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB MAX - roc 0.553026872832261 -pr 0.5179981026574977\n",
      "MAX (13542, 1000, 4)\n",
      "MAX MAX - roc 0.9973359659066395 -pr 0.9833171044958968\n",
      "MYC (8267, 1000, 4)\n",
      "MYC MAX - roc 0.9718835029809152 -pr 0.884647011388108\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB MAX - roc 0.6267329959138502 -pr 0.5833715759824956\n",
      "REST (10417, 1000, 4)\n",
      "REST MAX - roc 0.6097300524295487 -pr 0.4668700420639166\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF MAX - roc 0.7529279920654987 -pr 0.6865862187296672\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 MAX - roc 0.6257802360360292 -pr 0.5769717094905195\n",
      "JUN (23887, 1000, 4)\n",
      "JUN MAX - roc 0.6245368113553404 -pr 0.562462649568292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB MYC - roc 0.5651382110963494 -pr 0.5324834435055006\n",
      "MAX (13542, 1000, 4)\n",
      "MAX MYC - roc 0.9705818758408611 -pr 0.895553725357179\n",
      "MYC (8267, 1000, 4)\n",
      "MYC MYC - roc 0.9941902451087881 -pr 0.9741368544444436\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB MYC - roc 0.6785060475182634 -pr 0.6365637104034775\n",
      "REST (10417, 1000, 4)\n",
      "REST MYC - roc 0.6268566419908346 -pr 0.482539532175999\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF MYC - roc 0.7203247472866533 -pr 0.6319365557176087\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 MYC - roc 0.6832766413814014 -pr 0.6346286142992701\n",
      "JUN (23887, 1000, 4)\n",
      "JUN MYC - roc 0.6763282885110115 -pr 0.6130405864120304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB JUNB - roc 0.5 -pr 0.492563169747816\n",
      "MAX (13542, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX JUNB - roc 0.5 -pr 0.2315758381332152\n",
      "MYC (8267, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MYC JUNB - roc 0.5 -pr 0.24265150598766178\n",
      "JUNB (21498, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JUNB JUNB - roc 0.5 -pr 0.4891617824913945\n",
      "REST (10417, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "REST JUNB - roc 0.5 -pr 0.3738120380147835\n",
      "CTCF (41113, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CTCF JUNB - roc 0.5 -pr 0.4166808552039501\n",
      "FOSL2 (34556, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOSL2 JUNB - roc 0.5 -pr 0.4819423544391712\n",
      "JUN (23887, 1000, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JUN JUNB - roc 0.5 -pr 0.46573449993720434\n",
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB REST - roc 0.5265780561768275 -pr 0.5048667766229762\n",
      "MAX (13542, 1000, 4)\n",
      "MAX REST - roc 0.707470710211338 -pr 0.38689155082123733\n",
      "MYC (8267, 1000, 4)\n",
      "MYC REST - roc 0.6950229808896262 -pr 0.39781776943724845\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB REST - roc 0.5374681547778415 -pr 0.5137563241999179\n",
      "REST (10417, 1000, 4)\n",
      "REST REST - roc 0.9997672492443278 -pr 0.9995663589168552\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF REST - roc 0.7381502479280997 -pr 0.6277876501959041\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 REST - roc 0.5404482918133858 -pr 0.5063555339006481\n",
      "JUN (23887, 1000, 4)\n",
      "JUN REST - roc 0.5408770172686117 -pr 0.49181809477492505\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB CTCF - roc 0.5459572910276957 -pr 0.5245521358675166\n",
      "MAX (13542, 1000, 4)\n",
      "MAX CTCF - roc 0.6794448791072262 -pr 0.4862321334915193\n",
      "MYC (8267, 1000, 4)\n",
      "MYC CTCF - roc 0.6523403754556487 -pr 0.4369385054012508\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB CTCF - roc 0.5412346833460806 -pr 0.5139669043229579\n",
      "REST (10417, 1000, 4)\n",
      "REST CTCF - roc 0.6147214577378248 -pr 0.5011078761998482\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF CTCF - roc 0.998959010717965 -pr 0.9981199075558367\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 CTCF - roc 0.5378297275787716 -pr 0.5102711932732809\n",
      "JUN (23887, 1000, 4)\n",
      "JUN CTCF - roc 0.5291830064323686 -pr 0.4892337173668482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB FOSL2 - roc 0.5923763747640747 -pr 0.5748692697424683\n",
      "MAX (13542, 1000, 4)\n",
      "MAX FOSL2 - roc 0.5154284211522395 -pr 0.2640359621054602\n",
      "MYC (8267, 1000, 4)\n",
      "MYC FOSL2 - roc 0.5635726584819889 -pr 0.3250252763908231\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB FOSL2 - roc 0.9984777252988205 -pr 0.9977374962178245\n",
      "REST (10417, 1000, 4)\n",
      "REST FOSL2 - roc 0.47111928074662285 -pr 0.3622919049473192\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF FOSL2 - roc 0.4147950958451652 -pr 0.36726652020509226\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 FOSL2 - roc 0.9991416814953871 -pr 0.9987050271829412\n",
      "JUN (23887, 1000, 4)\n",
      "JUN FOSL2 - roc 0.9975136122160417 -pr 0.995948832760398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CEBPB (40407, 1000, 4)\n",
      "CEBPB JUN - roc 0.5511350048711671 -pr 0.544296178961535\n",
      "MAX (13542, 1000, 4)\n",
      "MAX JUN - roc 0.462073627680459 -pr 0.23214101698394773\n",
      "MYC (8267, 1000, 4)\n",
      "MYC JUN - roc 0.4964656023942229 -pr 0.27253928110500014\n",
      "JUNB (21498, 1000, 4)\n",
      "JUNB JUN - roc 0.9980793634509224 -pr 0.9972633763555971\n",
      "REST (10417, 1000, 4)\n",
      "REST JUN - roc 0.44280447416872115 -pr 0.3425964130238304\n",
      "CTCF (41113, 1000, 4)\n",
      "CTCF JUN - roc 0.40741994945998383 -pr 0.36251983883225086\n",
      "FOSL2 (34556, 1000, 4)\n",
      "FOSL2 JUN - roc 0.9979639089443872 -pr 0.9973383906523391\n",
      "JUN (23887, 1000, 4)\n",
      "JUN JUN - roc 0.9991580165132091 -pr 0.9982042720175412\n",
      "GC\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "# Validation\n",
    "#from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import precision_recall_curve,roc_auc_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef, cohen_kappa_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "tfs = ['CEBPB', 'MAX', 'MYC', 'JUNB', 'REST','CTCF', 'FOSL2','JUN']\n",
    "#tfs = ['CEBPB', 'MAX']\n",
    "cross_model_roc = pd.DataFrame(index=tfs, columns=tfs)\n",
    "cross_model_mcc = pd.DataFrame(index=tfs, columns=tfs)\n",
    "cross_model_kappa = pd.DataFrame(index=tfs, columns=tfs)\n",
    "cross_model_pr = pd.DataFrame(index=tfs, columns=tfs)\n",
    "\n",
    "mode = 'GC'\n",
    "#for saved_model in os.listdir('checkpoints'):\n",
    "for TF1 in tfs:\n",
    "    model = tf.keras.models.load_model('checkpoints/model_checkpoint_'+mode+'_'+TF1+'.h5')\n",
    "    #print(saved_model)\n",
    "    for TF in tfs:\n",
    "        x=np.load('/mnt/media/tfbind3/training_v5_'+mode+'/'+'x1_val_'+TF+'.npy')\n",
    "        y=np.load('/mnt/media/tfbind3/training_v5_'+mode+'/'+'y_val_'+TF+'.npy')\n",
    "        print(TF, x.shape)\n",
    "\n",
    "        y_pred = model.predict([x]).flatten() ###classification\n",
    "        y_act = y\n",
    "        y_mod = np.where(y_pred>0.5,1,0)\n",
    "\n",
    "        ###\n",
    "        try:\n",
    "            roc_auc_score_ = roc_auc_score(y_act, y_pred, average=None).tolist()\n",
    "        except:\n",
    "            roc_auc_score_ = 0.5\n",
    "        #fpr, tpr,_ = roc_curve(y_act, y_mod)\n",
    "\n",
    "        #precision, recall, _ = precision_recall_curve(y_act, y_mod)\n",
    "        average_precision_score_ = average_precision_score(y_act, y_pred, average=None).tolist()\n",
    "        mcc = matthews_corrcoef(y_act, y_mod)\n",
    "        kappa = cohen_kappa_score(y_act, y_mod)\n",
    "        \n",
    "        cross_model_roc[TF][TF1]=roc_auc_score_\n",
    "        cross_model_mcc[TF][TF1]=mcc\n",
    "        cross_model_kappa[TF][TF1]=kappa\n",
    "        cross_model_pr[TF][TF1]=average_precision_score_\n",
    "        print(TF, TF1, '- roc', roc_auc_score_, '-pr', average_precision_score_)\n",
    "    del model\n",
    "\n",
    "# Print\n",
    "print(mode)\n",
    "import time\n",
    "cross_model_pr.to_csv('220518_results/cross_model_pr_'+mode+'.csv')\n",
    "cross_model_roc.to_csv('220518_results/cross_model_roc_'+mode+'.csv')\n",
    "cross_model_mcc.to_csv('220518_results/cross_model_mcc_'+mode+'.csv')\n",
    "cross_model_kappa.to_csv('220518_results/cross_model_kappa_'+mode+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20220514-192846_cross_model_kappa_base.csv\n",
      "20220514-192846_cross_model_mcc_base.csv\n",
      "20220514-192846_cross_model_roc_base.csv\n",
      "20220514-190355_saliency_dict_ones_base.npy\n",
      "20220514-190356_saliency_dict_zero_base.npy\n",
      "20220514-192846_cross_model_pr_base.csv\n"
     ]
    }
   ],
   "source": [
    "# # Load cross model performances\n",
    "# import os \n",
    "# for f in os.listdir('results/'):\n",
    "#     if f.startswith('20220514'):a\n",
    "#         print(f)\n",
    "# cross_model_pr = pd.read_csv('results/20220514-192846_cross_model_pr_base.csv', index_col=0)\n",
    "# cross_model_roc= pd.read_csv('results/20220514-192846_cross_model_roc_base.csv', index_col=0)\n",
    "# cross_model_mcc = pd.read_csv('results/20220514-192846_cross_model_mcc_base.csv', index_col=0)\n",
    "# cross_model_kappa = pd.read_csv('results/20220514-192846_cross_model_kappa_base.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/client/session.py:1761: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n",
      "/home/dell15/miniconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: functions is not loaded, but a Lambda layer uses it. It may cause errors.\n",
      "  , UserWarning)\n"
     ]
    }
   ],
   "source": [
    "# Validation\n",
    "#from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import precision_recall_curve,roc_auc_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef, cohen_kappa_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import r2_score\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "\n",
    "config = ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)\n",
    "\n",
    "mode = 'base'\n",
    "TF = 'CEBPB'\n",
    "\n",
    "tfs = ['CEBPB', 'MAX', 'MYC', 'JUNB', 'REST','CTCF', 'FOSL2','JUN']\n",
    "stats_df = pd.DataFrame(columns=['roc', 'pr', 'mcc', 'kappa'], index=tfs)\n",
    "for TF in tfs:\n",
    "    model = tf.keras.models.load_model('checkpoints/model_checkpoint_'+mode+'_'+TF+'.h5')\n",
    "    x = np.load('/mnt/media/tfbind3/training_v5_'+mode+'/'+'x1_val_'+TF+'.npy')\n",
    "    y = np.load('/mnt/media/tfbind3/training_v5_'+mode+'/'+'y_val_'+TF+'.npy')\n",
    "\n",
    "    ##### predictions\n",
    "    y_pred = model.predict([x]).flatten() ###classification\n",
    "    y_act = y\n",
    "    y_mod = np.where(y_pred>0.5,1,0)\n",
    "\n",
    "    roc_auc_score_ = roc_auc_score(y_act, y_pred, average=None).tolist()\n",
    "    fpr, tpr,_ = roc_curve(y_act, y_pred, pos_label=1)\n",
    "    average_precision_score_ = average_precision_score(y_act, y_pred, average=None).tolist()\n",
    "    mcc = matthews_corrcoef(y_act, y_mod)\n",
    "    kappa = cohen_kappa_score(y_act, y_mod)\n",
    "\n",
    "    stats_df['roc'][TF]=roc_auc_score_\n",
    "    stats_df['pr'][TF]=average_precision_score_\n",
    "    stats_df['mcc'][TF]=mcc\n",
    "    stats_df['kappa'][TF]=kappa\n",
    "\n",
    "    del model\n",
    "\n",
    "stats_df.to_csv('220518_results/performance_stats_%s.csv'%(mode))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00,  6.97it/s]\n"
     ]
    }
   ],
   "source": [
    "# Data statistics\n",
    "import os\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import collections\n",
    "tfs = ['CEBPB', 'MAX', 'MYC', 'JUNB', 'REST','CTCF', 'FOSL2','JUN']\n",
    "sets = ['train_0', 'train_1', 'test_0', 'test_1', 'val_0', 'val_1']\n",
    "mode = 'CB'\n",
    "stats_data = pd.DataFrame(columns=sets, index=tfs)\n",
    "for set in tqdm(sets):\n",
    "    for TF in tfs:\n",
    "        y = np.load('/mnt/media/tfbind3/training_v5_'+mode+'/'+'y_%s_'%(set.split('_')[0])+TF+'.npy')\n",
    "        if set.endswith('0'):\n",
    "            stats_data[set][TF]=collections.Counter(y)[0]\n",
    "        if set.endswith('1'):\n",
    "            stats_data[set][TF]=collections.Counter(y)[1]\n",
    "        # print(TF, set, collections.Counter(y))\n",
    "\n",
    "stats_data.to_csv('results/stats_data_%s.csv'%(mode))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_0</th>\n",
       "      <th>train_1</th>\n",
       "      <th>test_0</th>\n",
       "      <th>test_1</th>\n",
       "      <th>val_0</th>\n",
       "      <th>val_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CEBPB</th>\n",
       "      <td>167634</td>\n",
       "      <td>167500</td>\n",
       "      <td>20927</td>\n",
       "      <td>20965</td>\n",
       "      <td>20898</td>\n",
       "      <td>20994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MAX</th>\n",
       "      <td>167220</td>\n",
       "      <td>167271</td>\n",
       "      <td>20932</td>\n",
       "      <td>20879</td>\n",
       "      <td>20905</td>\n",
       "      <td>20907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MYC</th>\n",
       "      <td>101875</td>\n",
       "      <td>101627</td>\n",
       "      <td>12554</td>\n",
       "      <td>12884</td>\n",
       "      <td>12760</td>\n",
       "      <td>12678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JUNB</th>\n",
       "      <td>162143</td>\n",
       "      <td>162202</td>\n",
       "      <td>20345</td>\n",
       "      <td>20198</td>\n",
       "      <td>20228</td>\n",
       "      <td>20316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REST</th>\n",
       "      <td>123230</td>\n",
       "      <td>123098</td>\n",
       "      <td>15423</td>\n",
       "      <td>15368</td>\n",
       "      <td>15302</td>\n",
       "      <td>15489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CTCF</th>\n",
       "      <td>179357</td>\n",
       "      <td>179278</td>\n",
       "      <td>22374</td>\n",
       "      <td>22455</td>\n",
       "      <td>22416</td>\n",
       "      <td>22414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FOSL2</th>\n",
       "      <td>164380</td>\n",
       "      <td>164724</td>\n",
       "      <td>20691</td>\n",
       "      <td>20447</td>\n",
       "      <td>20619</td>\n",
       "      <td>20519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JUN</th>\n",
       "      <td>163204</td>\n",
       "      <td>163160</td>\n",
       "      <td>20469</td>\n",
       "      <td>20327</td>\n",
       "      <td>20305</td>\n",
       "      <td>20491</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      train_0 train_1 test_0 test_1  val_0  val_1\n",
       "CEBPB  167634  167500  20927  20965  20898  20994\n",
       "MAX    167220  167271  20932  20879  20905  20907\n",
       "MYC    101875  101627  12554  12884  12760  12678\n",
       "JUNB   162143  162202  20345  20198  20228  20316\n",
       "REST   123230  123098  15423  15368  15302  15489\n",
       "CTCF   179357  179278  22374  22455  22416  22414\n",
       "FOSL2  164380  164724  20691  20447  20619  20519\n",
       "JUN    163204  163160  20469  20327  20305  20491"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==Performance Metrics==\n",
      "--------------------\n",
      "ROC_AUC 0.9703151317066452\n",
      "PR_AUC 0.971932002514446\n",
      "MCC 0.8346901620851059\n",
      "KAPPA 0.834595793470293\n",
      "\n",
      "==Confusion Matrix==\n",
      "[[387  30]\n",
      " [ 36 347]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAD6CAYAAAA/dPUzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAg/0lEQVR4nO3deXRU5f3H8feXABIsm4BAiRZQqkQFCiFsUgKiLOlpBFmlyvLTKMUFe1qhchDQHhA5WkWxKXUBTqsgFRQxVKkGFYhCEgsIKRRBwlZWUTbJ9vz+SLBpCmZCZubO8nmdwykzc5n53AY+Pve597ljzjlERKJVNa8DiIh4SSUoIlFNJSgiUU0lKCJRTSUoIlFNJSgiUa3CEjSzl83skJl9foHXzczmmNkOM9tkZh38H1NEJDCq+7DNfOB5YOEFXu8PtC791Rn4Q+n/fq9GjRq5Fi1a+BRSRMRX2dnZR5xzjX3dvsISdM59ZGYtvmeTFGChK7nq+hMzq29mzZxzB77vfVu0aEFWVpavOUVEzqu4uBgzw8wAMLPdlfnzvowEK9Ic2FPm8d7S5763BEWi0auf5vHWP/Z5HSOinDpygOqHc3nzmckX9ef9UYJ2nufOuxbPzFKBVIArr7zSDx8tUnXBLKZPdx0DoHPLy4LyeZHs1JED1G7YlEsbNaPZ5Zde9Pv4owT3AleUeRwH7D/fhs65ecA8gISEBC1ajgLhMPIJZjF1bnkZKe2bc3tnDQKqYv78+Yz73Tjmzp3L2LFjq/Re/ijB5cB9ZraIkhMiX1c0HyiR6XyFFw4jHxVT+Dh79iwPPPAA8+bNo3fv3vzsZz+r8ntWWIJm9hqQBDQys73AVKAGgHMuDUgHBgA7gNPAmCqnEr8LxojsfIWnghF/ycvLY/DgwWzYsIFJkybx+OOPU7161cdxvpwdHlHB6w4YX+UkUSpYh4vBGJGp8CSQNm/ezPbt21m2bBm33nqr397XH4fD4iMvDxdVUBKOiouL2bBhA507dyY5OZldu3bRoEEDv36GSjAALjS60+GiiO+OHz/OnXfeSXp6Ohs3buS6667zewGCSvCiVHQIe6HRnQpPxDebNm1i0KBB7N69m6effpr4+PiAfZZK0Edli6+iQ1iVncjF+/Of/0xqaioNGjRg9erVdO/ePaCfpxL00Vv/2MfWA98Q36yuSk4kgPLy8khMTGTx4sU0adIk4J9nXn3RUkJCggv1tcNlR3/nCnDxPV09TiUSefbu3cvu3bvp3r07xcXFFBcXX/TlL2aW7ZxL8HV7jQRLVXTmNr5ZXVLaN/cimkhEy8jIYNiwYVx66aVs376dGjVqUK1a8G51GtUlWNE8nw57RQLHOcfs2bP57W9/yzXXXMPSpUupUaNG0HNEbQm++mkejyzbDJSUnQpPJHi+/fZbRo4cydKlSxkyZAgvvfQSderU8SRLxJdgRdfszRh4g4pPJMguueQSateuzdNPP82ECRO+uxegFyK6BMuP9srSyE8k+BYtWkTHjh1p3bo1Cxcu9LT8zonoEjw3AtRoT8Rb+fn5/OY3v2HOnDncfffdzJs3LyQKECKwBMuf7Ojc8jIVoIiH9u/fz9ChQ1m7di0PPfQQs2bN8jrSf4moErzQyQ4R8caWLVu46aabOHnyJIsWLWLYsGFeR/ofYVuC33ddnw5/RUJDq1atSEpK4tFHHw3o+t+qCLsSPFd+uq5PJDSdOHGCqVOnMm3aNOrWrcuiRYu8jvS9wq4EzxWgCk8k9OTm5jJo0CC2b99OUlISP//5z72OVKGwK0EoGfFpDa9IaFmyZAljx46ldu3a/P3vf6dXr15eR/JJ8BboVdGrn+Yx7I+Z3x0Gi0joeOGFFxg6dCg33HADOTk5YVOAEEYjwfKHwSISOn7+85+zZ88epk+fTs2aNb2OUylhMxKE/xwGax5QxHtr1qxh7NixFBcXExcXx8yZM8OuACHMSlBEvOec49lnn6VXr158/PHHHDx40OtIVRIWJfjqp3maCxQJASdPnmTEiBFMmDCB5ORksrKyaNasmdexqiQsSvDcRdGaCxTx1qBBg1iyZAkzZsxg6dKl1KtXz+tIVRY2J0a0BljEO845zIxp06bx8MMP06dPH68j+U3YlKCIBF9hYSGPPPIIZsasWbPo1q2b15H8LuQPhzUfKOKNgwcPcvPNNzN79mxOnDiBV1/KFmghOxIsv0ZY84EiwZOZmcmQIUM4evQoCxYs4M477/Q6UsCEbAlqjbCIN7766iv69u1L48aNyczMpH379l5HCqiQLUHQGmGRYCooKKBGjRo0aNCAJUuWkJiYSIMGDbyOFXAhPycoIoH3r3/9i44dO/KXv/wFgL59+0ZFAUKIlqBOhogEz1tvvUVCQgL79u2jcePGXscJupAsQV0cLRJ4RUVFPPLII9x66638+Mc/Jicnh1tuucXrWEEXUnOC584Ibz3wjS6OFgmwDz74gJkzZ3L33XczZ84catWq5XUkT/g0EjSzfma2zcx2mNmk87xez8zeNrONZrbFzMZcTJhzBRjfrK5GgSIBcvz4cQBuvvlm1q1bx7x586K2AMGHEjSzGGAu0B+IB0aYWflvTBkPbHXOtQOSgKfMrFL31Dk3DxjfrK5ulyUSAM450tLS+NGPfkR2djYAXbvq6gtfRoKJwA7n3E7nXD6wCEgpt40D6ljJtyn/ADgGFFYmiOYBRQLn9OnTjBkzhnHjxtG9e3datmzpdaSQ4UsJNgf2lHm8t/S5sp4H2gD7gc3Ag8654sqG0TygiP998cUXdOvWjYULFzJt2jRWrFjBZZddVvEfjBK+nBix8zxXfhFhX+AfQG/gKmCVmX3snPvmv97ILBVIBbjySpWdSDAsWLCAvLw83nnnHfr37+91nJDjy0hwL3BFmcdxlIz4yhoDLHUldgC7gGvLv5Fzbp5zLsE5lxCN1yOJBEtRURG7du0C4NFHH2Xjxo0qwAvwpQQ3AK3NrGXpyY7hwPJy2+QBNwGYWRPgGmCnP4OKiG+OHj1KcnIy3bt35/jx41SvXp0rrrii4j8YpSo8HHbOFZrZfcC7QAzwsnNui5ndW/p6GvA4MN/MNlNy+DzROXfE1xDnzgx3bql5CpGqyMrKYvDgwRw4cIDnnnsuIu78HGg+XSztnEsH0ss9l1bm9/uBi77UXGeGRaruxRdfZPz48TRt2pQ1a9bQqVMnryOFhZBZNqczwyIXr7i4mGXLltGzZ0+ys7NVgJUQUsvmRKRydu3aRY0aNYiLi2Px4sXExsYSExPjdaywEjIjQRGpnJUrV9KxY0fuuusuAH7wgx+oAC+CSlAkzBQXFzN9+nSSk5O54ooreP75572OFNZ0OCwSRr766it+8YtfkJ6ezh133EFaWhq1a9f2OlZY00hQJIzExMSwZ88eXnjhBRYsWKAC9AONBEXCwBtvvMGAAQOoW7cuOTk5VK+uf7r+opGgSAg7e/Ys99xzD4MHD2bu3LkAKkA/0/+bIiEqLy+PwYMHs2HDBiZNmsSECRO8jhSRVIIiIejjjz9m4MCBFBQUsGzZMm699VavI0UsHQ6LhKBmzZrRpk0bsrKyVIAB5nkJ6us1RUocP36cp59+GuccV199NR999BGtW7f2OlbE87wEdfMEEdi4cSMJCQlMnDiRjRs3AlDybRUSaJ6XIOjmCRLdFi5cSNeuXTlz5gwffvgh7du39zpSVAmJEhSJVo888gijRo0iMTGRnJwcunXr5nWkqKOzwyIeSkpKorCwkBkzZuj6P494OhLUSRGJRh988AFz5swB4JZbbuHJJ59UAXrI0xLUSRGJJs45Zs2axc0338yf/vQnzp4963UkIQTmBHVSRKLB119/zaBBg5g0aRKDBw8mMzOTSy65xOtYguYERQIuPz+fbt26sX37dn7/+9/z4IMP6vKXEKISFAmwmjVrMmHCBK699lp69OjhdRwpx/PDYZFIlJ+fz4MPPsjbb78NwN13360CDFEqQRE/27dvH0lJScyZM4esrCyv40gFdDgs4kerV69m2LBhnDp1isWLFzN06FCvI0kFVIIifrJp0yb69OnD1VdfTUZGBvHx8V5HEh/ocFikipxzANxwww0899xzrF+/XgUYRlSCIlWwdetWEhMTyc3NxcwYN24cdevW9TqWVIJKUOQivf766yQmJpKXl8exY1r+Ga48K8Fjp/K1bljCUkFBAQ899BDDhg2jXbt25OTk0L17d69jyUXyrASPny4AtG5Yws9zzz3HM888w/33309GRgbNm+vvcDjz9Oyw1g1LODl79iyXXHIJ48eP55prriE5OdnrSOIHmhMUqYBzjmeeeYa2bdvy1Vdfcckll6gAI4hKUOR7nDx5khEjRvDQQw8RHx9PtWr6JxNp9BMVuYB//vOfdO7cmSVLlvDEE0+wdOlS6tWr53Us8TOfStDM+pnZNjPbYWaTLrBNkpn9w8y2mNmH/o0pEny//vWvOXz4MKtWrWLixIm6/VWEqvDEiJnFAHOBm4G9wAYzW+6c21pmm/rAC0A/51yemV0eoLwiAVVYWMipU6eoV68eL774IoWFhcTFxXkdSwLIl7PDicAO59xOADNbBKQAW8tsczuw1DmXB+CcO+TvoCKBdvDgQYYPH061atVYtWoVTZs29TqSBIEvh8PNgT1lHu8tfa6sHwMNzGy1mWWb2Z3+CigSDJmZmXTo0IFPP/2U0aNH6wRIFPHlJ32+iRBX7nF1oCOQDPQFppjZj//njcxSzSzLzLIKCgoqHVbE35xzPP/88/Ts2ZPY2FgyMzO54447vI4lQeRLCe4FrijzOA7Yf55t/uacO+WcOwJ8BLQr/0bOuXnOuQTnXEKNGjUuNrOI35w4cYLZs2fTr18/srKyaNfuf/7aSoTzpQQ3AK3NrKWZ1QSGA8vLbfMW0MPMqptZbaAzkOvfqCL+s3PnTgoKCqhbty7r1q3jzTffpH79+l7HEg9UWILOuULgPuBdSortdefcFjO718zuLd0mF/gbsAlYD7zonPs8cLFFLt6bb77JT37yE6ZOnQpA8+bNNQcYxXxaO+ycSwfSyz2XVu7xbGC2/6KJ+FdhYSFTpkzhiSeeICEhgXvvvdfrSBICdHt9iQqHDx9mxIgRvP/++6SmpvLss89Sq1Ytr2NJCFAJSlT497//zWeffcbLL7/MmDFjvI4jIUQlKBHLOcfq1avp1asXN9xwA19++SV16tTxOpaEGM0GS0Q6ffo0o0ePpnfv3qxcuRJABSjnpZGgRJwvvviCQYMGsXnzZqZPn07fvn29jiQhTCUoEeWdd95h5MiRVKtWjXfeeYf+/ft7HUlCnEpQIsq3337LVVddxV//+ldatmzpdRwJA5oTlLB35MgRli8vWcR02223sX79ehWg+EwjQQlrGzZsYPDgwRw7dowvv/yShg0bEhMT43UsCSMaCUpYcs7xpz/9iRtvvBGAjIwMGjZs6HEqCUcqQQk7zjlSU1NJTU0lKSmJ7OxsEhISvI4lYUolKGHHzIiLi2PKlCmkp6fTqFEjryNJGNOcoISNlStXEhsbS1JS0nd3gBGpKo0EJeQVFxczbdo0kpOTmTFjhtdxJMJoJCgh7dixY/ziF79g5cqVjBo1ihdeeMHrSBJhVIISsg4cOEC3bt3Yt28faWlppKam6rt/xe9UghKymjZtyoABA7jzzjvp3Lmz13EkQmlOUELKt99+y69+9St27dqFmTF37lwVoASUSlBCxu7du+nRowe///3veffdd72OI1FCh8MSEt577z1GjBhBYWEhb775JikpKV5HkiihkaB47q233qJfv3788Ic/JCsrSwUoQaUSFM/16dOHSZMm8cknn9C6dWuv40iUUQmKJzZu3EhKSgonT57k0ksvZcaMGVx66aVex5IopBKUoFu4cCFdunQhOzubvLw8r+NIlFMJStCcPXuWX/7yl4waNYquXbuSk5NDfHy817EkyqkEJWjuu+8+/vCHP/Dwww/z3nvvcfnll3sdSUSXyEjgOecwMyZPnsyAAQMYOHCg15FEvqORoASMc44nnniCIUOGUFxcTIsWLVSAEnJUghIQX3/9NQMHDuS3v/0t1atXJz8/3+tIIuelEhS/27x5MwkJCbzzzjs888wzvPbaa9SqVcvrWCLnpTlB8avCwkJSUlI4c+YMGRkZ330RkkioUgmKX+Tn5xMTE0P16tVZvHgxcXFxNGvWzOtYIhXS4bBU2b59+0hKSmL69OkAdOrUSQUoYUMlKFWyevVqOnTowKZNm7j++uu9jiNSaT6VoJn1M7NtZrbDzCZ9z3adzKzIzAb7L6KEIuccs2fPpk+fPlx22WWsX7+eoUOHeh1LpNIqLEEziwHmAv2BeGCEmf3PWqfS7WYBuhtmFNi2bRuTJ09m4MCBrF+/XsvfJGz5cmIkEdjhnNsJYGaLgBRga7nt7gfeADr5NaGElMOHD9O4cWOuvfZaNmzYQNu2bfXlRxLWfDkcbg7sKfN4b+lz3zGz5sBAIM1/0STULF68mFatWrF06VIA2rVrpwKUsOdLCZ7vb7kr9/gZYKJzruh738gs1cyyzCyroKDAx4jitYKCAh566CGGDx9O27Zt6dKli9eRRPzGl8PhvcAVZR7HAfvLbZMALCodFTQCBphZoXPuzbIbOefmAfMALvtRm/JFKiHowIEDDB06lDVr1vDAAw8we/Zsatas6XUsEb/xpQQ3AK3NrCWwDxgO3F52A+dcy3O/N7P5wIryBSjh6YMPPiAnJ4dXX32VESNGeB1HxO8qLEHnXKGZ3UfJWd8Y4GXn3BYzu7f0dc0DRhjnHP/85z9p06YNI0eOpFevXvzwhz/0OpZIQPi0bM45lw6kl3vuvOXnnBtd9VjilRMnTnDXXXfx9ttvs3nzZq666ioVoEQ0rR2W7+Tm5nLbbbexbds2Zs6cSatWrbyOJBJwKkEB4K9//StjxowhNjaWVatW0bt3b68jiQSFSlAAWLt2Lddffz1LliwhLi7O6zgiQaMSjGIHDx7k4MGDtG3blieffBLnnC5/kaijEoxS69atY8iQIdSpU4ctW7ZQo0YNryOJeEK30ooyzjnmzJlDz549iY2N5fXXXycmJsbrWCKeUQlGkTNnzjBy5EgefPBBBgwYQFZWFm3btvU6loinVIJRpGbNmhw7dowZM2awbNky6tev73UkEc9pTjAKLF++/Ltb3qenp1Otmv7bJ3KO/jVEsMLCQiZNmkRKSgq/+93vAFSAIuVoJBihDh06xPDhw8nIyOCee+7h6aef9jqSSEhSCUagzz//nH79+nH06FFeeeUVRo8e7XUkkZClY6MIFBcXR3x8PJmZmSpAkQqoBCPE6dOnmT59OmfPnqV+/fq89957tG/f3utYIiHPsxI8lV/o1UdHnB07dtC1a1emT5/O3//+d6/jiIQVT0eCKe2bV7yRfK/ly5eTkJDA3r17SU9PJzk52etIImHFsxK8tGZ1bu98pVcfHxHmzp1LSkoKV111FdnZ2fTr18/rSCJhR3OCYeymm25i/PjxrF27lhYtWngdRyQsqQTDzIYNG3j44YdxznHttdfy/PPPU6tWLa9jiYQtlWCYcM4xb948brzxRl5//XUOHz7sdSSRiKASDANnzpxh7Nix3HPPPfTu3Zvs7Gwuv/xyr2OJRASVYIhzzpGcnMz8+fOZOnUqK1asoGHDhl7HEokY5pzz5IMv+1Ebd2x3riefHW5WrlxJcXGxLn8R8YGZZTvnEnzdXmuHQ1BRURGPPfYY9erV41e/+hX9+/f3OpJIxNLhcIg5duwYP/vZz3jsscfYunUrXo3URaKFRoIhJCcnh9tuu439+/eTlpZGamoqZuZ1LJGIphIMEYcOHaJHjx40bNiQNWvW0KlTJ68jiUQFlaDHiouLqVatGpdffjmvvPIKvXv3plGjRl7HEokamhP00JdffkmXLl3429/+BsDQoUNVgCJBphL0yLvvvkvHjh3Ztm0bRUVFXscRiVoqwSArLi7m8ccfp3///jRv3pysrCxd/yfiIZVgkC1fvpxHH32UkSNHkpmZSevWrb2OJBLVdGIkSE6fPk3t2rVJSUlh5cqV9O3bV5e/iIQAjQSDYMGCBbRq1Yrt27djZvTr108FKBIifCpBM+tnZtvMbIeZTTrP6yPNbFPpr3Vm1s7/UcPP2bNnGTduHKNHjyY+Pp769et7HUlEyqmwBM0sBpgL9AfigRFmFl9us11AT+dcW+BxYJ6/g4abPXv28NOf/pS0tDQmTpzIe++9p9tfiYQgX+YEE4EdzrmdAGa2CEgBtp7bwDm3rsz2nwBx/gwZjp566ilyc3N54403GDRokNdxROQCfDkcbg7sKfN4b+lzF/J/wMqqhApXxcXFHDx4EICZM2eSnZ2tAhQJcb6U4Plm8M97axMz60VJCU68wOupZpZlZlkFBQW+pwwDx48fZ+DAgfTo0YNTp04RGxury19EwoAvJbgXuKLM4zhgf/mNzKwt8CKQ4pw7er43cs7Nc84lOOcSatSocTF5Q9KmTZtISEggPT2d++67j9q1a3sdSUR85EsJbgBam1lLM6sJDAeWl93AzK4ElgJ3OOe2+z9m6Przn/9Mly5dOH36NKtXr+aBBx7Q5S8iYaTCEyPOuUIzuw94F4gBXnbObTGze0tfTwMeBRoCL5QWQGFlbm8droqKipg7dy6dOnVi8eLFNG3a1OtIIlJJ+o6Ri7B3715iY2Np2LAhR44coV69ekTS4b1IOKvsd4xoxUglZWRk0KFDB+69914AGjVqpAIUCWMqQR8553jyySfp06cPjRo14rHHHvM6koj4gW6g4INvvvmGMWPGsHTpUoYMGcJLL71EnTp1vI4lIn6gkaAPzpw5Q3Z2Nk899RSLFy9WAYpEEI0Ev8eqVavo1asXTZo0ITc3l9jYWK8jiYifaSR4HgUFBUyYMIFbbrmFP/7xjwAqQJEIpZFgOQcOHGDIkCGsXbuWCRMmkJqa6nUkEQkglWAZ69atY9CgQZw4cYLXXnuN4cOHex1JRAJMJVhGrVq1aNKkCe+//z7XXXed13FEJAiifk7wxIkTLFiwAIAOHTrw2WefqQBFokhUl2Bubi6JiYmMHTuW3NySJXzVqkX1/yUiUSdq/8UvWbKExMREjh49yqpVq2jTpo3XkUTEA1FZglOmTGHo0KFcf/315OTk0Lt3b68jiYhHorIEr7vuOsaPH8+HH35IXFzUfx2KSFSLmltprV27li+//JKRI0cG7TNFJPh0K61ynHM8++yzJCUlMXPmTAoLC72OJCIhJKJL8OTJk9x+++1MmDCBAQMGsGbNGqpX16WRIvIfEdsIZ86coWvXrmzdupUZM2YwceJEXf4iIv8jYkswNjaW22+/nU6dOtGnTx+v44hIiIqoEyOFhYVMmTKF5ORkbrzxRr++t4iEh8qeGImYkeChQ4cYPnw4GRkZxMTEqARFxCcRUYKZmZkMGTKEo0ePMn/+fEaNGuV1JBEJE2FfgllZWfTs2ZO4uDgyMzNp376915FEJIyE/enSDh06MHXqVLKzs1WAIlJpYVmCO3bsoE+fPuzZs4dq1aoxefJkGjRo4HUsEQlDYVeCy5cvJyEhgc8++4zdu3d7HUdEwlzYlGBRURGTJ08mJSWFq6++muzsbJ0BFpEqC5sSnDVrFjNmzOCuu+5izZo1tGjRwutIIhIBQv7scFFRETExMdx///20atVKX34kIn4VsiNB5xxpaWl0796d06dPU6dOHRWgiPhdSJbgmTNnGDNmDOPGjaNBgwbk5+d7HUlEIlTIleDOnTvp1q0bCxcuZOrUqaxYsYL69et7HUtEIlTIzQmOHTuW3bt3s2LFCgYMGOB1HBGJcCFRgkVFReTn5xMbG8vLL7+MmdGyZUuvY4lIFPDpcNjM+pnZNjPbYWaTzvO6mdmc0tc3mVkHXwMcPXqU5ORkRo0ahXOOVq1aqQBFJGgqLEEziwHmAv2BeGCEmcWX26w/0Lr0VyrwB18+PCsri44dO5KRkaEbn4qIJ3w5HE4EdjjndgKY2SIgBdhaZpsUYKEruUPrJ2ZW38yaOecOXOhNz548Tvfu3WnSpAlr1qyhU6dOVdgNEZGL48vhcHNgT5nHe0ufq+w2/+XM8cP07NmTnJwcFaCIeMaXkaCd57ny9+T3ZRvMLJWSw2WAs6tWrfq8cePGPkQIO42AI16HCKBI3j/tW3gqu28/qswf9KUE9wJXlHkcB+y/iG1wzs0D5gGYWVZlvgcgnETyvkFk75/2LTxVZd98ORzeALQ2s5ZmVhMYDiwvt81y4M7Ss8RdgK+/bz5QRCRUVDgSdM4Vmtl9wLtADPCyc26Lmd1b+noakA4MAHYAp4ExgYssIuI/Pl0s7ZxLp6Toyj6XVub3Dhhfyc+eV8ntw0kk7xtE9v5p38LTRe+bZ987LCISCkLuBgoiIsEU8BIM5JI7r/mwbyNL92mTma0zs3Ze5LwYFe1bme06mVmRmQ0OZr6q8mX/zCzJzP5hZlvM7MNgZ7xYPvy9rGdmb5vZxtJ9C5s5fDN72cwOmdnnF3i98n3inAvYL0pOpHwBtAJqAhuB+HLbDABWUnKtYRfg00BmCvK+dQMalP6+fyTtW5ntPqBkvniw17n9/LOrT8mqqCtLH1/udW4/7tsjwKzS3zcGjgE1vc7u4/79FOgAfH6B1yvdJ4EeCX635M45lw+cW3JX1ndL7pxznwD1zaxZgHP5Q4X75pxb55z7qvThJ5RcPxkOfPm5AdwPvAEcCmY4P/Bl/24Hljrn8gCcc+Gyj77smwPqmJkBP6CkBAuDG/PiOOc+oiTvhVS6TwJdggFZchciKpv7/yj5L1Q4qHDfzKw5MBBII/z48rP7MdDAzFabWbaZ3Rm0dFXjy749D7ShZEHDZuBB51xxcOIFXKX7JND3E/TbkrsQ5HNuM+tFSQmGy3eE+rJvzwATnXNFJQOKsOLL/lUHOgI3AbFAppl94pzbHuhwVeTLvvUF/gH0Bq4CVpnZx865bwKcLRgq3SeBLkG/LbkLQT7lNrO2wItAf+fc0SBlqypf9i0BWFRagI2AAWZW6Jx7MygJq8bXv5dHnHOngFNm9hHQDgj1EvRl38YAT7iSSbQdZrYLuBZYH5yIAVX5PgnwJGZ1YCfQkv9M0l5Xbptk/nsic73Xk69+3LcrKVlF083rvP7et3Lbzye8Toz48rNrA7xfum1t4HPgeq+z+2nf/gBMK/19E2Af0Mjr7JXYxxZc+MRIpfskoCNBF8FL7nzct0eBhsALpSOmQhcGC9h93Lew5cv+OedyzexvwCagGHjROXfeyzJCiY8/u8eB+Wa2mZKymOicC4u7y5jZa0AS0MjM9gJTgRpw8X2iFSMiEtW0YkREoppKUESimkpQRKKaSlBEoppKUESimkpQRKKaSlBEoppKUESi2v8DTrtvRcTbn4gAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "###\n",
    "roc_auc_score_ = roc_auc_score(y_act, y_pred, average=None).tolist()\n",
    "fpr, tpr,_ = roc_curve(y_act, y_pred, pos_label=1)\n",
    "average_precision_score_ = average_precision_score(y_act, y_pred, average=None).tolist()\n",
    "mcc = matthews_corrcoef(y_act, y_mod)\n",
    "kappa = cohen_kappa_score(y_act, y_mod)\n",
    "\n",
    "fig, ax = plt.subplots(1, figsize=(5,4))\n",
    "ax.plot(fpr, tpr, alpha=1)\n",
    "ax.plot([0, 1], [0, 1], 'k--')\n",
    "ax.set_ylim([0.0, 1.01])\n",
    "ax.set_xlim([0.0, 1.01])\n",
    "\n",
    "\n",
    "print('==Performance Metrics==')\n",
    "print('--------------------')\n",
    "print('ROC_AUC', roc_auc_score_)\n",
    "print('PR_AUC', average_precision_score_)\n",
    "print('MCC', mcc)\n",
    "print('KAPPA', kappa)\n",
    "print('\\n==Confusion Matrix==')\n",
    "print(confusion_matrix(y_act, y_mod))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1, 0, 1, ..., 1, 0, 0]), array([1, 1, 1, ..., 1, 1, 1]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_act, y_mod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 67427, 0: 67199})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import collections\n",
    "collections.Counter(y_act)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ecb09b1366c5c74746f7d71cbb92674c9b708ed41682da618158b5308ff1abfd"
  },
  "kernelspec": {
   "display_name": "Python 3.7.13 ('tf_env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}