cortical_thickness_cross_species.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "626de92c-031e-482e-8833-f38687ecc92d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "import nibabel as nb\n",
    "from scipy import ndimage\n",
    "import scipy.stats as ss\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "205e977c-230c-49f4-8de2-094770135da4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('/Users/jayantadey/Human.parcellated_thickness.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "72ff36da-63c2-4ec8-9110-e2793d511a69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>sid</th>\n",
       "      <th>Markov.1</th>\n",
       "      <th>Markov.2</th>\n",
       "      <th>Markov.3</th>\n",
       "      <th>Markov.4</th>\n",
       "      <th>Markov.5</th>\n",
       "      <th>Markov.6</th>\n",
       "      <th>Markov.7</th>\n",
       "      <th>Markov.8</th>\n",
       "      <th>...</th>\n",
       "      <th>Schaefer217.191</th>\n",
       "      <th>Schaefer217.192</th>\n",
       "      <th>Schaefer217.193</th>\n",
       "      <th>Schaefer217.194</th>\n",
       "      <th>Schaefer217.195</th>\n",
       "      <th>Schaefer217.196</th>\n",
       "      <th>Schaefer217.197</th>\n",
       "      <th>Schaefer217.198</th>\n",
       "      <th>Schaefer217.199</th>\n",
       "      <th>Schaefer217.200</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>sub-OAS30876MRD4592</td>\n",
       "      <td>1.995032</td>\n",
       "      <td>2.203564</td>\n",
       "      <td>1.651978</td>\n",
       "      <td>1.969754</td>\n",
       "      <td>2.603206</td>\n",
       "      <td>2.295727</td>\n",
       "      <td>2.385144</td>\n",
       "      <td>2.719692</td>\n",
       "      <td>...</td>\n",
       "      <td>8.193966</td>\n",
       "      <td>7.736098</td>\n",
       "      <td>7.404804</td>\n",
       "      <td>7.431338</td>\n",
       "      <td>7.541022</td>\n",
       "      <td>7.433447</td>\n",
       "      <td>7.475594</td>\n",
       "      <td>7.460332</td>\n",
       "      <td>7.476401</td>\n",
       "      <td>7.609443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>sub-HBN_CBIC_NDARXC962XNK</td>\n",
       "      <td>2.557198</td>\n",
       "      <td>2.011555</td>\n",
       "      <td>2.175673</td>\n",
       "      <td>1.863080</td>\n",
       "      <td>2.473705</td>\n",
       "      <td>2.576267</td>\n",
       "      <td>2.392282</td>\n",
       "      <td>2.242582</td>\n",
       "      <td>...</td>\n",
       "      <td>6.927265</td>\n",
       "      <td>7.487809</td>\n",
       "      <td>7.098700</td>\n",
       "      <td>6.753360</td>\n",
       "      <td>6.841211</td>\n",
       "      <td>6.707229</td>\n",
       "      <td>7.189156</td>\n",
       "      <td>6.795299</td>\n",
       "      <td>6.823550</td>\n",
       "      <td>6.533783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>sub-AOMIC_0770</td>\n",
       "      <td>2.246607</td>\n",
       "      <td>2.295872</td>\n",
       "      <td>1.978412</td>\n",
       "      <td>2.069700</td>\n",
       "      <td>2.213602</td>\n",
       "      <td>2.449572</td>\n",
       "      <td>2.541624</td>\n",
       "      <td>2.777280</td>\n",
       "      <td>...</td>\n",
       "      <td>8.117053</td>\n",
       "      <td>7.775745</td>\n",
       "      <td>7.608771</td>\n",
       "      <td>7.579930</td>\n",
       "      <td>7.573511</td>\n",
       "      <td>7.607256</td>\n",
       "      <td>7.883723</td>\n",
       "      <td>7.630075</td>\n",
       "      <td>7.670835</td>\n",
       "      <td>7.354955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>sub-AOMIC_0344</td>\n",
       "      <td>2.219745</td>\n",
       "      <td>2.366237</td>\n",
       "      <td>2.036068</td>\n",
       "      <td>2.173696</td>\n",
       "      <td>2.508847</td>\n",
       "      <td>2.408997</td>\n",
       "      <td>2.430510</td>\n",
       "      <td>2.882698</td>\n",
       "      <td>...</td>\n",
       "      <td>7.908504</td>\n",
       "      <td>7.856069</td>\n",
       "      <td>7.756918</td>\n",
       "      <td>7.526684</td>\n",
       "      <td>7.410575</td>\n",
       "      <td>7.654072</td>\n",
       "      <td>7.952062</td>\n",
       "      <td>7.682724</td>\n",
       "      <td>7.444550</td>\n",
       "      <td>7.697996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>sub-Narratives_150</td>\n",
       "      <td>2.131236</td>\n",
       "      <td>2.432549</td>\n",
       "      <td>2.066158</td>\n",
       "      <td>2.352589</td>\n",
       "      <td>2.331565</td>\n",
       "      <td>2.799966</td>\n",
       "      <td>2.490590</td>\n",
       "      <td>2.818574</td>\n",
       "      <td>...</td>\n",
       "      <td>8.229232</td>\n",
       "      <td>7.847060</td>\n",
       "      <td>7.821367</td>\n",
       "      <td>7.739102</td>\n",
       "      <td>8.001608</td>\n",
       "      <td>7.803380</td>\n",
       "      <td>7.750091</td>\n",
       "      <td>7.730715</td>\n",
       "      <td>7.858832</td>\n",
       "      <td>7.892026</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 384 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                        sid  Markov.1  Markov.2  Markov.3  \\\n",
       "0           0        sub-OAS30876MRD4592  1.995032  2.203564  1.651978   \n",
       "1           1  sub-HBN_CBIC_NDARXC962XNK  2.557198  2.011555  2.175673   \n",
       "2           2             sub-AOMIC_0770  2.246607  2.295872  1.978412   \n",
       "3           3             sub-AOMIC_0344  2.219745  2.366237  2.036068   \n",
       "4           4         sub-Narratives_150  2.131236  2.432549  2.066158   \n",
       "\n",
       "   Markov.4  Markov.5  Markov.6  Markov.7  Markov.8  ...  Schaefer217.191  \\\n",
       "0  1.969754  2.603206  2.295727  2.385144  2.719692  ...         8.193966   \n",
       "1  1.863080  2.473705  2.576267  2.392282  2.242582  ...         6.927265   \n",
       "2  2.069700  2.213602  2.449572  2.541624  2.777280  ...         8.117053   \n",
       "3  2.173696  2.508847  2.408997  2.430510  2.882698  ...         7.908504   \n",
       "4  2.352589  2.331565  2.799966  2.490590  2.818574  ...         8.229232   \n",
       "\n",
       "   Schaefer217.192  Schaefer217.193  Schaefer217.194  Schaefer217.195  \\\n",
       "0         7.736098         7.404804         7.431338         7.541022   \n",
       "1         7.487809         7.098700         6.753360         6.841211   \n",
       "2         7.775745         7.608771         7.579930         7.573511   \n",
       "3         7.856069         7.756918         7.526684         7.410575   \n",
       "4         7.847060         7.821367         7.739102         8.001608   \n",
       "\n",
       "   Schaefer217.196  Schaefer217.197  Schaefer217.198  Schaefer217.199  \\\n",
       "0         7.433447         7.475594         7.460332         7.476401   \n",
       "1         6.707229         7.189156         6.795299         6.823550   \n",
       "2         7.607256         7.883723         7.630075         7.670835   \n",
       "3         7.654072         7.952062         7.682724         7.444550   \n",
       "4         7.803380         7.750091         7.730715         7.858832   \n",
       "\n",
       "   Schaefer217.200  \n",
       "0         7.609443  \n",
       "1         6.533783  \n",
       "2         7.354955  \n",
       "3         7.697996  \n",
       "4         7.892026  \n",
       "\n",
       "[5 rows x 384 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "79f09473-1d7c-4be1-93fb-87a260116c68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Dataset-ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-ABIDE1050339</td>\n",
       "      <td>18.0000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>ABIDE</td>\n",
       "      <td>50339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-ABIDE1050701</td>\n",
       "      <td>18.0000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>ABIDE</td>\n",
       "      <td>50701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sub-ABIDE1050445</td>\n",
       "      <td>18.1383</td>\n",
       "      <td>MALE</td>\n",
       "      <td>ABIDE</td>\n",
       "      <td>50445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sub-ABIDE1050459</td>\n",
       "      <td>18.1547</td>\n",
       "      <td>MALE</td>\n",
       "      <td>ABIDE</td>\n",
       "      <td>50459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sub-ABIDE1050341</td>\n",
       "      <td>18.2000</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>ABIDE</td>\n",
       "      <td>50341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID      Age     Sex Dataset Dataset-ID\n",
       "0  sub-ABIDE1050339  18.0000    MALE   ABIDE      50339\n",
       "1  sub-ABIDE1050701  18.0000    MALE   ABIDE      50701\n",
       "2  sub-ABIDE1050445  18.1383    MALE   ABIDE      50445\n",
       "3  sub-ABIDE1050459  18.1547    MALE   ABIDE      50459\n",
       "4  sub-ABIDE1050341  18.2000  FEMALE   ABIDE      50341"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sex = pd.read_excel('/Users/jayantadey/data_MRI/subjects_age_sex_data_MRI.xlsx')\n",
    "df_sex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a5c96b7-c722-4776-9fd8-746edd3a1e2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14465/14465 [00:12<00:00, 1144.69it/s]\n"
     ]
    }
   ],
   "source": [
    "X1 = []\n",
    "X2 = []\n",
    "y_human = []\n",
    "IDs = set(df['sid'])\n",
    "ref_IDs = set(df_sex['ID'])\n",
    "\n",
    "for subject in tqdm(IDs):\n",
    "    if subject in ref_IDs:\n",
    "        features = np.array(df[df['sid']==subject]).reshape(-1)[2:]\n",
    "        gender = list(df_sex[df_sex['ID']==subject]['Sex'])\n",
    "        sex = int(gender[0]=='FEMALE')\n",
    "             \n",
    "        X1.append(list(features[:182]))\n",
    "        X2.append(list(features[182:]))\n",
    "        y_human.append(sex)\n",
    "\n",
    "X1_human = np.array(X1)\n",
    "X2_human = np.array(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "70fb902e-dd1f-469d-aa77-fc529d2dcb17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10648, 182) (10648, 200)\n"
     ]
    }
   ],
   "source": [
    "print(X1_human.shape, X2_human.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "771d073c-18c6-4352-a4d3-b899f83021b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>participant_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>Markov.1</th>\n",
       "      <th>Markov.2</th>\n",
       "      <th>Markov.3</th>\n",
       "      <th>Markov.4</th>\n",
       "      <th>Markov.5</th>\n",
       "      <th>Markov.6</th>\n",
       "      <th>...</th>\n",
       "      <th>Schaefer217.191</th>\n",
       "      <th>Schaefer217.192</th>\n",
       "      <th>Schaefer217.193</th>\n",
       "      <th>Schaefer217.194</th>\n",
       "      <th>Schaefer217.195</th>\n",
       "      <th>Schaefer217.196</th>\n",
       "      <th>Schaefer217.197</th>\n",
       "      <th>Schaefer217.198</th>\n",
       "      <th>Schaefer217.199</th>\n",
       "      <th>Schaefer217.200</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>sub-1001</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "      <td>3.048436</td>\n",
       "      <td>3.908286</td>\n",
       "      <td>3.221595</td>\n",
       "      <td>3.615675</td>\n",
       "      <td>4.662432</td>\n",
       "      <td>3.707754</td>\n",
       "      <td>...</td>\n",
       "      <td>4.231826</td>\n",
       "      <td>4.908868</td>\n",
       "      <td>4.522730</td>\n",
       "      <td>2.294943</td>\n",
       "      <td>2.853976</td>\n",
       "      <td>3.406234</td>\n",
       "      <td>4.261370</td>\n",
       "      <td>4.131977</td>\n",
       "      <td>3.387978</td>\n",
       "      <td>3.451267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>sub-1002</td>\n",
       "      <td>1.783562</td>\n",
       "      <td>F</td>\n",
       "      <td>3.053520</td>\n",
       "      <td>3.748308</td>\n",
       "      <td>3.043567</td>\n",
       "      <td>3.764927</td>\n",
       "      <td>4.708283</td>\n",
       "      <td>4.060617</td>\n",
       "      <td>...</td>\n",
       "      <td>4.384853</td>\n",
       "      <td>4.849508</td>\n",
       "      <td>4.589500</td>\n",
       "      <td>2.443734</td>\n",
       "      <td>2.855187</td>\n",
       "      <td>3.344378</td>\n",
       "      <td>3.926697</td>\n",
       "      <td>3.477919</td>\n",
       "      <td>2.962553</td>\n",
       "      <td>3.474969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>sub-1003</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "      <td>3.211265</td>\n",
       "      <td>4.122524</td>\n",
       "      <td>3.374628</td>\n",
       "      <td>4.022762</td>\n",
       "      <td>4.759439</td>\n",
       "      <td>4.182558</td>\n",
       "      <td>...</td>\n",
       "      <td>4.570739</td>\n",
       "      <td>4.921833</td>\n",
       "      <td>4.770724</td>\n",
       "      <td>3.106145</td>\n",
       "      <td>3.094785</td>\n",
       "      <td>3.350355</td>\n",
       "      <td>4.562199</td>\n",
       "      <td>4.212585</td>\n",
       "      <td>3.582792</td>\n",
       "      <td>3.827813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>sub-1004</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "      <td>3.004275</td>\n",
       "      <td>3.681716</td>\n",
       "      <td>3.227427</td>\n",
       "      <td>3.762712</td>\n",
       "      <td>4.555942</td>\n",
       "      <td>3.984013</td>\n",
       "      <td>...</td>\n",
       "      <td>4.264869</td>\n",
       "      <td>4.935628</td>\n",
       "      <td>4.505048</td>\n",
       "      <td>3.337418</td>\n",
       "      <td>2.892611</td>\n",
       "      <td>3.690076</td>\n",
       "      <td>4.095378</td>\n",
       "      <td>4.328465</td>\n",
       "      <td>3.763171</td>\n",
       "      <td>3.758017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>sub-1005</td>\n",
       "      <td>1.742466</td>\n",
       "      <td>M</td>\n",
       "      <td>2.868796</td>\n",
       "      <td>3.837011</td>\n",
       "      <td>2.997172</td>\n",
       "      <td>3.724171</td>\n",
       "      <td>4.537298</td>\n",
       "      <td>3.816082</td>\n",
       "      <td>...</td>\n",
       "      <td>4.154663</td>\n",
       "      <td>4.817727</td>\n",
       "      <td>4.695378</td>\n",
       "      <td>3.965287</td>\n",
       "      <td>3.219764</td>\n",
       "      <td>3.268439</td>\n",
       "      <td>4.115168</td>\n",
       "      <td>3.889531</td>\n",
       "      <td>3.271547</td>\n",
       "      <td>4.040183</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 386 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0 participant_id       age sex  Markov.1  Markov.2  Markov.3  \\\n",
       "0           0       sub-1001  1.756164   M  3.048436  3.908286  3.221595   \n",
       "1           1       sub-1002  1.783562   F  3.053520  3.748308  3.043567   \n",
       "2           2       sub-1003  1.756164   M  3.211265  4.122524  3.374628   \n",
       "3           3       sub-1004  1.756164   M  3.004275  3.681716  3.227427   \n",
       "4           4       sub-1005  1.742466   M  2.868796  3.837011  2.997172   \n",
       "\n",
       "   Markov.4  Markov.5  Markov.6  ...  Schaefer217.191  Schaefer217.192  \\\n",
       "0  3.615675  4.662432  3.707754  ...         4.231826         4.908868   \n",
       "1  3.764927  4.708283  4.060617  ...         4.384853         4.849508   \n",
       "2  4.022762  4.759439  4.182558  ...         4.570739         4.921833   \n",
       "3  3.762712  4.555942  3.984013  ...         4.264869         4.935628   \n",
       "4  3.724171  4.537298  3.816082  ...         4.154663         4.817727   \n",
       "\n",
       "   Schaefer217.193  Schaefer217.194  Schaefer217.195  Schaefer217.196  \\\n",
       "0         4.522730         2.294943         2.853976         3.406234   \n",
       "1         4.589500         2.443734         2.855187         3.344378   \n",
       "2         4.770724         3.106145         3.094785         3.350355   \n",
       "3         4.505048         3.337418         2.892611         3.690076   \n",
       "4         4.695378         3.965287         3.219764         3.268439   \n",
       "\n",
       "   Schaefer217.197  Schaefer217.198  Schaefer217.199  Schaefer217.200  \n",
       "0         4.261370         4.131977         3.387978         3.451267  \n",
       "1         3.926697         3.477919         2.962553         3.474969  \n",
       "2         4.562199         4.212585         3.582792         3.827813  \n",
       "3         4.095378         4.328465         3.763171         3.758017  \n",
       "4         4.115168         3.889531         3.271547         4.040183  \n",
       "\n",
       "[5 rows x 386 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel('/Users/jayantadey/Macaque.parcellated_thickness.xlsx')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dacca8b8-5409-4db4-a3b2-da930e503dbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sub-1001</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sub-1002</td>\n",
       "      <td>1.783562</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sub-1003</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sub-1004</td>\n",
       "      <td>1.756164</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sub-1005</td>\n",
       "      <td>1.742466</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  participant_id       age sex\n",
       "0       sub-1001  1.756164   M\n",
       "1       sub-1002  1.783562   F\n",
       "2       sub-1003  1.756164   M\n",
       "3       sub-1004  1.756164   M\n",
       "4       sub-1005  1.742466   M"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sex = pd.read_csv('/Users/jayantadey/spmmouse_segment/uwmadison.csv')\n",
    "df_sex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "90af8ea7-31c7-40c7-8aed-ba984c542330",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 592/592 [00:00<00:00, 3647.03it/s]\n"
     ]
    }
   ],
   "source": [
    "X1 = []\n",
    "X2 = []\n",
    "y_monkey = []\n",
    "IDs = set(df['participant_id'])\n",
    "ref_IDs = set(df_sex['participant_id'])\n",
    "\n",
    "for subject in tqdm(IDs):\n",
    "    if subject in ref_IDs:\n",
    "        features = np.array(df[df['participant_id']==subject]).reshape(-1)[4:]\n",
    "        gender = list(df_sex[df_sex['participant_id']==subject]['sex'])\n",
    "        sex = int(gender[0]=='F')\n",
    "             \n",
    "        X1.append(list(features[:182]))\n",
    "        X2.append(list(features[182:]))\n",
    "        y_monkey.append(sex)\n",
    "\n",
    "X1_monkey = np.array(X1)\n",
    "X2_monkey = np.array(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6a8310c7-e79b-4d7b-879e-fccf9c32beb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(592, 182) (592, 200)\n"
     ]
    }
   ],
   "source": [
    "print(X1_monkey.shape, X2_monkey.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "848eb6fd-3718-4b92-98ba-be171ab0e406",
   "metadata": {},
   "source": [
    "### Try random forest (trained on Humans, tested on monkeys)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad9b2af4-71df-4587-ab19-4efad733e37b",
   "metadata": {},
   "source": [
    "# Markov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "464c2761-da21-441c-a57b-cfde674d1452",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:37<00:00,  7.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is  0.5162162162162162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "X1_human = np.nan_to_num(X1_human)\n",
    "X2_human = np.nan_to_num(X2_human)\n",
    "X1_monkey = np.nan_to_num(X1_monkey)\n",
    "X2_monkey = np.nan_to_num(X2_monkey)\n",
    "\n",
    "reps = 5\n",
    "accuracy = 0.0\n",
    "\n",
    "for ii in tqdm(range(reps)):\n",
    "    x_train, x_test, y_train, y_test = train_test_split(\n",
    "                    X1_human, y_human, train_size=0.8, random_state=ii, stratify=y_human)\n",
    "    clf = RandomForestClassifier(n_estimators=1000, n_jobs=-1)\n",
    "    clf.fit(x_train,y_train)\n",
    "    accuracy += np.mean(clf.predict(X1_monkey)==y_monkey)\n",
    "\n",
    "print('Accuracy is ',accuracy/reps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "618bd14e-4809-48ec-9da9-620461820db8",
   "metadata": {},
   "source": [
    "# Schaefer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b32c9005-0eac-49bf-a7e3-3666cc368a3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:37<00:00,  7.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is  0.48040540540540544\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "reps = 5\n",
    "accuracy = 0.0\n",
    "\n",
    "for ii in tqdm(range(reps)):\n",
    "    x_train, x_test, y_train, y_test = train_test_split(\n",
    "                    X2_human, y_human, train_size=0.8, random_state=ii, stratify=y_human)\n",
    "    clf = RandomForestClassifier(n_estimators=1000, n_jobs=-1)\n",
    "    clf.fit(x_train,y_train)\n",
    "    accuracy += np.mean(clf.predict(X2_monkey)==y_monkey)\n",
    "\n",
    "print('Accuracy is ',accuracy/reps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "83dfe2e5-a234-446b-9d29-ce3f273f376d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.44763513513513514"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_monkey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4165e74b-c50e-41fc-8960-2aa1d75accd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from proglearn.deciders import SimpleArgmaxAverage\n",
    "from proglearn.progressive_learner import ProgressiveLearner\n",
    "from proglearn.transformers import (\n",
    "    NeuralClassificationTransformer,\n",
    "    TreeClassificationTransformer,\n",
    ")\n",
    "from proglearn.voters import TreeClassificationVoter, KNNClassificationVoter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "44c001fa-3e43-485a-8f42-646b2acd29ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "default_transformer_class = TreeClassificationTransformer\n",
    "default_transformer_kwargs = {\n",
    "    \"kwargs\": {\"max_depth\": 30, \"max_features\": \"sqrt\"}\n",
    "}\n",
    "\n",
    "default_voter_class = TreeClassificationVoter\n",
    "default_voter_kwargs = {}\n",
    "\n",
    "default_decider_class = SimpleArgmaxAverage\n",
    "\n",
    "progressive_learner = ProgressiveLearner(\n",
    "    default_transformer_class=default_transformer_class,\n",
    "    default_transformer_kwargs=default_transformer_kwargs,\n",
    "    default_voter_class=default_voter_class,\n",
    "    default_voter_kwargs=default_voter_kwargs,\n",
    "    default_decider_class=default_decider_class,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5c41b91a-7626-4e83-97ed-ee5b728a84b9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<proglearn.progressive_learner.ProgressiveLearner at 0x2d907d760>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_human = np.array(y_human)\n",
    "y_monkey = np.array(y_monkey)\n",
    "\n",
    "x_train2, x_test2, y_train2, y_test2 = train_test_split(\n",
    "                    X1_monkey, y_monkey, train_size=0.8, random_state=0, stratify=y_monkey)\n",
    "\n",
    "\n",
    "progressive_learner.add_task(\n",
    "            X=X1_human,\n",
    "            y=y_human,\n",
    "            task_id=0,\n",
    "            num_transformers=1000,\n",
    "            transformer_voter_decider_split=[0.67, 0.33, 0],\n",
    "            decider_kwargs={\n",
    "                \"classes\": np.unique(\n",
    "                    y_human\n",
    "                )\n",
    "            },\n",
    "        )\n",
    "\n",
    "progressive_learner.add_task(\n",
    "            X=x_train2,\n",
    "            y=y_train2,\n",
    "            task_id=1,\n",
    "            num_transformers=1000,\n",
    "            transformer_voter_decider_split=[0.67, 0.33, 0],\n",
    "            decider_kwargs={\n",
    "                \"classes\": np.unique(\n",
    "                    y_train2\n",
    "                )\n",
    "            },\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e97a411f-f098-4f1f-aa74-3c32eb450bbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multi-task accuracy  0.6050420168067226\n",
      "Single-task accuracy  0.6470588235294118\n"
     ]
    }
   ],
   "source": [
    "multitask_label = progressive_learner.predict(x_test2, task_id=1)\n",
    "singletask_label = progressive_learner.predict(x_test2, task_id=1, transformer_ids=[1])\n",
    "\n",
    "print('Multi-task accuracy ', np.mean(multitask_label==y_test2))\n",
    "print('Single-task accuracy ', np.mean(singletask_label==y_test2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2db2bddb-e598-465f-96b8-b1b77800dfdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<proglearn.progressive_learner.ProgressiveLearner at 0x32fe6e8b0>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progressive_learner = ProgressiveLearner(\n",
    "    default_transformer_class=default_transformer_class,\n",
    "    default_transformer_kwargs=default_transformer_kwargs,\n",
    "    default_voter_class=default_voter_class,\n",
    "    default_voter_kwargs=default_voter_kwargs,\n",
    "    default_decider_class=default_decider_class,\n",
    ")\n",
    "\n",
    "x_train2, x_test2, y_train2, y_test2 = train_test_split(\n",
    "                    X2_monkey, y_monkey, train_size=0.8, random_state=0, stratify=y_monkey)\n",
    "\n",
    "\n",
    "progressive_learner.add_task(\n",
    "            X=X2_human,\n",
    "            y=y_human,\n",
    "            task_id=0,\n",
    "            num_transformers=1000,\n",
    "            transformer_voter_decider_split=[0.67, 0.33, 0],\n",
    "            decider_kwargs={\n",
    "                \"classes\": np.unique(\n",
    "                    y_human\n",
    "                )\n",
    "            },\n",
    "        )\n",
    "\n",
    "progressive_learner.add_task(\n",
    "            X=x_train2,\n",
    "            y=y_train2,\n",
    "            task_id=1,\n",
    "            num_transformers=1000,\n",
    "            transformer_voter_decider_split=[0.67, 0.33, 0],\n",
    "            decider_kwargs={\n",
    "                \"classes\": np.unique(\n",
    "                    y_train2\n",
    "                )\n",
    "            },\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e9d7e679",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multi-task accuracy  0.6638655462184874\n",
      "Single-task accuracy  0.6554621848739496\n"
     ]
    }
   ],
   "source": [
    "multitask_label = progressive_learner.predict(x_test2, task_id=1)\n",
    "singletask_label = progressive_learner.predict(x_test2, task_id=1, transformer_ids=[1])\n",
    "\n",
    "print('Multi-task accuracy ', np.mean(multitask_label==y_test2))\n",
    "print('Single-task accuracy ', np.mean(singletask_label==y_test2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72895085",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}