manage_papers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e3642e59-8794-4357-b043-1874eade3553",
   "metadata": {},
   "source": [
    "# Manage Papers\n",
    "\n",
    "This notebook selects papers to read from whom and parses references.\n",
    "It also filters the indices of passed papers from the search results and creates the table `papers_passed` that is annotated as `papers_annotated` in the google spreadsheet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3abe721-73ae-455a-911f-cac8415dc1c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d146ade6-f440-4e18-9c33-c19614973d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir(\"/home/jovyan/work/BCI/SLR_EEG_motion_artifact_correction_methods/\")\n",
    "import sys\n",
    "import time\n",
    "from datetime import datetime\n",
    "from unidecode import unidecode\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f7d9578-52fe-49a6-92c7-8af92596d1a5",
   "metadata": {},
   "source": [
    "## Load Search Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2de1deb-6036-479a-bb45-fc58d6bf42bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35443219</td>\n",
       "      <td>Naturalistic viewing conditions can increase t...</td>\n",
       "      <td>Welke D, Vessel EA.</td>\n",
       "      <td>Neuroimage. 2022 Aug 1;256:119218. doi: 10.101...</td>\n",
       "      <td>Welke D</td>\n",
       "      <td>Neuroimage</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/04/20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/j.neuroimage.2022.119218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35278015</td>\n",
       "      <td>Zapline-plus: A Zapline extension for automati...</td>\n",
       "      <td>Klug M, Kloosterman NA.</td>\n",
       "      <td>Hum Brain Mapp. 2022 Jun 15;43(9):2743-2758. d...</td>\n",
       "      <td>Klug M</td>\n",
       "      <td>Hum Brain Mapp</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/03/12</td>\n",
       "      <td>PMC9120550</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1002/hbm.25832</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PMID                                              Title                  Authors                                           Citation First Author    Journal/Book  Publication Year Create Date       PMCID NIHMS ID                               DOI\n",
       "1  35443219  Naturalistic viewing conditions can increase t...      Welke D, Vessel EA.  Neuroimage. 2022 Aug 1;256:119218. doi: 10.101...      Welke D      Neuroimage              2022  2022/04/20         NaN      NaN  10.1016/j.neuroimage.2022.119218\n",
       "2  35278015  Zapline-plus: A Zapline extension for automati...  Klug M, Kloosterman NA.  Hum Brain Mapp. 2022 Jun 15;43(9):2743-2758. d...       Klug M  Hum Brain Mapp              2022  2022/03/12  PMC9120550      NaN                 10.1002/hbm.25832"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"Search/all_searchresults.csv\")\n",
    "df.index += 1  # increment index such that it fits with the used indices\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "401f5c46-1864-4028-9f78-328b4e022e7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(90, 11)\n"
     ]
    }
   ],
   "source": [
    "passed = !ls Results/PASS\n",
    "passed_idxs = sorted([int(paper.split(\"]_\")[0].split(\"[\")[-1]) for paper in passed])\n",
    "df = df.loc[passed_idxs]\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8d675f0-be41-408f-ba7f-5bfcb8f73fb7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a0b876ed-f0ab-4a4b-a9b8-47891f42cd14",
   "metadata": {},
   "source": [
    "## Create unique IDs for the References in the style of 'Jung01b'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2abdd0bf-5f8e-49dd-8c00-ce1a737edac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create unique IDs\n",
    "def create_ref_id(row):\n",
    "    # print(row)\n",
    "    author = unidecode(row[\"First Author\"].split(\" \")[0])\n",
    "    if author in [\"van\", \"von\", \"de\", \"Abu\"]:\n",
    "        author = unidecode(row[\"First Author\"].split(\" \")[1])\n",
    "    year = str(row[\"Publication Year\"])[2:]\n",
    "    return author + year\n",
    "\n",
    "df[\"Ref\"] = df.apply(create_ref_id, axis=1)\n",
    "dupl = pd.value_counts(df[\"Ref\"])\n",
    "duplicates = dict({a: 0 for a in dupl[dupl>1].index})\n",
    "for idx, ref in df[\"Ref\"].iteritems():\n",
    "    if ref in duplicates.keys():\n",
    "        df.loc[idx, \"Ref\"] = ref + \"abcde\"[duplicates[ref]]\n",
    "        duplicates[ref] = duplicates[ref] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "146a3ca4-14e7-4136-861f-5d47b54ef4fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Ref</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>11559961</td>\n",
       "      <td>Analysis and visualization of single-trial eve...</td>\n",
       "      <td>Jung TP, Makeig S, Westerfield M, Townsend J, ...</td>\n",
       "      <td>Hum Brain Mapp. 2001 Nov;14(3):166-85. doi: 10...</td>\n",
       "      <td>Jung TP</td>\n",
       "      <td>Hum Brain Mapp</td>\n",
       "      <td>2001</td>\n",
       "      <td>2001/09/18</td>\n",
       "      <td>PMC6871967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1002/hbm.1050</td>\n",
       "      <td>Jung01a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>725</th>\n",
       "      <td>20824156</td>\n",
       "      <td>Imaging Brain Dynamics Using Independent Compo...</td>\n",
       "      <td>Jung TP, Makeig S, McKeown MJ, Bell AJ, Lee TW...</td>\n",
       "      <td>Proc IEEE Inst Electr Electron Eng. 2001 Jul 1...</td>\n",
       "      <td>Jung TP</td>\n",
       "      <td>Proc IEEE Inst Electr Electron Eng</td>\n",
       "      <td>2001</td>\n",
       "      <td>2010/09/09</td>\n",
       "      <td>PMC2932458</td>\n",
       "      <td>HHMIMS197143</td>\n",
       "      <td>10.1109/5.939827</td>\n",
       "      <td>Jung01b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         PMID                                              Title                                            Authors                                           Citation First Author                        Journal/Book  Publication Year Create Date       PMCID      NIHMS ID               DOI      Ref\n",
       "724  11559961  Analysis and visualization of single-trial eve...  Jung TP, Makeig S, Westerfield M, Townsend J, ...  Hum Brain Mapp. 2001 Nov;14(3):166-85. doi: 10...      Jung TP                      Hum Brain Mapp              2001  2001/09/18  PMC6871967           NaN  10.1002/hbm.1050  Jung01a\n",
       "725  20824156  Imaging Brain Dynamics Using Independent Compo...  Jung TP, Makeig S, McKeown MJ, Bell AJ, Lee TW...  Proc IEEE Inst Electr Electron Eng. 2001 Jul 1...      Jung TP  Proc IEEE Inst Electr Electron Eng              2001  2010/09/09  PMC2932458  HHMIMS197143  10.1109/5.939827  Jung01b"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the refs must be unique\n",
    "df[df[\"First Author\"].str.contains(\"Jung\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "16bc8e5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Ref</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>35459033</td>\n",
       "      <td>Improved Cognitive Vigilance Assessment after ...</td>\n",
       "      <td>Abu Farha N, Al-Shargie F, Tariq U, Al-Nashash H.</td>\n",
       "      <td>Sensors (Basel). 2022 Apr 15;22(8):3051. doi: ...</td>\n",
       "      <td>Abu Farha N</td>\n",
       "      <td>Sensors (Basel)</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/04/23</td>\n",
       "      <td>PMC9033092</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3390/s22083051</td>\n",
       "      <td>Farha22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        PMID                                              Title                                            Authors                                           Citation First Author     Journal/Book  Publication Year Create Date       PMCID NIHMS ID                DOI      Ref\n",
       "10  35459033  Improved Cognitive Vigilance Assessment after ...  Abu Farha N, Al-Shargie F, Tariq U, Al-Nashash H.  Sensors (Basel). 2022 Apr 15;22(8):3051. doi: ...  Abu Farha N  Sensors (Basel)              2022  2022/04/23  PMC9033092      NaN  10.3390/s22083051  Farha22"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the refs must be unique\n",
    "df[df[\"First Author\"].str.contains(\"Abu\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d21ccf16-1eb8-4691-939c-d76b68782bd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(df[\"Ref\"].unique()) == df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "143f1e51-dab1-469c-9c72-10bf8cd0eb51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Ref</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>35590859</td>\n",
       "      <td>Motion Artifacts Correction from Single-Channe...</td>\n",
       "      <td>Hossain MS, Chowdhury MEH, Reaz MBI, Ali SHM, ...</td>\n",
       "      <td>Sensors (Basel). 2022 Apr 21;22(9):3169. doi: ...</td>\n",
       "      <td>Hossain MS</td>\n",
       "      <td>Sensors (Basel)</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/05/20</td>\n",
       "      <td>PMC9102309</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3390/s22093169</td>\n",
       "      <td>Hossain22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>35459033</td>\n",
       "      <td>Improved Cognitive Vigilance Assessment after ...</td>\n",
       "      <td>Abu Farha N, Al-Shargie F, Tariq U, Al-Nashash H.</td>\n",
       "      <td>Sensors (Basel). 2022 Apr 15;22(8):3051. doi: ...</td>\n",
       "      <td>Abu Farha N</td>\n",
       "      <td>Sensors (Basel)</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/04/23</td>\n",
       "      <td>PMC9033092</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3390/s22083051</td>\n",
       "      <td>Farha22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>35093730</td>\n",
       "      <td>Automated Pipeline for Infants Continuous EEG ...</td>\n",
       "      <td>Fló A, Gennari G, Benjamin L, Dehaene-Lambertz G.</td>\n",
       "      <td>Dev Cogn Neurosci. 2022 Apr;54:101077. doi: 10...</td>\n",
       "      <td>Fló A</td>\n",
       "      <td>Dev Cogn Neurosci</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/01/30</td>\n",
       "      <td>PMC8804179</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/j.dcn.2022.101077</td>\n",
       "      <td>Flo22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>35085870</td>\n",
       "      <td>NEAR: An artifact removal pipeline for human n...</td>\n",
       "      <td>Kumaravel VP, Farella E, Parise E, Buiatti M.</td>\n",
       "      <td>Dev Cogn Neurosci. 2022 Apr;54:101068. doi: 10...</td>\n",
       "      <td>Kumaravel VP</td>\n",
       "      <td>Dev Cogn Neurosci</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022/01/27</td>\n",
       "      <td>PMC8800139</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1016/j.dcn.2022.101068</td>\n",
       "      <td>Kumaravel22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>34924988</td>\n",
       "      <td>A Toolbox and Crowdsourcing Platform for Autom...</td>\n",
       "      <td>Soghoyan G, Ledovsky A, Nekrashevich M, Martyn...</td>\n",
       "      <td>Front Neuroinform. 2021 Dec 2;15:720229. doi: ...</td>\n",
       "      <td>Soghoyan G</td>\n",
       "      <td>Front Neuroinform</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021/12/20</td>\n",
       "      <td>PMC8675888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3389/fninf.2021.720229</td>\n",
       "      <td>Soghoyan21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        PMID                                              Title                                            Authors                                           Citation  First Author       Journal/Book  Publication Year Create Date       PMCID NIHMS ID                        DOI          Ref\n",
       "8   35590859  Motion Artifacts Correction from Single-Channe...  Hossain MS, Chowdhury MEH, Reaz MBI, Ali SHM, ...  Sensors (Basel). 2022 Apr 21;22(9):3169. doi: ...    Hossain MS    Sensors (Basel)              2022  2022/05/20  PMC9102309      NaN          10.3390/s22093169    Hossain22\n",
       "10  35459033  Improved Cognitive Vigilance Assessment after ...  Abu Farha N, Al-Shargie F, Tariq U, Al-Nashash H.  Sensors (Basel). 2022 Apr 15;22(8):3051. doi: ...   Abu Farha N    Sensors (Basel)              2022  2022/04/23  PMC9033092      NaN          10.3390/s22083051      Farha22\n",
       "16  35093730  Automated Pipeline for Infants Continuous EEG ...  Fló A, Gennari G, Benjamin L, Dehaene-Lambertz G.  Dev Cogn Neurosci. 2022 Apr;54:101077. doi: 10...         Fló A  Dev Cogn Neurosci              2022  2022/01/30  PMC8804179      NaN  10.1016/j.dcn.2022.101077        Flo22\n",
       "17  35085870  NEAR: An artifact removal pipeline for human n...      Kumaravel VP, Farella E, Parise E, Buiatti M.  Dev Cogn Neurosci. 2022 Apr;54:101068. doi: 10...  Kumaravel VP  Dev Cogn Neurosci              2022  2022/01/27  PMC8800139      NaN  10.1016/j.dcn.2022.101068  Kumaravel22\n",
       "35  34924988  A Toolbox and Crowdsourcing Platform for Autom...  Soghoyan G, Ledovsky A, Nekrashevich M, Martyn...  Front Neuroinform. 2021 Dec 2;15:720229. doi: ...    Soghoyan G  Front Neuroinform              2021  2021/12/20  PMC8675888      NaN  10.3389/fninf.2021.720229   Soghoyan21"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d50efa8-fab9-45bc-a658-fd37f0027db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the table of passed papers into a file\n",
    "df.to_csv(\"Passed_papers/papers_passed.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50a25e43-40d3-41e6-90ea-be354e710576",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e229434a-512a-46f7-b051-74c95d351d76",
   "metadata": {},
   "source": [
    "## Partitionate the Papers for reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0939313e-84ca-4668-bd24-0d6ff3d9ef17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2, 13, 53, 41, 66, 30, 45, 43, 78, 89,  7, 26, 33, 63,  8, 16, 24,\n",
       "       56, 77, 42, 22,  6, 61, 48, 80, 54, 73, 79, 82,  3, 62, 75, 27, 18,\n",
       "       50, 51, 74, 59, 55,  4, 15, 17, 40, 38,  5, 81, 68,  0, 34, 28, 11,\n",
       "       35, 23, 52, 10, 31, 60, 57, 71,  1, 32, 76, 14, 85, 19, 29, 49, 87,\n",
       "       88, 69, 20, 84, 72, 25, 37, 46, 39, 65, 58, 12, 70, 36, 21, 83,  9,\n",
       "       86, 67, 64, 47, 44])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "rand_vec = np.random.permutation(df.shape[0])\n",
    "rand_vec"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "309cb869-547b-4184-8080-e8168dfa0493",
   "metadata": {},
   "source": [
    "## Papers for MST and for CS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1e6084cc-5d6c-49e3-9627-03d636331d99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([16, 86, 389, 281, 482, 228, 328, 300, 615, 725, 52, 194, 241, 451, 54, 106, 163, 405, 609, 293, 156, 47, 444, 347, 628, 392, 549, 624, 648, 17, 447, 551, 196, 126, 370, 377, 550, 420, 400, 35], dtype='int64')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Papers for Chris\n",
    "df.iloc[rand_vec[:40]].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "597f7e30-1618-4dca-9434-f938d7d970a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Ref</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>33328870</td>\n",
       "      <td>Refinement of High-Gamma EEG Features From TBI...</td>\n",
       "      <td>Li Y, Wang PT, Vaidya MP, Flint RD, Liu CY, Sl...</td>\n",
       "      <td>Front Neurosci. 2020 Nov 24;14:599010. doi: 10...</td>\n",
       "      <td>Li Y</td>\n",
       "      <td>Front Neurosci</td>\n",
       "      <td>2020</td>\n",
       "      <td>2020/12/17</td>\n",
       "      <td>PMC7732541</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3389/fnins.2020.599010</td>\n",
       "      <td>Li20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>32943621</td>\n",
       "      <td>Fully organic compliant dry electrodes self-ad...</td>\n",
       "      <td>Zhang L, Kumar KS, He H, Cai CJ, He X, Gao H, ...</td>\n",
       "      <td>Nat Commun. 2020 Sep 17;11(1):4683. doi: 10.10...</td>\n",
       "      <td>Zhang L</td>\n",
       "      <td>Nat Commun</td>\n",
       "      <td>2020</td>\n",
       "      <td>2020/09/18</td>\n",
       "      <td>PMC7499260</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/s41467-020-18503-8</td>\n",
       "      <td>Zhang20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>29492336</td>\n",
       "      <td>A new ICA-based fingerprint method for the aut...</td>\n",
       "      <td>Tamburro G, Fiedler P, Stone D, Haueisen J, Co...</td>\n",
       "      <td>PeerJ. 2018 Feb 23;6:e4380. doi: 10.7717/peerj...</td>\n",
       "      <td>Tamburro G</td>\n",
       "      <td>PeerJ</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018/03/02</td>\n",
       "      <td>PMC5826009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.7717/peerj.4380</td>\n",
       "      <td>Tamburro18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>29322469</td>\n",
       "      <td>Removal of muscular artifacts in EEG signals: ...</td>\n",
       "      <td>Frølich L, Dowding I.</td>\n",
       "      <td>Brain Inform. 2018 Mar;5(1):13-22. doi: 10.100...</td>\n",
       "      <td>Frølich L</td>\n",
       "      <td>Brain Inform</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018/01/12</td>\n",
       "      <td>PMC5893498</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1007/s40708-017-0074-6</td>\n",
       "      <td>Frolich18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>34833790</td>\n",
       "      <td>The Influence of Filters on EEG-ERP Testing: A...</td>\n",
       "      <td>Karpiel I, Kurasz Z, Kurasz R, Duch K.</td>\n",
       "      <td>Sensors (Basel). 2021 Nov 19;21(22):7711. doi:...</td>\n",
       "      <td>Karpiel I</td>\n",
       "      <td>Sensors (Basel)</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021/11/27</td>\n",
       "      <td>PMC8619013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.3390/s21227711</td>\n",
       "      <td>Karpiel21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         PMID                                              Title                                            Authors                                           Citation First Author     Journal/Book  Publication Year Create Date       PMCID NIHMS ID                         DOI         Ref\n",
       "104  33328870  Refinement of High-Gamma EEG Features From TBI...  Li Y, Wang PT, Vaidya MP, Flint RD, Liu CY, Sl...  Front Neurosci. 2020 Nov 24;14:599010. doi: 10...         Li Y   Front Neurosci              2020  2020/12/17  PMC7732541      NaN   10.3389/fnins.2020.599010        Li20\n",
       "117  32943621  Fully organic compliant dry electrodes self-ad...  Zhang L, Kumar KS, He H, Cai CJ, He X, Gao H, ...  Nat Commun. 2020 Sep 17;11(1):4683. doi: 10.10...      Zhang L       Nat Commun              2020  2020/09/18  PMC7499260      NaN  10.1038/s41467-020-18503-8     Zhang20\n",
       "275  29492336  A new ICA-based fingerprint method for the aut...  Tamburro G, Fiedler P, Stone D, Haueisen J, Co...  PeerJ. 2018 Feb 23;6:e4380. doi: 10.7717/peerj...   Tamburro G            PeerJ              2018  2018/03/02  PMC5826009      NaN          10.7717/peerj.4380  Tamburro18\n",
       "272  29322469  Removal of muscular artifacts in EEG signals: ...                              Frølich L, Dowding I.  Brain Inform. 2018 Mar;5(1):13-22. doi: 10.100...    Frølich L     Brain Inform              2018  2018/01/12  PMC5893498      NaN   10.1007/s40708-017-0074-6   Frolich18\n",
       "40   34833790  The Influence of Filters on EEG-ERP Testing: A...             Karpiel I, Kurasz Z, Kurasz R, Duch K.  Sensors (Basel). 2021 Nov 19;21(22):7711. doi:...    Karpiel I  Sensors (Basel)              2021  2021/11/27  PMC8619013      NaN           10.3390/s21227711   Karpiel21"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Papers for both of us\n",
    "df.iloc[rand_vec[40:45]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "915a68f7-e816-4cda-989d-d711d2f14143",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([640, 506, 8, 248, 207, 77, 259, 162, 384, 73, 235, 431, 406, 524, 10, 240, 583, 89, 707, 139, 216, 356, 723, 724, 509, 144, 684, 538, 171, 268, 333, 273, 465, 408, 82, 513, 265, 145, 672, 56, 713, 490, 455, 345, 301], dtype='int64')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Papers for Mat\n",
    "df.iloc[rand_vec[45:]].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d963960c-5677-47c9-b3c4-8b7dab20f4a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dc04e132-6b1f-4dd6-ac55-c95a9cef5ca2",
   "metadata": {},
   "source": [
    "## Create the bibtex-file\n",
    "\n",
    "Using a [webservice](https://scipython.com/blog/doi-to-bibtex/#comments) for the parsing."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9605d8d7-6669-4234-bfc5-58f61e633030",
   "metadata": {},
   "source": [
    "@Article{article,  \n",
    "\tauthor = \"Author1 LastName1 and Author2 LastName2 and Author3 LastName3\",  \n",
    "\ttitle = \"Article Title\",  \n",
    "\tvolume = \"30\",  \n",
    "\tnumber = \"30\",  \n",
    "\tpages = \"10127-10134\",  \n",
    "\tyear = \"2013\",  \n",
    "\tdoi = \"10.3389/fnins.2013.12345\",  \n",
    "\tURL = \"http://www.frontiersin.org/Journal/10.3389/fnins.2013.12345/abstract\",  \n",
    "\tjournal = \"Frontiers in Neuroscience\"  \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8060bdd5-b4be-4f4c-9163-bdbb3c2d87a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import urllib.request\n",
    "from urllib.error import HTTPError\n",
    "\n",
    "BASE_URL = 'http://dx.doi.org/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f6e3c530-9753-4180-8189-574f9402f39b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ref</th>\n",
       "      <th>DOI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Hossain22</td>\n",
       "      <td>10.3390/s22093169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Farha22</td>\n",
       "      <td>10.3390/s22083051</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Ref                DOI\n",
       "8   Hossain22  10.3390/s22093169\n",
       "10    Farha22  10.3390/s22083051"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\"Ref\", \"DOI\"]].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d633a238-a012-4091-aa09-469caf16dbfa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parsing Hossain22\n"
     ]
    }
   ],
   "source": [
    "reviewed_bibtex = str()\n",
    "\n",
    "for idx, row in df.iterrows():\n",
    "    print(f\"parsing {row['Ref']}\")\n",
    "    ref = row[\"Ref\"]\n",
    "    doi = row[\"DOI\"]\n",
    "    \n",
    "    # create specific doi-url\n",
    "    url = BASE_URL + doi\n",
    "    req = urllib.request.Request(url)\n",
    "    req.add_header('Accept', 'application/x-bibtex')\n",
    "    \n",
    "    # parse content\n",
    "    try:\n",
    "        with urllib.request.urlopen(req) as f:\n",
    "            bibtex = f.read().decode()\n",
    "    except HTTPError as e:\n",
    "        if e.code == 404:\n",
    "            print('DOI not found.')\n",
    "        else:\n",
    "            print('Service unavailable.')\n",
    "        print(row)\n",
    "\n",
    "    # replace the article id\n",
    "    original_id = bibtex.split(\"\\n\")[0].split(\"{\")[1]\n",
    "    bibtex = bibtex.split(\"{\")[0] + \"{\" + row[\"Ref\"] + \",\\n\" + \"\\n\".join(bibtex.split(\"\\n\")[1:])\n",
    "    \n",
    "    # append to bibtex file\n",
    "    reviewed_bibtex += bibtex\n",
    "    reviewed_bibtex += \"\\n\"\n",
    "    \n",
    "    break  # TODO remove break to parse all files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5f711f06-b367-442d-8f27-6389f42967bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@article{Hossain22,\n",
      "\tdoi = {10.3390/s22093169},\n",
      "\turl = {https://doi.org/10.3390%2Fs22093169},\n",
      "\tyear = 2022,\n",
      "\tmonth = {apr},\n",
      "\tpublisher = {{MDPI} {AG}},\n",
      "\tvolume = {22},\n",
      "\tnumber = {9},\n",
      "\tpages = {3169},\n",
      "\tauthor = {Md Shafayet Hossain and Muhammad E. H. Chowdhury and Mamun Bin Ibne Reaz and Sawal Hamid Md Ali and Ahmad Ashrif A. Bakar and Serkan Kiranyaz and Amith Khandakar and Mohammed Alhatou and Rumana Habib and Muhammad Maqsud Hossain},\n",
      "\ttitle = {Motion Artifacts Correction from Single-Channel {\n"
     ]
    }
   ],
   "source": [
    "# write to file if all papers have been parsed\n",
    "if reviewed_bibtex.count(\"@\") == df.shape[0]:\n",
    "    with open(\"output/reviewed_papers.bib\", \"w\") as f:\n",
    "        f.write(reviewed_bibtex)\n",
    "\n",
    "print(reviewed_bibtex[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "685d33b8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}