09-Task-Evaluate_Data/01c1_PRE_BEH_Event_Data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Behavioral Data\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "hdf_files = get_hdf_files_per_participant()\n",
    "sequence_orders = get_sequence_order_per_participant()\n",
    "assert (set(folders.keys()) == set(hdf_files.keys()))\n",
    "assert (set(folders.keys()) == set(sequence_orders.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "snippets_data = pd.read_csv(SNIPPETS_PATH, sep=SEPARATOR).fillna(0)\n",
    "snippets_data[SNIPPET_COLUMN_SNIPPET] = snippets_data[SNIPPET_COLUMN_SNIPPET].str.replace(\n",
    "    '\\'', '')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get behavioral data\n",
    "* store events for EEG annotations\n",
    "* transform events into usable dataframe\n",
    "* check sequence order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf833026dd11429eb571457826f09c2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15485a8452fc4e4d83c644db1575957c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E003\\Experiment_003_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 004\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05a75ac2bc884c11bf8557ead11d0255",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E004\\Experiment_004_part01.hdf5\n",
      "\tNo behavioral events found in file ..\\08-Data-Trial_Recordings\\raw\\E004\\Experiment_004_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E004\\Experiment_004_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 005\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0b3df8c016a4c0db0d11338ce9bfede",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E005\\Experiment_005_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 007\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d3cb9df63ff4e8fac935735110a61da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E007\\Experiment_007_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E007\\Experiment_007_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 008\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b91bb5e5d9e74027b86cbf2afc4fe635",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part01.hdf5\n",
      "\tNo behavioral events found in file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part02.hdf5\n",
      "\tNo behavioral events found in file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E008\\Experiment_008_part04.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 009\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "994fac7f528140b19b3c59f89fd0178d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E009\\Experiment_009_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E009\\Experiment_009_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 010\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20dd63fb3f0f48c8b5bdfd835011bdc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E010\\Experiment_010_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E010\\Experiment_010_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E010\\Experiment_010_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 011\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e409a1f644644269f9249bd70b4dadb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E011\\Experiment_011_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 012\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eab4574ab7054799a60c3a33d5564bb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E012\\Experiment_012_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E012\\Experiment_012_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E012\\Experiment_012_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E012\\Experiment_012_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 013\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2d4b19c0e29427abb088ea23774988e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E013\\Experiment_013_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 014\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb0d966004d14e208cb6b9aef301e559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part03.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part04.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part04.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part05.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part05.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part06.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E014\\Experiment_014_part07.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "\"Na\" values in start time identified, trying to fill it with fixation information\n",
      "Filling attempt failed occasionally (start after end), deleting rows\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "The number of trials in the condition files and the behavioral data from the hdf files differs. The snippets not given in the behavioral data will be excluded from further analysis.\n",
      "The order of trials in the condition files and those from the hdf files differs. Differences are in:\n",
      "        Snippet  Fixation_Start          End        Start  Duration Answer   \n",
      "0     13-obf-v0      110.707628   124.842301   115.705738  9.136563      5  \\\n",
      "1      7-obf-v0      132.870165   142.833258   137.854658  4.978600      6   \n",
      "2     -clean-v0      150.627998   162.715399   155.603914  7.111485   true   \n",
      "3   12-clean-v0      169.917255   180.531883   174.919806  5.612077      7   \n",
      "4   60-clean-v0      186.892024   199.898883   191.885796  8.013086     32   \n",
      "..          ...             ...          ...          ...       ...    ...   \n",
      "67  49-clean-v2     2262.603262  2271.758582  2267.607949  4.150633     42   \n",
      "68     5-obf-v2     2278.120954  2288.959291  2283.107310  5.851981  false   \n",
      "69  60-clean-v2     2295.937945  2306.208172  2300.923439  5.284732     40   \n",
      "70  54-clean-v2     2312.756113  2322.373861  2317.739467  4.634394      6   \n",
      "71          NaN             NaN          NaN          NaN       NaN    NaN   \n",
      "\n",
      "   Correctness Rating right_answer possible_wrong_answers  ...  version   \n",
      "0         True   easy            5                    6;3  ...       v0  \\\n",
      "1        False   easy            5                    6;7  ...       v0   \n",
      "2         True   easy         true                  false  ...       v0   \n",
      "3         True   easy            7                    6;5  ...       v0   \n",
      "4         True   easy           32                256;288  ...       v0   \n",
      "..         ...    ...          ...                    ...  ...      ...   \n",
      "67        True   easy           42                  21;80  ...       v2   \n",
      "68        True   easy        false                   true  ...       v2   \n",
      "69        True   easy           40                255;296  ...       v2   \n",
      "70       False   easy            0                    6;2  ...       v2   \n",
      "71         NaN    NaN          NaN                    NaN  ...      NaN   \n",
      "\n",
      "         Langhout AoC_category            AoC_category    Condition HDF_index   \n",
      "0           constant variables      confusingvariables    ambiguous       6.0  \\\n",
      "1     post increment decrement      prepostindecrement    ambiguous       6.0   \n",
      "2           implicit predicate  confusingboolwitharith  unambiguous       6.0   \n",
      "3      pre increment decrement      prepostindecrement  unambiguous       6.0   \n",
      "4              type_conversion   implementationdetails  unambiguous       6.0   \n",
      "..                         ...                     ...          ...       ...   \n",
      "67   dead_unreachable_repeated      confusingvariables  unambiguous       6.0   \n",
      "68   infix operator precedence  confusingboolwitharith    ambiguous       6.0   \n",
      "69             type_conversion   implementationdetails  unambiguous       6.0   \n",
      "70  change_of_literal_encoding   implementationdetails  unambiguous       6.0   \n",
      "71                         NaN                     NaN          NaN       NaN   \n",
      "\n",
      "   Participant  start_event   Snippet_so  Block Block_Index  \n",
      "0          014           12    62-obf-v0      1           1  \n",
      "1          014           12    13-obf-v0      1           2  \n",
      "2          014           12     7-obf-v0      1           3  \n",
      "3          014           11    -clean-v0      1           4  \n",
      "4          014           11  12-clean-v0      1           5  \n",
      "..         ...          ...          ...    ...         ...  \n",
      "67         014           12    17-obf-v2      3          20  \n",
      "68         014           11  49-clean-v2      3          21  \n",
      "69         014           12     5-obf-v2      3          22  \n",
      "70         014           11  60-clean-v2      3          23  \n",
      "71         NaN           11  54-clean-v2      3          24  \n",
      "\n",
      "[72 rows x 22 columns]\n",
      "Some rows with empty entries are removed       Snippet  Fixation_Start  End  Start  Duration Answer Correctness Rating   \n",
      "71  62-obf-v0             NaN  NaN    NaN       NaN    NaN         NaN    NaN  \\\n",
      "\n",
      "   right_answer possible_wrong_answers  ...  variant version   \n",
      "71          NaN                    NaN  ...      NaN     NaN  \\\n",
      "\n",
      "   Langhout AoC_category AoC_category Condition HDF_index  Participant   \n",
      "71                   NaN          NaN       NaN       NaN          NaN  \\\n",
      "\n",
      "   start_event  Block  Block_Index  \n",
      "71          12      1            1  \n",
      "\n",
      "[1 rows x 21 columns]\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 015\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89bea4aef2654cb7b090566c5f862f68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E015\\Experiment_015_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E015\\Experiment_015_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 016\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "080ba1d40e324749acc871bc77b52fda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E016\\Experiment_016_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E016\\Experiment_016_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E016\\Experiment_016_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E016\\Experiment_016_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 017\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46f7de4b8a2e484bb6974a650a21fbde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E017\\Experiment_017_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E017\\Experiment_017_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E017\\Experiment_017_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E017\\Experiment_017_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E017\\Experiment_017_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 018\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "014adf77abd34024b11c5d390c84fc15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E018\\Experiment_018_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 019\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e77d2751be9a407fb2ed3c62a1d308d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E019\\Experiment_019_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E019\\Experiment_019_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 020\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "526599753f0643b480d78b666d57d2d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E020\\Experiment_020_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E020\\Experiment_020_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "\"Na\" values in start time identified, trying to fill it with fixation information\n",
      "Filling attempt failed occasionally (start after end), deleting rows\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 021\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c729808fc7f40aeb5194eea537b4a0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E021\\Experiment_021_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E021\\Experiment_021_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E021\\Experiment_021_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 022\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e18762db97d54868a38ccc036e450825",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E022\\Experiment_022_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E022\\Experiment_022_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E022\\Experiment_022_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E022\\Experiment_022_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E022\\Experiment_022_part04.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 023\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15ad36938395479d9ba944aadef52db1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E023\\Experiment_023_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E023\\Experiment_023_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E023\\Experiment_023_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 024\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ec09d32c8b44b90ad0df1256c976090",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E024\\Experiment_024_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E024\\Experiment_024_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E024\\Experiment_024_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E024\\Experiment_024_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "085a2d73ad8541f2ac02e41ed010824c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E101\\Experiment_101_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a10dd2d8d0c4f5aa2b614b126efed92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E102\\Experiment_102_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E102\\Experiment_102_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "----------------------------------------------\n",
      "Participant 106\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1baef1a2410f4706bb1e30fa7f92c8a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part01.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part01.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part02.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part02.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part03.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part03.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part04.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part04.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part05.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part06.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part07.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "\tOnly 1 fixation cross event found in file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part07.hdf5.\n",
      "(01/10) Load behavioral events from file ..\\08-Data-Trial_Recordings\\raw\\E106\\Experiment_106_part08.hdf5\n",
      "(02/10) Retrieve fixation cross data\n",
      "(03/10) Retrieve / calculate timing data\n",
      "(04/10) Retrieve answer data\n",
      "(05/10) Retrieve perception / rating data\n",
      "(06/10) Transform snippets data\n",
      "(07/10) Prepare for concat of hdf file results\n",
      "(08/10) Prepare snippet events for eeg trial assignment\n",
      "(09/10) Check that sequence order is retained and complete\n",
      "(10/10) Save participant behavioral data\n",
      "(10/10) Save all behavioral data\n"
     ]
    }
   ],
   "source": [
    "# Iterate over all participants\n",
    "all_behavioral_data = []\n",
    "all_behavioral_events = []\n",
    "\n",
    "for participant in tqdm(folders):\n",
    "    print('----------------------------------------------')\n",
    "    print(PARTICIPANT, participant)\n",
    "\n",
    "    event_data_files = hdf_files[participant]\n",
    "\n",
    "    participant_behavioral_data = []\n",
    "    participant_behavioral_events = []\n",
    "\n",
    "    for i, hdf_file in tqdm(enumerate(event_data_files)):\n",
    "        print(f'(01/10) Load behavioral events from file {hdf_file}')\n",
    "        message_events = retrieve_message_events(hdf_file)\n",
    "        if message_events.empty:\n",
    "            print(f'\\tNo behavioral events found in file {hdf_file}.')\n",
    "            update_exclusions(participant, HDF_FILE, BEHAVIORAL, {\n",
    "                hdf_file.name: '01c1_PRE_BEH_Event_Data: No behavioral events found in file.'})\n",
    "            continue\n",
    "\n",
    "        # Behavioral: get data according to categories (FIXATION)\n",
    "        print('(02/10) Retrieve fixation cross data')\n",
    "        snippet_fixation_events = get_fixation_cross_data(message_events)\n",
    "        # display(snippet_fixation_events)\n",
    "        if (message_events.shape[0] == 1) and (message_events[MESSAGE_COLUMN_CATEGORY] == MESSAGE_CATEGORY_FIXATION).all():\n",
    "            print(f'\\tOnly 1 fixation cross event found in file {hdf_file}.')\n",
    "            behavioral_events = prepare_for_EEG_synchronization(\n",
    "                snippet_fixation_events, None)\n",
    "            behavioral_events[HDF_INDEX] = i\n",
    "            participant_behavioral_events.append(behavioral_events)\n",
    "            update_exclusions(participant, HDF_FILE, BEHAVIORAL, {\n",
    "                hdf_file.name: '01c1_PRE_BEH_Event_Data: No behavioral events found in file.'})\n",
    "            continue\n",
    "\n",
    "        # Behavioral: get data according to categories (SNIPPETVIEW for start and end snippet-view => answer time)\n",
    "        print('(03/10) Retrieve / calculate timing data')\n",
    "        snippet_duration_events = get_snippet_view_data(\n",
    "            message_events, snippet_fixation_events)\n",
    "        # display(snippet_duration_events)\n",
    "\n",
    "        # Behavioral: get data according to categories (ANSWER for correctness)\n",
    "        print('(04/10) Retrieve answer data')\n",
    "        snippet_answer_events = get_snippet_answer_data(message_events)\n",
    "        # display(snippet_answer_events)\n",
    "\n",
    "        # Behavioral: get data according to categories (PERCEPTION for subjective difficulty rating)\n",
    "        print('(05/10) Retrieve perception / rating data')\n",
    "        snippet_perception_events = get_snippet_perception_data(message_events)\n",
    "        # display(snippet_perception_events)\n",
    "\n",
    "        print('(06/10) Transform snippets data')\n",
    "        # merge all information retrieved from events\n",
    "        behavioral_data = transform_snippet_data(\n",
    "            snippet_fixation_events, snippet_duration_events, snippet_answer_events, snippet_perception_events, snippets_data)\n",
    "\n",
    "        print('(07/10) Prepare for concat of hdf file results')\n",
    "        # delete unnecessary columns\n",
    "        behavioral_data = behavioral_data.drop(columns=[SNIPPET_COLUMN_SNIPPET]+[\n",
    "                                               c for c in behavioral_data.columns if not (c in BEHAVIORAL_COLUMNS)])\n",
    "        # add information\n",
    "        behavioral_data[HDF_INDEX] = i\n",
    "        behavioral_data[PARTICIPANT] = participant\n",
    "        participant_behavioral_data.append(behavioral_data)\n",
    "        # display(behavioral_data)\n",
    "\n",
    "        print('(08/10) Prepare snippet events for eeg trial assignment')\n",
    "        # linear events and corresponding stimuli code\n",
    "        behavioral_events = prepare_for_EEG_synchronization(\n",
    "            snippet_fixation_events, snippet_duration_events)\n",
    "        behavioral_events[HDF_INDEX] = i\n",
    "        participant_behavioral_events.append(behavioral_events)\n",
    "\n",
    "    # add extracted snippets to exclusion file - respective after all files delete those that did not occur\n",
    "    print('(09/10) Check that sequence order is retained and complete')\n",
    "    behavioral_data = pd.concat([bd.sort_values(\n",
    "        [BEHAVIORAL_COLUMN_START, BEHAVIORAL_COLUMN_END]) for bd in participant_behavioral_data]).reset_index(drop=True)\n",
    "    behavioral_data = compare_behavioral_to_sequence_order(participant, behavioral_data, sequence_orders[participant],\n",
    "                                                           '01c1_PRE_BEH_Event_Data')\n",
    "    # display(behavioral_data)\n",
    "\n",
    "    print('(10/10) Save participant behavioral data')\n",
    "    # save behavioral data\n",
    "    behavioral_data.to_csv(get_behavioral_data_path(\n",
    "        participant, final_data_exclusion=False), index=False, sep=SEPARATOR)\n",
    "    behavioral_events = pd.concat(participant_behavioral_events)\n",
    "    behavioral_events[PARTICIPANT] = participant\n",
    "    behavioral_events.to_csv(get_behavioral_events_path(participant),\n",
    "                             index=False, sep=SEPARATOR)\n",
    "    all_behavioral_data.append(behavioral_data)\n",
    "    all_behavioral_events.append(behavioral_events)\n",
    "\n",
    "print('(10/10) Save all behavioral data')\n",
    "# save behavioral data\n",
    "behavioral_data = pd.concat(all_behavioral_data)\n",
    "behavioral_data.to_csv(get_behavioral_data_path(\n",
    "    final_data_exclusion=False), index=False, sep=SEPARATOR)\n",
    "behavioral_events = pd.concat(all_behavioral_events)\n",
    "behavioral_events[PARTICIPANT] = participant\n",
    "behavioral_events.to_csv(get_behavioral_events_path(),\n",
    "                         index=False, sep=SEPARATOR)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}