09-Task-Evaluate_Data/03d2_EVAL_FRP_fixation_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7c9a9ba0-1fa4-467c-be6d-de2c92c50a30",
   "metadata": {},
   "source": [
    "# FRP fixation-based statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed68a5f5-cc8e-44cd-b916-6c4c8f4a9809",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "774c436e-1ab4-477b-ba08-dcf2008f1774",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *\n",
    "from mne.stats import *\n",
    "import mne\n",
    "\n",
    "import scipy\n",
    "mne.set_log_level('CRITICAL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c5784a5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33c1432e34dc4326a811ceb1a6f9ee95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "..\\10-Data_Evaluation_Results\\EEG\\EEG_FRP_combs\\fixation_analysis\\combs_-300_1000_all_trial_erp frp offset.csv\n",
      "{}\n",
      "\tOffset from stimulus onset to fixation onset diff_ambiguous-unambiguous might be normally distributed due to 0.255 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9487491250038147, pvalue=0.25452032685279846)\n",
      "\tFor Offset from stimulus onset to fixation onset, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.998: H0 kept, H1 not proven; Result TtestResult(statistic=-3.2183696073925603, pvalue=0.998096150209999, df=23)\n",
      "\tFor Offset from stimulus onset to fixation onset, the t-test paired's stance on the H1 `confusing less than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=-3.2183696073925603, pvalue=0.001903849790001008, df=23)\n",
      "\t\tEffect size -0.38900303234440087 small\n",
      "{}\n",
      "\tFixation Duration diff_ambiguous-unambiguous might be normally distributed due to 0.784 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9747850894927979, pvalue=0.7840366363525391)\n",
      "\tFor Fixation Duration, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=3.173314883537209, pvalue=0.002120166336748664, df=23)\n",
      "\t\tEffect size 0.6314328786494646 medium\n",
      "{}\n",
      "\tDuration to next fixation diff_ambiguous-unambiguous might be normally distributed due to 0.729 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9725219011306763, pvalue=0.729282557964325)\n",
      "\tFor Duration to next fixation, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.001: H1 accepted; Result TtestResult(statistic=3.697645940104206, pvalue=0.0005940798111454209, df=23)\n",
      "\t\tEffect size 0.7402104390317827 medium\n",
      "{}\n",
      "\tOffset to First Fixation diff_ambiguous-unambiguous might be normally distributed due to 0.227 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9464946389198303, pvalue=0.22719767689704895)\n",
      "\tFor Offset to First Fixation, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.998: H0 kept, H1 not proven; Result TtestResult(statistic=-3.1859722952719136, pvalue=0.9979428935693659, df=23)\n",
      "\tFor Offset to First Fixation, the t-test paired's stance on the H1 `confusing less than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=-3.1859722952719136, pvalue=0.0020571064306340843, df=23)\n",
      "\t\tEffect size -0.38812525563755734 small\n",
      "..\\10-Data_Evaluation_Results\\EEG\\EEG_FRP_faois\\fixation_analysis\\faois_-300_1000_all_trial_erp frp offset.csv\n",
      "{}\n",
      "\tOffset from stimulus onset to fixation onset diff_ambiguous-unambiguous might be normally distributed due to 0.323 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9535549283027649, pvalue=0.3229120373725891)\n",
      "\tFor Offset from stimulus onset to fixation onset, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.998: H0 kept, H1 not proven; Result TtestResult(statistic=-3.3079569874853796, pvalue=0.9984645952981677, df=23)\n",
      "\tFor Offset from stimulus onset to fixation onset, the t-test paired's stance on the H1 `confusing less than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=-3.3079569874853796, pvalue=0.0015354047018322915, df=23)\n",
      "\t\tEffect size -0.4085958946113044 small\n",
      "{}\n",
      "\tFixation Duration diff_ambiguous-unambiguous might be normally distributed due to 0.481 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9620736241340637, pvalue=0.4814843237400055)\n",
      "\tFor Fixation Duration, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=3.1904204618222662, pvalue=0.0020353788546602822, df=23)\n",
      "\t\tEffect size 0.6324078676606695 medium\n",
      "{}\n",
      "\tDuration to next fixation diff_ambiguous-unambiguous might be normally distributed due to 0.363 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9560010433197021, pvalue=0.3634490668773651)\n",
      "\tFor Duration to next fixation, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.001: H1 accepted; Result TtestResult(statistic=3.557712423322171, pvalue=0.0008373480419084945, df=23)\n",
      "\t\tEffect size 0.7255930032787055 medium\n",
      "{}\n",
      "\tOffset to First Fixation diff_ambiguous-unambiguous might be normally distributed due to 0.337 greater equals to alpha 0.05; Result ShapiroResult(statistic=0.9544262290000916, pvalue=0.33689287304878235)\n",
      "\tFor Offset to First Fixation, the t-test paired's stance on the H1 `confusing greater than clean` indicates with alpha=0.05 and the p-value of 0.998: H0 kept, H1 not proven; Result TtestResult(statistic=-3.2794334565114256, pvalue=0.9983555158979517, df=23)\n",
      "\tFor Offset to First Fixation, the t-test paired's stance on the H1 `confusing less than clean` indicates with alpha=0.05 and the p-value of 0.002: H1 accepted; Result TtestResult(statistic=-3.2794334565114256, pvalue=0.001644484102048352, df=23)\n",
      "\t\tEffect size -0.4079439512892104 small\n"
     ]
    }
   ],
   "source": [
    "behavioral_data = pd.read_csv(get_behavioral_data_path(final_data_exclusion=True), sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "special_fixation_data = pd.read_csv(get_special_fixations_path(), sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "included_trials_path = get_all_erp_fixation_analysis_paths(False, 'erp frp offset')\n",
    "for path in tqdm(included_trials_path):\n",
    "    print(path)\n",
    "    included_trials = pd.read_csv(path, sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "    epoch_interval=[int(t) for t in path.stem.split('_')[1:3]]\n",
    "    frp=[fs_long for fs_long, fs_short in FIXATION_SELECTION_SHORT_VERSION.items() if fs_short == path.parts[3].split('_')[2]][0]\n",
    "    try:\n",
    "        snippet_group = path.parts[3].split('_')[3]\n",
    "    except IndexError:\n",
    "        snippet_group = SNIPPET_GROUP_ALL\n",
    "    description = get_erp_description(frp, False, [e/1000 for e in epoch_interval])\n",
    "\n",
    "    data = pd.DataFrame(columns=[PATH, ERP_PARAMETER_EPOCH_INTERVAL, SITUATION, ERP_PARAMETER_ERP_FRP, PARTICIPANT, CONDITION, \n",
    "                                 'Offset from stimulus onset to fixation onset', 'Fixation Duration','Duration to next fixation', 'Offset to First Fixation'])\n",
    "    for i, row in included_trials.iterrows():\n",
    "\n",
    "        participant, snippet=row[PARTICIPANT], row[SNIPPET]\n",
    "        # print(participant, snippet)\n",
    "        # identify corresponding fixation\n",
    "        special_fixation = special_fixation_data[(special_fixation_data[PARTICIPANT]==participant) & (special_fixation_data[SNIPPET]==snippet) & (special_fixation_data[FIXATION_SELECTION_ALGORITHM]==frp)].squeeze()\n",
    "        behavior = behavioral_data[(behavioral_data[PARTICIPANT]==participant) & (behavioral_data[SNIPPET]==snippet)].squeeze()\n",
    "        assert(len(special_fixation.shape)==1), f'''{participant} {snippet} {special_fixation}'''\n",
    "        assert(len(behavior.shape)==1)\n",
    "        frp_offset = special_fixation[FIXATION_COLUMN_START]-behavior[BEHAVIORAL_COLUMN_START]\n",
    "        if abs(row['Offset to ERP']-frp_offset)>=round_time_EEG(1/EEG_FREQUENCY):\n",
    "            print(f'''Big delta between offset from eeg file {row['Offset to ERP']} and fixation-behavioral {frp_offset}''')\n",
    "        fixation_data = pd.read_csv(get_selected_fixations_path(participant, snippet), sep=SEPARATOR, dtype={PARTICIPANT: str})\n",
    "        fixation = fixation_data.reset_index().merge(special_fixation.to_frame().T, how='inner', on=FIXATION_COLUMNS)\n",
    "        assert(fixation.shape[0]==1)\n",
    "        fixation_index=fixation['index'].values[0]\n",
    "        if fixation_index<fixation_data.index.max():\n",
    "            next_fixation_start = fixation_data.loc[fixation_index+1].squeeze()[FIXATION_COLUMN_START]\n",
    "        else:\n",
    "            next_fixation_start = behavior[BEHAVIORAL_COLUMN_END]\n",
    "        # calculate distance between fixation / behavioral and compare to given results in iteration\n",
    "        # get length of fixation, and distance to next fixation\n",
    "        # FIXATION_COLUMNS\n",
    "        data.loc[data.shape[0]] = {\n",
    "            PARTICIPANT:participant, \n",
    "            CONDITION:CONDITION_VARIANT_MATCH[get_snippet_variant(snippet)], \n",
    "            'Offset from stimulus onset to fixation onset': special_fixation[FIXATION_COLUMN_START]-behavior[BEHAVIORAL_COLUMN_START],\n",
    "            'Fixation Duration': special_fixation[FIXATION_COLUMN_DURATION],\n",
    "            'Duration to next fixation':(next_fixation_start-special_fixation[FIXATION_COLUMN_START]),\n",
    "            'Offset to First Fixation':special_fixation[FIXATION_COLUMN_START]-fixation_data.loc[0].squeeze()[FIXATION_COLUMN_START]\n",
    "        }\n",
    "    data[PATH] = path\n",
    "    data[SITUATION] = path.stem\n",
    "    data[FIXATION_SELECTION_ALGORITHM] = frp\n",
    "    data[ERP_PARAMETER_EPOCH_INTERVAL] = data.index.map(lambda i:epoch_interval)\n",
    "    data.to_csv(get_erp_fixation_analysis_path(frp, snippet_group, f'{description}_statistics', 'fixation analysis'), sep=SEPARATOR)\n",
    "    analyze_statistics_distribution(frp, snippet_group, description, data, 'erp frp offset2','Offset from stimulus onset to fixation onset')\n",
    "    analyze_statistics_distribution(frp, snippet_group, description, data, 'fixation duration','Fixation Duration')\n",
    "    analyze_statistics_distribution(frp, snippet_group, description, data, 'next fixation','Duration to next fixation')\n",
    "    analyze_statistics_distribution(frp, snippet_group, description, data, '1st fixation frp offset','Offset to First Fixation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3cfaada2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data['Offset from stimulus onset to fixation onset (ms)'] = data['Offset from stimulus onset to fixation onset']*1000\n",
    "# data['Fixation Duration (ms)'] = data['Fixation Duration']*1000\n",
    "# plt.rcParams.update({'font.size': 14})\n",
    "# fig, axis = plt.subplots(2, 1, figsize=(7,6))\n",
    "# for i, col in enumerate(['Fixation Duration (ms)','Offset from stimulus onset to fixation onset (ms)', ]):\n",
    "#     sns.violinplot(data, x=col, y=CONDITION, #hue=CONDITION, \n",
    "#                     legend=False, inner=\"box\", cut=0, ax=axis[i], inner_kws=dict(box_width=30, whis_width=2, color=\".2\"))\n",
    "#     axis[i].set_ylabel('')\n",
    "#     axis[i].set_xlabel('Time (ms)')\n",
    "#     axis[i].set_title(' ', loc='left')\n",
    "# plt.tight_layout()\n",
    "# plt.show()\n",
    "# plt.savefig('test.png', bbox_inches='tight', pad_inches=0)\n",
    "# plt.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}