09-Task-Evaluate_Data/03a2_EVAL_BEH_Confusion_LMEM Correctness.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7c9a9ba0-1fa4-467c-be6d-de2c92c50a30",
   "metadata": {},
   "source": [
    "# EVALUATION: Behavioral Correctness with LMEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "774c436e-1ab4-477b-ba08-dcf2008f1774",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\maure\\AppData\\Roaming\\Python\\Python311\\site-packages\\rpy2\\robjects\\packages.py:367: UserWarning: The symbol 'quartz' is not in this R namespace/package.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Lade nötiges Paket: Matrix\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "Attache Paket: 'lmerTest'\n",
       "\n",
       "Das folgende Objekt ist maskiert 'package:lme4':\n",
       "\n",
       "    lmer\n",
       "\n",
       "Das folgende Objekt ist maskiert 'package:stats':\n",
       "\n",
       "    step\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "ciTools version 0.6.1 (C) Institute for Defense Analyses\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "Attache Paket: 'nlme'\n",
       "\n",
       "Das folgende Objekt ist maskiert 'package:lme4':\n",
       "\n",
       "    lmList\n",
       "\n",
       "In addition: Warning message:\n",
       "Paket 'nlme' wurde unter R Version 4.3.3 erstellt \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Lade nötiges Paket: mgcv\n",
       "This is mgcv 1.9-0. For overview type 'help(\"mgcv-package\")'.\n",
       "Lade nötiges Paket: plotfunctions\n",
       "Loaded package itsadug 2.4 (see 'help(\"itsadug\")' ).\n",
       "In addition: Warning messages:\n",
       "1: Paket 'itsadug' wurde unter R Version 4.3.3 erstellt \n",
       "2: Paket 'plotfunctions' wurde unter R Version 4.3.3 erstellt \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from utils.utils import *\n",
    "\n",
    "%run utils/LMEM_helpers.ipynb"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e75c7be5-217d-4fcd-9a80-f6b83409a9a4",
   "metadata": {},
   "source": [
    "### Load behavioral data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "365c1ce4-2d53-4f4b-9ab7-7159c28446c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Snippet</th>\n",
       "      <th>Fixation_Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Start</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Correctness</th>\n",
       "      <th>Rating</th>\n",
       "      <th>right_answer</th>\n",
       "      <th>possible_wrong_answers</th>\n",
       "      <th>...</th>\n",
       "      <th>HDF_index</th>\n",
       "      <th>Participant</th>\n",
       "      <th>start_event</th>\n",
       "      <th>Block</th>\n",
       "      <th>Block_Index</th>\n",
       "      <th>Stimulus_id</th>\n",
       "      <th>Trial_No</th>\n",
       "      <th>Block_No</th>\n",
       "      <th>In_Block_No</th>\n",
       "      <th>Duration_log</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16-obf-v0</td>\n",
       "      <td>130.144131</td>\n",
       "      <td>165.194542</td>\n",
       "      <td>135.137310</td>\n",
       "      <td>30.057232</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>5;3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P003</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Stimulus/S 12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.403103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5-clean-v0</td>\n",
       "      <td>174.369258</td>\n",
       "      <td>203.895033</td>\n",
       "      <td>179.370753</td>\n",
       "      <td>24.524280</td>\n",
       "      <td>false</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>true</td>\n",
       "      <td>false</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P003</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Stimulus/S 11</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.199664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61-clean-v0</td>\n",
       "      <td>211.250803</td>\n",
       "      <td>221.093158</td>\n",
       "      <td>216.250725</td>\n",
       "      <td>4.842433</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2;0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P003</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Stimulus/S 11</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.577417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13-clean-v0</td>\n",
       "      <td>228.432896</td>\n",
       "      <td>237.992304</td>\n",
       "      <td>233.416647</td>\n",
       "      <td>4.575657</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6;3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P003</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Stimulus/S 11</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1.520750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10-obf-v0</td>\n",
       "      <td>245.032454</td>\n",
       "      <td>264.358911</td>\n",
       "      <td>250.001624</td>\n",
       "      <td>14.357288</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0;2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P003</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Stimulus/S 12</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2.664258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1722</th>\n",
       "      <td>18-clean-v2</td>\n",
       "      <td>650.097955</td>\n",
       "      <td>671.641398</td>\n",
       "      <td>655.097237</td>\n",
       "      <td>16.544161</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4;5</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>P106</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>Stimulus/S 11</td>\n",
       "      <td>68</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>2.806033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1723</th>\n",
       "      <td>54-obf-v2</td>\n",
       "      <td>678.104435</td>\n",
       "      <td>687.740039</td>\n",
       "      <td>683.096211</td>\n",
       "      <td>4.643827</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6;2</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>P106</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>Stimulus/S 12</td>\n",
       "      <td>69</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1.535539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1724</th>\n",
       "      <td>9-clean-v2</td>\n",
       "      <td>694.153446</td>\n",
       "      <td>707.572308</td>\n",
       "      <td>699.128912</td>\n",
       "      <td>8.443396</td>\n",
       "      <td>false</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>P106</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>Stimulus/S 11</td>\n",
       "      <td>70</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>2.133385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1725</th>\n",
       "      <td>12-obf-v2</td>\n",
       "      <td>713.678701</td>\n",
       "      <td>726.539764</td>\n",
       "      <td>718.661568</td>\n",
       "      <td>7.878196</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3;2</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>P106</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>Stimulus/S 12</td>\n",
       "      <td>71</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2.064099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1726</th>\n",
       "      <td>15-obf-v2</td>\n",
       "      <td>733.845005</td>\n",
       "      <td>755.571496</td>\n",
       "      <td>738.827485</td>\n",
       "      <td>16.744011</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>312.0;63.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>P106</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>Stimulus/S 12</td>\n",
       "      <td>72</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>2.818041</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1727 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Snippet  Fixation_Start         End       Start   Duration Answer   \n",
       "0       16-obf-v0      130.144131  165.194542  135.137310  30.057232      3  \\\n",
       "1      5-clean-v0      174.369258  203.895033  179.370753  24.524280  false   \n",
       "2     61-clean-v0      211.250803  221.093158  216.250725   4.842433      4   \n",
       "3     13-clean-v0      228.432896  237.992304  233.416647   4.575657      5   \n",
       "4       10-obf-v0      245.032454  264.358911  250.001624  14.357288      1   \n",
       "...           ...             ...         ...         ...        ...    ...   \n",
       "1722  18-clean-v2      650.097955  671.641398  655.097237  16.544161      6   \n",
       "1723    54-obf-v2      678.104435  687.740039  683.096211   4.643827      6   \n",
       "1724   9-clean-v2      694.153446  707.572308  699.128912   8.443396  false   \n",
       "1725    12-obf-v2      713.678701  726.539764  718.661568   7.878196      4   \n",
       "1726    15-obf-v2      733.845005  755.571496  738.827485  16.744011   62.0   \n",
       "\n",
       "      Correctness  Rating right_answer possible_wrong_answers  ... HDF_index   \n",
       "0               0       1            7                    5;3  ...       0.0  \\\n",
       "1               0       1         true                  false  ...       0.0   \n",
       "2               1       1            4                    2;0  ...       0.0   \n",
       "3               1       1            5                    6;3  ...       0.0   \n",
       "4               1       0            1                    0;2  ...       0.0   \n",
       "...           ...     ...          ...                    ...  ...       ...   \n",
       "1722            1       1            6                    4;5  ...       7.0   \n",
       "1723            0       1            0                    6;2  ...       7.0   \n",
       "1724            1       1        false                   true  ...       7.0   \n",
       "1725            1       1            4                    3;2  ...       7.0   \n",
       "1726            1       0         62.0             312.0;63.0  ...       7.0   \n",
       "\n",
       "     Participant start_event Block Block_Index    Stimulus_id  Trial_No   \n",
       "0           P003          12     1           1  Stimulus/S 12         1  \\\n",
       "1           P003          11     1           2  Stimulus/S 11         2   \n",
       "2           P003          11     1           3  Stimulus/S 11         3   \n",
       "3           P003          11     1           4  Stimulus/S 11         4   \n",
       "4           P003          12     1           5  Stimulus/S 12         5   \n",
       "...          ...         ...   ...         ...            ...       ...   \n",
       "1722        P106          11     3          20  Stimulus/S 11        68   \n",
       "1723        P106          12     3          21  Stimulus/S 12        69   \n",
       "1724        P106          11     3          22  Stimulus/S 11        70   \n",
       "1725        P106          12     3          23  Stimulus/S 12        71   \n",
       "1726        P106          12     3          24  Stimulus/S 12        72   \n",
       "\n",
       "     Block_No  In_Block_No  Duration_log  \n",
       "0           1            1      3.403103  \n",
       "1           1            2      3.199664  \n",
       "2           1            3      1.577417  \n",
       "3           1            4      1.520750  \n",
       "4           1            5      2.664258  \n",
       "...       ...          ...           ...  \n",
       "1722        3           20      2.806033  \n",
       "1723        3           21      1.535539  \n",
       "1724        3           22      2.133385  \n",
       "1725        3           23      2.064099  \n",
       "1726        3           24      2.818041  \n",
       "\n",
       "[1727 rows x 26 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Snippet': dtype('O'), 'Fixation_Start': dtype('float64'), 'End': dtype('float64'), 'Start': dtype('float64'), 'Duration': dtype('float64'), 'Answer': dtype('O'), 'Correctness': dtype('int64'), 'Rating': dtype('int64'), 'right_answer': dtype('O'), 'possible_wrong_answers': dtype('O'), 'number': dtype('O'), 'variant': dtype('O'), 'version': dtype('O'), 'Langhout AoC_category': dtype('O'), 'AoC_category': dtype('O'), 'Condition': dtype('O'), 'HDF_index': dtype('float64'), 'Participant': dtype('O'), 'start_event': dtype('int64'), 'Block': dtype('int64'), 'Block_Index': dtype('int64'), 'Stimulus_id': dtype('O'), 'Trial_No': dtype('int64'), 'Block_No': dtype('int64'), 'In_Block_No': dtype('int64'), 'Duration_log': dtype('float64')}\n"
     ]
    }
   ],
   "source": [
    "# Snippet data\n",
    "behavioral_data = pd.read_csv(get_behavioral_data_path(\n",
    "    final_data_exclusion=True, analysis_prep=True), sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "display(behavioral_data)\n",
    "print(behavioral_data.dtypes.to_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd385db2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + (1 + Condition | Participant) + (1 + Condition | number)\n",
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition + (1 + Condition | Participant) +  \n",
      "    (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1215.6   1259.3   -599.8   1199.6     1719 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.3594  0.0856  0.1974  0.3407  3.8404 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0033   1.0017        \n",
      "             Conditionconfusing 0.2941   0.5423   -0.65\n",
      " number      (Intercept)        2.2404   1.4968        \n",
      "             Conditionconfusing 1.6341   1.2783   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                   Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)          3.0374     0.4198   7.235 4.65e-13 ***\n",
      "Conditionconfusing  -1.5716     0.3692  -4.257 2.07e-05 ***\n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr)\n",
      "Cndtncnfsng -0.298\n",
      "Analysis of Variance Table\n",
      "          npar Sum Sq Mean Sq F value\n",
      "Condition    1 20.858  20.858  20.858\n"
     ]
    }
   ],
   "source": [
    "# Correctness: Binomial on normal values (0/1)\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion base model binomial')\n",
    "# best model according to AIC (see below for more complex models, reduction to this model performed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee480271",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition * Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.15758 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition * Block_No * In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1223.6   1299.9   -597.8   1195.6     1713 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.0012  0.0823  0.1982  0.3398  3.7563 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0317   1.0157        \n",
      "             Conditionconfusing 0.3217   0.5672   -0.66\n",
      " number      (Intercept)        2.2643   1.5048        \n",
      "             Conditionconfusing 1.6550   1.2865   0.10 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                                        Estimate Std. Error z value Pr(>|z|)   \n",
      "(Intercept)                              2.10742    0.79147   2.663  0.00775 **\n",
      "Conditionconfusing                      -0.52164    0.91686  -0.569  0.56940   \n",
      "Block_No                                 0.52240    0.33925   1.540  0.12360   \n",
      "In_Block_No                              0.03697    0.04805   0.769  0.44164   \n",
      "Conditionconfusing:Block_No             -0.63533    0.41486  -1.531  0.12566   \n",
      "Conditionconfusing:In_Block_No          -0.03597    0.06125  -0.587  0.55708   \n",
      "Block_No:In_Block_No                    -0.02192    0.02339  -0.937  0.34882   \n",
      "Conditionconfusing:Block_No:In_Block_No  0.02598    0.02916   0.891  0.37292   \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N Cn:B_N C:I_B_ B_N:I_\n",
      "Cndtncnfsng -0.683                                          \n",
      "Block_No    -0.784  0.677                                   \n",
      "In_Block_No -0.768  0.663  0.813                            \n",
      "Cndtncn:B_N  0.641 -0.853 -0.818 -0.665                     \n",
      "Cndtn:I_B_N  0.602 -0.820 -0.638 -0.784  0.805              \n",
      "Blc_N:I_B_N  0.709 -0.612 -0.896 -0.919  0.732  0.721       \n",
      "C:B_N:I_B_N -0.568  0.763  0.718  0.737 -0.885 -0.923 -0.802\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.15758 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                               npar  Sum Sq Mean Sq F value\n",
      "Condition                         1 20.2008 20.2008 20.2008\n",
      "Block_No                          1  0.2996  0.2996  0.2996\n",
      "In_Block_No                       1  0.1538  0.1538  0.1538\n",
      "Condition:Block_No                1  2.2831  2.2831  2.2831\n",
      "Condition:In_Block_No             1  0.3169  0.3169  0.3169\n",
      "Block_No:In_Block_No              1  0.1207  0.1207  0.1207\n",
      "Condition:Block_No:In_Block_No    1  0.7608  0.7608  0.7608\n"
     ]
    }
   ],
   "source": [
    "# 3wise interaction failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no in_block_no 3wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b1880f51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition * Block_No + Condition * In_Block_No + Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.00207913 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition * Block_No + Condition * In_Block_No +  \n",
      "    Block_No * In_Block_No + (1 + Condition | Participant) +  \n",
      "    (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1222.4   1293.3   -598.2   1196.4     1714 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.3868  0.0825  0.1991  0.3410  3.7400 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0213   1.0106        \n",
      "             Conditionconfusing 0.3102   0.5569   -0.66\n",
      " number      (Intercept)        2.2622   1.5041        \n",
      "             Conditionconfusing 1.6283   1.2761   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                                Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                     2.499841   0.656525   3.808  0.00014 ***\n",
      "Conditionconfusing             -1.143583   0.594101  -1.925  0.05424 .  \n",
      "Block_No                        0.310831   0.234528   1.325  0.18506    \n",
      "In_Block_No                     0.005851   0.032430   0.180  0.85682    \n",
      "Conditionconfusing:Block_No    -0.310969   0.192760  -1.613  0.10669    \n",
      "Conditionconfusing:In_Block_No  0.014431   0.023593   0.612  0.54076    \n",
      "Block_No:In_Block_No           -0.005418   0.013927  -0.389  0.69729    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N Cn:B_N C:I_B_\n",
      "Cndtncnfsng -0.479                                   \n",
      "Block_No    -0.666  0.306                            \n",
      "In_Block_No -0.638  0.250  0.618                     \n",
      "Cndtncn:B_N  0.366 -0.601 -0.560 -0.055              \n",
      "Cndtn:I_B_N  0.255 -0.473  0.071 -0.400 -0.043       \n",
      "Blc_N:I_B_N  0.518 -0.012 -0.766 -0.812  0.075 -0.084\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.00207913 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                      npar  Sum Sq Mean Sq F value\n",
      "Condition                1 20.5641 20.5641 20.5641\n",
      "Block_No                 1  0.3283  0.3283  0.3283\n",
      "In_Block_No              1  0.1216  0.1216  0.1216\n",
      "Condition:Block_No       1  2.3584  2.3584  2.3584\n",
      "Condition:In_Block_No    1  0.3218  0.3218  0.3218\n",
      "Block_No:In_Block_No     1  0.1464  0.1464  0.1464\n"
     ]
    }
   ],
   "source": [
    "# all 2wise interactions failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No + {CONDITION} * In_Block_No + Block_No * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no in_block_no all 2wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3dad36f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition * Block_No + Condition * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.00632755 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition * Block_No + Condition * In_Block_No +  \n",
      "    (1 + Condition | Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1220.5   1286.0   -598.3   1196.5     1715 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.5243  0.0827  0.1982  0.3421  3.7546 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0153   1.0076        \n",
      "             Conditionconfusing 0.3075   0.5545   -0.65\n",
      " number      (Intercept)        2.2603   1.5034        \n",
      "             Conditionconfusing 1.6367   1.2793   0.10 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                                Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                     2.630739   0.562837   4.674 2.95e-06 ***\n",
      "Conditionconfusing             -1.144795   0.595932  -1.921   0.0547 .  \n",
      "Block_No                        0.241010   0.150598   1.600   0.1095    \n",
      "In_Block_No                    -0.004395   0.018911  -0.232   0.8162    \n",
      "Conditionconfusing:Block_No    -0.305399   0.192136  -1.589   0.1119    \n",
      "Conditionconfusing:In_Block_No  0.013643   0.023499   0.581   0.5615    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N Cn:B_N\n",
      "Cndtncnfsng -0.555                            \n",
      "Block_No    -0.494  0.466                     \n",
      "In_Block_No -0.442  0.417  0.001              \n",
      "Cndtncn:B_N  0.387 -0.605 -0.784  0.000       \n",
      "Cndtn:I_B_N  0.355 -0.481  0.000 -0.804 -0.026\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.00632755 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                      npar  Sum Sq Mean Sq F value\n",
      "Condition                1 20.4849 20.4849 20.4849\n",
      "Block_No                 1  0.3273  0.3273  0.3273\n",
      "In_Block_No              1  0.1166  0.1166  0.1166\n",
      "Condition:Block_No       1  2.3798  2.3798  2.3798\n",
      "Condition:In_Block_No    1  0.3226  0.3226  0.3226\n",
      "Correctness ~ 1 + Condition * Block_No + Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.0172112 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition * Block_No + Block_No * In_Block_No +  \n",
      "    (1 + Condition | Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1220.7   1286.2   -598.4   1196.7     1715 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.1824  0.0825  0.1982  0.3422  3.7897 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0214   1.0106        \n",
      "             Conditionconfusing 0.3084   0.5553   -0.66\n",
      " number      (Intercept)        2.2707   1.5069        \n",
      "             Conditionconfusing 1.6267   1.2754   0.10 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                             Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                  2.400178   0.631437   3.801 0.000144 ***\n",
      "Conditionconfusing          -0.973751   0.522678  -1.863 0.062462 .  \n",
      "Block_No                     0.299942   0.231858   1.294 0.195788    \n",
      "In_Block_No                  0.013765   0.029720   0.463 0.643249    \n",
      "Conditionconfusing:Block_No -0.305533   0.192279  -1.589 0.112059    \n",
      "Block_No:In_Block_No        -0.004665   0.013873  -0.336 0.736645    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N Cn:B_N\n",
      "Cndtncnfsng -0.414                            \n",
      "Block_No    -0.707  0.380                     \n",
      "In_Block_No -0.599  0.061  0.703              \n",
      "Cndtncn:B_N  0.383 -0.705 -0.550 -0.064       \n",
      "Blc_N:I_B_N  0.554 -0.046 -0.761 -0.926  0.055\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.0172112 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                     npar  Sum Sq Mean Sq F value\n",
      "Condition               1 20.5958 20.5958 20.5958\n",
      "Block_No                1  0.3303  0.3303  0.3303\n",
      "In_Block_No             1  0.1164  0.1164  0.1164\n",
      "Condition:Block_No      1  2.3819  2.3819  2.3819\n",
      "Block_No:In_Block_No    1  0.1095  0.1095  0.1095\n",
      "Correctness ~ 1 + Condition * In_Block_No + Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.0161127 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition * In_Block_No + Block_No * In_Block_No +  \n",
      "    (1 + Condition | Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1222.8   1288.3   -599.4   1198.8     1715 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.1172  0.0852  0.1986  0.3379  3.7538 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0051   1.0026        \n",
      "             Conditionconfusing 0.3009   0.5486   -0.65\n",
      " number      (Intercept)        2.2346   1.4949        \n",
      "             Conditionconfusing 1.6283   1.2760   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                                Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                     2.899620   0.617508   4.696 2.66e-06 ***\n",
      "Conditionconfusing             -1.728039   0.473729  -3.648 0.000265 ***\n",
      "In_Block_No                     0.002886   0.032996   0.087 0.930295    \n",
      "Block_No                        0.098776   0.193917   0.509 0.610489    \n",
      "Conditionconfusing:In_Block_No  0.012869   0.023501   0.548 0.583964    \n",
      "In_Block_No:Block_No           -0.003746   0.013855  -0.270 0.786892    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc In_B_N Blck_N C:I_B_\n",
      "Cndtncnfsng -0.363                            \n",
      "In_Block_No -0.671  0.287                     \n",
      "Block_No    -0.614 -0.007  0.718              \n",
      "Cndtn:I_B_N  0.312 -0.626 -0.431  0.017       \n",
      "In_Bl_N:B_N  0.538  0.016 -0.820 -0.877 -0.037\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.0161127 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                      npar  Sum Sq Mean Sq F value\n",
      "Condition                1 20.8282 20.8282 20.8282\n",
      "In_Block_No              1  0.1384  0.1384  0.1384\n",
      "Block_No                 1  0.3326  0.3326  0.3326\n",
      "Condition:In_Block_No    1  0.2767  0.2767  0.2767\n",
      "In_Block_No:Block_No     1  0.0707  0.0707  0.0707\n"
     ]
    }
   ],
   "source": [
    "# all models with 2/3 2wise interactions failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No + {CONDITION} * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no in_block_no 2 2wise with c binomial')\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No + Block_No * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no in_block_no 2 2wise with b binomial')\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * In_Block_No + Block_No * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no in_block_no 2 2wise with i binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "efad5bb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.0571756 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition + Block_No * In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1221.1   1281.1   -599.6   1199.1     1716 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.8934  0.0851  0.1989  0.3408  3.7978 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0069   1.0034        \n",
      "             Conditionconfusing 0.3021   0.5497   -0.65\n",
      " number      (Intercept)        2.2450   1.4983        \n",
      "             Conditionconfusing 1.6255   1.2749   0.12 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                      Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)           2.792751   0.585681   4.768 1.86e-06 ***\n",
      "Conditionconfusing   -1.565759   0.369435  -4.238 2.25e-05 ***\n",
      "Block_No              0.097782   0.193499   0.505    0.613    \n",
      "In_Block_No           0.010945   0.029804   0.367    0.713    \n",
      "Block_No:In_Block_No -0.003553   0.013855  -0.256    0.798    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N\n",
      "Cndtncnfsng -0.222                     \n",
      "Block_No    -0.652  0.005              \n",
      "In_Block_No -0.623  0.019  0.803       \n",
      "Blc_N:I_B_N  0.578 -0.009 -0.877 -0.926\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.0571756 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                     npar  Sum Sq Mean Sq F value\n",
      "Condition               1 20.8190 20.8190 20.8190\n",
      "Block_No                1  0.3534  0.3534  0.3534\n",
      "In_Block_No             1  0.1150  0.1150  0.1150\n",
      "Block_No:In_Block_No    1  0.0636  0.0636  0.0636\n",
      "Correctness ~ 1 + Block_No + Condition * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.00979904 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Block_No + Condition * In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1220.9   1280.9   -599.5   1198.9     1716 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.2378  0.0858  0.1988  0.3381  3.7619 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0063   1.0031        \n",
      "             Conditionconfusing 0.3006   0.5483   -0.65\n",
      " number      (Intercept)        2.2354   1.4951        \n",
      "             Conditionconfusing 1.6341   1.2783   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                                Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                     2.990009   0.521005   5.739 9.53e-09 ***\n",
      "Block_No                        0.052800   0.093136   0.567 0.570772    \n",
      "Conditionconfusing             -1.727162   0.473879  -3.645 0.000268 ***\n",
      "In_Block_No                    -0.004423   0.018900  -0.234 0.814965    \n",
      "Conditionconfusing:In_Block_No  0.012639   0.023485   0.538 0.590479    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Blck_N Cndtnc In_B_N\n",
      "Block_No    -0.353                     \n",
      "Cndtncnfsng -0.442  0.017              \n",
      "In_Block_No -0.477  0.003  0.523       \n",
      "Cndtn:I_B_N  0.395 -0.035 -0.625 -0.805\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.00979904 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                      npar  Sum Sq Mean Sq F value\n",
      "Block_No                 1  0.3286  0.3286  0.3286\n",
      "Condition                1 20.8298 20.8298 20.8298\n",
      "In_Block_No              1  0.1159  0.1159  0.1159\n",
      "Condition:In_Block_No    1  0.2771  0.2771  0.2771\n"
     ]
    }
   ],
   "source": [
    "# 2/3 models with 1 2wise interaction failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + Block_No * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no i in_block_no 2wise binomial')\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + Block_No + {CONDITION} * In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion i in_block_no block_no 2wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6ce24c51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition * Block_No + In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n",
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition * Block_No + In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1218.8   1278.8   -598.4   1196.8     1716 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.3135  0.0829  0.1980  0.3422  3.7992 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0184   1.0092        \n",
      "             Conditionconfusing 0.3068   0.5539   -0.66\n",
      " number      (Intercept)        2.2699   1.5066        \n",
      "             Conditionconfusing 1.6343   1.2784   0.10 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                             Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                  2.517322   0.525789   4.788 1.69e-06 ***\n",
      "Conditionconfusing          -0.981402   0.522277  -1.879   0.0602 .  \n",
      "Block_No                     0.240972   0.150522   1.601   0.1094    \n",
      "In_Block_No                  0.004504   0.011225   0.401   0.6882    \n",
      "Conditionconfusing:Block_No -0.302330   0.191986  -1.575   0.1153    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N In_B_N\n",
      "Cndtncnfsng -0.468                     \n",
      "Block_No    -0.529  0.532              \n",
      "In_Block_No -0.275  0.049  0.002       \n",
      "Cndtncn:B_N  0.425 -0.704 -0.784 -0.037\n",
      "Analysis of Variance Table\n",
      "                   npar  Sum Sq Mean Sq F value\n",
      "Condition             1 20.5491 20.5491 20.5491\n",
      "Block_No              1  0.3275  0.3275  0.3275\n",
      "In_Block_No           1  0.1130  0.1130  0.1130\n",
      "Condition:Block_No    1  2.3871  2.3871  2.3871\n"
     ]
    }
   ],
   "source": [
    "# 1 models with 1 2wise interaction converged Correctness ~ 1 + Condition + Block_No * In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No + In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion i block_no in_block_no 2wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5d0fb008",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + Block_No + In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.00381907 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition + Block_No + In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1219.2   1273.7   -599.6   1199.2     1717 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.0047  0.0850  0.1988  0.3397  3.8059 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0031   1.0015        \n",
      "             Conditionconfusing 0.2978   0.5457   -0.65\n",
      " number      (Intercept)        2.2412   1.4971        \n",
      "             Conditionconfusing 1.6335   1.2781   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                    Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)         2.879897   0.477696   6.029 1.65e-09 ***\n",
      "Conditionconfusing -1.568187   0.369574  -4.243 2.20e-05 ***\n",
      "Block_No            0.054481   0.093049   0.586    0.558    \n",
      "In_Block_No         0.003814   0.011216   0.340    0.734    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N\n",
      "Cndtncnfsng -0.269              \n",
      "Block_No    -0.370 -0.006       \n",
      "In_Block_No -0.285  0.030 -0.043\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.00381907 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "            npar  Sum Sq Mean Sq F value\n",
      "Condition      1 20.8207 20.8207 20.8207\n",
      "Block_No       1  0.3525  0.3525  0.3525\n",
      "In_Block_No    1  0.1121  0.1121  0.1121\n"
     ]
    }
   ],
   "source": [
    "# reduction round 1 test 1: remove interaction failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + Block_No + In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion in_block_no block_no 1wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "60c33155",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition * Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In checkConv(attr(opt, \"derivs\"), opt$par, ctrl = control$checkConv,  :\n",
       "  Model failed to converge with max|grad| = 0.00324067 (tol = 0.002, component 1)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition * Block_No + (1 + Condition | Participant) +  \n",
      "    (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1217.0   1271.5   -598.5   1197.0     1717 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-8.4351  0.0833  0.1986  0.3411  3.8418 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0181   1.0090        \n",
      "             Conditionconfusing 0.3018   0.5493   -0.66\n",
      " number      (Intercept)        2.2689   1.5063        \n",
      "             Conditionconfusing 1.6367   1.2793   0.10 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                            Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)                   2.5754     0.5055   5.095 3.49e-07 ***\n",
      "Conditionconfusing           -0.9922     0.5214  -1.903   0.0571 .  \n",
      "Block_No                      0.2409     0.1506   1.600   0.1096    \n",
      "Conditionconfusing:Block_No  -0.2993     0.1919  -1.560   0.1188    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc Blck_N\n",
      "Cndtncnfsng -0.473              \n",
      "Block_No    -0.550  0.533       \n",
      "Cndtncn:B_N  0.431 -0.704 -0.785\n",
      "optimizer (Nelder_Mead) convergence code: 0 (OK)\n",
      "Model failed to converge with max|grad| = 0.00324067 (tol = 0.002, component 1)\n",
      "\n",
      "Analysis of Variance Table\n",
      "                   npar  Sum Sq Mean Sq F value\n",
      "Condition             1 20.5679 20.5679 20.5679\n",
      "Block_No              1  0.3258  0.3258  0.3258\n",
      "Condition:Block_No    1  2.3427  2.3427  2.3427\n"
     ]
    }
   ],
   "source": [
    "# reduction round 1 test 2: remove factor in_block_no failed to converge\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} * Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion i block_no 2wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "82395422",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n",
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: \n",
      "Correctness ~ 1 + Condition + Block_No + (1 + Condition | Participant) +  \n",
      "    (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1217.3   1266.4   -599.6   1199.3     1718 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.1091  0.0852  0.1988  0.3401  3.8423 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0046   1.002         \n",
      "             Conditionconfusing 0.2948   0.543    -0.65\n",
      " number      (Intercept)        2.2408   1.497         \n",
      "             Conditionconfusing 1.6353   1.279    0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                   Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)         2.92621    0.45792   6.390 1.66e-10 ***\n",
      "Conditionconfusing -1.57211    0.36934  -4.257 2.08e-05 ***\n",
      "Block_No            0.05623    0.09297   0.605    0.545    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc\n",
      "Cndtncnfsng -0.272       \n",
      "Block_No    -0.399 -0.005\n",
      "Analysis of Variance Table\n",
      "          npar  Sum Sq Mean Sq F value\n",
      "Condition    1 20.8303 20.8303 20.8303\n",
      "Block_No     1  0.3549  0.3549  0.3549\n"
     ]
    }
   ],
   "source": [
    "# further reduction only 2 factors condition block_no\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion block_no 1wise binomial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2a828fa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + In_Block_No + (1 + Condition | Participant) + (1 + Condition | number)\n",
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition + In_Block_No + (1 + Condition |  \n",
      "    Participant) + (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1217.5   1266.6   -599.8   1199.5     1718 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.2383  0.0851  0.1984  0.3404  3.8010 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0026   1.0013        \n",
      "             Conditionconfusing 0.2977   0.5456   -0.65\n",
      " number      (Intercept)        2.2421   1.4974        \n",
      "             Conditionconfusing 1.6315   1.2773   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                    Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)         2.984251   0.443765   6.725 1.76e-11 ***\n",
      "Conditionconfusing -1.567424   0.369459  -4.242 2.21e-05 ***\n",
      "In_Block_No         0.004126   0.011205   0.368    0.713    \n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr) Cndtnc\n",
      "Cndtncnfsng -0.292       \n",
      "In_Block_No -0.323  0.029\n",
      "Analysis of Variance Table\n",
      "            npar  Sum Sq Mean Sq F value\n",
      "Condition      1 20.8485 20.8485 20.8485\n",
      "In_Block_No    1  0.1315  0.1315  0.1315\n"
     ]
    }
   ],
   "source": [
    "# further reduction only 2 factors condition in_block_no\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + In_Block_No + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion in_block_no 1wise binomial')\n",
    "# best model compared to below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "138c2dec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctness ~ 1 + Condition + (1 + Condition | Participant) + (1 + Condition | number)\n",
      "Generalized linear mixed model fit by maximum likelihood (Laplace\n",
      "  Approximation) [glmerMod]\n",
      " Family: binomial  ( logit )\n",
      "Formula: Correctness ~ 1 + Condition + (1 + Condition | Participant) +  \n",
      "    (1 + Condition | number)\n",
      "   Data: data\n",
      "\n",
      "     AIC      BIC   logLik deviance df.resid \n",
      "  1215.6   1259.3   -599.8   1199.6     1719 \n",
      "\n",
      "Scaled residuals: \n",
      "    Min      1Q  Median      3Q     Max \n",
      "-9.3594  0.0856  0.1974  0.3407  3.8404 \n",
      "\n",
      "Random effects:\n",
      " Groups      Name               Variance Std.Dev. Corr \n",
      " Participant (Intercept)        1.0033   1.0017        \n",
      "             Conditionconfusing 0.2941   0.5423   -0.65\n",
      " number      (Intercept)        2.2404   1.4968        \n",
      "             Conditionconfusing 1.6341   1.2783   0.11 \n",
      "Number of obs: 1727, groups:  Participant, 24; number, 24\n",
      "\n",
      "Fixed effects:\n",
      "                   Estimate Std. Error z value Pr(>|z|)    \n",
      "(Intercept)          3.0374     0.4198   7.235 4.65e-13 ***\n",
      "Conditionconfusing  -1.5716     0.3692  -4.257 2.07e-05 ***\n",
      "---\n",
      "Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n",
      "\n",
      "Correlation of Fixed Effects:\n",
      "            (Intr)\n",
      "Cndtncnfsng -0.298\n",
      "Analysis of Variance Table\n",
      "          npar Sum Sq Mean Sq F value\n",
      "Condition    1 20.858  20.858  20.858\n"
     ]
    }
   ],
   "source": [
    "# ultimate reduction, converges\n",
    "res2 = create_glmer_in_R(\n",
    "    f'{BEHAVIORAL_COLUMN_CORRECTNESS} ~ 1 + {CONDITION} + ' +\n",
    "    f'(1 + {CONDITION} | {PARTICIPANT}) + ' +\n",
    "    f'(1 + {CONDITION} | {SNIPPET_COLUMN_NUMBER})',\n",
    "    behavioral_data, LMEM_FAMILY_BINOMIAL, description='correctness confusion binomial final')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}