calculate_word_frequencies_visual.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Source word and bigram frequency "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Syllables</th>\n",
       "      <th>Syllable Occurrence Counts</th>\n",
       "      <th>Bigrams</th>\n",
       "      <th>Bigram Occurrence Counts</th>\n",
       "      <th>Zipf</th>\n",
       "      <th>FREQcount</th>\n",
       "      <th>Lg10WF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>toen</td>\n",
       "      <td>[\"'tun\"]</td>\n",
       "      <td>[47826.0]</td>\n",
       "      <td>['to', 'oe', 'en']</td>\n",
       "      <td>[2236591.0, 837795.0, 8542439.0]</td>\n",
       "      <td>6.032333</td>\n",
       "      <td>47514.0</td>\n",
       "      <td>4.676822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>de</td>\n",
       "      <td>[\"'d@\"]</td>\n",
       "      <td>[1061177.0]</td>\n",
       "      <td>['de']</td>\n",
       "      <td>[2559661.0]</td>\n",
       "      <td>7.381291</td>\n",
       "      <td>1061177.0</td>\n",
       "      <td>6.025788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>barkeeper</td>\n",
       "      <td>[\"'bAr\", 'ki', 'p@r']</td>\n",
       "      <td>[3329.0, 3669.0, 20167.0]</td>\n",
       "      <td>['ba', 'ar', 'rk', 'ke', 'ee', 'ep', 'pe', 'er']</td>\n",
       "      <td>[584273.0, 1754491.0, 2174749.0, 3143026.0, 94...</td>\n",
       "      <td>3.622674</td>\n",
       "      <td>184.0</td>\n",
       "      <td>2.264818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>die</td>\n",
       "      <td>[\"'di\"]</td>\n",
       "      <td>[325903.0]</td>\n",
       "      <td>['di', 'ie']</td>\n",
       "      <td>[1050254.0, 2076673.0]</td>\n",
       "      <td>6.853871</td>\n",
       "      <td>315041.0</td>\n",
       "      <td>5.498367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>irritante</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['ir', 'rr', 'ri', 'it', 'ta', 'an', 'nt', 'te']</td>\n",
       "      <td>[2884214.0, 200692.0, 1787712.0, 333345.0, 892...</td>\n",
       "      <td>3.284921</td>\n",
       "      <td>84.0</td>\n",
       "      <td>1.924279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937</th>\n",
       "      <td>blije</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['bl', 'li', 'ij', 'je']</td>\n",
       "      <td>[152155.0, 3369152.0, 1639122.0, 913788.0]</td>\n",
       "      <td>3.485836</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.127105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938</th>\n",
       "      <td>jongeren</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>['jo', 'on', 'ng', 'ge', 'er', 're', 'en']</td>\n",
       "      <td>[866375.0, 2451854.0, 2927082.0, 3692145.0, 67...</td>\n",
       "      <td>3.965097</td>\n",
       "      <td>406.0</td>\n",
       "      <td>2.608526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1939</th>\n",
       "      <td>intens</td>\n",
       "      <td>['In', \"'tEns\"]</td>\n",
       "      <td>[39087.0, 264.0]</td>\n",
       "      <td>['in', 'nt', 'te', 'en', 'ns']</td>\n",
       "      <td>[2264626.0, 1477178.0, 3522714.0, 8542439.0, 6...</td>\n",
       "      <td>3.778748</td>\n",
       "      <td>264.0</td>\n",
       "      <td>2.421604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1940</th>\n",
       "      <td>plezier</td>\n",
       "      <td>['pl@', \"'zir\"]</td>\n",
       "      <td>[5981.0, 5884.0]</td>\n",
       "      <td>['pl', 'le', 'ez', 'zi', 'ie', 'er']</td>\n",
       "      <td>[97171.0, 4397946.0, 100963.0, 123482.0, 20766...</td>\n",
       "      <td>5.093966</td>\n",
       "      <td>5475.0</td>\n",
       "      <td>3.738384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1941</th>\n",
       "      <td>woonplaats</td>\n",
       "      <td>[\"'wom\", 'plats']</td>\n",
       "      <td>[175.0, 2617.0]</td>\n",
       "      <td>['wo', 'oo', 'on', 'np', 'pl', 'la', 'aa', 'at...</td>\n",
       "      <td>[933919.0, 3901750.0, 2451854.0, 16967.0, 9717...</td>\n",
       "      <td>3.241993</td>\n",
       "      <td>76.0</td>\n",
       "      <td>1.880814</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1942 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Word              Syllables Syllable Occurrence Counts  \\\n",
       "0           toen               [\"'tun\"]                  [47826.0]   \n",
       "1             de                [\"'d@\"]                [1061177.0]   \n",
       "2      barkeeper  [\"'bAr\", 'ki', 'p@r']  [3329.0, 3669.0, 20167.0]   \n",
       "3            die                [\"'di\"]                 [325903.0]   \n",
       "4      irritante                    NaN                         []   \n",
       "...          ...                    ...                        ...   \n",
       "1937       blije                    NaN                         []   \n",
       "1938    jongeren                    NaN                         []   \n",
       "1939      intens        ['In', \"'tEns\"]           [39087.0, 264.0]   \n",
       "1940     plezier        ['pl@', \"'zir\"]           [5981.0, 5884.0]   \n",
       "1941  woonplaats      [\"'wom\", 'plats']            [175.0, 2617.0]   \n",
       "\n",
       "                                                Bigrams  \\\n",
       "0                                    ['to', 'oe', 'en']   \n",
       "1                                                ['de']   \n",
       "2      ['ba', 'ar', 'rk', 'ke', 'ee', 'ep', 'pe', 'er']   \n",
       "3                                          ['di', 'ie']   \n",
       "4      ['ir', 'rr', 'ri', 'it', 'ta', 'an', 'nt', 'te']   \n",
       "...                                                 ...   \n",
       "1937                           ['bl', 'li', 'ij', 'je']   \n",
       "1938         ['jo', 'on', 'ng', 'ge', 'er', 're', 'en']   \n",
       "1939                     ['in', 'nt', 'te', 'en', 'ns']   \n",
       "1940               ['pl', 'le', 'ez', 'zi', 'ie', 'er']   \n",
       "1941  ['wo', 'oo', 'on', 'np', 'pl', 'la', 'aa', 'at...   \n",
       "\n",
       "                               Bigram Occurrence Counts      Zipf  FREQcount  \\\n",
       "0                      [2236591.0, 837795.0, 8542439.0]  6.032333    47514.0   \n",
       "1                                           [2559661.0]  7.381291  1061177.0   \n",
       "2     [584273.0, 1754491.0, 2174749.0, 3143026.0, 94...  3.622674      184.0   \n",
       "3                                [1050254.0, 2076673.0]  6.853871   315041.0   \n",
       "4     [2884214.0, 200692.0, 1787712.0, 333345.0, 892...  3.284921       84.0   \n",
       "...                                                 ...       ...        ...   \n",
       "1937         [152155.0, 3369152.0, 1639122.0, 913788.0]  3.485836      134.0   \n",
       "1938  [866375.0, 2451854.0, 2927082.0, 3692145.0, 67...  3.965097      406.0   \n",
       "1939  [2264626.0, 1477178.0, 3522714.0, 8542439.0, 6...  3.778748      264.0   \n",
       "1940  [97171.0, 4397946.0, 100963.0, 123482.0, 20766...  5.093966     5475.0   \n",
       "1941  [933919.0, 3901750.0, 2451854.0, 16967.0, 9717...  3.241993       76.0   \n",
       "\n",
       "        Lg10WF  \n",
       "0     4.676822  \n",
       "1     6.025788  \n",
       "2     2.264818  \n",
       "3     5.498367  \n",
       "4     1.924279  \n",
       "...        ...  \n",
       "1937  2.127105  \n",
       "1938  2.608526  \n",
       "1939  2.421604  \n",
       "1940  3.738384  \n",
       "1941  1.880814  \n",
       "\n",
       "[1942 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mous_words = pd.read_csv('mous_words_syllable_bigram_frequencies.csv')\n",
    "mous_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Word', 'Syllables', 'Syllable Occurrence Counts', 'Bigrams',\n",
       "       'Bigram Occurrence Counts', 'Zipf', 'FREQcount', 'Lg10WF'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mous_words.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = Path('/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source') #change this to the source directory that contains the 'transcription' CSVs \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sub-V1008\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1008/func/sub-V1008_transcription.csv\n",
      "sub-V1081\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1081/func/sub-V1081_transcription.csv\n",
      "sub-V1077\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1077/func/sub-V1077_transcription.csv\n",
      "sub-V1065\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1065/func/sub-V1065_transcription.csv\n",
      "sub-V1011\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1011/func/sub-V1011_transcription.csv\n",
      "sub-V1086\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1086/func/sub-V1086_transcription.csv\n",
      "sub-V1099\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1099/func/sub-V1099_transcription.csv\n",
      "sub-V1049\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1049/func/sub-V1049_transcription.csv\n",
      "sub-V1104\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1104/func/sub-V1104_transcription.csv\n",
      "sub-V1100\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1100/func/sub-V1100_transcription.csv\n",
      "sub-V1042\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1042/func/sub-V1042_transcription.csv\n",
      "sub-V1010\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1010/func/sub-V1010_transcription.csv\n",
      "sub-V1015\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1015/func/sub-V1015_transcription.csv\n",
      "sub-V1089\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1089/func/sub-V1089_transcription.csv\n",
      "sub-V1032\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1032/func/sub-V1032_transcription.csv\n",
      "sub-V1117\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1117/func/sub-V1117_transcription.csv\n",
      "sub-V1098\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1098/func/sub-V1098_transcription.csv\n",
      "sub-V1068\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1068/func/sub-V1068_transcription.csv\n",
      "sub-V1009\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1009/func/sub-V1009_transcription.csv\n",
      "sub-V1063\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1063/func/sub-V1063_transcription.csv\n",
      "sub-V1022\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1022/func/sub-V1022_transcription.csv\n",
      "sub-V1055\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1055/func/sub-V1055_transcription.csv\n",
      "sub-V1072\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1072/func/sub-V1072_transcription.csv\n",
      "sub-V1070\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1070/func/sub-V1070_transcription.csv\n",
      "sub-V1034\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1034/func/sub-V1034_transcription.csv\n",
      "sub-V1114\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1114/func/sub-V1114_transcription.csv\n",
      "sub-V1069\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1069/func/sub-V1069_transcription.csv\n",
      "sub-V1050\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1050/func/sub-V1050_transcription.csv\n",
      "sub-V1002\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1002/func/sub-V1002_transcription.csv\n",
      "sub-V1101\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1101/func/sub-V1101_transcription.csv\n",
      "sub-V1080\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1080/func/sub-V1080_transcription.csv\n",
      "sub-V1039\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1039/func/sub-V1039_transcription.csv\n",
      "sub-V1053\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1053/func/sub-V1053_transcription.csv\n",
      "sub-V1059\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1059/func/sub-V1059_transcription.csv\n",
      "sub-V1026\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1026/func/sub-V1026_transcription.csv\n",
      "sub-V1058\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1058/func/sub-V1058_transcription.csv\n",
      "sub-V1006\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1006/func/sub-V1006_transcription.csv\n",
      "sub-V1076\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1076/func/sub-V1076_transcription.csv\n",
      "sub-V1078\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1078/func/sub-V1078_transcription.csv\n",
      "sub-V1103\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1103/func/sub-V1103_transcription.csv\n",
      "sub-V1016\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1016/func/sub-V1016_transcription.csv\n",
      "sub-V1003\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1003/func/sub-V1003_transcription.csv\n",
      "sub-V1061\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1061/func/sub-V1061_transcription.csv\n",
      "sub-V1074\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1074/func/sub-V1074_transcription.csv\n",
      "sub-V1105\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1105/func/sub-V1105_transcription.csv\n",
      "sub-V1052\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1052/func/sub-V1052_transcription.csv\n",
      "sub-V1093\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1093/func/sub-V1093_transcription.csv\n",
      "sub-V1027\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1027/func/sub-V1027_transcription.csv\n",
      "sub-V1113\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1113/func/sub-V1113_transcription.csv\n",
      "sub-V1029\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1029/func/sub-V1029_transcription.csv\n",
      "sub-V1045\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1045/func/sub-V1045_transcription.csv\n",
      "sub-V1116\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1116/func/sub-V1116_transcription.csv\n",
      "sub-V1110\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1110/func/sub-V1110_transcription.csv\n",
      "sub-V1044\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1044/func/sub-V1044_transcription.csv\n",
      "sub-V1087\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1087/func/sub-V1087_transcription.csv\n",
      "sub-V1013\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1013/func/sub-V1013_transcription.csv\n",
      "sub-V1007\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1007/func/sub-V1007_transcription.csv\n",
      "sub-V1102\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1102/func/sub-V1102_transcription.csv\n",
      "sub-V1046\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1046/func/sub-V1046_transcription.csv\n",
      "sub-V1075\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1075/func/sub-V1075_transcription.csv\n",
      "sub-V1037\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1037/func/sub-V1037_transcription.csv\n",
      "sub-V1107\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1107/func/sub-V1107_transcription.csv\n",
      "sub-V1012\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1012/func/sub-V1012_transcription.csv\n",
      "sub-V1090\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1090/func/sub-V1090_transcription.csv\n",
      "sub-V1035\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1035/func/sub-V1035_transcription.csv\n",
      "sub-V1020\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1020/func/sub-V1020_transcription.csv\n",
      "sub-V1115\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1115/func/sub-V1115_transcription.csv\n",
      "sub-V1025\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1025/func/sub-V1025_transcription.csv\n",
      "sub-V1108\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1108/func/sub-V1108_transcription.csv\n",
      "sub-V1109\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1109/func/sub-V1109_transcription.csv\n",
      "sub-V1088\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1088/func/sub-V1088_transcription.csv\n",
      "sub-V1094\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1094/func/sub-V1094_transcription.csv\n",
      "sub-V1033\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1033/func/sub-V1033_transcription.csv\n",
      "sub-V1028\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1028/func/sub-V1028_transcription.csv\n",
      "sub-V1048\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1048/func/sub-V1048_transcription.csv\n",
      "sub-V1073\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1073/func/sub-V1073_transcription.csv\n",
      "sub-V1040\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1040/func/sub-V1040_transcription.csv\n",
      "sub-V1066\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1066/func/sub-V1066_transcription.csv\n",
      "sub-V1054\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1054/func/sub-V1054_transcription.csv\n",
      "sub-V1030\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1030/func/sub-V1030_transcription.csv\n",
      "sub-V1019\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1019/func/sub-V1019_transcription.csv\n",
      "sub-V1106\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1106/func/sub-V1106_transcription.csv\n",
      "sub-V1005\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1005/func/sub-V1005_transcription.csv\n",
      "sub-V1017\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1017/func/sub-V1017_transcription.csv\n",
      "sub-V1001\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1001/func/sub-V1001_transcription.csv\n",
      "sub-V1038\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1038/func/sub-V1038_transcription.csv\n",
      "sub-V1057\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1057/func/sub-V1057_transcription.csv\n",
      "sub-V1024\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1024/func/sub-V1024_transcription.csv\n",
      "sub-V1092\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1092/func/sub-V1092_transcription.csv\n",
      "sub-V1111\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1111/func/sub-V1111_transcription.csv\n",
      "sub-V1079\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1079/func/sub-V1079_transcription.csv\n",
      "sub-V1004\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1004/func/sub-V1004_transcription.csv\n",
      "sub-V1031\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1031/func/sub-V1031_transcription.csv\n",
      "sub-V1095\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1095/func/sub-V1095_transcription.csv\n",
      "sub-V1085\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1085/func/sub-V1085_transcription.csv\n",
      "sub-V1064\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1064/func/sub-V1064_transcription.csv\n",
      "sub-V1084\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1084/func/sub-V1084_transcription.csv\n",
      "sub-V1083\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1083/func/sub-V1083_transcription.csv\n",
      "sub-V1036\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1036/func/sub-V1036_transcription.csv\n",
      "sub-V1097\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1097/func/sub-V1097_transcription.csv\n",
      "sub-V1062\n",
      "/media/neel/MOUS/MOUS/MOUS/SynologyDrive/source/sub-V1062/func/sub-V1062_transcription.csv\n"
     ]
    }
   ],
   "source": [
    "for subject in source.iterdir():\n",
    "    if subject.is_dir() and subject.name.startswith('sub-V'):\n",
    "        print(subject.name)\n",
    "        transcription = subject / 'func' / f'{subject.name}_transcription.csv'\n",
    "        if transcription.exists():\n",
    "            print(transcription)\n",
    "            df = pd.read_csv(transcription)\n",
    "            # Merge df with mous_words on 'Word'\n",
    "            df = pd.merge(df, mous_words, on='Word', how='left')\n",
    "            # Ensure 'Bigram Occurrence Counts' are lists of numbers\n",
    "            df['Bigram Occurrence Counts'] = df['Bigram Occurrence Counts'].apply(lambda x: eval(x) if isinstance(x, str) else x)\n",
    "            # Log10 of the minimum value of Bigram Occurrence Counts\n",
    "            df['log10_Min_Bigram'] = df['Bigram Occurrence Counts'].apply(\n",
    "                lambda x: np.log10(min(x)) if isinstance(x, list) and len(x) > 0 and all(i > 0 for i in x) else 0)\n",
    "            # Log10 of the maximum value of Bigram Occurrence Counts\n",
    "            df['log10_Max_Bigram'] = df['Bigram Occurrence Counts'].apply(\n",
    "                lambda x: np.log10(max(x)) if isinstance(x, list) and len(x) > 0 and all(i > 0 for i in x) else 0)\n",
    "            # Log10 of the mean value of Bigram Occurrence Counts\n",
    "            df['log10_Mean_Bigram'] = df['Bigram Occurrence Counts'].apply(\n",
    "                lambda x: np.log10(np.mean(x)) if isinstance(x, list) and len(x) > 0 and all(i > 0 for i in x) else 0)\n",
    "            df['WordLength'] = df['Word'].apply(lambda x: len(x))\n",
    "            word_frequencies = df[['Onset','Word','Zipf','Lg10WF','WordLength']]\n",
    "            bigrams = df[['Onset','Word','Bigrams','log10_Min_Bigram','log10_Max_Bigram','log10_Mean_Bigram','WordLength']]\n",
    "            word_frequencies.to_csv(subject / 'func' / f'{subject.name}_word_frequencies.csv', index=False)\n",
    "            bigrams.to_csv(subject / 'func' / f'{subject.name}_bigrams_processed.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Onset</th>\n",
       "      <th>Word</th>\n",
       "      <th>Zipf</th>\n",
       "      <th>Lg10WF</th>\n",
       "      <th>WordLength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.142</td>\n",
       "      <td>de</td>\n",
       "      <td>7.381291</td>\n",
       "      <td>6.025788</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.674</td>\n",
       "      <td>ontworpen</td>\n",
       "      <td>4.061366</td>\n",
       "      <td>2.705008</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.842</td>\n",
       "      <td>die</td>\n",
       "      <td>6.853871</td>\n",
       "      <td>5.498367</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12.458</td>\n",
       "      <td>sprak</td>\n",
       "      <td>4.725275</td>\n",
       "      <td>3.369587</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13.260</td>\n",
       "      <td>heeft</td>\n",
       "      <td>6.559424</td>\n",
       "      <td>5.203919</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1375</th>\n",
       "      <td>1712.776</td>\n",
       "      <td>trompet</td>\n",
       "      <td>3.427384</td>\n",
       "      <td>2.068186</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1376</th>\n",
       "      <td>1713.742</td>\n",
       "      <td>uitkiest</td>\n",
       "      <td>2.770476</td>\n",
       "      <td>1.397940</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1377</th>\n",
       "      <td>1714.794</td>\n",
       "      <td>heeft</td>\n",
       "      <td>6.559424</td>\n",
       "      <td>5.203919</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1378</th>\n",
       "      <td>1715.578</td>\n",
       "      <td>geld</td>\n",
       "      <td>5.895894</td>\n",
       "      <td>4.540380</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1379</th>\n",
       "      <td>1716.278</td>\n",
       "      <td>genoeg</td>\n",
       "      <td>5.775656</td>\n",
       "      <td>4.420137</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1380 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Onset       Word      Zipf    Lg10WF  WordLength\n",
       "0       10.142         de  7.381291  6.025788           2\n",
       "1       10.674  ontworpen  4.061366  2.705008           9\n",
       "2       11.842        die  6.853871  5.498367           3\n",
       "3       12.458      sprak  4.725275  3.369587           5\n",
       "4       13.260      heeft  6.559424  5.203919           5\n",
       "...        ...        ...       ...       ...         ...\n",
       "1375  1712.776    trompet  3.427384  2.068186           7\n",
       "1376  1713.742   uitkiest  2.770476  1.397940           8\n",
       "1377  1714.794      heeft  6.559424  5.203919           5\n",
       "1378  1715.578       geld  5.895894  4.540380           4\n",
       "1379  1716.278     genoeg  5.775656  4.420137           6\n",
       "\n",
       "[1380 rows x 5 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Onset</th>\n",
       "      <th>Word</th>\n",
       "      <th>Bigrams</th>\n",
       "      <th>log10_Min_Bigram</th>\n",
       "      <th>log10_Max_Bigram</th>\n",
       "      <th>log10_Mean_Bigram</th>\n",
       "      <th>WordLength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.142</td>\n",
       "      <td>de</td>\n",
       "      <td>['de']</td>\n",
       "      <td>6.408182</td>\n",
       "      <td>6.408182</td>\n",
       "      <td>6.408182</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.674</td>\n",
       "      <td>ontworpen</td>\n",
       "      <td>['on', 'nt', 'tw', 'wo', 'or', 'rp', 'pe', 'en']</td>\n",
       "      <td>4.602624</td>\n",
       "      <td>6.931582</td>\n",
       "      <td>6.331510</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.842</td>\n",
       "      <td>die</td>\n",
       "      <td>['di', 'ie']</td>\n",
       "      <td>6.021294</td>\n",
       "      <td>6.317368</td>\n",
       "      <td>6.194088</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12.458</td>\n",
       "      <td>sprak</td>\n",
       "      <td>['sp', 'pr', 'ra', 'ak']</td>\n",
       "      <td>5.385215</td>\n",
       "      <td>6.346360</td>\n",
       "      <td>6.074015</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13.260</td>\n",
       "      <td>heeft</td>\n",
       "      <td>['he', 'ee', 'ef', 'ft']</td>\n",
       "      <td>5.228939</td>\n",
       "      <td>5.975295</td>\n",
       "      <td>5.723646</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1375</th>\n",
       "      <td>1712.776</td>\n",
       "      <td>trompet</td>\n",
       "      <td>['tr', 'ro', 'om', 'mp', 'pe', 'et']</td>\n",
       "      <td>4.709914</td>\n",
       "      <td>6.393353</td>\n",
       "      <td>6.118872</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1376</th>\n",
       "      <td>1713.742</td>\n",
       "      <td>uitkiest</td>\n",
       "      <td>['ui', 'it', 'tk', 'ki', 'ie', 'es', 'st']</td>\n",
       "      <td>4.368138</td>\n",
       "      <td>6.363066</td>\n",
       "      <td>6.031377</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1377</th>\n",
       "      <td>1714.794</td>\n",
       "      <td>heeft</td>\n",
       "      <td>['he', 'ee', 'ef', 'ft']</td>\n",
       "      <td>5.228939</td>\n",
       "      <td>5.975295</td>\n",
       "      <td>5.723646</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1378</th>\n",
       "      <td>1715.578</td>\n",
       "      <td>geld</td>\n",
       "      <td>['ge', 'el', 'ld']</td>\n",
       "      <td>5.924795</td>\n",
       "      <td>6.669429</td>\n",
       "      <td>6.486872</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1379</th>\n",
       "      <td>1716.278</td>\n",
       "      <td>genoeg</td>\n",
       "      <td>['ge', 'en', 'no', 'oe', 'eg']</td>\n",
       "      <td>5.923138</td>\n",
       "      <td>6.931582</td>\n",
       "      <td>6.478436</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1380 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Onset       Word                                           Bigrams  \\\n",
       "0       10.142         de                                            ['de']   \n",
       "1       10.674  ontworpen  ['on', 'nt', 'tw', 'wo', 'or', 'rp', 'pe', 'en']   \n",
       "2       11.842        die                                      ['di', 'ie']   \n",
       "3       12.458      sprak                          ['sp', 'pr', 'ra', 'ak']   \n",
       "4       13.260      heeft                          ['he', 'ee', 'ef', 'ft']   \n",
       "...        ...        ...                                               ...   \n",
       "1375  1712.776    trompet              ['tr', 'ro', 'om', 'mp', 'pe', 'et']   \n",
       "1376  1713.742   uitkiest        ['ui', 'it', 'tk', 'ki', 'ie', 'es', 'st']   \n",
       "1377  1714.794      heeft                          ['he', 'ee', 'ef', 'ft']   \n",
       "1378  1715.578       geld                                ['ge', 'el', 'ld']   \n",
       "1379  1716.278     genoeg                    ['ge', 'en', 'no', 'oe', 'eg']   \n",
       "\n",
       "      log10_Min_Bigram  log10_Max_Bigram  log10_Mean_Bigram  WordLength  \n",
       "0             6.408182          6.408182           6.408182           2  \n",
       "1             4.602624          6.931582           6.331510           9  \n",
       "2             6.021294          6.317368           6.194088           3  \n",
       "3             5.385215          6.346360           6.074015           5  \n",
       "4             5.228939          5.975295           5.723646           5  \n",
       "...                ...               ...                ...         ...  \n",
       "1375          4.709914          6.393353           6.118872           7  \n",
       "1376          4.368138          6.363066           6.031377           8  \n",
       "1377          5.228939          5.975295           5.723646           5  \n",
       "1378          5.924795          6.669429           6.486872           4  \n",
       "1379          5.923138          6.931582           6.478436           6  \n",
       "\n",
       "[1380 rows x 7 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dti",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}