work/dataset/2_inverse_normalize.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "163f62b8-7ac3-41f5-82f1-7a5c9aff67df",
   "metadata": {},
   "source": [
    "Now we need to perform an inverse text normalization to obtain examples to train an LLM on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4608e1d2-a7d7-4731-9336-2ec6cf7988d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from tqdm.notebook import tqdm\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a16fcf55-dfda-4608-ad85-53d40b3ec413",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict, Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d02c5a1-445b-4949-8715-67357b4bed79",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATHES = {\n",
    "    \"librusec\": {\n",
    "        \"input\": \"/home/jovyan/data/librusec.jsonl\",\n",
    "        \"output\": \"/home/jovyan/data/librusec_pairs.jsonl\",\n",
    "    },\n",
    "    \"ficbook\": {\n",
    "        \"input\": \"/home/jovyan/data/ficbook.jsonl\",\n",
    "        \"output\": \"/home/jovyan/data/ficbook_pairs.jsonl\",\n",
    "    },\n",
    "    \"pikabu\": {\n",
    "        \"input\": \"/home/jovyan/data/pikabu.jsonl\",\n",
    "        \"output\": \"/home/jovyan/data/pikabu_pairs.jsonl\",\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2e3e3bc-2840-42e6-a2d0-15a33a7a5799",
   "metadata": {},
   "source": [
    "Uncomment pairs of input and output file pathes below one by one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34c439b2-24ea-4084-adab-ab14e43601eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET = \"librusec\"\n",
    "DATASET = \"ficbook\"\n",
    "DATASET = \"pikabu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "809971b9-395a-407a-82fd-f338f3530bd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install jsonlines pysimdjson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b528e32c-999a-4fb3-8abc-64bc663bb1f6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import simdjson\n",
    "\n",
    "\n",
    "data = []\n",
    "parser = simdjson.Parser()\n",
    "with open(PATHES[DATASET][\"input\"]) as f:\n",
    "    for i, line in tqdm(enumerate(f)):\n",
    "        data.append(parser.parse(line).as_dict())\n",
    "random.choice(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb3cb29-3ff5-4228-b2a0-9b5a58cd49df",
   "metadata": {},
   "outputs": [],
   "source": [
    "for elem in tqdm(data):\n",
    "    for text_elem in elem[\"texts\"]:\n",
    "        text_elem[\"matches\"] = [_[\"form\"] for _ in text_elem[\"matches\"]]\n",
    "random.choice(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd154913-097f-4432-a480-85d0d065d664",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "~~I end up using `nemo_text_processing` as it is fast.~~\n",
    "~~This part is vital so better use another one here next time.~~\n",
    "\n",
    "~~Seems ok for a first attempt anyway.~~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "520b9522-e296-49e7-90b2-8ec56856f427",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer\n",
    "# inverse_normalizer = InverseNormalizer(lang='ru')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41e140cd-3174-46b2-9b0e-852a6aad6b8e",
   "metadata": {},
   "source": [
    "Use [another](https://github.com/flockentanz/word_to_number_ru) itn now as NeMo covers not much of what I want to.\n",
    "\n",
    "Do `pip install natasha==0.10.0 yargy==0.12.0` first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60916ab0-db85-4fc9-a636-24d7b85546ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from word_to_number_ru.extractor import NumberExtractor\n",
    "\n",
    "\n",
    "ne = NumberExtractor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "869c24c2-89c5-4d09-88eb-69b08506d7a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(len(_[\"texts\"]) for _ in data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03efe745-e7fb-40f0-8d50-0a7fb50fc8f0",
   "metadata": {},
   "source": [
    "Make some **ROUGH** balancing here.\n",
    "First, perform a quick distribution check."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbda9606-a856-4681-872f-07c6fd869113",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "counter = Counter()\n",
    "for elem in tqdm(data):\n",
    "    for text in elem[\"texts\"]:\n",
    "        counter.update(text[\"matches\"])\n",
    "sum(counter.values()), len(counter), counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4ceca62-9db6-43d3-b60a-5d1176195597",
   "metadata": {},
   "source": [
    "Seems too much of numbers less than 10.\n",
    "So we strip a part of the most common numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00d930e4-1b29-41e0-8942-15faeb52c54b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from multiprocessing import Process, Queue\n",
    "from multiprocessing import Pool\n",
    "from itertools import islice, chain\n",
    "\n",
    "\n",
    "queue = Queue()\n",
    "occurances = Counter()\n",
    "occurances_limit = sum(counter.values()) / 25  # len(counter)\n",
    "print(f\"{occurances_limit=}\")\n",
    "\n",
    "\n",
    "def inverse_normalize(elem):\n",
    "    result = []\n",
    "    for i_text, text in enumerate(elem[\"texts\"]):\n",
    "        matches = [match for match in text[\"matches\"] if occurances[match] < occurances_limit]\n",
    "        if not matches:\n",
    "            continue\n",
    "        occurances.update(text[\"matches\"])\n",
    "        # itn = inverse_normalizer.inverse_normalize(text[\"text\"], verbose=False)\n",
    "        itn = ne(text[\"text\"])\n",
    "        if itn == text[\"text\"]:\n",
    "            continue\n",
    "        result.append({\n",
    "            \"tn\": text[\"text\"],\n",
    "            \"itn\": itn,\n",
    "            \"orig_index\": elem[\"index\"],\n",
    "            \"text_index\": i_text\n",
    "        })\n",
    "    return result\n",
    "\n",
    "\n",
    "b_size = 10000  # you may wnat to decrease it in case of librusec as its texts are vast\n",
    "f = open(PATHES[DATASET][\"output\"], \"w\")\n",
    "i = 0\n",
    "pbar = tqdm(total=len(data))\n",
    "while i < len(data):\n",
    "    with Pool(15) as p:\n",
    "        result = p.imap_unordered(inverse_normalize, data[i:i + b_size])\n",
    "        for elem in chain(*result):\n",
    "            json.dump(elem, f, ensure_ascii=False)\n",
    "            f.write(\"\\n\")\n",
    "        i += b_size\n",
    "        pbar.update(b_size)\n",
    "pbar.close()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "151ddaa9-3f2b-453e-9b16-f19598bfa5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "occurances"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}