AMHARIC_NEWS_CLASSIFICATION.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0e6abd2-0f79-4b39-a27b-73c1f360285f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Error loading stopwords: <urlopen error [Errno 11002]\n",
      "[nltk_data]     getaddrinfo failed>\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "#import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import SVC\n",
    "import warnings\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "\n",
    "np.random.seed(42) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "763a1afa-71a3-49a1-94f2-6b473e744be3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>views</th>\n",
       "      <th>article</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44810</th>\n",
       "      <td>በመዲናዋ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና ለማስጀመር የ20 ቢሊ...</td>\n",
       "      <td>ፖለቲካ</td>\n",
       "      <td>September 14, 2017</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>በአዲስ አበባ የተጀመሩ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና አዳዲስ...</td>\n",
       "      <td>https://waltainfo.com/am/29360/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18329</th>\n",
       "      <td>አሸንዳ፣ ሻደይ፣ አሸንድዬ፣ ሶለል፣ ማሪያና ዓይኒ ዋሪ በዓል በዩኔስኮ አ...</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>Jan 8, 2020</td>\n",
       "      <td>387</td>\n",
       "      <td>አዲስ አበባ፣ ታህሳስ 29፣ 2012 (ኤፍ.ቢ.ሲ) የአሸንዳ፣ ሻደይ፣ አሸ...</td>\n",
       "      <td>https://www.fanabc.com/%e1%8b%a8%e1%8a%a0%e1%8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17644</th>\n",
       "      <td>ግምታዊ ዋጋቸው ከ31 ነጥብ 6 ሚሊየን ብር በላይ የሆኑ የኮንትሮባንድ ዕ...</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>Mar 4, 2020</td>\n",
       "      <td>634</td>\n",
       "      <td>አዲስ አበባ ፣ የካቲት 25 ፣ 2012 (ኤፍ ቢ ሲ) ባለፉት አምስት ቀና...</td>\n",
       "      <td>https://www.fanabc.com/%e1%8c%8d%e1%88%9d%e1%8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32773</th>\n",
       "      <td>​ባምላክ ተሰማ የዓለም ዋንጫ ለመምራት እጩ ከሆኑ ዳኞች ውስጥ ተካተተ</td>\n",
       "      <td>ስፖርት</td>\n",
       "      <td>November 19, 2017</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>ሩሲያ በ2018 ለምታስተናግደው የዓለም ዋንጫ ፊፋ ጨዋታዎችን ሊመሩ የሚች...</td>\n",
       "      <td>https://soccerethiopia.net/football/31318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4139</th>\n",
       "      <td>“መንገድ ለሰው” መርሐ ግብር በየሳምንቱ እንደሚካሄድ ተገለፀ</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>August 7, 2019</td>\n",
       "      <td>32</td>\n",
       "      <td>አዲስ አበባ፤- “መንገድ ለሰው” በሚል በአዲስ አበባ ከተማ ሲካሄድ የነ...</td>\n",
       "      <td>https://www.press.et/Ama/?p=15606</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                headline    category  \\\n",
       "44810  በመዲናዋ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና ለማስጀመር የ20 ቢሊ...        ፖለቲካ   \n",
       "18329  አሸንዳ፣ ሻደይ፣ አሸንድዬ፣ ሶለል፣ ማሪያና ዓይኒ ዋሪ በዓል በዩኔስኮ አ...  ሀገር አቀፍ ዜና   \n",
       "17644  ግምታዊ ዋጋቸው ከ31 ነጥብ 6 ሚሊየን ብር በላይ የሆኑ የኮንትሮባንድ ዕ...  ሀገር አቀፍ ዜና   \n",
       "32773       ​ባምላክ ተሰማ የዓለም ዋንጫ ለመምራት እጩ ከሆኑ ዳኞች ውስጥ ተካተተ        ስፖርት   \n",
       "4139              “መንገድ ለሰው” መርሐ ግብር በየሳምንቱ እንደሚካሄድ ተገለፀ  ሀገር አቀፍ ዜና   \n",
       "\n",
       "                     date    views  \\\n",
       "44810  September 14, 2017  Unknown   \n",
       "18329         Jan 8, 2020      387   \n",
       "17644         Mar 4, 2020      634   \n",
       "32773   November 19, 2017  Unknown   \n",
       "4139       August 7, 2019       32   \n",
       "\n",
       "                                                 article  \\\n",
       "44810  በአዲስ አበባ የተጀመሩ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና አዳዲስ...   \n",
       "18329  አዲስ አበባ፣ ታህሳስ 29፣ 2012 (ኤፍ.ቢ.ሲ) የአሸንዳ፣ ሻደይ፣ አሸ...   \n",
       "17644  አዲስ አበባ ፣ የካቲት 25 ፣ 2012 (ኤፍ ቢ ሲ) ባለፉት አምስት ቀና...   \n",
       "32773  ሩሲያ በ2018 ለምታስተናግደው የዓለም ዋንጫ ፊፋ ጨዋታዎችን ሊመሩ የሚች...   \n",
       "4139    አዲስ አበባ፤- “መንገድ ለሰው” በሚል በአዲስ አበባ ከተማ ሲካሄድ የነ...   \n",
       "\n",
       "                                                    link  \n",
       "44810                    https://waltainfo.com/am/29360/  \n",
       "18329  https://www.fanabc.com/%e1%8b%a8%e1%8a%a0%e1%8...  \n",
       "17644  https://www.fanabc.com/%e1%8c%8d%e1%88%9d%e1%8...  \n",
       "32773          https://soccerethiopia.net/football/31318  \n",
       "4139                   https://www.press.et/Ama/?p=15606  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r\"Amharic News Dataset.csv\")##C:\\Users\\TEST\\Documents\\CS\\ML\\sample.csv\n",
    "news = data\n",
    "data = shuffle(data)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c502cdb4-f81a-4362-b0ff-4428e765d25c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 51483 entries, 44810 to 15795\n",
      "Data columns (total 6 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   headline  51470 non-null  object\n",
      " 1   category  51482 non-null  object\n",
      " 2   date      51483 non-null  object\n",
      " 3   views     51483 non-null  object\n",
      " 4   article   51483 non-null  object\n",
      " 5   link      51483 non-null  object\n",
      "dtypes: object(6)\n",
      "memory usage: 2.7+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0fb05683-e442-4b22-b89a-78032a86f920",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.dropna(subset=['article'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f86b99fd-127f-436b-bb72-5e987382b401",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "https://amharic.voanews.com//a/amb-tibor-nagy-in-addis-ababa-on-sudan-06-14-19/4960197.html                                                                                                                                                                                                                                                 3\n",
       "https://amharic.voanews.com//a/covid-africa/5535902.html                                                                                                                                                                                                                                                                                    3\n",
       "https://amharic.voanews.com//a/looking-ahead-Ethiopian-renaissance-dam-weaspire-with-voa-alula-kebede-june-2020/5470822.html                                                                                                                                                                                                                3\n",
       "https://amharic.voanews.com//a/ethiocracy-new-economic-system-abate-kassa-wiath-voa-alula-kebede-july-14-2019/5004185.html                                                                                                                                                                                                                  3\n",
       "https://amharic.voanews.com//a/ethiopia-tigrai-mekele/5679989.html                                                                                                                                                                                                                                                                          3\n",
       "                                                                                                                                                                                                                                                                                                                                           ..\n",
       "https://www.ethiopianreporter.com/content/%E1%89%A0%E1%88%B1%E1%8B%B3%E1%8A%95-%E1%8B%88%E1%89%B3%E1%8B%B0%E1%88%AE%E1%89%BD-%E1%89%B3%E1%8D%8D%E1%8A%90%E1%8B%8D-%E1%8B%A8%E1%88%9A%E1%8B%88%E1%88%B0%E1%8B%B1-%E1%8B%9C%E1%8C%8E%E1%89%BD-%E1%89%81%E1%8C%A5%E1%88%AD-%E1%8A%A5%E1%8B%A8%E1%8C%A8%E1%88%98%E1%88%A8-%E1%8A%90%E1%8B%8D    1\n",
       "https://www.press.et/Ama/?p=27087                                                                                                                                                                                                                                                                                                           1\n",
       "https://www.ethiopianreporter.com/article/12848                                                                                                                                                                                                                                                                                             1\n",
       "https://soccerethiopia.net/football/12241                                                                                                                                                                                                                                                                                                   1\n",
       "https://www.fanabc.com/%e1%89%b0%e1%89%80%e1%8c%a3%e1%8c%a3%e1%8b%ad-%e1%8d%88%e1%8a%95%e1%8c%86%e1%89%bd-%e1%8d%a3-%e1%8a%a0%e1%88%9e%e1%8a%92%e1%8b%a8%e1%88%9d-%e1%8a%93%e1%8b%ad%e1%89%b5%e1%88%ac%e1%89%b5%e1%8a%93/                                                                                                                   1\n",
       "Name: link, Length: 50778, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['link'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e22096b-4d82-4b1d-b93a-9b12edaea537",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ፖለቲካ', 'ሀገር አቀፍ ዜና', 'ስፖርት', 'ዓለም አቀፍ ዜና', 'ቢዝነስ', 'መዝናኛ', nan],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.category.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66404503-fada-49a6-9805-77e481ee4aed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>views</th>\n",
       "      <th>article</th>\n",
       "      <th>link</th>\n",
       "      <th>word_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44810</th>\n",
       "      <td>በመዲናዋ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና ለማስጀመር የ20 ቢሊ...</td>\n",
       "      <td>ፖለቲካ</td>\n",
       "      <td>September 14, 2017</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>በአዲስ አበባ የተጀመሩ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና አዳዲስ...</td>\n",
       "      <td>https://waltainfo.com/am/29360/</td>\n",
       "      <td>302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18329</th>\n",
       "      <td>አሸንዳ፣ ሻደይ፣ አሸንድዬ፣ ሶለል፣ ማሪያና ዓይኒ ዋሪ በዓል በዩኔስኮ አ...</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>Jan 8, 2020</td>\n",
       "      <td>387</td>\n",
       "      <td>አዲስ አበባ፣ ታህሳስ 29፣ 2012 (ኤፍ.ቢ.ሲ) የአሸንዳ፣ ሻደይ፣ አሸ...</td>\n",
       "      <td>https://www.fanabc.com/%e1%8b%a8%e1%8a%a0%e1%8...</td>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17644</th>\n",
       "      <td>ግምታዊ ዋጋቸው ከ31 ነጥብ 6 ሚሊየን ብር በላይ የሆኑ የኮንትሮባንድ ዕ...</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>Mar 4, 2020</td>\n",
       "      <td>634</td>\n",
       "      <td>አዲስ አበባ ፣ የካቲት 25 ፣ 2012 (ኤፍ ቢ ሲ) ባለፉት አምስት ቀና...</td>\n",
       "      <td>https://www.fanabc.com/%e1%8c%8d%e1%88%9d%e1%8...</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32773</th>\n",
       "      <td>​ባምላክ ተሰማ የዓለም ዋንጫ ለመምራት እጩ ከሆኑ ዳኞች ውስጥ ተካተተ</td>\n",
       "      <td>ስፖርት</td>\n",
       "      <td>November 19, 2017</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>ሩሲያ በ2018 ለምታስተናግደው የዓለም ዋንጫ ፊፋ ጨዋታዎችን ሊመሩ የሚች...</td>\n",
       "      <td>https://soccerethiopia.net/football/31318</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4139</th>\n",
       "      <td>“መንገድ ለሰው” መርሐ ግብር በየሳምንቱ እንደሚካሄድ ተገለፀ</td>\n",
       "      <td>ሀገር አቀፍ ዜና</td>\n",
       "      <td>August 7, 2019</td>\n",
       "      <td>32</td>\n",
       "      <td>አዲስ አበባ፤- “መንገድ ለሰው” በሚል በአዲስ አበባ ከተማ ሲካሄድ የነ...</td>\n",
       "      <td>https://www.press.et/Ama/?p=15606</td>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                headline    category  \\\n",
       "44810  በመዲናዋ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና ለማስጀመር የ20 ቢሊ...        ፖለቲካ   \n",
       "18329  አሸንዳ፣ ሻደይ፣ አሸንድዬ፣ ሶለል፣ ማሪያና ዓይኒ ዋሪ በዓል በዩኔስኮ አ...  ሀገር አቀፍ ዜና   \n",
       "17644  ግምታዊ ዋጋቸው ከ31 ነጥብ 6 ሚሊየን ብር በላይ የሆኑ የኮንትሮባንድ ዕ...  ሀገር አቀፍ ዜና   \n",
       "32773       ​ባምላክ ተሰማ የዓለም ዋንጫ ለመምራት እጩ ከሆኑ ዳኞች ውስጥ ተካተተ        ስፖርት   \n",
       "4139              “መንገድ ለሰው” መርሐ ግብር በየሳምንቱ እንደሚካሄድ ተገለፀ  ሀገር አቀፍ ዜና   \n",
       "\n",
       "                     date    views  \\\n",
       "44810  September 14, 2017  Unknown   \n",
       "18329         Jan 8, 2020      387   \n",
       "17644         Mar 4, 2020      634   \n",
       "32773   November 19, 2017  Unknown   \n",
       "4139       August 7, 2019       32   \n",
       "\n",
       "                                                 article  \\\n",
       "44810  በአዲስ አበባ የተጀመሩ የጋራ መኖሪያ ቤቶችን ግንባታ ለማጠናቀቅና አዳዲስ...   \n",
       "18329  አዲስ አበባ፣ ታህሳስ 29፣ 2012 (ኤፍ.ቢ.ሲ) የአሸንዳ፣ ሻደይ፣ አሸ...   \n",
       "17644  አዲስ አበባ ፣ የካቲት 25 ፣ 2012 (ኤፍ ቢ ሲ) ባለፉት አምስት ቀና...   \n",
       "32773  ሩሲያ በ2018 ለምታስተናግደው የዓለም ዋንጫ ፊፋ ጨዋታዎችን ሊመሩ የሚች...   \n",
       "4139    አዲስ አበባ፤- “መንገድ ለሰው” በሚል በአዲስ አበባ ከተማ ሲካሄድ የነ...   \n",
       "\n",
       "                                                    link  word_len  \n",
       "44810                    https://waltainfo.com/am/29360/       302  \n",
       "18329  https://www.fanabc.com/%e1%8b%a8%e1%8a%a0%e1%8...       165  \n",
       "17644  https://www.fanabc.com/%e1%8c%8d%e1%88%9d%e1%8...       102  \n",
       "32773          https://soccerethiopia.net/football/31318       143  \n",
       "4139                   https://www.press.et/Ama/?p=15606       193  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['word_len'] = data['article'].str.split().str.len()\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3b4c7880-cf15-4fb5-a670-bad2536a8ab4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 51483 entries, 44810 to 15795\n",
      "Data columns (total 7 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   headline  51470 non-null  object\n",
      " 1   category  51482 non-null  object\n",
      " 2   date      51483 non-null  object\n",
      " 3   views     51483 non-null  object\n",
      " 4   article   51483 non-null  object\n",
      " 5   link      51483 non-null  object\n",
      " 6   word_len  51483 non-null  int64 \n",
      "dtypes: int64(1), object(6)\n",
      "memory usage: 3.1+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7be6db6a-ddaf-4d2a-bd57-888ee5c47ec9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "248.9586853912942"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.word_len.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5e90f71e-2667-4782-942e-ff44d6124524",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "#method to normalize character level missmatch such as ጸሀይ and ፀሐይ\n",
    "def normalize_char_level_missmatch(input_token):\n",
    "    rep1=re.sub('[ሃኅኃሐሓኻ]','ሀ',input_token)\n",
    "    rep2=re.sub('[ሑኁዅ]','ሁ',rep1)\n",
    "    rep3=re.sub('[ኂሒኺ]','ሂ',rep2)\n",
    "    rep4=re.sub('[ኌሔዄ]','ሄ',rep3)\n",
    "    rep5=re.sub('[ሕኅ]','ህ',rep4)\n",
    "    rep6=re.sub('[ኆሖኾ]','ሆ',rep5)\n",
    "    rep7=re.sub('[ሠ]','ሰ',rep6)\n",
    "    rep8=re.sub('[ሡ]','ሱ',rep7)\n",
    "    rep9=re.sub('[ሢ]','ሲ',rep8)\n",
    "    rep10=re.sub('[ሣ]','ሳ',rep9)\n",
    "    rep11=re.sub('[ሤ]','ሴ',rep10)\n",
    "    rep12=re.sub('[ሥ]','ስ',rep11)\n",
    "    rep13=re.sub('[ሦ]','ሶ',rep12)\n",
    "    rep14=re.sub('[ዓኣዐ]','አ',rep13)\n",
    "    rep15=re.sub('[ዑ]','ኡ',rep14)\n",
    "    rep16=re.sub('[ዒ]','ኢ',rep15)\n",
    "    rep17=re.sub('[ዔ]','ኤ',rep16)\n",
    "    rep18=re.sub('[ዕ]','እ',rep17)\n",
    "    rep19=re.sub('[ዖ]','ኦ',rep18)\n",
    "    rep20=re.sub('[ጸ]','ፀ',rep19)\n",
    "    rep21=re.sub('[ጹ]','ፁ',rep20)\n",
    "    rep22=re.sub('[ጺ]','ፂ',rep21)\n",
    "    rep23=re.sub('[ጻ]','ፃ',rep22)\n",
    "    rep24=re.sub('[ጼ]','ፄ',rep23)\n",
    "    rep25=re.sub('[ጽ]','ፅ',rep24)\n",
    "    rep26=re.sub('[ጾ]','ፆ',rep25)\n",
    "    #Normalizing words with Labialized Amharic characters such as በልቱዋል or  በልቱአል to  በልቷል  \n",
    "    rep27=re.sub('(ሉ[ዋአ])','ሏ',rep26)\n",
    "    rep28=re.sub('(ሙ[ዋአ])','ሟ',rep27)\n",
    "    rep29=re.sub('(ቱ[ዋአ])','ቷ',rep28)\n",
    "    rep30=re.sub('(ሩ[ዋአ])','ሯ',rep29)\n",
    "    rep31=re.sub('(ሱ[ዋአ])','ሷ',rep30)\n",
    "    rep32=re.sub('(ሹ[ዋአ])','ሿ',rep31)\n",
    "    rep33=re.sub('(ቁ[ዋአ])','ቋ',rep32)\n",
    "    rep34=re.sub('(ቡ[ዋአ])','ቧ',rep33)\n",
    "    rep35=re.sub('(ቹ[ዋአ])','ቿ',rep34)\n",
    "    rep36=re.sub('(ሁ[ዋአ])','ኋ',rep35)\n",
    "    rep37=re.sub('(ኑ[ዋአ])','ኗ',rep36)\n",
    "    rep38=re.sub('(ኙ[ዋአ])','ኟ',rep37)\n",
    "    rep39=re.sub('(ኩ[ዋአ])','ኳ',rep38)\n",
    "    rep40=re.sub('(ዙ[ዋአ])','ዟ',rep39)\n",
    "    rep41=re.sub('(ጉ[ዋአ])','ጓ',rep40)\n",
    "    rep42=re.sub('(ደ[ዋአ])','ዷ',rep41)\n",
    "    rep43=re.sub('(ጡ[ዋአ])','ጧ',rep42)\n",
    "    rep44=re.sub('(ጩ[ዋአ])','ጯ',rep43)\n",
    "    rep45=re.sub('(ጹ[ዋአ])','ጿ',rep44)\n",
    "    rep46=re.sub('(ፉ[ዋአ])','ፏ',rep45)\n",
    "    rep47=re.sub('[ቊ]','ቁ',rep46) #ቁ can be written as ቊ\n",
    "    rep48=re.sub('[ኵ]','ኩ',rep47) #ኩ can be also written as ኵ  \n",
    "    return rep48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6c61f66e-6c31-433c-b457-fd6ff216ef95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\devhr\\AppData\\Local\\Temp\\ipykernel_26332\\3727900363.py:1: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  data['article'] = data['article'].str.replace('[^\\w\\s]','')\n"
     ]
    }
   ],
   "source": [
    "data['article'] = data['article'].str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "64b4888a-9bd2-49ad-81ce-2db16e9100d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['article'] = data['article'].apply(lambda x: normalize_char_level_missmatch(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f88e3980-0a02-47d9-98ac-03b881b8e8ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_data = data[['article','category']]\n",
    "n_data.head()\n",
    "\n",
    "text,label = data['article'].values,data['category'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c655580e-3ed0-4228-a026-d33721ce3a2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "matrix = CountVectorizer(analyzer='word',max_features=1000,ngram_range=(1, 3))\n",
    "X = matrix.fit_transform(text).toarray()\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f93caf33-1415-47e2-8657-0f9b714d9a68",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_label = list(set(label))\n",
    "Y= []\n",
    "for i in label:\n",
    "    Y.append(unique_label.index(i))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "266d101a-b11f-4bb7-8f49-2452131130a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "feea605f-8a57-4081-ad66-c167c7ecc7e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6179469748470429"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SVM MODEL\n",
    "# Naive Bayes \n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "classifier = GaussianNB()\n",
    "classifier.fit(X_train, y_train)\n",
    "\n",
    "# Predict Class\n",
    "y_pred = classifier.predict(X_test)\n",
    "\n",
    "# Accuracy \n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "\n",
    "accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "623ed620-27ac-4e7b-80f5-34784a3240da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        ስፖርት       0.00      0.00      0.00         0\n",
      "        መዝናኛ       0.37      0.78      0.50       789\n",
      "  ሀገር አቀፍ ዜና       0.62      0.53      0.57      1927\n",
      "        ቢዝነስ       0.96      0.93      0.95      2023\n",
      "  ዓለም አቀፍ ዜና       0.82      0.38      0.52      4172\n",
      "        ፖለቲካ       0.44      0.91      0.59      1285\n",
      "                   0.26      0.77      0.39       101\n",
      "\n",
      "    accuracy                           0.62     10297\n",
      "   macro avg       0.50      0.61      0.50     10297\n",
      "weighted avg       0.72      0.62      0.62     10297\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\devhr\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\devhr\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\devhr\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred, target_names=['ስፖርት', 'መዝናኛ', 'ሀገር አቀፍ ዜና', 'ቢዝነስ', 'ዓለም አቀፍ ዜና', 'ፖለቲካ','']))#, 'nan'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}