3_TF-IDF(Term Frequency-Inverse Document Frequency).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "427c1b20",
   "metadata": {},
   "source": [
    "# 英文部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "531602e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 匯入套件\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f40a432",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['and' 'document' 'first' 'is' 'one' 'second' 'the' 'third' 'this']\n",
      "[[0.         0.46979139 0.58028582 0.38408524 0.         0.\n",
      "  0.38408524 0.         0.38408524]\n",
      " [0.         0.6876236  0.         0.28108867 0.         0.53864762\n",
      "  0.28108867 0.         0.28108867]\n",
      " [0.51184851 0.         0.         0.26710379 0.51184851 0.\n",
      "  0.26710379 0.51184851 0.26710379]\n",
      " [0.         0.46979139 0.58028582 0.38408524 0.         0.\n",
      "  0.38408524 0.         0.38408524]]\n"
     ]
    }
   ],
   "source": [
    "# 假設我們有這些英文文本\n",
    "docs = [\n",
    "    'This is the first document.',\n",
    "    'This document is the second document.',\n",
    "    'And this is the third one.',\n",
    "    'Is this the first document?',\n",
    "]\n",
    "\n",
    "# 創建 TF-IDF 範例\n",
    "vectorizer = TfidfVectorizer()\n",
    "\n",
    "# 進行 TF-IDF 轉換\n",
    "X = vectorizer.fit_transform(docs)\n",
    "\n",
    "# 顯示結果\n",
    "print(vectorizer.get_feature_names_out())\n",
    "print(X.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4fd0cabf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>and</th>\n",
       "      <th>document</th>\n",
       "      <th>first</th>\n",
       "      <th>is</th>\n",
       "      <th>one</th>\n",
       "      <th>second</th>\n",
       "      <th>the</th>\n",
       "      <th>third</th>\n",
       "      <th>this</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.469791</td>\n",
       "      <td>0.580286</td>\n",
       "      <td>0.384085</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384085</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.687624</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.281089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.538648</td>\n",
       "      <td>0.281089</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.281089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.511849</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.267104</td>\n",
       "      <td>0.511849</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.267104</td>\n",
       "      <td>0.511849</td>\n",
       "      <td>0.267104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.469791</td>\n",
       "      <td>0.580286</td>\n",
       "      <td>0.384085</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384085</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        and  document     first        is       one    second       the  \\\n",
       "0  0.000000  0.469791  0.580286  0.384085  0.000000  0.000000  0.384085   \n",
       "1  0.000000  0.687624  0.000000  0.281089  0.000000  0.538648  0.281089   \n",
       "2  0.511849  0.000000  0.000000  0.267104  0.511849  0.000000  0.267104   \n",
       "3  0.000000  0.469791  0.580286  0.384085  0.000000  0.000000  0.384085   \n",
       "\n",
       "      third      this  \n",
       "0  0.000000  0.384085  \n",
       "1  0.000000  0.281089  \n",
       "2  0.511849  0.267104  \n",
       "3  0.000000  0.384085  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 透過 pandas 來預覽結果\n",
    "df = pd.DataFrame(X.toarray(), columns=vectorizer.get_feature_names_out()); df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12d70c7e",
   "metadata": {},
   "source": [
    "# 中文部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "80db628d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\darren\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.630 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['份文件' '文件' '第一份' '第三' '第二份' '這是']\n",
      "[[0.         0.53256952 0.65782931 0.         0.         0.53256952]\n",
      " [0.64450299 0.41137791 0.         0.         0.64450299 0.        ]\n",
      " [0.         0.         0.         0.84292635 0.         0.53802897]\n",
      " [0.         0.53256952 0.65782931 0.         0.         0.53256952]]\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# 假設我們有這些中文文本\n",
    "documents = [\n",
    "    '這是第一份文件',\n",
    "    '這份文件是第二份文件',\n",
    "    '而這是第三個',\n",
    "    '這是第一份文件嗎'\n",
    "]\n",
    "\n",
    "# 先進行分詞\n",
    "documents = [' '.join(jieba.cut(doc)) for doc in documents]\n",
    "\n",
    "# 創建 TF-IDF 範例\n",
    "vectorizer = TfidfVectorizer()\n",
    "\n",
    "# 進行 TF-IDF 轉換\n",
    "X = vectorizer.fit_transform(documents)\n",
    "\n",
    "# 顯示結果\n",
    "print(vectorizer.get_feature_names_out())\n",
    "print(X.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bd40d181",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>份文件</th>\n",
       "      <th>文件</th>\n",
       "      <th>第一份</th>\n",
       "      <th>第三</th>\n",
       "      <th>第二份</th>\n",
       "      <th>這是</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.532570</td>\n",
       "      <td>0.657829</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.532570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.644503</td>\n",
       "      <td>0.411378</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.644503</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.842926</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.538029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.532570</td>\n",
       "      <td>0.657829</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.532570</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        份文件        文件       第一份        第三       第二份        這是\n",
       "0  0.000000  0.532570  0.657829  0.000000  0.000000  0.532570\n",
       "1  0.644503  0.411378  0.000000  0.000000  0.644503  0.000000\n",
       "2  0.000000  0.000000  0.000000  0.842926  0.000000  0.538029\n",
       "3  0.000000  0.532570  0.657829  0.000000  0.000000  0.532570"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 透過 pandas 來預覽結果\n",
    "df = pd.DataFrame(X.toarray(), columns=vectorizer.get_feature_names_out()); df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3@nlp",
   "language": "python",
   "name": "k_nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}