bert-base-turkish-bullying-best-parameter.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport torch\nfrom sklearn.preprocessing import LabelEncoder\nfrom transformers import BertTokenizer\nfrom torch.utils.data import TensorDataset, random_split\nfrom torch.utils.data import DataLoader, RandomSampler, SequentialSampler\nfrom transformers import BertForSequenceClassification, AdamW, BertConfig, Adafactor\nfrom transformers import get_linear_schedule_with_warmup\nimport datetime\nimport random\nimport seaborn as sns\nimport numpy as np\nimport time\nimport matplotlib.pyplot as plt\n\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-16T20:37:10.256469Z","iopub.execute_input":"2022-08-16T20:37:10.257075Z","iopub.status.idle":"2022-08-16T20:37:10.273795Z","shell.execute_reply.started":"2022-08-16T20:37:10.257036Z","shell.execute_reply":"2022-08-16T20:37:10.272887Z"},"trusted":true},"execution_count":34,"outputs":[]},{"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained('dbmdz/bert-base-turkish-128k-uncased', do_lower_case=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:10.276606Z","iopub.execute_input":"2022-08-16T20:37:10.277591Z","iopub.status.idle":"2022-08-16T20:37:12.117703Z","shell.execute_reply.started":"2022-08-16T20:37:10.277557Z","shell.execute_reply":"2022-08-16T20:37:12.116662Z"},"trusted":true},"execution_count":35,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('datasets/clean_data.csv')\n\ndf.head(10)\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.119775Z","iopub.execute_input":"2022-08-16T20:37:12.120177Z","iopub.status.idle":"2022-08-16T20:37:12.153165Z","shell.execute_reply.started":"2022-08-16T20:37:12.120140Z","shell.execute_reply":"2022-08-16T20:37:12.152068Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"markdown","source":"```\n<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 3388 entries, 0 to 3387\nData columns (total 6 columns):\n #   Column       Non-Null Count  Dtype \n---  ------       --------------  ----- \n 0   scraped_id   3388 non-null   int64 \n 1   text         3388 non-null   object\n 2   tagger       3375 non-null   object\n 3   tagged_date  3388 non-null   object\n 4   label        3388 non-null   object\n 5   clean_data   3388 non-null   object\ndtypes: int64(1), object(5)\nmemory usage: 158.9+ KB\n```","metadata":{}},{"cell_type":"code","source":"# check GPU\ndevice_name = tf.test.gpu_device_name()\nif device_name == '/device:GPU:0':\n    device = torch.device(\"cuda\")\n    print('GPU:', torch.cuda.get_device_name(0))\nelse:\n    raise SystemError('GPU device not found')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.154488Z","iopub.execute_input":"2022-08-16T20:37:12.155392Z","iopub.status.idle":"2022-08-16T20:37:12.168453Z","shell.execute_reply.started":"2022-08-16T20:37:12.155345Z","shell.execute_reply":"2022-08-16T20:37:12.167484Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"markdown","source":"```\nGPU: Tesla P100-PCIE-16GB\n2022-08-16 20:37:12.159662: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:37:12.160856: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:37:12.161558: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:37:12.162267: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:37:12.162857: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:37:12.163383: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /device:GPU:0 with 15047 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n```\n","metadata":{}},{"cell_type":"code","source":"df.groupby('label').size()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.171179Z","iopub.execute_input":"2022-08-16T20:37:12.172387Z","iopub.status.idle":"2022-08-16T20:37:12.181540Z","shell.execute_reply.started":"2022-08-16T20:37:12.172348Z","shell.execute_reply":"2022-08-16T20:37:12.180263Z"},"trusted":true},"execution_count":38,"outputs":[]},{"cell_type":"markdown","source":"```\nlabel\nCinsiyetçilik     601\nIrkçılık          490\nKızdırma          910\nNötr             1387\ndtype: int64\n```","metadata":{}},{"cell_type":"code","source":"df['scraped_id'].size","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.182992Z","iopub.execute_input":"2022-08-16T20:37:12.183565Z","iopub.status.idle":"2022-08-16T20:37:12.192832Z","shell.execute_reply.started":"2022-08-16T20:37:12.183527Z","shell.execute_reply":"2022-08-16T20:37:12.191665Z"},"trusted":true},"execution_count":39,"outputs":[]},{"cell_type":"markdown","source":"```\n3388\n```","metadata":{}},{"cell_type":"code","source":"df['label'] = LabelEncoder().fit_transform(df['label'])","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.195553Z","iopub.execute_input":"2022-08-16T20:37:12.195876Z","iopub.status.idle":"2022-08-16T20:37:12.203593Z","shell.execute_reply.started":"2022-08-16T20:37:12.195846Z","shell.execute_reply":"2022-08-16T20:37:12.202673Z"},"trusted":true},"execution_count":40,"outputs":[]},{"cell_type":"code","source":"training = df.groupby('label').apply(lambda x : x.sample(frac = 0.8))\ntest = pd.concat([df,training]).drop_duplicates(keep=False)\nresult = test.groupby('text')['label'].count().sum()\nprint(df['scraped_id'].size - result)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.238236Z","iopub.execute_input":"2022-08-16T20:37:12.238815Z","iopub.status.idle":"2022-08-16T20:37:12.257095Z","shell.execute_reply.started":"2022-08-16T20:37:12.238779Z","shell.execute_reply":"2022-08-16T20:37:12.256059Z"},"trusted":true},"execution_count":42,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.259092Z","iopub.execute_input":"2022-08-16T20:37:12.259683Z","iopub.status.idle":"2022-08-16T20:37:12.277282Z","shell.execute_reply.started":"2022-08-16T20:37:12.259648Z","shell.execute_reply":"2022-08-16T20:37:12.276371Z"},"trusted":true},"execution_count":43,"outputs":[]},{"cell_type":"code","source":"print(\"Training: \", len(training))\nprint(\"Test: \", len(test))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.282782Z","iopub.execute_input":"2022-08-16T20:37:12.283447Z","iopub.status.idle":"2022-08-16T20:37:12.289498Z","shell.execute_reply.started":"2022-08-16T20:37:12.283413Z","shell.execute_reply":"2022-08-16T20:37:12.288466Z"},"trusted":true},"execution_count":44,"outputs":[]},{"cell_type":"markdown","source":"```\nTraining:  2372\nTest:  1016\n```","metadata":{}},{"cell_type":"code","source":"training_texts = training.clean_data.values\ntraining_labels = training.label.values","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.290886Z","iopub.execute_input":"2022-08-16T20:37:12.291406Z","iopub.status.idle":"2022-08-16T20:37:12.299604Z","shell.execute_reply.started":"2022-08-16T20:37:12.291371Z","shell.execute_reply":"2022-08-16T20:37:12.298516Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"training_labels","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.301166Z","iopub.execute_input":"2022-08-16T20:37:12.301805Z","iopub.status.idle":"2022-08-16T20:37:12.311014Z","shell.execute_reply.started":"2022-08-16T20:37:12.301770Z","shell.execute_reply":"2022-08-16T20:37:12.309980Z"},"trusted":true},"execution_count":46,"outputs":[]},{"cell_type":"markdown","source":"```\narray([0, 0, 0, ..., 3, 3, 3])\n```","metadata":{}},{"cell_type":"code","source":"input_ids = []\nattention_masks = []\nmax_len = 100\n\n\nfor text in training_texts:\n    encoded_dict = tokenizer.encode_plus(\n                        str(text),                     \n                        add_special_tokens = True,\n                        max_length = max_len,      \n                        pad_to_max_length = True,\n                        return_attention_mask = True, \n                        return_tensors = 'pt',\n                   )\n    \n    input_ids.append(encoded_dict['input_ids'])\n    attention_masks.append(encoded_dict['attention_mask'])\n\ninput_ids = torch.cat(input_ids, dim=0)\nattention_masks = torch.cat(attention_masks, dim=0)\nlabels = torch.tensor(training_labels)\n\nprint('Original: ', training_texts[0])\nprint('Token IDs:', input_ids[0])","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:12.312683Z","iopub.execute_input":"2022-08-16T20:37:12.313273Z","iopub.status.idle":"2022-08-16T20:37:14.232185Z","shell.execute_reply.started":"2022-08-16T20:37:12.313238Z","shell.execute_reply":"2022-08-16T20:37:14.231242Z"},"trusted":true},"execution_count":47,"outputs":[]},{"cell_type":"markdown","source":"```\nOriginal:  boyle pasta yapamayan da kendine kizim demesin\nToken IDs: tensor([    2, 21181,  9692, 43783,  1972,  4852, 68907, 86075,     3,     0,\n            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n            0,     0,     0,     0,     0])\n```","metadata":{}},{"cell_type":"code","source":"train_dataset = TensorDataset(input_ids, attention_masks, labels)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:14.233600Z","iopub.execute_input":"2022-08-16T20:37:14.233940Z","iopub.status.idle":"2022-08-16T20:37:14.239142Z","shell.execute_reply.started":"2022-08-16T20:37:14.233912Z","shell.execute_reply":"2022-08-16T20:37:14.238042Z"},"trusted":true},"execution_count":48,"outputs":[]},{"cell_type":"code","source":"batch_size = 16\n\ntrain_dataloader = DataLoader(\n            train_dataset,  \n            sampler = RandomSampler(train_dataset), \n            batch_size = batch_size \n        )","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:14.240745Z","iopub.execute_input":"2022-08-16T20:37:14.241449Z","iopub.status.idle":"2022-08-16T20:37:14.249538Z","shell.execute_reply.started":"2022-08-16T20:37:14.241414Z","shell.execute_reply":"2022-08-16T20:37:14.248480Z"},"trusted":true},"execution_count":49,"outputs":[]},{"cell_type":"code","source":"number_of_categories = len(df['label'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:14.251532Z","iopub.execute_input":"2022-08-16T20:37:14.253238Z","iopub.status.idle":"2022-08-16T20:37:14.260618Z","shell.execute_reply.started":"2022-08-16T20:37:14.253204Z","shell.execute_reply":"2022-08-16T20:37:14.259646Z"},"trusted":true},"execution_count":50,"outputs":[]},{"cell_type":"code","source":"model = BertForSequenceClassification.from_pretrained(\n    \"dbmdz/bert-base-turkish-128k-uncased\",\n    num_labels = number_of_categories, \n    output_attentions = False,\n    output_hidden_states = False,\n)\n\nmodel.cuda()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:14.262079Z","iopub.execute_input":"2022-08-16T20:37:14.262794Z","iopub.status.idle":"2022-08-16T20:37:18.675707Z","shell.execute_reply.started":"2022-08-16T20:37:14.262653Z","shell.execute_reply":"2022-08-16T20:37:18.674627Z"},"trusted":true},"execution_count":51,"outputs":[]},{"cell_type":"markdown","source":"```\nBertForSequenceClassification(\n  (bert): BertModel(\n    (embeddings): BertEmbeddings(\n      (word_embeddings): Embedding(128000, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (token_type_embeddings): Embedding(2, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (encoder): BertEncoder(\n      (layer): ModuleList(\n        (0): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (1): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (2): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (3): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (4): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (5): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (6): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (7): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (8): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (9): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (10): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n        (11): BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (pooler): BertPooler(\n      (dense): Linear(in_features=768, out_features=768, bias=True)\n      (activation): Tanh()\n    )\n  )\n  (dropout): Dropout(p=0.1, inplace=False)\n  (classifier): Linear(in_features=768, out_features=4, bias=True)\n)\n```","metadata":{}},{"cell_type":"code","source":"    epochs = 8 #denemelerim sonucu kayıp 0 a 8. epochta yaklaşıyor\n\n    optimizer = AdamW(model.parameters(),\n                      lr = 5e-5,\n                      eps = 1e-8 \n                    )\n    #bu optimazer kaldırılacakmış yakında yeni versiyona uygun torch.optim.AdamW kullanalım.\n\n    total_steps = len(train_dataloader) * epochs\n    scheduler = get_linear_schedule_with_warmup(optimizer, \n                                                num_warmup_steps = 0,\n                                                num_training_steps = total_steps)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:18.677361Z","iopub.execute_input":"2022-08-16T20:37:18.678037Z","iopub.status.idle":"2022-08-16T20:37:18.691231Z","shell.execute_reply.started":"2022-08-16T20:37:18.677968Z","shell.execute_reply":"2022-08-16T20:37:18.690233Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"code","source":"def format_time(elapsed):\n    elapsed_rounded = int(round((elapsed)))\n    return str(datetime.timedelta(seconds=elapsed_rounded))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:18.692754Z","iopub.execute_input":"2022-08-16T20:37:18.693364Z","iopub.status.idle":"2022-08-16T20:37:18.699296Z","shell.execute_reply.started":"2022-08-16T20:37:18.693320Z","shell.execute_reply":"2022-08-16T20:37:18.698182Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"seed_val = 3000\n\nrandom.seed(seed_val)\nnp.random.seed(seed_val)\ntorch.manual_seed(seed_val)\ntorch.cuda.manual_seed_all(seed_val)\n\ntraining_stats = []\ntotal_t0 = time.time()\n\nfor epoch_i in range(0, epochs):\n    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n    t0 = time.time()\n    total_train_loss = 0\n    model.train()\n    \n    for step, batch in enumerate(train_dataloader):\n        if step % 10 == 0 and not step == 0:\n            elapsed = format_time(time.time() - t0)\n            print('Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n\n        b_input_ids = batch[0].to(device)\n        b_input_mask = batch[1].to(device)\n        b_labels = batch[2].to(device)\n\n        model.zero_grad()        \n        output = model(b_input_ids, \n                             token_type_ids=None, \n                             attention_mask=b_input_mask, \n                             labels=b_labels)\n        loss = output['loss']\n        logits = output['logits']\n        total_train_loss += loss.item()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        optimizer.step()\n        scheduler.step()\n\n    avg_train_loss = total_train_loss / len(train_dataloader)            \n    training_time = format_time(time.time() - t0)\n\n    print(\"Average training loss: {0:.2f}\".format(avg_train_loss))\n    print(\"Training epoch took: {:}\".format(training_time))\n\n    training_stats.append(\n        {\n            'epoch': epoch_i + 1,\n            'Training Loss': avg_train_loss,\n            'Training Time': training_time,\n        }\n    )\n\nprint(\"Training completed in {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:37:18.700676Z","iopub.execute_input":"2022-08-16T20:37:18.701262Z","iopub.status.idle":"2022-08-16T20:39:03.357651Z","shell.execute_reply.started":"2022-08-16T20:37:18.701228Z","shell.execute_reply":"2022-08-16T20:39:03.356531Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"markdown","source":"```\n======== Epoch 1 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:04.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:11.\nBatch   130  of    149.    Elapsed: 0:00:11.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.67\nTraining epoch took: 0:00:13\n======== Epoch 2 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:04.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:10.\nBatch   130  of    149.    Elapsed: 0:00:11.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.32\nTraining epoch took: 0:00:13\n======== Epoch 3 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:03.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:09.\nBatch   120  of    149.    Elapsed: 0:00:10.\nBatch   130  of    149.    Elapsed: 0:00:11.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.18\nTraining epoch took: 0:00:13\n======== Epoch 4 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:04.\nBatch    50  of    149.    Elapsed: 0:00:05.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:11.\nBatch   130  of    149.    Elapsed: 0:00:12.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.09\nTraining epoch took: 0:00:13\n======== Epoch 5 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:03.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:11.\nBatch   130  of    149.    Elapsed: 0:00:12.\nBatch   140  of    149.    Elapsed: 0:00:13.\nAverage training loss: 0.05\nTraining epoch took: 0:00:13\n======== Epoch 6 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:03.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:11.\nBatch   130  of    149.    Elapsed: 0:00:11.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.01\nTraining epoch took: 0:00:13\n======== Epoch 7 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:04.\nBatch    50  of    149.    Elapsed: 0:00:05.\nBatch    60  of    149.    Elapsed: 0:00:06.\nBatch    70  of    149.    Elapsed: 0:00:07.\nBatch    80  of    149.    Elapsed: 0:00:08.\nBatch    90  of    149.    Elapsed: 0:00:09.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:10.\nBatch   120  of    149.    Elapsed: 0:00:11.\nBatch   130  of    149.    Elapsed: 0:00:12.\nBatch   140  of    149.    Elapsed: 0:00:13.\nAverage training loss: 0.01\nTraining epoch took: 0:00:14\n======== Epoch 8 / 8 ========\nBatch    10  of    149.    Elapsed: 0:00:01.\nBatch    20  of    149.    Elapsed: 0:00:02.\nBatch    30  of    149.    Elapsed: 0:00:03.\nBatch    40  of    149.    Elapsed: 0:00:03.\nBatch    50  of    149.    Elapsed: 0:00:04.\nBatch    60  of    149.    Elapsed: 0:00:05.\nBatch    70  of    149.    Elapsed: 0:00:06.\nBatch    80  of    149.    Elapsed: 0:00:07.\nBatch    90  of    149.    Elapsed: 0:00:08.\nBatch   100  of    149.    Elapsed: 0:00:09.\nBatch   110  of    149.    Elapsed: 0:00:09.\nBatch   120  of    149.    Elapsed: 0:00:10.\nBatch   130  of    149.    Elapsed: 0:00:11.\nBatch   140  of    149.    Elapsed: 0:00:12.\nAverage training loss: 0.00\nTraining epoch took: 0:00:13\nTraining completed in 0:01:45 (h:mm:ss)\n```\n","metadata":{}},{"cell_type":"markdown","source":"* ## Burada model oluşmuş oluyor. model save işlemi yapmamız ve sonra bir text gönderip ne etiket bastıgına bakmamız gerekiyor.\n","metadata":{}},{"cell_type":"code","source":"tokenizer.save_pretrained(\"./bigscience_t0_tokenizer\")\nmodel.save_pretrained(\"./bigscience_t0_model\")","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:03.358930Z","iopub.execute_input":"2022-08-16T20:39:03.359624Z","iopub.status.idle":"2022-08-16T20:39:06.063570Z","shell.execute_reply.started":"2022-08-16T20:39:03.359586Z","shell.execute_reply":"2022-08-16T20:39:06.062352Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"df_stats = pd.DataFrame(data=training_stats)\nplt.plot(df_stats['Training Loss'], label=\"Training\")\nplt.title(\"Training Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.xticks([1, 2, 3, 4])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:06.065154Z","iopub.execute_input":"2022-08-16T20:39:06.065886Z","iopub.status.idle":"2022-08-16T20:39:06.243830Z","shell.execute_reply.started":"2022-08-16T20:39:06.065840Z","shell.execute_reply":"2022-08-16T20:39:06.242897Z"},"trusted":true},"execution_count":56,"outputs":[]},{"cell_type":"code","source":"test_texts = test.text.values\ntest_labels = test.label.values\n\ninput_ids = []\nattention_masks = []\n\nfor text in test_texts:\n    encoded_dict = tokenizer.encode_plus(\n                        text,                     \n                        add_special_tokens = True, \n                        max_length = max_len,          \n                        pad_to_max_length = True,\n                        return_attention_mask = True,  \n                        return_tensors = 'pt',   \n                   )\n    \n    input_ids.append(encoded_dict['input_ids'])\n    attention_masks.append(encoded_dict['attention_mask'])\n\ninput_ids = torch.cat(input_ids, dim=0)\nattention_masks = torch.cat(attention_masks, dim=0)\nlabels = torch.tensor(test_labels)\n\nbatch_size = 32  \n\nprediction_data = TensorDataset(input_ids, attention_masks, labels)\nprediction_sampler = SequentialSampler(prediction_data)\nprediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:06.246677Z","iopub.execute_input":"2022-08-16T20:39:06.246951Z","iopub.status.idle":"2022-08-16T20:39:07.036076Z","shell.execute_reply.started":"2022-08-16T20:39:06.246926Z","shell.execute_reply":"2022-08-16T20:39:07.035115Z"},"trusted":true},"execution_count":57,"outputs":[]},{"cell_type":"code","source":"print('Prediction started on test data')\nmodel.eval()\npredictions , true_labels = [], []\n\nfor batch in prediction_dataloader:\n  batch = tuple(t.to(device) for t in batch)\n  b_input_ids, b_input_mask, b_labels = batch\n\n  with torch.no_grad():\n      outputs = model(b_input_ids, token_type_ids=None, \n                      attention_mask=b_input_mask)\n\n  logits = outputs[0]\n  logits = logits.detach().cpu().numpy()\n  label_ids = b_labels.to('cpu').numpy()\n  \n  predictions.append(logits)\n  true_labels.append(label_ids)\n\nprint('Prediction completed')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:07.037701Z","iopub.execute_input":"2022-08-16T20:39:07.038080Z","iopub.status.idle":"2022-08-16T20:39:07.958450Z","shell.execute_reply.started":"2022-08-16T20:39:07.038041Z","shell.execute_reply":"2022-08-16T20:39:07.957348Z"},"trusted":true},"execution_count":58,"outputs":[]},{"cell_type":"markdown","source":"```\nPrediction started on test data\nPrediction completed\n```","metadata":{}},{"cell_type":"code","source":"prediction_set = []\n\nfor i in range(len(true_labels)):\n  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n  prediction_set.append(pred_labels_i)\n\nprediction_scores = [item for sublist in prediction_set for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:07.962988Z","iopub.execute_input":"2022-08-16T20:39:07.965330Z","iopub.status.idle":"2022-08-16T20:39:07.974184Z","shell.execute_reply.started":"2022-08-16T20:39:07.965290Z","shell.execute_reply":"2022-08-16T20:39:07.973053Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"f_score = f1_score(test_labels, prediction_scores, average='macro')\nprecision = precision_score(test_labels, prediction_scores, average='macro')\nrecall = recall_score(test_labels, prediction_scores, average='macro')\naccr = accuracy_score(test_labels, prediction_scores)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:07.979506Z","iopub.execute_input":"2022-08-16T20:39:07.981879Z","iopub.status.idle":"2022-08-16T20:39:07.998630Z","shell.execute_reply.started":"2022-08-16T20:39:07.981831Z","shell.execute_reply":"2022-08-16T20:39:07.997629Z"},"trusted":true},"execution_count":60,"outputs":[]},{"cell_type":"code","source":"print(\"F-Score: \", f_score)\nprint(\"Recall: \", recall)\nprint(\"Precision: \", precision)\nprint(\"Accuracy: \", accr)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.002807Z","iopub.execute_input":"2022-08-16T20:39:08.005766Z","iopub.status.idle":"2022-08-16T20:39:08.015789Z","shell.execute_reply.started":"2022-08-16T20:39:08.005729Z","shell.execute_reply":"2022-08-16T20:39:08.014588Z"},"trusted":true},"execution_count":61,"outputs":[]},{"cell_type":"markdown","source":"```\nF-Score:  0.8684970081779455\nRecall:  0.866681111547183\nPrecision:  0.8704137206600524\nAccuracy:  0.8710629921259843\n```","metadata":{}},{"cell_type":"code","source":"report = pd.DataFrame(classification_report(test_labels, prediction_scores, output_dict=True))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.026215Z","iopub.execute_input":"2022-08-16T20:39:08.028837Z","iopub.status.idle":"2022-08-16T20:39:08.044857Z","shell.execute_reply.started":"2022-08-16T20:39:08.028802Z","shell.execute_reply":"2022-08-16T20:39:08.043902Z"},"trusted":true},"execution_count":62,"outputs":[]},{"cell_type":"code","source":"report = report.rename(columns={\n                                '0':'Cinsiyetçilik',\n                                '1':'Irkçılık',\n                                '2':'Kızdırma',\n                                '3':'Nötr'})\nreport","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.048863Z","iopub.execute_input":"2022-08-16T20:39:08.051587Z","iopub.status.idle":"2022-08-16T20:39:08.074974Z","shell.execute_reply.started":"2022-08-16T20:39:08.051547Z","shell.execute_reply":"2022-08-16T20:39:08.074068Z"},"trusted":true},"execution_count":63,"outputs":[]},{"cell_type":"markdown","source":"## Bundan alt kısım veri çoğaltma çabalarım... ","metadata":{}},{"cell_type":"code","source":"\"\"\"df = pd.read_csv('../input/nane-limon-son/clean_data (2) - clean_data (2).csv')\ndf = df.sample(10)\ndf.head(10)\ndf.info()\n\n!pip install nlpaug\nimport nlpaug.augmenter.word as naw\n\n\ndef augmentation(text):\n    aug = naw.ContextualWordEmbsAug(\n        model_path='dbmdz/bert-base-turkish-128k-uncased', action=\"insert\", top_k=0)\n    augmented_text = aug.augment(text)\n    return augmented_text\n\n\ndf['augmentation'] = df.clean_data.apply(augmentation)\ndf.to_csv('augmentation.csv')\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.078791Z","iopub.execute_input":"2022-08-16T20:39:08.081513Z","iopub.status.idle":"2022-08-16T20:39:08.091915Z","shell.execute_reply.started":"2022-08-16T20:39:08.081478Z","shell.execute_reply":"2022-08-16T20:39:08.090064Z"},"trusted":true},"execution_count":64,"outputs":[]},{"cell_type":"code","source":"print(df.head(10))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.094514Z","iopub.execute_input":"2022-08-16T20:39:08.095126Z","iopub.status.idle":"2022-08-16T20:39:08.111833Z","shell.execute_reply.started":"2022-08-16T20:39:08.095091Z","shell.execute_reply":"2022-08-16T20:39:08.110805Z"},"trusted":true},"execution_count":65,"outputs":[]},{"cell_type":"markdown","source":"```\n\n   scraped_id                                               text  \\\n0           1       -185 altı kendine erkeğim demesin diyen kasa   \n1           2  \"feminen bir erkek neden olamıyorsun, neden be...   \n2           3  \"Kızlar böyle tırnağı olmayan da kendine kızım...   \n3           4  \"Şık olmalı kadın dediğin.Gelisi, gülüşü, bakı...   \n4           5  “ Çokta …de “ demek için erkek olmak isterdim....   \n5           6  “ Şöyle olmayan kendine kadınım demesin böyle ...   \n6           7  “Ben öldüğüm zaman, beni gece gömün ki ! Namah...   \n7           8  *Sakallı lavuk trans kız olduğunu iddia eden t...   \n8           9  *Twittera girerim*\\nEğer benim gibi değilseniz...   \n9          10  #kücükköy Kadınlar  gibi  takdir  değil, her  ...   \n\n                         tagger                 tagged_date  label  \\\n0  model_tarafından_etiketlendi  2022-07-31 22:11:42.000000      0   \n1  model_tarafından_etiketlendi  2022-07-31 22:11:47.000000      0   \n2  model_tarafından_etiketlendi  2022-07-31 22:11:49.000000      0   \n3  model_tarafından_etiketlendi  2022-07-31 22:11:51.000000      0   \n4  model_tarafından_etiketlendi  2022-07-31 22:11:53.000000      0   \n5  model_tarafından_etiketlendi  2022-07-31 22:11:55.000000      0   \n6  model_tarafından_etiketlendi  2022-07-31 22:11:57.000000      0   \n7  model_tarafından_etiketlendi  2022-07-31 22:11:59.000000      0   \n8  model_tarafından_etiketlendi  2022-07-31 22:12:01.000000      0   \n9  model_tarafından_etiketlendi  2022-07-31 22:12:03.000000      0   \n\n                                          clean_data  \n0            alti kendine erkegim demesin diyen kasa  \n1  feminen bir erkek neden olamiyorsun neden ben ...  \n2  kizlar boyle tirnagi olmayan da kendine kizim ...  \n3  sik olmali kadin dedigingelisi gulusu bakisi d...  \n4   cokta de demek icin erkek olmak isterdim cok ...  \n5   soyle olmayan kendine kadinim demesin boyle o...  \n6  ben oldugum zaman beni gece gomun ki namahrem ...  \n7  sakalli lavuk trans kiz oldugunu iddia eden tw...  \n8  twittera girerim eger benim gibi degilseniz ba...  \n9      kadinlar gibi takdir degil her deprenis anlar  \n```","metadata":{}},{"cell_type":"code","source":"\nfrom transformers import BertTokenizer,BertTokenizerFast, TFBertForSequenceClassification, BertConfig, TFBertModel\nconfig = BertConfig.from_json_file(\"./bigscience_t0_model/config.json\")\nmodel_path = \"./bigscience_t0_model\"\ntokenizer_path = \"./bigscience_t0_tokenizer\"\n#model = TFBertModel.from_pretrained(model_path, from_pt=True, config=config)\nmodel = TFBertForSequenceClassification.from_pretrained(model_path, from_pt=True) # modify labels as needed.\n\"\"\"\n id2label={'0':'Cinsiyetçilik',\n                                                                                   '1':'Irkçılık',\n                                                                                   '2':'Kızdırma',\n                                                                                   '3':'Nötr'}\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:08.115359Z","iopub.execute_input":"2022-08-16T20:39:08.115939Z","iopub.status.idle":"2022-08-16T20:39:11.136314Z","shell.execute_reply.started":"2022-08-16T20:39:08.115904Z","shell.execute_reply":"2022-08-16T20:39:11.135218Z"},"trusted":true},"execution_count":66,"outputs":[]},{"cell_type":"markdown","source":"```\n2022-08-16 20:39:09.086630: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.087439: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.087989: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.089037: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.089599: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.090225: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.090982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.091566: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-08-16 20:39:09.092111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15047 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n2022-08-16 20:39:09.130184: I tensorflow/stream_executor/cuda/cuda_driver.cc:732] failed to allocate 14.69G (15778709504 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory\n2022-08-16 20:39:09.132511: I tensorflow/stream_executor/cuda/cuda_driver.cc:732] failed to allocate 13.22G (14200838144 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory\n2022-08-16 20:39:09.134833: I tensorflow/stream_executor/cuda/cuda_driver.cc:732] failed to allocate 11.90G (12780753920 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory\n2022-08-16 20:39:09.137573: I tensorflow/stream_executor/cuda/cuda_driver.cc:732] failed to allocate 10.71G (11502678016 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory\nSome weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForSequenceClassification: ['bert.embeddings.position_ids']\n- This IS expected if you are initializing TFBertForSequenceClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing TFBertForSequenceClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\nAll the weights of TFBertForSequenceClassification were initialized from the PyTorch model.\nIf your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForSequenceClassification for predictions without further training.\n```","metadata":{}},{"cell_type":"code","source":"from transformers import TextClassificationPipeline\n\ntext = [\"Selam herkese bugün güzel bir gün\",\n        \"Aptal zihniyetinizde bir Yunan yatıyor\",\n        \"Akşam halısahaya giderken karısından izin alanda kendine erkeğim demesin!\",\n        \"kör olası çöpçüler aşkımı süpürmüşler\",\n        \"sınıfımdaki deve hörgüçleri\",\n       \"bugün de ölmedik\",\n       'seninle iyi anlaştık',\n       'seni sevmek ümitli şey ama artık ümit yetmiyor bana',\n       'Selam sen hariç piç',\n       'bana bak kadın']\n\npipe = TextClassificationPipeline(model=model, tokenizer=tokenizer)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:11.138262Z","iopub.execute_input":"2022-08-16T20:39:11.139049Z","iopub.status.idle":"2022-08-16T20:39:13.022805Z","shell.execute_reply.started":"2022-08-16T20:39:11.138990Z","shell.execute_reply":"2022-08-16T20:39:13.021706Z"},"trusted":true},"execution_count":67,"outputs":[]},{"cell_type":"code","source":"[print(f\"{text[index]} - {i['label']}\") for index, i in enumerate(pipe(text))]","metadata":{"execution":{"iopub.status.busy":"2022-08-16T20:39:13.024548Z","iopub.execute_input":"2022-08-16T20:39:13.024979Z","iopub.status.idle":"2022-08-16T20:39:16.636101Z","shell.execute_reply.started":"2022-08-16T20:39:13.024929Z","shell.execute_reply":"2022-08-16T20:39:16.635052Z"},"trusted":true},"execution_count":68,"outputs":[]},{"cell_type":"markdown","source":"```\nSelam herkese bugün güzel bir gün - LABEL_3\nAptal zihniyetinizde bir Yunan yatıyor - LABEL_1\nAkşam halısahaya giderken karısından izin alanda kendine erkeğim demesin! - LABEL_0\nkör olası çöpçüler aşkımı süpürmüşler - LABEL_2\nsınıfımdaki deve hörgüçleri - LABEL_2\nbugün de ölmedik - LABEL_3\nakilli erdogan! - LABEL_2\nSen çok adi piç bir insansın diyebilirdim lakin doğru olmazdı - LABEL_2\nseninle iyi anlaştık - LABEL_3\nseni sevmek ümitli şey ama artık ümit yetmiyor bana - LABEL_3\nSelam sen hariç piç - LABEL_2\nbana bak kadın - LABEL_3\n```","metadata":{}}]}