train_custom_transformer_model.py

import torch
import pandas as pd
import numpy as np 
import random
from sklearn.preprocessing import LabelEncoder
from sklearn.utils.class_weight import compute_class_weight
import time
import datetime
from sklearn.metrics import classification_report,confusion_matrix
import random
import time
import torch.nn as nn
from transformers import (
    AutoModel , 
    AutoConfig , 
    AutoTokenizer , 
    AdamW ,
    get_linear_schedule_with_warmup 
)
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib import rc
from matplotlib.ticker import MaxNLocator
from os import getcwd , listdir 
from os.path import join
from utils import (
    check_accuracy , 
    eval_custom_model , 
    format_time , 
    get_predictions_on_test ,
    get_predictions ,
    good_update_interval ,
    make_smart_batches_on_test ,
    make_smart_batches ,
    plot_training_history , 
    show_confusion_matrix,
    Custom_Model,
    nll_loss_with_class_weights,
    sadice_loss
)

Path_To_Save_Model = 'your path'
Path_To_Save_Submission_DataFrame = 'your path'

# checking for GPU availability
if torch.cuda.is_available():    

    # Tell PyTorch to use the GPU.    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('We will use the GPU:', torch.cuda.get_device_name(0))

# If not...
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")



# Use plot styling from seaborn.
sns.set(style='darkgrid')
# Increase the plot size and font size.
sns.set(font_scale=1.5)
plt.rcParams["figure.figsize"] = (16,12)


# Set the seed value all over the place to make this reproducible.
seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)



# Importing dataset directories"
curr_dir = getcwd()
drive_dir = join(curr_dir , 'drive','MyDrive')
dataset_dir = join(drive_dir,'Codalab','Offensive Language Identification','Dataset')


## Loading training data
train_df_path = join(dataset_dir,'train.csv')
# Load the dataset into a pandas dataframe.
train_df = pd.read_csv(train_df_path)
# Report the number of sentences.
print('Number of training sentences: {:,}\n'.format(train_df.shape[0]))
# Display 10 random rows from the data.
train_df.sample(10)

# labelencoding the target vars
le = LabelEncoder()
train_df['label'] = le.fit_transform(train_df['label'])

# Loading `train_sentences` and `train_labels`
train_sentences = train_df['text'].values
train_labels = train_df['label'].values

# Saving Class Names
class_names = le.inverse_transform(range(0,6))


# Loading the validation data
val_df_path = join(dataset_dir,'dev.csv')
# Load the dataset into a pandas dataframe.
val_df = pd.read_csv(val_df_path)
# Report the number of sentences.
print('Number of validation sentences: {:,}\n'.format(val_df.shape[0]))
# Display 10 random rows from the data.
val_df.sample(10)
val_df['label'] = le.transform(val_df['label'])

# Loading `val_sentences` and `val_labels`
val_sentences = val_df['text'].values
val_labels = val_df['label'].values

# Loading the test data
test_df_path = join(dataset_dir,'test.csv')
# Load the dataset into a pandas dataframe.
test_df = pd.read_csv(test_df_path)
# Report the number of sentences.
print('Number of test sentences: {:,}\n'.format(test_df.shape[0]))

# Display 10 random rows from the data.
test_df.sample(10)
test_df['label'] = le.transform(test_df['label'])

# Loading `test_sentences` and `test_labels`
test_sentences = test_df['text'].values
test_labels = test_df['label'].values

#computing the class weights
class_wts = compute_class_weight('balanced', np.unique(train_labels), train_labels)
print(f"Class weights => {class_wts}")

# Smart Batching of the training data"

model_name = MODEL_NAME ## could be any transformer model
# Load the BERT tokenizer.
print(f'Loading {model_name} tokenizer...')
tokenizer = AutoTokenizer.from_pretrained(model_name,use_fast=False)  ## use_fast flag is needed for IndicBERT model

lengths = []
for text in train_sentences:
  lengths.append(len(text))

## visualizing before tokenizing
plt.scatter(range(0, len(lengths)), lengths, marker="|")
plt.xlabel('Sample Number')
plt.ylabel('Sequence Length')
plt.title('Samples BEFORE Tokenizing')
plt.show()

# Tokenizing the sequences

## setting max_input_length 
max_input_length = 400


full_input_ids = []
labels = []

## Tokenizing each sample
print('Tokenizing {:,} training samples...'.format(len(train_sentences)))

# Choose an interval on which to print progress updates.
update_interval = good_update_interval(total_iters=len(train_sentences), num_desired_updates=10)

# For each training example...
for text in train_sentences:
    
    # Report progress.
    if ((len(full_input_ids) % update_interval) == 0):
        print('  Tokenized {:,} samples.'.format(len(full_input_ids)))

    # Tokenize the sentence.
    input_ids = tokenizer.encode(text=text,           
                                 add_special_tokens=True, 
                                 max_length=max_input_length,  
                                 truncation=True,     
                                 padding=False)       
                                 
    # Add the tokenized result to our list.
    full_input_ids.append(input_ids)
    
print('DONE.')
print('{:>10,} samples'.format(len(full_input_ids)))



# Get all of the lengths.
unsorted_lengths = [len(x) for x in full_input_ids]

## Visualizations after tokenizing 
plt.scatter(range(0, len(unsorted_lengths)), unsorted_lengths, marker="|")
plt.xlabel('Sample Number')
plt.ylabel('Sequence Length')
plt.title('Samples BEFORE Sorting')

plt.show()

# Sort the two lists together by the length of the input sequence.
train_samples = sorted(zip(full_input_ids, train_labels), key=lambda x: len(x[0]))

print(f"Shortest sample: { len(train_samples[0][0]) }")
print(f"Longest sample: { len(train_samples[-1][0]) }")

# Get the new list of lengths after sorting.
sorted_lengths = [len(s[0]) for s in train_samples]

## plotting lenghts of sequences after sorting
plt.plot(range(0, len(sorted_lengths)), sorted_lengths)
plt.xlabel('Sample Number')
plt.ylabel('Sequence Length')
plt.title('Samples after Sorting')

plt.show()

#################### Random Batch Selections ####################



## setting our batch size
batch_size = 16



# List of batches that we'll construct.
batch_ordered_sentences = []
batch_ordered_labels = []

print('Creating training batches of size {:}'.format(batch_size))

# Loop over all of the input samples...    
while len(train_samples) > 0:
    
    # Report progress.
    if ((len(batch_ordered_sentences) % 100) == 0):
        print('  Selected {:,} batches.'.format(len(batch_ordered_sentences)))

    # `to_take` is our actual batch size. It will be `batch_size` until 
    # we get to the last batch, which may be smaller. 
    to_take = min(batch_size, len(train_samples))

    # Pick a random index in the list of remaining samples to start
    # our batch at.
    select = random.randint(0, len(train_samples) - to_take)

    # Select a contiguous batch of samples starting at `select`.
    batch = train_samples[select:(select + to_take)]

    # Each sample is a tuple--split them apart to create a separate list of 
    # sequences and a list of labels for this batch.
    batch_ordered_sentences.append([s[0] for s in batch])
    batch_ordered_labels.append([s[1] for s in batch])

    # Remove these samples from the list.
    del train_samples[select:select + to_take]

print('\n  DONE - {:,} batches.'.format(len(batch_ordered_sentences)))

print(batch_ordered_sentences[0])


############ Padding #########

py_inputs = []
py_attn_masks = []
py_labels = []

# For each batch...
for (batch_inputs, batch_labels) in zip(batch_ordered_sentences, batch_ordered_labels):

    batch_padded_inputs = []
    batch_attn_masks = []
    
    # First, find the longest sample in the batch. 
    # Note that the sequences do currently include the special tokens!
    max_size = max([len(sen) for sen in batch_inputs])

    #print('Max size:', max_size)

    # For each input in this batch...
    for sen in batch_inputs:
        
        # How many pad tokens do we need to add?
        num_pads = max_size - len(sen)

        # Add `num_pads` padding tokens to the end of the sequence.
        padded_input = sen + [tokenizer.pad_token_id]*num_pads

        # Define the attention mask--it's just a `1` for every real token
        # and a `0` for every padding token.
        attn_mask = [1] * len(sen) + [0] * num_pads

        # Add the padded results to the batch.
        batch_padded_inputs.append(padded_input)
        batch_attn_masks.append(attn_mask)

    # Our batch has been padded, so we need to save this updated batch.
    # We also need the inputs to be PyTorch tensors, so we'll do that here.
    py_inputs.append(torch.tensor(batch_padded_inputs))
    py_attn_masks.append(torch.tensor(batch_attn_masks))
    py_labels.append(torch.tensor(batch_labels))

# Check the number of token reductions because of smart batching

# Get the new list of lengths after sorting.

padded_lengths = []

# For each batch...
for batch in py_inputs:
    
    # For each sample...
    for s in batch:
    
        # Record its length.
        padded_lengths.append(len(s))



######################## Checking token reduction ###############

# Sum up the lengths to the get the total number of tokens after smart batching.
smart_token_count = np.sum(padded_lengths)

# To get the total number of tokens in the dataset using fixed padding, it's
# as simple as the number of samples times our `max_len` parameter (that we
# would pad everything to).
fixed_token_count = len(train_sentences) * max_input_length

# Calculate the percentage reduction.
prcnt_reduced = (fixed_token_count - smart_token_count) / float(fixed_token_count) 

print('Total tokens:')
print('   Fixed Padding: {:,}'.format(fixed_token_count))
print('  Smart Batching: {:,}  ({:.1%} less)'.format(smart_token_count, prcnt_reduced))



# Load the model configuration from the transformers library using AutoConfig

# Load the Config object, with an output configured for classification.
config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_name,
                                    num_labels=len(class_names))

print('Config type:', str(type(config)), '\n')



# Load the model from the transformers library using AutoModel"

# Load the pre-trained model for classification, passing in the `config` from above.
model = AutoModel.from_pretrained(
    pretrained_model_name_or_path=model_name,
    config=config)

print('\nModel type:', str(type(model)))



# freeze all the parameters
for param in model.parameters():
    param.requires_grad = False


print('\nLoading model ...')
# pass the pre-trained model to our defined architecture
model = Custom_Model(model,num_labels=len(class_names))
# set the model on cuda
model.cuda()


# custom loss function
nll_loss  = nll_loss_with_class_weights(class_wts,device)

# Loading Optimizer

optimizer = AdamW(model.parameters(),
                  lr = 2e-5, 
                  eps = 1e-8 
                )

# Loading lr scheduler

epochs = 4

# Total number of training steps is [number of batches] x [number of epochs]. 
# Note that it's the number of *batches*, not *samples*!
total_steps = len(py_inputs) * epochs

# Create the learning rate scheduler.
scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps = 0, # Default value in run_glue.py
                                            num_training_steps = total_steps)

# Training Loop

# We'll store a number of quantities such as training and validation loss, 
# validation accuracy, and timings.
training_stats = {
    'epoch':[],
    'train_loss':[],
    'Training Time':[],
    'val_loss':[],
    'Validation Time':[],
    'train_acc':[],
    'val_acc':[]
}

# Update every `update_interval` batches.
update_interval = good_update_interval(total_iters=len(py_inputs), num_desired_updates=10)

# Measure the total training time for the whole run.
total_t0 = time.time()
best_accuracy = 0
# For each epoch...
for epoch_i in range(0, epochs):

    predictions = []
    true_labels = []
    
    # ========================================
    #               Training
    # ========================================
    
    # Perform one full pass over the training set.

    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
    
    # At the start of each epoch (except for the first) we need to re-randomize
    # our training data.
    if epoch_i > 0:
        # Use our `make_smart_batches` function (from 6.1.) to re-shuffle the 
        # dataset into new batches.
        (py_inputs, py_attn_masks, py_labels) = make_smart_batches(train_sentences, train_labels, batch_size,tokenizer,max_input_length)
    
    print('Training on {:,} batches...'.format(len(py_inputs)))

    # Measure how long the training epoch takes.
    t0 = time.time()

    # Reset the total loss for this epoch.
    total_train_loss = 0

    model.train()

    # For each batch of training data...
    for step in range(0, len(py_inputs)):

        # Progress update every, e.g., 100 batches.
        if step % update_interval == 0 and not step == 0:
            # Calculate elapsed time in minutes.
            elapsed = format_time(time.time() - t0)
            
            # Calculate the time remaining based on our progress.
            steps_per_sec = (time.time() - t0) / step
            remaining_sec = steps_per_sec * (len(py_inputs) - step)
            remaining = format_time(remaining_sec)

            # Report progress.
            print('  Batch {:>7,}  of  {:>7,}.    Elapsed: {:}.  Remaining: {:}'.format(step, len(py_inputs), elapsed, remaining))

        # Copy the current training batch to the GPU using the `to` method.
        b_input_ids = py_inputs[step].to(device)
        b_input_mask = py_attn_masks[step].to(device)
        b_labels = py_labels[step].to(device)

        # clearing any previously calculated gradients before performing a backward pass.
        model.zero_grad()        

        # Perform a forward pass (evaluate the model on this training batch).
        logits = model(b_input_ids, b_input_mask)

        # Accumulate the training loss over all of the batches so that we can
        # calculate the average loss at the end.
        loss = nll_loss(logits,b_labels)
        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()
      
        # Store predictions and true labels
        predictions.append(logits)
        true_labels.append(label_ids)
        
        # Accumulate the training loss over all of the batches so that we can
        # calculate the average loss at the end.
        total_train_loss += loss.item()

        # Perform a backward pass to calculate the gradients.
        loss.backward()

        # Clip the norm of the gradients to 1.0.
        # This is to help prevent the "exploding gradients" problem.
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        # Update parameters and take a step using the computed gradient.
        optimizer.step()

        # Update the learning rate.
        scheduler.step()

    # Calculate the average loss over all of the batches.
    avg_train_loss = total_train_loss / len(py_inputs)     
       
    training_accuracy = check_accuracy(predictions,true_labels)       
    
    # Measure how long this epoch took.
    training_time = format_time(time.time() - t0)

    print("")
    print("  Average training loss: {0:.2f}".format(avg_train_loss))
    print("  Training Accuracy: {0:.2f}".format(training_accuracy))
    print("  Training epoch took: {:}".format(training_time))
        
    (py_inputs, py_attn_masks, py_labels) = make_smart_batches(val_sentences, val_labels, batch_size ,tokenizer,max_input_length)
    val_loss,val_accuracy,validation_time = eval_custom_model(model,py_inputs, py_attn_masks, py_labels,nll_loss)

    if val_accuracy > best_accuracy:
        torch.save(model.state_dict(), 'best_model_state.bin')
        best_accuracy = val_accuracy


    # Record all statistics from this epoch.

    print("")
    print("  Average validation loss: {0:.2f}".format(val_loss))
    print("  Validation Accuracy: {0:.2f}".format(val_accuracy))
    print("  Validation epoch took: {:}".format(validation_time))
    
    
    training_stats['epoch'].append(epoch_i + 1)
    training_stats['train_loss'].append(avg_train_loss)
    training_stats['Training Time'].append(training_time)
    training_stats['val_loss'].append(val_loss)
    training_stats['Validation Time'].append(validation_time)
    training_stats['train_acc'].append(training_accuracy)
    training_stats['val_acc'].append(val_accuracy)

print(f'Best val accuracy: {best_accuracy}')
  
model.load_state_dict(torch.load('best_model_state.bin'))

print("")
print("Training complete!")

print("Total training took {:} (h:mm:ss)".format(format_time(time.time()-total_t0)))

plot_training_history(training_stats)

## Evaluating Performance Over Training Set

(py_inputs, py_attn_masks, py_labels) = make_smart_batches(train_sentences, train_labels, batch_size ,tokenizer,max_input_length)

y_pred , y_true = get_predictions(py_inputs, py_attn_masks, py_labels)

print(classification_report(y_true, y_pred, target_names=class_names))

cm = confusion_matrix(y_true, y_pred)
show_confusion_matrix(cm, class_names)

## Evaluating Performance Over Validation Set

(py_inputs, py_attn_masks, py_labels) = make_smart_batches(val_sentences, val_labels, batch_size ,tokenizer,max_input_length)

y_pred , y_true = get_predictions(py_inputs, py_attn_masks, py_labels)

print(classification_report(y_true, y_pred, target_names=class_names))

cm = confusion_matrix(y_true, y_pred)
show_confusion_matrix(cm, class_names)

## Making Predictions on Test Set"

test_df_path = join(dataset_dir,'test.csv')
test_df = pd.read_csv(test_df_path)

## Loading `test_sentences`
test_sentences = test_df['text'].values 
test_ids = test_df.index.values

(py_inputs, py_attn_masks, py_ids) = make_smart_batches_on_test(test_sentences, test_ids, batch_size,tokenizer,max_input_length)

## Evaluating accuracy over test set

test_ids , test_preds = get_predictions_on_test(py_inputs, py_attn_masks,py_ids)

print(test_preds)

le.inverse_transform(test_preds)

sns.countplot(y =le.inverse_transform(test_preds))

## Saving the model

torch.save(model,Path_To_Save_Model)

## Creating Submission DataFrame

df_new = pd.DataFrame({
    'id':test_ids,
    'label':le.inverse_transform(test_preds)
})

test_df['id'] = test_df.index

df_f = pd.merge(test_df,df_new,on = 'id')

df_f = df_f[['id','text','label']]
df_f

## Saving Submission DataFrame
path = Path_To_Save_Submission_DataFrame

df_f.to_csv(join(path,'submission.csv'),index=False)