GPT2_LoRA(fine_tuning).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyP5Rns2vw2P9ok0yw81irWG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "810817624a614c7f8a44e63a57cdb53b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_075a8b537f654caa8cbca12583a80ca0",
              "IPY_MODEL_706ac39b286f4ca6925ec575980b418b",
              "IPY_MODEL_f3920ac2d59648a0af34a65557c924d6"
            ],
            "layout": "IPY_MODEL_a435cf9c1d5e41ada7214842d3bc3daa"
          }
        },
        "075a8b537f654caa8cbca12583a80ca0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a73c1a4340114ad7b56476ee026bac6b",
            "placeholder": "​",
            "style": "IPY_MODEL_431a2cf800c040d1a756e02a2efec9ab",
            "value": "Map: 100%"
          }
        },
        "706ac39b286f4ca6925ec575980b418b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0c13175ed7b48a4bbb4cde4ba9885f3",
            "max": 40,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4efeca5d1f04416db1aa1bf29cc33090",
            "value": 40
          }
        },
        "f3920ac2d59648a0af34a65557c924d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77800403be324c1b97817d3c9e68d5b6",
            "placeholder": "​",
            "style": "IPY_MODEL_bf6e2fbcae1947e89fdccefbe743cbe2",
            "value": " 40/40 [00:00&lt;00:00, 393.98 examples/s]"
          }
        },
        "a435cf9c1d5e41ada7214842d3bc3daa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a73c1a4340114ad7b56476ee026bac6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "431a2cf800c040d1a756e02a2efec9ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0c13175ed7b48a4bbb4cde4ba9885f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4efeca5d1f04416db1aa1bf29cc33090": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "77800403be324c1b97817d3c9e68d5b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf6e2fbcae1947e89fdccefbe743cbe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52e0265fbab945b4b69b50abd66fa1d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fb23fc07860d4b3eb131fd92ac1e99b3",
              "IPY_MODEL_cad953862db343ebbcb354ca1342de06",
              "IPY_MODEL_3a57a0e20fc14041a6523320e6829467"
            ],
            "layout": "IPY_MODEL_152aed08ad31444f8c2d24fed240ccd5"
          }
        },
        "fb23fc07860d4b3eb131fd92ac1e99b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d25894d771642c3b40ce30ace6d3796",
            "placeholder": "​",
            "style": "IPY_MODEL_c9903d449103463eaa06e0b4292f88c1",
            "value": "Map: 100%"
          }
        },
        "cad953862db343ebbcb354ca1342de06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_106a9e3d23e649e1b1bc3a8e3bd4f276",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_acec2df8d80f4aa0b8753425017a0662",
            "value": 10
          }
        },
        "3a57a0e20fc14041a6523320e6829467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ceef067c853644fd9282a4b0c929c4d8",
            "placeholder": "​",
            "style": "IPY_MODEL_95463c68eeb34466baadce85de30d240",
            "value": " 10/10 [00:00&lt;00:00, 120.59 examples/s]"
          }
        },
        "152aed08ad31444f8c2d24fed240ccd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d25894d771642c3b40ce30ace6d3796": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9903d449103463eaa06e0b4292f88c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "106a9e3d23e649e1b1bc3a8e3bd4f276": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acec2df8d80f4aa0b8753425017a0662": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ceef067c853644fd9282a4b0c929c4d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95463c68eeb34466baadce85de30d240": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Neel7317/Generative-Models/blob/main/GPT2_LoRA(fine_tuning).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "V7EZLjm6_weV",
        "outputId": "c575a3b3-25d5-4723-dcdc-52edc0797bdb"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.47.1)\n",
            "Collecting datasets\n",
            "  Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.27.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.11.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.21.0)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.67.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess<0.70.17 (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets)\n",
            "  Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.11.10)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.2)\n",
            "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.18.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.12.14)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n",
            "Downloading datasets-3.2.0-py3-none-any.whl (480 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: xxhash, fsspec, dill, multiprocess, datasets\n",
            "  Attempting uninstall: fsspec\n",
            "    Found existing installation: fsspec 2024.10.0\n",
            "    Uninstalling fsspec-2024.10.0:\n",
            "      Successfully uninstalled fsspec-2024.10.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-3.2.0 dill-0.3.8 fsspec-2024.9.0 multiprocess-0.70.16 xxhash-3.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from datasets import Dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import GPT2Tokenizer, GPT2ForSequenceClassification, Trainer, TrainingArguments\n",
        "from peft import get_peft_model, LoraConfig\n",
        "from sklearn.metrics import accuracy_score\n",
        "import numpy as np\n",
        "\n",
        "# Sample Dataset\n",
        "data = [\n",
        "    {\"text\": \"The soccer team clinched a thrilling victory in overtime.\", \"label\": 0},\n",
        "    {\"text\": \"The cricket championship finals attracted global attention.\", \"label\": 0},\n",
        "    {\"text\": \"A new record was set in the 100-meter sprint.\", \"label\": 0},\n",
        "    {\"text\": \"Tennis legends face off in a historic grand slam final.\", \"label\": 0},\n",
        "    {\"text\": \"The basketball league announces the upcoming season schedule.\", \"label\": 0},\n",
        "    {\"text\": \"Fans cheer as their favorite football club wins the title.\", \"label\": 0},\n",
        "    {\"text\": \"Olympic gold medalist announces retirement from the sport.\", \"label\": 0},\n",
        "    {\"text\": \"The marathon saw record participation this year.\", \"label\": 0},\n",
        "    {\"text\": \"Baseball's star player hits a remarkable home run.\", \"label\": 0},\n",
        "    {\"text\": \"The Formula 1 race witnessed a dramatic last-lap overtake.\", \"label\": 0},\n",
        "    {\"text\": \"New training facilities unveiled for the national hockey team.\", \"label\": 0},\n",
        "    {\"text\": \"The underdog team surprises everyone with a stunning win.\", \"label\": 0},\n",
        "    {\"text\": \"The swimming championship showcased extraordinary talent.\", \"label\": 0},\n",
        "    {\"text\": \"The coach shares strategies ahead of the semi-finals.\", \"label\": 0},\n",
        "    {\"text\": \"Athletes gear up for the international sports meet.\", \"label\": 0},\n",
        "    {\"text\": \"The volleyball team secures a crucial win in the tournament.\", \"label\": 0},\n",
        "    {\"text\": \"The government unveils a new healthcare policy.\", \"label\": 1},\n",
        "    {\"text\": \"The president addresses the nation on economic reforms.\", \"label\": 1},\n",
        "    {\"text\": \"The election results spark debates across the country.\", \"label\": 1},\n",
        "    {\"text\": \"Parliament passes a bill to boost renewable energy.\", \"label\": 1},\n",
        "    {\"text\": \"Diplomatic talks between the two nations yield progress.\", \"label\": 1},\n",
        "    {\"text\": \"The mayor announces plans for urban development.\", \"label\": 1},\n",
        "    {\"text\": \"The opposition criticizes the new tax policies.\", \"label\": 1},\n",
        "    {\"text\": \"Campaigns heat up as candidates rally for voter support.\", \"label\": 1},\n",
        "    {\"text\": \"The new law aims to strengthen cybersecurity measures.\", \"label\": 1},\n",
        "    {\"text\": \"The prime minister visits a neighboring country for trade talks.\", \"label\": 1},\n",
        "    {\"text\": \"Protests erupt against the controversial policy decision.\", \"label\": 1},\n",
        "    {\"text\": \"A landmark decision by the court shapes future legislation.\", \"label\": 1},\n",
        "    {\"text\": \"The minister highlights plans to improve education systems.\", \"label\": 1},\n",
        "    {\"text\": \"National security is at the forefront of the political agenda.\", \"label\": 1},\n",
        "    {\"text\": \"The governor outlines disaster relief efforts after the floods.\", \"label\": 1},\n",
        "    {\"text\": \"A major treaty is signed to address climate change issues.\", \"label\": 1},\n",
        "    {\"text\": \"Students prepare for university entrance exams.\", \"label\": 2},\n",
        "    {\"text\": \"The new study techniques promise better learning outcomes.\", \"label\": 2},\n",
        "    {\"text\": \"The education board announces changes in the curriculum.\", \"label\": 2},\n",
        "    {\"text\": \"A groundbreaking study explores the effects of online learning.\", \"label\": 2},\n",
        "    {\"text\": \"A new scholarship program supports underprivileged students.\", \"label\": 2},\n",
        "    {\"text\": \"Teachers attend workshops on integrating technology in classrooms.\", \"label\": 2},\n",
        "    {\"text\": \"The library adds hundreds of new academic resources.\", \"label\": 2},\n",
        "    {\"text\": \"Researchers publish findings on improving memory retention.\", \"label\": 2},\n",
        "    {\"text\": \"Study groups form to tackle challenging science topics.\", \"label\": 2},\n",
        "    {\"text\": \"Students collaborate on innovative projects for the science fair.\", \"label\": 2},\n",
        "    {\"text\": \"The school introduces coding classes for young learners.\", \"label\": 2},\n",
        "    {\"text\": \"An education conference discusses the future of AI in learning.\", \"label\": 2},\n",
        "    {\"text\": \"Experts share tips for mastering difficult math concepts.\", \"label\": 2},\n",
        "    {\"text\": \"The university announces free courses on climate studies.\", \"label\": 2},\n",
        "    {\"text\": \"A recent survey highlights the need for mental health support in schools.\", \"label\": 2},\n",
        "    {\"text\": \"The tutoring center reports an increase in attendance.\", \"label\": 2},\n",
        "    {\"text\": \"Students celebrate as their project wins the national innovation award.\", \"label\": 2},\n",
        "    {\"text\": \"A new app helps students track their study habits effectively.\", \"label\": 2},\n",
        "]\n",
        "\n",
        "# Split the data into training and validation sets\n",
        "train_texts, val_texts = train_test_split(data, test_size=0.2, random_state=42)\n",
        "train_dataset = Dataset.from_list(train_texts)\n",
        "val_dataset = Dataset.from_list(val_texts)\n",
        "\n",
        "# Tokenizer and Base Model\n",
        "tokenizer = GPT2Tokenizer.from_pretrained(\"distilgpt2\")\n",
        "tokenizer.pad_token = tokenizer.eos_token  # Set padding token as EOS\n",
        "base_model = GPT2ForSequenceClassification.from_pretrained(\"distilgpt2\", num_labels=3)\n",
        "base_model.config.pad_token_id = tokenizer.pad_token_id  # Set pad_token_id explicitly\n",
        "\n",
        "# LoRA Configuration\n",
        "lora_config = LoraConfig(\n",
        "    r=8,  # Rank of low-rank adapters\n",
        "    lora_alpha=16,  # Scaling factor for LoRA\n",
        "    target_modules=[\"attn.c_attn\", \"attn.c_proj\", \"mlp.c_fc\", \"mlp.c_proj\"],  # Target modules\n",
        "    task_type=\"SEQ_CLS\",  # Task type for sequence classification\n",
        ")\n",
        "\n",
        "# Wrap Base Model with LoRA\n",
        "model = get_peft_model(base_model, lora_config)\n",
        "\n",
        "# Tokenization function\n",
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True, max_length=512)\n",
        "\n",
        "# Tokenize datasets\n",
        "tokenized_train = train_dataset.map(tokenize_function, batched=True)\n",
        "tokenized_val = val_dataset.map(tokenize_function, batched=True)\n",
        "\n",
        "# Remove unneeded columns\n",
        "tokenized_train = tokenized_train.remove_columns([\"text\"])\n",
        "tokenized_val = tokenized_val.remove_columns([\"text\"])\n",
        "\n",
        "# Format to Torch\n",
        "tokenized_train.set_format(\"torch\")\n",
        "tokenized_val.set_format(\"torch\")\n",
        "\n",
        "# Training Arguments\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./results_lora_finetune_gpt2\",\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=4,\n",
        "    per_device_eval_batch_size=4,\n",
        "    num_train_epochs=3,\n",
        "    weight_decay=0.01,\n",
        "    logging_dir=\"./logs_lora_finetune_gpt2\",\n",
        "    logging_steps=10,\n",
        "    no_cuda=True,  # Ensure training on CPU\n",
        ")\n",
        "\n",
        "# Trainer\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=tokenized_train,\n",
        "    eval_dataset=tokenized_val,\n",
        "    tokenizer=tokenizer,\n",
        ")\n",
        "\n",
        "# Train the model\n",
        "trainer.train()\n",
        "\n",
        "# Save the model and tokenizer\n",
        "model.save_pretrained(\"./lora_finetune_gpt2_model\")\n",
        "tokenizer.save_pretrained(\"./lora_finetune_gpt2_model\")\n",
        "\n",
        "# Evaluation\n",
        "eval_results = trainer.evaluate()\n",
        "\n",
        "# Print evaluation results\n",
        "print(\"Evaluation Results:\")\n",
        "print(f\"Loss: {eval_results['eval_loss']:.4f}\")\n",
        "print(f\"Perplexity: {torch.exp(torch.tensor(eval_results['eval_loss'])).item():.4f}\")\n",
        "\n",
        "# If desired, add additional evaluation metrics such as accuracy\n",
        "predictions = trainer.predict(tokenized_val)\n",
        "\n",
        "# Move tensors to the same device (CPU)\n",
        "preds = torch.tensor(np.argmax(predictions.predictions, axis=-1), device='cpu')\n",
        "labels = torch.tensor(tokenized_val[\"label\"], device='cpu')\n",
        "\n",
        "# Calculate accuracy\n",
        "accuracy = accuracy_score(labels.numpy(), preds.numpy())\n",
        "print(f\"Accuracy: {accuracy:.4f}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 531,
          "referenced_widgets": [
            "810817624a614c7f8a44e63a57cdb53b",
            "075a8b537f654caa8cbca12583a80ca0",
            "706ac39b286f4ca6925ec575980b418b",
            "f3920ac2d59648a0af34a65557c924d6",
            "a435cf9c1d5e41ada7214842d3bc3daa",
            "a73c1a4340114ad7b56476ee026bac6b",
            "431a2cf800c040d1a756e02a2efec9ab",
            "a0c13175ed7b48a4bbb4cde4ba9885f3",
            "4efeca5d1f04416db1aa1bf29cc33090",
            "77800403be324c1b97817d3c9e68d5b6",
            "bf6e2fbcae1947e89fdccefbe743cbe2",
            "52e0265fbab945b4b69b50abd66fa1d6",
            "fb23fc07860d4b3eb131fd92ac1e99b3",
            "cad953862db343ebbcb354ca1342de06",
            "3a57a0e20fc14041a6523320e6829467",
            "152aed08ad31444f8c2d24fed240ccd5",
            "4d25894d771642c3b40ce30ace6d3796",
            "c9903d449103463eaa06e0b4292f88c1",
            "106a9e3d23e649e1b1bc3a8e3bd4f276",
            "acec2df8d80f4aa0b8753425017a0662",
            "ceef067c853644fd9282a4b0c929c4d8",
            "95463c68eeb34466baadce85de30d240"
          ]
        },
        "id": "mz1XBuvm_lAG",
        "outputId": "54ec205d-5971-4e9e-bb22-621186fcb92e"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at distilgpt2 and are newly initialized: ['score.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.10/dist-packages/peft/tuners/lora/layer.py:1264: UserWarning: fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/40 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "810817624a614c7f8a44e63a57cdb53b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/10 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "52e0265fbab945b4b69b50abd66fa1d6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1590: FutureWarning: using `no_cuda` is deprecated and will be removed in version 5.0 of 🤗 Transformers. Use `use_cpu` instead\n",
            "  warnings.warn(\n",
            "<ipython-input-6-9f5f35f78572>:116: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
            "  trainer = Trainer(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [30/30 06:33, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.984700</td>\n",
              "      <td>1.895181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.785000</td>\n",
              "      <td>1.852549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.770700</td>\n",
              "      <td>1.836410</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation Results:\n",
            "Loss: 1.8364\n",
            "Perplexity: 6.2740\n",
            "Accuracy: 0.5000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-6-9f5f35f78572>:144: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  labels = torch.tensor(tokenized_val[\"label\"], device='cpu')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vQynHl2ACHXz"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}