mistral_7b_fine_tuning_with_peft_lora_and_wandb.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.12",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [
        {
          "sourceId": 5111,
          "sourceType": "modelInstanceVersion",
          "modelInstanceId": 3899
        },
        {
          "sourceId": 5110,
          "sourceType": "modelInstanceVersion",
          "isSourceIdPinned": true,
          "modelInstanceId": 3898
        }
      ],
      "dockerImageVersionId": 30559,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# You only need to run this once per machine\n",
        "!pip install -q -U bitsandbytes\n",
        "!pip install -q -U transformers\n",
        "!pip install -q -U peft\n",
        "!pip install -q -U accelerate\n",
        "!pip install -q -U datasets scipy ipywidgets\n",
        "!pip install -q -U trl"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:10:58.424681Z",
          "iopub.execute_input": "2023-11-22T02:10:58.425085Z",
          "iopub.status.idle": "2023-11-22T02:12:45.433495Z",
          "shell.execute_reply.started": "2023-11-22T02:10:58.425053Z",
          "shell.execute_reply": "2023-11-22T02:12:45.432232Z"
        },
        "trusted": true,
        "id": "iCczIBZT-ykI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from kaggle_secrets import UserSecretsClient\n",
        "import wandb\n",
        "\n",
        "user_secrets = UserSecretsClient()\n",
        "secret_hf = user_secrets.get_secret(\"HUGGINGFACE_API\")\n",
        "secret_wandb = user_secrets.get_secret(\"wandb\")\n",
        "\n",
        "!huggingface-cli login --token $secret_hf\n",
        "\n",
        "wandb.login(key = secret_wandb)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:12:45.435454Z",
          "iopub.execute_input": "2023-11-22T02:12:45.435742Z",
          "iopub.status.idle": "2023-11-22T02:12:50.464710Z",
          "shell.execute_reply.started": "2023-11-22T02:12:45.435713Z",
          "shell.execute_reply": "2023-11-22T02:12:50.463712Z"
        },
        "trusted": true,
        "id": "xYh-wXYM-ykP",
        "outputId": "2e683a3e-72ae-451a-f7bc-ea4bde4fbfd2"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\nToken is valid (permission: read).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n",
          "output_type": "stream"
        },
        {
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
          "output_type": "stream"
        },
        {
          "execution_count": 2,
          "output_type": "execute_result",
          "data": {
            "text/plain": "True"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "from copy import deepcopy\n",
        "from random import randrange\n",
        "from functools import partial\n",
        "\n",
        "import torch\n",
        "import accelerate\n",
        "import bitsandbytes as bnb\n",
        "\n",
        "from datasets import load_dataset\n",
        "from transformers import (\n",
        "    AutoModelForCausalLM,\n",
        "    AutoTokenizer,\n",
        "    BitsAndBytesConfig,\n",
        "    TrainingArguments,\n",
        "    Trainer,\n",
        "    DataCollatorForLanguageModeling\n",
        ")\n",
        "from transformers.integrations import WandbCallback\n",
        "from peft import (\n",
        "    LoraConfig,\n",
        "    prepare_model_for_kbit_training,\n",
        "    get_peft_model,\n",
        "    PeftModel\n",
        ")\n",
        "from trl import SFTTrainer"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:12:50.466203Z",
          "iopub.execute_input": "2023-11-22T02:12:50.466786Z",
          "iopub.status.idle": "2023-11-22T02:13:05.655111Z",
          "shell.execute_reply.started": "2023-11-22T02:12:50.466746Z",
          "shell.execute_reply": "2023-11-22T02:13:05.654095Z"
        },
        "trusted": true,
        "id": "Xi4hzrie-ykR",
        "outputId": "b37ce5dc-c553-41d1-b81b-c65715833da0"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "<frozen importlib._bootstrap>:241: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n/opt/conda/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.\n  warnings.warn(\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = \"/kaggle/input/mistral/pytorch/7b-v0.1-hf/1\""
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:13:05.658086Z",
          "iopub.execute_input": "2023-11-22T02:13:05.658774Z",
          "iopub.status.idle": "2023-11-22T02:13:05.663898Z",
          "shell.execute_reply.started": "2023-11-22T02:13:05.658745Z",
          "shell.execute_reply": "2023-11-22T02:13:05.662242Z"
        },
        "trusted": true,
        "id": "VABSdLpk-ykS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "tokenizer.pad_token = tokenizer.eos_token"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:13:05.665130Z",
          "iopub.execute_input": "2023-11-22T02:13:05.665492Z",
          "iopub.status.idle": "2023-11-22T02:13:05.794135Z",
          "shell.execute_reply.started": "2023-11-22T02:13:05.665468Z",
          "shell.execute_reply": "2023-11-22T02:13:05.793367Z"
        },
        "trusted": true,
        "id": "1rjNqdjP-ykS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_compute_dtype=torch.bfloat16,\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:13:05.795167Z",
          "iopub.execute_input": "2023-11-22T02:13:05.795471Z",
          "iopub.status.idle": "2023-11-22T02:13:05.801371Z",
          "shell.execute_reply.started": "2023-11-22T02:13:05.795445Z",
          "shell.execute_reply": "2023-11-22T02:13:05.800475Z"
        },
        "trusted": true,
        "id": "jkoUgnzf-ykT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "    model_name,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map=\"auto\",  # Auto selects device to put model on.\n",
        ")\n",
        "model.config.use_cache = False"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:13:05.802940Z",
          "iopub.execute_input": "2023-11-22T02:13:05.803313Z",
          "iopub.status.idle": "2023-11-22T02:15:28.074801Z",
          "shell.execute_reply.started": "2023-11-22T02:13:05.803269Z",
          "shell.execute_reply": "2023-11-22T02:15:28.073750Z"
        },
        "trusted": true,
        "colab": {
          "referenced_widgets": [
            "088cd94ad39c4ce890daa1e717b80447"
          ]
        },
        "id": "DQjeVr0x-ykT",
        "outputId": "73fca0ba-aa8e-4583-bd5f-465498e65e85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "088cd94ad39c4ce890daa1e717b80447"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# model.gradient_checkpointing_enable() # use the line below instead\n",
        "model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)  # Explicitly specify!"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.076116Z",
          "iopub.execute_input": "2023-11-22T02:15:28.076492Z",
          "iopub.status.idle": "2023-11-22T02:15:28.093544Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.076458Z",
          "shell.execute_reply": "2023-11-22T02:15:28.092600Z"
        },
        "trusted": true,
        "id": "l9d13mQV-ykU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.095167Z",
          "iopub.execute_input": "2023-11-22T02:15:28.095465Z",
          "iopub.status.idle": "2023-11-22T02:15:28.103830Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.095440Z",
          "shell.execute_reply": "2023-11-22T02:15:28.102799Z"
        },
        "trusted": true,
        "id": "TlGOzXzl-ykU",
        "outputId": "409ba835-3ef2-4f30-a5a9-d6d5c27480ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 9,
          "output_type": "execute_result",
          "data": {
            "text/plain": "MistralForCausalLM(\n  (model): MistralModel(\n    (embed_tokens): Embedding(32000, 4096)\n    (layers): ModuleList(\n      (0-31): 32 x MistralDecoderLayer(\n        (self_attn): MistralAttention(\n          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (k_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n          (v_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (rotary_emb): MistralRotaryEmbedding()\n        )\n        (mlp): MistralMLP(\n          (gate_proj): Linear4bit(in_features=4096, out_features=14336, bias=False)\n          (up_proj): Linear4bit(in_features=4096, out_features=14336, bias=False)\n          (down_proj): Linear4bit(in_features=14336, out_features=4096, bias=False)\n          (act_fn): SiLUActivation()\n        )\n        (input_layernorm): MistralRMSNorm()\n        (post_attention_layernorm): MistralRMSNorm()\n      )\n    )\n    (norm): MistralRMSNorm()\n  )\n  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def find_all_linear_names(model):\n",
        "    cls = bnb.nn.Linear4bit #if args.bits == 4 else (bnb.nn.Linear8bitLt if args.bits == 8 else torch.nn.Linear)\n",
        "    lora_module_names = set()\n",
        "    for name, module in model.named_modules():\n",
        "        if isinstance(module, cls):\n",
        "            names = name.split('.')\n",
        "            lora_module_names.add(names[0] if len(names) == 1 else names[-1])\n",
        "\n",
        "\n",
        "    # lm_head is often excluded.\n",
        "    if 'lm_head' in lora_module_names:  # needed for 16-bit\n",
        "        lora_module_names.remove('lm_head')\n",
        "    return list(lora_module_names)\n",
        "\n",
        "\n",
        "modules = find_all_linear_names(model)\n",
        "modules"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.107641Z",
          "iopub.execute_input": "2023-11-22T02:15:28.107939Z",
          "iopub.status.idle": "2023-11-22T02:15:28.120840Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.107907Z",
          "shell.execute_reply": "2023-11-22T02:15:28.119789Z"
        },
        "trusted": true,
        "id": "_gMb90sB-ykV",
        "outputId": "8b285933-1453-4fcf-8503-b916e0d58123"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 10,
          "output_type": "execute_result",
          "data": {
            "text/plain": "['q_proj', 'o_proj', 'k_proj', 'v_proj', 'gate_proj', 'down_proj', 'up_proj']"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "peft_config = LoraConfig(\n",
        "    lora_alpha=16,\n",
        "    lora_dropout=0.1,\n",
        "    target_modules=modules,\n",
        "    r=8,\n",
        "    bias=\"none\",\n",
        "    task_type=\"CAUSAL_LM\"\n",
        ")\n",
        "\n",
        "model = get_peft_model(model, peft_config)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.122021Z",
          "iopub.execute_input": "2023-11-22T02:15:28.122961Z",
          "iopub.status.idle": "2023-11-22T02:15:28.623661Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.122903Z",
          "shell.execute_reply": "2023-11-22T02:15:28.622786Z"
        },
        "trusted": true,
        "id": "mXdyWuiE-ykW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainable, total = model.get_nb_trainable_parameters()\n",
        "print(f\"Trainable: {trainable} | total: {total} | Percentage: {trainable/total*100:.4f}%\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.624959Z",
          "iopub.execute_input": "2023-11-22T02:15:28.625351Z",
          "iopub.status.idle": "2023-11-22T02:15:28.642033Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.625315Z",
          "shell.execute_reply": "2023-11-22T02:15:28.641040Z"
        },
        "trusted": true,
        "id": "_WXnkMFs-ykW",
        "outputId": "89cbbe57-a921-4e2f-a9fe-974659c6d102"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Trainable: 20971520 | total: 7262703616 | Percentage: 0.2888%\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:28.643482Z",
          "iopub.execute_input": "2023-11-22T02:15:28.643897Z",
          "iopub.status.idle": "2023-11-22T02:15:29.829908Z",
          "shell.execute_reply.started": "2023-11-22T02:15:28.643862Z",
          "shell.execute_reply": "2023-11-22T02:15:29.828942Z"
        },
        "trusted": true,
        "id": "YbdPZLsQ-ykW",
        "outputId": "23ce8d3f-1692-4bc1-d444-1ed46979299e"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Wed Nov 22 02:15:29 2023       \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 470.161.03   Driver Version: 470.161.03   CUDA Version: 11.4     |\n|-------------------------------+----------------------+----------------------+\n| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n|                               |                      |               MIG M. |\n|===============================+======================+======================|\n|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n| N/A   48C    P0    27W /  70W |   3274MiB / 15109MiB |      0%      Default |\n|                               |                      |                  N/A |\n+-------------------------------+----------------------+----------------------+\n|   1  Tesla T4            Off  | 00000000:00:05.0 Off |                    0 |\n| N/A   50C    P0    27W /  70W |   4072MiB / 15109MiB |      0%      Default |\n|                               |                      |                  N/A |\n+-------------------------------+----------------------+----------------------+\n                                                                               \n+-----------------------------------------------------------------------------+\n| Processes:                                                                  |\n|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n|        ID   ID                                                   Usage      |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"LDJnr/Puffin\", split=\"train\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:29.831495Z",
          "iopub.execute_input": "2023-11-22T02:15:29.831903Z",
          "iopub.status.idle": "2023-11-22T02:15:32.775653Z",
          "shell.execute_reply.started": "2023-11-22T02:15:29.831855Z",
          "shell.execute_reply": "2023-11-22T02:15:32.774662Z"
        },
        "trusted": true,
        "colab": {
          "referenced_widgets": [
            "36716f6731f0439ab772861695b524e3",
            "aa9191115aba47a292cafe6ffa93204c",
            "da826388c6b643ce87e6502befe5a49c",
            "ccc4ca1e65d546f599dfe99c75c5041b",
            "bbbd058621044380bbb6c14a15ce550f"
          ]
        },
        "id": "7Hb7LiBi-ykX",
        "outputId": "3e0b57fc-45eb-44d1-f3fc-bfe293447a8e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Downloading readme:   0%|          | 0.00/2.70k [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "36716f6731f0439ab772861695b524e3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aa9191115aba47a292cafe6ffa93204c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Downloading data:   0%|          | 0.00/14.4M [00:00<?, ?B/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "da826388c6b643ce87e6502befe5a49c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ccc4ca1e65d546f599dfe99c75c5041b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Generating train split: 0 examples [00:00, ? examples/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bbbd058621044380bbb6c14a15ce550f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_sample = dataset[randrange(len(dataset))]\n",
        "print(type(random_sample))\n",
        "print(random_sample.keys())\n",
        "print(random_sample['id'])\n",
        "print(random_sample[\"conversations\"])"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:32.777107Z",
          "iopub.execute_input": "2023-11-22T02:15:32.778051Z",
          "iopub.status.idle": "2023-11-22T02:15:32.785991Z",
          "shell.execute_reply.started": "2023-11-22T02:15:32.778021Z",
          "shell.execute_reply": "2023-11-22T02:15:32.784882Z"
        },
        "trusted": true,
        "id": "V2sXLsie-ykX",
        "outputId": "65a565d7-67f9-4522-ece3-1404640215ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "<class 'dict'>\ndict_keys(['id', 'conversations'])\n2820\n[{'from': 'human', 'value': 'can you explain this in laymans terms?\\n\\n(B) Schema of the haplo-cord stem cell transplant. The participant received an allogeneic stem cell transplant per institutional standard care. Conditioning regimen was fludarabine 30 mg/m2 daily on days −7 to −3, melphalan 140 mg/m2 × 1 dose (day −2), and total body irradiation at 400 CGy on days −7 to −6. Haploidentical stem cells were infused on day 0, and CCR5 Δ32/Δ32 cord stem cells were infused on day +1. Graft versus host (GVH) disease prophylaxis included: antithymocyte globulin (ATG) 1.5 mg/kg on days −5, −3, and −1; mycophenolate mofetil (MMF) 1 g three times daily on day −2 through day +28; and tacrolimus from day −2 to day 180 post-transplant.'}, {'from': 'gpt', 'value': 'In simpler terms, this passage is describing a procedure called \"haplo-cord stem cell transplant\" for a patient. This is a type of transplant where stem cells from a partially matched donor (haploidentical) and cord blood stem cells with a specific genetic mutation (CCR5 Δ32/Δ32) are used.\\n\\nThe patient first undergoes a \"conditioning regimen\" to prepare their body for the transplant. This involves receiving a series of drugs and radiation treatments over several days to weaken their immune system and make space for the new stem cells. The drugs used include fludarabine, melphalan, and antithymocyte globulin (ATG). The radiation therapy is called total body irradiation.\\n\\nAfter the conditioning regimen, the patient receives the haploidentical stem cells on day 0, followed by the CCR5 Δ32/Δ32 cord stem cells on day +1. To prevent the new stem cells from attacking the patient\\'s body (a condition called graft versus host disease), the patient is given additional medications, including mycophenolate mofetil (MMF) and tacrolimus, for a specific period after the transplant.'}]\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(random_sample[\"conversations\"]))\n",
        "print(random_sample[\"conversations\"][0])\n",
        "print(random_sample[\"conversations\"][1])"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:32.787408Z",
          "iopub.execute_input": "2023-11-22T02:15:32.787735Z",
          "iopub.status.idle": "2023-11-22T02:15:32.819138Z",
          "shell.execute_reply.started": "2023-11-22T02:15:32.787710Z",
          "shell.execute_reply": "2023-11-22T02:15:32.818240Z"
        },
        "trusted": true,
        "id": "UWPW25B4-ykY",
        "outputId": "dbe42536-3763-465c-ee11-a2a93429e1db"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "2\n{'from': 'human', 'value': 'can you explain this in laymans terms?\\n\\n(B) Schema of the haplo-cord stem cell transplant. The participant received an allogeneic stem cell transplant per institutional standard care. Conditioning regimen was fludarabine 30 mg/m2 daily on days −7 to −3, melphalan 140 mg/m2 × 1 dose (day −2), and total body irradiation at 400 CGy on days −7 to −6. Haploidentical stem cells were infused on day 0, and CCR5 Δ32/Δ32 cord stem cells were infused on day +1. Graft versus host (GVH) disease prophylaxis included: antithymocyte globulin (ATG) 1.5 mg/kg on days −5, −3, and −1; mycophenolate mofetil (MMF) 1 g three times daily on day −2 through day +28; and tacrolimus from day −2 to day 180 post-transplant.'}\n{'from': 'gpt', 'value': 'In simpler terms, this passage is describing a procedure called \"haplo-cord stem cell transplant\" for a patient. This is a type of transplant where stem cells from a partially matched donor (haploidentical) and cord blood stem cells with a specific genetic mutation (CCR5 Δ32/Δ32) are used.\\n\\nThe patient first undergoes a \"conditioning regimen\" to prepare their body for the transplant. This involves receiving a series of drugs and radiation treatments over several days to weaken their immune system and make space for the new stem cells. The drugs used include fludarabine, melphalan, and antithymocyte globulin (ATG). The radiation therapy is called total body irradiation.\\n\\nAfter the conditioning regimen, the patient receives the haploidentical stem cells on day 0, followed by the CCR5 Δ32/Δ32 cord stem cells on day +1. To prevent the new stem cells from attacking the patient\\'s body (a condition called graft versus host disease), the patient is given additional medications, including mycophenolate mofetil (MMF) and tacrolimus, for a specific period after the transplant.'}\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# These are multi-turn conversations. They don't end after GPT-4 responds to the user's first message.\n",
        "\n",
        "for i in dataset:\n",
        "  if len(i[\"conversations\"]) > 2:\n",
        "    for j in i[\"conversations\"]:\n",
        "      print(j)  # Conversations are multi-turn (>= 2) and always even in count (human then gpt response).\n",
        "    break"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:32.820440Z",
          "iopub.execute_input": "2023-11-22T02:15:32.820807Z",
          "iopub.status.idle": "2023-11-22T02:15:32.831116Z",
          "shell.execute_reply.started": "2023-11-22T02:15:32.820770Z",
          "shell.execute_reply": "2023-11-22T02:15:32.830138Z"
        },
        "trusted": true,
        "id": "a47bFJIt-ykY",
        "outputId": "43534d9a-0809-433e-923d-b4300454deb1"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "{'from': 'human', 'value': 'How do I center a text element vertically in a row in jetpack compose? The height of the row is determined by a button next to the text element.'}\n{'from': 'gpt', 'value': 'In Jetpack Compose, you can center a text element vertically within a row by using the `Arrangement.Center` parameter for the `verticalAlignment` property in the `Row` composable. Here\\'s an example:\\n```kotlin\\nimport androidx.compose.foundation.layout.Arrangement\\nimport androidx.compose.foundation.layout.Row\\nimport androidx.compose.material.Button\\nimport androidx.compose.material.Text\\nimport androidx.compose.runtime.Composable\\nimport androidx.compose.ui.Alignment\\n\\n@Composable\\nfun CenteredTextWithButton() {\\n    Row(\\n        verticalAlignment = Alignment.CenterVertically\\n    ) {\\n        Text(\"Centered Text\")\\n        Button(onClick = { /* Do something on button click */ }) {\\n            Text(\"Button\")\\n        }\\n    }\\n}\\n```\\nIn this example, the `Text` element will be centered vertically in the `Row`, and the height of the row will be determined by the `Button`.'}\n{'from': 'human', 'value': 'Add some spacing between the text and the button'}\n{'from': 'gpt', 'value': 'To add spacing between the `Text` and `Button` elements within the `Row`, you can use the `Spacer` composable along with a specific width. Here\\'s an updated version of the previous example with added spacing:\\n```kotlin\\nimport androidx.compose.foundation.layout.Arrangement\\nimport androidx.compose.foundation.layout.Row\\nimport androidx.compose.foundation.layout.Spacer\\nimport androidx.compose.foundation.layout.width\\nimport androidx.compose.material.Button\\nimport androidx.compose.material.Text\\nimport androidx.compose.runtime.Composable\\nimport androidx.compose.ui.Alignment\\nimport androidx.compose.ui.Modifier\\nimport androidx.compose.ui.unit.dp\\n\\n@Composable\\nfun CenteredTextWithButton() {\\n    Row(\\n        verticalAlignment = Alignment.CenterVertically\\n    ) {\\n        Text(\"Centered Text\")\\n        Spacer(modifier = Modifier.width(16.dp)) // Add 16 dp of spacing\\n        Button(onClick = { /* Do something on button click */ }) {\\n            Text(\"Button\")\\n        }\\n    }\\n}\\n```\\nIn this example, we\\'ve added a `Spacer` with a width of 16 dp between the `Text` and `Button` elements. You can adjust the width value according to your desired spacing.'}\n{'from': 'human', 'value': 'Instead of using a spacer how do I give some padding to the text element instead?'}\n{'from': 'gpt', 'value': 'You can add padding to the `Text` element by using the `Modifier.padding()` function. Here\\'s an example with padding added to the `Text` element instead of using a `Spacer`:\\n```kotlin\\nimport androidx.compose.foundation.layout.Arrangement\\nimport androidx.compose.foundation.layout.Row\\nimport androidx.compose.material.Button\\nimport androidx.compose.material.Text\\nimport androidx.compose.runtime.Composable\\nimport androidx.compose.ui.Alignment\\nimport androidx.compose.ui.Modifier\\nimport androidx.compose.ui.unit.dp\\n\\n@Composable\\nfun CenteredTextWithButton() {\\n    Row(\\n        verticalAlignment = Alignment.CenterVertically\\n    ) {\\n        Text(\\n            \"Centered Text\",\\n            modifier = Modifier.padding(end = 16.dp) // Add 16 dp of padding to the end (right) side of the text\\n        )\\n        Button(onClick = { /* Do something on button click */ }) {\\n            Text(\"Button\")\\n        }\\n    }\\n}\\n```\\nIn this example, we\\'ve added 16 dp of padding to the end (right) side of the `Text` element, which creates space between the `Text` and the `Button`. You can adjust the padding value as needed.'}\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run = wandb.init(\n",
        "    project=\"Fine tuning mistral 7B\",  # Project name.\n",
        "    name=\"log_dataset\",          # name of the run within this project.\n",
        "    config={                     # Configuration dictionary.\n",
        "        \"split\": \"train\"\n",
        "    },\n",
        "    group=\"dataset\",             # Group runs. This run belongs in \"dataset\".\n",
        "    tags=[\"dataset\"],            # Tags. More dynamic, low-level grouping.\n",
        "    notes=\"Logging subset of Puffin dataset.\",  # Description about the run.\n",
        "    job_type=\"training\",\n",
        ")  # Check out the other parameters in the `wandb.init`!"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:15:32.832645Z",
          "iopub.execute_input": "2023-11-22T02:15:32.833004Z",
          "iopub.status.idle": "2023-11-22T02:16:03.546724Z",
          "shell.execute_reply.started": "2023-11-22T02:15:32.832973Z",
          "shell.execute_reply": "2023-11-22T02:16:03.545788Z"
        },
        "trusted": true,
        "id": "abq2e2hH-ykZ",
        "outputId": "450f7114-ff53-451b-f4ea-2ad4d8c68582"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33malinourian10\u001b[0m (\u001b[33msut-ee\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "wandb version 0.16.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.15.9"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/kaggle/working/wandb/run-20231122_021532-uauimqb9</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B/runs/uauimqb9' target=\"_blank\">log_dataset</a></strong> to <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B/runs/uauimqb9' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B/runs/uauimqb9</a>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = []\n",
        "for i in range(1000):  # Log 1000 instances.\n",
        "    x = dataset[i]\n",
        "    id_ = x[\"id\"]\n",
        "    conversations = x[\"conversations\"]\n",
        "    for idx, response in enumerate(conversations):\n",
        "        data.append([id_, idx, response[\"from\"], response[\"value\"]])\n",
        "\n",
        "\n",
        "table = wandb.Table(data=data, columns=[\"id\", \"idx\", \"from\", \"value\"])\n",
        "run.log({\"first1000_Puffin\": table})\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:03.548885Z",
          "iopub.execute_input": "2023-11-22T02:16:03.549785Z",
          "iopub.status.idle": "2023-11-22T02:16:04.722228Z",
          "shell.execute_reply.started": "2023-11-22T02:16:03.549756Z",
          "shell.execute_reply": "2023-11-22T02:16:04.721252Z"
        },
        "trusted": true,
        "id": "GIP46vul-ykc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run.finish()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:04.723459Z",
          "iopub.execute_input": "2023-11-22T02:16:04.723738Z",
          "iopub.status.idle": "2023-11-22T02:16:34.213595Z",
          "shell.execute_reply.started": "2023-11-22T02:16:04.723711Z",
          "shell.execute_reply": "2023-11-22T02:16:34.212781Z"
        },
        "trusted": true,
        "id": "HOleyONU-ykc",
        "outputId": "4e097fbc-28b7-4ee3-9f6c-ce813074bad2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run <strong style=\"color:#cdcd00\">log_dataset</strong> at: <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B/runs/uauimqb9' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B/runs/uauimqb9</a><br/>Synced 5 W&B file(s), 1 media file(s), 1 artifact file(s) and 0 other file(s)"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Find logs at: <code>./wandb/run-20231122_021532-uauimqb9/logs</code>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below is a conversation between a user and you.\n",
        "\n",
        "\n",
        "<human>: <value>\n",
        "<gpt>: <value>\n",
        "...\n",
        "\n",
        "\n",
        "Instruction: Write a response appropriate to the conversation.\n"
      ],
      "metadata": {
        "id": "dNaxuVwx-ykc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def format_prompt(sample):\n",
        "    \"\"\"Given a sample dictionary with key \"conversations\", format the conversation into a prompt.\n",
        "\n",
        "\n",
        "    Args:\n",
        "      sample: A sample dictionary from a Hugging Face dataset.\n",
        "\n",
        "\n",
        "    Returns:\n",
        "      sample: sample dictionary with \"text\" key for the formatted prompt.\n",
        "    \"\"\"\n",
        "\n",
        "\n",
        "    INTRO = \"Below is a conversation between a user and you.\"\n",
        "    END = \"Instruction: Write a response appropriate to the conversation.\"\n",
        "\n",
        "\n",
        "    conversations = \"\"\n",
        "    for response in sample[\"conversations\"]:\n",
        "      from_, value = response[\"from\"], response[\"value\"]\n",
        "      conversations += f\"<{from_}>: \" + value + \"\\n\"\n",
        "\n",
        "\n",
        "    sample[\"text\"] = \"\\n\\n\".join([INTRO, conversations, END])\n",
        "\n",
        "\n",
        "    return sample\n",
        "\n",
        "format_prompt(random_sample)[\"text\"]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:34.214720Z",
          "iopub.execute_input": "2023-11-22T02:16:34.214994Z",
          "iopub.status.idle": "2023-11-22T02:16:34.224247Z",
          "shell.execute_reply.started": "2023-11-22T02:16:34.214970Z",
          "shell.execute_reply": "2023-11-22T02:16:34.223118Z"
        },
        "trusted": true,
        "id": "h5Fi9h_k-ykf",
        "outputId": "676be08a-4f09-4956-9c35-247a5a3a1ae7"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 21,
          "output_type": "execute_result",
          "data": {
            "text/plain": "'Below is a conversation between a user and you.\\n\\n<human>: can you explain this in laymans terms?\\n\\n(B) Schema of the haplo-cord stem cell transplant. The participant received an allogeneic stem cell transplant per institutional standard care. Conditioning regimen was fludarabine 30 mg/m2 daily on days −7 to −3, melphalan 140 mg/m2 × 1 dose (day −2), and total body irradiation at 400 CGy on days −7 to −6. Haploidentical stem cells were infused on day 0, and CCR5 Δ32/Δ32 cord stem cells were infused on day +1. Graft versus host (GVH) disease prophylaxis included: antithymocyte globulin (ATG) 1.5 mg/kg on days −5, −3, and −1; mycophenolate mofetil (MMF) 1 g three times daily on day −2 through day +28; and tacrolimus from day −2 to day 180 post-transplant.\\n<gpt>: In simpler terms, this passage is describing a procedure called \"haplo-cord stem cell transplant\" for a patient. This is a type of transplant where stem cells from a partially matched donor (haploidentical) and cord blood stem cells with a specific genetic mutation (CCR5 Δ32/Δ32) are used.\\n\\nThe patient first undergoes a \"conditioning regimen\" to prepare their body for the transplant. This involves receiving a series of drugs and radiation treatments over several days to weaken their immune system and make space for the new stem cells. The drugs used include fludarabine, melphalan, and antithymocyte globulin (ATG). The radiation therapy is called total body irradiation.\\n\\nAfter the conditioning regimen, the patient receives the haploidentical stem cells on day 0, followed by the CCR5 Δ32/Δ32 cord stem cells on day +1. To prevent the new stem cells from attacking the patient\\'s body (a condition called graft versus host disease), the patient is given additional medications, including mycophenolate mofetil (MMF) and tacrolimus, for a specific period after the transplant.\\n\\n\\nInstruction: Write a response appropriate to the conversation.'"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_max_length(model):\n",
        "    conf = model.config\n",
        "    max_length = None\n",
        "    for length_setting in [\"n_positions\", \"max_position_embeddings\", \"seq_length\"]:\n",
        "        max_length = getattr(model.config, length_setting, None)\n",
        "        if max_length:\n",
        "            print(f\"Found max length: {max_length}\")\n",
        "            break\n",
        "    if not max_length:\n",
        "        max_length = 1024\n",
        "        print(f\"Using default max length: {max_length}\")\n",
        "    return max_length\n",
        "\n",
        "\n",
        "# Change the max length depending on hardware constraints.\n",
        "max_length = get_max_length(model)\n",
        "print(max_length)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:34.225549Z",
          "iopub.execute_input": "2023-11-22T02:16:34.226113Z",
          "iopub.status.idle": "2023-11-22T02:16:34.235445Z",
          "shell.execute_reply.started": "2023-11-22T02:16:34.226088Z",
          "shell.execute_reply": "2023-11-22T02:16:34.234468Z"
        },
        "trusted": true,
        "id": "xXN6wAmt-ykf",
        "outputId": "348798f1-9ce5-46d8-d91f-379529dfe682"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Found max length: 32768\n32768\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(\n",
        "    random_sample[\"text\"],\n",
        "    max_length=max_length,\n",
        "    truncation=True\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:34.236590Z",
          "iopub.execute_input": "2023-11-22T02:16:34.236856Z",
          "iopub.status.idle": "2023-11-22T02:16:34.255132Z",
          "shell.execute_reply.started": "2023-11-22T02:16:34.236832Z",
          "shell.execute_reply": "2023-11-22T02:16:34.254187Z"
        },
        "trusted": true,
        "id": "R1F6EOZ3-ykg",
        "outputId": "5071c52d-2b03-441d-8dc3-6a5675b7eb08"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 23,
          "output_type": "execute_result",
          "data": {
            "text/plain": "{'input_ids': [1, 20811, 349, 264, 7114, 1444, 264, 2188, 304, 368, 28723, 13, 13, 28789, 18529, 9670, 541, 368, 7282, 456, 297, 4897, 20661, 3471, 28804, 13, 13, 28732, 28760, 28731, 24051, 302, 272, 295, 377, 731, 28733, 19056, 17854, 3601, 1203, 18071, 28723, 415, 28503, 3874, 396, 544, 17344, 294, 17854, 3601, 1203, 18071, 660, 28211, 4787, 1656, 28723, 28237, 288, 983, 21538, 403, 972, 554, 283, 375, 473, 28705, 28770, 28734, 18144, 28748, 28719, 28750, 6790, 356, 2202, 8798, 28787, 298, 8798, 28770, 28725, 8970, 721, 282, 276, 28705, 28740, 28781, 28734, 18144, 28748, 28719, 28750, 15770, 28705, 28740, 20222, 325, 1466, 8798, 28750, 557, 304, 3102, 2187, 4139, 4306, 6752, 438, 28705, 28781, 28734, 28734, 334, 28777, 28724, 356, 2202, 8798, 28787, 298, 8798, 28784, 28723, 382, 377, 731, 1129, 745, 17854, 8894, 654, 4319, 3436, 356, 1370, 28705, 28734, 28725, 304, 334, 5728, 28782, 28705, 29475, 28770, 28750, 28748, 29475, 28770, 28750, 16732, 17854, 8894, 654, 4319, 3436, 356, 1370, 648, 28740, 28723, 420, 2869, 17063, 3434, 325, 28777, 28790, 28769, 28731, 8030, 430, 721, 2951, 8405, 4658, 28747, 2725, 372, 1082, 26720, 424, 15737, 24726, 325, 962, 28777, 28731, 28705, 28740, 28723, 28782, 18144, 28748, 8087, 356, 2202, 8798, 28782, 28725, 8798, 28770, 28725, 304, 8798, 28740, 28745, 586, 11917, 540, 19807, 290, 1009, 299, 309, 325, 4962, 28765, 28731, 28705, 28740, 319, 1712, 2421, 6790, 356, 1370, 8798, 28750, 1059, 1370, 648, 28750, 28783, 28745, 304, 261, 323, 1438, 321, 381, 477, 1370, 8798, 28750, 298, 1370, 28705, 28740, 28783, 28734, 1704, 28733, 1281, 18071, 28723, 13, 28789, 28721, 447, 9670, 560, 27013, 3471, 28725, 456, 12280, 349, 18063, 264, 10722, 1987, 345, 28716, 377, 731, 28733, 19056, 17854, 3601, 1203, 18071, 28739, 354, 264, 7749, 28723, 851, 349, 264, 1212, 302, 1203, 18071, 970, 17854, 8894, 477, 264, 19966, 16933, 949, 271, 325, 28716, 377, 731, 1129, 745, 28731, 304, 16732, 4242, 17854, 8894, 395, 264, 2948, 19869, 4548, 352, 325, 4020, 28754, 28782, 28705, 29475, 28770, 28750, 28748, 29475, 28770, 28750, 28731, 460, 1307, 28723, 13, 13, 1014, 7749, 907, 916, 1644, 274, 264, 345, 12058, 288, 983, 21538, 28739, 298, 10221, 652, 2187, 354, 272, 1203, 18071, 28723, 851, 14657, 11864, 264, 3518, 302, 10747, 304, 19308, 19724, 754, 2856, 2202, 298, 478, 2589, 652, 20930, 1587, 304, 1038, 2764, 354, 272, 633, 17854, 8894, 28723, 415, 10747, 1307, 3024, 972, 554, 283, 375, 473, 28725, 8970, 721, 282, 276, 28725, 304, 2725, 372, 1082, 26720, 424, 15737, 24726, 325, 962, 28777, 609, 415, 19308, 12238, 349, 1987, 3102, 2187, 4139, 4306, 6752, 28723, 13, 13, 9169, 272, 4644, 288, 983, 21538, 28725, 272, 7749, 21415, 272, 295, 377, 731, 1129, 745, 17854, 8894, 356, 1370, 28705, 28734, 28725, 4961, 486, 272, 334, 5728, 28782, 28705, 29475, 28770, 28750, 28748, 29475, 28770, 28750, 16732, 17854, 8894, 356, 1370, 648, 28740, 28723, 1791, 5297, 272, 633, 17854, 8894, 477, 23627, 272, 7749, 28742, 28713, 2187, 325, 28708, 4644, 1987, 319, 2869, 17063, 3434, 8030, 557, 272, 7749, 349, 2078, 4870, 28050, 28725, 2490, 586, 11917, 540, 19807, 290, 1009, 299, 309, 325, 4962, 28765, 28731, 304, 261, 323, 1438, 321, 381, 28725, 354, 264, 2948, 3216, 1024, 272, 1203, 18071, 28723, 13, 13, 13, 18066, 28747, 12018, 264, 2899, 7658, 298, 272, 7114, 28723], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# https://github.com/databrickslabs/dolly/blob/master/training/trainer.py\n",
        "def preprocess_dataset(tokenizer: AutoTokenizer, max_length: int, dataset: str, seed: int = 42):\n",
        "    # Format each prompt.\n",
        "    print(\"Preprocessing dataset...\")\n",
        "    dataset = dataset.map(format_prompt)\n",
        "\n",
        "\n",
        "    # https://blog.ovhcloud.com/fine-tuning-llama-2-models-using-a-single-gpu-qlora-and-ai-notebooks/\n",
        "    def preprocess_batch(batch, tokenizer, max_length):\n",
        "        return tokenizer(\n",
        "            batch[\"text\"],\n",
        "            max_length=max_length,\n",
        "            truncation=True,\n",
        "        )\n",
        "\n",
        "\n",
        "    # Apply preprocessing to each batch of the dataset & and remove \"conversations\" and \"text\" fields.\n",
        "    _preprocessing_function = partial(preprocess_batch, max_length=max_length, tokenizer=tokenizer)\n",
        "    dataset = dataset.map(\n",
        "        _preprocessing_function,\n",
        "        batched=True,\n",
        "        remove_columns=[\"conversations\", \"text\"],\n",
        "    )\n",
        "\n",
        "\n",
        "    # Filter out samples that have input_ids exceeding max_length.\n",
        "    # Not needed as the tokenizer truncates all prompts over max length.\n",
        "    # dataset = dataset.filter(lambda sample: len(sample[\"input_ids\"]) < max_length)\n",
        "\n",
        "\n",
        "    # Shuffle dataset.\n",
        "    dataset = dataset.shuffle(seed=seed)\n",
        "\n",
        "\n",
        "    return dataset"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:34.256334Z",
          "iopub.execute_input": "2023-11-22T02:16:34.258038Z",
          "iopub.status.idle": "2023-11-22T02:16:34.265202Z",
          "shell.execute_reply.started": "2023-11-22T02:16:34.258007Z",
          "shell.execute_reply": "2023-11-22T02:16:34.264265Z"
        },
        "trusted": true,
        "id": "0Jf-1c6h-ykg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "formatted_dataset = deepcopy(dataset).map(format_prompt)\n",
        "dataset = preprocess_dataset(tokenizer, max_length, dataset)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:34.266477Z",
          "iopub.execute_input": "2023-11-22T02:16:34.268710Z",
          "iopub.status.idle": "2023-11-22T02:16:40.972229Z",
          "shell.execute_reply.started": "2023-11-22T02:16:34.268676Z",
          "shell.execute_reply": "2023-11-22T02:16:40.971249Z"
        },
        "trusted": true,
        "colab": {
          "referenced_widgets": [
            "78447cb960464daab01869e21d5f7f32",
            "58b736aeed8b4f568c1ec70476e1c8c5"
          ]
        },
        "id": "4OhbUWie-ykg",
        "outputId": "710ac425-ea0c-4c4d-cb77-3bb1a4015bcf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Map:   0%|          | 0/3000 [00:00<?, ? examples/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "78447cb960464daab01869e21d5f7f32"
            }
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "text": "Preprocessing dataset...\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Map:   0%|          | 0/3000 [00:00<?, ? examples/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "58b736aeed8b4f568c1ec70476e1c8c5"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "run = wandb.init(\n",
        "    project=\"Fine tuning mistral 7B preprocessing\",  # Project name.\n",
        "    name=\"log_prep_dataset\",     # name of the run within this project.\n",
        "    config={                     # Configuration dictionary.\n",
        "        \"split\": \"train\"\n",
        "    },\n",
        "    group=\"dataset\",             # Group runs. This run belongs in \"dataset\".\n",
        "    tags=[\"dataset\"],            # Tags. More dynamic, low-level grouping.\n",
        "    notes=\"Logging preprocessed subset of Puffin dataset.\"  # Description about the run.\n",
        ")  # Check out the other parameters in the `wandb.init`!\n",
        "\n",
        "\n",
        "data = []\n",
        "for i in range(1000):  # Log 1000 instances.\n",
        "    x = formatted_dataset[i]\n",
        "    id_ = x[\"id\"]\n",
        "    conversation = x[\"text\"]\n",
        "    data.append([id_, conversation])\n",
        "\n",
        "\n",
        "table = wandb.Table(data=data, columns=[\"id\", \"value\"])\n",
        "run.log({\"first1000_prep_Puffin\": table})\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:16:40.973546Z",
          "iopub.execute_input": "2023-11-22T02:16:40.973806Z",
          "iopub.status.idle": "2023-11-22T02:17:12.042633Z",
          "shell.execute_reply.started": "2023-11-22T02:16:40.973783Z",
          "shell.execute_reply": "2023-11-22T02:17:12.041640Z"
        },
        "trusted": true,
        "id": "BeUuqoZz-ykh",
        "outputId": "6747706f-fbb1-4bbc-e79f-a72fd195c371"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "wandb version 0.16.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.15.9"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/kaggle/working/wandb/run-20231122_021641-1mkn6grh</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing/runs/1mkn6grh' target=\"_blank\">log_prep_dataset</a></strong> to <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing/runs/1mkn6grh' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing/runs/1mkn6grh</a>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset.save_to_disk(\"Puffin_prep.hf\")\n",
        "\n",
        "artifact = wandb.Artifact(name=\"Puffin_prep\", type=\"dataset\")\n",
        "artifact.add_dir(\"./Puffin_prep.hf\", name=\"train\")\n",
        "run.log_artifact(artifact)\n",
        "run.finish()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:17:12.044063Z",
          "iopub.execute_input": "2023-11-22T02:17:12.044464Z",
          "iopub.status.idle": "2023-11-22T02:17:42.122690Z",
          "shell.execute_reply.started": "2023-11-22T02:17:12.044428Z",
          "shell.execute_reply": "2023-11-22T02:17:42.121795Z"
        },
        "trusted": true,
        "colab": {
          "referenced_widgets": [
            "fa6f733b283a44219bce369df10bdc75"
          ]
        },
        "id": "szby1-Cu-ykh",
        "outputId": "8eeba1ab-e1a2-4896-d92f-fd6e96f61937"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "Saving the dataset (0/1 shards):   0%|          | 0/3000 [00:00<?, ? examples/s]",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fa6f733b283a44219bce369df10bdc75"
            }
          },
          "metadata": {}
        },
        {
          "name": "stderr",
          "text": "\u001b[34m\u001b[1mwandb\u001b[0m: Adding directory to artifact (./Puffin_prep.hf)... Done. 0.1s\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run <strong style=\"color:#cdcd00\">log_prep_dataset</strong> at: <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing/runs/1mkn6grh' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20preprocessing/runs/1mkn6grh</a><br/>Synced 6 W&B file(s), 1 media file(s), 4 artifact file(s) and 0 other file(s)"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Find logs at: <code>./wandb/run-20231122_021641-1mkn6grh/logs</code>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./outputs\",\n",
        "    per_device_train_batch_size=1,  # Best practice: https://huggingface.co/docs/transformers/main/main_classes/quantization#tips-and-best-practices\n",
        "    gradient_accumulation_steps=1,  # Powers of 2.\n",
        "    learning_rate=2e-4,\n",
        "    max_grad_norm=1.0,\n",
        "    max_steps=20,\n",
        "    lr_scheduler_type=\"linear\",\n",
        "    warmup_steps=5,\n",
        "    fp16=True,\n",
        "    logging_strategy=\"steps\",\n",
        "    logging_steps=1,\n",
        "    save_strategy=\"steps\",\n",
        "    save_steps=5,\n",
        "    optim=\"paged_adamw_8bit\",\n",
        "    report_to=\"wandb\"\n",
        ")\n",
        "\n",
        "# training_arguments = TrainingArguments(\n",
        "#     weight_decay=0.001,\n",
        "#     bf16=False,\n",
        "#     warmup_ratio=0.03,\n",
        "#     group_by_length=True,\n",
        "# )"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:17:42.131727Z",
          "iopub.execute_input": "2023-11-22T02:17:42.132070Z",
          "iopub.status.idle": "2023-11-22T02:17:42.139511Z",
          "shell.execute_reply.started": "2023-11-22T02:17:42.132045Z",
          "shell.execute_reply": "2023-11-22T02:17:42.138592Z"
        },
        "trusted": true,
        "id": "bpT1cynf-yki"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run = wandb.init(\n",
        "    project=\"Fine tuning mistral 7B Main\",\n",
        "    name=\"train_run0\",  # Sometimes I use the run name as short descriptor for the run.\n",
        "    config={\n",
        "        \"split\": \"train\",\n",
        "        # Optionally, you can add all hyperparameters and configs here for better reproducibility!\n",
        "    },\n",
        "    group=\"train\",\n",
        "    tags=[\"train\", \"AdamW\"],  # Add tags for what might characterize this run.\n",
        "    notes=\"Initial finetuning.\"\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:17:42.141128Z",
          "iopub.execute_input": "2023-11-22T02:17:42.141488Z",
          "iopub.status.idle": "2023-11-22T02:18:13.471146Z",
          "shell.execute_reply.started": "2023-11-22T02:17:42.141455Z",
          "shell.execute_reply": "2023-11-22T02:18:13.470274Z"
        },
        "trusted": true,
        "id": "YdvLCXCf-yki",
        "outputId": "1a84066a-7b71-4bbb-ce64-b1240771db4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "wandb version 0.16.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Tracking run with wandb version 0.15.9"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Run data is saved locally in <code>/kaggle/working/wandb/run-20231122_021742-bgxidy3o</code>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Syncing run <strong><a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main/runs/bgxidy3o' target=\"_blank\">train_run0</a></strong> to <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View project at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main</a>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run at <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main/runs/bgxidy3o' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main/runs/bgxidy3o</a>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
        "    train_dataset=dataset,\n",
        "#     dataset_text_field=dataset[\"text\"]\n",
        "#     packing=True\n",
        ")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:18:13.472709Z",
          "iopub.execute_input": "2023-11-22T02:18:13.473057Z",
          "iopub.status.idle": "2023-11-22T02:18:13.488329Z",
          "shell.execute_reply.started": "2023-11-22T02:18:13.473025Z",
          "shell.execute_reply": "2023-11-22T02:18:13.487103Z"
        },
        "trusted": true,
        "id": "KJQrSShX-yki"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "results = trainer.train()  # Now we just run train()!\n",
        "run.finish()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2023-11-22T02:19:07.855638Z",
          "iopub.execute_input": "2023-11-22T02:19:07.856405Z",
          "iopub.status.idle": "2023-11-22T02:21:57.068362Z",
          "shell.execute_reply.started": "2023-11-22T02:19:07.856370Z",
          "shell.execute_reply": "2023-11-22T02:21:57.067466Z"
        },
        "trusted": true,
        "colab": {
          "referenced_widgets": [
            "d422638bb4ee427e923278f6dc28dd34"
          ]
        },
        "id": "xP__rLyt-ykj",
        "outputId": "256c7e7b-1cf6-40ae-e0bc-cb2fff040f6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "\n    <div>\n      \n      <progress value='20' max='20' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [20/20 02:27, Epoch 0/1]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Step</th>\n      <th>Training Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>1.326600</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>1.021800</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>1.941500</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>1.063900</td>\n    </tr>\n    <tr>\n      <td>5</td>\n      <td>1.066000</td>\n    </tr>\n    <tr>\n      <td>6</td>\n      <td>0.735800</td>\n    </tr>\n    <tr>\n      <td>7</td>\n      <td>0.776600</td>\n    </tr>\n    <tr>\n      <td>8</td>\n      <td>1.206500</td>\n    </tr>\n    <tr>\n      <td>9</td>\n      <td>1.164100</td>\n    </tr>\n    <tr>\n      <td>10</td>\n      <td>0.509800</td>\n    </tr>\n    <tr>\n      <td>11</td>\n      <td>0.691200</td>\n    </tr>\n    <tr>\n      <td>12</td>\n      <td>1.156900</td>\n    </tr>\n    <tr>\n      <td>13</td>\n      <td>0.775100</td>\n    </tr>\n    <tr>\n      <td>14</td>\n      <td>0.677700</td>\n    </tr>\n    <tr>\n      <td>15</td>\n      <td>0.729800</td>\n    </tr>\n    <tr>\n      <td>16</td>\n      <td>0.619100</td>\n    </tr>\n    <tr>\n      <td>17</td>\n      <td>0.346800</td>\n    </tr>\n    <tr>\n      <td>18</td>\n      <td>0.585800</td>\n    </tr>\n    <tr>\n      <td>19</td>\n      <td>0.720600</td>\n    </tr>\n    <tr>\n      <td>20</td>\n      <td>0.893300</td>\n    </tr>\n  </tbody>\n</table><p>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "VBox(children=(Label(value='0.001 MB of 0.026 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=0.050795…",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d422638bb4ee427e923278f6dc28dd34"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "<style>\n    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n    </style>\n<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/epoch</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁███████</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▄▅▇██▇▇▆▆▅▅▄▄▃▃▂▂▁▁</td></tr><tr><td>train/loss</td><td>▅▄█▄▄▃▃▅▅▂▃▅▃▂▃▂▁▂▃▃</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/epoch</td><td>0.01</td></tr><tr><td>train/global_step</td><td>20</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.8933</td></tr><tr><td>train/total_flos</td><td>1004233574375424.0</td></tr><tr><td>train/train_loss</td><td>0.90044</td></tr><tr><td>train/train_runtime</td><td>165.4855</td></tr><tr><td>train/train_samples_per_second</td><td>0.121</td></tr><tr><td>train/train_steps_per_second</td><td>0.121</td></tr></table><br/></div></div>"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": " View run <strong style=\"color:#cdcd00\">train_run0</strong> at: <a href='https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main/runs/bgxidy3o' target=\"_blank\">https://wandb.ai/sut-ee/Fine%20tuning%20mistral%207B%20Main/runs/bgxidy3o</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<IPython.core.display.HTML object>",
            "text/html": "Find logs at: <code>./wandb/run-20231122_021742-bgxidy3o/logs</code>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Inference Using Mistral 7B"
      ],
      "metadata": {
        "id": "l_vZhBjl-ykj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = \"/kaggle/input/mistral/pytorch/7b-v0.1-hf/1\"\n",
        "\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_compute_dtype=torch.bfloat16,\n",
        ")\n",
        "\n",
        "\n",
        "# You can just use model.\n",
        "inf_model = AutoModelForCausalLM.from_pretrained(\n",
        "    model_name,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map=\"auto\",\n",
        ")\n"
      ],
      "metadata": {
        "trusted": true,
        "id": "_6bKPmXL-ykj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "run = wandb.init(project=\"Fine tuning mistral 7B Main\")  # MAKE SURE TO PASS IN YOUR PROJECT NAME!\n",
        "artifact = run.use_artifact('vincenttu/finetuning_mistral7b/model-t6rw0dav:v0', type='model')\n",
        "artifact_dir = artifact.download()\n",
        "run.finish()"
      ],
      "metadata": {
        "id": "-YcmmNNj-ykk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = PeftModel.from_pretrained(inf_model, \"/content/artifacts/model-t6rw0dav:v0\")"
      ],
      "metadata": {
        "id": "4o19fixk-ykk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"What is a neural network??\"\n",
        "\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "model_input = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
        "\n",
        "\n",
        "_ = model.eval()\n",
        "with torch.no_grad():\n",
        "    out = model.generate(**model_input, max_new_tokens=100)\n",
        "\n",
        "\n",
        "print(tokenizer.decode(out[0], skip_special_tokens=True))"
      ],
      "metadata": {
        "id": "R4_jO9K4-ykk"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}