finetune_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "048bb70f-3751-4156-be43-d1e897d98d03",
   "metadata": {},
   "source": [
    "# -> Processing the data ->"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f3ff02ad-64ac-4e77-995d-a17b8977882e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "801d4d2d-675c-4be8-bf8f-c1950cf09f70",
   "metadata": {},
   "source": [
    "## Example retrain"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdd13da4-6db7-44b1-9599-40d33447447f",
   "metadata": {},
   "source": [
    "(two new instances, one little backprop step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f585de3-7cdb-4242-9c08-37db77e20856",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "# Same as in 'using_transformers.ipynb'\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "sequences = [\n",
    "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
    "    \"This course is amazing!\",\n",
    "]\n",
    "batch = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "batch[\"labels\"] = torch.tensor([1, 1])\n",
    "\n",
    "optimizer = AdamW(model.parameters())\n",
    "loss = model(**batch).loss\n",
    "loss.backward()\n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ca09f8ae-8420-42fe-8fda-bbe848298420",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
      "          2607,  2026,  2878,  2166,  1012,   102],\n",
      "        [  101,  2023,  2607,  2003,  6429,   999,   102,     0,     0,     0,\n",
      "             0,     0,     0,     0,     0,     0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'labels': tensor([1, 1])}\n"
     ]
    }
   ],
   "source": [
    "print(batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5107f0c6-7797-44c2-87a7-75d8824b4a39",
   "metadata": {},
   "source": [
    "## Loading a dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ca79486-b877-4620-b38c-fc7d3d1f2d70",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "raw_datasets = load_dataset(\"glue\", \"mrpc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f3589d6f-eefb-4e94-a26a-18d9ec1b1c1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 3668\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 408\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 1725\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8ca4e2ba-7bd8-48f4-9060-06020741b89b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': Value(dtype='string', id=None),\n",
       " 'sentence2': Value(dtype='string', id=None),\n",
       " 'label': ClassLabel(names=['not_equivalent', 'equivalent'], id=None),\n",
       " 'idx': Value(dtype='int32', id=None)}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets[\"train\"].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0c550d37-d14a-40e2-94f2-87d29a42d478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': 'Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence .',\n",
       " 'sentence2': 'Referring to him as only \" the witness \" , Amrozi accused his brother of deliberately distorting his evidence .',\n",
       " 'label': 1,\n",
       " 'idx': 0}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets[\"train\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f7f61a9-4040-4fd3-9a12-04ad1d2ea941",
   "metadata": {},
   "source": [
    "## dealing with sentence pairs as input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1c162104-7b35-44da-a6b4-ae2b3c840877",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8b84489-ed14-4b88-8da5-3693085ce25f",
   "metadata": {},
   "source": [
    "this will tokenize the two sentences as a pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b0ee1ea1-be8c-4ef4-ae80-5abdb40d1def",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2023, 2003, 1996, 2034, 6251, 1012, 102, 2023, 2003, 1996, 2117, 2028, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tokenizer(\"This is the first sentence.\", \"This is the second one.\")\n",
    "inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d37b1e-9d14-4db3-9bb3-c95973f3af10",
   "metadata": {},
   "source": [
    "✏️ Try it out! Take element 15 of the training set and tokenize the two sentences separately and as a pair. What’s the difference between the two results?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b13d449f-78b7-4697-b81d-540c7c0f4613",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] rudder was most recently senior vice president for the developer & platform evangelism business. [SEP]\n",
      "[CLS] senior vice president eric rudder, formerly head of the developer and platform evangelism unit, will lead the new entity. [SEP]\n",
      "[CLS] rudder was most recently senior vice president for the developer & platform evangelism business. [SEP] senior vice president eric rudder, formerly head of the developer and platform evangelism unit, will lead the new entity. [SEP]\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(tokenizer(raw_datasets[\"train\"][15][\"sentence1\"])[\"input_ids\"]))\n",
    "print(tokenizer.decode(tokenizer(raw_datasets[\"train\"][15][\"sentence2\"])[\"input_ids\"]))\n",
    "print(tokenizer.decode(tokenizer(raw_datasets[\"train\"][15][\"sentence1\"], raw_datasets[\"train\"][15][\"sentence2\"])[\"input_ids\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6a37806-1e90-49ba-b4bf-cb58bad43279",
   "metadata": {},
   "source": [
    "this will deal with all examples as pairs, BUT returned in memory as a dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fecf8061-6232-4afa-8346-a475eb7176b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])\n",
      "torch.Size([3668, 103])\n",
      "torch.Size([3668, 103])\n",
      "torch.Size([3668, 103])\n"
     ]
    }
   ],
   "source": [
    "tokenized_dataset = tokenizer(\n",
    "    raw_datasets[\"train\"][\"sentence1\"],\n",
    "    raw_datasets[\"train\"][\"sentence2\"],\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\"\n",
    ")\n",
    "print(tokenized_dataset.keys())\n",
    "print(tokenized_dataset[\"input_ids\"].shape)\n",
    "print(tokenized_dataset[\"token_type_ids\"].shape)\n",
    "print(tokenized_dataset[\"attention_mask\"].shape)\n",
    "\n",
    "# model(**tokenized_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4669a18-818e-4a18-a426-bd6fe83ae3c3",
   "metadata": {},
   "source": [
    "this will deal with all examples as pairs, but kept as a dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "1aff20ba-8dbe-473b-844a-7dc366eccbe3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-ff78205d0d3e4be8.arrow\n",
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-4c15cdedd2965be7.arrow\n",
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9835ccadb0ead017.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 3668\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 408\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 1725\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize_function(example):\n",
    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97e4d378-aa79-4b93-a248-4983634d7bd2",
   "metadata": {},
   "source": [
    "notice the keys input_ids, token_type_ids, and attention_mask have all been added now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "99116717-e809-4b87-aa0b-3edb8fb5d84f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': Value(dtype='string', id=None),\n",
       " 'sentence2': Value(dtype='string', id=None),\n",
       " 'label': ClassLabel(names=['not_equivalent', 'equivalent'], id=None),\n",
       " 'idx': Value(dtype='int32', id=None),\n",
       " 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None),\n",
       " 'token_type_ids': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None),\n",
       " 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets[\"train\"].features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1be64234-acc1-40ca-bae2-03c30b58fb6c",
   "metadata": {},
   "source": [
    "but samples are variable length still, e.g."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "4f6aa41f-aa34-4afa-93ca-ef6274bba3c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[50, 59, 47, 67, 59, 50, 62, 32]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples = tokenized_datasets[\"train\"][:8]\n",
    "samples = {k: v for k, v in samples.items() if k not in [\"idx\", \"sentence1\", \"sentence2\", \"label\"]}\n",
    "[len(sample) for sample in samples[\"input_ids\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9403510c-bbd2-4a43-b1a0-de5867bcb1bd",
   "metadata": {},
   "source": [
    "## dynamic padding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c7e9840-da94-415e-a324-f08dc22ba812",
   "metadata": {},
   "source": [
    "(padding only when creating each batch. recommended unless you're using TPU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "642911d6-0a56-4038-992e-a7223cc9d07c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': torch.Size([8, 67]),\n",
       " 'token_type_ids': torch.Size([8, 67]),\n",
       " 'attention_mask': torch.Size([8, 67])}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "\n",
    "batch = data_collator(samples)\n",
    "{k: v.shape for k, v in batch.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98324a86-f3e5-49f9-85ea-524730288f65",
   "metadata": {},
   "source": [
    "# -> Fine-tuning a model with the Trainer API ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bb626a4-e603-4760-892a-c4f084cd49d7",
   "metadata": {},
   "source": [
    "## Summary of what we know so far"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4254680-8ced-4d1b-b47b-8bf3949627a7",
   "metadata": {},
   "source": [
    "need to load dataset, choose checkpoint, load tokenizer, model, datacollator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "e51e4a2e-1f5a-469b-8a82-61319d361599",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset glue (/Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
      "100%|████████████████████████████████████████████| 3/3 [00:00<00:00, 558.35it/s]\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, DataCollatorWithPadding\n",
    "\n",
    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "# since the below model wasn't pretrained with 2 labels, setting num_labels=2 means its head will be discarded\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
    "collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e13f3b23-6105-47ff-a81d-a3c3ae992634",
   "metadata": {},
   "source": [
    "define tokenizer function and map it to all examples in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "15cc98b2-6555-4088-9bdc-69974a1a0a04",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-062fa6f7d2bb6c2b.arrow\n",
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-3f53a58402d3075c.arrow\n",
      "Loading cached processed dataset at /Users/davidg/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-2956d94d6a022eae.arrow\n"
     ]
    }
   ],
   "source": [
    "def tokenize(example):\n",
    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(tokenize, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3363a34-ea95-4d33-bd51-9d49a5dd30d2",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3030b7f1-1047-4514-b568-9c143ec89273",
   "metadata": {},
   "source": [
    "need to pass PreTrainedModel and TrainingArguments objects to a Trainer object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "3520f54b-563a-4a99-8c2f-e840b83a02bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The following columns in the training set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2. If sentence1, idx, sentence2 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running training *****\n",
      "  Num examples = 400\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 50\n",
      "  Number of trainable parameters = 109483778\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='50' max='50' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [50/50 01:44, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=50, training_loss=0.37279773712158204, metrics={'train_runtime': 107.0255, 'train_samples_per_second': 3.737, 'train_steps_per_second': 0.467, 'total_flos': 14491663596000.0, 'train_loss': 0.37279773712158204, 'epoch': 1.0})"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "# we'll just use 400 training examples and 1 epoch since we don't have gpu\n",
    "training_args = TrainingArguments(\"test-trainer\", report_to=\"all\", optim=\"adamw_torch\", num_train_epochs=1)\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"].select(range(400)),\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=collator,\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edadf5b3-4a41-4b0a-9309-0f650cacd01d",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "114c58cf-8c75-47df-b532-4af8256d71ce",
   "metadata": {},
   "source": [
    "run inferences on validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "e6112891-1353-4e86-91bb-ba8c8e9dbefc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2. If sentence1, idx, sentence2 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 408\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(408, 2) (408,)\n"
     ]
    }
   ],
   "source": [
    "predictions = trainer.predict(tokenized_datasets[\"validation\"])\n",
    "print(predictions.predictions.shape, predictions.label_ids.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c097c07-ee00-4861-a537-24a7a2b80f85",
   "metadata": {},
   "source": [
    "convert to predicted labels and compute metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "63bf9e23-d933-4bb3-8df4-55dfb543fc9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.7377450980392157, 'f1': 0.821963394342762}"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "pred_labels = np.argmax(predictions.predictions, axis=-1)\n",
    "\n",
    "metric = evaluate.load(\"glue\", \"mrpc\")\n",
    "metric.compute(predictions=pred_labels, references=predictions.label_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "170aa4cf-eb86-4279-bbb6-69c4c6842e51",
   "metadata": {},
   "source": [
    "wrap together and include during training\n",
    "\n",
    "have to instantiate model object again or you're just continuing training from before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "bc6961b9-572a-44fa-a1b6-601c94ab5cf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "loading configuration file config.json from cache at /Users/davidg/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"bert-base-uncased\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.25.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file pytorch_model.bin from cache at /Users/davidg/.cache/huggingface/hub/models--bert-base-uncased/snapshots/0a6aa9128b6194f4f3c4db429b6cb4891cdb421b/pytorch_model.bin\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "The following columns in the training set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2. If sentence1, idx, sentence2 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running training *****\n",
      "  Num examples = 400\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 50\n",
      "  Number of trainable parameters = 109483778\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='50' max='50' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [50/50 02:06, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.593635</td>\n",
       "      <td>0.713235</td>\n",
       "      <td>0.824060</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2. If sentence1, idx, sentence2 are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 8\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=50, training_loss=0.5956118011474609, metrics={'train_runtime': 130.1899, 'train_samples_per_second': 3.072, 'train_steps_per_second': 0.384, 'total_flos': 14491663596000.0, 'train_loss': 0.5956118011474609, 'epoch': 1.0})"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compute_metrics(eval_preds):\n",
    "    metric = evaluate.load(\"glue\", \"mrpc\")\n",
    "    logits, labels = eval_preds\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)\n",
    "\n",
    "training_args = TrainingArguments(\"test-trainer\", report_to=\"all\", optim=\"adamw_torch\", num_train_epochs=1, evaluation_strategy=\"epoch\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"].select(range(400)),\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afd35509-b18a-4090-a0d9-f20e03e41f9d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface",
   "language": "python",
   "name": "huggingface"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}