notebooks/counterfactual_analysis_imdb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "522875dc-8d8e-4253-b010-5ae8e71062b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import evaluate\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import textdistance\n",
    "import torch\n",
    "from nltk.tokenize import regexp_tokenize\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ab6cbc0e-cd76-435d-90c3-4771e644c307",
   "metadata": {},
   "outputs": [],
   "source": [
    "perplexity = evaluate.load(\"perplexity\", module_type=\"metric\")\n",
    "sacrebleu = evaluate.load(\"sacrebleu\")\n",
    "t5_tokenizer = AutoTokenizer.from_pretrained('t5-small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cbb851f6-3b7e-474d-96bc-23d3a2cd5b6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"mtreviso/roberta-base-imdb\"\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
    "model = model.cuda()\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bf23d4f-cd6b-4656-9364-81e02551152e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, tokenizer, inp):\n",
    "    enc = tokenizer(\n",
    "        inp, \n",
    "        padding=True,\n",
    "        max_length=512,\n",
    "        truncation=True,\n",
    "        return_tensors=\"pt\",\n",
    "    )\n",
    "    out = model(\n",
    "        input_ids=enc.input_ids.cuda(), \n",
    "        attention_mask=enc.attention_mask.cuda()\n",
    "    )\n",
    "    return out.logits.argmax(-1).item()\n",
    "\n",
    "\n",
    "def get_predictions(model, tokenizer, inputs, verbose=True):\n",
    "    model.eval()\n",
    "    outs = []\n",
    "    with torch.no_grad():\n",
    "        gen = tqdm(inputs) if verbose else inputs\n",
    "        for inp in gen:\n",
    "            outs.append(predict(model, tokenizer, inp))\n",
    "    return outs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "99976827-a4ee-4b6f-a145-3cc004544bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def trim(text):\n",
    "    text = text.replace('<', ' <').replace('>', '> ')\n",
    "    # text = text.replace('\"\"', ' \"').replace(\"''\", \"'\")\n",
    "    text = text.replace('<unk> br', '<br')\n",
    "    text = re.sub(r'( </s>)+', ' </s>', text)\n",
    "    text = text.replace('</s> </s>', '</s>')\n",
    "    text = text.replace(\"br />\", \"<br />\").replace(\"<<\", \"<\")\n",
    "    text = text.replace(\"<br />\", \"\")\n",
    "    text = re.sub(r'</s>[\\S\\ ]+', '</s>', text)\n",
    "    text = text.replace('</s>', '')\n",
    "    text = text.replace('<unk>', '')\n",
    "    text = re.sub(r'\\ +', ' ', text).strip()\n",
    "    return text\n",
    "\n",
    "\n",
    "def decode_and_trim(text):\n",
    "    text = t5_tokenizer.decode(\n",
    "        t5_tokenizer.convert_tokens_to_ids(text.strip().split()),\n",
    "        skip_special_tokens=True, \n",
    "        clean_up_tokenization_spaces=True\n",
    "    )\n",
    "    text = trim(text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def read_edits(fname, valids_only=False, return_refs=False):\n",
    "    df = pd.read_csv(\n",
    "        fname, \n",
    "        sep='\\t', \n",
    "        usecols=['orig_texts', 'orig_labels', 'orig_predictions', 'orig_z', \n",
    "                 'edits_texts', 'edits_labels', 'edits_predictions', 'edits_z_pre', 'edits_z_pos']\n",
    "    )\n",
    "    # df = df[df['orig_labels'] != 'neutral']\n",
    "    if valids_only:\n",
    "        df = df[df['edits_labels'] == df['edits_predictions']]\n",
    "    edits = df['edits_texts'].map(decode_and_trim).tolist()\n",
    "    refs = df['orig_texts'].map(decode_and_trim).tolist()\n",
    "    try:\n",
    "        edits_labels = df['edits_labels'].map(int).tolist()\n",
    "        edits_preds = df['edits_predictions'].map(int).tolist()\n",
    "        refs_labels = df['orig_labels'].map(int).tolist()\n",
    "        refs_preds = df['orig_predictions'].map(int).tolist()\n",
    "    except:\n",
    "        label_map = {'Negative': 0, 'Positive':1}\n",
    "        edits_labels = df['edits_labels'].apply(label_map.__getitem__).tolist()\n",
    "        edits_preds = df['edits_predictions'].apply(label_map.__getitem__).tolist()\n",
    "        refs_labels = df['orig_labels'].apply(label_map.__getitem__).tolist()\n",
    "        refs_preds = df['orig_predictions'].apply(label_map.__getitem__).tolist()\n",
    "    return edits, edits_labels, edits_preds, refs, refs_labels, refs_preds\n",
    "\n",
    "\n",
    "def read_edits_mice(fname, use_last_search_step=False, valids_only=False):\n",
    "    \n",
    "    global dataset_ff_tmp, dataset_cf_tmp\n",
    "    \n",
    "    def get_mice_counterfactuals(df_mice):\n",
    "        # \"\"\" MiCE writes all edits that are found in Stage 2, \n",
    "        # but we only want to evaluate the smallest per input. \n",
    "        df_test = df_mice[df_mice['sorted_idx'] == 0]\n",
    "        # reset index\n",
    "        df_test = df_test.reset_index(drop=True)\n",
    "        df_test = df_test.sort_values(by='data_idx')\n",
    "        # sort by minimality:\n",
    "        # df_test = df_mice.sort_values(by='minimality', ascending=False)\n",
    "        # df_test = df_test.groupby('data_idx').last().reset_index(drop=True)\n",
    "        return df_test.reset_index(drop=True)\n",
    "\n",
    "    def get_mice_counterfactuals_max(df_mice):\n",
    "        # \"\"\" MiCE writes all edits that are found in Stage 2, \n",
    "        # but we only want to evaluate the longest per input. \n",
    "        df_test = df_mice.groupby('data_idx').last()\n",
    "        # reset index\n",
    "        df_test = df_test.reset_index(drop=True)\n",
    "        \n",
    "        # sort by minimality:\n",
    "        # df_test = df_mice.sort_values(by='minimality', ascending=True)\n",
    "        # df_test = df_test.groupby('data_idx').last().reset_index(drop=True)\n",
    "        return df_test\n",
    "    \n",
    "    try:\n",
    "        df_mice = pd.read_csv(fname, delimiter='\\t')\n",
    "    except:\n",
    "        df_mice = pd.read_csv(fname, delimiter='\\t', lineterminator='\\n')\n",
    "    \n",
    "    if not use_last_search_step:\n",
    "        df_mice_test = get_mice_counterfactuals(df_mice)\n",
    "    else:\n",
    "        df_mice_test = get_mice_counterfactuals_max(df_mice)\n",
    "    \n",
    "    valid_rows = ~df_mice_test['new_pred'].isna()\n",
    "    df_mice_test = df_mice_test[valid_rows].reset_index(drop=True)\n",
    "    \n",
    "    if valids_only:\n",
    "        df_mice_test = df_mice_test[df_mice_test['contrast_label'] == df_mice_test['new_pred']]\n",
    "    \n",
    "    refs = df_mice_test['orig_input'].map(trim).tolist()\n",
    "    refs_labels = df_mice_test['gold_label'].apply(int).tolist()\n",
    "    refs_preds = df_mice_test['orig_pred'].apply(int).tolist()\n",
    "    \n",
    "    edits = df_mice_test['edited_input'].map(trim).tolist()\n",
    "    edits_labels = df_mice_test['contrast_label'].apply(int).tolist()\n",
    "    edits_preds = df_mice_test['new_pred'].apply(int).tolist()\n",
    "    \n",
    "    return edits, edits_labels, edits_preds, refs, refs_labels, refs_preds\n",
    "\n",
    "\n",
    "def get_tokenized_texts(texts):\n",
    "    return [' '.join(regexp_tokenize(text, '\\w+|\\$[\\d\\.]+|\\S+')) for text in texts]\n",
    "\n",
    "\n",
    "def dist_ratio(es, rs):\n",
    "    return np.mean([\n",
    "        textdistance.levenshtein.normalized_distance(e.split(), r.split())\n",
    "        for e, r in zip(es, rs)\n",
    "    ])\n",
    "\n",
    "\n",
    "def clean(text):\n",
    "    text = text.replace('</s>', '')\n",
    "    text = text.replace('[SEP]', '')\n",
    "    text = re.sub(r'\\ +', ' ', text).strip()\n",
    "    return text\n",
    "\n",
    "\n",
    "def print_eval(filename, valids_only=False, use_last_search_step=False):\n",
    "    if 'mice' in filename:\n",
    "        edits, edits_labels, edits_preds, refs, refs_labels, refs_preds = read_edits_mice(\n",
    "            filename, use_last_search_step=use_last_search_step, valids_only=valids_only\n",
    "        )\n",
    "    else:\n",
    "        edits, edits_labels, edits_preds, refs, refs_labels, refs_preds = read_edits(\n",
    "            filename, valids_only=valids_only\n",
    "        )\n",
    "    \n",
    "    acc = accuracy_score(refs_labels, refs_preds)\n",
    "    f1 = f1_score(refs_labels, refs_preds, average='macro')\n",
    "    print('Ref Rat. Acc: {:.4f}'.format(acc))\n",
    "    print('Ref Rat. F1: {:.4f}'.format(f1))\n",
    "    \n",
    "    acc = accuracy_score(edits_labels, edits_preds)\n",
    "    f1 = f1_score(edits_labels, edits_preds, average='macro')\n",
    "    print('Edit Rat. Acc: {:.4f}'.format(acc))\n",
    "    print('Edit Rat. F1: {:.4f}'.format(f1))\n",
    "    \n",
    "    print('---')\n",
    "    \n",
    "    y_pred = list(get_predictions(model, tokenizer, refs, verbose=False))\n",
    "    acc = accuracy_score(refs_labels, y_pred)\n",
    "    f1 = f1_score(refs_labels, y_pred, average='macro')\n",
    "    print('Ref Valid. Acc: {:.4f}'.format(acc))\n",
    "    print('Ref Valid. F1: {:.4f}'.format(f1))\n",
    "    \n",
    "    y_pred = list(get_predictions(model, tokenizer, edits, verbose=False))\n",
    "    acc = accuracy_score(edits_labels, y_pred)\n",
    "    f1 = f1_score(edits_labels, y_pred, average='macro')\n",
    "    print('Edit Valid. Acc: {:.4f}'.format(acc))\n",
    "    print('Edit Valid. F1: {:.4f}'.format(f1))\n",
    "    \n",
    "    print('---')\n",
    "    \n",
    "    edits = list(map(clean, edits))\n",
    "    refs = list(map(clean, refs))\n",
    "    \n",
    "    res = dist_ratio(get_tokenized_texts(edits), get_tokenized_texts(refs))\n",
    "    print('Levensh. dist: {:.2f}'.format(res))\n",
    "    res = np.mean(list(map(lambda x: len(x.split()), get_tokenized_texts(refs))))\n",
    "    print('Num. tokens ref: {:.1f}'.format(res))\n",
    "    res = np.mean(list(map(lambda x: len(x.split()), get_tokenized_texts(edits))))\n",
    "    print('Num. tokens edit: {:.1f}'.format(res))\n",
    "    \n",
    "    print('---')\n",
    "    \n",
    "    res = sacrebleu.compute(predictions=edits, references=refs)\n",
    "    print('Self-bleu: {:.2f}'.format(res['score']))\n",
    "    \n",
    "    print('---')\n",
    "    \n",
    "    res = perplexity.compute(predictions=refs + edits, model_id='gpt2-large')\n",
    "    perp_refs = res['perplexities'][:len(refs)]\n",
    "    perp_edits = res['perplexities'][len(refs):] \n",
    "    print('Ref Perpl: {:.2f}'.format(np.mean(perp_refs)))\n",
    "    print('Edit Perpl: {:.2f}'.format(np.mean(perp_edits)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6e7c5e21-b284-4dad-8e53-a00d280914ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Rat. Acc: 0.9734\n",
      "Ref Rat. F1: 0.9734\n",
      "Edit Rat. Acc: 0.8115\n",
      "Edit Rat. F1: 0.8115\n",
      "---\n",
      "Ref Valid. Acc: 0.9754\n",
      "Ref Valid. F1: 0.9754\n",
      "Edit Valid. Acc: 0.7582\n",
      "Edit Valid. F1: 0.7582\n",
      "---\n",
      "Levensh. dist: 0.33\n",
      "Num. tokens ref: 182.9\n",
      "Num. tokens edit: 180.9\n",
      "---\n",
      "Self-bleu: 57.58\n",
      "---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3611922fa04324a7f01f5c4ea57117",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/61 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Perpl: 68.20\n",
      "Edit Perpl: 67.29\n"
     ]
    }
   ],
   "source": [
    "print_eval(f'../data/edits_paper/imdb/crest_30p.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "648ce723-8dac-4b9b-a638-09103e066e7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Rat. Acc: 0.9754\n",
      "Ref Rat. F1: 0.9754\n",
      "Edit Rat. Acc: 0.9324\n",
      "Edit Rat. F1: 0.9324\n",
      "---\n",
      "Ref Valid. Acc: 0.9754\n",
      "Ref Valid. F1: 0.9754\n",
      "Edit Valid. Acc: 0.9365\n",
      "Edit Valid. F1: 0.9365\n",
      "---\n",
      "Levensh. dist: 0.67\n",
      "Num. tokens ref: 182.9\n",
      "Num. tokens edit: 193.9\n",
      "---\n",
      "Self-bleu: 23.08\n",
      "---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcb22ff3543e486b8ec18bd3e5c33752",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/61 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Perpl: 68.20\n",
      "Edit Perpl: 50.68\n"
     ]
    }
   ],
   "source": [
    "print_eval(f'../data/edits_paper/imdb/crest_50p.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e1ca7e3e-8f04-4f10-a1a9-8edd6040074c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Rat. Acc: 0.5102\n",
      "Ref Rat. F1: 0.5101\n",
      "Edit Rat. Acc: 0.7520\n",
      "Edit Rat. F1: 0.7519\n",
      "---\n",
      "Ref Valid. Acc: 0.5102\n",
      "Ref Valid. F1: 0.5102\n",
      "Edit Valid. Acc: 0.7213\n",
      "Edit Valid. F1: 0.7209\n",
      "---\n",
      "Levensh. dist: 0.20\n",
      "Num. tokens ref: 183.0\n",
      "Num. tokens edit: 171.3\n",
      "---\n",
      "Self-bleu: 73.76\n",
      "---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e33e5de7aa14d1781c6c7d0aab2701c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/61 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Perpl: 67.82\n",
      "Edit Perpl: 76.72\n"
     ]
    }
   ],
   "source": [
    "print_eval(f'../data/edits_paper/imdb/mice_binary_search.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "da0bc575-abac-4a53-ac67-d5fa001033a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Rat. Acc: 0.5092\n",
      "Ref Rat. F1: 0.5092\n",
      "Edit Rat. Acc: 0.7659\n",
      "Edit Rat. F1: 0.7659\n",
      "---\n",
      "Ref Valid. Acc: 0.5092\n",
      "Ref Valid. F1: 0.5092\n",
      "Edit Valid. Acc: 0.7680\n",
      "Edit Valid. F1: 0.7680\n",
      "---\n",
      "Levensh. dist: 0.39\n",
      "Num. tokens ref: 182.7\n",
      "Num. tokens edit: 161.2\n",
      "---\n",
      "Self-bleu: 49.64\n",
      "---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9520e3e435ae4d24b4ee6cdc16222c13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/61 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Perpl: 67.93\n",
      "Edit Perpl: 79.32\n"
     ]
    }
   ],
   "source": [
    "print_eval(f'../data/edits_paper/imdb/mice_30p.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ca8b8d20-c5c6-4255-a156-35a0f7589a65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Rat. Acc: 0.5102\n",
      "Ref Rat. F1: 0.5101\n",
      "Edit Rat. Acc: 0.8484\n",
      "Edit Rat. F1: 0.8481\n",
      "---\n",
      "Ref Valid. Acc: 0.5102\n",
      "Ref Valid. F1: 0.5102\n",
      "Edit Valid. Acc: 0.8320\n",
      "Edit Valid. F1: 0.8320\n",
      "---\n",
      "Levensh. dist: 0.65\n",
      "Num. tokens ref: 183.0\n",
      "Num. tokens edit: 115.7\n",
      "---\n",
      "Self-bleu: 20.70\n",
      "---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b73c49cdfd8141f7943ee61841ba34ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/61 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ref Perpl: 67.82\n",
      "Edit Perpl: 89.92\n"
     ]
    }
   ],
   "source": [
    "print_eval(f'../data/edits_paper/imdb/mice_50p.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5f743de-d641-40fb-87e6-a1f3ee72a7f3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}