tawa_ai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OpenAI Chatbot with TAWA data (v2)\n",
    "\n",
    "This TAWA output is from TAR 295, which modelled several single tier FTC changes, one of which was chosen for Budget 22."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "from aipy.ai import *\n",
    "import pandas as pd\n",
    "import yaml\n",
    "import jinja2\n",
    "import tiktoken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Available models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = openai.Model.list().data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPT-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gpt-4-1106-preview',\n",
       " 'gpt-4-vision-preview',\n",
       " 'gpt-4',\n",
       " 'gpt-4-0314',\n",
       " 'gpt-4-0613']"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m['id'] for m in models if m['id'].startswith('gpt-4')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPT-3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gpt-3.5-turbo-instruct',\n",
       " 'gpt-3.5-turbo-instruct-0914',\n",
       " 'gpt-3.5-turbo-0613',\n",
       " 'gpt-3.5-turbo-0301',\n",
       " 'gpt-3.5-turbo',\n",
       " 'gpt-3.5-turbo-16k-0613',\n",
       " 'gpt-3.5-turbo-1106',\n",
       " 'gpt-3.5-turbo-16k']"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m['id'] for m in models if m['id'].startswith('gpt-3.5')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tawa_details contains the tax year and the reform scenario descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tawa_details.yaml') as f:\n",
    "    tawa_details = yaml.load(f, Loader=yaml.FullLoader)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "tawa = pd.read_csv(\"input/tawa.csv\")\n",
    "\n",
    "tawa.drop(columns=['Name', 'Rounding_Rule', 'Population', 'Margin_Of_Error'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "fiscals = tawa[\n",
    "    (tawa.Topic == 'Fiscals') & (tawa.Variable == 'Disposable_Income')][\n",
    "        ['Scenario', 'Value']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "wnl_cols = [\n",
    "    'Scenario', 'Variable', 'Winner_Loser', 'Eq_DI_Quantile','Value']\n",
    "wnl_vars = ['Mean_Weekly_Change', 'Population_In_Category']\n",
    "\n",
    "wnl = tawa[\n",
    "    (tawa.Topic == \"WnL\") & tawa.Variable.isin(wnl_vars) & \n",
    "    (tawa.WnL_Group == \"All Households\")][wnl_cols]\n",
    "wnl.Eq_DI_Quantile = pd.Categorical(\n",
    "    wnl.Eq_DI_Quantile, categories=[str(i) for i in range(1, 11)]+['All'], \n",
    "    ordered=True)\n",
    "wnl.Winner_Loser = pd.Categorical(\n",
    "    wnl.Winner_Loser, categories=['Winners', 'Losers', 'All'], ordered=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "wnl_wide = wnl.pivot_table(index=['Scenario', 'Eq_DI_Quantile'], columns=['Winner_Loser', 'Variable'], values='Value', aggfunc='first')\n",
    "wnl_wide.columns = ['_'.join(col).strip() for col in wnl_wide.columns.values]\n",
    "wnl_wide.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "poverty = tawa[\n",
    "    (tawa.Topic == \"Poverty\") & (tawa.Variable == \"Change_In_Population_In_Poverty\")\n",
    "    & (tawa.Population_Type == \"Children\")][\n",
    "        ['Scenario', 'Poverty_Type', 'Value']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "poverty_wide = poverty.pivot_table(index='Scenario', columns='Poverty_Type', values='Value', aggfunc='first')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prime model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tawa_priming.jinja2') as f:\n",
    "    priming_template = jinja2.Template(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "chat = Chat()\n",
    "chat.add_context(\n",
    "    priming_template.render(\n",
    "        tawa_details=tawa_details, fiscals=fiscals, wnl_wide=wnl_wide, \n",
    "        poverty_wide=poverty_wide))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display number of tokens in the context before asking any questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
    "    encoding = tiktoken.encoding_for_model(encoding_name)\n",
    "    num_tokens = len(encoding.encode(string))\n",
    "    return num_tokens\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = '\\n'.join([m['content'] for m in chat.messages])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 2169\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of tokens: {num_tokens_from_string(content, 'gpt-4')}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This number of tokens gives us several options for which model to use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ask some questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The best reform to support low income households from a value for money perspective appears to be FTC7.5a42.7k27. This is because it has the second highest fiscal cost of 145 million dollars but is noticeably more effective at reducing poverty. It reduces BHC by 7000 and AHC50 Fixed by 11000, which are the highest reductions among all reforms. It also leads to the highest mean weekly increase for the poorest households (16 dollars for 1st quantile) while only slightly causing losers in the higher quantiles. Its overall mean weekly change is also quite high at 2 dollars. So it seems to provide the most substantial support for lower-income groups with relatively considerable cost.'"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat.ask(\n",
    "    \"From a value for money perspective, which reform best supports low income households?\",\n",
    "    model = \"gpt-4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best reform to support low income households from a value for money perspective appears to be FTC7.5a42.7k27. This is because it has the second highest fiscal cost of 145 million dollars but is noticeably more effective at reducing poverty. It reduces BHC by 7000 and AHC50 Fixed by 11000, which are the highest reductions among all reforms. It also leads to the highest mean weekly increase for the poorest households (16 dollars for 1st quantile) while only slightly causing losers in the higher quantiles. Its overall mean weekly change is also quite high at 2 dollars. So it seems to provide the most substantial support for lower-income groups with relatively considerable cost.\n"
     ]
    }
   ],
   "source": [
    "print(chat.messages[-1]['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The reform that appears to be most tightly targeted to low incomes is FTC5a42.7k27. The mean weekly income change for winning households in quantile 1 (lowest income) is 10 dollars and it gradually decreases to 5 dollars for quantile 6. This reform also starts introducing losers from quantile 3 onwards with the losers mean weekly change incrementally worsening as we move up income quantiles. The fiscal cost of this reform is also significantly lower at 68 million dollars. This indicates that this reform is offering a more focused support to lower-income households while also drawing back support from higher-income ones. Therefore, it could be seen as more tightly targeted to low incomes.'"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat.ask(\"which reform is the most tightly targeted to low incomes?\", model = \"gpt-4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The reform that appears to be most tightly targeted to low incomes is FTC5a42.7k27. The mean weekly income change for winning households in quantile 1 (lowest income) is 10 dollars and it gradually decreases to 5 dollars for quantile 6. This reform also starts introducing losers from quantile 3 onwards with the losers mean weekly change incrementally worsening as we move up income quantiles. The fiscal cost of this reform is also significantly lower at 68 million dollars. This indicates that this reform is offering a more focused support to lower-income households while also drawing back support from higher-income ones. Therefore, it could be seen as more tightly targeted to low incomes.\n"
     ]
    }
   ],
   "source": [
    "print(chat.messages[-1]['content'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and a final check of the content tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 2481\n"
     ]
    }
   ],
   "source": [
    "content = '\\n'.join([m['content'] for m in chat.messages])\n",
    "print(f\"Number of tokens: {num_tokens_from_string(content, 'gpt-4')}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved chat log to output/tawa_ai.txt\n"
     ]
    }
   ],
   "source": [
    "chat.write_txt(path='output/tawa_ai.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "aipy-7XVxUsX7",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}