111A Pulling Tweets via the Twitter API v2.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #1 Define Functions to Interact with the Twitter API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "import json \n",
    "import pandas as pd\n",
    "\n",
    "# imports the twitter_secrets python file in which we store the twitter API keys\n",
    "# place the twitter_secrets file under <User>/anaconda3/Lib\n",
    "from twitter_secrets import twitter_secrets as ts\n",
    "\n",
    "# puts the bearer token in the request header\n",
    "def create_headers(bearer_token):\n",
    "    headers = {\"Authorization\": \"Bearer {}\".format(bearer_token)}\n",
    "    return headers\n",
    "        \n",
    "# sets the rules on which tweets to retrieve    \n",
    "def set_rules(headers, delete, bearer_token, rules):\n",
    "    payload = {\"add\": rules}\n",
    "    response = requests.post(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\",\n",
    "        headers=headers,\n",
    "        json=payload,\n",
    "    )\n",
    "    if response.status_code != 201:\n",
    "        raise Exception(\n",
    "            \"Cannot add rules (HTTP {}): {}\".format(response.status_code, response.text)\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "    \n",
    "# retrieves the current set of rules from the API     \n",
    "def get_rules(headers, bearer_token):\n",
    "    response = requests.get(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\", headers=headers\n",
    "    )\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot get rules (HTTP {}): {}\".format(response.status_code, response.text)\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "    return response.json()\n",
    "\n",
    "# tells the API to delete our current rule configuration \n",
    "def delete_all_rules(headers, bearer_token, rules):\n",
    "    if rules is None or \"data\" not in rules:\n",
    "        return None\n",
    "\n",
    "    ids = list(map(lambda rule: rule[\"id\"], rules[\"data\"]))\n",
    "    payload = {\"delete\": {\"ids\": ids}}\n",
    "    response = requests.post(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream/rules\",\n",
    "        headers=headers,\n",
    "        json=payload\n",
    "    )\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot delete rules (HTTP {}): {}\".format(\n",
    "                response.status_code, response.text\n",
    "            )\n",
    "        )\n",
    "    print(json.dumps(response.json()))\n",
    "\n",
    "# starts the stream, iterates through the lines of the response and for each line calls the save_tweets function\n",
    "def get_stream(headers, set, bearer_token, expansions, fields):\n",
    "    data = []\n",
    "    response = requests.get(\n",
    "        \"https://api.twitter.com/2/tweets/search/stream\" + expansions + fields, headers=headers, stream=True,\n",
    "    )\n",
    "    print(response.status_code)\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(\n",
    "            \"Cannot get stream (HTTP {}): {}\".format(\n",
    "                response.status_code, response.text\n",
    "            )\n",
    "        )\n",
    "    i = 0\n",
    "    for response_line in response.iter_lines():\n",
    "        i += 1\n",
    "        if i == max_results:\n",
    "            break\n",
    "        else:\n",
    "            json_response = json.loads(response_line)\n",
    "            #print(json.dumps(json_response, indent=4, sort_keys=True))\n",
    "            try:\n",
    "                save_tweets(json_response)\n",
    "            except (json.JSONDecodeError, KeyError) as err:\n",
    "                # In case the JSON fails to decode, we skip this tweet\n",
    "                print(f\"{i}/{max_results}: ERROR: encountered a problem with a line of data... \\n\")\n",
    "                continue\n",
    "\n",
    "# appends information from tweets to a dataframe                \n",
    "def save_tweets(tweet):\n",
    "    print(json.dumps(tweet, indent=4, sort_keys=True))\n",
    "    data = tweet['data']\n",
    "    public_metrics = data['public_metrics']\n",
    "    tweet_list.append([data['id'], data['author_id'], data['created_at'], data['text'], public_metrics['like_count']])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step #2 Subscribe to the Tweet Streaming Service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the max number of tweets that will be returned\n",
    "max_results = 20\n",
    "\n",
    "# You can adjust the rules if needed\n",
    "search_rules = [\n",
    "    {\"value\": \"dog has:images\", \"tag\": \"dog pictures\", \"lang\": \"en\"},\n",
    "    {\"value\": \"cat has:images -grumpy\", \"tag\": \"cat pictures\", \"lang\": \"en\"},\n",
    "]\n",
    "\n",
    "# defines the fields which we want to retrieve\n",
    "tweet_fields = \"?tweet.fields=attachments,author_id,created_at,public_metrics\"\n",
    "\n",
    "# we only retrieve the tweet object, but if we wanted to retrieve other objects (e.g., media), we would add them to the expansions string\n",
    "expansions = \"\"\n",
    "tweet_list = []\n",
    "\n",
    "bearer_token = ts.BEARER_TOKEN\n",
    "headers = create_headers(bearer_token)\n",
    "rules = get_rules(headers, bearer_token)\n",
    "delete = delete_all_rules(headers, bearer_token, rules)\n",
    "set = set_rules(headers, delete, bearer_token, search_rules)\n",
    "get_stream(headers, set, bearer_token, expansions, tweet_fields)\n",
    "\n",
    "df = pd.DataFrame (tweet_list, columns = ['tweetid', 'author_id' , 'created_at', 'text', 'like_count'])\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}