YouGovAPI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"data\":{\"Company default data\":\"https:\\/\\/api.brandindex.com\\/v0\\/company\\/defaults\"},\"ok\":true}\n",
      "{\"data\":{\"Company default data\":\"https:\\/\\/api.brandindex.com\\/v0\\/company\\/defaults\"},\"ok\":true}\n",
      "{\"data\":{\"Company default data\":\"https:\\/\\/api.brandindex.com\\/v0\\/company\\/defaults\"},\"ok\":true}\n",
      "{\"data\":{\"Company default data\":\"https:\\/\\/api.brandindex.com\\/v0\\/company\\/defaults\"},\"ok\":true}\n"
     ]
    }
   ],
   "source": [
    "### 4th May 2020\n",
    "### Nishant Das @ IESE (ndas@iese.edu)\n",
    "### Code to extract data from YouGov API\n",
    "### Only for USA because we have excel file that lists nessesary Parameters for data extraction\n",
    "\n",
    "import csv\n",
    "import requests\n",
    "import urllib.parse\n",
    "import pathlib\n",
    "import os\n",
    "from itertools import islice\n",
    "\n",
    "#Enter your login credinetials here & where the Excel file path is saved here\n",
    "email = 'ndas@iese.edu'  \n",
    "password = 'XXX'\n",
    "inFile = \"/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/Test_YouGov_DataSet.csv\"\n",
    "#region='us'\n",
    "\n",
    "#Loop that will read the excel file, extract the data from YouGov API and save it into a new excel file\n",
    "# File will save as USA_SECTOR ID NUMBER_COMPANY NAME.csv\n",
    "# For example, USA_1_US Airways.csv\n",
    "# Save this .py file in a new folder and all YouGov files will be saved in the same folder\n",
    "    \n",
    "    \n",
    "with open(inFile, 'r') as read_obj:\n",
    "    csv_reader=csv.reader(read_obj)\n",
    "    header = next(csv_reader)\n",
    "    if header != None:\n",
    "        for col in csv_reader:\n",
    "            try:\n",
    "                brand_code= col[0]\n",
    "                active=col[1]\n",
    "                url=col[2]\n",
    "                sector_id= col[3]\n",
    "                brand_name= col[4].split('/')[0]\n",
    "                #sector_name=col[5].split('/')[0]\n",
    "                region=url[url.find('region=')+7:url.find('region=')+9]\n",
    "                already_downloaded=col[6]\n",
    "                if already_downloaded==\"Already Downloaded\":\n",
    "                    pass\n",
    "                else:\n",
    "    #extracting & saving the data\n",
    "                    with requests.Session() as session:\n",
    "                        response = session.post('https://api.brandindex.com/v0/login', \n",
    "                                                {'email': email, 'password': password})\n",
    "                        print(response.text)\n",
    "                        # get start and end dates\n",
    "                        r = session.get(url)\n",
    "                        start_date = r.json()['data']['validity_periods'][0][0]\n",
    "                        end_date = r.json()['data']['validity_periods'][0][1]\n",
    "                        if end_date == None:\n",
    "                            end_date = '2020-05-09'\n",
    "\n",
    "\n",
    "                        apiParameters={'region':region, 'sector':sector_id, 'brand_id': brand_code, 'start_date':start_date, 'end_date':end_date}\n",
    "                        baseUrl = 'https://api.brandindex.com/v0/timeline/file.csv?scoring=total&moving_average=1&'\n",
    "                        url = baseUrl + urllib.parse.urlencode(apiParameters)\n",
    "                        download = session.get(url)\n",
    "                        decoded_content = download.content.decode('utf-8')\n",
    "\n",
    "        #creating industry level Sub-directories/Folders\n",
    "                        #fileName = brand_name+\"_sector_\"+sector_id+'_'+sector_name+'.csv' \n",
    "                        #folderName= os.path.join(region, 'sector_'+sector_id+'_'+sector_name)\n",
    "                        fileName = brand_name+\"_sector_\"+sector_id+'_'+'.csv'\n",
    "                        folderName= os.path.join(region, region+'_'+'sector_'+sector_id)\n",
    "                        folderPath = pathlib.Path(folderName).mkdir(parents=True, exist_ok=True)\n",
    "                        outFile=os.path.join(folderName, fileName)\n",
    "                        with open(outFile, 'wb') as f:  \n",
    "                            f.write(download.content)\n",
    "\n",
    "\n",
    "    # Creating a log file with downloaded file details\n",
    "                        fileSize=os.path.getsize(outFile)\n",
    "                        if fileSize > 2000:\n",
    "                            downloadStatus = \"Seems Okay\"\n",
    "                        else:\n",
    "                            downloadStatus = 'ALERT - File is Small'\n",
    "                        with open('logFile.csv', 'a+') as f:\n",
    "                            f.write(('{},{},{},{}, {}'.format(str(sector_id),\n",
    "                                                              str(brand_code), \n",
    "                                                              str(brand_name), \n",
    "                                                              str(fileSize),\n",
    "                                                              str(downloadStatus)))+'\\n')\n",
    "\n",
    "    # Creating Master file per industry with all brands saved                   \n",
    "                        #masterFileName = \"000_Master_\"+region+'_'+'sector_'+sector_id+'_'+sector_name+'.csv'\n",
    "                        masterFileName = \"000_Master_\"+region+'_'+'sector_'+sector_id+'_'+'.csv'\n",
    "                        masterOutFile=os.path.join(folderName, masterFileName)\n",
    "                        with open(outFile, 'r') as input_file:   \n",
    "                            with open(masterOutFile, 'a') as output_file:\n",
    "                                writer=csv.writer(output_file)\n",
    "                                header = ['brand_code', 'brand name']\n",
    "                                fieldNames=header+next(csv.reader(input_file))\n",
    "                                writer.writerow(fieldNames)\n",
    "                                for row in csv.reader(input_file):\n",
    "                                    writer.writerow([brand_code]+[brand_name]+row)\n",
    "\n",
    "            except:\n",
    "                pass  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/logFile.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-3757f931425b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogFile_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mlogFile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0mdownloaded_brandIds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcsv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogFile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdownloaded_brandIds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/logFile.csv'"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import requests\n",
    "import urllib.parse\n",
    "import pathlib\n",
    "import os\n",
    "from itertools import islice\n",
    "\n",
    "\n",
    "logFile_path=\"/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/logFile.csv\"\n",
    "inFile_path = \"/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/Test_YouGov_DataSet.csv\"\n",
    "\n",
    "\n",
    "with open(logFile_path, 'r') as logFile:\n",
    "    downloaded_brandIds = dict((r[1], i) for i, r in enumerate(csv.reader(logFile)))\n",
    "    print(downloaded_brandIds)\n",
    "\n",
    "with open(inFile_path, 'r') as inFile:\n",
    "    with open('results.csv', 'w') as results:    \n",
    "        reader = csv.reader(inFile)\n",
    "        writer = csv.writer(results)\n",
    "\n",
    "        writer.writerow(next(reader, []) + ['RESULTS'])\n",
    "\n",
    "        for row in reader:\n",
    "            index = downloaded_brandIds.get(row[0])\n",
    "            print(index)\n",
    "            if index is not None:\n",
    "                message = 'Already Downloaded'\n",
    "            else:\n",
    "                message = 'NOT Downloaded'\n",
    "            writer.writerow(row + [message])\n",
    "    os.rename(r'/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/results.csv',r'/Users/nishant/Desktop/1-RESEARCH/Data&Code/YouGov/YouGovAPI/Test_YouGov_DataSet.csv')\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests in /Users/nishant/opt/anaconda3/lib/python3.7/site-packages (2.22.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/nishant/opt/anaconda3/lib/python3.7/site-packages (from requests) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Users/nishant/opt/anaconda3/lib/python3.7/site-packages (from requests) (2.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/nishant/opt/anaconda3/lib/python3.7/site-packages (from requests) (1.24.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/nishant/opt/anaconda3/lib/python3.7/site-packages (from requests) (2019.9.11)\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1.1 is available.\n",
      "You should consider upgrading via the '/Users/nishant/opt/anaconda3/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: Could not find a version that satisfies the requirement urllib.parse (from versions: none)\u001b[0m\n",
      "\u001b[31mERROR: No matching distribution found for urllib.parse\u001b[0m\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1.1 is available.\n",
      "You should consider upgrading via the '/Users/nishant/opt/anaconda3/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install urllib.parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pathlib\n",
      "  Downloading pathlib-1.0.1.tar.gz (49 kB)\n",
      "\u001b[K     |████████████████████████████████| 49 kB 1.9 MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: pathlib\n",
      "  Building wheel for pathlib (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pathlib: filename=pathlib-1.0.1-py3-none-any.whl size=14347 sha256=a07e6775a225ef94b6b224432f33a29626fcb296f98b7c26a7181cc9a441debe\n",
      "  Stored in directory: /Users/nishant/Library/Caches/pip/wheels/6e/96/b8/10037fe231e23970bac58361d7c93571ab983a7bbc55e68550\n",
      "Successfully built pathlib\n",
      "Installing collected packages: pathlib\n",
      "Successfully installed pathlib-1.0.1\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1.1 is available.\n",
      "You should consider upgrading via the '/Users/nishant/opt/anaconda3/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install pathlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}