Generate Latex Bibliography.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import unicodedata\n",
    "from unidecode import unidecode"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proof flag\n",
    "Change the following to `IS_PROOF = False` if links should go to public IsisCB site.\n",
    "\n",
    "Change `INCLUDE_ALREADY_PRINTED` to `INCLUDE_ALREADY_PRINTED = True` if already printed publications should be included (publications that have a date in the column 'Published Print'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IS_PROOF = True\n",
    "INCLUDE_ALREADY_PRINTED = False\n",
    "IGNORE_DATASET = False\n",
    "IGNORE_CATEGORIES = False\n",
    "\n",
    "DEV_MODE = True # change this to true to use production file paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATABASE_TO_PRINT = \"Isis Bibliography of the History of Science (Stephen P. Weldon, ed.)\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File paths\n",
    "Set the following paths to files that this notebook should read or write.\n",
    "- `export_file`: path to file that contains the IsisCB export (should be a csv file)\n",
    "- `result_file_path`: path to file that will contain all publications except book reviews\n",
    "- `reviews_file_path`: path to file that will contain all book reviews\n",
    "- `errors_file_path`: path to file that will contain all errors\n",
    "- `author_index_file_path`: path to file that will contain the author index\n",
    "\n",
    "If you run this notebook on a Windows machine, make sure to encode all backslashes in path names (e.g. `\\` should be `\\\\`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if DEV_MODE:\n",
    "    export_file = '/Users/jdamerow/UpConsulting/latexBibGenerator/files/printdata-211005.csv' #'/Users/jdamerow/UpConsulting/files/latex/printdata-media.csv'  #\n",
    "    result_file_path = '/Users/jdamerow/UpConsulting/files/latex/result1.txt'\n",
    "    reviews_file_path = '/Users/jdamerow/UpConsulting/files/latex/reviews1.txt'\n",
    "    errors_file_path = '/Users/jdamerow/UpConsulting/files/latex/errors1.csv'\n",
    "    author_index_file_path = '/Users/jdamerow/UpConsulting/files/latex/index1.tex'\n",
    "    subject_index_file_path = '/Users/jdamerow/UpConsulting/files/latex/subj_index.tex'\n",
    "else:\n",
    "    export_file = \"C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\printdata.csv\"\n",
    "    result_file_path = 'C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\biblist.tex'\n",
    "    reviews_file_path = 'C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\bookreviews.tex'\n",
    "    errors_file_path = 'C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\errors.csv'\n",
    "    author_index_file_path = 'C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\author_index.tex'\n",
    "    subject_index_file_path = 'C:\\\\Users\\\\Stephen\\\\Dropbox (Univ. of Oklahoma)\\\\CB Directory\\\\PRINT\\\\subj_index.tex'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SUBJECTS_TO_EXCLUDE = [\n",
    "    'CBA000113709'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code\n",
    "The following cells contain the code that will turn the exported csv file into latex. Do not change unless you know what you are doing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_template = \"\\\\textsc{%s}, %s\"\n",
    "author_template_organization = \"\\\\textsc{%s}\"\n",
    "isbn_template = \"\\\\textsc{isbn}: %s.\"\n",
    "CBB_link_template = \"\\\\textsc{\\\\href{https://data.isiscb.org/isis/citation/%s}{%s}}\"\n",
    "CBB_link_template_proof = \"\\\\textsc{\\\\href{https://data.isiscb.org/isis/curation/citation/%s}{%s}}\"\n",
    "\n",
    "latex_template_description = \"\"\"\n",
    "\\\\begin{isisdescription} %s\\\\end{isisdescription}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTHOR_SEPARATER = \"; \"\n",
    "CHAPTER_SEPARATER = \"; \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DocumentError():\n",
    "    def __init__(self, doc, error, error_type):\n",
    "        self.doc = doc\n",
    "        self.error = error\n",
    "        self.error_type = error_type\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_link_latex(cbb_nr):\n",
    "    if IS_PROOF:\n",
    "        return CBB_link_template_proof%(cbb_nr, cbb_nr)\n",
    "    return CBB_link_template%(cbb_nr, cbb_nr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_last_first(name):\n",
    "    name = name.strip()\n",
    "    if ',' in name:\n",
    "        last = name[:name.index(',')].strip()\n",
    "        first = name[name.index(',')+1:].strip()\n",
    "    elif \" \" in name:\n",
    "        last = name[name.rfind(\" \"):].strip()\n",
    "        first = name[:name.rfind(\" \")].strip()\n",
    "    else:\n",
    "        last = name\n",
    "        first = \"\"\n",
    "    return last, first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Author:\n",
    "    def __init__(self):\n",
    "        self.name = ''\n",
    "        self.order = 0\n",
    "        self.author_type = \"\"\n",
    "        \n",
    "    def get_last_name(self):\n",
    "        if self.author_type == \"Person\":\n",
    "            return get_last_first(self.name)[0]\n",
    "        return self.name\n",
    "        \n",
    "    def get_first_name(self):\n",
    "        if self.author_type == \"Person\":\n",
    "            return get_last_first(self.name)[1]\n",
    "        return \"\"\n",
    "        \n",
    "    def get_name_for_sort(self):\n",
    "        return unicodedata.normalize('NFD', self.get_last_name().lower()).encode('ascii', 'ignore').decode('utf8') + \", \" + unicodedata.normalize('NFD', self.get_first_name().lower()).encode('ascii', 'ignore').decode('utf8')\n",
    "    \n",
    "    def get_order(self):\n",
    "        try:\n",
    "            float(self.order)\n",
    "            return float(self.order)\n",
    "        except:\n",
    "            print(\"Not a valid order value \" + str(self.order))\n",
    "            return 0\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_persons_latex_list(persons):\n",
    "    persons_latex = []\n",
    "    if persons:\n",
    "        for p in persons:\n",
    "            if p.author_type == 'Person':\n",
    "                last, first = get_last_first(p.name)\n",
    "                if first:\n",
    "                    persons_latex.append(author_template%(last, first))\n",
    "                else:\n",
    "                    persons_latex.append(author_template_organization%(last))\n",
    "            else:\n",
    "                persons_latex.append(author_template_organization%(p.name))\n",
    "    return persons_latex\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_author_editor_string(author_editors_latex):\n",
    "    author_editor_string = AUTHOR_SEPARATER.join(author_editors_latex).strip()\n",
    "    if author_editor_string:\n",
    "        if not author_editor_string.endswith(\".\"):\n",
    "            author_editor_string = author_editor_string + \".\"\n",
    "        return author_editor_string + \" \"\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_title(title_string, append_period=True):\n",
    "    title_string = title_string.replace(\" - \", \"--\").replace(\" -- \", \"---\")\n",
    "    if title_string[len(title_string)-1:] not in [\"?\", \"!\", \":\", \";\", \".\"] and append_period:\n",
    "        return title_string + \".\"\n",
    "    return title_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_special_chars(text_string):\n",
    "    text_string = re.sub(r\"( |^)'([a-zA-Z0-9\\\"])\", r\"\\1`\\2\", text_string)\n",
    "    text_string = re.sub(r\"( |^)[\\\"“„]([a-zA-Z0-9\\[\\]`])\", r\"\\1``\\2\", text_string)\n",
    "    text_string = re.sub(r\"([a-zA-Z0-9\\[\\]\\.\\?!])[\\\"”“]([ \\.:!\\?\\[\\]]|$)\", r\"\\1''\\2\", text_string)\n",
    "    text_string = re.sub(\"’\", \"'\", text_string)\n",
    "    if \"\\\"'\" in text_string:\n",
    "        text_string = text_string.replace(\"\\\"'\", \"''\\\\hspace{0.4mm}'\")\n",
    "    text_string = re.sub(\"_\", \"\\\\_\", text_string)\n",
    "    text_string = text_string.replace(\"─\", \"---\").replace(\"–\", \"---\").replace(\"…\", \"\\\\dots\")\n",
    "    return text_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_html_tags(text_string, doc, set_non_italic=False):\n",
    "    if not set_non_italic:\n",
    "        text_string = re.sub(r\"<em>(.+?)</em>\", r\"\\\\textit{\\1}\", text_string)\n",
    "        text_string = re.sub(r\"<i>(.+?)</i>\", r\"\\\\textit{\\1}\", text_string)\n",
    "    else:\n",
    "        text_string = re.sub(r\"<em>(.+?)</em>\", r\"{\\\\normalfont \\1}\", text_string)\n",
    "        text_string = re.sub(r\"<i>(.+?)</i>\", r\"{\\\\normalfont \\1}\", text_string)\n",
    "    text_string = re.sub(r\"<sub>(.+?)</sub>\", r\"\\\\textsubscript{\\1}\", text_string)\n",
    "    \n",
    "    tag_regex = re.compile(r'<.*?>')\n",
    "    found_tag = tag_regex.search(text_string)\n",
    "    if found_tag:\n",
    "        errors.append(DocumentError(doc, \"Could not handle tag: \" + found_tag.group(0), \"WARNING\"))\n",
    "    return text_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_space_before_quotes(text_string):\n",
    "    if text_string.startswith(\"`\") or text_string.startswith(\"\\\"\"):\n",
    "        text_string = \"\\\\hspace{0.4mm}\" + text_string\n",
    "    return text_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_edition_string(edition_string):\n",
    "    edition_string = edition_string.strip()\n",
    "    if edition_string and not edition_string.endswith(\".\"):\n",
    "        edition_string = edition_string.strip() + \".\"\n",
    "    if edition_string:\n",
    "        edition_string = \" \" + edition_string\n",
    "    return edition_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def include_based_on_print_date(print_date):\n",
    "    if INCLUDE_ALREADY_PRINTED:\n",
    "        return True\n",
    "    \n",
    "    return print_date.strip() == \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following cells contain the publication classes and their latex templates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Document:\n",
    "    def __init__(self):\n",
    "        self.category_number = ''\n",
    "        self.print_date = ''\n",
    "        self.index = 0\n",
    "        self.isiscb_id = ''\n",
    "        self.title = ''\n",
    "        self.year = 0\n",
    "        self.database = \"\"\n",
    "        self.edition = \"\"\n",
    "        self.description = \"\"\n",
    "        self.publisher = ''\n",
    "        self.subjects = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Subject:\n",
    "    def __init__(self):\n",
    "        self.name = ''\n",
    "        self.id = ''\n",
    "        self.display_name = ''\n",
    "        self.authority_type = ''\n",
    "        self.acr_type = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_book = \"\"\"\n",
    " \n",
    " \n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(index)s}\\\\hspace{0.5em} %(authors)s\\\\textit{%(title)s}%(edition)s%(editors)s %(publisher)s, %(year)s. %(isbn)s %(link)s.\n",
    "%(desc)s\n",
    "%(chapters)s\n",
    "%(reviews)s\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "latex_template_book_chapters = \"\"\"\n",
    "\\\\begin{isisdescription} Includes:  %s.\\\\end{isisdescription}\n",
    "\"\"\"\n",
    "\n",
    "latex_template_book_chapter_item = \"\"\"\n",
    "%s ``\\\\selectlanguage{french}%s\\\\selectlanguage{english}''%s%s\n",
    "\"\"\"\n",
    "\n",
    "chapter_ref=\" [ref.~%s]\"\n",
    "editors_template = \", %s (Eds.)\"\n",
    "pages_template = \" %s\"\n",
    "\n",
    "reviews_template = \"\"\"\n",
    "\\\\begin{isisdescription} Reviews: %s\\\\end{isisdescription}\n",
    "\"\"\"\n",
    "\n",
    "class Book(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.editors = []\n",
    "        self.title = ''\n",
    "        self.year = 0\n",
    "        self.isbn = ''\n",
    "        self.chapters = []\n",
    "        self.chapter_ids = []\n",
    "       \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors + self.editors\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in (self.authors if self.authors else self.editors)])\n",
    "    \n",
    "    def get_author_string(self):\n",
    "        return \", \".join(build_persons_latex_list(self.authors)) if self.authors else \"\"\n",
    "        \n",
    "    def get_editor_string(self, append_eds=True):\n",
    "        editors_latex = []\n",
    "        if self.editors:\n",
    "            for e in self.editors:\n",
    "                last, first = get_last_first(e.name)\n",
    "                if last and first:\n",
    "                    editors_latex.append(author_template%(last, first))\n",
    "                else:\n",
    "                    editors_latex.append(author_template_organization%(last))\n",
    "        eds_suffix = \"\"\n",
    "        if editors_latex and append_eds:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "                \n",
    "        return prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "        \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = build_persons_latex_list(self.authors)\n",
    "        editors_latex = build_persons_latex_list(self.editors)\n",
    "        \n",
    "        author_editor_string = \"\"\n",
    "        editors_string = \"\"\n",
    "        if editors_latex:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "                    \n",
    "        if authors_latex:\n",
    "            author_editor_string = prepare_author_editor_string(authors_latex)\n",
    "            if editors_latex:\n",
    "                editors_string = \", \" + prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "        elif editors_latex:\n",
    "            author_editor_string = prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "\n",
    "        isbn = ''\n",
    "        if self.isbn:\n",
    "            isbn = isbn_template%(self.isbn)\n",
    "            \n",
    "        # render chapters\n",
    "        chapter_string = \"\"\n",
    "        if self.chapters:\n",
    "            chapter_latex = []\n",
    "            for chap in self.chapters:\n",
    "                print(chap.isiscb_id)\n",
    "                chapter_authors = build_persons_latex_list(chap.authors)\n",
    "                \n",
    "                chap_pages = ''\n",
    "                if hasattr(chap, 'pages') and chap.pages:\n",
    "                    chap_pages = pages_template%(chap.pages)\n",
    "                    \n",
    "                chap_ref = \"\"\n",
    "                if chap.index:\n",
    "                    chap_ref = chapter_ref%(chap.index)\n",
    "\n",
    "                chapter_latex.append((latex_template_book_chapter_item%(prepare_author_editor_string(chapter_authors), handle_html_tags(handle_special_chars(chap.title), chap), chap_pages, chap_ref)).strip())\n",
    "            \n",
    "            if chapter_latex:\n",
    "                chapter_string = latex_template_book_chapters%(CHAPTER_SEPARATER.join(chapter_latex))\n",
    "                \n",
    "        reviews_string = \"\"\n",
    "        if reviews:\n",
    "            reviews_string = reviews_template%(\"[\" + str(reviews.index) + \"]\")\n",
    "            \n",
    "        edition_string = get_edition_string(self.edition)\n",
    "        if edition_string.endswith(\".\"):\n",
    "            edition_string = edition_string[:-1]\n",
    "            \n",
    "        title = handle_html_tags(handle_special_chars(self.title), self, set_non_italic=True)\n",
    "        if edition_string or not editors_string:\n",
    "            title = prepare_title(title)\n",
    "            \n",
    "        return latex_template_book%{\"index\":counter, \"authors\":author_editor_string, \"title\":title, \"edition\":edition_string, \"editors\":editors_string,\n",
    "                                    \"publisher\":self.publisher, \"year\":self.year, \"isbn\":isbn, \"link\":generate_link_latex(self.isiscb_id), \"desc\":latex_template_description%(self.description) if self.description else \"\", \"chapters\":chapter_string, \"reviews\":reviews_string}\n",
    "                                    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_article = \"\"\"\n",
    " \n",
    " \n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(counter)s}\\\\hspace{0.5em} %(author)s``%(title)s''%(edition)s \\\\textit{%(journal)s}%(volume)s%(issue)s (%(year)s)%(pages)s.%(publisher)s %(link)s.\n",
    "\n",
    "%(description)s\n",
    "\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "latex_template_article_publisher = \" %s.\"\n",
    "\n",
    "volume_template = \" %s\"\n",
    "issue_template = \", no. %s\"\n",
    "pages_template = \": %s\"\n",
    "class Article(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.title = ''\n",
    "        self.year = 0\n",
    "        self.journal = ''\n",
    "        self.journal_abbr = ''\n",
    "        self.isbn = ''\n",
    "        self.volume = ''\n",
    "        self.issue = ''\n",
    "        self.pages = ''\n",
    "        \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in self.authors])\n",
    "    \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = []\n",
    "        for a in self.authors:\n",
    "            name = a.name\n",
    "            last, first = get_last_first(name)\n",
    "            if first:\n",
    "                authors_latex.append(author_template%(last, first))\n",
    "            else:\n",
    "                authors_latex.append(author_template_organization%(last))\n",
    "\n",
    "        vol = ''\n",
    "        if self.volume:\n",
    "            vol = volume_template%(self.volume)\n",
    "        issue = ''\n",
    "        if self.issue:\n",
    "            issue = issue_template%(self.issue)\n",
    "        pages = ''\n",
    "        if self.pages:\n",
    "            pages = pages_template%(self.pages)\n",
    "            \n",
    "        journal_text = self.journal_abbr if self.journal_abbr.strip() else self.journal\n",
    "        if not journal_text:\n",
    "            errors.append(DocumentError(self, \"No Journal abbreviation or title found.\", \"ERROR\"))\n",
    "        return latex_template_article%{\n",
    "            \"counter\":counter, \n",
    "            \"author\": prepare_author_editor_string(authors_latex), \n",
    "            \"title\": handle_html_tags(add_space_before_quotes(prepare_title(handle_special_chars(self.title))), self), \n",
    "            \"edition\": get_edition_string(self.edition), \n",
    "            \"journal\": journal_text, \n",
    "            \"volume\": vol, \n",
    "            \"issue\": issue, \n",
    "            \"year\": self.year, \n",
    "            \"pages\": pages, \n",
    "            \"publisher\": latex_template_article_publisher%(self.publisher) if self.publisher else \"\",\n",
    "            \"link\": generate_link_latex(self.isiscb_id), \n",
    "            \"description\": latex_template_description%(self.description)  if self.description else \"\"\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_chapter = \"\"\"\n",
    " \n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(counter)s}\\\\hspace{0.5em} %(author)s``%(title)s''%(edition)s In \\\\textit{%(book)s}%(book_ref)s%(book_editor)s%(publisher)s (%(year)s)%(pages)s. %(link)s.\n",
    "\n",
    "%(description)s\n",
    "\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "latex_template_book_ref = \" [ref. %s]\"\n",
    "latex_template_book_editors = \", edited by %s\"\n",
    "latex_template_book_authors = \" by %s\"\n",
    "latex_template_book_pages = \", %s\"\n",
    "latex_template_chapter_publisher = \" %s\"\n",
    "\n",
    "class Chapter(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.editors = []\n",
    "        self.title = ''\n",
    "        self.year = 0\n",
    "        self.book = None\n",
    "        self.pages = ''\n",
    "        \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors + self.editors\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in self.authors])\n",
    "    \n",
    "    def get_editor_string(self):\n",
    "        editors_latex = []\n",
    "        if self.editors:\n",
    "            for e in self.editors:\n",
    "                last, first = get_last_first(e.name)\n",
    "                if last and first:\n",
    "                    editors_latex.append(author_template%(last, first))\n",
    "                else:\n",
    "                    editors_latex.append(author_template_organization%(last))\n",
    "        eds_suffix = \"\"\n",
    "        if editors_latex:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "        \n",
    "        return prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "        \n",
    "        \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = []\n",
    "        for a in self.authors:\n",
    "            name = a.name\n",
    "            last, first = get_last_first(name)\n",
    "            if first:\n",
    "                authors_latex.append(author_template%(last, first))\n",
    "            else:\n",
    "                authors_latex.append(author_template_organization%(last))\n",
    "\n",
    "        book_ref = \"\"\n",
    "        if self.book.index > 0:\n",
    "            book_ref = latex_template_book_ref%(self.book.index)\n",
    "        \n",
    "        print(self.isiscb_id)\n",
    "        book_authors = self.book.get_author_string()\n",
    "        book_editors = self.book.get_editor_string(append_eds=False) if self.book.get_editor_string() else self.get_editor_string()\n",
    "        book_editors_or_authors_string = \"\"\n",
    "        if book_editors and not book_authors:\n",
    "            book_editors_or_authors_string = latex_template_book_editors%(book_editors)\n",
    "        else:\n",
    "            book_editors_or_authors_string = latex_template_book_authors%(book_authors)\n",
    "        \n",
    "        pages_string = \"\"\n",
    "        if self.pages:\n",
    "            pages_string = latex_template_book_pages%(self.pages)\n",
    "        return latex_template_chapter%{\n",
    "            \"counter\": counter, \n",
    "            \"author\": prepare_author_editor_string(authors_latex), \n",
    "            \"title\": handle_html_tags(prepare_title(handle_special_chars(self.title)), self), \n",
    "            \"edition\": get_edition_string(self.edition), \n",
    "            \"book\": handle_html_tags(self.book.title, self.book, set_non_italic=True), \n",
    "            \"book_ref\": book_ref, \n",
    "            \"book_editor\": book_editors_or_authors_string, \n",
    "            \"year\": self.year, \n",
    "            \"pages\": pages_string, \n",
    "            \"publisher\": latex_template_chapter_publisher%(self.book.publisher) if self.book.publisher else \"\",\n",
    "            \"link\": generate_link_latex(self.isiscb_id), \n",
    "            \"description\": latex_template_description%(self.description) if self.description else \"\"\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_thesis = \"\"\"\n",
    " \n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(counter)s}\\\\hspace{0.5em} %(author)s``%(title)s''%(edition)s Dissertation at %(school)s  (%(year)s).%(publisher)s%(advisor)s %(link)s.\n",
    "\n",
    "%(description)s\n",
    "\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "latex_template_thesis_advisor = \" Advisor(s): %s.\"\n",
    "latex_template_thesis_publisher = \" %s.\"\n",
    "\n",
    "class Thesis(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.isbn = ''\n",
    "        self.school = ''\n",
    "        self.advisors = []\n",
    "        \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in self.authors])\n",
    "    \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = []\n",
    "        for a in self.authors:\n",
    "            name = a.name\n",
    "            last, first = get_last_first(name)\n",
    "            if first:\n",
    "                authors_latex.append(author_template%(last, first))\n",
    "            else:\n",
    "                authors_latex.append(author_template_organization%(last))\n",
    "            \n",
    "        advisors_latex = \"\"\n",
    "        advisors_list = []\n",
    "        if self.advisors:\n",
    "            for a in self.advisors:\n",
    "                name = a.name\n",
    "                if ',' in name:\n",
    "                    last = name[:name.index(',')]\n",
    "                    first = name[name.index(',')+1:]\n",
    "                else:\n",
    "                    last = name[name.rfind(\" \"):].strip()\n",
    "                    first = name[:name.rfind(\" \")].strip()\n",
    "                advisors_list.append(author_template%(last, first))\n",
    "            advisors_latex = latex_template_thesis_advisor%(\", \".join(advisors_list))\n",
    "\n",
    "        return latex_template_thesis%{\n",
    "            \"counter\":counter, \n",
    "            \"author\":prepare_author_editor_string(authors_latex), \n",
    "            \"title\":handle_html_tags(add_space_before_quotes(prepare_title(handle_special_chars(self.title))), self), \n",
    "            \"edition\": get_edition_string(self.edition), \n",
    "            \"school\":self.school, \n",
    "            \"year\":self.year, \n",
    "            \"advisor\":advisors_latex, \n",
    "            \"link\":generate_link_latex(self.isiscb_id), \n",
    "            \"publisher\": latex_template_thesis_publisher%(self.publisher) if self.publisher else \"\", \n",
    "            \"description\": latex_template_description%(self.description) if self.description else \"\"\n",
    "        }\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_media = \"\"\"\n",
    "\n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%s}\\\\hspace{0.5em} %s``%s''%s%s (%s). %s.\n",
    "\n",
    "%s\n",
    "\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "latex_template_media_publisher = \" %s.\"\n",
    "\n",
    "class Media(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.publisher = None\n",
    "        \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in self.authors])\n",
    "    \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = []\n",
    "        for a in self.authors:\n",
    "            name = a.name\n",
    "            last, first = get_last_first(name)\n",
    "            if first:\n",
    "                authors_latex.append(author_template%(last, first))\n",
    "            else:\n",
    "                authors_latex.append(author_template_organization%(last))\n",
    "            \n",
    "        publisher_string = latex_template_media_publisher%(self.publisher) if self.publisher else \"\"\n",
    "\n",
    "        return latex_template_media%(counter, prepare_author_editor_string(authors_latex), handle_html_tags(add_space_before_quotes(prepare_title(handle_special_chars(self.title))), self), get_edition_string(self.edition), publisher_string, self.year, generate_link_latex(self.isiscb_id), latex_template_description%(self.description) if self.description else \"\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_essay_review = \"\"\"\n",
    "\n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(counter)s}\\\\hspace{0.5em} %(author)s``%(title)s''%(edition)s \\\\textit{%(journal)s} %(volume)s%(issue)s (%(year)s)%(pages)s.%(publisher)s %(link)s.\n",
    "\n",
    "%(description)s\n",
    "\n",
    "\\\\begin{isisdescription}Essay review of %(book)s\\\\end{isisdescription}\n",
    "\n",
    "\\\\vspace{2ex} \\\\end{footnotesize}\n",
    "\\\\paragraph{}\n",
    "\n",
    "\"\"\"\n",
    "latex_template_essay_review_publisher = \" %s.\"\n",
    "\n",
    "volume_template = \" %s\"\n",
    "issue_template = \", no. %s\"\n",
    "pages_template = \": %s\"\n",
    "ref_template = \" [ref. %s]\"\n",
    "essay_review_entry_template = \"%s \\\\emph{%s}  (%s)%s\"\n",
    "\n",
    "class EssayReview(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.journal = ''\n",
    "        self.journal_abbr = ''\n",
    "        self.volume = ''\n",
    "        self.issue = ''\n",
    "        self.pages = ''\n",
    "        self.books = []\n",
    "        self.reviewed_book_id = ''\n",
    "        self.active = True\n",
    "     \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors\n",
    "    \n",
    "    def get_sort_by_value(self):\n",
    "        return \"; \".join([a.get_name_for_sort() for a in self.authors])\n",
    "   \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = []\n",
    "        for a in self.authors:\n",
    "            name = a.name\n",
    "            last, first = get_last_first(name)\n",
    "            if first:\n",
    "                authors_latex.append(author_template%(last, first))\n",
    "            else:\n",
    "                authors_latex.append(author_template_organization%(last))\n",
    "\n",
    "        vol = ''\n",
    "        if self.volume:\n",
    "            vol = volume_template%(self.volume)\n",
    "        issue = ''\n",
    "        if self.issue:\n",
    "            issue = issue_template%(self.issue)\n",
    "        pages = ''\n",
    "        if self.pages:\n",
    "            pages = pages_template%(self.pages)\n",
    "            \n",
    "        books_info = []\n",
    "        for book in self.books:\n",
    "            book_authors_latex = build_persons_latex_list(book.authors)\n",
    "            book_editors_latex = build_persons_latex_list(book.editors) if type(book) == Book else \"\"\n",
    "\n",
    "            book_author_editor_string = \"\"\n",
    "            book_editors_string = \"\"\n",
    "            if book_editors_latex:\n",
    "                eds_suffix = \" (Eds.) \"\n",
    "                if len(book_editors_latex) == 1:\n",
    "                    eds_suffix = \" (Ed.) \"\n",
    "\n",
    "            if book_authors_latex:\n",
    "                book_author_editor_string = prepare_author_editor_string(book_authors_latex)\n",
    "                if book_editors_latex:\n",
    "                    book_editors_string = \", \" + prepare_author_editor_string(book_editors_latex) + eds_suffix\n",
    "            elif book_editors_latex:\n",
    "                book_author_editor_string = prepare_author_editor_string(book_editors_latex) + eds_suffix\n",
    "\n",
    "            ref_string = \"\"\n",
    "            if book.index:\n",
    "                ref_string = ref_template%(book.index)\n",
    "                \n",
    "            books_info.append(essay_review_entry_template%(book_author_editor_string, handle_html_tags(add_space_before_quotes(handle_special_chars(book.title)), book, set_non_italic=True), book.year, ref_string))\n",
    "            \n",
    "            \n",
    "        journal_text = self.journal_abbr if self.journal_abbr.strip() else self.journal\n",
    "        if not journal_text:\n",
    "            errors.append(DocumentError(self, \"No Journal abbreviation or title found.\", \"ERROR\"))\n",
    "        \n",
    "        return latex_template_essay_review%{\n",
    "            \"counter\": counter, \n",
    "            \"author\": prepare_author_editor_string(authors_latex), \n",
    "            \"title\": handle_html_tags(add_space_before_quotes(prepare_title(handle_special_chars(self.title))), self), \n",
    "            \"edition\": get_edition_string(self.edition), \n",
    "            \"journal\": journal_text, \n",
    "            \"volume\": vol, \n",
    "            \"issue\": issue, \n",
    "            \"year\": self.year, \n",
    "            \"pages\": pages, \n",
    "            \"publisher\": latex_template_essay_review_publisher%(self.publisher) if self.publisher else \"\",\n",
    "            \"link\": generate_link_latex(self.isiscb_id), \n",
    "            \"description\": latex_template_description%(self.description) if self.description else \"\", \n",
    "            \"book\": \"; \".join(books_info)\n",
    "        }\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_template_book_review=\"\"\"\n",
    "\\\\noindent\\\\begin{footnotesize}\\\\textbf{%(index)s}\\\\hspace{0.5em}%(authors)s \\\\textit{%(title)s}. %(year)s.%(ref)s %(link)s\n",
    "\n",
    "%(desc)s\n",
    "\n",
    "\\\\begin{isisdescription}%(reviews)s \\\\end{isisdescription}\n",
    "\n",
    " \\\\end{footnotesize}\n",
    "\n",
    "\\\\vspace{0.75ex}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "latex_book_review = \"%(author)s \\\\textit{%(journal)s}  %(vol)s (%(year)s)%(pages)s\"\n",
    "book_review_ref_template = \" [ref. %s]\"\n",
    "template_book_review_publisher = \" %s.\"\n",
    "\n",
    "class BookReview(Document):\n",
    "    def __init__(self):\n",
    "        Document.__init__(self)\n",
    "        self.authors = []\n",
    "        self.journal = ''\n",
    "        self.journal_abbr = ''\n",
    "        self.volume = ''\n",
    "        self.issue = ''\n",
    "        self.pages = ''\n",
    "        self.reviewed_book_id = ''\n",
    "        self.active = True\n",
    "        \n",
    "    def get_contributors_to_index(self):\n",
    "        return self.authors\n",
    "        \n",
    "class BookReviews():\n",
    "    def __init__(self):\n",
    "        self.book = None\n",
    "        self.reviews = []\n",
    "        self.edition = \"\"\n",
    "        self.description = \"\"\n",
    "        \n",
    "    def get_sort_by_value(self):\n",
    "        return self.book.get_sort_by_value()\n",
    "   \n",
    "    def get_editor_string(self):\n",
    "        editors_latex = []\n",
    "        if self.book.editors:\n",
    "            for e in self.book.editors:\n",
    "                last, first = get_last_first(e.name)\n",
    "                if last and first:\n",
    "                    editors_latex.append(author_template%(last, first))\n",
    "                else:\n",
    "                    editors_latex.append(author_template_organization%(last))\n",
    "        eds_suffix = \"\"\n",
    "        if editors_latex:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "                \n",
    "        return prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "        \n",
    "    def render(self, counter, reviews=None):\n",
    "        authors_latex = build_persons_latex_list(self.book.authors)\n",
    "        editors_latex = build_persons_latex_list(self.book.editors) if type(self.book) == Book else \"\"\n",
    "        \n",
    "        author_editor_string = \"\"\n",
    "        editors_string = \"\"\n",
    "        if editors_latex:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "                    \n",
    "        if authors_latex:\n",
    "            author_editor_string = prepare_author_editor_string(authors_latex)\n",
    "            if editors_latex:\n",
    "                editors_string = \", \" + prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "        elif editors_latex:\n",
    "            author_editor_string = prepare_author_editor_string(editors_latex) + eds_suffix\n",
    "\n",
    "        review_latexes = []\n",
    "        for review in self.reviews:\n",
    "            review_authors_latex = []\n",
    "            for a in review.authors:\n",
    "                name = a.name\n",
    "                last, first = get_last_first(name)\n",
    "                if first:\n",
    "                    review_authors_latex.append(author_template%(last, first))\n",
    "                else:\n",
    "                    review_authors_latex.append(author_template_organization%(last))\n",
    "            \n",
    "            vol = ''\n",
    "            if review.volume:\n",
    "                vol = volume_template%(review.volume)\n",
    "        \n",
    "            pages = ''\n",
    "            if review.pages:\n",
    "                pages = pages_template%(review.pages)\n",
    "                \n",
    "            journal_text = review.journal_abbr if review.journal_abbr.strip() else review.journal\n",
    "            if not journal_text:\n",
    "                errors.append(DocumentError(review, \"No Journal abbreviation or title found.\", \"ERROR\"))\n",
    "        \n",
    "            review_latexes.append(latex_book_review%{\"author\":prepare_author_editor_string(review_authors_latex), \"journal\":journal_text, \"vol\":vol, \"year\":review.year, \"pages\":pages})\n",
    "            \n",
    "        book_authors_latex = build_persons_latex_list(self.book.authors)\n",
    "        book_editors_latex = build_persons_latex_list(self.book.editors) if type(self.book) == Book else \"\"\n",
    "        \n",
    "        book_author_editor_string = \"\"\n",
    "        book_editors_string = \"\"\n",
    "        if book_editors_latex:\n",
    "            eds_suffix = \" (Eds.) \"\n",
    "            if len(book_editors_latex) == 1:\n",
    "                eds_suffix = \" (Ed.) \"\n",
    "                    \n",
    "        if book_authors_latex:\n",
    "            book_author_editor_string = prepare_author_editor_string(book_authors_latex)\n",
    "            if book_editors_latex:\n",
    "                book_editors_string = \", \" + prepare_author_editor_string(book_editors_latex) + eds_suffix\n",
    "        elif book_editors_latex:\n",
    "            book_author_editor_string = prepare_author_editor_string(book_editors_latex) + eds_suffix\n",
    "            \n",
    "        book_ref_info = \"\"\n",
    "        if self.book.index:\n",
    "            book_ref_info = book_review_ref_template%(self.book.index)\n",
    "  \n",
    "        return latex_template_book_review%{\n",
    "            \"index\":counter, \n",
    "            \"authors\":book_author_editor_string, \n",
    "            \"title\":handle_html_tags(handle_special_chars(self.book.title), self.book, set_non_italic=True), \n",
    "            \"year\":self.book.year, \n",
    "            \"ref\":book_ref_info,\n",
    "            \"link\":generate_link_latex(self.book.isiscb_id) if IS_PROOF else \"\", \n",
    "            \"desc\":latex_template_description%(self.description) if self.description else \"\",\n",
    "            \"reviews\":\"; \".join(review_latexes)}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following cells contain function to create publication objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_author(author_string):\n",
    "    author = Author()\n",
    "    parts = author_string.split('||')\n",
    "    if len(parts) == 1:\n",
    "        return None\n",
    "    for part in parts:\n",
    "        part = part.strip()\n",
    "        if part.startswith('AuthorityName '):\n",
    "            author.name = handle_special_chars(part[len('AuthorityName '):].strip())\n",
    "            continue\n",
    "        if part.startswith('ACRDisplayOrder '):\n",
    "            author.order = part[len('ACRDisplayOrder '):].strip()\n",
    "        if part.startswith('AuthorityType '):\n",
    "            author.author_type = part[len('AuthorityType '):]\n",
    "    return author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_publisher(publisher_string):\n",
    "    if not publisher_string:\n",
    "        return \"\"\n",
    "    publishers = publisher_string.split('//')\n",
    "    for publisher in publishers:\n",
    "        authority_name = ''\n",
    "        authority_type = ''\n",
    "        parts = publisher.split('||')\n",
    "        for part in parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith('AuthorityName '):\n",
    "                authority_name = part[len('AuthorityName '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('ACRType '):\n",
    "                authority_type = part[len('ACRType '):].strip()\n",
    "        \n",
    "        if authority_type == 'Publisher':\n",
    "            return authority_name\n",
    "        \n",
    "    return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_journal_or_publisher(publisher_string):\n",
    "    if not publisher_string:\n",
    "        return \"\"\n",
    "    publishers = publisher_string.split('//')\n",
    "    journal_name = ''\n",
    "    publisher_name = ''\n",
    "    serial_pub = ''\n",
    "    for publisher in publishers:\n",
    "        authority_name = ''\n",
    "        authority_type = ''\n",
    "        parts = publisher.split('||')\n",
    "        for part in parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith('AuthorityName '):\n",
    "                authority_name = part[len('AuthorityName '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('ACRType '):\n",
    "                authority_type = part[len('ACRType '):].strip()\n",
    "        \n",
    "        if authority_type == 'Publisher':\n",
    "            publisher_name = authority_name\n",
    "        if authority_type == 'Periodical':\n",
    "            journal_name = authority_name\n",
    "        if authority_type == 'Serial Publication':\n",
    "            serial_pub = authority_name\n",
    "        \n",
    "    return journal_name if journal_name else serial_pub if serial_pub else publisher_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_journal(row, doc):\n",
    "    parts = row['Journal Link'].split(\" || \")\n",
    "    if len(parts) <= 1:\n",
    "        doc.journal = row['Journal Link'].strip()\n",
    "        return\n",
    "    for part in parts:\n",
    "        part = part.strip()\n",
    "        if part.startswith('AuthorityName '):\n",
    "            doc.journal = part[len('AuthorityName '):].strip().replace('&', '\\\\&')\n",
    "        if part.startswith('Abbreviation '):\n",
    "            doc.journal_abbr = part[len('Abbreviation '):].strip().replace('&', '\\\\&')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_subjects(row, doc):\n",
    "    subjects = row['Subjects'].split(\" // \")\n",
    "    for subject in subjects:\n",
    "        parts = subject.split(\" || \")\n",
    "        subject = Subject()\n",
    "        if len(parts) <= 1:\n",
    "            subject.name = row['Subjects'].strip()\n",
    "            doc.subjects.append(subject)\n",
    "            return\n",
    "        for part in parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith('AuthorityName '):\n",
    "                subject.name = part[len('AuthorityName '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('AuthorityType '):\n",
    "                subject.authority_type = part[len('AuthorityType '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('AuthorityID '):\n",
    "                subject.id = part[len('AuthorityID '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('ACRNameForDisplayInCitation '):\n",
    "                subject.display_name = part[len('ACRNameForDisplayInCitation '):].strip().replace('&', '\\\\&')\n",
    "            if part.startswith('ACRType '):\n",
    "                subject.acr_type = part[len('ACRType '):].strip().replace('&', '\\\\&')\n",
    "        if subject.acr_type == \"Subject\":\n",
    "            doc.subjects.append(subject)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_pages(row):\n",
    "    pages = row['Pages Free Text']\n",
    "    if pages:\n",
    "        pages = pages[:pages.index(\"(From\")].strip() if \"(From\" in pages else \"\"\n",
    "        pages = pages.replace(\"&\", \"\\&\")\n",
    "    return pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_volume(row):\n",
    "    vol = row['Journal Volume']\n",
    "    if vol:\n",
    "        vol = vol[:vol.index(\"(From\")].strip() if \"(From\" in vol else \"\"\n",
    "        vol = vol.replace(\"&\", \"\\&\")\n",
    "    return vol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_issue(row):\n",
    "    issue = row['Journal Issue']\n",
    "    if issue:\n",
    "        issue = issue[:issue.index(\"(From\")].strip() if \"(From\" in issue else \"\"\n",
    "        issue = issue.replace(\"&\", \"\\&\")\n",
    "    return issue    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_book(row):\n",
    "    book = Book()\n",
    "    book.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        if a.strip():\n",
    "            author = create_author(a)\n",
    "            if author:\n",
    "                book.authors.append(author)\n",
    "        \n",
    "    def get_author_order(author):\n",
    "        return author.get_order()\n",
    "    book.authors.sort(key=get_author_order)\n",
    "        \n",
    "    editor_strings = row['Editor'].split('//')\n",
    "    for e in editor_strings:\n",
    "        editor = create_author(e)\n",
    "        if editor:\n",
    "            book.editors.append(editor)\n",
    "    \n",
    "    book.title = row['Title'].replace('&', '\\\\&')\n",
    "    book.year = row['Year of publication']\n",
    "    book.publisher = get_publisher(row['Related Authorities'])\n",
    "    book.isbn = row['ISBN']\n",
    "    \n",
    "    related_citations = row['Related Citations'].split('//')\n",
    "    for rel_cit in related_citations:\n",
    "        if rel_cit.strip():\n",
    "            cit_info = rel_cit.split(\"||\")\n",
    "            cit_id = ''\n",
    "            type_is_chapter = False\n",
    "            for info in cit_info:\n",
    "                info = info.strip()\n",
    "                if info.startswith(\"CitationID \"):\n",
    "                    cit_id = info[len(\"CitationID \"):].strip()\n",
    "                if info.startswith(\"CCRType  \"):\n",
    "                    ccrType = info[len(\"CCRType \"):].strip()\n",
    "                    if ccrType.strip() == \"Includes Chapter\":\n",
    "                        type_is_chapter = True\n",
    "                    \n",
    "            if type_is_chapter and cit_id:\n",
    "                book.chapter_ids.append(cit_id)                    \n",
    "    \n",
    "    return book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_chapter(row):\n",
    "    chapter = Chapter()\n",
    "    chapter.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        author = create_author(a)\n",
    "        if author:\n",
    "            chapter.authors.append(author)\n",
    "        \n",
    "    def get_order(author):\n",
    "        return author.order\n",
    "    chapter.authors.sort(key=get_order)\n",
    "    \n",
    "    chapter.title = row['Title'].replace('&', '\\\\&')\n",
    "    chapter.year = row['Year of publication']\n",
    "    \n",
    "    editor_strings = row['Editor'].split('//')\n",
    "    for e in editor_strings:\n",
    "        editor = create_author(e)\n",
    "        if editor:\n",
    "            chapter.editors.append(editor)\n",
    "    \n",
    "    related_citations = row['Related Citations'].split(\"//\")\n",
    "    for cit in related_citations:\n",
    "        cit_parts = cit.split(\"||\")\n",
    "        book_title = \"\"\n",
    "        book_id = \"\"\n",
    "        is_book_info = True\n",
    "        for part in cit_parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith(\"CCRType \"):\n",
    "                ccrType = part[len(\"CCRType \"):]\n",
    "                if ccrType.strip() != \"Includes Chapter\":\n",
    "                    is_book_info = False\n",
    "            if part.startswith(\"CitationTitle \"):\n",
    "                book_title = part[len(\"CitationTitle \"):]\n",
    "            if part.startswith(\"CitationID \"):\n",
    "                book_id = part[len(\"CitationID \"):].strip()\n",
    "                \n",
    "        if is_book_info:\n",
    "            chapter.book = Book()\n",
    "            chapter.book.title = book_title\n",
    "            chapter.book.isiscb_id = book_id\n",
    "            break;\n",
    "        \n",
    "    chapter.pages = parse_pages(row)\n",
    "    \n",
    "    return chapter        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_article(row):\n",
    "    article = Article()\n",
    "    \n",
    "    article.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        author = create_author(a)\n",
    "        if author:\n",
    "            article.authors.append(author)\n",
    "    \n",
    "    article.title = row['Title'].replace('&', '\\\\&')\n",
    "    article.year = row['Year of publication']\n",
    "    article.publisher = get_publisher(row['Related Authorities'])\n",
    "    parse_journal(row, article)\n",
    "    \n",
    "    article.volume = parse_volume(row)\n",
    "    \n",
    "    article.issue = parse_issue(row)\n",
    "    \n",
    "    article.pages = parse_pages(row)\n",
    "    return article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_thesis(row):\n",
    "    thesis = Thesis()\n",
    "    \n",
    "    thesis.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        author = create_author(a)\n",
    "        if author:\n",
    "            thesis.authors.append(author)\n",
    "            \n",
    "    advisor_strings = row['Advisor'].split('//')\n",
    "    for a in advisor_strings:\n",
    "        advisor = create_author(a)\n",
    "        if advisor:\n",
    "            thesis.advisors.append(advisor)\n",
    "    \n",
    "    thesis.title = row['Title'].replace('&', '\\\\&')\n",
    "    thesis.year = row['Year of publication']\n",
    "    \n",
    "    thesis.publisher = get_publisher(row['Related Authorities'])\n",
    "    \n",
    "    school_string = row['School']\n",
    "    for part in school_string.split('||'):\n",
    "        if part.strip().startswith('AuthorityName'):\n",
    "            thesis.school = part[len(\"AuthorityName \"):].strip()\n",
    "            \n",
    "    return thesis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_media(row):\n",
    "    media = Media()\n",
    "    media.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        author = create_author(a)\n",
    "        if author:\n",
    "            media.authors.append(author)\n",
    "   \n",
    "    media.title = row['Title'].replace('&', '\\\\&')\n",
    "    media.year = row['Year of publication']\n",
    "    \n",
    "    media.publisher = get_journal_or_publisher(row['Related Authorities'])\n",
    "    \n",
    "    return media"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_essay_review(row):\n",
    "    essay_review = EssayReview()\n",
    "    \n",
    "    essay_review.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        author = create_author(a)\n",
    "        if author:\n",
    "            essay_review.authors.append(author)\n",
    "    \n",
    "    essay_review.title = row['Title'].replace('&', '\\\\&')\n",
    "    essay_review.year = row['Year of publication']\n",
    "    essay_review.publisher = get_publisher(row['Place Publisher'])\n",
    "    parse_journal(row, essay_review)\n",
    "    \n",
    "    essay_review.volume = parse_volume(row)\n",
    "    essay_review.issue  = parse_issue(row)\n",
    "    essay_review.pages = parse_pages(row)\n",
    "         \n",
    "    related_citations = row['Related Citations'].split(\"//\")\n",
    "    for cit in related_citations:\n",
    "        cit_parts = cit.split(\"||\")\n",
    "        book_title = \"\"\n",
    "        book_id = \"\"\n",
    "        is_book_info = True\n",
    "        for part in cit_parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith(\"CCRType \"):\n",
    "                ccrType = part[len(\"CCRType \"):]\n",
    "                if ccrType.strip() != \"Is Reviewed By\":\n",
    "                    is_book_info = False\n",
    "            if part.startswith(\"CitationTitle \"):\n",
    "                book_title = part[len(\"CitationTitle \"):]\n",
    "            if part.startswith(\"CitationID \"):\n",
    "                book_id = part[len(\"CitationID \"):].strip()\n",
    "                \n",
    "        if is_book_info:\n",
    "            book = Book()\n",
    "            book.title = book_title\n",
    "            book.isiscb_id = book_id\n",
    "            essay_review.books.append(book)\n",
    "            \n",
    "    return essay_review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_book_review(row):\n",
    "    book_review = BookReview()\n",
    "    \n",
    "    book_review.isiscb_id = row['Record ID']\n",
    "    \n",
    "    author_strings = row['Author'].split('//')\n",
    "    for a in author_strings:\n",
    "        if a.strip():\n",
    "            author = create_author(a)\n",
    "            if author:\n",
    "                book_review.authors.append(author)\n",
    "        \n",
    "    def get_order(author):\n",
    "        return author.order\n",
    "    book_review.authors.sort(key=get_order)\n",
    "    \n",
    "    book_review.year = row['Year of publication']\n",
    "    parse_journal(row, book_review)\n",
    "    \n",
    "    book_review.volume = parse_volume(row)\n",
    "    book_review.issue = parse_issue(row)\n",
    "    book_review.pages = parse_pages(row)\n",
    "        \n",
    "    status = row['Record Nature']\n",
    "    if not status.startswith('Active'):\n",
    "        book_review.active = False\n",
    "        \n",
    "    related_citations = row['Related Citations'].split(\"//\")\n",
    "    for cit in related_citations:\n",
    "        cit_parts = cit.split(\"||\")\n",
    "        book_title = \"\"\n",
    "        book_id = \"\"\n",
    "        is_book_info = True\n",
    "        for part in cit_parts:\n",
    "            part = part.strip()\n",
    "            if part.startswith(\"CCRType \"):\n",
    "                ccrType = part[len(\"CCRType \"):]\n",
    "                if ccrType.strip() != \"Is Reviewed By\":\n",
    "                    is_book_info = False\n",
    "            if part.startswith(\"CitationTitle \"):\n",
    "                book_title = part[len(\"CitationTitle \"):]\n",
    "            if part.startswith(\"CitationID \"):\n",
    "                book_id = part[len(\"CitationID \"):].strip()\n",
    "                \n",
    "        if is_book_info:\n",
    "            book_review.reviewed_book_id = book_id\n",
    "            break;\n",
    "            \n",
    "    return book_review"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following cells deal with the classifications."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classification():\n",
    "    def __init__(self):\n",
    "        self.code = ''\n",
    "        self.code_print = ''\n",
    "        self.code_for_sorting = -1\n",
    "        self.name = ''\n",
    "        self.code_main = -1\n",
    "        self.code_sub = -1\n",
    "        self.acr_type = ''\n",
    "        self.authority_type = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_classification_object(class_code, doc):\n",
    "    classi = Classification()\n",
    "    \n",
    "    for part in class_code.split('||'):\n",
    "        if part.strip().startswith('ClassificationCode'):\n",
    "            code_parts = part[len(\"ClassificationCode \"):].strip().split('-')\n",
    "            classi.code = part[len(\"ClassificationCode \"):].strip()\n",
    "            while \"\" in code_parts: code_parts.remove(\"\")\n",
    "            try:\n",
    "                if len(code_parts) == 1:\n",
    "                    classi.code_for_sorting = int(code_parts[0].strip())/1000\n",
    "                    classi.code_print = code_parts[0].strip()\n",
    "                    classi.code_main = int(code_parts[0].strip())\n",
    "                    classi.code_sub = 0\n",
    "                if len(code_parts) == 2:\n",
    "                    classi.code_for_sorting = float(code_parts[1].strip()) + float(code_parts[0].strip())/1000\n",
    "                    classi.code_print = code_parts[1].strip() + \"-\" + code_parts[0].strip()\n",
    "                    classi.code_main = int(code_parts[1].strip())\n",
    "                    classi.code_sub = int(code_parts[0].strip())\n",
    "            except Exception:\n",
    "                print(\"Category code not valid \" + classi.code)\n",
    "                doc.category_number = \"\"\n",
    "                return None\n",
    "\n",
    "        if part.strip().startswith('AuthorityName'):\n",
    "            classi.name = part[len(\"AuthorityName \"):].strip()\n",
    "            \n",
    "        if part.strip().startswith('ACRType'):\n",
    "            classi.acr_type = part[len(\"ACRType \"):].strip()\n",
    "            \n",
    "        if part.strip().startswith('AuthorityType'):\n",
    "            classi.authority_type = part[len(\"AuthorityType \"):].strip()\n",
    "    \n",
    "    if (classi.acr_type != 'Category' or classi.authority_type != 'Classification Term'):\n",
    "        print(\"Skipping classification due to wrong type %s, %s.\"%(classi.acr_type, classi.authority_type))\n",
    "        return None\n",
    "    \n",
    "    return classi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main\n",
    "\n",
    "Below is the main code that reads and writes files and generates the bilbiography."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decide which type of doc to build:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_doc(row):\n",
    "    doc = None\n",
    "    if row['Record Type'] == \"Book\":\n",
    "        doc = build_book(row)\n",
    "    elif row['Record Type'] == \"Article\":\n",
    "        doc = build_article(row)\n",
    "    elif row['Record Type'] == \"Chapter\":\n",
    "        doc = build_chapter(row)\n",
    "    elif row['Record Type'] == \"Thesis\":\n",
    "        doc = build_thesis(row)\n",
    "    elif row['Record Type'] == \"Multimedia Object\":\n",
    "        doc = build_media(row)\n",
    "    elif row['Record Type'] == \"Essay Review\":\n",
    "        doc = build_essay_review(row)\n",
    "    elif row['Record Type'] == \"Review\":\n",
    "        doc = build_book_review(row)\n",
    "    \n",
    "    if doc:\n",
    "        parse_subjects(row, doc)\n",
    "    return doc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set basic info:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_basic_info(doc, row):\n",
    "    doc.database = row['Dataset']\n",
    "    doc.category_number = row['CategoryNumbers'].strip()\n",
    "    doc.print_date = row['Published Print'].strip()\n",
    "    doc.edition = row['Edition Details']\n",
    "    doc.edition = doc.edition.replace(\"<em>\", \"\\emph{\").replace(\"</em>\", \"}\") if doc.edition else \"\"\n",
    "    doc.description = row['Description']\n",
    "    doc.description = doc.description.replace(\"<em>\", \"\\emph{\").replace(\"</em>\", \"}\") if doc.description else \"\"\n",
    "    doc.description = doc.description.replace(\"&\", \"\\&\") if doc.description else \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_classifications(row, doc, classifications_by_nr, docs_by_classification):\n",
    "    class_codes = row['CategoryNumbers'].split('//')\n",
    "    doc.classifications = []\n",
    "    all_classies = []\n",
    "    \n",
    "    if IGNORE_CATEGORIES:\n",
    "        classi = Classification()\n",
    "        classi.code_for_sorting = 0\n",
    "        classi.code_print = \"0\"\n",
    "        classi.code_main = 0\n",
    "        classi.code_sub = 0\n",
    "        classi.acr_type = 'Category' \n",
    "        classi.authority_type = 'Classification Term'\n",
    "        doc.category_number = \"0\"\n",
    "        all_classies.append(classi)\n",
    "    else:\n",
    "        # create all classifications\n",
    "        for class_code in class_codes:\n",
    "            classi = create_classification_object(class_code, doc)\n",
    "            if classi:\n",
    "                all_classies.append(classi)\n",
    "    # if there are more than one, we have an error\n",
    "    if len(all_classies) > 1:\n",
    "        errors.append(DocumentError(doc, \"There are more than two classifications.\", \"ERROR\"))\n",
    "    # otherwise append classification to doc\n",
    "    elif len(all_classies) == 1:\n",
    "        classi = all_classies[0]\n",
    "        if classi.code_print.strip():\n",
    "            doc.classifications.append(classi)\n",
    "            if classi.code_for_sorting not in classifications_by_nr:\n",
    "                classifications_by_nr[classi.code_for_sorting] = classi\n",
    "            if classi.code_for_sorting not in docs_by_classification:\n",
    "                docs_by_classification[classi.code_for_sorting] = []\n",
    "            docs_by_classification[classi.code_for_sorting].append(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to check if document should be included based on database:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def include_based_on_database(doc):\n",
    "    if not IGNORE_DATASET:\n",
    "        return doc.database.strip() == DATABASE_TO_PRINT\n",
    "    else:\n",
    "        return True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read csv file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_csv(export_file, docs, docs_by_id, classifications_by_nr, docs_by_classification, reviews_by_book_id):\n",
    "    with open(export_file, encoding='utf-8') as csvfile:\n",
    "        reader = csv.DictReader(csvfile)\n",
    "\n",
    "        for row in reader:\n",
    "            doc = create_doc(row)\n",
    "\n",
    "            if doc and not doc.isiscb_id in docs_by_id:\n",
    "                set_basic_info(doc, row)\n",
    "\n",
    "                if type(doc) not in [BookReview]: \n",
    "                    docs.append(doc)\n",
    "                    docs_by_id[doc.isiscb_id] = doc\n",
    "\n",
    "                    if include_based_on_database(doc): \n",
    "                        # classification\n",
    "                        handle_classifications(row, doc, classifications_by_nr, docs_by_classification) \n",
    "                else:\n",
    "                    reviews_list = reviews_by_book_id.get(doc.reviewed_book_id, None)\n",
    "                    if not reviews_list:\n",
    "                        reviews_by_book_id[doc.reviewed_book_id] = []\n",
    "\n",
    "                    if doc.active and include_based_on_print_date(doc.print_date) and include_based_on_database(doc):\n",
    "                        reviews_by_book_id[doc.reviewed_book_id].append(doc)\n",
    "                    elif not include_based_on_database(doc):\n",
    "                        errors.append(DocumentError(doc, \"Document has wrong database (%s). Skipping.\"%(doc.database), \"WARNING\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sort_by_value(doc):\n",
    "    return doc.get_sort_by_value()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create index for publications:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_index(docs_by_classification):\n",
    "    counter = 1\n",
    "    for classi_code in sorted(docs_by_classification):\n",
    "        sorted_docs = docs_by_classification[classi_code]\n",
    "        sorted_docs.sort(key=get_sort_by_value)\n",
    "        for doc in sorted_docs:\n",
    "            if doc.category_number and include_based_on_print_date(doc.print_date) and include_based_on_database(doc):\n",
    "                doc.index = counter\n",
    "                counter += 1\n",
    "            elif not include_based_on_database(doc):\n",
    "                errors.append(DocumentError(doc, \"Document has wrong database (%s). Skipping.\"%(doc.database), \"WARNING\"))\n",
    "        docs_by_classification[classi_code] = sorted_docs\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add books to reviews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_books_to_reviews(reviews_by_book_id, docs_by_id, all_book_reviews, book_reviews_obj_by_book_id):\n",
    "    for book_id in reviews_by_book_id:\n",
    "        if book_id in docs_by_id:\n",
    "            book = docs_by_id[book_id]\n",
    "\n",
    "            if book:\n",
    "                book_reviews = BookReviews()\n",
    "                book_reviews.book = book\n",
    "                book_reviews.reviews = reviews_by_book_id[book_id]\n",
    "                all_book_reviews.append(book_reviews)\n",
    "                book_reviews_obj_by_book_id[book.isiscb_id] = book_reviews"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create index for reviews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_index_for_reviews(all_book_reviews):\n",
    "    counter = 1\n",
    "    for doc in all_book_reviews:\n",
    "        doc.index = \"R\" + str(counter)\n",
    "        counter += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add books to index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_to_index(doc, author_sorted_index, author_index, review_idx = -1):\n",
    "    to_index = doc.get_contributors_to_index()\n",
    "    for author in to_index:\n",
    "        sort_name = author.get_name_for_sort()\n",
    "        display_name = \", \".join([author.get_last_name(), author.get_first_name()]) if author.get_first_name() else author.get_last_name()\n",
    "        if sort_name not in author_sorted_index:\n",
    "            author_sorted_index[sort_name] = display_name\n",
    "        if display_name not in author_index:\n",
    "            author_index[display_name] = []\n",
    "        # for reviews we need to add their review id (R..) not by the book id\n",
    "        author_index[display_name].append(str(doc.index) if review_idx == -1 else str(review_idx))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_to_subject_index(doc, subject_sorted_index, subject_index):\n",
    "    for subject in doc.subjects:\n",
    "        # we want to exclude predefined subjects\n",
    "        # print(\"subject: \" + subject.id)\n",
    "        if subject.id in SUBJECTS_TO_EXCLUDE:\n",
    "            continue\n",
    "        subject_sorted_index[unidecode(subject.name.lower())] = subject\n",
    "        if subject.name not in subject_index:\n",
    "            subject_index[subject.name] = []\n",
    "        subject_index[subject.name].append(str(doc.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print publications:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heading_main_section_latex_template = \"\"\"\n",
    "\\\\addtocontents{toc}{\\\\vspace{1em}}\\\\setcounter{section}{%s}\n",
    "\\\\section*{%s.\\\\hspace{0.5em}%s}\n",
    "\\\\markboth{%s}{%s}\n",
    "\\\\addcontentsline{toc}{section}{\\\\protect\\\\numberline{%s}\\\\textbf{%s}}\n",
    "\"\"\"\n",
    "    \n",
    "heading_latex_template = \"\"\"\n",
    "\\\\setcounter{subsection}{%s}\n",
    "\\\\vspace{-5pt}\\\\nopagebreak\\\\subsection*{%s.\\\\hspace{0.5em}%s}\n",
    "\\\\addcontentsline{toc}{subsection}{\\\\protect\\\\numberline{%s.}%s}\n",
    "\"\"\"\n",
    "\n",
    "def print_publications(result_file_path, docs_by_classification, classifications_by_nr, author_sorted_index, author_index, subject_sorted_index, subject_index):\n",
    "    with open(result_file_path, 'w', encoding='utf-8') as result_file:\n",
    "        # which one is the last main section seen\n",
    "        last_main_sec = -1\n",
    "        for classi_code in sorted(docs_by_classification):\n",
    "            classification = classifications_by_nr[classi_code]\n",
    "            docs_to_print = list(filter(lambda d : d.category_number and include_based_on_print_date(d.print_date) and d.index > 0, docs_by_classification[classi_code]))\n",
    "            #print(str(len(docs_to_print)) + \" in category \" + str(classi_code))\n",
    "            main_title = classification.name\n",
    "            sub_title = \"\"\n",
    "            if \" -- \" in main_title:\n",
    "                main_title = classification.name[classification.name.index(\" -- \")+4:]\n",
    "                sub_title = classification.name[:classification.name.index(\" -- \")]\n",
    "\n",
    "            if last_main_sec != classification.code_main and docs_to_print and not IGNORE_CATEGORIES:\n",
    "                main_sec_heading = heading_main_section_latex_template%(str(classification.code_main), str(classification.code_main), main_title, main_title, main_title, str(classification.code_main), main_title)\n",
    "                print(\"Writing main section: \" + main_title)\n",
    "                result_file.write(\"\\n\" + main_sec_heading + \"\\n\")\n",
    "                last_main_sec = classification.code_main\n",
    "\n",
    "            if sub_title and docs_to_print and not IGNORE_CATEGORIES:\n",
    "                heading = heading_latex_template%(str(classification.code_sub), classification.code_print, sub_title, str(classification.code_sub), sub_title)\n",
    "                print(\"Writing \" + sub_title)\n",
    "                result_file.write(heading + \"\\n\")\n",
    "\n",
    "            for doc in docs_by_classification[classi_code]:\n",
    "                if (type(doc) in [Chapter]) and doc.book.isiscb_id in docs_by_id:\n",
    "                    book = docs_by_id[doc.book.isiscb_id]\n",
    "                    if book:\n",
    "                        doc.book = book\n",
    "                if type(doc) == EssayReview:\n",
    "                    final_book_list = []\n",
    "                    for book in doc.books:\n",
    "                        if book.isiscb_id in docs_by_id:\n",
    "                            final_book_list.append(docs_by_id[book.isiscb_id])\n",
    "                        else:\n",
    "                            final_book_list.append(doc)\n",
    "                    doc.books = final_book_list\n",
    "\n",
    "                if type(doc) == Book and doc.chapter_ids:\n",
    "                    for chap_id in doc.chapter_ids:\n",
    "                        if chap_id in docs_by_id:\n",
    "                            doc.chapters.append(docs_by_id[chap_id])\n",
    "                # print all classified records (e.g., have Category Number) \n",
    "                # that have not been previously printed (e.g., have Published Print data).\n",
    "                if doc.category_number and include_based_on_print_date(doc.print_date) and doc.index > 0:\n",
    "                    print(\"Printing: \" + str(doc.index))\n",
    "                    reviews = []\n",
    "                    if doc.isiscb_id in book_reviews_obj_by_book_id:\n",
    "                        reviews = book_reviews_obj_by_book_id[doc.isiscb_id]\n",
    "                    line = doc.render(doc.index, reviews=reviews).strip() + \"\\n\"\n",
    "                    result_file.write(line)\n",
    "\n",
    "                    # add to author index\n",
    "                    add_to_index(doc, author_sorted_index, author_index)\n",
    "                    if type(doc) != EssayReview:\n",
    "                        add_to_subject_index(doc, subject_sorted_index, subject_index)\n",
    "                    if type(doc) == EssayReview:\n",
    "                        for book in doc.books:\n",
    "                            add_to_index(book, author_sorted_index, author_index, doc.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print reviews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_reviews(reviews_file_path, all_book_reviews, author_sorted_index, author_index):\n",
    "    with open(reviews_file_path, 'w', encoding='utf-8') as reviews_file:\n",
    "        for reviews in all_book_reviews:\n",
    "            print(\"Writing Reviews for\" + str(reviews.book.isiscb_id))\n",
    "            reviews_file.write(reviews.render(reviews.index).strip() + \"\\n\")\n",
    "\n",
    "            add_to_index(reviews.book, author_sorted_index, author_index, reviews.index)\n",
    "            # add to author index\n",
    "            for review in reviews.reviews:\n",
    "                add_to_index(review, author_sorted_index, author_index, reviews.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write errors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_errors(errors_file_path):\n",
    "    with open(errors_file_path, 'w', encoding='utf-8') as error_file:\n",
    "        error_file.write(\"Document Id, Type, Error Message\\n\")\n",
    "        for error in errors:\n",
    "            error_file.write(error.doc.isiscb_id + \", \" + error.error_type + \", \" + error.error + \"\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print author index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_author_index(author_index_file_path, author_sorted_index, author_index):\n",
    "    index_template = \"\"\"\n",
    "    \\\\begin{cbindex}%s %s\\\\end{cbindex}\n",
    "    \"\"\"\n",
    "    with open(author_index_file_path, 'w', encoding=\"utf-8\") as index_file:\n",
    "        for author in sorted(author_sorted_index):\n",
    "            print(author)\n",
    "            line = index_template%(author_sorted_index[author], \", \".join(author_index[author_sorted_index[author]]))\n",
    "            index_file.write(line + \"\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print subject index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_subject_index(subject_index_file_path, subjects_sorted_index, subject_index):\n",
    "    index_template = \"\"\"\n",
    "    \\\\begin{cbindex}%s %s\\\\end{cbindex}\n",
    "    \"\"\"\n",
    "    with open(subject_index_file_path, 'w', encoding=\"utf-8\") as index_file:\n",
    "        for subject in sorted(subjects_sorted_index):\n",
    "            print(subject)\n",
    "            line = index_template%(subjects_sorted_index[subject].name, \", \".join(subject_index[subjects_sorted_index[subject].name]))\n",
    "            index_file.write(line + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = []\n",
    "docs_by_id = {}\n",
    "reviews_by_book_id = {}\n",
    "classifications_by_nr = {}\n",
    "docs_by_classification = {}\n",
    "\n",
    "# ------------ Reading publications ------------------\n",
    "read_csv(export_file, docs, docs_by_id, classifications_by_nr, docs_by_classification, reviews_by_book_id)\n",
    "create_index(docs_by_classification)                \n",
    "    \n",
    "\n",
    "# ------------- Dealing with reviews ----------------------\n",
    "all_book_reviews = []\n",
    "book_reviews_obj_by_book_id = {}\n",
    "\n",
    "add_books_to_reviews(reviews_by_book_id, docs_by_id, all_book_reviews, book_reviews_obj_by_book_id)\n",
    "\n",
    "all_book_reviews.sort(key=get_sort_by_value)\n",
    "create_index_for_reviews(all_book_reviews)\n",
    "\n",
    "author_sorted_index = {}\n",
    "author_index = {}\n",
    "\n",
    "# the sorted dictionary will be used to sort all subjects\n",
    "subject_sorted_index = {}\n",
    "# this index will be used to hold the publication numbers for a subjecg\n",
    "subject_index = {}\n",
    "\n",
    "# ---------------- Print publications -------------\n",
    "print_publications(result_file_path, docs_by_classification, classifications_by_nr, author_sorted_index, author_index, subject_sorted_index, subject_index)\n",
    "\n",
    "# ---------------- Print reviews ---------------\n",
    "print_reviews(reviews_file_path, all_book_reviews, author_sorted_index, author_index)\n",
    "        \n",
    "# ---------------- Print errors -----------------\n",
    "print_errors(errors_file_path)\n",
    "\n",
    "# ---------------- Print author index ---------------\n",
    "print_author_index(author_index_file_path, author_sorted_index, author_index)\n",
    "\n",
    "# ---------------- Print subject index --------------\n",
    "print_subject_index(subject_index_file_path, subject_sorted_index, subject_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}