update.py

"""
Update django's dictionary using official documentation's index

Set VERSION and FILENAME before running :

```sh
pip install -r requirements.txt
python update.py
```
"""

from bs4 import BeautifulSoup
import requests

BASE_URL = "https://docs.djangoproject.com/en/{}/genindex/"
VERSION = "4.2"  # update to match current stable version number
FILENAME = "src/django.txt"


def update():
    """
    Update FILENAME contents

    1. get https://docs.djangoproject.com/en/VERSION/genindex/ content
    2. extract all DT tags from the index
    3. cleanup (remove details, links, parenthesis, etc.)
    4. split dotted terms if needed (django.db -> django AND db)
    5. deduplicate & sort the list
    6. write each words in FILENAME
    """

    fullPage = requests.get(BASE_URL.format(VERSION)).text
    soup = BeautifulSoup(fullPage, "html.parser")
    data = soup.find("div", role="main")  # ignore the rest of the page

    # init with terms missing from index (mostly closing template tags)
    terms_list = [
        "elif",
        "empty",
        "endautoescape",
        "endblocktrans",
        "endcomment",
        "endfor",
        "endfilter",
        "endif",
        "endifequal",
        "endifnotequal",
        "endifchanged",
        "endspaceless",
        "endverbatim",
        "endwith",
        "openblock",
        "closeblock",
        "openvariable",
        "closevariable",
        "openbrace",
        "closebrace",
        "opencomment",
        "closecomment",
    ]

    for index_entry in data.find_all("dt"):
        if index_entry.a:
            entry = index_entry.a
        else:
            entry = index_entry
        try:
            # cleanup spaces & parenthesis
            term = entry.string.strip().split(" ", 1)[0].split("(", 1)[0]
        except AttributeError:
            continue
        if term.startswith("--"):  # ignore command line params
            continue
        for word in term.split("."):
            terms_list.append(word)

    with open(FILENAME, "w") as output:
        for word in sorted(list(set(terms_list))):
            output.write(word + "\n")


if __name__ == "__main__":
    update()