✨ Implement Notebook app (#65)

* ✨ Start notebooks app
* 🔧 Fix nullable fields for notebooks
* 🔧 Display user-friendly name for `Notebook`s
* 🔧 Allow filtering in the notebook admin page
* 🗒 Improve README
* 🗒 Improve README again
* ⬆ Add bluelib to the dependencies of the frontend
* 🧹 Prepare a good frontend base for development
* ✨ Port and improve useStorageState  
  Original: https://github.com/pds-nest/nest/blob/main/nest_frontend/hooks/useLocalStorageState.js
* 🧹 Remove React logo
* ⬆ Add `docker` to the dependencies
* ⬆ Add `axios` to the dependencies
* 🔨 Mark `src` as sources root
* ✨ Add API routes to view Notebooks
* 🔧 Use a router for the `by-project` route
* 🐛 Fix deletion failing on `SophonViewSet`
* 🔧 Abstract notebook methods
* ✨ Create a base docker client
* 🚧 Proof of concept for notebook starter
* 📔 Document the contents of the Django apps
* 🚧 Incomplete container implementation
* 🚧 Working container implementation
* 💥 Leftovers from an experiment
* ✨ Correct implementation of the proxy configuration
  (Apache config file is still missing)
* 💥 Improve code
* 💥 Improve more things
* 🔧 Remove duplicated `/project` in project app urls
* ✨ Add basic Apache proxy config file
* 🔧 User should have sudo access on the notebook
* ✨ Implement the Internet access field (currently ignored)
* 🧹 Cleanup code

.gitignore

@@ -0,0 +1 @@
+proxy.dbm

README.md

@@ -15,12 +15,12 @@ The project consists of a **single-page-app with React** on the frontend and a *
 
 Development progress is tracked on [issue #20](https://github.com/Steffo99/sophon/issues/20).
 
-Development is currently focusing on the **backend**.
-
 ### Tools
 
 Sophon is being developed using [IntelliJ IDEA Ultimate](https://www.jetbrains.com/idea/): its metadata is included in the `.idea` directory so that the code style and tools are consistent across all machines used during the development.
 
+Run configurations for *running the backend*, *testing the backend* and *running the frontend* are included.
+
 ### Commits
 
 Commits names are prefixed with a variant of [Gitmoji](https://gitmoji.dev/) which follows roughly this legend:

backend/pyproject.toml

@@ -16,6 +16,7 @@ pydantic = "~1.7.3"
 django-pam = "^2.0.0"
 django-colorfield = "^0.4.2"
 deprecation = "^2.1.0"
+docker = "^5.0.2"
 
 [tool.poetry.dev-dependencies]
 

backend/sophon/core/__init__.py

@@ -0,0 +1,3 @@
+"""
+The :mod:`sophon.core` module provides the base Sophon functionality, such as users and research groups.
+"""

backend/sophon/core/views.py

@@ -141,7 +141,7 @@ def destroy(self, request, *args, **kwargs):
         except HTTPException as e:
             return e.as_response()
 
-        self.perform_destroy(instance)
+        instance.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def get_serializer_class(self):

backend/sophon/notebooks/__init__.py

@@ -0,0 +1,21 @@
+"""
+The :mod:`sophon.notebooks` module provides the JupyterLab connection and functionality.
+
+It depends on the following :mod:`django` apps:
+
+- `sophon.core`
+- `sophon.projects`
+
+It can be configured with the following :data:`os.environ` keys:
+
+- ``DOCKER_HOST``: The URL to the Docker host.
+- ``DOCKER_TLS_VERIFY``: Verify the host against a CA certificate.
+- ``DOCKER_CERT_PATH``: A path to a directory containing TLS certificates to use when connecting to the Docker host.
+- ``APACHE_PROXY_EXPRESS_DBM``: The filename of the ``proxy_express`` DBM file to write to.
+- ``APACHE_PROXY_BASE_DOMAIN``: The base domain for virtualhost reverse proxying.
+- ``APACHE_PROXY_HTTP_PROTOCOL``: The http_protocol that Apache uses to make the containers available to the public.
+- ``APACHE_PROXY_WS_PROTOCOL``: The http_protocol that Apache uses to make the Jupyter websocket available to the public.
+- ``SOPHON_CONTAINER_PREFIX``: The prefix added to the names of notebooks' containers will have.
+- ``SOPHON_VOLUME_PREFIX``: The prefix added to the names of notebooks' volumes will have.
+- ``SOPHON_NETWORK_PREFIX``: The prefix added to the names of notebooks' volumes will have.
+"""

backend/sophon/notebooks/admin.py

@@ -0,0 +1,61 @@
+from django.contrib import admin
+
+from sophon.core.admin import SophonAdmin
+from . import models
+
+
+@admin.register(models.Notebook)
+class NotebookAdmin(SophonAdmin):
+    list_display = (
+        "slug",
+        "name",
+        "project",
+        "locked_by",
+        "container_image",
+        "container_id",
+        "port",
+    )
+
+    list_filter = (
+        "container_image",
+    )
+
+    ordering = (
+        "slug",
+    )
+
+    fieldsets = (
+        (
+            None, {
+                "fields": (
+                    "slug",
+                    "name",
+                    "project",
+                    "locked_by",
+                ),
+            },
+        ),
+        (
+            "Docker", {
+                "fields": (
+                    "container_image",
+                    "container_id",
+                    "internet_access",
+                ),
+            },
+        ),
+        (
+            "Proxy", {
+                "fields": (
+                    "port",
+                ),
+            },
+        ),
+        (
+            "Jupyter", {
+                "fields": (
+                    "jupyter_token",
+                ),
+            },
+        ),
+    )

backend/sophon/notebooks/apache.conf

@@ -0,0 +1,20 @@
+# Log rewrite at the maximum level
+# DO NOT ENABLE THIS IN PRODUCTION OR YOUR LOGS WILL BE FLOODED!
+# LogLevel "rewrite:trace8"
+
+# Enable rewriting
+RewriteEngine on
+RewriteMap "sophonproxy" "dbm=gdbm:/mnt/tebi/ext4/workspace/sophon/proxy.dbm"
+
+# Preserve host
+ProxyPreserveHost on
+
+# Proxy websockets
+RewriteCond "%{HTTP_HOST}" ".dev.sophon.steffo.eu$" [NC]
+RewriteCond "%{HTTP:Connection}" "Upgrade" [NC]
+RewriteCond "%{HTTP:Upgrade}" "websocket" [NC]
+RewriteRule "/(.*)" "ws://${sophonproxy:%{HTTP_HOST}}/$1" [P,L]
+
+# Proxy regular requests
+RewriteCond "%{HTTP_HOST}" ".dev.sophon.steffo.eu$" [NC]
+RewriteRule "/(.*)" "http://${sophonproxy:%{HTTP_HOST}}/$1" [P,L]

backend/sophon/notebooks/apache.py

@@ -0,0 +1,69 @@
+import dbm.gnu
+import logging
+import os
+import socket
+import typing as t
+
+log = logging.getLogger(__name__)
+db_name = os.environ.get("APACHE_PROXY_EXPRESS_DBM", "proxy.dbm")
+base_domain = os.environ["APACHE_PROXY_BASE_DOMAIN"]
+http_protocol = os.environ.get("APACHE_PROXY_HTTP_PROTOCOL", "https")
+
+
+class ApacheDB:
+    def __init__(self, filename: str):
+        self.filename: str = filename
+        log.debug(f"{self.filename}: Initializing...")
+        with dbm.open(self.filename, "c"):
+            pass
+
+    @staticmethod
+    def convert_to_bytes(item: t.Union[str, bytes]) -> bytes:
+        if isinstance(item, str):
+            log.debug(f"Encoding {item!r} as ASCII...")
+            item = item.encode("ascii")
+        return item
+
+    def __getitem__(self, key: t.Union[str, bytes]) -> bytes:
+        key = self.convert_to_bytes(key)
+        log.debug(f"{self.filename}: Getting {key!r}...")
+        with dbm.open(self.filename, "r") as adb:
+            return adb[key]
+
+    def __setitem__(self, key: bytes, value: bytes) -> None:
+        key = self.convert_to_bytes(key)
+        value = self.convert_to_bytes(value)
+        log.debug(f"{self.filename}: Setting {key!r} → {value!r}...")
+        with dbm.open(self.filename, "w") as adb:
+            adb[key] = value
+
+    def __delitem__(self, key):
+        key = self.convert_to_bytes(key)
+        log.debug(f"{self.filename}: Deleting {key!r}...")
+        with dbm.open(self.filename, "w") as adb:
+            del adb[key]
+
+
+log.info(f"Creating proxy_express database: {db_name}")
+db = ApacheDB(db_name)
+log.info(f"Created proxy_express database!")
+
+
+def get_ephemeral_port() -> int:
+    """
+    Request a free TCP port from the operating system by opening and immediately closing a TCP socket.
+
+    :return: A free port number.
+
+    .. warning:: Prone to race conditions, be sure to bind something to the obtained port as soon as it is retrieved!
+
+    .. seealso:: https://stackoverflow.com/a/36331860/4334568
+    """
+
+    sock: socket.socket = socket.socket()
+    sock.bind(("localhost", 0))
+
+    port: int
+    _, port = sock.getsockname()
+
+    return port

backend/sophon/notebooks/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class NotebooksConfig(AppConfig):
+    name = 'sophon.notebooks'
+    verbose_name = "Sophon Notebooks"