churn: replace sqlalchemy with odmantic -> use mongodb instead of sqlite

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
     rev: v1.8.0
     hooks:
       - id: mypy
-        additional_dependencies: [types-requests]
+        additional_dependencies: [types-requests, types-aiofiles]
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.9.0.6
     hooks:

DEPENDENCIES

@@ -16,10 +16,10 @@ pypi/pypi/-/chevron/0.14.0
 pypi/pypi/-/click/8.1.7
 pypi/pypi/-/colorama/0.4.6
 pypi/pypi/-/cryptography/42.0.2
+pypi/pypi/-/dnspython/2.5.0
 pypi/pypi/-/exceptiongroup/1.2.0
 pypi/pypi/-/flask/3.0.2
 pypi/pypi/-/flask-minify/0.42
-pypi/pypi/-/flask-sqlalchemy/3.1.1
 pypi/pypi/-/frozenlist/1.4.1
 pypi/pypi/-/ghp-import/2.1.0
 pypi/pypi/-/gojsonnet/0.20.0
@@ -47,14 +47,16 @@ pypi/pypi/-/mergedeep/1.3.4
 pypi/pypi/-/mintotp/0.3.0
 pypi/pypi/-/mkdocs/1.5.3
 pypi/pypi/-/mkdocs-exclude/1.0.2
-pypi/pypi/-/mkdocs-material/9.5.7
+pypi/pypi/-/mkdocs-material/9.5.8
 pypi/pypi/-/mkdocs-material-extensions/1.3.1
+pypi/pypi/-/motor/3.3.2
 pypi/pypi/-/multidict/6.0.5
+pypi/pypi/-/odmantic/1.0.0
 pypi/pypi/-/packaging/23.2
 pypi/pypi/-/paginate/0.5.6
 pypi/pypi/-/pathspec/0.12.1
 pypi/pypi/-/platformdirs/4.2.0
-pypi/pypi/-/playwright/1.41.1
+pypi/pypi/-/playwright/1.41.2
 pypi/pypi/-/ply/3.11
 pypi/pypi/-/priority/2.0.0
 pypi/pypi/-/pycparser/2.21
@@ -63,6 +65,7 @@ pypi/pypi/-/pydantic-core/2.16.2
 pypi/pypi/-/pyee/11.0.1
 pypi/pypi/-/pygments/2.17.2
 pypi/pypi/-/pymdown-extensions/10.7
+pypi/pypi/-/pymongo/4.6.1
 pypi/pypi/-/pynacl/1.5.0
 pypi/pypi/-/python-dateutil/2.8.2
 pypi/pypi/-/python-decouple/3.8
@@ -81,13 +84,12 @@ pypi/pypi/-/requests/2.31.0
 pypi/pypi/-/requests-cache/1.1.1
 pypi/pypi/-/rpds-py/0.17.1
 pypi/pypi/-/six/1.16.0
-pypi/pypi/-/sqlalchemy/2.0.25
 pypi/pypi/-/taskgroup/0.0.0a4
 pypi/pypi/-/tomli/2.0.1
 pypi/pypi/-/typing-extensions/4.9.0
 pypi/pypi/-/url-normalize/1.4.3
 pypi/pypi/-/urllib3/2.2.0
-pypi/pypi/-/watchdog/3.0.0
+pypi/pypi/-/watchdog/4.0.0
 pypi/pypi/-/werkzeug/3.0.1
 pypi/pypi/-/wsproto/1.2.0
 pypi/pypi/-/wtforms/3.1.2

docker/docker-compose.base.yml

@@ -3,6 +3,8 @@ version: '3.8'
 services:
   webapp:
     restart: always
+    depends_on:
+      - mongodb
     env_file: ../.env
     build:
       context: ..
@@ -13,13 +15,17 @@ services:
     volumes:
       - ../github-app:/app/work/config
 
-#  nginx:
-#    container_name: nginx
-#    restart: always
-#    image: "nginx:latest"
-#    ports:
-#      - "80:80"
-#    volumes:
-#      - ./nginx:/etc/nginx/conf.d
-#    depends_on:
-#      - webapp
+  init-webapp:
+    image: curlimages/curl:8.2.1
+    command: sh /init_webapp.sh webapp 5000
+    volumes:
+      - ./init_webapp.sh:/init_webapp.sh
+    depends_on:
+      webapp:
+          condition: service_started
+
+  mongodb:
+    image: mongo:7.0.5
+    command: mongod --quiet --logpath /dev/null
+    ports:
+      - 27017:27017

docker/docker-compose.dev.yml

@@ -3,5 +3,10 @@ version: '3.8'
 services:
   webapp:
     volumes:
+      - ../docker/hypercorn-cfg.toml:/app/hypercorn-cfg.toml
       - ../otterdog:/app/otterdog
       - ../approot:/app/work
+
+  mongodb:
+    volumes:
+      - ../approot/db:/data/db

docker/init_webapp.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+#
+#  *******************************************************************************
+#  Copyright (c) 2024 Eclipse Foundation and others.
+#  This program and the accompanying materials are made available
+#  under the terms of the Eclipse Public License 2.0
+#  which is available at http://www.eclipse.org/legal/epl-v20.html
+#  SPDX-License-Identifier: EPL-2.0
+#  *******************************************************************************
+#
+
+# exits if any of your variables is not set
+set -o nounset
+
+HOST="${1:-webapp}"
+PORT="${2:-5000}"
+DELAY_BETWEEN_ATTEMPTS=1
+
+HEAL_URL="http://${HOST}:${PORT}/health"
+INIT_URL="http://${HOST}:${PORT}/init"
+
+while true; do
+  echo "Checking health url ${HEAL_URL}..."
+
+  status_code=$(curl -s -I "${HEAL_URL}" -o /dev/null -w "%{http_code}")
+  if [ "${status_code}" == "200" ]; then
+    echo "Triggering init via ${INIT_URL}..."
+    curl -s "${INIT_URL}"
+    exit 0
+  fi
+
+  sleep "${DELAY_BETWEEN_ATTEMPTS}"
+done

otterdog/app.py

@@ -57,7 +57,6 @@
     app.logger.info("Environment = " + config_mode)
     app.logger.info("QUART_APP   = " + app_config.QUART_APP)
     app.logger.info("APP_ROOT    = " + app_config.APP_ROOT)
-    app.logger.info("DB          = " + app_config.SQLALCHEMY_DATABASE_URI)
 
 
 def run():

otterdog/config.py

@@ -105,28 +105,16 @@ def of(
         cls,
         project_name: str,
         github_id: str,
+        config_repo: str,
+        base_template: str,
         credential_data: dict[str, Any],
         work_dir: str,
-        otterdog_config: OtterdogConfig,
     ) -> OrganizationConfig:
-        # get the default organization config to retrieve some properties
-        try:
-            org_config = otterdog_config.get_organization_config(project_name)
-            config_repo = org_config.config_repo
-            base_template = org_config.base_template
-        except RuntimeError:
-            # if no organization config can be found for the project, assume the defaults
-            config_repo = otterdog_config.default_config_repo
-            base_template = otterdog_config.default_base_template
-
-        # construct a custom jsonnet config based on the given work_dir
-        base_dir = os.path.join(otterdog_config.jsonnet_base_dir, work_dir)
-
         jsonnet_config = JsonnetConfig(
             github_id,
-            base_dir,
+            work_dir,
             base_template,
-            otterdog_config.local_mode,
+            False,
         )
 
         return cls(project_name, github_id, config_repo, base_template, jsonnet_config, credential_data)

otterdog/providers/github/rest/__init__.py

@@ -23,7 +23,7 @@ class RestApi:
     _GH_API_VERSION = "2022-11-28"
     _GH_API_URL_ROOT = "https://api.github.com"
 
-    def __init__(self, auth_strategy: AuthStrategy):
+    def __init__(self, auth_strategy: Optional[AuthStrategy] = None):
         self._auth_strategy = auth_strategy
         self._requester = Requester(auth_strategy, self._GH_API_URL_ROOT, self._GH_API_VERSION)
 

otterdog/providers/github/rest/content_client.py

@@ -37,7 +37,7 @@ async def get_content_object(
             tb = ex.__traceback__
             raise RuntimeError(f"failed retrieving content '{path}' from repo '{repo_name}':\n{ex}").with_traceback(tb)
 
-    async def get_content(self, org_id: str, repo_name: str, path: str, ref: Optional[str]) -> str:
+    async def get_content(self, org_id: str, repo_name: str, path: str, ref: Optional[str] = None) -> str:
         json_response = await self.get_content_object(org_id, repo_name, path, ref)
         return base64.b64decode(json_response["content"]).decode("utf-8")
 

otterdog/providers/github/rest/requester.py

@@ -23,9 +23,9 @@
 
 
 class Requester:
-    def __init__(self, auth_strategy: AuthStrategy, base_url: str, api_version: str):
+    def __init__(self, auth_strategy: Optional[AuthStrategy], base_url: str, api_version: str):
         self._base_url = base_url
-        self._auth = auth_strategy.get_auth()
+        self._auth = auth_strategy.get_auth() if auth_strategy is not None else None
 
         self._headers = {
             "Accept": "application/vnd.github+json",
@@ -176,7 +176,8 @@ async def async_request_raw(
         print_trace(f"async '{method}' url = {url_path}, data = {data}, headers = {self._headers}")
 
         headers = self._headers.copy()
-        self._auth.update_headers_with_authorization(headers)
+        if self._auth is not None:
+            self._auth.update_headers_with_authorization(headers)
 
         async with AsyncCachedSession(cache=FileBackend(cache_name=_AIOHTTP_CACHE_DIR, use_temp=False)) as session:
             url = self._build_url(url_path)

otterdog/webapp/__init__.py

@@ -10,21 +10,20 @@
 from importlib.util import find_spec
 
 import quart_flask_patch  # type: ignore # noqa: F401
-from flask_sqlalchemy import SQLAlchemy
 from quart import Quart
 from quart_auth import QuartAuth
 
 from .config import AppConfig
-from .db import Base
+from .db import Mongo, init_mongo_database
 
 _BLUEPRINT_MODULES: list[str] = ["home"]
 
-db = SQLAlchemy(model_class=Base)
+mongo = Mongo()
 auth_manager = QuartAuth()
 
 
 def register_extensions(app):
-    db.init_app(app)
+    mongo.init_app(app)
     auth_manager.init_app(app)
 
 
@@ -47,21 +46,9 @@ def register_blueprints(app):
 
 def configure_database(app):
     @app.before_serving
-    async def create_tables():
+    async def configure():
         async with app.app_context():
-            models_fqn = "otterdog.webapp.db.models"
-            import_module(models_fqn)
-            db.create_all()
-
-    @app.before_serving
-    async def fill_database():
-        from otterdog.webapp.db.service import fill_organization_table
-
-        await fill_organization_table(app)
-
-    @app.teardown_request
-    def shutdown_session(exception=None):
-        db.session.remove()
+            await init_mongo_database(mongo)
 
 
 def create_app(app_config: AppConfig):

otterdog/webapp/config.py

@@ -28,12 +28,11 @@ class AppConfig(object):
     if not os.path.exists(DB_ROOT):
         os.makedirs(DB_ROOT)
 
-    SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join(DB_ROOT, "otterdog-db.sqlite3")
-    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    MONGO_URI = config("MONGO_URI", default="mongodb://mongodb:27017/otterdog")
 
-    OTTERDOG_CONFIG_URL = config("OTTERDOG_CONFIG_URL", default=None)
     OTTERDOG_CONFIG_OWNER = config("OTTERDOG_CONFIG_OWNER", default=None)
     OTTERDOG_CONFIG_REPO = config("OTTERDOG_CONFIG_REPO", default=None)
+    OTTERDOG_CONFIG_PATH = config("OTTERDOG_CONFIG_PATH", default=None)
 
     # Set up the App SECRET_KEY
     SECRET_KEY = config("SECRET_KEY", default=None)

otterdog/webapp/db/__init__.py

@@ -6,8 +6,40 @@
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
 
-from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass
+import re
+from typing import Optional
 
+from motor.motor_asyncio import AsyncIOMotorClient
+from odmantic import AIOEngine
+from quart import Quart
 
-class Base(MappedAsDataclass, DeclarativeBase):
-    pass
+
+class Mongo:
+    def __init__(self) -> None:
+        self._client: Optional[AsyncIOMotorClient] = None
+        self._engine: Optional[AIOEngine] = None
+
+    def init_app(self, app: Quart) -> None:
+        connection_uri = app.config["MONGO_URI"]
+
+        m = re.match(r"^((mongodb:(?:/{2})?)((\w+?):(\w+?)@|:?@?)(\w+?):(\d+)/)(\w+?)$", connection_uri)
+
+        if m is not None:
+            server_uri = m.group(1)
+            database = m.group(8)
+        else:
+            raise RuntimeError(f"failed to parse mongo connection uri: '{connection_uri}'")
+
+        self._client = AsyncIOMotorClient(server_uri)
+        self._engine = AIOEngine(client=self._client, database=database)
+
+    @property
+    def odm(self) -> AIOEngine:
+        assert self._engine is not None
+        return self._engine
+
+
+async def init_mongo_database(mongo: Mongo) -> None:
+    from .models import OrganizationConfigModel, TaskModel
+
+    await mongo.odm.configure_database([OrganizationConfigModel, TaskModel])  # type: ignore