From 2c3146600f5653c54a68faf3bfe7faee2a1fc56a Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 16 Dec 2024 09:03:26 +0100 Subject: [PATCH] add multidb migrations (#243) Changes: - add real multidb support for migrations - split test settings - add multi db tests and improve existing - add templates - splitout hashing - allow passing a different registry for preparation - extra names are checked - allow selecting the main database via a space character - documentation update --- docs/migrations/migrations.md | 163 ++++++---- docs/release-notes.md | 17 + docs/tips-and-tricks.md | 6 + docs_src/migrations/accounts_models.py | 2 +- docs_src/migrations/attaching.py | 50 +-- docs_src/migrations/lru.py | 5 +- docs_src/migrations/model.py | 2 +- docs_src/migrations/starlette.py | 4 +- edgy/__init__.py | 11 +- edgy/cli/operations/init.py | 2 +- edgy/cli/templates/default/env.py | 91 ++++-- edgy/cli/templates/default/script.py.mako | 33 +- edgy/cli/templates/plain/README | 1 + edgy/cli/templates/plain/alembic.ini.mako | 50 +++ edgy/cli/templates/plain/env.py | 147 +++++++++ edgy/cli/templates/plain/script.py.mako | 47 +++ edgy/cli/templates/url/README | 1 + edgy/cli/templates/url/alembic.ini.mako | 50 +++ edgy/cli/templates/url/env.py | 151 +++++++++ edgy/cli/templates/url/script.py.mako | 63 ++++ edgy/conf/global_settings.py | 1 + edgy/core/connection/registry.py | 24 +- edgy/core/db/fields/foreign_keys.py | 1 + edgy/core/db/models/metaclasses.py | 4 +- edgy/core/db/querysets/base.py | 35 ++- edgy/core/utils/db.py | 28 +- edgy/utils/hashing.py | 19 ++ tests/cli/{custom => custom_multidb}/README | 0 .../alembic.ini.mako | 0 tests/cli/custom_multidb/env.py | 142 +++++++++ tests/cli/custom_multidb/script.py.mako | 44 +++ tests/cli/custom_singledb/README | 1 + tests/cli/custom_singledb/alembic.ini.mako | 50 +++ tests/cli/{custom => custom_singledb}/env.py | 2 +- .../script.py.mako | 0 tests/cli/main_multidb.py | 63 ++++ tests/cli/test_multidb_templates.py | 291 ++++++++++++++++++ tests/cli/test_templates.py | 59 +++- tests/cli/utils.py | 8 +- tests/conftest.py | 2 +- tests/{settings.py => settings/__init__.py} | 11 +- tests/settings/default.py | 11 + tests/settings/multidb.py | 7 + 43 files changed, 1488 insertions(+), 211 deletions(-) create mode 100644 edgy/cli/templates/plain/README create mode 100644 edgy/cli/templates/plain/alembic.ini.mako create mode 100644 edgy/cli/templates/plain/env.py create mode 100644 edgy/cli/templates/plain/script.py.mako create mode 100644 edgy/cli/templates/url/README create mode 100644 edgy/cli/templates/url/alembic.ini.mako create mode 100644 edgy/cli/templates/url/env.py create mode 100644 edgy/cli/templates/url/script.py.mako create mode 100644 edgy/utils/hashing.py rename tests/cli/{custom => custom_multidb}/README (100%) rename tests/cli/{custom => custom_multidb}/alembic.ini.mako (100%) create mode 100644 tests/cli/custom_multidb/env.py create mode 100644 tests/cli/custom_multidb/script.py.mako create mode 100644 tests/cli/custom_singledb/README create mode 100644 tests/cli/custom_singledb/alembic.ini.mako rename tests/cli/{custom => custom_singledb}/env.py (98%) rename tests/cli/{custom => custom_singledb}/script.py.mako (100%) create mode 100644 tests/cli/main_multidb.py create mode 100644 tests/cli/test_multidb_templates.py rename tests/{settings.py => settings/__init__.py} (54%) create mode 100644 tests/settings/default.py create mode 100644 tests/settings/multidb.py diff --git a/docs/migrations/migrations.md b/docs/migrations/migrations.md index e0d3da91..47f746dc 100644 --- a/docs/migrations/migrations.md +++ b/docs/migrations/migrations.md @@ -198,26 +198,23 @@ As you can see, it is quite structured but let us focus specifically on `account There is where your models for the `accounts` application will be placed. Something like this: -```python +```python title="myproject/apps/accounts/models.py" {!> ../docs_src/migrations/accounts_models.py !} ``` -Now we want to tell the **Instance** object to make sure it knows about this. +Now we use `preloads` to load the file containing the models: -```python +```python title="myproject/configs/settings.py" {!> ../docs_src/migrations/attaching.py !} ``` +It is maybe also required to set the `migrate_databases` in case of extra databases should be used in migrations. + ## Generating and working with migrations Now this is the juicy part, right? Yes but before jumping right into this, please make sure you read properly the [migration](#migration) section and you have everything in place. -**It is recommended that you follow** the [environment variables](#environment-variables) -suggestions. - -This will depend heavily on this and **everything works around the registry**. - Edgy has the internal client that manages and handles the migration process for you in a clean fashion and it called `edgy`. @@ -275,47 +272,46 @@ that same principle. ### Environment variables -When generating migrations, Edgy **expects at least one environment variable to be present**. +When generating migrations, Edgy can use following environment variables to modify the migrations. -* **EDGY_DATABASE_URL** - The database url for your database. +* **EDGY_DATABASE** - Restrict to this database metadata in migrations. Use **one** whitespace for selecting the main database. There is a special mode when used with **EDGY_DATABASE_URL** together. +* **EDGY_DATABASE_URL** - Has two modes: + 1. **EDGY_DATABASE** is empty. Here is tried to retrieve the metadata of the database in the registry via the url. When none is matching the default database is used but with the differing url. + 2. **EDGY_DATABASE** is not empty. Here the metadata of the database of the name is used but with a different URL. -The reason for this is because Edgy is agnostic to any framework and this way it makes it easier -to work with the `migrations`. +You most probably won't need the variables. Instead you can use the setting +[`migrate_databases`](#migration-settings) for selecting the databases. -Also, gives a clean design for the time where it is needed to go to production as the procedure is -very likely to be done using environment variables. - -**This variable must be present**. So to save time you can simply do: - -``` -$ export EDGY_DATABASE_URL=postgresql+asyncpg://user:pass@localhost:5432/my_database -``` +!!! Warning + Spaces are often not visible. When having **EDGY_DATABASE** in the environment you may have to check carefully if it consist of spaces or other whitespace characters. -Or whatever connection string you are using. +!!! Tip + In case you want a different name for the main database than " ", you can change the MAIN_DATABASE_NAME variable in the generated `env.py`. ### Initialize the migrations folder -It is now time to generate the migrations folder. As mentioned before in the -[environment variables section](#environment-variables), Edgy does need to have the -`EDGY_DATABASE_URL` to generate the `migrations` folder. So, without further ado let us generate +It is now time to generate the migrations folder. So, without further ado let us generate our `migrations`. ```shell -edgy --app myproject.main init +# code is in myproject.main +edgy init +# or you want to specify an entrypoint module explicit +# edgy --app myproject.main_test init ``` -What is happenening here? Well, `edgy` is always expecting an `--app` parameter to be -provided. +What is happenening here? The [discovery mechanism](./discovery.md) finds the entrypoint automatically +but you can also provide it explicit via `--app`. -This `--app` is the location of your application in `module_app` format and this is because of +The optional `--app` is the location of your application in `module_app` format and this is because of the fact of being **framework agnostic**. Edgy needs the module automatically setting the instance (see [Connections](../connection.md)) to know the registry which shall be used as well as the application object. Remember when it was mentioned that is important the location where you generate the migrations -folder? Well, this is why, because when you do `my_project.main` you are telling that -your application is inside the `myproject/main/app.py` and your migration folder should be placed +folder? Well, this is why, because when you do `my_project.main_test` you are telling that +your application is inside the `myproject/main_test.py` and your migration folder should be placed **where the command was executed**. In other words, the place you execute the `init` command it will be where the migrations will be @@ -370,25 +366,35 @@ You have now generated your migrations folder and came with gifts. A lot of files were generated automatically for you and they are specially tailored for the needs and complexity of **Edgy**. -Do you remember when it was mentioned in the [environment variables](#environment-variables) that -edgy is expecting the `EDGY_DATABASE_URL` to be available? +#### Templates -Well, this is another reason, inside the generated `migrations/env.py` the `get_engine_url()` is -also expecting that value. +Sometimes you don't want to start with a migration template which uses hashed names for upgrade and downgrade. +Or you want to use the database url instead for the name generation. -```python title="migrations/env.py" -# Code above +Edgy has different flavors called templates: -def get_engine_url(): - return os.environ.get("EDGY_DATABASE_URL") +- default - (Default) The default template. Uses hashed database names. `env.py` is compatible to flask-migrate multidb migrations. +- plain - Uses plain database names (means: databases in extra should be identifiers). `env.py` is compatible to flask-migrate multidb migrations. +- url - Uses database urls instead of names for hashing. `env.py` is NOT compatible to flask-migrate multidb migrations. You need to adapt them. -# Code below +You can use them with: + +```shell +edgy init -t plain ``` -!!! Warning - You do not need to use this environment variable. This is the `default` provided by Edgy. - You can change the value to whatever you want/need but be careful when doing it as it might - cause Edgy not to work properly with migrations if this value is not updated properly. +or list all available templates with: + +```shell +edgy list_templates +``` + +You can also use templates from the filesystem + +```shell title="Example how to use the singledb template from tests" +edgy --app myproject.main init -t tests/cli/custom_singledb +``` +Templates are always just the starting point. You most probably want to adapt the result. ### Generate the first migrations @@ -411,16 +417,15 @@ from .models import User ``` !!! Note - Since Edgy is agnostic to any framework, there aren't automatic mechanisms that detects - Edgy models in the same fashion that Django does with the `INSTALLED_APPS`. So this is - one way of exposing your models in the application. + Since Edgy is agnostic to any framework, it has no hard-coded detection algorithm like Django has + with `INSTALLED_APPS`. Instead use the `preloads` feature and imports for loading all models. There are many ways of exposing your models of course, so feel free to use any approach you want. Now it is time to generate the migration. ```shell -$ edgy --app my_project.main makemigrations +$ edgy makemigrations ``` Yes, it is this simple 😁 @@ -445,7 +450,7 @@ Your new migration should now be inside `migrations/versions/`. Something like t Or you can attach a message your migration that will then added to the file name as well. ```shell -$ edgy --app my_project.main makemigrations -m "Initial migrations" +$ edgy makemigrations -m "Initial migrations" ``` ```shell hl_lines="10" @@ -470,7 +475,7 @@ Now comes the easiest part where you need to apply the migrations. Simply run: ```shell -$ edgy --app my_project.main:app migrate +$ edgy migrate ``` And that is about it 🎉🎉 @@ -485,13 +490,13 @@ for any other ORM and when you are happy run the migrations and apply them again **Generate new migrations** ```shell -$ edgy --app my_project.main makemigrations +$ edgy makemigrations ``` **Apply them to your database** ```shell -$ edgy --app my_project.main migrate +$ edgy migrate ``` ### More migration commands @@ -551,7 +556,12 @@ into a more friendly and intuitive way. For those familiar with Django, the names came from those same operations. -## Migrate from flask-migrate +## Multi-database migrations + +Edgy added recently support for multi database migrations. You can simply continue using the old style +single database migrations. Or update your `env.py` and existing migrations for multi-database migrations. + +### Migrate from flask-migrate `flask-migrate` was the blueprint for the original `Migrate` object which was the way to enable migrations but is deprecated nowadays. @@ -560,17 +570,45 @@ The new way are the `edgy.Instance` class and the migration settings. `edgy.Instance` takes as arguments `(registry, app=None)` instead of flask-migrate `Migrate` arguments: `(app, database)`. Also settings are not set here anymore, they are set in the edgy settings object. -### Multi-schema migrations +#### Migrate env.py -If you want to migrate multiple schemes you just have to turn on `multi_schema` in the [Migration settings](#migration-settings). -You might want to filter via the schema parameters what schemes should be migrated. +Let's assume we have flask-migrate with the multiple db feature: + +Just exchanging the env.py by the default one of edgy should be enough. +Otherwise we need to adjust the migrations. See below. + +### Migrate from single-database migrations + +In case you want to use the new edgy multidb migration feature you need to adapt old migrations. +It is quite easy: + +1. Adding an parameter named `engine_name` to the `upgrade`/`downgrade` functions in all migrations which defaults to ''. +2. Preventing the execution in case the `engine_name` parameter isn't empty. + +That is all. + +In case of a different default database for old migrations add the database to extra and prevent the execution for all other names +then the extra name. + +**Example** -### Multi-database migrations +``` python +def downgrade(): + ... +``` + +becomes + +``` python +def downgrade(engine_name: str = ""): + if engine_name != "": # or dbname you want + return +``` -Currently it is only possible to select the database used for the migration and to overwrite the folder. -The multi-db migrations of flask are not supported yet in this way. -But you can script them by calling with different `EDGY_DATABASE` or `EDGY_DATABASE_URL` environment -variables. +## Multi-schema migrations + +If you want to migrate multiple schemes you just have to turn on `multi_schema` in the [Migration settings](#migration-settings). +You might want to filter via the schema parameters what schemes should be migrated. ## Migration Settings @@ -581,10 +619,7 @@ Some important settings are: - `multi_schema` - (Default: False). Include the schemes in the migrations, `True` for all schemes, a regex for some schemes. - `ignore_schema_pattern` - (Default: "information_schema"). Exclude patterns for `multi_schema`. +- `migrate_databases` - (Default: (None,)) Databases which should be migrated. - `migration_directory` - (Default: "migrations"). Path to the alembic migration folder. This overwritable per command via `-d`, `--directory` parameter. - `alembic_ctx_kwargs` - (Default: `{"compare_type": True, "render_as_batch": True}`). Extra arguments for alembic. - -## Very important - -Check the [environment variables](#environment-variables) for more details and making sure you follow the right steps. diff --git a/docs/release-notes.md b/docs/release-notes.md index 14427098..3236a05e 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -6,6 +6,23 @@ hide: # Release Notes +## 0.24.0 + +### Added + +- True multi-database migrations. + You may need to rework your migrations in case you want to use it. +- Generalized `hash_to_identifier` function. +- `get_name` function on `metadata_by_url` dict. +- Differing databases can be passed via `database` attribute on models. + +### Changed + +- Breaking: empty names are not allowed anymore for extra. This includes names consisting of spaces. + +### Fixed + +- ForeignKey remote check failed for objects with different database but same registry. ## 0.23.3 diff --git a/docs/tips-and-tricks.md b/docs/tips-and-tricks.md index 12507347..d963138d 100644 --- a/docs/tips-and-tricks.md +++ b/docs/tips-and-tricks.md @@ -62,6 +62,12 @@ the `lru_cache` technique for our `db_connection`. This will make sure that from now on you will always use the same connection and registry within your appliction by importing the `get_db_connection()` anywhere is needed. +Why don't we use `edgy.monkay.instance.registry` instead? It is a chicken-egg problem: + +It is not set before the preloads are executed. You are running into circular import issues. + +There is also a second advantage of using the lru cache: you can have multiple registries. + ## Pratical example Let us now assemble everything and generate an application that will have: diff --git a/docs_src/migrations/accounts_models.py b/docs_src/migrations/accounts_models.py index 96d39173..fb064b79 100644 --- a/docs_src/migrations/accounts_models.py +++ b/docs_src/migrations/accounts_models.py @@ -4,7 +4,7 @@ import edgy -_, registry = get_db_connection() +registry = get_db_connection() class User(edgy.Model): diff --git a/docs_src/migrations/attaching.py b/docs_src/migrations/attaching.py index f4390652..5ad33021 100644 --- a/docs_src/migrations/attaching.py +++ b/docs_src/migrations/attaching.py @@ -1,46 +1,10 @@ -#!/usr/bin/env python -import os -import sys -from pathlib import Path +from typing import Optional, Union -from my_project.utils import get_db_connection +from edgy import EdgySettings -from edgy import Instance, monkay -from esmerald import Esmerald, Include - -def build_path(): - """ - Builds the path of the project and project root. - """ - Path(__file__).resolve().parent.parent - SITE_ROOT = os.path.dirname(os.path.realpath(__file__)) - - if SITE_ROOT not in sys.path: - sys.path.append(SITE_ROOT) - sys.path.append(os.path.join(SITE_ROOT, "apps")) - - -def get_application(): - """ - This is optional. The function is only used for organisation purposes. - """ - build_path() - registry = get_db_connection() - - app = registry.asgi( - Esmerald( - routes=[Include(namespace="my_project.urls")], - ) - ) - - monkay.set_instance( - Instance( - app=app, - registry=registry, - ) - ) - return app - - -app = get_application() +class MyMigrationSettings(EdgySettings): + # here we notify about the models import path + preloads: list[str] = ["myproject.apps.accounts.models"] + # here we can set the databases which should be used in migrations, by default (None,) + migrate_databases: Union[list[Union[str, None]], tuple[Union[str, None], ...]] = (None,) diff --git a/docs_src/migrations/lru.py b/docs_src/migrations/lru.py index 5d4cad6b..7097eba9 100644 --- a/docs_src/migrations/lru.py +++ b/docs_src/migrations/lru.py @@ -5,5 +5,6 @@ @lru_cache() def get_db_connection(): - database = Database("postgresql+asyncpg://user:pass@localhost:5432/my_database") - return database, Registry(database=database) + # use echo=True for getting the connection infos printed + database = Database("postgresql+asyncpg://user:pass@localhost:5432/my_database", echo=True) + return Registry(database=database) diff --git a/docs_src/migrations/model.py b/docs_src/migrations/model.py index 96d39173..fb064b79 100644 --- a/docs_src/migrations/model.py +++ b/docs_src/migrations/model.py @@ -4,7 +4,7 @@ import edgy -_, registry = get_db_connection() +registry = get_db_connection() class User(edgy.Model): diff --git a/docs_src/migrations/starlette.py b/docs_src/migrations/starlette.py index 05776ce8..564f1ccc 100644 --- a/docs_src/migrations/starlette.py +++ b/docs_src/migrations/starlette.py @@ -3,7 +3,7 @@ import sys from pathlib import Path -from lilya.apps import Lilya +from starlette.applications import Starlette from my_project.utils import get_db_connection from edgy import monkay, Instance @@ -28,7 +28,7 @@ def get_application(): build_path() registry = get_db_connection() - app = registry.asgi(Lilya(__name__)) + app = registry.asgi(Starlette()) monkay.set_instance(Instance(app=app, registry=registry)) return app diff --git a/edgy/__init__.py b/edgy/__init__.py index 91492dc9..e354c06a 100644 --- a/edgy/__init__.py +++ b/edgy/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -__version__ = "0.23.3" +__version__ = "0.24.0" from typing import TYPE_CHECKING from ._monkay import Instance, create_monkay @@ -107,11 +107,12 @@ del create_monkay -def get_migration_prepared_registry() -> Registry: +def get_migration_prepared_registry(registry: Registry | None = None) -> Registry: """Get registry with applied restrictions, usable for migrations.""" - instance = monkay.instance - assert instance is not None - registry = instance.registry + if registry is None: + instance = monkay.instance + assert instance is not None + registry = instance.registry assert registry is not None registry.refresh_metadata( multi_schema=monkay.settings.multi_schema, diff --git a/edgy/cli/operations/init.py b/edgy/cli/operations/init.py index 8d2cb67d..8714c6e3 100644 --- a/edgy/cli/operations/init.py +++ b/edgy/cli/operations/init.py @@ -6,7 +6,7 @@ @add_migration_directory_option @click.option( - "-t", "--template", default=None, help=('Repository template to use (default is "flask")') + "-t", "--template", default=None, help=('Repository template to use (default is "default")') ) @click.option( "--package", diff --git a/edgy/cli/templates/default/env.py b/edgy/cli/templates/default/env.py index 299b54ff..a005cab7 100644 --- a/edgy/cli/templates/default/env.py +++ b/edgy/cli/templates/default/env.py @@ -3,14 +3,15 @@ import asyncio import logging import os +from collections.abc import Generator from logging.config import fileConfig -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Literal, Optional, Union from alembic import context from rich.console import Console import edgy -from edgy.core.connection import Database +from edgy.core.connection import Database, Registry if TYPE_CHECKING: import sqlalchemy @@ -26,29 +27,40 @@ # This line sets up loggers basically. fileConfig(config.config_file_name) logger = logging.getLogger("alembic.env") +MAIN_DATABASE_NAME: str = " " -def get_engine_url_and_metadata() -> tuple[str, "sqlalchemy.MetaData"]: +def iter_databases( + registry: Registry, +) -> Generator[tuple[str, Database, "sqlalchemy.MetaData"], None, None]: url: Optional[str] = os.environ.get("EDGY_DATABASE_URL") - _name = None - registry = edgy.get_migration_prepared_registry() - _metadata = registry.metadata_by_name[None] - if not url: - db_name: Optional[str] = os.environ.get("EDGY_DATABASE") - if db_name: - url = str(registry.extra[db_name].url) - if not url: - url = str(registry.database.url) + name: Union[str, Literal[False], None] = os.environ.get("EDGY_DATABASE") or False + if url and not name: + try: + name = registry.metadata_by_url.get_name(url) + except KeyError: + name = None + if name is False: + db_names = edgy.monkay.settings.migrate_databases + for name in db_names: + if name is None: + yield (None, registry.database, registry.metadata_by_name[None]) + else: + yield (name, registry.extra[name], registry.metadata_by_name[name]) else: - _metadata = registry.metadata_by_url.get(url, _metadata) - return url, _metadata - - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -target_url, target_metadata = get_engine_url_and_metadata() -config.set_main_option("sqlalchemy.url", target_url) + if name == MAIN_DATABASE_NAME: + name = None + if url: + database = Database(url) + elif name is None: + database = registry.database + else: + database = registry.extra[name] + yield ( + name, + database, + registry.metadata_by_name[name], + ) # other values from the config, defined by the needs of env.py, @@ -69,33 +81,47 @@ def run_migrations_offline() -> Any: Calls to context.execute() here emit the given string to the script output. """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + registry = edgy.get_migration_prepared_registry() + for name, db, metadata in iter_databases(registry): + context.configure( + url=str(db.url), + target_metadata=metadata, + literal_binds=True, + ) - with context.begin_transaction(): - context.run_migrations() + with context.begin_transaction(): + # for compatibility with flask migrate multidb kwarg is called engine_name + context.run_migrations(engine_name=name or "") -def do_run_migrations(connection: Any) -> Any: +def do_run_migrations(connection: Any, name: str, metadata: "sqlalchemy.Metadata") -> Any: # this callback is used to prevent an auto-migration from being generated # when there are no changes to the schema # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html def process_revision_directives(context, revision, directives) -> Any: # type: ignore if getattr(config.cmd_opts, "autogenerate", False): script = directives[0] - if script.upgrade_ops.is_empty(): + empty = True + for upgrade_ops in script.upgrade_ops_list: + if not upgrade_ops.is_empty(): + empty = False + break + if empty: directives[:] = [] console.print("[bright_red]No changes in schema detected.") context.configure( connection=connection, - target_metadata=target_metadata, + target_metadata=metadata, + upgrade_token=f"{name or ''}_upgrades", + downgrade_token=f"{name or ''}_downgrades", process_revision_directives=process_revision_directives, **edgy.monkay.settings.alembic_ctx_kwargs, ) with context.begin_transaction(): - context.run_migrations() + # for compatibility with flask migrate multidb kwarg is called engine_name + context.run_migrations(engine_name=name or "") async def run_migrations_online() -> Any: @@ -108,8 +134,11 @@ async def run_migrations_online() -> Any: """ # the original script checked for the async compatibility # we are only compatible with async drivers so just use Database - async with Database(target_url) as database: - await database.run_sync(do_run_migrations) + registry = edgy.get_migration_prepared_registry() + async with registry: + for name, db, metadata in iter_databases(registry): + async with db as database: + await database.run_sync(do_run_migrations, name, metadata) if context.is_offline_mode(): diff --git a/edgy/cli/templates/default/script.py.mako b/edgy/cli/templates/default/script.py.mako index 2c015630..18c18b73 100644 --- a/edgy/cli/templates/default/script.py.mako +++ b/edgy/cli/templates/default/script.py.mako @@ -7,6 +7,7 @@ Create Date: ${create_date} """ from alembic import op import sqlalchemy as sa +from edgy.utils.hashing import hash_to_identifier ${imports if imports else ""} # revision identifiers, used by Alembic. @@ -15,10 +16,34 @@ down_revision = ${repr(down_revision)} branch_labels = ${repr(branch_labels)} depends_on = ${repr(depends_on)} +def upgrade(engine_name: str = "") -> None: + fn = globals().get(f"upgrade_{hash_to_identifier(engine_name)}") + if fn is not None: + fn() -def upgrade(): - ${upgrades if upgrades else "pass"} +def downgrade(engine_name: str = "") -> None: + fn = globals().get(f"downgrade_{hash_to_identifier(engine_name)}") + if fn is not None: + fn() -def downgrade(): - ${downgrades if downgrades else "pass"} + +<% + from edgy import monkay + from edgy.utils.hashing import hash_to_identifier + db_names = monkay.settings.migrate_databases +%> + +## generate an "upgrade_() / downgrade_()" function +## according to edgy migrate settings + +% for db_name in db_names: + +def ${f"upgrade_{hash_to_identifier(db_name or '')}"}(): + ${context.get(f"{db_name or ''}_upgrades", "pass")} + + +def ${f"downgrade_{hash_to_identifier(db_name or '')}"}(): + ${context.get(f"{db_name or ''}_downgrades", "pass")} + +% endfor diff --git a/edgy/cli/templates/plain/README b/edgy/cli/templates/plain/README new file mode 100644 index 00000000..e687f0ff --- /dev/null +++ b/edgy/cli/templates/plain/README @@ -0,0 +1 @@ +Database configuration with Alembic. diff --git a/edgy/cli/templates/plain/alembic.ini.mako b/edgy/cli/templates/plain/alembic.ini.mako new file mode 100644 index 00000000..dd5924b4 --- /dev/null +++ b/edgy/cli/templates/plain/alembic.ini.mako @@ -0,0 +1,50 @@ +# A generic database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,edgy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_edgy] +level = INFO +handlers = +qualname = edgy + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/edgy/cli/templates/plain/env.py b/edgy/cli/templates/plain/env.py new file mode 100644 index 00000000..a005cab7 --- /dev/null +++ b/edgy/cli/templates/plain/env.py @@ -0,0 +1,147 @@ +# Default env template + +import asyncio +import logging +import os +from collections.abc import Generator +from logging.config import fileConfig +from typing import TYPE_CHECKING, Any, Literal, Optional, Union + +from alembic import context +from rich.console import Console + +import edgy +from edgy.core.connection import Database, Registry + +if TYPE_CHECKING: + import sqlalchemy + +# The console used for the outputs +console = Console() + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config: Any = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") +MAIN_DATABASE_NAME: str = " " + + +def iter_databases( + registry: Registry, +) -> Generator[tuple[str, Database, "sqlalchemy.MetaData"], None, None]: + url: Optional[str] = os.environ.get("EDGY_DATABASE_URL") + name: Union[str, Literal[False], None] = os.environ.get("EDGY_DATABASE") or False + if url and not name: + try: + name = registry.metadata_by_url.get_name(url) + except KeyError: + name = None + if name is False: + db_names = edgy.monkay.settings.migrate_databases + for name in db_names: + if name is None: + yield (None, registry.database, registry.metadata_by_name[None]) + else: + yield (name, registry.extra[name], registry.metadata_by_name[name]) + else: + if name == MAIN_DATABASE_NAME: + name = None + if url: + database = Database(url) + elif name is None: + database = registry.database + else: + database = registry.extra[name] + yield ( + name, + database, + registry.metadata_by_name[name], + ) + + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> Any: + """ + Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + registry = edgy.get_migration_prepared_registry() + for name, db, metadata in iter_databases(registry): + context.configure( + url=str(db.url), + target_metadata=metadata, + literal_binds=True, + ) + + with context.begin_transaction(): + # for compatibility with flask migrate multidb kwarg is called engine_name + context.run_migrations(engine_name=name or "") + + +def do_run_migrations(connection: Any, name: str, metadata: "sqlalchemy.Metadata") -> Any: + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives) -> Any: # type: ignore + if getattr(config.cmd_opts, "autogenerate", False): + script = directives[0] + empty = True + for upgrade_ops in script.upgrade_ops_list: + if not upgrade_ops.is_empty(): + empty = False + break + if empty: + directives[:] = [] + console.print("[bright_red]No changes in schema detected.") + + context.configure( + connection=connection, + target_metadata=metadata, + upgrade_token=f"{name or ''}_upgrades", + downgrade_token=f"{name or ''}_downgrades", + process_revision_directives=process_revision_directives, + **edgy.monkay.settings.alembic_ctx_kwargs, + ) + + with context.begin_transaction(): + # for compatibility with flask migrate multidb kwarg is called engine_name + context.run_migrations(engine_name=name or "") + + +async def run_migrations_online() -> Any: + """ + Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + # the original script checked for the async compatibility + # we are only compatible with async drivers so just use Database + registry = edgy.get_migration_prepared_registry() + async with registry: + for name, db, metadata in iter_databases(registry): + async with db as database: + await database.run_sync(do_run_migrations, name, metadata) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/edgy/cli/templates/plain/script.py.mako b/edgy/cli/templates/plain/script.py.mako new file mode 100644 index 00000000..0b9beead --- /dev/null +++ b/edgy/cli/templates/plain/script.py.mako @@ -0,0 +1,47 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +def upgrade(engine_name: str = "") -> None: + fn = globals().get(f"upgrade_{engine_name}") + if fn is not None: + fn() + + +def downgrade(engine_name: str = "") -> None: + fn = globals().get(f"downgrade_{engine_name}") + if fn is not None: + fn() + + +<% + from edgy import monkay + db_names = monkay.settings.migrate_databases +%> + +## generate an "upgrade_() / downgrade_()" function +## according to edgy migrate settings + +% for db_name in db_names: + +def ${f"upgrade_{db_name or ''}"}(): + ${context.get(f"{db_name or ''}_upgrades", "pass")} + + +def ${f"downgrade_{db_name or ''}"}(): + ${context.get(f"{db_name or ''}_downgrades", "pass")} + +% endfor diff --git a/edgy/cli/templates/url/README b/edgy/cli/templates/url/README new file mode 100644 index 00000000..e687f0ff --- /dev/null +++ b/edgy/cli/templates/url/README @@ -0,0 +1 @@ +Database configuration with Alembic. diff --git a/edgy/cli/templates/url/alembic.ini.mako b/edgy/cli/templates/url/alembic.ini.mako new file mode 100644 index 00000000..dd5924b4 --- /dev/null +++ b/edgy/cli/templates/url/alembic.ini.mako @@ -0,0 +1,50 @@ +# A generic database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,edgy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_edgy] +level = INFO +handlers = +qualname = edgy + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/edgy/cli/templates/url/env.py b/edgy/cli/templates/url/env.py new file mode 100644 index 00000000..df3b026b --- /dev/null +++ b/edgy/cli/templates/url/env.py @@ -0,0 +1,151 @@ +# Default env template + +import asyncio +import logging +import os +from collections.abc import Generator +from logging.config import fileConfig +from typing import TYPE_CHECKING, Any, Literal, Optional, Union + +from alembic import context +from rich.console import Console + +import edgy +from edgy.core.connection import Database, DatabaseURL, Registry + +if TYPE_CHECKING: + import sqlalchemy + +# The console used for the outputs +console = Console() + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config: Any = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") +MAIN_DATABASE_NAME: str = " " + + +def iter_databases( + registry: Registry, +) -> Generator[tuple[str, Database, "sqlalchemy.MetaData"], None, None]: + url: Optional[str] = os.environ.get("EDGY_DATABASE_URL") + name: Union[str, Literal[False], None] = os.environ.get("EDGY_DATABASE") or False + if url and not name: + try: + name = registry.metadata_by_url.get_name(url) + except KeyError: + name = None + if name is False: + db_names = edgy.monkay.settings.migrate_databases + for name in db_names: + if name is None: + yield (None, registry.database, registry.metadata_by_name[None]) + else: + yield (name, registry.extra[name], registry.metadata_by_name[name]) + else: + if name == MAIN_DATABASE_NAME: + name = None + if url: + database = Database(url) + elif name is None: + database = registry.database + else: + database = registry.extra[name] + yield ( + name, + database, + registry.metadata_by_name[name], + ) + + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> Any: + """ + Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + registry = edgy.get_migration_prepared_registry() + for name, db, metadata in iter_databases(registry): + # db is maybe overwritten, so use the original url + orig_url = registry.database.url if name is None else registry.extra[name].url + context.configure( + url=str(db.url), + target_metadata=metadata, + literal_binds=True, + ) + + with context.begin_transaction(): + context.run_migrations(url=orig_url) + + +def do_run_migrations( + connection: Any, url: str, orig_url: DatabaseURL, name: str, metadata: "sqlalchemy.Metadata" +) -> Any: + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives) -> Any: # type: ignore + if getattr(config.cmd_opts, "autogenerate", False): + script = directives[0] + empty = True + for upgrade_ops in script.upgrade_ops_list: + if not upgrade_ops.is_empty(): + empty = False + break + if empty: + directives[:] = [] + console.print("[bright_red]No changes in schema detected.") + + context.configure( + connection=connection, + target_metadata=metadata, + upgrade_token=f"{name or ''}_upgrades", + downgrade_token=f"{name or ''}_downgrades", + process_revision_directives=process_revision_directives, + **edgy.monkay.settings.alembic_ctx_kwargs, + ) + + with context.begin_transaction(): + context.run_migrations(url=orig_url) + + +async def run_migrations_online() -> Any: + """ + Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + # the original script checked for the async compatibility + # we are only compatible with async drivers so just use Database + registry = edgy.get_migration_prepared_registry() + async with registry: + for name, db, metadata in iter_databases(registry): + # db is maybe overwritten, so use the original url + orig_url = registry.database.url if name is None else registry.extra[name].url + async with db as database: + await database.run_sync(do_run_migrations, str(db.url), orig_url, name, metadata) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/edgy/cli/templates/url/script.py.mako b/edgy/cli/templates/url/script.py.mako new file mode 100644 index 00000000..e08e2a78 --- /dev/null +++ b/edgy/cli/templates/url/script.py.mako @@ -0,0 +1,63 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import TYPE_CHECKING, Optional + +from alembic import op +import sqlalchemy as sa +from edgy.utils.hashing import hash_to_identifier +${imports if imports else ""} + +if TYPE_CHECKING: + from edgy.core.connection import DatabaseURL + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +def upgrade(url: Optional["DatabaseURL"] = None) -> None: + urlstring = "" if url is None else f"{url.username}:{url.netloc}" + fn = globals().get(f"upgrade_{hash_to_identifier(urlstring)}") + if fn is not None: + fn() + + +def downgrade(url: Optional["DatabaseURL"] = None) -> None: + urlstring = "" if url is None else f"{url.username}:{url.netloc}" + fn = globals().get(f"downgrade_{hash_to_identifier(urlstring)}") + if fn is not None: + fn() + + +<% + from edgy import monkay + from edgy.utils.hashing import hash_to_identifier + db_names = monkay.settings.migrate_databases + + def url_for_name(name): + if name: + url = monkay.instance.registry.extra[name].url + else: + url = monkay.instance.registry.database.url + return f"{url.username}:{url.netloc}" +%> + +## generate an "upgrade_() / downgrade_()" function +## according to edgy migrate settings + +% for db_name in db_names: + +def ${f"upgrade_{hash_to_identifier(url_for_name(db_name))}"}(): + ${context.get(f"{db_name or ''}_upgrades", "pass")} + + +def ${f"downgrade_{hash_to_identifier(url_for_name(db_name))}"}(): + ${context.get(f"{db_name or ''}_downgrades", "pass")} + +% endfor diff --git a/edgy/conf/global_settings.py b/edgy/conf/global_settings.py index 1c0dafce..4a795992 100644 --- a/edgy/conf/global_settings.py +++ b/edgy/conf/global_settings.py @@ -36,6 +36,7 @@ class MediaSettings(BaseSettings): class MigrationSettings(BaseSettings): multi_schema: Union[bool, re.Pattern, str] = False ignore_schema_pattern: Union[None, re.Pattern, str] = "information_schema" + migrate_databases: Union[list[Union[str, None]], tuple[Union[str, None], ...]] = (None,) migration_directory: Union[str, os.PathLike] = Path("migrations/") # extra keyword arguments to pass to alembic alembic_ctx_kwargs: dict = { diff --git a/edgy/core/connection/registry.py b/edgy/core/connection/registry.py index c20d921c..69598813 100644 --- a/edgy/core/connection/registry.py +++ b/edgy/core/connection/registry.py @@ -44,7 +44,7 @@ def __init__(self, registry: "Registry") -> None: def __getitem__(self, key: Union[str, None]) -> sqlalchemy.MetaData: if key not in self.registry.extra and key is not None: - raise KeyError("Key does not exist") + raise KeyError(f'Extra database "{key}" does not exist.') return super().__getitem__(key) def get(self, key: str, default: Any = None) -> sqlalchemy.MetaData: @@ -87,6 +87,10 @@ def get(self, key: str, default: Any = None) -> sqlalchemy.MetaData: except KeyError: return default + def get_name(self, key: str) -> Optional[str]: + """Return name to url or raise a KeyError in case it isn't available.""" + return cast(Optional[str], super().__getitem__(key)) + def __copy__(self) -> "MetaDataByUrlDict": return MetaDataByUrlDict(registry=self.registry) @@ -135,11 +139,29 @@ def __init__( self.extra: dict[str, Database] = { k: v if isinstance(v, Database) else Database(v) for k, v in extra.items() } + # we want to get all problems before failing + assert all( + [self.extra_name_check(x) for x in self.extra] # noqa: C419 + ), "Invalid name in extra detected. See logs for details." self.metadata_by_url = MetaDataByUrlDict(registry=self) if with_content_type is not False: self._set_content_type(with_content_type) + def extra_name_check(self, name: Any) -> bool: + if not isinstance(name, str): + logger.error(f"Extra database name: {name!r} is not a string.") + return False + elif not name.strip(): + logger.error(f'Extra database name: "{name}" is empty.') + return False + + if name.strip() != name: + logger.warning( + f'Extra database name: "{name}" starts or ends with whitespace characters.' + ) + return True + def __copy__(self) -> "Registry": content_type: Union[bool, type[BaseModelType]] = False if self.content_type is not None: diff --git a/edgy/core/db/fields/foreign_keys.py b/edgy/core/db/fields/foreign_keys.py index 924f42e2..f1060b6d 100644 --- a/edgy/core/db/fields/foreign_keys.py +++ b/edgy/core/db/fields/foreign_keys.py @@ -318,6 +318,7 @@ def get_global_constraints( # this does not work because fks are checked in metadata # this implies is_cross_db and is just a stronger version or self.owner.meta.registry is not self.target.meta.registry + or self.owner.database is not self.target.database ) if not no_constraint: target = self.target diff --git a/edgy/core/db/models/metaclasses.py b/edgy/core/db/models/metaclasses.py index c75dacf9..129563b0 100644 --- a/edgy/core/db/models/metaclasses.py +++ b/edgy/core/db/models/metaclasses.py @@ -34,6 +34,7 @@ from edgy.exceptions import ImproperlyConfigured, TableBuildError if TYPE_CHECKING: + from edgy.core.connection import Database from edgy.core.db.models import Model from edgy.core.db.models.types import BaseModelType @@ -598,6 +599,7 @@ def __new__( attrs.pop("_pkcolumns", None) attrs.pop("_pknames", None) attrs.pop("_table", None) + database: Union[Literal["keep"], None, Database, bool] = attrs.pop("database", "keep") # Extract fields and managers and include them in attrs attrs = extract_fields_and_managers(bases, attrs) @@ -789,7 +791,7 @@ def __new__( if not meta.registry: new_class.model_rebuild(force=True) return new_class - new_class.add_to_registry(meta.registry) + new_class.add_to_registry(meta.registry, database=database) return new_class def get_db_schema(cls) -> Union[str, None]: diff --git a/edgy/core/db/querysets/base.py b/edgy/core/db/querysets/base.py index 60ea86ed..04932e42 100644 --- a/edgy/core/db/querysets/base.py +++ b/edgy/core/db/querysets/base.py @@ -23,7 +23,7 @@ from edgy.core.db.models.types import BaseModelType from edgy.core.db.models.utils import apply_instance_extras from edgy.core.db.relationships.utils import crawl_relationship -from edgy.core.utils.db import check_db_connection, hash_tablekey +from edgy.core.utils.db import CHECK_DB_CONNECTION_SILENCED, check_db_connection, hash_tablekey from edgy.core.utils.sync import run_sync from edgy.exceptions import MultipleObjectsReturned, ObjectNotFound, QuerySetError from edgy.types import Undefined @@ -1429,20 +1429,25 @@ async def create(self, *args: Any, **kwargs: Any) -> EdgyEmbedTarget: """ # for tenancy queryset: QuerySet = self._clone() - instance = self.model_class(*args, **kwargs) - apply_instance_extras( - instance, - self.model_class, - schema=self.using_schema, - table=queryset.table, - database=queryset.database, - ) - # values=kwargs is required for ensuring all kwargs are seen as explicit kwargs - instance = await instance.save(force_insert=True, values=set(kwargs.keys())) - result = await self._embed_parent_in_result(instance) - self._clear_cache(True) - self._cache.update([result]) - return cast(EdgyEmbedTarget, result[1]) + check_db_connection(queryset.database) + token = CHECK_DB_CONNECTION_SILENCED.set(True) + try: + instance = queryset.model_class(*args, **kwargs) + apply_instance_extras( + instance, + self.model_class, + schema=self.using_schema, + table=queryset.table, + database=queryset.database, + ) + # values=kwargs is required for ensuring all kwargs are seen as explicit kwargs + instance = await instance.save(force_insert=True, values=set(kwargs.keys())) + result = await self._embed_parent_in_result(instance) + self._clear_cache(True) + self._cache.update([result]) + return cast(EdgyEmbedTarget, result[1]) + finally: + CHECK_DB_CONNECTION_SILENCED.reset(token) async def bulk_create(self, objs: Iterable[Union[dict[str, Any], EdgyModel]]) -> None: """ diff --git a/edgy/core/utils/db.py b/edgy/core/utils/db.py index 2cfb190e..fc548b40 100644 --- a/edgy/core/utils/db.py +++ b/edgy/core/utils/db.py @@ -1,37 +1,35 @@ import warnings -from base64 import b32encode +from contextvars import ContextVar from functools import lru_cache -from hashlib import blake2b from typing import TYPE_CHECKING from edgy.exceptions import DatabaseNotConnectedWarning +from edgy.utils.hashing import hash_to_identifier if TYPE_CHECKING: from edgy.core.connection.database import Database +# for silencing warning +CHECK_DB_CONNECTION_SILENCED = ContextVar("CHECK_DB_CONNECTION_SILENCED", default=False) + def check_db_connection(db: "Database", stacklevel: int = 3) -> None: if not db.is_connected: # with force_rollback the effects are even worse, so fail if db.force_rollback: raise RuntimeError("db is not connected.") - # db engine will be created and destroyed afterwards - warnings.warn( - "Database not connected. Executing operation is inperformant.", - DatabaseNotConnectedWarning, - stacklevel=stacklevel, - ) + if not CHECK_DB_CONNECTION_SILENCED.get(): + # db engine will be created and destroyed afterwards + warnings.warn( + "Database not connected. Executing operation is inperformant.", + DatabaseNotConnectedWarning, + stacklevel=stacklevel, + ) @lru_cache(512, typed=False) def _hash_tablekey(tablekey: str, prefix: str) -> str: - tablehash = ( - b32encode(blake2b(f"{tablekey}_{prefix}".encode(), digest_size=16).digest()) - .decode() - .rstrip("=") - ) - - return f"_join_{tablehash}" + return f'_join{hash_to_identifier(f"{tablekey}_{prefix}")}' def hash_tablekey(*, tablekey: str, prefix: str) -> str: diff --git a/edgy/utils/hashing.py b/edgy/utils/hashing.py new file mode 100644 index 00000000..848411a3 --- /dev/null +++ b/edgy/utils/hashing.py @@ -0,0 +1,19 @@ +from base64 import b32encode +from hashlib import blake2b +from typing import Union + + +def hash_to_identifier(key: Union[str, bytes]) -> str: + """ + A generic hasher for keys, which output stays a valid name for python + and other languages. + It is NOT supposed for the use in security critical contexts. + + It is for shortening and flattening known names and urls like database names or database urls or pathes. + + See edgy/cli/templates/default/script.py or edgy/core/db/querysets/base.py for the usage. + """ + if isinstance(key, str): + key = key.encode() + # prefix with _ for preventing a name starting with a number + return f"_{b32encode(blake2b(key, digest_size=16).digest()).decode().rstrip('=')}" diff --git a/tests/cli/custom/README b/tests/cli/custom_multidb/README similarity index 100% rename from tests/cli/custom/README rename to tests/cli/custom_multidb/README diff --git a/tests/cli/custom/alembic.ini.mako b/tests/cli/custom_multidb/alembic.ini.mako similarity index 100% rename from tests/cli/custom/alembic.ini.mako rename to tests/cli/custom_multidb/alembic.ini.mako diff --git a/tests/cli/custom_multidb/env.py b/tests/cli/custom_multidb/env.py new file mode 100644 index 00000000..762bed7d --- /dev/null +++ b/tests/cli/custom_multidb/env.py @@ -0,0 +1,142 @@ +# Custom env template +import asyncio +import logging +import os +from collections.abc import Generator +from logging.config import fileConfig +from typing import TYPE_CHECKING, Any, Literal, Optional, Union + +from alembic import context +from rich.console import Console + +import edgy +from edgy.core.connection import Database, Registry + +if TYPE_CHECKING: + import sqlalchemy + +# The console used for the outputs +console = Console() + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config: Any = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") +MAIN_DATABASE_NAME: str = " " + + +def iter_databases(registry: Registry) -> Generator[tuple[str, Database, "sqlalchemy.MetaData"]]: + url: Optional[str] = os.environ.get("EDGY_DATABASE_URL") + name: Union[str, Literal[False], None] = os.environ.get("EDGY_DATABASE") or False + if url and not name: + try: + name = registry.metadata_by_url.get_name(url) + except KeyError: + name = None + if name is False: + db_names = edgy.monkay.settings.migrate_databases + for name in db_names: + if name is None: + yield (None, registry.database, registry.metadata_by_name[None]) + else: + yield (name, registry.extra[name], registry.metadata_by_name[name]) + else: + if name == MAIN_DATABASE_NAME: + name = None + if url: + database = Database(url) + elif name is None: + database = registry.database + else: + database = registry.extra[name] + yield ( + name, + database, + registry.metadata_by_name[name], + ) + + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> Any: + """ + Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + registry = edgy.get_migration_prepared_registry() + for name, db, metadata in iter_databases(registry): + context.configure( + url=str(db.url), + target_metadata=metadata, + literal_binds=True, + ) + + with context.begin_transaction(): + context.run_migrations(edgy_dbname=name or "") + + +def do_run_migrations(connection: Any, name: str, metadata: "sqlalchemy.Metadata") -> Any: + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives) -> Any: # type: ignore + if getattr(config.cmd_opts, "autogenerate", False): + script = directives[0] + empty = True + for upgrade_ops in script.upgrade_ops_list: + if not upgrade_ops.is_empty(): + empty = False + break + if empty: + directives[:] = [] + console.print("[bright_red]No changes in schema detected.") + + context.configure( + connection=connection, + target_metadata=metadata, + upgrade_token=f"{name or ''}_upgrades", + downgrade_token=f"{name or ''}_downgrades", + process_revision_directives=process_revision_directives, + **edgy.monkay.settings.alembic_ctx_kwargs, + ) + + with context.begin_transaction(): + context.run_migrations(edgy_dbname=name or "") + + +async def run_migrations_online() -> Any: + """ + Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + # the original script checked for the async compatibility + # we are only compatible with async drivers so just use Database + registry = edgy.get_migration_prepared_registry() + async with registry: + for name, db, metadata in iter_databases(registry): + async with db as database: + await database.run_sync(do_run_migrations, name, metadata) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/tests/cli/custom_multidb/script.py.mako b/tests/cli/custom_multidb/script.py.mako new file mode 100644 index 00000000..f1d3a11e --- /dev/null +++ b/tests/cli/custom_multidb/script.py.mako @@ -0,0 +1,44 @@ +# Custom mako template +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +def upgrade(edgy_dbname: str = "") -> None: + globals()[f"upgrade_{edgy_dbname}"]() + + +def downgrade(edgy_dbname: str = "") -> None: + globals()[f"downgrade_{edgy_dbname}"]() + + +<% + from edgy import monkay + db_names = monkay.settings.migrate_databases +%> + +## generate an "upgrade_() / downgrade_()" function +## according to edgy migrate settings + +% for db_name in db_names: + +def ${f"upgrade_{db_name or ''}"}(): + ${context.get(f"{db_name or ''}_upgrades", "pass")} + + +def ${f"downgrade_{db_name or ''}"}(): + ${context.get(f"{db_name or ''}_downgrades", "pass")} + +% endfor diff --git a/tests/cli/custom_singledb/README b/tests/cli/custom_singledb/README new file mode 100644 index 00000000..58c93de4 --- /dev/null +++ b/tests/cli/custom_singledb/README @@ -0,0 +1 @@ +Custom template diff --git a/tests/cli/custom_singledb/alembic.ini.mako b/tests/cli/custom_singledb/alembic.ini.mako new file mode 100644 index 00000000..57ba8a58 --- /dev/null +++ b/tests/cli/custom_singledb/alembic.ini.mako @@ -0,0 +1,50 @@ +# A custom generic database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,saffier + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_saffier] +level = INFO +handlers = +qualname = saffier + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/tests/cli/custom/env.py b/tests/cli/custom_singledb/env.py similarity index 98% rename from tests/cli/custom/env.py rename to tests/cli/custom_singledb/env.py index 02e50c12..f6e61896 100644 --- a/tests/cli/custom/env.py +++ b/tests/cli/custom_singledb/env.py @@ -32,7 +32,7 @@ def get_engine_url(): # for 'autogenerate' support config.set_main_option("sqlalchemy.url", get_engine_url()) -target_db = edgy.monkay.instance.registry +target_db = edgy.get_migration_prepared_registry() # other values from the config, defined by the needs of env.py, # can be acquired: diff --git a/tests/cli/custom/script.py.mako b/tests/cli/custom_singledb/script.py.mako similarity index 100% rename from tests/cli/custom/script.py.mako rename to tests/cli/custom_singledb/script.py.mako diff --git a/tests/cli/main_multidb.py b/tests/cli/main_multidb.py new file mode 100644 index 00000000..0b70ca9b --- /dev/null +++ b/tests/cli/main_multidb.py @@ -0,0 +1,63 @@ +import os + +import pytest + +import edgy +from edgy import Instance +from edgy.contrib.permissions import BasePermission +from tests.settings import TEST_ALTERNATIVE_DATABASE, TEST_DATABASE + +pytestmark = pytest.mark.anyio +models = edgy.Registry( + database=TEST_DATABASE, + extra={"another": TEST_ALTERNATIVE_DATABASE}, + with_content_type=True, +) +basedir = os.path.abspath(os.path.dirname(__file__)) + + +class User(edgy.StrictModel): + name = edgy.fields.CharField(max_length=100) + + class Meta: + registry = models + + +class Group(edgy.StrictModel): + name = edgy.fields.CharField(max_length=100) + users = edgy.fields.ManyToMany("User", embed_through=False) + + class Meta: + registry = models + + +class Permission(BasePermission): + users = edgy.fields.ManyToMany("User", embed_through=False) + groups = edgy.fields.ManyToMany("Group", embed_through=False) + name_model: str = edgy.fields.CharField(max_length=100, null=True) + obj = edgy.fields.ForeignKey("ContentType", null=True) + + class Meta: + registry = models + unique_together = [("name", "name_model", "obj")] + + +class Signal(edgy.StrictModel): + user = edgy.fields.ForeignKey(User, no_constraint=True) + signal_type = edgy.fields.CharField(max_length=100) + database = models.extra["another"] + + class Meta: + registry = models + + +class Unrelated(edgy.StrictModel): + name = edgy.fields.CharField(max_length=100) + database = models.extra["another"] + content_type = edgy.fields.ExcludeField() + + class Meta: + registry = models + + +edgy.monkay.set_instance(Instance(registry=models)) diff --git a/tests/cli/test_multidb_templates.py b/tests/cli/test_multidb_templates.py new file mode 100644 index 00000000..93d6a0cc --- /dev/null +++ b/tests/cli/test_multidb_templates.py @@ -0,0 +1,291 @@ +import contextlib +import os +import shutil +import sys +from asyncio import run +from pathlib import Path + +import pytest +import sqlalchemy +from sqlalchemy.exc import ProgrammingError +from sqlalchemy.ext.asyncio import create_async_engine + +from tests.cli.utils import arun_cmd +from tests.settings import ( + DATABASE_ALTERNATIVE_URL, + DATABASE_URL, + TEST_ALTERNATIVE_DATABASE, + TEST_DATABASE, +) + +pytestmark = pytest.mark.anyio + +base_path = Path(os.path.abspath(__file__)).absolute().parent + + +@pytest.fixture(scope="function", autouse=True) +def cleanup_folders(): + with contextlib.suppress(OSError): + shutil.rmtree(str(base_path / "migrations")) + with contextlib.suppress(OSError): + shutil.rmtree(str(base_path / "migrations2")) + + yield + with contextlib.suppress(OSError): + shutil.rmtree(str(base_path / "migrations")) + with contextlib.suppress(OSError): + shutil.rmtree(str(base_path / "migrations2")) + + +@pytest.fixture(scope="function", autouse=True) +async def prepare_db(): + for url in [DATABASE_ALTERNATIVE_URL, DATABASE_URL]: + engine = create_async_engine(url, isolation_level="AUTOCOMMIT") + try: + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("DROP DATABASE test_edgy")) + except Exception: + pass + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("CREATE DATABASE test_edgy")) + await engine.dispose() + + +@pytest.fixture(scope="function", autouse=True) +async def cleanup_db(): + for url in [DATABASE_ALTERNATIVE_URL, DATABASE_URL]: + engine = create_async_engine(url, isolation_level="AUTOCOMMIT") + try: + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("DROP DATABASE test_edgy")) + except Exception: + pass + await engine.dispose() + + +@pytest.mark.parametrize("app_flag", ["explicit", "explicit_env"]) +@pytest.mark.parametrize( + "template_param", + ["", " -t default", " -t plain", " -t url", " -t ./custom_multidb"], + ids=["default_empty", "default", "plain", "url", "custom"], +) +async def test_migrate_upgrade_multidb(app_flag, template_param): + os.chdir(base_path) + assert not (base_path / "migrations").exists() + app_param = "--app tests.cli.main_multidb " if app_flag == "explicit" else "" + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}init{template_param}", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + assert ss == 0 + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}makemigrations", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + assert ss == 0 + assert b"No changes in schema detected" not in o + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}migrate", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + assert ss == 0 + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"hatch run python {__file__} test_migrate_upgrade_multidb", + with_app_environment=False, + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + assert ss == 0 + + if "custom" in template_param: + with open("migrations/README") as f: + assert f.readline().strip() == "Custom template" + with open("migrations/alembic.ini") as f: + assert f.readline().strip() == "# A custom generic database configuration." + with open("migrations/env.py") as f: + assert f.readline().strip() == "# Custom env template" + with open("migrations/script.py.mako") as f: + assert f.readline().strip() == "# Custom mako template" + else: + with open("migrations/README") as f: + assert f.readline().strip() == "Database configuration with Alembic." + with open("migrations/alembic.ini") as f: + assert f.readline().strip() == "# A generic database configuration." + with open("migrations/env.py") as f: + assert f.readline().strip() == "# Default env template" + + +@pytest.mark.parametrize("app_flag", ["explicit", "explicit_env"]) +@pytest.mark.parametrize( + "template_param", + ["", " -t default", " -t plain", " -t url", " -t ./custom_multidb"], + ids=["default_empty", "default", "plain", "url", "custom"], +) +async def test_different_directory(app_flag, template_param): + os.chdir(base_path) + assert not (base_path / "migrations2").exists() + app_param = "--app tests.cli.main_multidb " if app_flag == "explicit" else "" + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}init -d migrations2 {template_param}", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}makemigrations -d migrations2", + with_app_environment=app_flag == "explicit_env", + ) + assert ss == 0 + assert b"No changes in schema detected" not in o + if "custom" in template_param: + with open("migrations2/README") as f: + assert f.readline().strip() == "Custom template" + with open("migrations2/alembic.ini") as f: + assert f.readline().strip() == "# A custom generic database configuration." + with open("migrations2/env.py") as f: + assert f.readline().strip() == "# Custom env template" + with open("migrations2/script.py.mako") as f: + assert f.readline().strip() == "# Custom mako template" + else: + with open("migrations2/README") as f: + assert f.readline().strip() == "Database configuration with Alembic." + with open("migrations2/alembic.ini") as f: + assert f.readline().strip() == "# A generic database configuration." + with open("migrations2/env.py") as f: + assert f.readline().strip() == "# Default env template" + + +@pytest.mark.parametrize( + "env_extra", + [ + {"EDGY_DATABASE": "another"}, + {"EDGY_DATABASE_URL": TEST_DATABASE, "EDGY_DATABASE": "another"}, + {"EDGY_DATABASE_URL": TEST_ALTERNATIVE_DATABASE, "EDGY_DATABASE": " "}, + {"EDGY_DATABASE_URL": TEST_ALTERNATIVE_DATABASE}, + ], + ids=[ + "only_another", + "only_another_and_custom_url", + "only_main_and_custom_url", + "retrieve_via_url", + ], +) +@pytest.mark.parametrize( + "template_param", + ["", " -t default", " -t plain", " -t url", " -t ./custom_multidb"], + ids=["default_empty", "default", "plain", "url", "custom"], +) +async def test_single_db(template_param, env_extra): + app_flag = "explicit" + os.chdir(base_path) + assert not (base_path / "migrations").exists() + app_param = "--app tests.cli.main_multidb " if app_flag == "explicit" else "" + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}init {template_param}", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings", **env_extra}, + ) + + assert ss == 0 + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}makemigrations", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings", **env_extra}, + ) + assert ss == 0 + assert b"No changes in schema detected" not in o + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"edgy {app_param}migrate", + with_app_environment=app_flag == "explicit_env", + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings", **env_extra}, + ) + assert ss == 0 + + (o, e, ss) = await arun_cmd( + "tests.cli.main_multidb", + f"hatch run python {__file__} test_single_db", + with_app_environment=False, + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings", **env_extra}, + ) + assert ss == 0 + + if "custom" in template_param: + with open("migrations/README") as f: + assert f.readline().strip() == "Custom template" + with open("migrations/alembic.ini") as f: + assert f.readline().strip() == "# A custom generic database configuration." + with open("migrations/env.py") as f: + assert f.readline().strip() == "# Custom env template" + with open("migrations/script.py.mako") as f: + assert f.readline().strip() == "# Custom mako template" + else: + with open("migrations/README") as f: + assert f.readline().strip() == "Database configuration with Alembic." + with open("migrations/alembic.ini") as f: + assert f.readline().strip() == "# A generic database configuration." + with open("migrations/env.py") as f: + assert f.readline().strip() == "# Default env template" + + +async def main(): + if sys.argv[1] == "test_single_db": + from tests.cli import main_multidb as main + + if os.environ.get("EDGY_DATABASE_URL") and os.environ.get("EDGY_DATABASE") == "another": + async with main.models: + await main.Unrelated.query.using(database=None).create(name="foo") + # should fail + with pytest.raises(ProgrammingError): + await main.Unrelated.query.create(name="foo") + # should fail + with pytest.raises(ProgrammingError): + await main.User.query.create(name="edgy") + elif os.environ.get("EDGY_DATABASE_URL") and os.environ.get("EDGY_DATABASE") == " ": + async with main.models: + # othewise content type crash + main.models.content_type.database = main.models.extra["another"] + await main.User.query.using(database="another").create(name="edgy") + # should fail + with pytest.raises(ProgrammingError): + await main.User.query.create(name="edgy2") + # should fail + with pytest.raises(ProgrammingError): + await main.Unrelated.query.create(name="foo") + elif os.environ.get("EDGY_DATABASE_URL") or os.environ.get("EDGY_DATABASE") == "another": + # should fail + with pytest.raises(ProgrammingError): + await main.User.query.create(name="edgy") + # should fail + with pytest.raises(ProgrammingError): + await main.User.query.using(database="another").create(name="edgy") + await main.Unrelated.query.create(name="foo") + elif sys.argv[1] == "test_migrate_upgrade_multidb": + from tests.cli import main_multidb as main + + async with main.models: + user = await main.User.query.create(name="edgy") + + signal = await main.Signal.query.create(user=user, signal_type="foo") + assert signal.user == user + permission = await main.Permission.query.create(users=[user], name="view") + assert await main.Permission.query.users("view").get() == user + assert await main.Permission.query.permissions_of(user).get() == permission + + +if __name__ == "__main__": + run(main()) diff --git a/tests/cli/test_templates.py b/tests/cli/test_templates.py index 0495bdbf..ce02a2b9 100644 --- a/tests/cli/test_templates.py +++ b/tests/cli/test_templates.py @@ -1,11 +1,12 @@ import contextlib import os import shutil +import sys +from asyncio import run from pathlib import Path import pytest import sqlalchemy -from esmerald import Esmerald from sqlalchemy.ext.asyncio import create_async_engine from tests.cli.utils import arun_cmd @@ -13,8 +14,6 @@ pytestmark = pytest.mark.anyio -app = Esmerald(routes=[]) - base_path = Path(os.path.abspath(__file__)).absolute().parent @@ -45,12 +44,15 @@ async def cleanup_prepare_db(): @pytest.mark.parametrize("app_flag", ["explicit", "explicit_env", "autosearch"]) -@pytest.mark.parametrize("template_type", ["default", "custom"]) -async def test_migrate_upgrade(app_flag, template_type): +@pytest.mark.parametrize( + "template_param", + ["", " -t default", " -t plain", " -t url", " -t ./custom_singledb"], + ids=["default_empty", "default", "plain", "url", "custom"], +) +async def test_migrate_upgrade(app_flag, template_param): os.chdir(base_path) assert not (base_path / "migrations").exists() app_param = "--app tests.cli.main " if app_flag == "explicit" else "" - template_param = " -t ./custom" if template_type == "custom" else "" (o, e, ss) = await arun_cmd( "tests.cli.main", f"edgy {app_param}init{template_param}", @@ -64,6 +66,7 @@ async def test_migrate_upgrade(app_flag, template_type): with_app_environment=app_flag == "explicit_env", ) assert ss == 0 + assert b"No changes in schema detected" not in o (o, e, ss) = await arun_cmd( "tests.cli.main", @@ -72,7 +75,15 @@ async def test_migrate_upgrade(app_flag, template_type): ) assert ss == 0 - if template_type == "custom": + (o, e, ss) = await arun_cmd( + "tests.cli.main", + f"hatch run python {__file__} test_migrate_upgrade", + with_app_environment=False, + extra_env={"EDGY_SETTINGS_MODULE": "tests.settings.multidb.TestSettings"}, + ) + assert ss == 0 + + if "custom" in template_param: with open("migrations/README") as f: assert f.readline().strip() == "Custom template" with open("migrations/alembic.ini") as f: @@ -91,18 +102,29 @@ async def test_migrate_upgrade(app_flag, template_type): @pytest.mark.parametrize("app_flag", ["explicit", "explicit_env", "autosearch"]) -@pytest.mark.parametrize("template_type", ["default", "custom"]) -async def test_different_directory(app_flag, template_type): +@pytest.mark.parametrize( + "template_param", + ["", " -t default", " -t plain", " -t url", " -t ./custom_singledb"], + ids=["default_empty", "default", "plain", "url", "custom"], +) +async def test_different_directory(app_flag, template_param): os.chdir(base_path) assert not (base_path / "migrations2").exists() app_param = "--app tests.cli.main " if app_flag == "explicit" else "" - template_param = " -t ./custom" if template_type == "custom" else "" (o, e, ss) = await arun_cmd( "tests.cli.main", f"edgy {app_param}init -d migrations2 {template_param}", with_app_environment=app_flag == "explicit_env", ) - if template_type == "custom": + + (o, e, ss) = await arun_cmd( + "tests.cli.main", + f"edgy {app_param}makemigrations -d migrations2", + with_app_environment=app_flag == "explicit_env", + ) + assert ss == 0 + assert b"No changes in schema detected" not in o + if "custom" in template_param: with open("migrations2/README") as f: assert f.readline().strip() == "Custom template" with open("migrations2/alembic.ini") as f: @@ -118,3 +140,18 @@ async def test_different_directory(app_flag, template_type): assert f.readline().strip() == "# A generic database configuration." with open("migrations2/env.py") as f: assert f.readline().strip() == "# Default env template" + + +async def main(): + if sys.argv[1] == "test_migrate_upgrade": + from tests.cli import main + + async with main.models: + user = await main.User.query.create(name="edgy") + permission = await main.Permission.query.create(users=[user], name="view") + assert await main.Permission.query.users("view").get() == user + assert await main.Permission.query.permissions_of(user).get() == permission + + +if __name__ == "__main__": + run(main()) diff --git a/tests/cli/utils.py b/tests/cli/utils.py index 536c7b4f..5493ee9a 100644 --- a/tests/cli/utils.py +++ b/tests/cli/utils.py @@ -3,11 +3,13 @@ import subprocess -def run_cmd(app, cmd, with_app_environment=True): +def run_cmd(app, cmd, with_app_environment=True, extra_env=None): env = dict(os.environ) env.setdefault("PYTHONPATH", env["PWD"]) if with_app_environment: env["EDGY_DEFAULT_APP"] = app + if extra_env: + env.update(extra_env) cmd = f"hatch --env test run {cmd}" result = subprocess.run(cmd, capture_output=True, env=env, shell=True) @@ -17,11 +19,13 @@ def run_cmd(app, cmd, with_app_environment=True): return result.stdout, result.stderr, result.returncode -async def arun_cmd(app, cmd, with_app_environment=True): +async def arun_cmd(app, cmd, with_app_environment=True, extra_env=None): env = dict(os.environ) env.setdefault("PYTHONPATH", env["PWD"]) if with_app_environment: env["EDGY_DEFAULT_APP"] = app + if extra_env: + env.update(extra_env) cmd = f"hatch --env test run {cmd}" process = await asyncio.create_subprocess_shell( diff --git a/tests/conftest.py b/tests/conftest.py index 4e2e12cc..39c6c12f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import pytest -os.environ.setdefault("EDGY_SETTINGS_MODULE", "tests.settings.TestSettings") +os.environ.setdefault("EDGY_SETTINGS_MODULE", "tests.settings.default.TestSettings") @pytest.fixture(scope="module") diff --git a/tests/settings.py b/tests/settings/__init__.py similarity index 54% rename from tests/settings.py rename to tests/settings/__init__.py index 37187746..d8da5430 100644 --- a/tests/settings.py +++ b/tests/settings/__init__.py @@ -1,8 +1,4 @@ import os -from pathlib import Path -from typing import Union - -from edgy.contrib.multi_tenancy.settings import TenancySettings DATABASE_URL = os.environ.get( "TEST_DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/edgy" @@ -14,9 +10,4 @@ ) TEST_DATABASE = "postgresql+asyncpg://postgres:postgres@localhost:5432/test_edgy" - - -class TestSettings(TenancySettings): - tenant_model: str = "Tenant" - auth_user_model: str = "User" - media_root: Union[str, os.PathLike] = Path(__file__).parent.parent / "test_media/" +TEST_ALTERNATIVE_DATABASE = "postgresql+asyncpg://postgres:postgres@localhost:5433/test_edgy" diff --git a/tests/settings/default.py b/tests/settings/default.py new file mode 100644 index 00000000..e3bc0bce --- /dev/null +++ b/tests/settings/default.py @@ -0,0 +1,11 @@ +import os +from pathlib import Path +from typing import Union + +from edgy.contrib.multi_tenancy.settings import TenancySettings + + +class TestSettings(TenancySettings): + tenant_model: str = "Tenant" + auth_user_model: str = "User" + media_root: Union[str, os.PathLike] = Path(__file__).parent.parent / "test_media/" diff --git a/tests/settings/multidb.py b/tests/settings/multidb.py new file mode 100644 index 00000000..d4dacaff --- /dev/null +++ b/tests/settings/multidb.py @@ -0,0 +1,7 @@ +from typing import Union + +from edgy.conf.global_settings import EdgySettings + + +class TestSettings(EdgySettings): + migrate_databases: list[Union[str, None]] = [None, "another"]