Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve migrate script even further #23

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Improve migrate script even further #23

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 148 additions & 53 deletions bin/migrate
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,73 +1,107 @@
#!/usr/bin/env python3
# pylint: disable=print-used

import logging
import re
from pathlib import Path

from click_odoo_contrib.update import main

from odoo import release, sql_db
from odoo.modules import module
from odoo.modules.migration import MigrationManager
from odoo.modules.registry import Registry
from odoo import sql_db
from datetime import datetime
import os
import logging
from odoo.tools import config

_logger = logging.getLogger(__name__)


def get_version_last_digit(version):
return ".".join(version.split(".")[-3:])

def get_new_version(current_version):
version = datetime.now().strftime("%y%m.%d.0")
# increment last digit if needed
if current_version >= version:
year_month, day, inc = current_version.split(".")
inc = str(int(inc) + 1)
version = ".".join([year_month, day, inc])
return version

# Patch odoo native migration manager to always run migration script in the
# directory "migrations/0.0.0"
# Indeed odoo will only run the migration script if the version have been bump
# inside the module.
# So if an migration directory 0.0.0 exist "dynamically" increment the version
# for odoo/local-src modules

def have_pending_migration(self, pkg):
return bool(self.migrations[pkg.name].get("module", {}).get("0.0.0"))
TOP_MODULE_PATH = Path("/") / "odoo" / "local-src" / "custom_all"

FAKE_VERSION = f"{release.major_version}.9999.9.9"

ori_load_information_from_description_file = (
module.load_information_from_description_file
)


# Special custom_all/migrations/{version}/ migrations are in the form:
# pre-global.py, pre-global-module_name.py
# post-global.py, post-global-module_name.py
SPECIAL_RE = re.compile(r"^(pre|post)-global(?:-(\w+))?.py$")
special_zero_migrations_path = TOP_MODULE_PATH / "migrations" / "0.0.0"
special_zero_migrations_modules = (
{
SPECIAL_RE.match(file.name).group(2) or "base"
for file in special_zero_migrations_path.iterdir()
if SPECIAL_RE.match(file.name)
}
if special_zero_migrations_path.exists()
else set()
)


# As odoo only run migration script when the version change
# We patch odoo when loading the manifest to increment virtually the version when
# a "pending" migration script exist
# The version is always set to the number X.X.9999.9.9
# Note: odoo natively support to process the migration in the directory 0.0.0
# so we do not need to hack this part
def load_information_from_description_file(module_name, mod_path=None):
info = ori_load_information_from_description_file(module_name, mod_path=mod_path)
if not mod_path:
mod_path = module.get_module_path(module_name, downloaded=True)
mod_path = Path(mod_path)
zero_path = mod_path / "migrations" / "0.0.0"
if ( # Migrate all local modules that have a 0.0.0 pending migration
"local-src" in mod_path.parts
and zero_path.exists()
and any(file.suffix in [".sql", ".py"] for file in zero_path.iterdir())
) or ( # Migrate all modules that have a 0.0.0 pending migration in custom_all
module_name in special_zero_migrations_modules
):
_logger.info(
"Module %s has pending migration, set version to %s",
module_name,
FAKE_VERSION,
)
info["version"] = FAKE_VERSION
return info


def update_version(pkg):
current_version = pkg.data["version"]
pkg.data["version"] = get_new_version(current_version)
module.load_information_from_description_file = load_information_from_description_file

ori_migrate_module = MigrationManager.migrate_module

def migrate_module(self, pkg, stage):
if have_pending_migration(self, pkg):
update_version(pkg)
return ori_migrate_module(self, pkg, stage)
# Process before-XXX.sql script in custom_all/migrations/{version}/

MigrationManager.migrate_module = migrate_module

def add_sql_migration(todo, version):
path = TOP_MODULE_PATH / "migrations" / version
for filename in path.iterdir():
if filename.stem.startswith("before") and filename.suffix == ".sql":
file_path = path / filename
with open(file_path) as f:
todo.append((file_path, f.read()))

# Process before.sql script in custom_all/migrations/{version}/before.sql

def get_before_request(cr):
cr.execute(
"SELECT latest_version FROM ir_module_module WHERE name='custom_all'"
)
cr.execute("SELECT latest_version FROM ir_module_module WHERE name='custom_all'")
todo = []
current_version = cr.fetchone()
if not current_version:
db_version = cr.fetchone()
if not db_version or not db_version[0]:
_logger.error("No version found for custom_all, skip begin script")
current_version = current_version[0]
migr_path = "/odoo/local-src/custom_all/migrations"
if os.path.exists(migr_path):
for version in os.listdir(migr_path):
if version == "0.0.0" or version > current_version:
file_path = f"{migr_path}/{version}/before.sql"
if os.path.exists(file_path):
todo.append((file_path, open(file_path, "r").read()))

return []
db_version = db_version[0]
migr_path = TOP_MODULE_PATH / "migrations"
if migr_path.exists():
versions = sorted(str(path) for path in migr_path.iterdir())
if versions and versions[0] == "0.0.0":
# always run pending version add the end
versions.append(versions.pop(0))
# Run all version that are superior to the db version
# And run version of 0.0.0 if FAKE_VERSION is not applied
for version in versions:
if version > db_version or version == "0.0.0" and FAKE_VERSION > db_version:
add_sql_migration(todo, version)
return todo


Expand All @@ -76,18 +110,79 @@ ori_new = Registry.new

@classmethod
def new(cls, db_name, force_demo=False, status=None, update_module=False):
# Mark base to update if it has pending migration
if (
"base" in special_zero_migrations_modules
and "base" not in config["update"]
and "all" not in config["update"]
):
config["update"]["base"] = 1

conn = sql_db.db_connect(db_name)
with conn.cursor() as cr:
for file_path, requests in get_before_request(cr):
_logger.info("Execute before sql request \n===\n%s===\n", requests)
_logger.info(
"Execute before sql request \n=== %s \n%s===\n", file_path, requests
)
cr.execute(requests)
return ori_new(
db_name, force_demo=force_demo, status=status,
update_module=update_module)
db_name, force_demo=force_demo, status=status, update_module=update_module
)


Registry.new = new


ori_get_files = MigrationManager._get_files


def _get_files(self):
# Add custom_all to graph if not already present to load the migration scripts

class FakeNode:
def __init__(self):
self.depth = 0
self.name = "custom_all"
self.state = "to upgrade"

if "custom_all" not in self.graph:
self.graph["custom_all"] = FakeNode()

ori_get_files(self)

if isinstance(self.graph["custom_all"], FakeNode):
# Remove fake custom_all from graph and migrations
del self.graph["custom_all"]
migrations = self.migrations.pop("custom_all")
else:
migrations = self.migrations["custom_all"]

# Iterate over custom_all migrations and move special ones
# in their respective modules:

for type_, version_migrations in migrations.items():
for version, migrations in version_migrations.items():
for migration in migrations[:]:
migration_file = Path(migration)
match = SPECIAL_RE.match(migration_file.name)
if match:
migrations.remove(migration)
module = match.group(2) or "base"
if module in self.migrations:
_logger.info(
"Adding special migration %s to %s",
migration_file,
module,
)
versions = self.migrations[module][type_]
if version not in versions:
versions[version] = []
versions[version].insert(0, migration)


MigrationManager._get_files = _get_files


# Call native click-odoo-update

if __name__ == "__main__": # pragma: no cover
Expand Down