From c79ba01dba9672753f2eeb7506ee4e4811de8104 Mon Sep 17 00:00:00 2001 From: Eric Callahan Date: Sun, 22 Dec 2024 07:38:30 -0500 Subject: [PATCH] install: support for package requirement specifiers Initial support for pip-like requirement specifiers applicable to system packages. This allows for a package dependencies specific to distribution version. Signed-off-by: Eric Callahan --- .../components/update_manager/app_deploy.py | 53 ++++++++- scripts/install-moonraker.sh | 43 +++++-- scripts/sync_dependencies.py | 111 +++++++++++++++--- 3 files changed, 180 insertions(+), 27 deletions(-) diff --git a/moonraker/components/update_manager/app_deploy.py b/moonraker/components/update_manager/app_deploy.py index ef3abce01..a637b9b31 100644 --- a/moonraker/components/update_manager/app_deploy.py +++ b/moonraker/components/update_manager/app_deploy.py @@ -262,6 +262,51 @@ async def restart_service(self) -> None: svc = kconn.unit_name await machine.do_service_action("restart", svc) + def _convert_version(self, version: str) -> Tuple[str | int, ...]: + version = version.strip() + ver_match = re.match(r"\d+(\.\d+)*((?:-|\.).+)?", version) + if ver_match is not None: + return tuple([ + int(part) if part.isdigit() else part + for part in re.split(r"\.|-", version) + ]) + return (version,) + + def _parse_system_dep(self, full_spec: str) -> str | None: + parts = full_spec.split(";", maxsplit=1) + if len(parts) == 1: + return full_spec + dep_parts = re.split(r"(==|!=|<=|>=|<|>)", parts[1].strip()) + if len(dep_parts) != 3 or dep_parts[0].strip().lower() != "distro_version": + logging.info(f"Invalid requirement specifier: {full_spec}") + return None + pkg_name = parts[0].strip() + operator = dep_parts[1].strip() + distro_ver = self._convert_version(distro.version()) + req_version = self._convert_version(dep_parts[2].strip()) + try: + if operator == "<": + if distro_ver < req_version: + return pkg_name + elif operator == ">": + if distro_ver > req_version: + return pkg_name + elif operator == "==": + if distro_ver == req_version: + return pkg_name + elif operator == "!=": + if distro_ver != req_version: + return pkg_name + elif operator == ">=": + if distro_ver >= req_version: + return pkg_name + elif operator == "<=": + if distro_ver <= req_version: + return pkg_name + except TypeError: + pass + return None + async def _read_system_dependencies(self) -> List[str]: eventloop = self.server.get_event_loop() if self.system_deps_json is not None: @@ -281,7 +326,13 @@ async def _read_system_dependencies(self) -> List[str]: f"Dependency file '{deps_json.name}' contains an empty " f"package definition for linux distro '{distro_id}'" ) - return dep_info[distro_id] + continue + processed_deps: List[str] = [] + for dep in dep_info[distro_id]: + parsed_dep = self._parse_system_dep(dep) + if parsed_dep is not None: + processed_deps.append(parsed_dep) + return processed_deps else: self.log_info( f"Dependency file '{deps_json.name}' has no package definition " diff --git a/scripts/install-moonraker.sh b/scripts/install-moonraker.sh index 93823ac16..fe9ecd108 100755 --- a/scripts/install-moonraker.sh +++ b/scripts/install-moonraker.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script installs Moonraker on Debian based Linux distros. -SUPPORTED_DISTROS="debian" +SUPPORTED_DISTROS="debian ubuntu" PYTHONDIR="${MOONRAKER_VENV:-${HOME}/moonraker-env}" SYSTEMDDIR="/etc/systemd/system" REBUILD_ENV="${MOONRAKER_REBUILD_ENV:-n}" @@ -15,6 +15,7 @@ INSTANCE_ALIAS="${MOONRAKER_ALIAS:-moonraker}" SPEEDUPS="${MOONRAKER_SPEEDUPS:-n}" SERVICE_VERSION="1" DISTRIBUTION="" +DISTRO_VERSION="" IS_SRC_DIST="n" PACKAGES="" @@ -37,11 +38,34 @@ if [ -f "${SRCDIR}/moonraker/__init__.py" ]; then IS_SRC_DIST="y" fi +compare_version () { + if [ -z "$DISTRO_VERSION" ]; then + return 1 + fi + compare_script=$(cat << EOF +import re +def convert_ver(ver): + ver = ver.strip() + ver_match = re.match(r"\d+(\.\d+)*((?:-|\.).+)?", ver) + if ver_match is None: + return (ver,) + return tuple([int(p) if p.isdigit() else p for p in re.split(r"\.|-", ver)]) +dist_version = convert_ver("$DISTRO_VERSION") +req_version = convert_ver("$2") +exit(int(not dist_version $1 req_version)) +EOF +) + python3 -c "$compare_script" +} + # Detect Current Distribution detect_distribution() { distro_list="" + orig_id="" if [ -f "/etc/os-release" ]; then - distro_list="$( grep -Po "^ID=\K.+" /etc/os-release || true )" + DISTRO_VERSION="$( grep -Po "^VERSION_ID=\"?\K[^\"]+" /etc/os-release || true )" + orig_id="$( grep -Po "^ID=\K.+" /etc/os-release || true )" + distro_list=$orig_id like_str="$( grep -Po "^ID_LIKE=\K.+" /etc/os-release || true )" if [ ! -z "${like_str}" ]; then distro_list="${distro_list} ${like_str}" @@ -71,19 +95,23 @@ detect_distribution() { [ ! -z "$DISTRIBUTION" ] && break done + if [ "$DISTRIBUTION" != "$orig_id" ]; then + DISTRO_VERSION="" + fi + if [ -z "$DISTRIBUTION" ] && [ -x "$( which apt-get || true )" ]; then - # Fall back to debian if apt-get is deteted + # Fall back to debian if apt-get is detected echo "Found apt-get, falling back to debian distribution" DISTRIBUTION="debian" fi - # *** AUTO GENERATED OS PACKAGE DEPENDENCES START *** + # *** AUTO GENERATED OS PACKAGE DEPENDENCIES START *** if [ ${DISTRIBUTION} = "debian" ]; then PACKAGES="python3-virtualenv python3-dev libopenjp2-7 libsodium-dev zlib1g-dev" PACKAGES="${PACKAGES} libjpeg-dev packagekit wireless-tools curl" PACKAGES="${PACKAGES} build-essential" fi - # *** AUTO GENERATED OS PACKAGE DEPENDENCES END *** + # *** AUTO GENERATED OS PACKAGE DEPENDENCIES END *** } # Step 2: Clean up legacy installation @@ -106,13 +134,14 @@ install_packages() echo "Bypassing system package installation." return fi + report_status "Installing Moonraker System Packages..." + echo "Linux Distribution: ${DISTRIBUTION} ${DISTRO_VERSION}" + echo "Packages: ${PACKAGES}" # Update system package info report_status "Running apt-get update..." sudo apt-get update --allow-releaseinfo-change # Install desired packages - report_status "Installing Moonraker Dependencies:" - report_status "${PACKAGES}" sudo apt-get install --yes ${PACKAGES} } diff --git a/scripts/sync_dependencies.py b/scripts/sync_dependencies.py index f540c7ae9..62e6d13cd 100755 --- a/scripts/sync_dependencies.py +++ b/scripts/sync_dependencies.py @@ -11,19 +11,27 @@ import tomllib import json import re -from typing import Dict, List +from typing import Dict, List, Tuple MAX_LINE_LENGTH = 88 SCRIPTS_PATH = pathlib.Path(__file__).parent -INST_PKG_HEADER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCES START ***" -INST_PKG_FOOTER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCES END ***" +INST_PKG_HEADER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCIES START ***" +INST_PKG_FOOTER = "# *** AUTO GENERATED OS PACKAGE DEPENDENCIES END ***" -def gen_multline_var(var_name: str, values: List[str], indent: int = 0) -> str: +def gen_multline_var( + var_name: str, + values: List[str], + indent: int = 0, + is_first: bool = True +) -> str: idt = " " * indent if not values: return f'{idt}{var_name}=""' line_list: List[str] = [] - current_line = f"{idt}{var_name}=\"{values.pop(0)}" + if is_first: + current_line = f"{idt}{var_name}=\"{values.pop(0)}" + else: + current_line = (f"{idt}{var_name}=\"${{{var_name}}} {values.pop(0)}") for val in values: if len(current_line) + len(val) + 2 > MAX_LINE_LENGTH: line_list.append(f'{current_line}"') @@ -33,14 +41,35 @@ def gen_multline_var(var_name: str, values: List[str], indent: int = 0) -> str: line_list.append(f'{current_line}"') return "\n".join(line_list) +def parse_sysdeps_file() -> Dict[str, List[Tuple[str, str, str]]]: + sys_deps_file = SCRIPTS_PATH.joinpath("system-dependencies.json") + base_deps: Dict[str, List[str]] = json.loads(sys_deps_file.read_bytes()) + parsed_deps: Dict[str, List[Tuple[str, str, str]]] = {} + for distro, pkgs in base_deps.items(): + parsed_deps[distro] = [] + for dep in pkgs: + parts = dep.split(";", maxsplit=1) + if len(parts) == 1: + parsed_deps[distro].append((dep.strip(), "", "")) + else: + pkg_name = parts[0].strip() + dep_parts = re.split(r"(==|!=|<=|>=|<|>)", parts[1].strip()) + comp_var = dep_parts[0].strip().lower() + if len(dep_parts) != 3 or comp_var != "distro_version": + continue + operator = dep_parts[1].strip() + req_version = dep_parts[2].strip() + parsed_deps[distro].append((pkg_name, operator, req_version)) + return parsed_deps + def sync_packages() -> int: inst_script = SCRIPTS_PATH.joinpath("install-moonraker.sh") - sys_deps_file = SCRIPTS_PATH.joinpath("system-dependencies.json") - new_deps: Dict[str, List[str]] = json.loads(sys_deps_file.read_bytes()) + new_deps = parse_sysdeps_file() # Copy install script in memory. install_data: List[str] = [] - prev_deps: Dict[str, List[str]] = {} + prev_deps: Dict[str, List[Tuple[str, str, str]]] = {} distro_name = "" + cur_spec: Tuple[str, str] | None = None skip_data = False with inst_script.open("r") as inst_file: for line in inst_file: @@ -56,12 +85,30 @@ def sync_packages() -> int: if distro_match is not None: distro_name = distro_match.group(1) prev_deps[distro_name] = [] - elif cur_line.startswith("PACKAGES"): - pkgs = cur_line.split("=", maxsplit=1)[1].strip('"') - pkg_list = pkgs.split() - if pkg_list and pkg_list[0] == "${PACKAGES}": - pkg_list.pop(0) - prev_deps[distro_name].extend(pkg_list) + else: + if cur_spec is not None and cur_line == "fi": + cur_spec = None + else: + req_match = re.match( + r"if \( compare_version \"(<|>|==|!=|<=|>=)\" " + r"\"([a-zA-Z0-9._-]+)\" \); then", + cur_line + ) + if req_match is not None: + parts = req_match.groups() + cur_spec = (parts[0], parts[1]) + elif cur_line.startswith("PACKAGES"): + pkgs = cur_line.split("=", maxsplit=1)[1].strip('"') + pkg_list = pkgs.split() + if pkg_list and pkg_list[0] == "${PACKAGES}": + pkg_list.pop(0) + operator, req_version = "", "" + if cur_spec is not None: + operator, req_version = cur_spec + for pkg in pkg_list: + prev_deps[distro_name].append( + (pkg, operator, req_version) + ) if cur_line == INST_PKG_HEADER: skip_data = True elif cur_line == INST_PKG_FOOTER: @@ -69,9 +116,9 @@ def sync_packages() -> int: install_data.append(line) # Check if an update is necessary if set(prev_deps.keys()) == set(new_deps.keys()): - for distro, pkg_list in prev_deps.items(): + for distro, prev_pkgs in prev_deps.items(): new_pkgs = new_deps[distro] - if set(pkg_list) != set(new_pkgs): + if set(prev_pkgs) != set(new_pkgs): break else: # Dependencies match, exit @@ -93,9 +140,35 @@ def sync_packages() -> int: inst_file.write( f'{prefix} [ ${{DISTRIBUTION}} = "{distro}" ]; then\n' ) - pkg_var = gen_multline_var("PACKAGES", packages, indent_count + 4) - inst_file.write(pkg_var) - inst_file.write("\n") + pkgs_by_op: Dict[Tuple[str, str], List[str]] = {} + base_list: List[str] = [] + for pkg_spec in packages: + if not pkg_spec[1] or not pkg_spec[2]: + base_list.append(pkg_spec[0]) + else: + key = (pkg_spec[1], pkg_spec[2]) + pkgs_by_op.setdefault(key, []).append(pkg_spec[0]) + is_first = True + if base_list: + pkg_var = gen_multline_var( + "PACKAGES", base_list, indent_count + 4 + ) + inst_file.write(pkg_var) + inst_file.write("\n") + is_first = False + if pkgs_by_op: + for (operator, req_ver), pkg_list in pkgs_by_op.items(): + req_idt = idt + " " * 4 + inst_file.write( + f"{req_idt}if ( compare_version \"{operator}\" " + f"\"{req_ver}\" ); then\n" + ) + req_pkgs = gen_multline_var( + "PACKAGES", pkg_list, indent_count + 8, is_first + ) + inst_file.write(req_pkgs) + inst_file.write("\n") + inst_file.write(f"{req_idt}fi\n") inst_file.write(f"{idt}fi\n") return 1