From 608a7753794f673de59229f84a829bd301a05924 Mon Sep 17 00:00:00 2001 From: Matas Misiunas Date: Wed, 22 Nov 2023 13:10:29 +0200 Subject: [PATCH 1/4] Bump changelog --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 36111e897..f2b48176d 100644 --- a/changelog.md +++ b/changelog.md @@ -47,6 +47,7 @@ * LLT-4124: Add IPv6 feature flag * LLT-3950: Enable IPv6 for wg-stun. * LLT-4502: Implement ICMP, UDP and TCP conntrack tracking for each peer separately +* LLT-4508: Fetch windows dependencies from local gitlab registry
From 450e604dbe3c2287daae15ba1611e4df15f29766 Mon Sep 17 00:00:00 2001 From: Matas Misiunas Date: Mon, 27 Nov 2023 16:45:54 +0200 Subject: [PATCH 2/4] Format ci py scripts --- ci/build_libtelio.py | 43 +++++++++--------- ci/moose_utils.py | 104 ++++++++++++++++++++++--------------------- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/ci/build_libtelio.py b/ci/build_libtelio.py index 827c2a9a8..d2edbef46 100755 --- a/ci/build_libtelio.py +++ b/ci/build_libtelio.py @@ -198,24 +198,12 @@ def copy_bindings(config): }, "linux": { "archs": { - "x86_64": { - "strip_path": "/usr/bin/strip" - }, - "aarch64": { - "strip_path": "/usr/aarch64-linux-gnu/bin/strip" - }, - "arm64": { - "strip_path": "/usr/aarch64-linux-gnu/bin/strip" - }, - "i686": { - "strip_path": "/usr/i686-linux-gnu/bin/strip" - }, - "armv7": { - "strip_path": "/usr/arm-linux-gnueabihf/bin/strip" - }, - "armv5": { - "strip_path": "/usr/arm-linux-gnueabi/bin/strip" - } + "x86_64": {"strip_path": "/usr/bin/strip"}, + "aarch64": {"strip_path": "/usr/aarch64-linux-gnu/bin/strip"}, + "arm64": {"strip_path": "/usr/aarch64-linux-gnu/bin/strip"}, + "i686": {"strip_path": "/usr/i686-linux-gnu/bin/strip"}, + "armv7": {"strip_path": "/usr/arm-linux-gnueabihf/bin/strip"}, + "armv5": {"strip_path": "/usr/arm-linux-gnueabi/bin/strip"}, }, "env": { "RUSTFLAGS": ([" -C debuginfo=2 "], "set"), @@ -252,7 +240,9 @@ def main() -> None: parser = rutils.create_cli_parser() build_parser = parser._subparsers._group_actions[0].choices["build"] build_parser.add_argument("--moose", action="store_true", help="Use libmoose") - build_parser.add_argument("--msvc", action="store_true", help="Use MSVC toolchain for Windows build") + build_parser.add_argument( + "--msvc", action="store_true", help="Use MSVC toolchain for Windows build" + ) args = parser.parse_args() @@ -300,8 +290,13 @@ def exec_build(args): moose_utils.unset_cargo_dependencies() if args.msvc: - GLOBAL_CONFIG["windows"]["archs"]["x86_64"]["rust_target"] = "x86_64-pc-windows-msvc" - GLOBAL_CONFIG["windows"]["env"]["RUSTFLAGS"] = ([" -C target-feature=-crt-static "], "set") + GLOBAL_CONFIG["windows"]["archs"]["x86_64"][ + "rust_target" + ] = "x86_64-pc-windows-msvc" + GLOBAL_CONFIG["windows"]["env"]["RUSTFLAGS"] = ( + [" -C target-feature=-crt-static "], + "set", + ) if args.moose: moose_utils.create_msvc_import_library() @@ -366,6 +361,7 @@ def _create_debug_symbol(path: str, strip_bin: str): ) _create_debug_symbol(f"{dist_dir}/{renamed_arch}/{lib_name}", strip_bin=strip) + def strip_binaries(config): if config.debug or config.target_os != "linux": return @@ -388,10 +384,13 @@ def _strip_debug_symbols(path: str, strip_bin: str): subprocess.check_call(strip_debug_symbols) strip = LIBTELIO_CONFIG["linux"]["archs"][config.arch]["strip_path"] - binaries = [bin for bin in LIBTELIO_CONFIG["linux"]["packages"].keys() if bin != NAME] + binaries = [ + bin for bin in LIBTELIO_CONFIG["linux"]["packages"].keys() if bin != NAME + ] for binary in binaries: _strip_debug_symbols(f"{dist_dir}/{binary}", strip_bin=strip) + def call_build(config): rutils.config_local_env_vars(config, LIBTELIO_CONFIG) diff --git a/ci/moose_utils.py b/ci/moose_utils.py index 828106ba4..8e1c6af6b 100644 --- a/ci/moose_utils.py +++ b/ci/moose_utils.py @@ -7,8 +7,18 @@ PROJECT_ROOT = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/..") + def _output_dir(opsys: str, arch: str) -> str: - return os.path.join(PROJECT_ROOT, "3rd-party", "libmoose", LIBTELIO_ENV_MOOSE_RELEASE_TAG, "bin", "common", opsys, arch) + return os.path.join( + PROJECT_ROOT, + "3rd-party", + "libmoose", + LIBTELIO_ENV_MOOSE_RELEASE_TAG, + "bin", + "common", + opsys, + arch, + ) def _download_moose_file(opsys: str, arch: str, file_name: str): @@ -27,20 +37,16 @@ def _download_moose_file(opsys: str, arch: str, file_name: str): nexus_url = os.environ.get("LIBTELIO_ENV_SEC_NEXUS_URL", None) if nexus_credentials is None: - raise ValueError( - "LIBTELIO_ENV_SEC_NEXUS_CREDENTIALS not set" - ) + raise ValueError("LIBTELIO_ENV_SEC_NEXUS_CREDENTIALS not set") if nexus_url is None: - raise ValueError( - "LIBTELIO_ENV_SEC_NEXUS_URL not set" - ) + raise ValueError("LIBTELIO_ENV_SEC_NEXUS_URL not set") - url = ( - f"{nexus_url}/repository/ll-gitlab-release/{MOOSE_PROJECT_ID}/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/{opsys}/{arch}/{file_name}" - ) + url = f"{nexus_url}/repository/ll-gitlab-release/{MOOSE_PROJECT_ID}/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/{opsys}/{arch}/{file_name}" - subprocess.check_call(["curl", "-f", "-u", nexus_credentials, url, "-o", output_path]) + subprocess.check_call( + ["curl", "-f", "-u", nexus_credentials, url, "-o", output_path] + ) def fetch_moose_dependencies(opsys: str, arch: str): @@ -53,7 +59,7 @@ def fetch_moose_dependencies(opsys: str, arch: str): def create_msvc_import_library(): def execute_dumpbin(file_path: str) -> list[str]: output = subprocess.check_output(["dumpbin", "/EXPORTS", file_path]) - output_lines = output.decode().split('\n')[19:] + output_lines = output.decode().split("\n")[19:] for i, line in enumerate(output_lines): if not line.strip(): output_lines = output_lines[:i] @@ -68,7 +74,9 @@ def write_exports(exports: list[str], file_path: str): f.write(f" {export}\n") def create_lib(def_path: str, lib_path: str): - subprocess.check_call(["lib", "/DEF:" + def_path, "/OUT:" + lib_path, "/MACHINE:X64"]) + subprocess.check_call( + ["lib", "/DEF:" + def_path, "/OUT:" + lib_path, "/MACHINE:X64"] + ) output_dir = _output_dir("windows", "x86_64") dll_path = os.path.join(output_dir, "sqlite3.dll") @@ -80,19 +88,21 @@ def create_lib(def_path: str, lib_path: str): create_lib(def_path, lib_path) -def _write_file(file_name, contents) : +def _write_file(file_name, contents): with open(file_name, "w") as cargoFile: cargoFile.write(contents) def set_cargo_dependencies(): - libtelio_env_sec_gitlab_repository = os.environ.get("LIBTELIO_ENV_SEC_GITLAB_REPOSITORY", None) + libtelio_env_sec_gitlab_repository = os.environ.get( + "LIBTELIO_ENV_SEC_GITLAB_REPOSITORY", None + ) if libtelio_env_sec_gitlab_repository is None: - raise ValueError('LIBTELIO_ENV_SEC_GITLAB_REPOSITORY not set.') + raise ValueError("LIBTELIO_ENV_SEC_GITLAB_REPOSITORY not set.") MOOSEMESHNETAPP_DEP = ( - r'\nmoosemeshnetapp = { ' + r"\nmoosemeshnetapp = { " f'git = "https://{libtelio_env_sec_gitlab_repository}/low-level-hacks/moose/moose-events",' f' tag = "{LIBTELIO_ENV_MOOSE_RELEASE_TAG}" }}' ) @@ -104,23 +114,19 @@ def set_cargo_dependencies(): if match_lana: if "features" not in match_lana.group(0): replaced_moose = re.sub( - r'( \})', r', features = ["moose"]\1', - match_lana.group(0) + r"( \})", r', features = ["moose"]\1', match_lana.group(0) ) cargo_contents = cargo_contents.replace( - match_lana.group(0), - replaced_moose - ); + match_lana.group(0), replaced_moose + ) _write_file(f"{PROJECT_ROOT}/Cargo.toml", cargo_contents) elif '"moose"' not in match_lana.group(0): replaced_moose = re.sub( - r'(features.*\[)(.*\])', r'\1"moose", \2', - match_lana.group(0) + r"(features.*\[)(.*\])", r'\1"moose", \2', match_lana.group(0) ) cargo_contents = cargo_contents.replace( - match_lana.group(0), - replaced_moose - ); + match_lana.group(0), replaced_moose + ) _write_file("./Cargo.toml", cargo_contents) # add moosemeshnetapp and moose feature dependency to telio-lana/Cargo.toml @@ -130,20 +136,18 @@ def set_cargo_dependencies(): if "moose = []" not in lana_cargo_contents: match_feature = re.search(r"\[features\]", lana_cargo_contents) if match_feature: - lana_cargo_contents += 'moose = []\n' + lana_cargo_contents += "moose = []\n" else: - lana_cargo_contents += '\n[features]\nmoose = []\n' + lana_cargo_contents += "\n[features]\nmoose = []\n" if "moosemeshnetapp" not in lana_cargo_contents: match_dependencies = re.search(r"\[dependencies\]", lana_cargo_contents) replaced_dependencies = re.sub( - r'$', MOOSEMESHNETAPP_DEP, - match_dependencies.group(0) + r"$", MOOSEMESHNETAPP_DEP, match_dependencies.group(0) ) lana_cargo_contents = lana_cargo_contents.replace( - match_dependencies.group(0), - replaced_dependencies - ); + match_dependencies.group(0), replaced_dependencies + ) _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents) @@ -154,14 +158,14 @@ def unset_cargo_dependencies(): match_lana = re.search(r'telio-lana.*"moose"', cargo_contents) if match_lana: replaced_moose = re.sub( - r'(telio-lana.*)"moose"(, )*(.*})', r'\1\3', - cargo_contents + r'(telio-lana.*)"moose"(, )*(.*})', r"\1\3", cargo_contents + ) + empty_features = re.search( + r'(telio-lana.*features.*)\[[^"]*\]', replaced_moose ) - empty_features = re.search(r'(telio-lana.*features.*)\[[^"]*\]', replaced_moose) if empty_features: replaced_moose = re.sub( - r'(telio-lana.*)(,\sfeatures.*)\[[^"]*\]', r'\1', - replaced_moose + r'(telio-lana.*)(,\sfeatures.*)\[[^"]*\]', r"\1", replaced_moose ) _write_file(f"{PROJECT_ROOT}/Cargo.toml", replaced_moose) @@ -170,21 +174,19 @@ def unset_cargo_dependencies(): lana_cargo_contents = lana_cargo_file.read() if "moosemeshnetapp" in lana_cargo_contents: lana_cargo_contents = re.sub( - r'\nmoosemeshnetapp.*\n', '\n', - lana_cargo_contents + r"\nmoosemeshnetapp.*\n", "\n", lana_cargo_contents + ) + _write_file( + f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents ) - _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents) if "moose" in lana_cargo_contents: - empty_features = re.search(r'\[features\]\nmo', lana_cargo_contents) + empty_features = re.search(r"\[features\]\nmo", lana_cargo_contents) if empty_features: lana_cargo_contents = re.sub( - r'\n\[features\]\nmoose = \[\]\n', '', - lana_cargo_contents + r"\n\[features\]\nmoose = \[\]\n", "", lana_cargo_contents ) else: - lana_cargo_contents = re.sub( - r'\nmoose.*\n', '\n', - lana_cargo_contents - ) - _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents) - + lana_cargo_contents = re.sub(r"\nmoose.*\n", "\n", lana_cargo_contents) + _write_file( + f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents + ) From 87956bd3c3cec6eee30ada2fc9fe538a7534866e Mon Sep 17 00:00:00 2001 From: Matas Misiunas Date: Tue, 14 Nov 2023 17:01:34 +0200 Subject: [PATCH 3/4] Fetch windows deps from gitlab package registry --- ci/build_libtelio.py | 42 ------------------------------------------ ci/env.py | 2 ++ 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/ci/build_libtelio.py b/ci/build_libtelio.py index d2edbef46..02f3af27a 100755 --- a/ci/build_libtelio.py +++ b/ci/build_libtelio.py @@ -1,11 +1,8 @@ #!/usr/bin/env python3 import os -import shutil import sys import subprocess -import urllib.request -import zipfile import moose_utils from pathlib import Path @@ -37,42 +34,6 @@ ) -def finalize_win(config, moose): - def get_dependency(url, name, dll_name): - zip_name = name + ".zip" - with urllib.request.urlopen(url) as f: - with open(zip_name, "wb") as w: - w.write(f.read()) - with zipfile.ZipFile(zip_name, "r") as zip_ref: - zip_ref.extractall(".") - shutil.copyfile( - name + "/bin/amd64/" + dll_name, - PROJECT_CONFIG.get_distribution_path( - config.target_os, config.arch, dll_name, config.debug - ), - ) - shutil.rmtree(name) - os.remove(zip_name) - - get_dependency( - "https://www.wintun.net/builds/wintun-0.14.1.zip", "wintun", "wintun.dll" - ) - get_dependency( - "https://download.wireguard.com/wireguard-nt/wireguard-nt-0.10.1.zip", - "wireguard-nt", - "wireguard.dll", - ) - - if moose: - sqlite_path = f"{PROJECT_ROOT}/3rd-party/libmoose/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/windows/{config.arch}/sqlite3.dll" - shutil.copyfile( - sqlite_path, - PROJECT_CONFIG.get_distribution_path( - config.target_os, config.arch, "sqlite3.dll", config.debug - ), - ) - - def copy_bindings(config): if "binding_src" in LIBTELIO_CONFIG[config.target_os]: telio_bindings = f"{PROJECT_CONFIG.root_dir}/{LIBTELIO_CONFIG[config.target_os]['binding_src']}" @@ -308,9 +269,6 @@ def exec_build(args): rutils.check_config(config) call_build(config) - if args.os == "windows": - finalize_win(config, args.moose) - def create_debug_symbols(config): if config.debug: diff --git a/ci/env.py b/ci/env.py index 1c94a5da5..1affa18dc 100755 --- a/ci/env.py +++ b/ci/env.py @@ -8,6 +8,7 @@ LIBTELIO_ENV_LINUX_BUILDER_TAG = "v0.0.2" LIBTELIO_ENV_WINDOWS_BUILDER_TAG = "v0.0.2" + def set_sh(): print(f"export LIBTELIO_ENV_MOOSE_RELEASE_TAG={LIBTELIO_ENV_MOOSE_RELEASE_TAG}") print(f"export LIBTELIO_ENV_NAT_LAB_DEPS_TAG={LIBTELIO_ENV_NAT_LAB_DEPS_TAG}") @@ -15,6 +16,7 @@ def set_sh(): print(f"export LIBTELIO_ENV_LINUX_BUILDER_TAG={LIBTELIO_ENV_LINUX_BUILDER_TAG}") print(f"export LIBTELIO_ENV_WINDOWS_BUILDER_TAG={LIBTELIO_ENV_WINDOWS_BUILDER_TAG}") + def set_ps1(): print(f"$env:LIBTELIO_ENV_MOOSE_RELEASE_TAG=\"{LIBTELIO_ENV_MOOSE_RELEASE_TAG}\"") print(f"$env:LIBTELIO_ENV_NAT_LAB_DEPS_TAG=\"{LIBTELIO_ENV_NAT_LAB_DEPS_TAG}\"") From edb0657a2c35877c54bd51190bd8af904f40b40e Mon Sep 17 00:00:00 2001 From: Matas Misiunas Date: Wed, 29 Nov 2023 10:09:24 +0200 Subject: [PATCH 4/4] Bump libtelio-build --- .github/workflows/gitlab.yml | 2 +- .gitlab-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index fe81a61e2..410fbffa3 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -31,4 +31,4 @@ jobs: project-id: ${{ secrets.PROJECT_ID }} schedule: ${{ github.event_name == 'schedule' }} cancel-outdated-pipelines: ${{ github.ref_name != 'main' }} - triggered-ref: v0.4.4 + triggered-ref: v0.4.5 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e8864902..1a3831fb5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,5 +15,5 @@ libtelio-build-pipeline: trigger: project: $LIBTELIO_BUILD_PROJECT_PATH - branch: v0.4.4 + branch: v0.4.5 strategy: depend