Merge pull request #208 from NordSecurity/LLT-4508_store_win_deps_loc…

…ally

LLT-4508 store win deps locally

.github/workflows/gitlab.yml

@@ -31,4 +31,4 @@ jobs:
           project-id: ${{ secrets.PROJECT_ID }}
           schedule: ${{ github.event_name == 'schedule' }}
           cancel-outdated-pipelines: ${{ github.ref_name != 'main' }}
-          triggered-ref: v0.4.4
+          triggered-ref: v0.4.5

.gitlab-ci.yml

@@ -15,5 +15,5 @@ libtelio-build-pipeline:
 
   trigger:
     project: $LIBTELIO_BUILD_PROJECT_PATH
-    branch: v0.4.4
+    branch: v0.4.5
     strategy: depend

changelog.md

@@ -47,6 +47,7 @@
 * LLT-4124: Add IPv6 feature flag
 * LLT-3950: Enable IPv6 for wg-stun.
 * LLT-4502: Implement ICMP, UDP and TCP conntrack tracking for each peer separately
+* LLT-4508: Fetch windows dependencies from local gitlab registry
 
 <br>
 

ci/build_libtelio.py

@@ -1,11 +1,8 @@
 #!/usr/bin/env python3
 
 import os
-import shutil
 import sys
 import subprocess
-import urllib.request
-import zipfile
 import moose_utils
 from pathlib import Path
 
@@ -37,42 +34,6 @@
 )
 
 
-def finalize_win(config, moose):
-    def get_dependency(url, name, dll_name):
-        zip_name = name + ".zip"
-        with urllib.request.urlopen(url) as f:
-            with open(zip_name, "wb") as w:
-                w.write(f.read())
-        with zipfile.ZipFile(zip_name, "r") as zip_ref:
-            zip_ref.extractall(".")
-        shutil.copyfile(
-            name + "/bin/amd64/" + dll_name,
-            PROJECT_CONFIG.get_distribution_path(
-                config.target_os, config.arch, dll_name, config.debug
-            ),
-        )
-        shutil.rmtree(name)
-        os.remove(zip_name)
-
-    get_dependency(
-        "https://www.wintun.net/builds/wintun-0.14.1.zip", "wintun", "wintun.dll"
-    )
-    get_dependency(
-        "https://download.wireguard.com/wireguard-nt/wireguard-nt-0.10.1.zip",
-        "wireguard-nt",
-        "wireguard.dll",
-    )
-
-    if moose:
-        sqlite_path = f"{PROJECT_ROOT}/3rd-party/libmoose/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/windows/{config.arch}/sqlite3.dll"
-        shutil.copyfile(
-            sqlite_path,
-            PROJECT_CONFIG.get_distribution_path(
-                config.target_os, config.arch, "sqlite3.dll", config.debug
-            ),
-        )
-
-
 def copy_bindings(config):
     if "binding_src" in LIBTELIO_CONFIG[config.target_os]:
         telio_bindings = f"{PROJECT_CONFIG.root_dir}/{LIBTELIO_CONFIG[config.target_os]['binding_src']}"
@@ -198,24 +159,12 @@ def copy_bindings(config):
     },
     "linux": {
         "archs": {
-            "x86_64": {
-                "strip_path": "/usr/bin/strip"
-            },
-            "aarch64": {
-                "strip_path": "/usr/aarch64-linux-gnu/bin/strip"
-            },
-            "arm64": {
-                "strip_path": "/usr/aarch64-linux-gnu/bin/strip"
-            },
-            "i686": {
-                "strip_path": "/usr/i686-linux-gnu/bin/strip"
-            },
-            "armv7": {
-                "strip_path": "/usr/arm-linux-gnueabihf/bin/strip"
-            },
-            "armv5": {
-                "strip_path": "/usr/arm-linux-gnueabi/bin/strip"
-            }
+            "x86_64": {"strip_path": "/usr/bin/strip"},
+            "aarch64": {"strip_path": "/usr/aarch64-linux-gnu/bin/strip"},
+            "arm64": {"strip_path": "/usr/aarch64-linux-gnu/bin/strip"},
+            "i686": {"strip_path": "/usr/i686-linux-gnu/bin/strip"},
+            "armv7": {"strip_path": "/usr/arm-linux-gnueabihf/bin/strip"},
+            "armv5": {"strip_path": "/usr/arm-linux-gnueabi/bin/strip"},
         },
         "env": {
             "RUSTFLAGS": ([" -C debuginfo=2 "], "set"),
@@ -252,7 +201,9 @@ def main() -> None:
     parser = rutils.create_cli_parser()
     build_parser = parser._subparsers._group_actions[0].choices["build"]
     build_parser.add_argument("--moose", action="store_true", help="Use libmoose")
-    build_parser.add_argument("--msvc", action="store_true", help="Use MSVC toolchain for Windows build")
+    build_parser.add_argument(
+        "--msvc", action="store_true", help="Use MSVC toolchain for Windows build"
+    )
 
     args = parser.parse_args()
 
@@ -300,8 +251,13 @@ def exec_build(args):
         moose_utils.unset_cargo_dependencies()
 
     if args.msvc:
-        GLOBAL_CONFIG["windows"]["archs"]["x86_64"]["rust_target"] = "x86_64-pc-windows-msvc"
-        GLOBAL_CONFIG["windows"]["env"]["RUSTFLAGS"] = ([" -C target-feature=-crt-static "], "set")
+        GLOBAL_CONFIG["windows"]["archs"]["x86_64"][
+            "rust_target"
+        ] = "x86_64-pc-windows-msvc"
+        GLOBAL_CONFIG["windows"]["env"]["RUSTFLAGS"] = (
+            [" -C target-feature=-crt-static "],
+            "set",
+        )
         if args.moose:
             moose_utils.create_msvc_import_library()
 
@@ -313,9 +269,6 @@ def exec_build(args):
     rutils.check_config(config)
     call_build(config)
 
-    if args.os == "windows":
-        finalize_win(config, args.moose)
-
 
 def create_debug_symbols(config):
     if config.debug:
@@ -366,6 +319,7 @@ def _create_debug_symbol(path: str, strip_bin: str):
         )
         _create_debug_symbol(f"{dist_dir}/{renamed_arch}/{lib_name}", strip_bin=strip)
 
+
 def strip_binaries(config):
     if config.debug or config.target_os != "linux":
         return
@@ -388,10 +342,13 @@ def _strip_debug_symbols(path: str, strip_bin: str):
         subprocess.check_call(strip_debug_symbols)
 
     strip = LIBTELIO_CONFIG["linux"]["archs"][config.arch]["strip_path"]
-    binaries = [bin for bin in LIBTELIO_CONFIG["linux"]["packages"].keys() if bin != NAME]
+    binaries = [
+        bin for bin in LIBTELIO_CONFIG["linux"]["packages"].keys() if bin != NAME
+    ]
     for binary in binaries:
         _strip_debug_symbols(f"{dist_dir}/{binary}", strip_bin=strip)
 
+
 def call_build(config):
     rutils.config_local_env_vars(config, LIBTELIO_CONFIG)
 

ci/env.py

@@ -8,13 +8,15 @@
 LIBTELIO_ENV_LINUX_BUILDER_TAG = "v0.0.2"
 LIBTELIO_ENV_WINDOWS_BUILDER_TAG = "v0.0.2"
 
+
 def set_sh():
     print(f"export LIBTELIO_ENV_MOOSE_RELEASE_TAG={LIBTELIO_ENV_MOOSE_RELEASE_TAG}")
     print(f"export LIBTELIO_ENV_NAT_LAB_DEPS_TAG={LIBTELIO_ENV_NAT_LAB_DEPS_TAG}")
     print(f"export LIBTELIO_ENV_ANDROID_BUILDER_TAG={LIBTELIO_ENV_ANDROID_BUILDER_TAG}")
     print(f"export LIBTELIO_ENV_LINUX_BUILDER_TAG={LIBTELIO_ENV_LINUX_BUILDER_TAG}")
     print(f"export LIBTELIO_ENV_WINDOWS_BUILDER_TAG={LIBTELIO_ENV_WINDOWS_BUILDER_TAG}")
 
+
 def set_ps1():
     print(f"$env:LIBTELIO_ENV_MOOSE_RELEASE_TAG=\"{LIBTELIO_ENV_MOOSE_RELEASE_TAG}\"")
     print(f"$env:LIBTELIO_ENV_NAT_LAB_DEPS_TAG=\"{LIBTELIO_ENV_NAT_LAB_DEPS_TAG}\"")

ci/moose_utils.py

@@ -7,8 +7,18 @@
 
 PROJECT_ROOT = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/..")
 
+
 def _output_dir(opsys: str, arch: str) -> str:
-    return os.path.join(PROJECT_ROOT, "3rd-party", "libmoose", LIBTELIO_ENV_MOOSE_RELEASE_TAG, "bin", "common", opsys, arch)
+    return os.path.join(
+        PROJECT_ROOT,
+        "3rd-party",
+        "libmoose",
+        LIBTELIO_ENV_MOOSE_RELEASE_TAG,
+        "bin",
+        "common",
+        opsys,
+        arch,
+    )
 
 
 def _download_moose_file(opsys: str, arch: str, file_name: str):
@@ -27,20 +37,16 @@ def _download_moose_file(opsys: str, arch: str, file_name: str):
     nexus_url = os.environ.get("LIBTELIO_ENV_SEC_NEXUS_URL", None)
 
     if nexus_credentials is None:
-        raise ValueError(
-            "LIBTELIO_ENV_SEC_NEXUS_CREDENTIALS not set"
-        )
+        raise ValueError("LIBTELIO_ENV_SEC_NEXUS_CREDENTIALS not set")
 
     if nexus_url is None:
-        raise ValueError(
-            "LIBTELIO_ENV_SEC_NEXUS_URL not set"
-        )
+        raise ValueError("LIBTELIO_ENV_SEC_NEXUS_URL not set")
 
-    url = (
-        f"{nexus_url}/repository/ll-gitlab-release/{MOOSE_PROJECT_ID}/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/{opsys}/{arch}/{file_name}"
-    )
+    url = f"{nexus_url}/repository/ll-gitlab-release/{MOOSE_PROJECT_ID}/{LIBTELIO_ENV_MOOSE_RELEASE_TAG}/bin/common/{opsys}/{arch}/{file_name}"
 
-    subprocess.check_call(["curl", "-f", "-u", nexus_credentials, url, "-o", output_path])
+    subprocess.check_call(
+        ["curl", "-f", "-u", nexus_credentials, url, "-o", output_path]
+    )
 
 
 def fetch_moose_dependencies(opsys: str, arch: str):
@@ -53,7 +59,7 @@ def fetch_moose_dependencies(opsys: str, arch: str):
 def create_msvc_import_library():
     def execute_dumpbin(file_path: str) -> list[str]:
         output = subprocess.check_output(["dumpbin", "/EXPORTS", file_path])
-        output_lines = output.decode().split('\n')[19:]
+        output_lines = output.decode().split("\n")[19:]
         for i, line in enumerate(output_lines):
             if not line.strip():
                 output_lines = output_lines[:i]
@@ -68,7 +74,9 @@ def write_exports(exports: list[str], file_path: str):
                 f.write(f"    {export}\n")
 
     def create_lib(def_path: str, lib_path: str):
-        subprocess.check_call(["lib", "/DEF:" + def_path, "/OUT:" + lib_path, "/MACHINE:X64"])
+        subprocess.check_call(
+            ["lib", "/DEF:" + def_path, "/OUT:" + lib_path, "/MACHINE:X64"]
+        )
 
     output_dir = _output_dir("windows", "x86_64")
     dll_path = os.path.join(output_dir, "sqlite3.dll")
@@ -80,19 +88,21 @@ def create_lib(def_path: str, lib_path: str):
     create_lib(def_path, lib_path)
 
 
-def _write_file(file_name, contents) :
+def _write_file(file_name, contents):
     with open(file_name, "w") as cargoFile:
         cargoFile.write(contents)
 
 
 def set_cargo_dependencies():
-    libtelio_env_sec_gitlab_repository = os.environ.get("LIBTELIO_ENV_SEC_GITLAB_REPOSITORY", None)
+    libtelio_env_sec_gitlab_repository = os.environ.get(
+        "LIBTELIO_ENV_SEC_GITLAB_REPOSITORY", None
+    )
 
     if libtelio_env_sec_gitlab_repository is None:
-        raise ValueError('LIBTELIO_ENV_SEC_GITLAB_REPOSITORY not set.')
+        raise ValueError("LIBTELIO_ENV_SEC_GITLAB_REPOSITORY not set.")
 
     MOOSEMESHNETAPP_DEP = (
-        r'\nmoosemeshnetapp = { '
+        r"\nmoosemeshnetapp = { "
         f'git = "https://{libtelio_env_sec_gitlab_repository}/low-level-hacks/moose/moose-events",'
         f' tag = "{LIBTELIO_ENV_MOOSE_RELEASE_TAG}" }}'
     )
@@ -104,23 +114,19 @@ def set_cargo_dependencies():
         if match_lana:
             if "features" not in match_lana.group(0):
                 replaced_moose = re.sub(
-                    r'( \})', r', features = ["moose"]\1',
-                    match_lana.group(0)
+                    r"( \})", r', features = ["moose"]\1', match_lana.group(0)
                 )
                 cargo_contents = cargo_contents.replace(
-                    match_lana.group(0),
-                    replaced_moose
-                );
+                    match_lana.group(0), replaced_moose
+                )
                 _write_file(f"{PROJECT_ROOT}/Cargo.toml", cargo_contents)
             elif '"moose"' not in match_lana.group(0):
                 replaced_moose = re.sub(
-                    r'(features.*\[)(.*\])', r'\1"moose", \2',
-                    match_lana.group(0)
+                    r"(features.*\[)(.*\])", r'\1"moose", \2', match_lana.group(0)
                 )
                 cargo_contents = cargo_contents.replace(
-                    match_lana.group(0),
-                    replaced_moose
-                );
+                    match_lana.group(0), replaced_moose
+                )
                 _write_file("./Cargo.toml", cargo_contents)
 
     # add moosemeshnetapp and moose feature dependency to telio-lana/Cargo.toml
@@ -130,20 +136,18 @@ def set_cargo_dependencies():
         if "moose = []" not in lana_cargo_contents:
             match_feature = re.search(r"\[features\]", lana_cargo_contents)
             if match_feature:
-                lana_cargo_contents += 'moose = []\n'
+                lana_cargo_contents += "moose = []\n"
             else:
-                lana_cargo_contents += '\n[features]\nmoose = []\n'
+                lana_cargo_contents += "\n[features]\nmoose = []\n"
 
         if "moosemeshnetapp" not in lana_cargo_contents:
             match_dependencies = re.search(r"\[dependencies\]", lana_cargo_contents)
             replaced_dependencies = re.sub(
-                r'$', MOOSEMESHNETAPP_DEP,
-                match_dependencies.group(0)
+                r"$", MOOSEMESHNETAPP_DEP, match_dependencies.group(0)
             )
             lana_cargo_contents = lana_cargo_contents.replace(
-                match_dependencies.group(0),
-                replaced_dependencies
-            );
+                match_dependencies.group(0), replaced_dependencies
+            )
         _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents)
 
 
@@ -154,14 +158,14 @@ def unset_cargo_dependencies():
         match_lana = re.search(r'telio-lana.*"moose"', cargo_contents)
         if match_lana:
             replaced_moose = re.sub(
-                r'(telio-lana.*)"moose"(, )*(.*})', r'\1\3',
-                cargo_contents
+                r'(telio-lana.*)"moose"(, )*(.*})', r"\1\3", cargo_contents
+            )
+            empty_features = re.search(
+                r'(telio-lana.*features.*)\[[^"]*\]', replaced_moose
             )
-            empty_features = re.search(r'(telio-lana.*features.*)\[[^"]*\]', replaced_moose)
             if empty_features:
                 replaced_moose = re.sub(
-                r'(telio-lana.*)(,\sfeatures.*)\[[^"]*\]', r'\1',
-                replaced_moose
+                    r'(telio-lana.*)(,\sfeatures.*)\[[^"]*\]', r"\1", replaced_moose
                 )
             _write_file(f"{PROJECT_ROOT}/Cargo.toml", replaced_moose)
 
@@ -170,21 +174,19 @@ def unset_cargo_dependencies():
         lana_cargo_contents = lana_cargo_file.read()
         if "moosemeshnetapp" in lana_cargo_contents:
             lana_cargo_contents = re.sub(
-                r'\nmoosemeshnetapp.*\n', '\n',
-                lana_cargo_contents
+                r"\nmoosemeshnetapp.*\n", "\n", lana_cargo_contents
+            )
+            _write_file(
+                f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents
             )
-            _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents)
         if "moose" in lana_cargo_contents:
-            empty_features = re.search(r'\[features\]\nmo', lana_cargo_contents)
+            empty_features = re.search(r"\[features\]\nmo", lana_cargo_contents)
             if empty_features:
                 lana_cargo_contents = re.sub(
-                    r'\n\[features\]\nmoose = \[\]\n', '',
-                    lana_cargo_contents
+                    r"\n\[features\]\nmoose = \[\]\n", "", lana_cargo_contents
                 )
             else:
-                lana_cargo_contents = re.sub(
-                    r'\nmoose.*\n', '\n',
-                    lana_cargo_contents
-                )
-            _write_file(f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents)
-
+                lana_cargo_contents = re.sub(r"\nmoose.*\n", "\n", lana_cargo_contents)
+            _write_file(
+                f"{PROJECT_ROOT}/crates/telio-lana/Cargo.toml", lana_cargo_contents
+            )