Merge pull request #873 from NordSecurity/LLT-5690_natlab_tcpdump_wra…

…pper

add tcpdump wrapper

.github/workflows/linters.yml

@@ -82,23 +82,23 @@ jobs:
             command: fmt
             args: -- --check --color always
   python-format-black:
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-24.04
       steps:
         - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         - uses: extractions/setup-just@dd310ad5a97d8e7b41793f8ef055398d51ad4de6 # v2.0.0
           with:
             just-version: 1.37.0
         - run: just black
   python-format-isort:
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-24.04
       steps:
         - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         - run: pip3 install --no-deps -r requirements.txt
         - run: pipenv install --system
         - run: isort --check-only --diff .
           working-directory: nat-lab
   python-format-autoflake:
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-24.04
       steps:
         - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         - run: pip3 install --no-deps -r requirements.txt
@@ -107,7 +107,7 @@ jobs:
           working-directory: nat-lab
   python-lint:
       needs: uniffi-bindings
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-24.04
       steps:
         - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
@@ -121,7 +121,7 @@ jobs:
         - run: just pylint
   natlab-typecheck:
       needs: uniffi-bindings
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-24.04
       steps:
         - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2

.gitignore

@@ -19,6 +19,7 @@ tcli.log
 /nat-lab/3rd-party/netfilter-full-cone-nat
 /nat-lab/tests/uniffi/telio_bindings.py
 /nat-lab/tests/uniffi/libtelio.so
+/nat-lab/tests/uniffi/telio.dll
 /nat-lab/logs
 /nat-lab/coredumps
 /src/telio.py

Justfile

@@ -65,7 +65,7 @@ black fix="false":
     fi
 
 pylint:
-    docker run --rm -t -v$(pwd):/code 'ubuntu:22.04' sh -c "apt-get update && apt-get -y install python3-pip && cd code && pip3 install --no-deps -r requirements.txt && pipenv install --system && cd nat-lab && pipenv install --system && pylint -f colorized . --ignore telio_bindings.py"
+    docker run --rm -t -v$(pwd):/code 'ubuntu:24.04' sh -c "apt-get update && apt-get -y install python3-pip && cd code && pip3 install --no-deps -r requirements.txt && pipenv install --system && cd nat-lab && pipenv install --system && pylint -f colorized . --ignore telio_bindings.py"
 
 # Start a dev web cgi server, for local teliod cgi development
 web:

nat-lab/bin/cleanup_natlab_processes

@@ -4,12 +4,16 @@ echo "Executing natlab process cleanup script"
 
 for pid in $(ps -e -o pid=); do
     # Skip non-testing processes
-    if ! grep --null-data --text KILL_ID /proc/${pid}/environ; then
+    if ! grep --null-data --text "KILL_ID" /proc/${pid}/environ; then
+        continue
+    fi
+
+    if grep --null-data --text "DO_NOT_KILL" /proc/${pid}/environ; then
         continue
     fi
 
     # KILL
-    cmd=$(tr -d '\000' < /proc/${pid}/cmdline || echo "N/A")
+    cmd=$(tr -d '\000' </proc/${pid}/cmdline || echo "N/A")
     echo "$(date) Cleaning up process: ${pid} ${cmd}"
     kill "${pid}" || echo "Kill command failed with $?, perhaps processs already exited?"
 done

nat-lab/bin/derp-server

@@ -2,6 +2,10 @@
 
 set -e
 
+# Backup IP tables
+iptables-save -f iptables_backup
+ip6tables-save -f ip6tables_backup
+
 /usr/bin/nordderper &
 
 sleep infinity

nat-lab/bin/dns-server.sh

@@ -1,2 +1,7 @@
 #!/bin/sh
+
+# Backup IP tables
+iptables-save -f iptables_backup
+ip6tables-save -f ip6tables_backup
+
 /opt/bin/dns-server 2>&1 | tee /dns-server.log

nat-lab/docker-compose.yml

@@ -424,6 +424,8 @@ services:
       - net.netfilter.nf_conntrack_tcp_timeout_fin_wait=15
     cap_drop:
       - ALL
+    cap_add:
+      - NET_ADMIN
     security_opt:
       - no-new-privileges:true
     networks:
@@ -550,6 +552,10 @@ services:
     hostname: dns-server-1
     image: nat-lab:base
     entrypoint: /opt/bin/dns-server.sh
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_ADMIN
     environment:
       PYTHONUNBUFFERED: 1
     networks:
@@ -562,6 +568,10 @@ services:
     hostname: dns-server-2
     image: nat-lab:base
     entrypoint: /opt/bin/dns-server.sh
+    cap_drop:
+      - ALL
+    cap_add:
+      - NET_ADMIN
     environment:
       PYTHONUNBUFFERED: 1
     networks:

nat-lab/tests/config.py

@@ -46,6 +46,8 @@ def get_root_path(path: str) -> str:
 IPERF_BINARY_MAC = "/var/root/iperf3/iperf3"
 IPERF_BINARY_WINDOWS = "C:/workspace/iperf3/iperf3.exe".replace("/", "\\")
 
+WINDUMP_BINARY_WINDOWS = "C:/workspace/WinDump.exe".replace("/", "\\")
+
 # JIRA issue: LLT-1664
 # The directories between host and Docker container are shared via
 # Docker volumes. Mounting `libtelio/dist` is a nogo, since when host

nat-lab/tests/conftest.py

@@ -5,25 +5,25 @@
 import shutil
 import subprocess
 from config import DERP_PRIMARY
+from contextlib import AsyncExitStack
 from datetime import datetime
 from helpers import SetupParameters
 from interderp_cli import InterDerpClient
 from itertools import combinations
-from mesh_api import start_tcpdump, stop_tcpdump
 from typing import Dict, List, Tuple
 from utils.bindings import TelioAdapterType
 from utils.connection import DockerConnection
 from utils.connection_util import (
-    ConnectionTag,
-    container_id,
     LAN_ADDR_MAP,
+    ConnectionTag,
     new_connection_raw,
     new_connection_with_conn_tracker,
 )
 from utils.ping import ping
 from utils.process import ProcessExecError
 from utils.router import IPStack
-from utils.vm import windows_vm_util, mac_vm_util
+from utils.tcpdump import make_tcpdump
+from utils.vm import mac_vm_util, windows_vm_util
 
 DERP_SERVER_1_ADDR = "http://10.0.10.1:8765"
 DERP_SERVER_2_ADDR = "http://10.0.10.2:8765"
@@ -36,6 +36,10 @@
 SETUP_CHECK_CONNECTIVITY_TIMEOUT = 60
 SETUP_CHECK_CONNECTIVITY_RETRIES = 1
 
+RUNNER = asyncio.Runner()
+# pylint: disable=unnecessary-dunder-call
+SESSION_SCOPE_EXIT_STACK = RUNNER.run(AsyncExitStack().__aenter__())
+
 
 def _cancel_all_tasks(loop: asyncio.AbstractEventLoop):
     to_cancel = asyncio.tasks.all_tasks(loop)
@@ -256,24 +260,28 @@ async def setup_check_connectivity():
 
 
 async def setup_check_interderp():
-    async with new_connection_raw(ConnectionTag.DOCKER_CONE_CLIENT_1) as connection:
-        if not isinstance(connection, DockerConnection):
-            raise Exception("Not docker connection")
-        containers = [
-            connection.container_name(),
-            "nat-lab-derp-01-1",
-            "nat-lab-derp-02-1",
-            "nat-lab-derp-03-1",
+    async with AsyncExitStack() as exit_stack:
+        connections = [
+            await exit_stack.enter_async_context(new_connection_raw(conn_tag))
+            for conn_tag in [
+                ConnectionTag.DOCKER_CONE_CLIENT_1,
+                ConnectionTag.DOCKER_DERP_1,
+                ConnectionTag.DOCKER_DERP_2,
+                ConnectionTag.DOCKER_DERP_3,
+            ]
         ]
-        start_tcpdump(containers)
-        try:
+
+        if not isinstance(connections[0], DockerConnection):
+            raise Exception("Not docker connection")
+
+        async with make_tcpdump(connections):
             for idx, (server1, server2) in enumerate(
                 combinations(
                     [DERP_SERVER_1_ADDR, DERP_SERVER_2_ADDR, DERP_SERVER_3_ADDR], 2
                 )
             ):
                 derp_test = InterDerpClient(
-                    connection,
+                    connections[0],
                     server1,
                     server2,
                     DERP_SERVER_1_SECRET_KEY,
@@ -282,8 +290,6 @@ async def setup_check_interderp():
                 )
                 await derp_test.execute()
                 await derp_test.save_logs()
-        finally:
-            stop_tcpdump(containers)
 
 
 SETUP_CHECKS = [
@@ -346,9 +352,6 @@ async def perform_pretest_cleanups():
 
 
 async def _copy_vm_binaries(tag: ConnectionTag):
-    is_ci = os.environ.get("CUSTOM_ENV_GITLAB_CI") is not None and os.environ.get(
-        "CUSTOM_ENV_GITLAB_CI"
-    )
     if tag in [ConnectionTag.WINDOWS_VM_1, ConnectionTag.WINDOWS_VM_2]:
         try:
             print(f"copying for {tag}")
@@ -357,7 +360,7 @@ async def _copy_vm_binaries(tag: ConnectionTag):
             ):
                 pass
         except OSError as e:
-            if is_ci:
+            if os.environ.get("GITLAB_CI"):
                 raise e
             print(e)
     elif tag is ConnectionTag.MAC_VM:
@@ -367,7 +370,7 @@ async def _copy_vm_binaries(tag: ConnectionTag):
             ):
                 pass
         except OSError as e:
-            if is_ci:
+            if os.environ.get("GITLAB_CI"):
                 raise e
             print(e)
 
@@ -432,10 +435,6 @@ async def _save_macos_logs(conn, suffix):
 
 
 async def collect_kernel_logs(items, suffix):
-    is_ci = os.environ.get("CUSTOM_ENV_GITLAB_CI") is not None and os.environ.get(
-        "CUSTOM_ENV_GITLAB_CI"
-    )
-
     log_dir = "logs"
     os.makedirs(log_dir, exist_ok=True)
 
@@ -448,7 +447,7 @@ async def collect_kernel_logs(items, suffix):
                 async with mac_vm_util.new_connection() as conn:
                     await _save_macos_logs(conn, suffix)
             except OSError as e:
-                if is_ci:
+                if os.environ.get("GITLAB_CI"):
                     raise e
 
 
@@ -467,39 +466,44 @@ def pytest_runtest_setup():
     asyncio.run(perform_pretest_cleanups())
 
 
-# pylint: disable=unused-argument
-def pytest_runtest_call(item):
-    start_tcpdump([f"nat-lab-dns-server-{i}-1" for i in range(1, 3)])
-
-
-# pylint: disable=unused-argument
-def pytest_runtest_makereport(item, call):
-    if call.when == "call":
-        stop_tcpdump([f"nat-lab-dns-server-{i}-1" for i in range(1, 3)])
-
-
 # pylint: disable=unused-argument
 def pytest_sessionstart(session):
     if os.environ.get("NATLAB_SAVE_LOGS") is None:
         return
 
-    if not session.config.option.collectonly:
-        start_tcpdump(
-            {container_id(gw_tag) for gw_tag in ConnectionTag if "_GW" in gw_tag.name}
+    async def async_context():
+        connections = [
+            await SESSION_SCOPE_EXIT_STACK.enter_async_context(
+                new_connection_raw(gw_tag)
+            )
+            for gw_tag in ConnectionTag
+            if "_GW" in gw_tag.name
+        ]
+
+        connections += [
+            await SESSION_SCOPE_EXIT_STACK.enter_async_context(
+                new_connection_raw(conn_tag)
+            )
+            for conn_tag in [
+                ConnectionTag.DOCKER_DNS_SERVER_1,
+                ConnectionTag.DOCKER_DNS_SERVER_2,
+            ]
+        ]
+
+        await SESSION_SCOPE_EXIT_STACK.enter_async_context(
+            make_tcpdump(connections, session=True)
         )
 
+    RUNNER.run(async_context())
+
 
 # pylint: disable=unused-argument
 def pytest_sessionfinish(session, exitstatus):
     if os.environ.get("NATLAB_SAVE_LOGS") is None:
         return
 
     if not session.config.option.collectonly:
-        stop_tcpdump(
-            {container_id(gw_tag) for gw_tag in ConnectionTag if "_GW" in gw_tag.name},
-            "./logs",
-        )
-
+        RUNNER.close()
         collect_nordderper_logs()
         collect_dns_server_logs()
         asyncio.run(collect_kernel_logs(session.items, "after_tests"))
@@ -533,7 +537,8 @@ def copy_file_from_container(container_name, src_path, dst_path):
     try:
         subprocess.run(docker_cp_command, shell=True, check=True)
         print(
-            f"Log file {src_path} copied successfully from {container_name} to {dst_path}"
+            f"Log file {src_path} copied successfully from {container_name} to"
+            f" {dst_path}"
         )
     except subprocess.CalledProcessError:
         print(f"Error copying log file {src_path} from {container_name} to {dst_path}")