From 518512bfca2069979c2729930d0261ecd3ef19fe Mon Sep 17 00:00:00 2001 From: Oskar Jung Date: Wed, 20 Nov 2024 09:20:38 -0500 Subject: [PATCH 01/15] symdb tests --- tests/debugger/test_debugger_symdb.py | 94 +++++++++++++++++++++++++++ tests/debugger/utils.py | 15 +++++ utils/_context/_scenarios/__init__.py | 16 +++++ utils/_remote_config.py | 57 +++++++++++++--- 4 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 tests/debugger/test_debugger_symdb.py diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py new file mode 100644 index 0000000000..327e6a0a77 --- /dev/null +++ b/tests/debugger/test_debugger_symdb.py @@ -0,0 +1,94 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +import os +import gzip +import json +import tests.debugger.utils as debugger +from utils import features, scenarios +from utils import remote_config as rc +from jsonschema import Draft7Validator + + +@features.debugger +@scenarios.debugger_symdb +class Test_Debugger_SymDb(debugger._Base_Debugger_Test): + ############ setup ############ + def _setup(self): + self.rc_state = rc.send_symdb_command() + + ############ assert ############ + def _assert(self): + self.collect() + self.assert_rc_state_not_error() + self._assert_symbols_uploaded() + + def _assert_symbols_uploaded(self): + assert len(self.symbols) > 0, "No symbol files were found" + + symbol_type = { + "type": "object", + "required": ["line", "name", "symbol_type", "type"], + "properties": { + "language_specifics": {"type": "object"}, + "line": {"type": "integer"}, + "name": {"type": "string"}, + "symbol_type": {"type": "string"}, + "type": {"type": ["string", "null"]}, + }, + } + + scope_type = { + "type": "object", + "required": ["end_line", "scope_type", "source_file", "start_line"], + "properties": { + "end_line": {"type": "integer"}, + "language_specifics": {"type": ["object", "null"]}, + "name": {"type": ["string", "null"]}, + "scope_type": {"type": "string"}, + "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, + "source_file": {"type": "string"}, + "start_line": {"type": "integer"}, + "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}}, + }, + } + + schema = { + "type": "object", + "required": ["env", "language", "scopes", "service", "version"], + "properties": { + "env": {"type": "string", "const": "system-tests"}, + "language": {"type": "string"}, + "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, + "service": {"type": "string", "const": "weblog"}, + "version": {"type": "string", "const": "1.0.0"}, + }, + "definitions": {"scope": scope_type, "symbol": symbol_type}, + } + + validator = Draft7Validator(schema) + + for file_path in self.symbols: + assert os.path.exists(file_path), f"Symbol file not found at {file_path}" + assert file_path.endswith(".gz"), f"Symbol file {file_path} is not a .gz file" + + try: + with gzip.open(file_path, "rb") as f: + content = json.loads(f.read().decode("utf-8")) + validation_errors = list(validator.iter_errors(content)) + assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( + f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" + for error in validation_errors + ) + except gzip.BadGzipFile: + assert False, f"File {file_path} is not a valid gzip archive" + except json.JSONDecodeError: + assert False, f"File {file_path} does not contain valid JSON" + + ############ test ############ + def setup_symdb_upload(self): + self._setup() + + def test_symdb_upload(self): + self._assert() diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index f3eeba8a3b..8edb071d69 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -7,6 +7,8 @@ import os import os.path import uuid +import gzip +import io from utils import interfaces, remote_config, weblog, context from utils.tools import logger @@ -17,6 +19,7 @@ _DEBUGGER_PATH = "/api/v2/debugger" _LOGS_PATH = "/api/v2/logs" _TRACES_PATH = "/api/v0.2/traces" +_SYMBOLS_PATH = "/symdb/v1/input" _CUR_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -65,6 +68,7 @@ class _Base_Debugger_Test: probe_diagnostics = {} probe_snapshots = {} probe_spans = {} + symbols = [] rc_state = None weblog_responses = [] @@ -261,6 +265,7 @@ def collect(self): self._collect_probe_diagnostics() self._collect_snapshots() self._collect_spans() + self._collect_symbols() def _collect_probe_diagnostics(self): def _read_data(): @@ -384,6 +389,16 @@ def _get_spans_hash(self): self.probe_spans = _get_spans_hash(self) + def _collect_symbols(self): + raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + + for data in raw_data: + if isinstance(data, dict) and "request" in data: + content = data["request"].get("content", []) + for part in content: + if isinstance(part, dict) and "system-tests-file-path" in part: + self.symbols.append(part["system-tests-file-path"]) + def get_tracer(self): if not _Base_Debugger_Test.tracer: _Base_Debugger_Test.tracer = { diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index bf1da0d643..f263a418d7 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -589,6 +589,22 @@ class _Scenarios: scenario_groups=[ScenarioGroup.DEBUGGER], ) + debugger_symdb = EndToEndScenario( + "DEBUGGER_SYMDB", + rc_api_enabled=True, + weblog_env={ + "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "1", + "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "1", + "DD_REMOTE_CONFIG_ENABLED": "true", + "DD_INTERNAL_RCM_POLL_INTERVAL": "2000", + "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1", + "DD_THIRD_PARTY_EXCLUDES": "org.springframework.samples.petclinic", + }, + library_interface_timeout=5, + doc="Test scenario for checking symdb.", + scenario_groups=[ScenarioGroup.DEBUGGER], + ) + fuzzer = DockerScenario("_FUZZER", doc="Fake scenario for fuzzing (launch without pytest)", github_workflow=None) # Single Step Instrumentation scenarios (HOST and CONTAINER) diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 3dfd652026..b43387ee87 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -163,17 +163,12 @@ def all_payload_sent(data) -> bool: library.wait_for(all_payload_sent, timeout=timeout) -def build_debugger_command(probes: list, version: int): - def _json_to_base64(json_object): - json_string = json.dumps(json_object).encode("utf-8") - return base64.b64encode(json_string).decode("utf-8") +def _create_base_rcm(): + return {"targets": "", "target_files": [], "client_configs": []} - def _sha256(value): - return hashlib.sha256(base64.b64decode(value)).hexdigest() - rcm = {"targets": "", "target_files": [], "client_configs": []} - - signed = { +def _create_base_signed(version: int): + return { "signed": { "_type": "targets", "custom": {"opaque_backend_state": "eyJmb28iOiAiYmFyIn0="}, # where does this come from ? @@ -192,6 +187,11 @@ def _sha256(value): ], } + +def build_debugger_command(probes: list | None, version: int): + rcm = _create_base_rcm() + signed = _create_base_signed(version) + if probes is None: rcm["targets"] = _json_to_base64(signed) else: @@ -201,7 +201,7 @@ def _sha256(value): probe_64 = _json_to_base64(probe) target["hashes"]["sha256"] = _sha256(probe_64) - target["length"] = len(json.dumps(probe).encode("utf-8")) + target["length"] = len(base64.b64decode(probe_64)) probe_path = re.sub(r"_([a-z])", lambda match: match.group(1).upper(), probe["type"].lower()) path = "datadog/2/LIVE_DEBUGGING/" + probe_path + "_" + probe["id"] + "/config" @@ -222,11 +222,48 @@ def send_debugger_command(probes: list, version: int) -> dict: return send_state(raw_payload) +def build_symdb_command(): + rcm = _create_base_rcm() + signed = _create_base_signed(version=1) + + target = {"custom": {"v": 1}, "hashes": {"sha256": ""}, "length": 0} + target_file = {"path": "", "raw": ""} + + payload = {"upload_symbols": True} + payload_64 = _json_to_base64(payload) + payload_length = len(base64.b64decode(payload_64)) + + path = "datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config" + + target["hashes"]["sha256"] = _sha256(payload_64) + target["length"] = payload_length + + signed["signed"]["targets"][path] = target + + target_file["path"] = path + target_file["raw"] = payload_64 + + rcm["target_files"].append(target_file) + rcm["client_configs"].append(path) + rcm["targets"] = _json_to_base64(signed) + + return rcm + + +def send_symdb_command() -> dict: + raw_payload = build_symdb_command() + return send_state(raw_payload) + + def _json_to_base64(json_object): json_string = json.dumps(json_object, indent=2).encode("utf-8") return base64.b64encode(json_string).decode("utf-8") +def _sha256(value): + return hashlib.sha256(base64.b64decode(value)).hexdigest() + + class ClientConfig: _store: dict[str, "ClientConfig"] = {} config_file_version: int = 1 From 15b5a088671764d91fc8dc649622e378f2213430 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Wed, 1 Jan 2025 16:32:23 +0200 Subject: [PATCH 02/15] fix test the test --- tests/debugger/test_debugger_symdb.py | 1 - tests/test_the_test/test_remote_config.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index 327e6a0a77..456b4eb23b 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -70,7 +70,6 @@ def _assert_symbols_uploaded(self): validator = Draft7Validator(schema) for file_path in self.symbols: - assert os.path.exists(file_path), f"Symbol file not found at {file_path}" assert file_path.endswith(".gz"), f"Symbol file {file_path} is not a .gz file" try: diff --git a/tests/test_the_test/test_remote_config.py b/tests/test_the_test/test_remote_config.py index 0c2fa781f7..ed3ad1cf3b 100644 --- a/tests/test_the_test/test_remote_config.py +++ b/tests/test_the_test/test_remote_config.py @@ -4,7 +4,7 @@ @scenarios.test_the_test def test_debugger_command_none(): expected = { - "targets": "eyJzaWduZWQiOiB7Il90eXBlIjogInRhcmdldHMiLCAiY3VzdG9tIjogeyJvcGFxdWVfYmFja2VuZF9zdGF0ZSI6ICJleUptYjI4aU9pQWlZbUZ5SW4wPSJ9LCAiZXhwaXJlcyI6ICIzMDAwLTAxLTAxVDAwOjAwOjAwWiIsICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwgInRhcmdldHMiOiB7fSwgInZlcnNpb24iOiAwfSwgInNpZ25hdHVyZXMiOiBbeyJrZXlpZCI6ICJlZDc2NzJjOWEyNGFiZGE3ODg3MmVlMzJlZTcxYzdjYjFkNTIzNWU4ZGI0ZWNiZjFjYTI4YjljNTBlYjc1ZDllIiwgInNpZyI6ICJlMjI3OWE1NTRkNTI1MDNmNWJkNjhlMGE5OTEwYzdlOTBjOWJiODE3NDRmZTljODgyNGVhMzczN2IyNzlkOWU2OWIzY2U1ZjRiNDYzYzQwMmViZTM0OTY0ZmI3YTY5NjI1ZWIwZTkxZDNkZGJkMzkyY2M4YjMyMTAzNzNkOWIwZiJ9XX0=", + "targets": "ewogICJzaWduZWQiOiB7CiAgICAiX3R5cGUiOiAidGFyZ2V0cyIsCiAgICAiY3VzdG9tIjogewogICAgICAib3BhcXVlX2JhY2tlbmRfc3RhdGUiOiAiZXlKbWIyOGlPaUFpWW1GeUluMD0iCiAgICB9LAogICAgImV4cGlyZXMiOiAiMzAwMC0wMS0wMVQwMDowMDowMFoiLAogICAgInNwZWNfdmVyc2lvbiI6ICIxLjAiLAogICAgInRhcmdldHMiOiB7fSwKICAgICJ2ZXJzaW9uIjogMAogIH0sCiAgInNpZ25hdHVyZXMiOiBbCiAgICB7CiAgICAgICJrZXlpZCI6ICJlZDc2NzJjOWEyNGFiZGE3ODg3MmVlMzJlZTcxYzdjYjFkNTIzNWU4ZGI0ZWNiZjFjYTI4YjljNTBlYjc1ZDllIiwKICAgICAgInNpZyI6ICJlMjI3OWE1NTRkNTI1MDNmNWJkNjhlMGE5OTEwYzdlOTBjOWJiODE3NDRmZTljODgyNGVhMzczN2IyNzlkOWU2OWIzY2U1ZjRiNDYzYzQwMmViZTM0OTY0ZmI3YTY5NjI1ZWIwZTkxZDNkZGJkMzkyY2M4YjMyMTAzNzNkOWIwZiIKICAgIH0KICBdCn0=", "target_files": [], "client_configs": [], } @@ -29,11 +29,11 @@ def test_debugger_command_one_probe(): ] expected = { - "targets": "eyJzaWduZWQiOiB7Il90eXBlIjogInRhcmdldHMiLCAiY3VzdG9tIjogeyJvcGFxdWVfYmFja2VuZF9zdGF0ZSI6ICJleUptYjI4aU9pQWlZbUZ5SW4wPSJ9LCAiZXhwaXJlcyI6ICIzMDAwLTAxLTAxVDAwOjAwOjAwWiIsICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwgInRhcmdldHMiOiB7ImRhdGFkb2cvMi9MSVZFX0RFQlVHR0lORy9sb2dQcm9iZV9sb2cxNzBhYS1hY2RhLTQ0NTMtOTExMS0xNDc4YTZtZXRob2QvY29uZmlnIjogeyJjdXN0b20iOiB7InYiOiAxfSwgImhhc2hlcyI6IHsic2hhMjU2IjogIjM1ZTQ3NzRhZmQzMWVhYWExMzAwNGU3MmU1ZTUyMWI3OGU3MGMxYTY5ZDBmZDI1MzI1NmU2ZTIwNTNkOGNkNjAifSwgImxlbmd0aCI6IDI0OX19LCAidmVyc2lvbiI6IDF9LCAic2lnbmF0dXJlcyI6IFt7ImtleWlkIjogImVkNzY3MmM5YTI0YWJkYTc4ODcyZWUzMmVlNzFjN2NiMWQ1MjM1ZThkYjRlY2JmMWNhMjhiOWM1MGViNzVkOWUiLCAic2lnIjogImUyMjc5YTU1NGQ1MjUwM2Y1YmQ2OGUwYTk5MTBjN2U5MGM5YmI4MTc0NGZlOWM4ODI0ZWEzNzM3YjI3OWQ5ZTY5YjNjZTVmNGI0NjNjNDAyZWJlMzQ5NjRmYjdhNjk2MjVlYjBlOTFkM2RkYmQzOTJjYzhiMzIxMDM3M2Q5YjBmIn1dfQ==", + "targets": "ewogICJzaWduZWQiOiB7CiAgICAiX3R5cGUiOiAidGFyZ2V0cyIsCiAgICAiY3VzdG9tIjogewogICAgICAib3BhcXVlX2JhY2tlbmRfc3RhdGUiOiAiZXlKbWIyOGlPaUFpWW1GeUluMD0iCiAgICB9LAogICAgImV4cGlyZXMiOiAiMzAwMC0wMS0wMVQwMDowMDowMFoiLAogICAgInNwZWNfdmVyc2lvbiI6ICIxLjAiLAogICAgInRhcmdldHMiOiB7CiAgICAgICJkYXRhZG9nLzIvTElWRV9ERUJVR0dJTkcvbG9nUHJvYmVfbG9nMTcwYWEtYWNkYS00NDUzLTkxMTEtMTQ3OGE2bWV0aG9kL2NvbmZpZyI6IHsKICAgICAgICAiY3VzdG9tIjogewogICAgICAgICAgInYiOiAxCiAgICAgICAgfSwKICAgICAgICAiaGFzaGVzIjogewogICAgICAgICAgInNoYTI1NiI6ICJlY2YzNDdmYjBlYTQ2MTZmZTU1NzZjMjI4M2FhZjU2ZTI2MWZiY2NkMzE2MmJhMjFmM2NmZDA0MmMzZWM5YWM1IgogICAgICAgIH0sCiAgICAgICAgImxlbmd0aCI6IDI4OQogICAgICB9CiAgICB9LAogICAgInZlcnNpb24iOiAxCiAgfSwKICAic2lnbmF0dXJlcyI6IFsKICAgIHsKICAgICAgImtleWlkIjogImVkNzY3MmM5YTI0YWJkYTc4ODcyZWUzMmVlNzFjN2NiMWQ1MjM1ZThkYjRlY2JmMWNhMjhiOWM1MGViNzVkOWUiLAogICAgICAic2lnIjogImUyMjc5YTU1NGQ1MjUwM2Y1YmQ2OGUwYTk5MTBjN2U5MGM5YmI4MTc0NGZlOWM4ODI0ZWEzNzM3YjI3OWQ5ZTY5YjNjZTVmNGI0NjNjNDAyZWJlMzQ5NjRmYjdhNjk2MjVlYjBlOTFkM2RkYmQzOTJjYzhiMzIxMDM3M2Q5YjBmIgogICAgfQogIF0KfQ==", "target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config", - "raw": "eyJsYW5ndWFnZSI6ICIiLCAiaWQiOiAibG9nMTcwYWEtYWNkYS00NDUzLTkxMTEtMTQ3OGE2bWV0aG9kIiwgInR5cGUiOiAiTE9HX1BST0JFIiwgIndoZXJlIjogeyJ0eXBlTmFtZSI6ICJBQ1RVQUxfVFlQRV9OQU1FIiwgIm1ldGhvZE5hbWUiOiAiUGlpIiwgInNvdXJjZUZpbGUiOiBudWxsfSwgImV2YWx1YXRlQXQiOiAiRVhJVCIsICJjYXB0dXJlU25hcHNob3QiOiB0cnVlLCAiY2FwdHVyZSI6IHsibWF4RmllbGRDb3VudCI6IDIwMH19", + "raw": "ewogICJsYW5ndWFnZSI6ICIiLAogICJpZCI6ICJsb2cxNzBhYS1hY2RhLTQ0NTMtOTExMS0xNDc4YTZtZXRob2QiLAogICJ0eXBlIjogIkxPR19QUk9CRSIsCiAgIndoZXJlIjogewogICAgInR5cGVOYW1lIjogIkFDVFVBTF9UWVBFX05BTUUiLAogICAgIm1ldGhvZE5hbWUiOiAiUGlpIiwKICAgICJzb3VyY2VGaWxlIjogbnVsbAogIH0sCiAgImV2YWx1YXRlQXQiOiAiRVhJVCIsCiAgImNhcHR1cmVTbmFwc2hvdCI6IHRydWUsCiAgImNhcHR1cmUiOiB7CiAgICAibWF4RmllbGRDb3VudCI6IDIwMAogIH0KfQ==", } ], "client_configs": ["datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config"], From 8225ee21b26eda0a1be0f0464570e9e98000890c Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Thu, 2 Jan 2025 13:28:53 +0200 Subject: [PATCH 03/15] format + yml --- manifests/cpp.yml | 2 ++ manifests/dotnet.yml | 2 ++ manifests/golang.yml | 2 ++ manifests/java.yml | 2 ++ manifests/nodejs.yml | 2 ++ manifests/php.yml | 2 ++ manifests/python.yml | 2 ++ manifests/ruby.yml | 4 ++++ tests/debugger/test_debugger_symdb.py | 4 ++-- 9 files changed, 20 insertions(+), 2 deletions(-) diff --git a/manifests/cpp.yml b/manifests/cpp.yml index c89f987641..f1284b0c12 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -105,6 +105,8 @@ tests/: Test_Debugger_Probe_Snaphots: irrelevant test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: irrelevant + test_debugger_symdb.py: + Test_Debugger_SymDb: irrelevant integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index a6e22bec4c..9f66ba1888 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -363,6 +363,8 @@ tests/: Test_Debugger_Probe_Snaphots: v2.53.0 test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: v2.53.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v2.53.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/golang.yml b/manifests/golang.yml index 7deb02009a..715b99cd0b 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -439,6 +439,8 @@ tests/: Test_Debugger_Probe_Snaphots: missing_feature (feature not implented) test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: missing_feature (feature not implented) + test_debugger_symdb.py: + Test_Debugger_SymDb: missing_feature (feature not implented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/java.yml b/manifests/java.yml index 2666fe7a79..c9d7d798d2 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1483,6 +1483,8 @@ tests/: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v1.38.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index dd6ade61ef..bab5d39a33 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -637,6 +637,8 @@ tests/: "*": irrelevant express4: v5.32.0 express4-typescript: v5.32.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: missing_feature (feature not implented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/php.yml b/manifests/php.yml index 48cc7e607b..82cf33c90d 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -277,6 +277,8 @@ tests/: Test_Debugger_Probe_Snaphots: irrelevant test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: irrelevant + test_debugger_symdb.py: + Test_Debugger_SymDb: irrelevant integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/python.yml b/manifests/python.yml index 281cd64ade..292e4103bc 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -617,6 +617,8 @@ tests/: flask-poc: v2.11.0 uds-flask: v2.11.0 uwsgi-poc: v2.11.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v2.11.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index b2d7ded6a1..088565948f 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -320,6 +320,10 @@ tests/: Test_Debugger_Probe_Statuses: "*": irrelevant rails70: v2.8.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: + "*": irrelevant + rails70: missing_feature (feature not implemented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index 456b4eb23b..b22a1130aa 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -2,11 +2,10 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -import os import gzip import json import tests.debugger.utils as debugger -from utils import features, scenarios +from utils import features, scenarios, bug, context from utils import remote_config as rc from jsonschema import Draft7Validator @@ -89,5 +88,6 @@ def _assert_symbols_uploaded(self): def setup_symdb_upload(self): self._setup() + @bug(context.library == "dotnet", reason="DEBUG-3298") def test_symdb_upload(self): self._assert() From a952670d06f7aff07b54628afd9d807696e9e380 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Thu, 2 Jan 2025 13:58:38 +0200 Subject: [PATCH 04/15] update expected --- .../rc_expected_requests_live_debugging.json | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/remote_config/rc_expected_requests_live_debugging.json b/tests/remote_config/rc_expected_requests_live_debugging.json index 25fdcbdb9c..0844fe79ff 100644 --- a/tests/remote_config/rc_expected_requests_live_debugging.json +++ b/tests/remote_config/rc_expected_requests_live_debugging.json @@ -38,11 +38,11 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] } @@ -68,11 +68,11 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 321, + "length": 365, "hashes": [ { "algorithm": "sha256", - "hash": "4180218218444e1e126a4e4ec41a475b89813d27537fba5c92662fe3510b951d" + "hash": "32c04cce9cd0820470f5b2d6d92609d51ad3c5619fa947fdc91445f641d977dc" } ] } @@ -113,21 +113,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -158,21 +158,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/spanProbe_kepf0cf2-9top-45cf-9f39-59installed/config", - "length": 164, + "length": 188, "hashes": [ { "algorithm": "sha256", - "hash": "8e897cbaccc2d604fbd5f4c910512e9c5d2ac7fee1f984929bf93e596d7a7461" + "hash": "d22df7cf36e9f2b0134c4f6535a7340b9a4435876b79280f91d80942c9562b5b" } ] } @@ -208,31 +208,31 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/spanProbe_kepf0cf2-9top-45cf-9f39-59installed/config", - "length": 164, + "length": 188, "hashes": [ { "algorithm": "sha256", - "hash": "8e897cbaccc2d604fbd5f4c910512e9c5d2ac7fee1f984929bf93e596d7a7461" + "hash": "d22df7cf36e9f2b0134c4f6535a7340b9a4435876b79280f91d80942c9562b5b" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 321, + "length": 365, "hashes": [ { "algorithm": "sha256", - "hash": "3712620153e0f4d41f03ca93fc8618b9f6714918a1f117cf110f25e6131eef52" + "hash": "4f12b33894fd7178f2464d3fc2c63223c3ee2a29a5cf0936de60ceee88fd0656" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -263,21 +263,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -308,21 +308,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -353,21 +353,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } From 5e8e0df23f6cf97d3726a1ef0d1639fae7ac0c93 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Sun, 5 Jan 2025 13:26:03 +0200 Subject: [PATCH 05/15] remove redundant env --- utils/_context/_scenarios/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index f263a418d7..f3c146b83e 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -597,8 +597,7 @@ class _Scenarios: "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "1", "DD_REMOTE_CONFIG_ENABLED": "true", "DD_INTERNAL_RCM_POLL_INTERVAL": "2000", - "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1", - "DD_THIRD_PARTY_EXCLUDES": "org.springframework.samples.petclinic", + "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1" }, library_interface_timeout=5, doc="Test scenario for checking symdb.", From 19ccc58fac8b8ba0333bb3cbe90557191821ba31 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Tue, 7 Jan 2025 13:46:31 +0200 Subject: [PATCH 06/15] format --- utils/_context/_scenarios/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index f3c146b83e..314aab2328 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -597,7 +597,7 @@ class _Scenarios: "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "1", "DD_REMOTE_CONFIG_ENABLED": "true", "DD_INTERNAL_RCM_POLL_INTERVAL": "2000", - "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1" + "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1", }, library_interface_timeout=5, doc="Test scenario for checking symdb.", From bc8947719e7ad23e08c6fd6e474c6f61efdd7a69 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Sun, 19 Jan 2025 22:33:11 +0200 Subject: [PATCH 07/15] schema + re-use rc methods --- tests/debugger/test_debugger_symdb.py | 18 ++--- utils/_remote_config.py | 80 +++++++++---------- .../agent/api/v2/debugger-request.json | 26 ++++-- .../library/symdb/v1/input-request.json | 30 +++++++ 4 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 utils/interfaces/schemas/library/symdb/v1/input-request.json diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index b22a1130aa..fbf8c245fc 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -2,7 +2,6 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -import gzip import json import tests.debugger.utils as debugger from utils import features, scenarios, bug, context @@ -69,21 +68,18 @@ def _assert_symbols_uploaded(self): validator = Draft7Validator(schema) for file_path in self.symbols: - assert file_path.endswith(".gz"), f"Symbol file {file_path} is not a .gz file" - try: - with gzip.open(file_path, "rb") as f: + with open(file_path, "rb") as f: content = json.loads(f.read().decode("utf-8")) - validation_errors = list(validator.iter_errors(content)) - assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( - f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" - for error in validation_errors - ) - except gzip.BadGzipFile: - assert False, f"File {file_path} is not a valid gzip archive" except json.JSONDecodeError: assert False, f"File {file_path} does not contain valid JSON" + validation_errors = list(validator.iter_errors(content)) + assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( + f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" + for error in validation_errors + ) + ############ test ############ def setup_symdb_upload(self): self._setup() diff --git a/utils/_remote_config.py b/utils/_remote_config.py index b43387ee87..10872fb4cf 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -188,66 +188,60 @@ def _create_base_signed(version: int): } -def build_debugger_command(probes: list | None, version: int): +def _build_base_command(path_payloads: dict[str, dict | list], version: int): + """Helper function to build a remote config command with common logic. + + Args: + path_payloads: Dictionary mapping paths to their corresponding payloads + version: The version number for the signed data + """ rcm = _create_base_rcm() signed = _create_base_signed(version) - if probes is None: + if not path_payloads: rcm["targets"] = _json_to_base64(signed) - else: - for probe in probes: - target = {"custom": {"v": 1}, "hashes": {"sha256": ""}, "length": 0} - target_file = {"path": "", "raw": ""} - - probe_64 = _json_to_base64(probe) - target["hashes"]["sha256"] = _sha256(probe_64) - target["length"] = len(base64.b64decode(probe_64)) + return rcm - probe_path = re.sub(r"_([a-z])", lambda match: match.group(1).upper(), probe["type"].lower()) - path = "datadog/2/LIVE_DEBUGGING/" + probe_path + "_" + probe["id"] + "/config" - signed["signed"]["targets"][path] = target + for path, payload in path_payloads.items(): + payload_64 = _json_to_base64(payload) + payload_length = len(base64.b64decode(payload_64)) - target_file["path"] = path - target_file["raw"] = probe_64 + target = {"custom": {"v": 1}, "hashes": {"sha256": ""}, "length": 0} + target["hashes"]["sha256"] = _sha256(payload_64) + target["length"] = payload_length + signed["signed"]["targets"][path] = target - rcm["target_files"].append(target_file) - rcm["client_configs"].append(path) + target_file = {"path": path, "raw": payload_64} + rcm["target_files"].append(target_file) + rcm["client_configs"].append(path) - rcm["targets"] = _json_to_base64(signed) + rcm["targets"] = _json_to_base64(signed) return rcm -def send_debugger_command(probes: list, version: int) -> dict: - raw_payload = build_debugger_command(probes, version) - return send_state(raw_payload) - - -def build_symdb_command(): - rcm = _create_base_rcm() - signed = _create_base_signed(version=1) - - target = {"custom": {"v": 1}, "hashes": {"sha256": ""}, "length": 0} - target_file = {"path": "", "raw": ""} - - payload = {"upload_symbols": True} - payload_64 = _json_to_base64(payload) - payload_length = len(base64.b64decode(payload_64)) +def build_debugger_command(probes: list | None, version: int): + if probes is None: + return _build_base_command({}, version) - path = "datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config" + path_payloads = {} + for probe in probes: + probe_path = re.sub(r"_([a-z])", lambda match: match.group(1).upper(), probe["type"].lower()) + path = f"datadog/2/LIVE_DEBUGGING/{probe_path}_{probe['id']}/config" + path_payloads[path] = probe - target["hashes"]["sha256"] = _sha256(payload_64) - target["length"] = payload_length + return _build_base_command(path_payloads, version) - signed["signed"]["targets"][path] = target - target_file["path"] = path - target_file["raw"] = payload_64 +def build_symdb_command(): + path_payloads = { + "datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config": {"upload_symbols": True} + } + return _build_base_command(path_payloads, version=1) - rcm["target_files"].append(target_file) - rcm["client_configs"].append(path) - rcm["targets"] = _json_to_base64(signed) - return rcm +def send_debugger_command(probes: list, version: int) -> dict: + raw_payload = build_debugger_command(probes, version) + return send_state(raw_payload) def send_symdb_command() -> dict: diff --git a/utils/interfaces/schemas/agent/api/v2/debugger-request.json b/utils/interfaces/schemas/agent/api/v2/debugger-request.json index 4d8177052c..0a26fccb63 100644 --- a/utils/interfaces/schemas/agent/api/v2/debugger-request.json +++ b/utils/interfaces/schemas/agent/api/v2/debugger-request.json @@ -3,11 +3,27 @@ "type": "array", "items": { "type": "object", - "properties":{ + "anyOf": [ + { + "required": ["content"] + }, + { + "required": ["headers"] + } + ], + "properties": { "content": { - "type": "array" + "type": [ + "array", + "object" + ] + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { "type": "string" } + } } - }, - "required": ["content"] + } } -} +} \ No newline at end of file diff --git a/utils/interfaces/schemas/library/symdb/v1/input-request.json b/utils/interfaces/schemas/library/symdb/v1/input-request.json new file mode 100644 index 0000000000..6952ffd7ba --- /dev/null +++ b/utils/interfaces/schemas/library/symdb/v1/input-request.json @@ -0,0 +1,30 @@ +{ + "$id": "/library/symdb/v1/input-request.json", + "type": "array", + "items": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "patternProperties": { + ".*": { "type": "string" } + } + }, + "content": { + "type": ["array", "object"], + "items": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": { + "Content-Type": { "type": "string" }, + "Content-Disposition": { "type": "string" } + } + } + } + } + } + } + } +} \ No newline at end of file From 4da86a30fe94f231ba921eea18bbbccbef120bfd Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Sun, 19 Jan 2025 22:40:42 +0200 Subject: [PATCH 08/15] format --- tests/debugger/test_debugger_symdb.py | 3 +-- utils/_remote_config.py | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index fbf8c245fc..589247786a 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -76,8 +76,7 @@ def _assert_symbols_uploaded(self): validation_errors = list(validator.iter_errors(content)) assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( - f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" - for error in validation_errors + f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" for error in validation_errors ) ############ test ############ diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 10872fb4cf..bf1b5f1cb5 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -190,10 +190,11 @@ def _create_base_signed(version: int): def _build_base_command(path_payloads: dict[str, dict | list], version: int): """Helper function to build a remote config command with common logic. - + Args: path_payloads: Dictionary mapping paths to their corresponding payloads version: The version number for the signed data + """ rcm = _create_base_rcm() signed = _create_base_signed(version) @@ -233,9 +234,7 @@ def build_debugger_command(probes: list | None, version: int): def build_symdb_command(): - path_payloads = { - "datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config": {"upload_symbols": True} - } + path_payloads = {"datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config": {"upload_symbols": True}} return _build_base_command(path_payloads, version=1) From e9b78c4a41284d01a32997b19e6306ab950893e7 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Mon, 20 Jan 2025 15:47:33 +0100 Subject: [PATCH 09/15] always consider data inside /symdb/v1/input as json --- utils/proxy/_deserializer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 3b0a682a59..7477465d2c 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -205,10 +205,10 @@ def json_load(): item["system-tests-error"] = "Can't decompress gzip data" continue - _deserialize_file_in_multipart_form_data(item, headers, export_content_files_to, content) + _deserialize_file_in_multipart_form_data(path, item, headers, export_content_files_to, content) elif content_type_part == "application/octet-stream": - _deserialize_file_in_multipart_form_data(item, headers, export_content_files_to, part.content) + _deserialize_file_in_multipart_form_data(path, item, headers, export_content_files_to, part.content) else: try: @@ -227,7 +227,7 @@ def json_load(): def _deserialize_file_in_multipart_form_data( - item: dict, headers: dict, export_content_files_to: str, content: bytes + path: str, item: dict, headers: dict, export_content_files_to: str, content: bytes ) -> None: content_disposition = headers.get("Content-Disposition", "") @@ -253,7 +253,10 @@ def _deserialize_file_in_multipart_form_data( filename = filename[:-3] content_is_deserialized = False - if filename.lower().endswith(".json"): + if filename.lower().endswith(".json") or path == "/symdb/v1/input": + # when path == /symdb/v1/input, the content may be either raw json, or gizipped json + # though, the file name may not always contains .json, so for this use case + # we always try to deserialize the content as json try: item["content"] = json.loads(content) content_is_deserialized = True From b138673060be779e03f4974b51e2678ef9764656 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Tue, 21 Jan 2025 08:30:54 +0200 Subject: [PATCH 10/15] validate schema --- tests/debugger/test_debugger_symdb.py | 62 +++------------- tests/debugger/utils.py | 22 +++--- .../library/symdb/v1/input-request.json | 70 ++++++++++++++----- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index 589247786a..b96ee876ae 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -25,64 +25,20 @@ def _assert(self): def _assert_symbols_uploaded(self): assert len(self.symbols) > 0, "No symbol files were found" - symbol_type = { - "type": "object", - "required": ["line", "name", "symbol_type", "type"], - "properties": { - "language_specifics": {"type": "object"}, - "line": {"type": "integer"}, - "name": {"type": "string"}, - "symbol_type": {"type": "string"}, - "type": {"type": ["string", "null"]}, - }, - } + errors = [] + for symbol in self.symbols: + error = symbol.get("system-tests-error") + if error is not None: + errors.append( + f"Error is: {error}, exported to file: {symbol.get('system-tests-file-path', 'No file path')}" + ) - scope_type = { - "type": "object", - "required": ["end_line", "scope_type", "source_file", "start_line"], - "properties": { - "end_line": {"type": "integer"}, - "language_specifics": {"type": ["object", "null"]}, - "name": {"type": ["string", "null"]}, - "scope_type": {"type": "string"}, - "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, - "source_file": {"type": "string"}, - "start_line": {"type": "integer"}, - "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}}, - }, - } - - schema = { - "type": "object", - "required": ["env", "language", "scopes", "service", "version"], - "properties": { - "env": {"type": "string", "const": "system-tests"}, - "language": {"type": "string"}, - "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, - "service": {"type": "string", "const": "weblog"}, - "version": {"type": "string", "const": "1.0.0"}, - }, - "definitions": {"scope": scope_type, "symbol": symbol_type}, - } - - validator = Draft7Validator(schema) - - for file_path in self.symbols: - try: - with open(file_path, "rb") as f: - content = json.loads(f.read().decode("utf-8")) - except json.JSONDecodeError: - assert False, f"File {file_path} does not contain valid JSON" - - validation_errors = list(validator.iter_errors(content)) - assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( - f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" for error in validation_errors - ) + assert not errors, "Found system-tests-errors:\n" + "\n".join(f"- {err}" for err in errors) ############ test ############ def setup_symdb_upload(self): self._setup() - @bug(context.library == "dotnet", reason="DEBUG-3298") + # @bug(context.library == "dotnet", reason="DEBUG-3298") def test_symdb_upload(self): self._assert() diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index 2012427540..c41754d8eb 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -390,14 +390,20 @@ def _get_spans_hash(self): self.probe_spans = _get_spans_hash(self) def _collect_symbols(self): - raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) - - for data in raw_data: - if isinstance(data, dict) and "request" in data: - content = data["request"].get("content", []) - for part in content: - if isinstance(part, dict) and "system-tests-file-path" in part: - self.symbols.append(part["system-tests-file-path"]) + def _get_symbols(): + result = [] + raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + + for data in raw_data: + if isinstance(data, dict) and "request" in data: + contents = data["request"].get("content", []) + for content in contents: + if isinstance(content, dict) and "system-tests-filename" in content: + result.append(content) + + return result + + self.symbols = _get_symbols() def get_tracer(self): if not _Base_Debugger_Test.tracer: diff --git a/utils/interfaces/schemas/library/symdb/v1/input-request.json b/utils/interfaces/schemas/library/symdb/v1/input-request.json index 6952ffd7ba..dd7b66c0c4 100644 --- a/utils/interfaces/schemas/library/symdb/v1/input-request.json +++ b/utils/interfaces/schemas/library/symdb/v1/input-request.json @@ -1,30 +1,68 @@ { "$id": "/library/symdb/v1/input-request.json", "type": "array", - "items": { + "definitions": { + "scope": { "type": "object", + "required": ["end_line", "scope_type", "source_file", "start_line"], "properties": { - "headers": { + "end_line": {"type": "integer"}, + "language_specifics": {"type": ["object", "null"]}, + "name": {"type": ["string", "null"]}, + "scope_type": {"type": "string"}, + "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, + "source_file": {"type": "string"}, + "start_line": {"type": "integer"}, + "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}} + } + }, + "symbol": { + "type": "object", + "required": ["line", "name", "symbol_type", "type"], + "properties": { + "language_specifics": {"type": "object"}, + "line": {"type": "integer"}, + "name": {"type": "string"}, + "symbol_type": {"type": "string"}, + "type": {"type": ["string", "null"]} + } + } + }, + "items": { + "type": "object", + "required": ["headers", "content"], + "properties": { + "headers": { + "type": "object", + "patternProperties": { + ".*": { "type": "string" } + } + }, + "content": { + "oneOf": [ + { "type": "object", - "patternProperties": { - ".*": { "type": "string" } + "required": ["ddsource", "service", "runtimeId"], + "properties": { + "ddsource": { "type": "string" }, + "service": { "type": "string" }, + "runtimeId": { "type": "string" } } }, - "content": { - "type": ["array", "object"], - "items": { + { "type": "object", + "required": ["env", "language", "scopes", "service", "version"], "properties": { - "headers": { - "type": "object", - "properties": { - "Content-Type": { "type": "string" }, - "Content-Disposition": { "type": "string" } - } - } + "env": {"type": "string", "const": "system-tests"}, + "language": {"type": "string"}, + "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, + "service": {"type": "string", "const": "weblog"}, + "version": {"type": "string", "const": "1.0.0"} } } - } - } + ] + }, + "system-tests-filename": { "type": "string" } + } } } \ No newline at end of file From d59d519081897b5f9d4bcec56b310259ba6086db Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Tue, 21 Jan 2025 09:26:11 +0200 Subject: [PATCH 11/15] mark dotnet as bug --- tests/debugger/test_debugger_symdb.py | 2 +- utils/_context/_scenarios/endtoend.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index b96ee876ae..6ee1c72f72 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -39,6 +39,6 @@ def _assert_symbols_uploaded(self): def setup_symdb_upload(self): self._setup() - # @bug(context.library == "dotnet", reason="DEBUG-3298") + @bug(context.library == "dotnet", reason="DEBUG-3298") def test_symdb_upload(self): self._assert() diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 2c2c9bcdb8..18d9d11225 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -593,6 +593,12 @@ def pytest_sessionfinish(self, session, exitstatus): and self.name == "DEBUGGER_EXPRESSION_LANGUAGE", ticket="APMRP-360", ), + _SchemaBug( + endpoint="/symdb/v1/input", + data_path="$", + condition=context.library == "dotnet", + ticket="DEBUG-3246", + ), ] self._test_schemas(session, interfaces.library, library_bugs) @@ -625,6 +631,12 @@ def pytest_sessionfinish(self, session, exitstatus): condition=context.library < "nodejs@5.31.0", ticket="DEBUG-2864", ), + _SchemaBug( + endpoint="/api/v2/debugger", + data_path="$", + condition=context.library == "dotnet", + ticket="DEBUG-3246", + ), ] self._test_schemas(session, interfaces.agent, agent_bugs) From 1fd4f27c17d02d760ea9ec11ad8cabe412894d60 Mon Sep 17 00:00:00 2001 From: "Alexander S." Date: Tue, 21 Jan 2025 16:39:35 +0200 Subject: [PATCH 12/15] remove express4-typescript: v5.32.0 Co-authored-by: Thomas Watson --- manifests/nodejs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 331c3c3921..48121fa657 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -635,7 +635,6 @@ tests/: Test_Debugger_Probe_Statuses: "*": irrelevant express4: v5.32.0 - express4-typescript: v5.32.0 test_debugger_symdb.py: Test_Debugger_SymDb: missing_feature (feature not implented) integrations/: From 3da9e91cd453aa97f96ac789fafe98ebe2638d37 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Tue, 21 Jan 2025 21:54:35 +0200 Subject: [PATCH 13/15] fix schema validations --- tests/debugger/test_debugger_symdb.py | 1 - utils/_context/_scenarios/endtoend.py | 18 +-- .../agent/api/v2/debugger-request.json | 26 +--- .../library/symdb/v1/input-request.json | 125 +++++++++--------- utils/proxy/_deserializer.py | 4 +- 5 files changed, 82 insertions(+), 92 deletions(-) diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py index 6ee1c72f72..0a3ca60958 100644 --- a/tests/debugger/test_debugger_symdb.py +++ b/tests/debugger/test_debugger_symdb.py @@ -6,7 +6,6 @@ import tests.debugger.utils as debugger from utils import features, scenarios, bug, context from utils import remote_config as rc -from jsonschema import Draft7Validator @features.debugger diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 18d9d11225..1c3385d643 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -38,7 +38,7 @@ @dataclass class _SchemaBug: endpoint: str - data_path: str + data_path: str | None # None means that all data_path will be considered as bug condition: bool ticket: str @@ -595,9 +595,10 @@ def pytest_sessionfinish(self, session, exitstatus): ), _SchemaBug( endpoint="/symdb/v1/input", - data_path="$", - condition=context.library == "dotnet", - ticket="DEBUG-3246", + data_path=None, + condition=context.library == "dotnet" + and self.name == "DEBUGGER_SYMDB", + ticket="DEBUG-3298", ), ] self._test_schemas(session, interfaces.library, library_bugs) @@ -633,9 +634,10 @@ def pytest_sessionfinish(self, session, exitstatus): ), _SchemaBug( endpoint="/api/v2/debugger", - data_path="$", - condition=context.library == "dotnet", - ticket="DEBUG-3246", + data_path="$[]", + condition=context.library == "dotnet" + and self.name == "DEBUGGER_SYMDB", + ticket="DEBUG-3298", ), ] self._test_schemas(session, interfaces.agent, agent_bugs) @@ -648,7 +650,7 @@ def _test_schemas(self, session, interface: ProxyBasedInterfaceValidator, known_ excluded_points = {(bug.endpoint, bug.data_path) for bug in known_bugs if bug.condition} for error in interface.get_schemas_errors(): - if (error.endpoint, error.data_path) not in excluded_points: + if (error.endpoint, error.data_path) not in excluded_points and (error.endpoint, None) not in excluded_points: long_repr.append(f"* {error.message}") if len(long_repr) != 0: diff --git a/utils/interfaces/schemas/agent/api/v2/debugger-request.json b/utils/interfaces/schemas/agent/api/v2/debugger-request.json index 0a26fccb63..701f765db3 100644 --- a/utils/interfaces/schemas/agent/api/v2/debugger-request.json +++ b/utils/interfaces/schemas/agent/api/v2/debugger-request.json @@ -3,27 +3,11 @@ "type": "array", "items": { "type": "object", - "anyOf": [ - { - "required": ["content"] - }, - { - "required": ["headers"] - } - ], - "properties": { + "properties":{ "content": { - "type": [ - "array", - "object" - ] - }, - "headers": { - "type": "object", - "patternProperties": { - ".*": { "type": "string" } - } + "type": ["array", "object"] } - } + }, + "required": ["content"] } -} \ No newline at end of file +} diff --git a/utils/interfaces/schemas/library/symdb/v1/input-request.json b/utils/interfaces/schemas/library/symdb/v1/input-request.json index dd7b66c0c4..1b306315cc 100644 --- a/utils/interfaces/schemas/library/symdb/v1/input-request.json +++ b/utils/interfaces/schemas/library/symdb/v1/input-request.json @@ -1,68 +1,73 @@ { - "$id": "/library/symdb/v1/input-request.json", - "type": "array", - "definitions": { - "scope": { - "type": "object", - "required": ["end_line", "scope_type", "source_file", "start_line"], - "properties": { - "end_line": {"type": "integer"}, - "language_specifics": {"type": ["object", "null"]}, - "name": {"type": ["string", "null"]}, - "scope_type": {"type": "string"}, - "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, - "source_file": {"type": "string"}, - "start_line": {"type": "integer"}, - "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}} + "$id": "/library/symdb/v1/input-request.json", + "definitions": { + "scope": { + "type": "object", + "required": ["end_line", "scope_type", "source_file", "start_line"], + "properties": { + "end_line": {"type": "integer"}, + "language_specifics": {"type": ["object", "null"]}, + "name": {"type": ["string", "null"]}, + "scope_type": {"type": "string"}, + "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, + "source_file": {"type": "string"}, + "start_line": {"type": "integer"}, + "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}} + } + }, + "symbol": { + "type": "object", + "required": ["line", "name", "symbol_type", "type"], + "properties": { + "language_specifics": {"type": "object"}, + "line": {"type": "integer"}, + "name": {"type": "string"}, + "symbol_type": {"type": "string"}, + "type": {"type": ["string", "null"]} + } + }, + "json_content": { + "type": "object", + "required": ["ddsource", "service", "runtimeId"], + "properties": { + "ddsource": { "type": "string" }, + "service": { "type": "string" }, + "runtimeId": { "type": "string" } + } + }, + "gzip_content": { + "type": "object", + "required": ["env", "language", "scopes", "service", "version"], + "properties": { + "env": {"type": "string", "const": "system-tests"}, + "language": {"type": "string"}, + "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, + "service": {"type": "string", "const": "weblog"}, + "version": {"type": "string", "const": "1.0.0"} + } } }, - "symbol": { + "type": "array", + "items": { "type": "object", - "required": ["line", "name", "symbol_type", "type"], + "required": ["headers", "content"], "properties": { - "language_specifics": {"type": "object"}, - "line": {"type": "integer"}, - "name": {"type": "string"}, - "symbol_type": {"type": "string"}, - "type": {"type": ["string", "null"]} - } - } - }, - "items": { - "type": "object", - "required": ["headers", "content"], - "properties": { - "headers": { - "type": "object", - "patternProperties": { - ".*": { "type": "string" } - } - }, - "content": { - "oneOf": [ - { - "type": "object", - "required": ["ddsource", "service", "runtimeId"], - "properties": { - "ddsource": { "type": "string" }, - "service": { "type": "string" }, - "runtimeId": { "type": "string" } - } - }, - { - "type": "object", - "required": ["env", "language", "scopes", "service", "version"], - "properties": { - "env": {"type": "string", "const": "system-tests"}, - "language": {"type": "string"}, - "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, - "service": {"type": "string", "const": "weblog"}, - "version": {"type": "string", "const": "1.0.0"} - } + "headers": { + "type": "object", + "patternProperties": { + ".*": { "type": "string" } } - ] + } }, - "system-tests-filename": { "type": "string" } + "allOf": [ + { + "if": {"properties": {"headers": {"properties": {"content-type": {"const": "application/json"}}}}}, + "then": {"properties": {"content": {"$ref": "#/definitions/json_content"}}} + }, + { + "if": {"properties": {"headers": {"properties": {"content-type": {"const": "application/gzip"}}}}}, + "then": {"properties": {"content": {"$ref": "#/definitions/gzip_content"}}} + } + ] } - } -} \ No newline at end of file + } \ No newline at end of file diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 7477465d2c..0d6b884014 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -253,8 +253,8 @@ def _deserialize_file_in_multipart_form_data( filename = filename[:-3] content_is_deserialized = False - if filename.lower().endswith(".json") or path == "/symdb/v1/input": - # when path == /symdb/v1/input, the content may be either raw json, or gizipped json + if filename.lower().endswith(".json") or path == "/symdb/v1/input" or path == "/api/v2/debugger": + # when path == /symdb/v1/input or /api/v2/debugger, the content may be either raw json, or gizipped json # though, the file name may not always contains .json, so for this use case # we always try to deserialize the content as json try: From b41cbf09f766fed04b46523d7b8c6211d19594e6 Mon Sep 17 00:00:00 2001 From: Alexander Seleznyov Date: Wed, 22 Jan 2025 07:00:00 +0200 Subject: [PATCH 14/15] format --- utils/_context/_scenarios/endtoend.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 1c3385d643..f91e08434d 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -596,8 +596,7 @@ def pytest_sessionfinish(self, session, exitstatus): _SchemaBug( endpoint="/symdb/v1/input", data_path=None, - condition=context.library == "dotnet" - and self.name == "DEBUGGER_SYMDB", + condition=context.library == "dotnet" and self.name == "DEBUGGER_SYMDB", ticket="DEBUG-3298", ), ] @@ -635,8 +634,7 @@ def pytest_sessionfinish(self, session, exitstatus): _SchemaBug( endpoint="/api/v2/debugger", data_path="$[]", - condition=context.library == "dotnet" - and self.name == "DEBUGGER_SYMDB", + condition=context.library == "dotnet" and self.name == "DEBUGGER_SYMDB", ticket="DEBUG-3298", ), ] @@ -650,7 +648,10 @@ def _test_schemas(self, session, interface: ProxyBasedInterfaceValidator, known_ excluded_points = {(bug.endpoint, bug.data_path) for bug in known_bugs if bug.condition} for error in interface.get_schemas_errors(): - if (error.endpoint, error.data_path) not in excluded_points and (error.endpoint, None) not in excluded_points: + if (error.endpoint, error.data_path) not in excluded_points and ( + error.endpoint, + None, + ) not in excluded_points: long_repr.append(f"* {error.message}") if len(long_repr) != 0: From f46005beaaa4e5b0e97d80d1846813ffe742954a Mon Sep 17 00:00:00 2001 From: "Alexander S." Date: Wed, 22 Jan 2025 14:38:58 +0200 Subject: [PATCH 15/15] in instead of == Co-authored-by: Charles de Beauchesne --- utils/proxy/_deserializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 0d6b884014..2b82012f79 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -253,7 +253,7 @@ def _deserialize_file_in_multipart_form_data( filename = filename[:-3] content_is_deserialized = False - if filename.lower().endswith(".json") or path == "/symdb/v1/input" or path == "/api/v2/debugger": + if filename.lower().endswith(".json") or path in ("/symdb/v1/input", "/api/v2/debugger"): # when path == /symdb/v1/input or /api/v2/debugger, the content may be either raw json, or gizipped json # though, the file name may not always contains .json, so for this use case # we always try to deserialize the content as json