diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 7eb456e0ce..be5b75fa02 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -104,6 +104,8 @@ tests/: Test_Debugger_Probe_Snaphots: irrelevant test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: irrelevant + test_debugger_symdb.py: + Test_Debugger_SymDb: irrelevant integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e3419ae522..ac27f32f92 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -361,6 +361,8 @@ tests/: Test_Debugger_Probe_Snaphots: v2.53.0 test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: v2.53.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v2.53.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/golang.yml b/manifests/golang.yml index c6f5de78b6..f2161b5a00 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -437,6 +437,8 @@ tests/: Test_Debugger_Probe_Snaphots: missing_feature (feature not implented) test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: missing_feature (feature not implented) + test_debugger_symdb.py: + Test_Debugger_SymDb: missing_feature (feature not implented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/java.yml b/manifests/java.yml index 4206bbf99c..6f72215195 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1476,6 +1476,8 @@ tests/: spring-boot-undertow: v1.38.0 spring-boot-wildfly: v1.38.0 uds-spring-boot: v1.38.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v1.38.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 3dc728acea..c0930290e2 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -648,6 +648,8 @@ tests/: Test_Debugger_Probe_Snaphots: missing_feature (feature not implented) test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: missing_feature (feature not implented) + test_debugger_symdb.py: + Test_Debugger_SymDb: missing_feature (feature not implented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/php.yml b/manifests/php.yml index 034c701885..c6334e0e16 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -275,6 +275,8 @@ tests/: Test_Debugger_Probe_Snaphots: irrelevant test_debugger_probe_status.py: Test_Debugger_Probe_Statuses: irrelevant + test_debugger_symdb.py: + Test_Debugger_SymDb: irrelevant integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/python.yml b/manifests/python.yml index 2f4e65023d..4cdd539ec1 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -630,6 +630,8 @@ tests/: flask-poc: v2.11.0 uds-flask: v2.11.0 uwsgi-poc: v2.11.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: v2.11.0 integrations/: crossed_integrations/: test_kafka.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 72e648c63c..e0db8dc2b8 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -318,6 +318,10 @@ tests/: Test_Debugger_Probe_Statuses: "*": irrelevant rails70: v2.8.0 + test_debugger_symdb.py: + Test_Debugger_SymDb: + "*": irrelevant + rails70: missing_feature (feature not implemented) integrations/: crossed_integrations/: test_kafka.py: diff --git a/tests/debugger/test_debugger_symdb.py b/tests/debugger/test_debugger_symdb.py new file mode 100644 index 0000000000..b22a1130aa --- /dev/null +++ b/tests/debugger/test_debugger_symdb.py @@ -0,0 +1,93 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +import gzip +import json +import tests.debugger.utils as debugger +from utils import features, scenarios, bug, context +from utils import remote_config as rc +from jsonschema import Draft7Validator + + +@features.debugger +@scenarios.debugger_symdb +class Test_Debugger_SymDb(debugger._Base_Debugger_Test): + ############ setup ############ + def _setup(self): + self.rc_state = rc.send_symdb_command() + + ############ assert ############ + def _assert(self): + self.collect() + self.assert_rc_state_not_error() + self._assert_symbols_uploaded() + + def _assert_symbols_uploaded(self): + assert len(self.symbols) > 0, "No symbol files were found" + + symbol_type = { + "type": "object", + "required": ["line", "name", "symbol_type", "type"], + "properties": { + "language_specifics": {"type": "object"}, + "line": {"type": "integer"}, + "name": {"type": "string"}, + "symbol_type": {"type": "string"}, + "type": {"type": ["string", "null"]}, + }, + } + + scope_type = { + "type": "object", + "required": ["end_line", "scope_type", "source_file", "start_line"], + "properties": { + "end_line": {"type": "integer"}, + "language_specifics": {"type": ["object", "null"]}, + "name": {"type": ["string", "null"]}, + "scope_type": {"type": "string"}, + "scopes": {"type": ["array", "null"], "items": {"$ref": "#/definitions/scope"}}, + "source_file": {"type": "string"}, + "start_line": {"type": "integer"}, + "symbols": {"type": ["array", "null"], "items": {"$ref": "#/definitions/symbol"}}, + }, + } + + schema = { + "type": "object", + "required": ["env", "language", "scopes", "service", "version"], + "properties": { + "env": {"type": "string", "const": "system-tests"}, + "language": {"type": "string"}, + "scopes": {"type": "array", "items": {"$ref": "#/definitions/scope"}}, + "service": {"type": "string", "const": "weblog"}, + "version": {"type": "string", "const": "1.0.0"}, + }, + "definitions": {"scope": scope_type, "symbol": symbol_type}, + } + + validator = Draft7Validator(schema) + + for file_path in self.symbols: + assert file_path.endswith(".gz"), f"Symbol file {file_path} is not a .gz file" + + try: + with gzip.open(file_path, "rb") as f: + content = json.loads(f.read().decode("utf-8")) + validation_errors = list(validator.iter_errors(content)) + assert not validation_errors, f"Schema validation errors in {file_path}:\n" + "\n".join( + f"- {error.message} (at path: {' -> '.join(str(p) for p in error.path)})" + for error in validation_errors + ) + except gzip.BadGzipFile: + assert False, f"File {file_path} is not a valid gzip archive" + except json.JSONDecodeError: + assert False, f"File {file_path} does not contain valid JSON" + + ############ test ############ + def setup_symdb_upload(self): + self._setup() + + @bug(context.library == "dotnet", reason="DEBUG-3298") + def test_symdb_upload(self): + self._assert() diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index 3bd4e321fa..dee3fed6a5 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -7,6 +7,8 @@ import os import os.path import uuid +import gzip +import io from utils import interfaces, remote_config, weblog, context from utils.tools import logger @@ -17,6 +19,7 @@ _DEBUGGER_PATH = "/api/v2/debugger" _LOGS_PATH = "/api/v2/logs" _TRACES_PATH = "/api/v0.2/traces" +_SYMBOLS_PATH = "/symdb/v1/input" _CUR_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -65,6 +68,7 @@ class _Base_Debugger_Test: probe_diagnostics = {} probe_snapshots = {} probe_spans = {} + symbols = [] rc_state = None weblog_responses = [] @@ -259,6 +263,7 @@ def collect(self): self._collect_probe_diagnostics() self._collect_snapshots() self._collect_spans() + self._collect_symbols() def _collect_probe_diagnostics(self): def _read_data(): @@ -380,6 +385,16 @@ def _get_spans_hash(self): self.probe_spans = _get_spans_hash(self) + def _collect_symbols(self): + raw_data = list(interfaces.library.get_data(_SYMBOLS_PATH)) + + for data in raw_data: + if isinstance(data, dict) and "request" in data: + content = data["request"].get("content", []) + for part in content: + if isinstance(part, dict) and "system-tests-file-path" in part: + self.symbols.append(part["system-tests-file-path"]) + def get_tracer(self): if not _Base_Debugger_Test.tracer: _Base_Debugger_Test.tracer = { diff --git a/tests/remote_config/rc_expected_requests_live_debugging.json b/tests/remote_config/rc_expected_requests_live_debugging.json index 25fdcbdb9c..0844fe79ff 100644 --- a/tests/remote_config/rc_expected_requests_live_debugging.json +++ b/tests/remote_config/rc_expected_requests_live_debugging.json @@ -38,11 +38,11 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] } @@ -68,11 +68,11 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 321, + "length": 365, "hashes": [ { "algorithm": "sha256", - "hash": "4180218218444e1e126a4e4ec41a475b89813d27537fba5c92662fe3510b951d" + "hash": "32c04cce9cd0820470f5b2d6d92609d51ad3c5619fa947fdc91445f641d977dc" } ] } @@ -113,21 +113,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -158,21 +158,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/spanProbe_kepf0cf2-9top-45cf-9f39-59installed/config", - "length": 164, + "length": 188, "hashes": [ { "algorithm": "sha256", - "hash": "8e897cbaccc2d604fbd5f4c910512e9c5d2ac7fee1f984929bf93e596d7a7461" + "hash": "d22df7cf36e9f2b0134c4f6535a7340b9a4435876b79280f91d80942c9562b5b" } ] } @@ -208,31 +208,31 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/spanProbe_kepf0cf2-9top-45cf-9f39-59installed/config", - "length": 164, + "length": 188, "hashes": [ { "algorithm": "sha256", - "hash": "8e897cbaccc2d604fbd5f4c910512e9c5d2ac7fee1f984929bf93e596d7a7461" + "hash": "d22df7cf36e9f2b0134c4f6535a7340b9a4435876b79280f91d80942c9562b5b" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 321, + "length": 365, "hashes": [ { "algorithm": "sha256", - "hash": "3712620153e0f4d41f03ca93fc8618b9f6714918a1f117cf110f25e6131eef52" + "hash": "4f12b33894fd7178f2464d3fc2c63223c3ee2a29a5cf0936de60ceee88fd0656" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -263,21 +263,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -308,21 +308,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } @@ -353,21 +353,21 @@ "cached_target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/metricProbe_33a64d99-fbed-5eab-bb10-80735405c09b/config", - "length": 316, + "length": 360, "hashes": [ { "algorithm": "sha256", - "hash": "bbbaf6ac0011c040d5c15287f461b0b7fd39552958d546e9b41c8c07442634da" + "hash": "6daaa0eb13996d340d99983bb014ef17453bad39edf19041f24a87a159ff94fe" } ] }, { "path": "datadog/2/LIVE_DEBUGGING/logProbe_22953c88-eadc-4f9a-aa0f-7f6243f4bf8a/config", - "length": 209, + "length": 239, "hashes": [ { "algorithm": "sha256", - "hash": "aec211179c64eff2acbeec327a6fb2866350882bddb9fed066cdbb3423ceffd0" + "hash": "8176095e451a5f4d49db40e5eadf7d79b0ca6956cf28c83f87d18f4d66ea2583" } ] } diff --git a/tests/test_the_test/test_remote_config.py b/tests/test_the_test/test_remote_config.py index 0c2fa781f7..ed3ad1cf3b 100644 --- a/tests/test_the_test/test_remote_config.py +++ b/tests/test_the_test/test_remote_config.py @@ -4,7 +4,7 @@ @scenarios.test_the_test def test_debugger_command_none(): expected = { - "targets": "eyJzaWduZWQiOiB7Il90eXBlIjogInRhcmdldHMiLCAiY3VzdG9tIjogeyJvcGFxdWVfYmFja2VuZF9zdGF0ZSI6ICJleUptYjI4aU9pQWlZbUZ5SW4wPSJ9LCAiZXhwaXJlcyI6ICIzMDAwLTAxLTAxVDAwOjAwOjAwWiIsICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwgInRhcmdldHMiOiB7fSwgInZlcnNpb24iOiAwfSwgInNpZ25hdHVyZXMiOiBbeyJrZXlpZCI6ICJlZDc2NzJjOWEyNGFiZGE3ODg3MmVlMzJlZTcxYzdjYjFkNTIzNWU4ZGI0ZWNiZjFjYTI4YjljNTBlYjc1ZDllIiwgInNpZyI6ICJlMjI3OWE1NTRkNTI1MDNmNWJkNjhlMGE5OTEwYzdlOTBjOWJiODE3NDRmZTljODgyNGVhMzczN2IyNzlkOWU2OWIzY2U1ZjRiNDYzYzQwMmViZTM0OTY0ZmI3YTY5NjI1ZWIwZTkxZDNkZGJkMzkyY2M4YjMyMTAzNzNkOWIwZiJ9XX0=", + "targets": "ewogICJzaWduZWQiOiB7CiAgICAiX3R5cGUiOiAidGFyZ2V0cyIsCiAgICAiY3VzdG9tIjogewogICAgICAib3BhcXVlX2JhY2tlbmRfc3RhdGUiOiAiZXlKbWIyOGlPaUFpWW1GeUluMD0iCiAgICB9LAogICAgImV4cGlyZXMiOiAiMzAwMC0wMS0wMVQwMDowMDowMFoiLAogICAgInNwZWNfdmVyc2lvbiI6ICIxLjAiLAogICAgInRhcmdldHMiOiB7fSwKICAgICJ2ZXJzaW9uIjogMAogIH0sCiAgInNpZ25hdHVyZXMiOiBbCiAgICB7CiAgICAgICJrZXlpZCI6ICJlZDc2NzJjOWEyNGFiZGE3ODg3MmVlMzJlZTcxYzdjYjFkNTIzNWU4ZGI0ZWNiZjFjYTI4YjljNTBlYjc1ZDllIiwKICAgICAgInNpZyI6ICJlMjI3OWE1NTRkNTI1MDNmNWJkNjhlMGE5OTEwYzdlOTBjOWJiODE3NDRmZTljODgyNGVhMzczN2IyNzlkOWU2OWIzY2U1ZjRiNDYzYzQwMmViZTM0OTY0ZmI3YTY5NjI1ZWIwZTkxZDNkZGJkMzkyY2M4YjMyMTAzNzNkOWIwZiIKICAgIH0KICBdCn0=", "target_files": [], "client_configs": [], } @@ -29,11 +29,11 @@ def test_debugger_command_one_probe(): ] expected = { - "targets": "eyJzaWduZWQiOiB7Il90eXBlIjogInRhcmdldHMiLCAiY3VzdG9tIjogeyJvcGFxdWVfYmFja2VuZF9zdGF0ZSI6ICJleUptYjI4aU9pQWlZbUZ5SW4wPSJ9LCAiZXhwaXJlcyI6ICIzMDAwLTAxLTAxVDAwOjAwOjAwWiIsICJzcGVjX3ZlcnNpb24iOiAiMS4wIiwgInRhcmdldHMiOiB7ImRhdGFkb2cvMi9MSVZFX0RFQlVHR0lORy9sb2dQcm9iZV9sb2cxNzBhYS1hY2RhLTQ0NTMtOTExMS0xNDc4YTZtZXRob2QvY29uZmlnIjogeyJjdXN0b20iOiB7InYiOiAxfSwgImhhc2hlcyI6IHsic2hhMjU2IjogIjM1ZTQ3NzRhZmQzMWVhYWExMzAwNGU3MmU1ZTUyMWI3OGU3MGMxYTY5ZDBmZDI1MzI1NmU2ZTIwNTNkOGNkNjAifSwgImxlbmd0aCI6IDI0OX19LCAidmVyc2lvbiI6IDF9LCAic2lnbmF0dXJlcyI6IFt7ImtleWlkIjogImVkNzY3MmM5YTI0YWJkYTc4ODcyZWUzMmVlNzFjN2NiMWQ1MjM1ZThkYjRlY2JmMWNhMjhiOWM1MGViNzVkOWUiLCAic2lnIjogImUyMjc5YTU1NGQ1MjUwM2Y1YmQ2OGUwYTk5MTBjN2U5MGM5YmI4MTc0NGZlOWM4ODI0ZWEzNzM3YjI3OWQ5ZTY5YjNjZTVmNGI0NjNjNDAyZWJlMzQ5NjRmYjdhNjk2MjVlYjBlOTFkM2RkYmQzOTJjYzhiMzIxMDM3M2Q5YjBmIn1dfQ==", + "targets": "ewogICJzaWduZWQiOiB7CiAgICAiX3R5cGUiOiAidGFyZ2V0cyIsCiAgICAiY3VzdG9tIjogewogICAgICAib3BhcXVlX2JhY2tlbmRfc3RhdGUiOiAiZXlKbWIyOGlPaUFpWW1GeUluMD0iCiAgICB9LAogICAgImV4cGlyZXMiOiAiMzAwMC0wMS0wMVQwMDowMDowMFoiLAogICAgInNwZWNfdmVyc2lvbiI6ICIxLjAiLAogICAgInRhcmdldHMiOiB7CiAgICAgICJkYXRhZG9nLzIvTElWRV9ERUJVR0dJTkcvbG9nUHJvYmVfbG9nMTcwYWEtYWNkYS00NDUzLTkxMTEtMTQ3OGE2bWV0aG9kL2NvbmZpZyI6IHsKICAgICAgICAiY3VzdG9tIjogewogICAgICAgICAgInYiOiAxCiAgICAgICAgfSwKICAgICAgICAiaGFzaGVzIjogewogICAgICAgICAgInNoYTI1NiI6ICJlY2YzNDdmYjBlYTQ2MTZmZTU1NzZjMjI4M2FhZjU2ZTI2MWZiY2NkMzE2MmJhMjFmM2NmZDA0MmMzZWM5YWM1IgogICAgICAgIH0sCiAgICAgICAgImxlbmd0aCI6IDI4OQogICAgICB9CiAgICB9LAogICAgInZlcnNpb24iOiAxCiAgfSwKICAic2lnbmF0dXJlcyI6IFsKICAgIHsKICAgICAgImtleWlkIjogImVkNzY3MmM5YTI0YWJkYTc4ODcyZWUzMmVlNzFjN2NiMWQ1MjM1ZThkYjRlY2JmMWNhMjhiOWM1MGViNzVkOWUiLAogICAgICAic2lnIjogImUyMjc5YTU1NGQ1MjUwM2Y1YmQ2OGUwYTk5MTBjN2U5MGM5YmI4MTc0NGZlOWM4ODI0ZWEzNzM3YjI3OWQ5ZTY5YjNjZTVmNGI0NjNjNDAyZWJlMzQ5NjRmYjdhNjk2MjVlYjBlOTFkM2RkYmQzOTJjYzhiMzIxMDM3M2Q5YjBmIgogICAgfQogIF0KfQ==", "target_files": [ { "path": "datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config", - "raw": "eyJsYW5ndWFnZSI6ICIiLCAiaWQiOiAibG9nMTcwYWEtYWNkYS00NDUzLTkxMTEtMTQ3OGE2bWV0aG9kIiwgInR5cGUiOiAiTE9HX1BST0JFIiwgIndoZXJlIjogeyJ0eXBlTmFtZSI6ICJBQ1RVQUxfVFlQRV9OQU1FIiwgIm1ldGhvZE5hbWUiOiAiUGlpIiwgInNvdXJjZUZpbGUiOiBudWxsfSwgImV2YWx1YXRlQXQiOiAiRVhJVCIsICJjYXB0dXJlU25hcHNob3QiOiB0cnVlLCAiY2FwdHVyZSI6IHsibWF4RmllbGRDb3VudCI6IDIwMH19", + "raw": "ewogICJsYW5ndWFnZSI6ICIiLAogICJpZCI6ICJsb2cxNzBhYS1hY2RhLTQ0NTMtOTExMS0xNDc4YTZtZXRob2QiLAogICJ0eXBlIjogIkxPR19QUk9CRSIsCiAgIndoZXJlIjogewogICAgInR5cGVOYW1lIjogIkFDVFVBTF9UWVBFX05BTUUiLAogICAgIm1ldGhvZE5hbWUiOiAiUGlpIiwKICAgICJzb3VyY2VGaWxlIjogbnVsbAogIH0sCiAgImV2YWx1YXRlQXQiOiAiRVhJVCIsCiAgImNhcHR1cmVTbmFwc2hvdCI6IHRydWUsCiAgImNhcHR1cmUiOiB7CiAgICAibWF4RmllbGRDb3VudCI6IDIwMAogIH0KfQ==", } ], "client_configs": ["datadog/2/LIVE_DEBUGGING/logProbe_log170aa-acda-4453-9111-1478a6method/config"], diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 50b05629e0..2acc1ad436 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -601,6 +601,21 @@ def all_endtoend_scenarios(test_object): scenario_groups=[ScenarioGroup.DEBUGGER], ) + debugger_symdb = EndToEndScenario( + "DEBUGGER_SYMDB", + rc_api_enabled=True, + weblog_env={ + "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "1", + "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "1", + "DD_REMOTE_CONFIG_ENABLED": "true", + "DD_INTERNAL_RCM_POLL_INTERVAL": "2000", + "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "1", + }, + library_interface_timeout=5, + doc="Test scenario for checking symdb.", + scenario_groups=[ScenarioGroup.DEBUGGER], + ) + fuzzer = DockerScenario("_FUZZER", doc="Fake scenario for fuzzing (launch without pytest)", github_workflow=None) # Single Step Instrumentation scenarios (HOST and CONTAINER) diff --git a/utils/_remote_config.py b/utils/_remote_config.py index 3dfd652026..b43387ee87 100644 --- a/utils/_remote_config.py +++ b/utils/_remote_config.py @@ -163,17 +163,12 @@ def all_payload_sent(data) -> bool: library.wait_for(all_payload_sent, timeout=timeout) -def build_debugger_command(probes: list, version: int): - def _json_to_base64(json_object): - json_string = json.dumps(json_object).encode("utf-8") - return base64.b64encode(json_string).decode("utf-8") +def _create_base_rcm(): + return {"targets": "", "target_files": [], "client_configs": []} - def _sha256(value): - return hashlib.sha256(base64.b64decode(value)).hexdigest() - rcm = {"targets": "", "target_files": [], "client_configs": []} - - signed = { +def _create_base_signed(version: int): + return { "signed": { "_type": "targets", "custom": {"opaque_backend_state": "eyJmb28iOiAiYmFyIn0="}, # where does this come from ? @@ -192,6 +187,11 @@ def _sha256(value): ], } + +def build_debugger_command(probes: list | None, version: int): + rcm = _create_base_rcm() + signed = _create_base_signed(version) + if probes is None: rcm["targets"] = _json_to_base64(signed) else: @@ -201,7 +201,7 @@ def _sha256(value): probe_64 = _json_to_base64(probe) target["hashes"]["sha256"] = _sha256(probe_64) - target["length"] = len(json.dumps(probe).encode("utf-8")) + target["length"] = len(base64.b64decode(probe_64)) probe_path = re.sub(r"_([a-z])", lambda match: match.group(1).upper(), probe["type"].lower()) path = "datadog/2/LIVE_DEBUGGING/" + probe_path + "_" + probe["id"] + "/config" @@ -222,11 +222,48 @@ def send_debugger_command(probes: list, version: int) -> dict: return send_state(raw_payload) +def build_symdb_command(): + rcm = _create_base_rcm() + signed = _create_base_signed(version=1) + + target = {"custom": {"v": 1}, "hashes": {"sha256": ""}, "length": 0} + target_file = {"path": "", "raw": ""} + + payload = {"upload_symbols": True} + payload_64 = _json_to_base64(payload) + payload_length = len(base64.b64decode(payload_64)) + + path = "datadog/2/LIVE_DEBUGGING_SYMBOL_DB/symDb/config" + + target["hashes"]["sha256"] = _sha256(payload_64) + target["length"] = payload_length + + signed["signed"]["targets"][path] = target + + target_file["path"] = path + target_file["raw"] = payload_64 + + rcm["target_files"].append(target_file) + rcm["client_configs"].append(path) + rcm["targets"] = _json_to_base64(signed) + + return rcm + + +def send_symdb_command() -> dict: + raw_payload = build_symdb_command() + return send_state(raw_payload) + + def _json_to_base64(json_object): json_string = json.dumps(json_object, indent=2).encode("utf-8") return base64.b64encode(json_string).decode("utf-8") +def _sha256(value): + return hashlib.sha256(base64.b64decode(value)).hexdigest() + + class ClientConfig: _store: dict[str, "ClientConfig"] = {} config_file_version: int = 1 diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 204fb70b9e..631ecf45fa 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -5,7 +5,6 @@ from ast import literal_eval import base64 import gzip -import io import json import logging from hashlib import md5 @@ -193,13 +192,7 @@ def json_load(): if content_type_part.startswith("application/json"): item["content"] = json.loads(part.content) - elif content_type_part == "application/gzip": - with gzip.GzipFile(fileobj=io.BytesIO(part.content)) as gz_file: - content = gz_file.read() - - _deserialize_file_in_multipart_form_data(item, headers, export_content_files_to, content) - - elif content_type_part == "application/octet-stream": + elif content_type_part == "application/gzip" or content_type_part == "application/octet-stream": _deserialize_file_in_multipart_form_data(item, headers, export_content_files_to, part.content) else: