diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a65174a30..b01db491f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: swap-storage: true - uses: actions/checkout@v4 - name: Install Rust - run: rustup install nightly-2024-10-21 && rustup default nightly-2024-10-21 + run: rustup install nightly-2024-12-16 && rustup default nightly-2024-12-16 - name: Install Protoc Binary shell: bash run: chmod +x ./scripts/install-protoc.sh && ./scripts/install-protoc.sh $HOME diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 76b295c61..209dfdcab 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -16,8 +16,8 @@ jobs: run: | set -e rustup set profile minimal - rustup toolchain install nightly-2024-10-21 - rustup default nightly-2024-10-21 + rustup toolchain install nightly-2024-12-16 + rustup default nightly-2024-12-16 - uses: taiki-e/install-action@v2 with: tool: cargo-bolero diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 231091526..094273e66 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,8 +21,8 @@ jobs: uses: actions/checkout@v4 - name: Cache uses: ./.github/actions/cache - - name: Install nightly-2024-10-21 toolchain and rustfmt - run: rustup install nightly-2024-10-21 && rustup default nightly-2024-10-21 && rustup component add rustfmt + - name: Install nightly-2024-12-16 toolchain and rustfmt + run: rustup install nightly-2024-12-16 && rustup default nightly-2024-12-16 && rustup component add rustfmt - run: cargo fmt --all -- --check clippy: name: "clippy #${{ matrix.platform }} ${{ matrix.rust_version }}" @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - rust_version: ["1.76.0", "stable", "nightly-2024-10-21"] + rust_version: ["1.78.0", "stable", "nightly-2024-12-16"] platform: [windows-latest, ubuntu-latest] steps: - name: Checkout sources @@ -48,7 +48,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.76.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.78.0 ] || [ ${{ matrix.rust_version }} = stable ] && echo -Aunknown-lints -Ainvalid_reference_casting -Aclippy::redundant-closure-call) licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index 0722ff193..481633d38 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -14,8 +14,8 @@ jobs: run: | set -e rustup set profile minimal - rustup toolchain install nightly-2024-10-21 --component miri - rustup default nightly-2024-10-21 + rustup toolchain install nightly-2024-12-16 --component miri + rustup default nightly-2024-12-16 - name: Install Protoc Binary shell: bash run: chmod +x ./scripts/install-protoc.sh && ./scripts/install-protoc.sh $HOME diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 218dfde69..7183a1758 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: [push] env: CARGO_TERM_COLOR: always - RUST_VERSION: 1.76.0 + RUST_VERSION: 1.78.0 jobs: test: diff --git a/.github/workflows/verify-proto-files.yml b/.github/workflows/verify-proto-files.yml index ae0728b3c..32a2cd54a 100644 --- a/.github/workflows/verify-proto-files.yml +++ b/.github/workflows/verify-proto-files.yml @@ -5,7 +5,7 @@ on: env: DATADOG_AGENT_TAG: "7.55.0-rc.3" - rust_version: "1.76.0" + rust_version: "1.78.0" jobs: verify-proto-files: diff --git a/Cargo.lock b/Cargo.lock index 7886b957d..f8d9ce672 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1324,10 +1324,12 @@ name = "data-pipeline-ffi" version = "14.3.1" dependencies = [ "build_common", - "bytes", "data-pipeline", + "datadog-trace-utils", "ddcommon-ffi", - "libc", + "httpmock", + "rmp-serde", + "tinybytes", ] [[package]] @@ -1375,7 +1377,9 @@ dependencies = [ "datadog-crashtracker", "ddcommon", "ddcommon-ffi", + "function_name", "hyper 0.14.31", + "libc", "symbolic-common", "symbolic-demangle", ] @@ -1548,7 +1552,7 @@ dependencies = [ [[package]] name = "datadog-protos" version = "0.1.0" -source = "git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751#3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" +source = "git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222#c89b58e5784b985819baf11f13f7d35876741222" dependencies = [ "bytes", "prost 0.13.3", @@ -1708,13 +1712,13 @@ dependencies = [ "ddcommon", "duplicate", "hyper 0.14.31", - "log", "rmp-serde", "serde", "serde_json", "serial_test", "tempfile", "tokio", + "tracing", ] [[package]] @@ -1814,7 +1818,7 @@ dependencies = [ "maplit", "pin-project", "regex", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-native-certs 0.7.3", "serde", "static_assertions", @@ -1840,10 +1844,10 @@ dependencies = [ [[package]] name = "ddsketch-agent" version = "0.1.0" -source = "git+https://github.com/DataDog/saluki-backport/?rev=3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751#3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" +source = "git+https://github.com/DataDog/saluki/?rev=c89b58e5784b985819baf11f13f7d35876741222#c89b58e5784b985819baf11f13f7d35876741222" dependencies = [ "datadog-protos", - "float_eq", + "float-cmp", "ordered-float 4.5.0", "smallvec", ] @@ -2203,12 +2207,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "float_eq" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" - [[package]] name = "fnv" version = "1.0.7" @@ -2230,6 +2228,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "function_name" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1ab577a896d09940b5fe12ec5ae71f9d8211fff62c919c03a3750a9901e98a7" +dependencies = [ + "function_name-proc-macro", +] + +[[package]] +name = "function_name-proc-macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673464e1e314dd67a0fd9544abc99e8eb28d0c7e3b69b033bcff9b2d00b87333" + [[package]] name = "futures" version = "0.3.31" @@ -2805,7 +2818,7 @@ dependencies = [ "http 1.1.0", "hyper 1.5.0", "hyper-util", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -3226,9 +3239,9 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libloading" @@ -4288,7 +4301,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.16", + "rustls 0.23.18", "socket2", "thiserror", "tokio", @@ -4305,7 +4318,7 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.16", + "rustls 0.23.18", "slab", "thiserror", "tinyvec", @@ -4503,7 +4516,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pemfile", "rustls-pki-types", "serde", @@ -4652,9 +4665,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "aws-lc-rs", "once_cell", @@ -5708,7 +5721,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pki-types", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index a662ed0bd..f9e63e590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,15 @@ resolver = "2" # These are used by many packages, but not all. For instance, the sidecar and # serverless packages wanted to maintain their own version numbers. Some of # their depenencies also remain under their own versioning. +# +# When bumping the Rust version, make sure that the version is less than or +# equal to latest Alpine Linux version, and also the latest RHEL 8.x and 9.x +# releases (not stream): +# - https://rpms.remirepo.net/rpmphp/zoom.php?rpm=rust +# The RHEL restrictions are for the dd-trace-php repository. Someone in the +# community, Remi Collet, packages the extension for these systems. [workspace.package] -rust-version = "1.76.0" +rust-version = "1.78.0" edition = "2021" version = "14.3.1" license = "Apache-2.0" diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 103f3821d..452f115cb 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -11161,15 +11161,6 @@ third_party_libraries: LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -- package_name: float_eq - package_version: 1.0.1 - repository: https://github.com/jtempest/float_eq-rs - license: MIT OR Apache-2.0 - licenses: - - license: MIT - text: "Copyright (c) 2020 jtempest\r\n\r\nPermission is hereby granted, free of charge, to any person obtaining a copy\r\nof this software and associated documentation files (the \"Software\"), to deal\r\nin the Software without restriction, including without limitation the rights\r\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\r\ncopies of the Software, and to permit persons to whom the Software is\r\nfurnished to do so, subject to the following conditions:\r\n\r\nThe above copyright notice and this permission notice shall be included in all\r\ncopies or substantial portions of the Software.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\r\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\r\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\r\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\r\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\r\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r\nSOFTWARE." - - license: Apache-2.0 - text: "\r\n Apache License\r\n Version 2.0, January 2004\r\n http://www.apache.org/licenses/\r\n\r\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\r\n\r\n 1. Definitions.\r\n\r\n \"License\" shall mean the terms and conditions for use, reproduction,\r\n and distribution as defined by Sections 1 through 9 of this document.\r\n\r\n \"Licensor\" shall mean the copyright owner or entity authorized by\r\n the copyright owner that is granting the License.\r\n\r\n \"Legal Entity\" shall mean the union of the acting entity and all\r\n other entities that control, are controlled by, or are under common\r\n control with that entity. For the purposes of this definition,\r\n \"control\" means (i) the power, direct or indirect, to cause the\r\n direction or management of such entity, whether by contract or\r\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\r\n outstanding shares, or (iii) beneficial ownership of such entity.\r\n\r\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\r\n exercising permissions granted by this License.\r\n\r\n \"Source\" form shall mean the preferred form for making modifications,\r\n including but not limited to software source code, documentation\r\n source, and configuration files.\r\n\r\n \"Object\" form shall mean any form resulting from mechanical\r\n transformation or translation of a Source form, including but\r\n not limited to compiled object code, generated documentation,\r\n and conversions to other media types.\r\n\r\n \"Work\" shall mean the work of authorship, whether in Source or\r\n Object form, made available under the License, as indicated by a\r\n copyright notice that is included in or attached to the work\r\n (an example is provided in the Appendix below).\r\n\r\n \"Derivative Works\" shall mean any work, whether in Source or Object\r\n form, that is based on (or derived from) the Work and for which the\r\n editorial revisions, annotations, elaborations, or other modifications\r\n represent, as a whole, an original work of authorship. For the purposes\r\n of this License, Derivative Works shall not include works that remain\r\n separable from, or merely link (or bind by name) to the interfaces of,\r\n the Work and Derivative Works thereof.\r\n\r\n \"Contribution\" shall mean any work of authorship, including\r\n the original version of the Work and any modifications or additions\r\n to that Work or Derivative Works thereof, that is intentionally\r\n submitted to Licensor for inclusion in the Work by the copyright owner\r\n or by an individual or Legal Entity authorized to submit on behalf of\r\n the copyright owner. For the purposes of this definition, \"submitted\"\r\n means any form of electronic, verbal, or written communication sent\r\n to the Licensor or its representatives, including but not limited to\r\n communication on electronic mailing lists, source code control systems,\r\n and issue tracking systems that are managed by, or on behalf of, the\r\n Licensor for the purpose of discussing and improving the Work, but\r\n excluding communication that is conspicuously marked or otherwise\r\n designated in writing by the copyright owner as \"Not a Contribution.\"\r\n\r\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\r\n on behalf of whom a Contribution has been received by Licensor and\r\n subsequently incorporated within the Work.\r\n\r\n 2. Grant of Copyright License. Subject to the terms and conditions of\r\n this License, each Contributor hereby grants to You a perpetual,\r\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\r\n copyright license to reproduce, prepare Derivative Works of,\r\n publicly display, publicly perform, sublicense, and distribute the\r\n Work and such Derivative Works in Source or Object form.\r\n\r\n 3. Grant of Patent License. Subject to the terms and conditions of\r\n this License, each Contributor hereby grants to You a perpetual,\r\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\r\n (except as stated in this section) patent license to make, have made,\r\n use, offer to sell, sell, import, and otherwise transfer the Work,\r\n where such license applies only to those patent claims licensable\r\n by such Contributor that are necessarily infringed by their\r\n Contribution(s) alone or by combination of their Contribution(s)\r\n with the Work to which such Contribution(s) was submitted. If You\r\n institute patent litigation against any entity (including a\r\n cross-claim or counterclaim in a lawsuit) alleging that the Work\r\n or a Contribution incorporated within the Work constitutes direct\r\n or contributory patent infringement, then any patent licenses\r\n granted to You under this License for that Work shall terminate\r\n as of the date such litigation is filed.\r\n\r\n 4. Redistribution. You may reproduce and distribute copies of the\r\n Work or Derivative Works thereof in any medium, with or without\r\n modifications, and in Source or Object form, provided that You\r\n meet the following conditions:\r\n\r\n (a) You must give any other recipients of the Work or\r\n Derivative Works a copy of this License; and\r\n\r\n (b) You must cause any modified files to carry prominent notices\r\n stating that You changed the files; and\r\n\r\n (c) You must retain, in the Source form of any Derivative Works\r\n that You distribute, all copyright, patent, trademark, and\r\n attribution notices from the Source form of the Work,\r\n excluding those notices that do not pertain to any part of\r\n the Derivative Works; and\r\n\r\n (d) If the Work includes a \"NOTICE\" text file as part of its\r\n distribution, then any Derivative Works that You distribute must\r\n include a readable copy of the attribution notices contained\r\n within such NOTICE file, excluding those notices that do not\r\n pertain to any part of the Derivative Works, in at least one\r\n of the following places: within a NOTICE text file distributed\r\n as part of the Derivative Works; within the Source form or\r\n documentation, if provided along with the Derivative Works; or,\r\n within a display generated by the Derivative Works, if and\r\n wherever such third-party notices normally appear. The contents\r\n of the NOTICE file are for informational purposes only and\r\n do not modify the License. You may add Your own attribution\r\n notices within Derivative Works that You distribute, alongside\r\n or as an addendum to the NOTICE text from the Work, provided\r\n that such additional attribution notices cannot be construed\r\n as modifying the License.\r\n\r\n You may add Your own copyright statement to Your modifications and\r\n may provide additional or different license terms and conditions\r\n for use, reproduction, or distribution of Your modifications, or\r\n for any such Derivative Works as a whole, provided Your use,\r\n reproduction, and distribution of the Work otherwise complies with\r\n the conditions stated in this License.\r\n\r\n 5. Submission of Contributions. Unless You explicitly state otherwise,\r\n any Contribution intentionally submitted for inclusion in the Work\r\n by You to the Licensor shall be under the terms and conditions of\r\n this License, without any additional terms or conditions.\r\n Notwithstanding the above, nothing herein shall supersede or modify\r\n the terms of any separate license agreement you may have executed\r\n with Licensor regarding such Contributions.\r\n\r\n 6. Trademarks. This License does not grant permission to use the trade\r\n names, trademarks, service marks, or product names of the Licensor,\r\n except as required for reasonable and customary use in describing the\r\n origin of the Work and reproducing the content of the NOTICE file.\r\n\r\n 7. Disclaimer of Warranty. Unless required by applicable law or\r\n agreed to in writing, Licensor provides the Work (and each\r\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\r\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\r\n implied, including, without limitation, any warranties or conditions\r\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\r\n PARTICULAR PURPOSE. You are solely responsible for determining the\r\n appropriateness of using or redistributing the Work and assume any\r\n risks associated with Your exercise of permissions under this License.\r\n\r\n 8. Limitation of Liability. In no event and under no legal theory,\r\n whether in tort (including negligence), contract, or otherwise,\r\n unless required by applicable law (such as deliberate and grossly\r\n negligent acts) or agreed to in writing, shall any Contributor be\r\n liable to You for damages, including any direct, indirect, special,\r\n incidental, or consequential damages of any character arising as a\r\n result of this License or out of the use or inability to use the\r\n Work (including but not limited to damages for loss of goodwill,\r\n work stoppage, computer failure or malfunction, or any and all\r\n other commercial damages or losses), even if such Contributor\r\n has been advised of the possibility of such damages.\r\n\r\n 9. Accepting Warranty or Additional Liability. While redistributing\r\n the Work or Derivative Works thereof, You may choose to offer,\r\n and charge a fee for, acceptance of support, warranty, indemnity,\r\n or other liability obligations and/or rights consistent with this\r\n License. However, in accepting such obligations, You may act only\r\n on Your own behalf and on Your sole responsibility, not on behalf\r\n of any other Contributor, and only if You agree to indemnify,\r\n defend, and hold each Contributor harmless for any liability\r\n incurred by, or claims asserted against, such Contributor by reason\r\n of your accepting any such warranty or additional liability.\r\n\r\n END OF TERMS AND CONDITIONS\r\n\r\n APPENDIX: How to apply the Apache License to your work.\r\n\r\n To apply the Apache License to your work, attach the following\r\n boilerplate notice, with the fields enclosed by brackets \"[]\"\r\n replaced with your own identifying information. (Don't include\r\n the brackets!) The text should be enclosed in the appropriate\r\n comment syntax for the file format. We also recommend that a\r\n file or class name and description of purpose be included on the\r\n same \"printed page\" as the copyright notice for easier\r\n identification within third-party archives.\r\n\r\n Copyright [yyyy] [name of copyright owner]\r\n\r\n Licensed under the Apache License, Version 2.0 (the \"License\");\r\n you may not use this file except in compliance with the License.\r\n You may obtain a copy of the License at\r\n\r\n http://www.apache.org/licenses/LICENSE-2.0\r\n\r\n Unless required by applicable law or agreed to in writing, software\r\n distributed under the License is distributed on an \"AS IS\" BASIS,\r\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n See the License for the specific language governing permissions and\r\n limitations under the License.\r\n" - package_name: fnv package_version: 1.0.7 repository: https://github.com/servo/rust-fnv @@ -11238,6 +11229,41 @@ third_party_libraries: DEALINGS IN THE SOFTWARE. - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: function_name + package_version: 0.3.0 + repository: https://github.com/danielhenrymantilla/rust-function_name + license: MIT + licenses: + - license: MIT + text: | + MIT License + + Copyright (c) 2019 Daniel Henry-Mantilla + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +- package_name: function_name-proc-macro + package_version: 0.3.0 + repository: https://github.com/danielhenrymantilla/rust-function_name + license: MIT + licenses: + - license: MIT + text: NOT FOUND - package_name: futures package_version: 0.3.31 repository: https://github.com/rust-lang/futures-rs @@ -15890,7 +15916,7 @@ third_party_libraries: - license: MIT text: NOT FOUND - package_name: libc - package_version: 0.2.161 + package_version: 0.2.167 repository: https://github.com/rust-lang/libc license: MIT OR Apache-2.0 licenses: @@ -22948,7 +22974,7 @@ third_party_libraries: IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - package_name: rustls - package_version: 0.23.16 + package_version: 0.23.18 repository: https://github.com/rustls/rustls license: Apache-2.0 OR ISC OR MIT licenses: @@ -24337,7 +24363,37 @@ third_party_libraries: limitations under the License. - license: BSD-3-Clause - text: NOT FOUND + text: |+ + BSD 3-Clause License + + Copyright (c) 2019, Standard Cognition + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + - package_name: serde package_version: 1.0.214 repository: https://github.com/serde-rs/serde @@ -42272,7 +42328,31 @@ third_party_libraries: license: BSD-2-Clause OR Apache-2.0 OR MIT licenses: - license: BSD-2-Clause - text: NOT FOUND + text: | + Copyright 2019 The Fuchsia Authors. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - license: Apache-2.0 text: |2+ Apache License @@ -42511,7 +42591,31 @@ third_party_libraries: license: BSD-2-Clause OR Apache-2.0 OR MIT licenses: - license: BSD-2-Clause - text: NOT FOUND + text: | + Copyright 2019 The Fuchsia Authors. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - license: Apache-2.0 text: |2+ Apache License diff --git a/README.md b/README.md index ff852b715..71c94a991 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ bash build-profiling-ffi.sh /opt/libdatadog #### Build Dependencies -- Rust 1.76.0 or newer with cargo +- Rust 1.78.0 or newer with cargo. See the Cargo.toml for information about bumping this version. - `cbindgen` 0.26 - `cmake` and `protoc` @@ -66,7 +66,7 @@ cargo nextest run The simplest way to install [cargo-nextest][nt] is to use `cargo install` like this. ```bash -cargo install --locked 'cargo-nextest@0.9.81' +cargo install --locked 'cargo-nextest@0.9.85' ``` #### Skipping tracing integration tests @@ -77,6 +77,6 @@ Tracing integration tests require docker to be installed and running. If you don cargo nextest run -E '!test(tracing_integration_tests::)' ``` -Please note that the locked version is to make sure that it can be built using rust `1.76.0`, and if you are using a newer rust version, then it's enough to limit the version to `0.9.*`. +Please note that the locked version is to make sure that it can be built using rust `1.78.0`, and if you are using a newer rust version, then it's enough to limit the version to `0.9.*`. [nt]: https://nexte.st/ diff --git a/bin_tests/src/lib.rs b/bin_tests/src/lib.rs index 6867fcfd8..e7d0dd3fd 100644 --- a/bin_tests/src/lib.rs +++ b/bin_tests/src/lib.rs @@ -5,7 +5,6 @@ pub mod modes; use std::{collections::HashMap, env, ops::DerefMut, path::PathBuf, process, sync::Mutex}; -use anyhow::Ok; use once_cell::sync::OnceCell; /// This crate implements an abstraction over compilation with cargo with the purpose @@ -69,6 +68,11 @@ fn inner_build_artifact(c: &ArtifactsBuild) -> anyhow::Result { /// it's directory static ARTIFACT_DIR: OnceCell = OnceCell::new(); let artifact_dir = ARTIFACT_DIR.get_or_init(|| { + // If the CARGO_TARGET_DIR env var is set, then just use that. + if let Ok(env_target_dir) = env::var("CARGO_TARGET_DIR") { + return PathBuf::from(env_target_dir); + } + let test_bin_location = PathBuf::from(env::args().next().unwrap()); let mut location_components = test_bin_location.components().rev().peekable(); loop { diff --git a/build-common/src/cbindgen.rs b/build-common/src/cbindgen.rs index 4f89f61bf..07275dac9 100644 --- a/build-common/src/cbindgen.rs +++ b/build-common/src/cbindgen.rs @@ -9,12 +9,19 @@ use std::str; pub const HEADER_PATH: &str = "include/datadog"; +pub struct OutPaths { + // Path to the default `./target directory where cargo outputs the result of + // it's build process ` + pub cargo_target_dir: PathBuf, + + // Directory identified by the DESTDIR env variable. + // It is taken realtive to the cargo project root. + // if the env variable is not defined this defaults to cargo_target_dir + pub deliverables_dir: PathBuf, +} + /// Determines the cargo target directory and deliverables directory. -/// -/// # Returns -/// -/// * `(PathBuf, PathBuf)` - The cargo target directory and deliverables directory. -pub fn determine_paths() -> (PathBuf, PathBuf) { +pub fn determine_paths() -> OutPaths { let cargo_target_dir = match env::var_os("CARGO_TARGET_DIR") { Some(dir) => PathBuf::from(dir), None => { @@ -60,8 +67,10 @@ pub fn determine_paths() -> (PathBuf, PathBuf) { .expect("CARGO_TARGET_DIR does not have a parent directory, aborting build."); deliverables_dir = parent_dir.join(&deliverables_dir); } - - (cargo_target_dir, deliverables_dir) + OutPaths { + cargo_target_dir, + deliverables_dir, + } } /// Configure the header generation using environment variables. @@ -78,9 +87,11 @@ pub fn determine_paths() -> (PathBuf, PathBuf) { /// * `header_name` - The name of the header file to generate. pub fn generate_and_configure_header(header_name: &str) { let crate_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let (_, deliverables_dir) = determine_paths(); + let OutPaths { + cargo_target_dir, .. + } = determine_paths(); - generate_header(crate_dir, header_name, deliverables_dir); + generate_header(crate_dir, header_name, cargo_target_dir); } /// Generates a C header file using `cbindgen` for the specified crate. @@ -124,11 +135,9 @@ pub fn generate_header(crate_dir: PathBuf, header_name: &str, output_base_dir: P /// This behaves in a similar way as `generate_and_configure_header`. pub fn copy_and_configure_headers() { let crate_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let (_cargo_target_dir, deliverables_dir) = determine_paths(); - - if !deliverables_dir.exists() { - fs::create_dir_all(&deliverables_dir).expect("Failed to create deliverables directory"); - } + let OutPaths { + cargo_target_dir, .. + } = determine_paths(); let src_dir = crate_dir.join("src"); if src_dir.is_dir() { @@ -137,7 +146,7 @@ pub fn copy_and_configure_headers() { let path = entry.path(); if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("h") { - copy_header(&path, &deliverables_dir); + copy_header(&path, &cargo_target_dir); } } } diff --git a/build-profiling-ffi.sh b/build-profiling-ffi.sh index 064716029..46094b4d1 100755 --- a/build-profiling-ffi.sh +++ b/build-profiling-ffi.sh @@ -224,15 +224,28 @@ echo "Building tools" DESTDIR=$destdir cargo build --package tools --bins echo "Generating $destdir/include/libdatadog headers..." -# ADD headers based on selected features. -HEADERS="$destdir/include/datadog/common.h $destdir/include/datadog/profiling.h $destdir/include/datadog/telemetry.h $destdir/include/datadog/crashtracker.h" +rm -r $destdir/include/datadog/ +mkdir $destdir/include/datadog/ + +CBINDGEN_HEADERS="common.h profiling.h telemetry.h crashtracker.h data-pipeline.h" +# When optional features are added, don't forget to also include thei headers here case $ARG_FEATURES in - *data-pipeline-ffi*) - HEADERS="$HEADERS $destdir/include/datadog/data-pipeline.h" - ;; esac +CBINDGEN_HEADERS_DESTS="" +for header in $CBINDGEN_HEADERS; do + HEADER_DEST="$destdir/include/datadog/$header" + cp "$CARGO_TARGET_DIR/include/datadog/$header" "$HEADER_DEST" + CBINDGEN_HEADERS_DESTS="$CBINDGEN_HEADERS_DESTS $HEADER_DEST" +done + +"$CARGO_TARGET_DIR"/debug/dedup_headers $CBINDGEN_HEADERS_DESTS + +if [[ "$symbolizer" -eq 1 ]]; then + # Copy the blazesym header separately because The blazesym header isn't auto-generated by cbindgen + # so we don't need to remove definitions that are already present in `common.h` using dedup_headers + cp "$CARGO_TARGET_DIR/include/datadog/blazesym.h" "$destdir/include/datadog/blazesym.h" +fi -"$CARGO_TARGET_DIR"/debug/dedup_headers $HEADERS # Don't build the crashtracker on windows if [[ "$target" != "x86_64-pc-windows-msvc" ]]; then diff --git a/builder/src/arch/apple.rs b/builder/src/arch/apple.rs index 6b409aa80..6811e0a10 100644 --- a/builder/src/arch/apple.rs +++ b/builder/src/arch/apple.rs @@ -1,6 +1,7 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use std::os::unix::process::ExitStatusExt; use std::process::Command; use std::ffi::OsStr; @@ -15,17 +16,33 @@ pub const REMOVE_RPATH: bool = true; pub const BUILD_CRASHTRACKER: bool = true; pub const RUSTFLAGS: [&str; 2] = ["-C", "relocation-model=pic"]; -#[allow(clippy::zombie_processes)] pub fn fix_rpath(lib_path: &str) { if REMOVE_RPATH { let lib_name = lib_path.split('/').last().unwrap(); - Command::new("install_name_tool") + let exit_status = Command::new("install_name_tool") .arg("-id") .arg("@rpath/".to_string() + lib_name) .arg(lib_path) - .spawn() - .expect("Failed to fix rpath"); + .status() + .expect("Failed to fix rpath using install_name_tool"); + match exit_status.code() { + Some(0) => {} + Some(rc) => panic!( + "Failed to fix rpath using install_name_tool: return code {}", + rc + ), + None => match exit_status.signal() { + Some(sig) => panic!( + "Failed to fix rpath using install_name_tool: killed by signal {}", + sig + ), + None => panic!( + "Failed to fix rpath using install_name_tool: exit status {:?}", + exit_status + ), + }, + } } } diff --git a/builder/src/bin/release.rs b/builder/src/bin/release.rs index 03388c858..f61d91c47 100644 --- a/builder/src/bin/release.rs +++ b/builder/src/bin/release.rs @@ -3,7 +3,7 @@ use std::env; -use build_common::determine_paths; +use build_common::{determine_paths, OutPaths}; use builder::builder::Builder; use builder::common::Common; @@ -40,7 +40,10 @@ impl From for ReleaseArgs { pub fn main() { let args: ReleaseArgs = pico_args::Arguments::from_env().into(); - let (_, source_path) = determine_paths(); + let OutPaths { + cargo_target_dir: source_path, + .. + } = determine_paths(); let profile = env::var("PROFILE").unwrap(); let version = env::var("CARGO_PKG_VERSION").unwrap(); diff --git a/builder/src/builder.rs b/builder/src/builder.rs index 92672adc6..08850dc78 100644 --- a/builder/src/builder.rs +++ b/builder/src/builder.rs @@ -139,7 +139,12 @@ impl Builder { pub fn add_cmake(&self) { let libs = arch::NATIVE_LIBS.to_owned(); - let cmake_path: PathBuf = [&self.target_dir, "DatadogConfig.cmake"].iter().collect(); + let cmake_dir: PathBuf = [&self.target_dir, "cmake"].iter().collect(); + fs::create_dir_all(cmake_dir).expect("Failed to create cmake dir"); + + let cmake_path: PathBuf = [&self.target_dir, "cmake", "DatadogConfig.cmake"] + .iter() + .collect(); let mut origin = project_root(); origin.push("cmake"); origin.push("DatadogConfig.cmake.in"); diff --git a/builder/src/crashtracker.rs b/builder/src/crashtracker.rs index be2c7603b..489e9c173 100644 --- a/builder/src/crashtracker.rs +++ b/builder/src/crashtracker.rs @@ -22,11 +22,14 @@ pub struct CrashTracker { impl CrashTracker { fn gen_binaries(&self) -> Result<()> { if arch::BUILD_CRASHTRACKER { + let mut datadog_root = project_root(); + datadog_root.push(self.target_dir.as_ref()); + let mut crashtracker_dir = project_root(); crashtracker_dir.push("crashtracker"); let _dst = cmake::Config::new(crashtracker_dir.to_str().unwrap()) - .define("Datadog_ROOT", self.target_dir.as_ref()) - .define("CMAKE_INSTALL_PREFIX", self.target_dir.as_ref()) + .define("Datadog_ROOT", datadog_root.to_str().unwrap()) + .define("CMAKE_INSTALL_PREFIX", self.target_dir.to_string()) .build(); } diff --git a/crashtracker-ffi/Cargo.toml b/crashtracker-ffi/Cargo.toml index c6de3f695..75c86a32f 100644 --- a/crashtracker-ffi/Cargo.toml +++ b/crashtracker-ffi/Cargo.toml @@ -31,3 +31,5 @@ ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false } hyper = {version = "0.14", features = ["backports", "deprecated"], default-features = false} symbolic-demangle = { version = "12.8.0", default-features = false, features = ["rust", "cpp", "msvc"], optional = true } symbolic-common = { version = "12.8.0", default-features = false, optional = true } +function_name = "0.3.0" +libc = "0.2.167" diff --git a/crashtracker-ffi/cbindgen.toml b/crashtracker-ffi/cbindgen.toml index b5571f563..b6267bc93 100644 --- a/crashtracker-ffi/cbindgen.toml +++ b/crashtracker-ffi/cbindgen.toml @@ -34,6 +34,7 @@ renaming_overrides_prefixing = true "Timespec" = "ddog_Timespec" "Vec_Tag" = "ddog_Vec_Tag" "Vec_U8" = "ddog_Vec_U8" +"VoidResult" = "ddog_VoidResult" [export.mangle] rename_types = "PascalCase" diff --git a/crashtracker-ffi/src/collector/counters.rs b/crashtracker-ffi/src/collector/counters.rs index 7e9075abb..b3bb4396b 100644 --- a/crashtracker-ffi/src/collector/counters.rs +++ b/crashtracker-ffi/src/collector/counters.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use super::datatypes::OpTypes; -use crate::Result; -use anyhow::Context; +use ::function_name::named; +use ddcommon_ffi::{wrap_with_void_ffi_result, VoidResult}; /// Resets all counters to 0. /// Expected to be used after a fork, to reset the counters on the child @@ -15,34 +15,31 @@ use anyhow::Context; /// No safety concerns. #[no_mangle] #[must_use] -pub unsafe extern "C" fn ddog_crasht_reset_counters() -> Result { - datadog_crashtracker::reset_counters() - .context("ddog_crasht_reset_counters failed") - .into() +#[named] +pub unsafe extern "C" fn ddog_crasht_reset_counters() -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::reset_counters()? }) } #[no_mangle] #[must_use] +#[named] /// Atomically increments the count associated with `op`. /// Useful for tracking what operations were occuring when a crash occurred. /// /// # Safety /// No safety concerns. -pub unsafe extern "C" fn ddog_crasht_begin_op(op: OpTypes) -> Result { - datadog_crashtracker::begin_op(op) - .context("ddog_crasht_begin_op failed") - .into() +pub unsafe extern "C" fn ddog_crasht_begin_op(op: OpTypes) -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::begin_op(op)? }) } #[no_mangle] #[must_use] +#[named] /// Atomically decrements the count associated with `op`. /// Useful for tracking what operations were occuring when a crash occurred. /// /// # Safety /// No safety concerns. -pub unsafe extern "C" fn ddog_crasht_end_op(op: OpTypes) -> Result { - datadog_crashtracker::end_op(op) - .context("ddog_crasht_end_op failed") - .into() +pub unsafe extern "C" fn ddog_crasht_end_op(op: OpTypes) -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::end_op(op)? }) } diff --git a/crashtracker-ffi/src/collector/datatypes.rs b/crashtracker-ffi/src/collector/datatypes.rs index c73fa354b..847b910d0 100644 --- a/crashtracker-ffi/src/collector/datatypes.rs +++ b/crashtracker-ffi/src/collector/datatypes.rs @@ -1,7 +1,6 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::option_from_char_slice; pub use datadog_crashtracker::{OpTypes, StacktraceCollection}; use ddcommon::Endpoint; use ddcommon_ffi::slice::{AsBytes, CharSlice}; @@ -30,23 +29,20 @@ impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerRecei let args = { let mut vec = Vec::with_capacity(value.args.len()); for x in value.args.iter() { - vec.push(x.try_to_utf8()?.to_string()); + vec.push(x.try_to_string()?); } vec }; let env = { let mut vec = Vec::with_capacity(value.env.len()); for x in value.env.iter() { - vec.push(( - x.key.try_to_utf8()?.to_string(), - x.val.try_to_utf8()?.to_string(), - )); + vec.push((x.key.try_to_string()?, x.val.try_to_string()?)); } vec }; - let path_to_receiver_binary = value.path_to_receiver_binary.try_to_utf8()?.to_string(); - let stderr_filename = option_from_char_slice(value.optional_stderr_filename)?; - let stdout_filename = option_from_char_slice(value.optional_stdout_filename)?; + let path_to_receiver_binary = value.path_to_receiver_binary.try_to_string()?; + let stderr_filename = value.optional_stderr_filename.try_to_string_option()?; + let stdout_filename = value.optional_stdout_filename.try_to_string_option()?; Self::new( args, env, @@ -80,7 +76,7 @@ impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerConfiguration let additional_files = { let mut vec = Vec::with_capacity(value.additional_files.len()); for x in value.additional_files.iter() { - vec.push(x.try_to_utf8()?.to_string()); + vec.push(x.try_to_string()?); } vec }; @@ -89,7 +85,7 @@ impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerConfiguration let endpoint = value.endpoint.cloned(); let resolve_frames = value.resolve_frames; let timeout_ms = value.timeout_ms; - let unix_socket_path = option_from_char_slice(value.optional_unix_socket_filename)?; + let unix_socket_path = value.optional_unix_socket_filename.try_to_string_option()?; Self::new( additional_files, create_alt_stack, @@ -102,22 +98,6 @@ impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerConfiguration } } -#[repr(C)] -pub enum UsizeResult { - Ok(usize), - #[allow(dead_code)] - Err(Error), -} - -impl From> for UsizeResult { - fn from(value: anyhow::Result) -> Self { - match value { - Ok(x) => Self::Ok(x), - Err(err) => Self::Err(err.into()), - } - } -} - #[repr(C)] pub enum CrashtrackerGetCountersResult { Ok([i64; OpTypes::SIZE as usize]), diff --git a/crashtracker-ffi/src/collector/mod.rs b/crashtracker-ffi/src/collector/mod.rs index 830279694..c705bd5a9 100644 --- a/crashtracker-ffi/src/collector/mod.rs +++ b/crashtracker-ffi/src/collector/mod.rs @@ -5,11 +5,12 @@ mod datatypes; mod spans; use super::crash_info::Metadata; -use crate::Result; use anyhow::Context; pub use counters::*; use datadog_crashtracker::CrashtrackerReceiverConfig; pub use datatypes::*; +use ddcommon_ffi::{wrap_with_void_ffi_result, VoidResult}; +use function_name::named; pub use spans::*; #[no_mangle] @@ -29,7 +30,7 @@ pub use spans::*; /// # Atomicity /// This function is not atomic. A crash during its execution may lead to /// unexpected crash-handling behaviour. -pub unsafe extern "C" fn ddog_crasht_shutdown() -> Result { +pub unsafe extern "C" fn ddog_crasht_shutdown() -> VoidResult { datadog_crashtracker::shutdown_crash_handler() .context("ddog_crasht_shutdown failed") .into() @@ -37,6 +38,7 @@ pub unsafe extern "C" fn ddog_crasht_shutdown() -> Result { #[no_mangle] #[must_use] +#[named] /// Reinitialize the crash-tracking infrastructure after a fork. /// This should be one of the first things done after a fork, to minimize the /// chance that a crash occurs between the fork, and this call. @@ -58,19 +60,19 @@ pub unsafe extern "C" fn ddog_crasht_update_on_fork( config: Config, receiver_config: ReceiverConfig, metadata: Metadata, -) -> Result { - (|| { - let config = config.try_into()?; - let receiver_config = receiver_config.try_into()?; - let metadata = metadata.try_into()?; - datadog_crashtracker::on_fork(config, receiver_config, metadata) - })() - .context("ddog_crasht_update_on_fork failed") - .into() +) -> VoidResult { + wrap_with_void_ffi_result!({ + datadog_crashtracker::on_fork( + config.try_into()?, + receiver_config.try_into()?, + metadata.try_into()?, + )?; + }) } #[no_mangle] #[must_use] +#[named] /// Initialize the crash-tracking infrastructure. /// /// # Preconditions @@ -85,19 +87,19 @@ pub unsafe extern "C" fn ddog_crasht_init( config: Config, receiver_config: ReceiverConfig, metadata: Metadata, -) -> Result { - (|| { - let config = config.try_into()?; - let receiver_config = receiver_config.try_into()?; - let metadata = metadata.try_into()?; - datadog_crashtracker::init(config, receiver_config, metadata) - })() - .context("ddog_crasht_init failed") - .into() +) -> VoidResult { + wrap_with_void_ffi_result!({ + datadog_crashtracker::init( + config.try_into()?, + receiver_config.try_into()?, + metadata.try_into()?, + )?; + }) } #[no_mangle] #[must_use] +#[named] /// Initialize the crash-tracking infrastructure without launching the receiver. /// /// # Preconditions @@ -111,31 +113,21 @@ pub unsafe extern "C" fn ddog_crasht_init( pub unsafe extern "C" fn ddog_crasht_init_without_receiver( config: Config, metadata: Metadata, -) -> Result { - (|| { - let config: datadog_crashtracker::CrashtrackerConfiguration = config.try_into()?; - let metadata = metadata.try_into()?; - +) -> VoidResult { + wrap_with_void_ffi_result!({ // If the unix domain socket path is not set, then we throw an error--there's currently no // other way to specify communication between an async receiver and a collector, so this // isn't a valid configuration. - if config.unix_socket_path.is_none() { - return Err(anyhow::anyhow!("config.unix_socket_path must be set")); - } - if config.unix_socket_path.as_ref().unwrap().is_empty() { - return Err(anyhow::anyhow!("config.unix_socket_path can't be empty")); - } + anyhow::ensure!( + !config.optional_unix_socket_filename.is_empty(), + "config.optional_unix_socket_filename must be set in this configuration" + ); - // Populate an empty receiver config - let receiver_config = CrashtrackerReceiverConfig { - args: vec![], - env: vec![], - path_to_receiver_binary: "".to_string(), - stderr_filename: None, - stdout_filename: None, - }; - datadog_crashtracker::init(config, receiver_config, metadata) - })() - .context("ddog_crasht_init failed") - .into() + // No receiver, use an empty receiver config + datadog_crashtracker::init( + config.try_into()?, + CrashtrackerReceiverConfig::default(), + metadata.try_into()?, + )? + }) } diff --git a/crashtracker-ffi/src/collector/spans.rs b/crashtracker-ffi/src/collector/spans.rs index 4bf162228..e9f356961 100644 --- a/crashtracker-ffi/src/collector/spans.rs +++ b/crashtracker-ffi/src/collector/spans.rs @@ -1,9 +1,8 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::{Result, UsizeResult}; -use anyhow::Context; - +use ddcommon_ffi::{wrap_with_ffi_result, wrap_with_void_ffi_result, Result, VoidResult}; +use function_name::named; /// Resets all stored spans to 0. /// Expected to be used after a fork, to reset the spans on the child /// ATOMICITY: @@ -14,10 +13,9 @@ use anyhow::Context; /// No safety concerns. #[no_mangle] #[must_use] -pub unsafe extern "C" fn ddog_crasht_clear_span_ids() -> Result { - datadog_crashtracker::clear_spans() - .context("ddog_crasht_clear_span_ids failed") - .into() +#[named] +pub unsafe extern "C" fn ddog_crasht_clear_span_ids() -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::clear_spans()? }) } /// Resets all stored traces to 0. @@ -30,14 +28,14 @@ pub unsafe extern "C" fn ddog_crasht_clear_span_ids() -> Result { /// No safety concerns. #[no_mangle] #[must_use] -pub unsafe extern "C" fn ddog_crasht_clear_trace_ids() -> Result { - datadog_crashtracker::clear_traces() - .context("ddog_crasht_clear_trace_ids failed") - .into() +#[named] +pub unsafe extern "C" fn ddog_crasht_clear_trace_ids() -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::clear_traces()? }) } #[no_mangle] #[must_use] +#[named] /// Atomically registers an active traceId. /// Useful for tracking what operations were occurring when a crash occurred. /// 0 is reserved for "NoId" @@ -57,15 +55,16 @@ pub unsafe extern "C" fn ddog_crasht_clear_trace_ids() -> Result { /// /// # Safety /// No safety concerns. -pub unsafe extern "C" fn ddog_crasht_insert_trace_id(id_high: u64, id_low: u64) -> UsizeResult { - let id: u128 = (id_high as u128) << 64 | (id_low as u128); - datadog_crashtracker::insert_trace(id) - .context("ddog_crasht_insert_trace_id failed") - .into() +pub unsafe extern "C" fn ddog_crasht_insert_trace_id(id_high: u64, id_low: u64) -> Result { + wrap_with_ffi_result!({ + let id: u128 = (id_high as u128) << 64 | (id_low as u128); + datadog_crashtracker::insert_trace(id) + }) } #[no_mangle] #[must_use] +#[named] /// Atomically registers an active SpanId. /// Useful for tracking what operations were occurring when a crash occurred. /// 0 is reserved for "NoId". @@ -85,15 +84,16 @@ pub unsafe extern "C" fn ddog_crasht_insert_trace_id(id_high: u64, id_low: u64) /// /// # Safety /// No safety concerns. -pub unsafe extern "C" fn ddog_crasht_insert_span_id(id_high: u64, id_low: u64) -> UsizeResult { - let id: u128 = (id_high as u128) << 64 | (id_low as u128); - datadog_crashtracker::insert_span(id) - .context("ddog_crasht_insert_span_id failed") - .into() +pub unsafe extern "C" fn ddog_crasht_insert_span_id(id_high: u64, id_low: u64) -> Result { + wrap_with_ffi_result!({ + let id: u128 = (id_high as u128) << 64 | (id_low as u128); + datadog_crashtracker::insert_span(id) + }) } #[no_mangle] #[must_use] +#[named] /// Atomically removes a completed SpanId. /// Useful for tracking what operations were occurring when a crash occurred. /// 0 is reserved for "NoId" @@ -118,15 +118,16 @@ pub unsafe extern "C" fn ddog_crasht_remove_span_id( id_high: u64, id_low: u64, idx: usize, -) -> Result { - let id: u128 = (id_high as u128) << 64 | (id_low as u128); - datadog_crashtracker::remove_span(id, idx) - .context("ddog_crasht_remove_span_id failed") - .into() +) -> VoidResult { + wrap_with_void_ffi_result!({ + let id: u128 = (id_high as u128) << 64 | (id_low as u128); + datadog_crashtracker::remove_span(id, idx)? + }) } #[no_mangle] #[must_use] +#[named] /// Atomically removes a completed TraceId. /// Useful for tracking what operations were occurring when a crash occurred. /// 0 is reserved for "NoId" @@ -151,9 +152,9 @@ pub unsafe extern "C" fn ddog_crasht_remove_trace_id( id_high: u64, id_low: u64, idx: usize, -) -> Result { - let id: u128 = (id_high as u128) << 64 | (id_low as u128); - datadog_crashtracker::remove_trace(id, idx) - .context("ddog_crasht_remove_trace_id failed") - .into() +) -> VoidResult { + wrap_with_void_ffi_result!({ + let id: u128 = (id_high as u128) << 64 | (id_low as u128); + datadog_crashtracker::remove_trace(id, idx)? + }) } diff --git a/crashtracker-ffi/src/crash_info/api.rs b/crashtracker-ffi/src/crash_info/api.rs new file mode 100644 index 000000000..4ced394f8 --- /dev/null +++ b/crashtracker-ffi/src/crash_info/api.rs @@ -0,0 +1,69 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use datadog_crashtracker::rfc5_crash_info::CrashInfo; +use ddcommon::Endpoint; +use ddcommon_ffi::{wrap_with_void_ffi_result, Handle, ToInner, VoidResult}; +use function_name::named; + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Frame +/// made by this module, which has not previously been dropped. +#[no_mangle] +pub unsafe extern "C" fn ddog_crasht_CrashInfo_drop(builder: *mut Handle) { + // Technically, this function has been designed so if it's double-dropped + // then it's okay, but it's not something that should be relied on. + if !builder.is_null() { + drop((*builder).take()) + } +} + +/// # Safety +/// The `crash_info` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +#[cfg(unix)] +pub unsafe extern "C" fn ddog_crasht_CrashInfo_normalize_ips( + mut crash_info: *mut Handle, + pid: u32, +) -> VoidResult { + wrap_with_void_ffi_result!({ + crash_info.to_inner_mut()?.normalize_ips(pid)?; + }) +} + +/// # Safety +/// The `crash_info` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +#[cfg(unix)] +pub unsafe extern "C" fn ddog_crasht_CrashInfo_resolve_names( + mut crash_info: *mut Handle, + pid: u32, +) -> VoidResult { + wrap_with_void_ffi_result!({ + crash_info.to_inner_mut()?.resolve_names(pid)?; + }) +} + +/// # Safety +/// The `crash_info` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfo_upload_to_endpoint( + mut crash_info: *mut Handle, + endpoint: Option<&Endpoint>, +) -> VoidResult { + wrap_with_void_ffi_result!({ + crash_info + .to_inner_mut()? + .upload_to_endpoint(&endpoint.cloned())?; + }) +} diff --git a/crashtracker-ffi/src/crash_info/builder.rs b/crashtracker-ffi/src/crash_info/builder.rs new file mode 100644 index 000000000..97b450a9e --- /dev/null +++ b/crashtracker-ffi/src/crash_info/builder.rs @@ -0,0 +1,403 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::{Metadata, OsInfo, ProcInfo, SigInfo, Span, ThreadData}; +use ::function_name::named; +use datadog_crashtracker::rfc5_crash_info::{CrashInfo, CrashInfoBuilder, ErrorKind, StackTrace}; +use ddcommon_ffi::{ + slice::AsBytes, wrap_with_ffi_result, wrap_with_void_ffi_result, CharSlice, Handle, Result, + Slice, Timespec, ToInner, VoidResult, +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// FFI API // +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Create a new CrashInfoBuilder, and returns an opaque reference to it. +/// # Safety +/// No safety issues. +#[no_mangle] +#[must_use] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_new() -> Result> { + ddcommon_ffi::Result::Ok(CrashInfoBuilder::new().into()) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Frame +/// made by this module, which has not previously been dropped. +#[no_mangle] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_drop(builder: *mut Handle) { + // Technically, this function has been designed so if it's double-dropped + // then it's okay, but it's not something that should be relied on. + if !builder.is_null() { + drop((*builder).take()) + } +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_build( + mut builder: *mut Handle, +) -> Result> { + wrap_with_ffi_result!({ anyhow::Ok(builder.take()?.build()?.into()) }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_counter( + mut builder: *mut Handle, + name: CharSlice, + value: i64, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_counter(name.try_to_string()?, value)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The Kind must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_kind( + mut builder: *mut Handle, + kind: ErrorKind, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_kind(kind)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_file( + mut builder: *mut Handle, + filename: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_file( + filename + .try_to_string_option()? + .context("filename cannot be empty string")?, + )?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_file_and_contents( + mut builder: *mut Handle, + filename: CharSlice, + contents: Slice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + let filename = filename + .try_to_string_option()? + .context("filename cannot be empty string")?; + let contents = { + let mut accum = Vec::with_capacity(contents.len()); + for line in contents.iter() { + let line = line.try_to_string()?; + accum.push(line); + } + accum + }; + + builder + .to_inner_mut()? + .with_file_and_contents(filename, contents)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_fingerprint( + mut builder: *mut Handle, + fingerprint: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_fingerprint(fingerprint.try_to_string()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_incomplete( + mut builder: *mut Handle, + incomplete: bool, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_incomplete(incomplete)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_log_message( + mut builder: *mut Handle, + message: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_log_message(message.try_to_string()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_metadata( + mut builder: *mut Handle, + metadata: Metadata, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_metadata(metadata.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_os_info( + mut builder: *mut Handle, + os_info: OsInfo, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_os_info(os_info.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_os_info_this_machine( + mut builder: *mut Handle, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_os_info_this_machine()?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_proc_info( + mut builder: *mut Handle, + proc_info: ProcInfo, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_proc_info(proc_info.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_sig_info( + mut builder: *mut Handle, + sig_info: SigInfo, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_sig_info(sig_info.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_span_id( + mut builder: *mut Handle, + span_id: Span, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_span_id(span_id.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +/// Consumes the stack argument. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_stack( + mut builder: *mut Handle, + mut stack: *mut Handle, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_stack(*stack.take()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +/// Consumes the stack argument. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_thread( + mut builder: *mut Handle, + thread: ThreadData, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_thread(thread.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_timestamp( + mut builder: *mut Handle, + ts: Timespec, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_timestamp(ts.into())?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_timestamp_now( + mut builder: *mut Handle, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_timestamp_now()?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// All arguments must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_trace_id( + mut builder: *mut Handle, + trace_id: Span, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder + .to_inner_mut()? + .with_trace_id(trace_id.try_into()?)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_uuid( + mut builder: *mut Handle, + uuid: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + let uuid = uuid + .try_to_string_option()? + .context("UUID cannot be empty string")?; + builder.to_inner_mut()?.with_uuid(uuid)?; + }) +} + +/// # Safety +/// The `builder` can be null, but if non-null it must point to a Builder made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_CrashInfoBuilder_with_uuid_random( + mut builder: *mut Handle, +) -> VoidResult { + wrap_with_void_ffi_result!({ + builder.to_inner_mut()?.with_uuid_random()?; + }) +} diff --git a/crashtracker-ffi/src/crash_info/datatypes.rs b/crashtracker-ffi/src/crash_info/datatypes.rs index 4fa2fa197..2139d9db2 100644 --- a/crashtracker-ffi/src/crash_info/datatypes.rs +++ b/crashtracker-ffi/src/crash_info/datatypes.rs @@ -1,68 +1,11 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::option_from_char_slice; -use ddcommon::tag::Tag; use ddcommon_ffi::{ slice::{AsBytes, ByteSlice}, - CharSlice, Error, Slice, + CharSlice, Slice, }; -/// Represents a CrashInfo. Do not access its member for any reason, only use -/// the C API functions on this struct. -#[repr(C)] -pub struct CrashInfo { - // This may be null, but if not it will point to a valid CrashInfo. - inner: *mut datadog_crashtracker::CrashInfo, -} - -impl CrashInfo { - pub(super) fn new(crash_info: datadog_crashtracker::CrashInfo) -> Self { - CrashInfo { - inner: Box::into_raw(Box::new(crash_info)), - } - } - - pub(super) fn take(&mut self) -> Option> { - // Leaving a null will help with double-free issues that can - // arise in C. Of course, it's best to never get there in the - // first place! - let raw = std::mem::replace(&mut self.inner, std::ptr::null_mut()); - - if raw.is_null() { - None - } else { - Some(unsafe { Box::from_raw(raw) }) - } - } -} - -impl Drop for CrashInfo { - fn drop(&mut self) { - drop(self.take()) - } -} - -pub(crate) unsafe fn crashinfo_ptr_to_inner<'a>( - crashinfo_ptr: *mut CrashInfo, -) -> anyhow::Result<&'a mut datadog_crashtracker::CrashInfo> { - match crashinfo_ptr.as_mut() { - None => anyhow::bail!("crashinfo pointer was null"), - Some(inner_ptr) => match inner_ptr.inner.as_mut() { - Some(crashinfo) => Ok(crashinfo), - None => anyhow::bail!("crashinfo's inner pointer was null (indicates use-after-free)"), - }, - } -} - -/// Returned by [ddog_prof_Profile_new]. -#[repr(C)] -pub enum CrashInfoNewResult { - Ok(CrashInfo), - #[allow(dead_code)] - Err(Error), -} - #[repr(C)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum NormalizedAddressTypes { @@ -166,9 +109,9 @@ impl<'a> TryFrom<&StackFrameNames<'a>> for datadog_crashtracker::StackFrameNames fn try_from(value: &StackFrameNames<'a>) -> Result { let colno = (&value.colno).into(); - let filename = option_from_char_slice(value.filename)?; + let filename = value.filename.try_to_string_option()?; let lineno = (&value.lineno).into(); - let name = option_from_char_slice(value.name)?; + let name = value.name.try_to_string_option()?; Ok(Self { colno, filename, @@ -179,7 +122,7 @@ impl<'a> TryFrom<&StackFrameNames<'a>> for datadog_crashtracker::StackFrameNames } #[repr(C)] -pub struct StackFrame<'a> { +pub struct StackFrameOld<'a> { build_id: CharSlice<'a>, ip: usize, module_base_address: usize, @@ -189,10 +132,10 @@ pub struct StackFrame<'a> { symbol_address: usize, } -impl<'a> TryFrom<&StackFrame<'a>> for datadog_crashtracker::StackFrame { +impl<'a> TryFrom<&StackFrameOld<'a>> for datadog_crashtracker::StackFrame { type Error = anyhow::Error; - fn try_from(value: &StackFrame<'a>) -> Result { + fn try_from(value: &StackFrameOld<'a>) -> Result { fn to_hex(v: usize) -> Option { if v == 0 { None @@ -224,61 +167,3 @@ impl<'a> TryFrom<&StackFrame<'a>> for datadog_crashtracker::StackFrame { }) } } - -#[repr(C)] -pub struct SigInfo<'a> { - pub signum: u64, - pub signame: CharSlice<'a>, -} - -impl<'a> TryFrom> for datadog_crashtracker::SigInfo { - type Error = anyhow::Error; - - fn try_from(value: SigInfo<'a>) -> Result { - let signum = value.signum; - let signame = option_from_char_slice(value.signame)?; - let faulting_address = None; // TODO: Expose this to FFI - Ok(Self { - signum, - signame, - faulting_address, - }) - } -} - -#[repr(C)] -pub struct ProcInfo { - pub pid: u32, -} - -impl TryFrom for datadog_crashtracker::ProcessInfo { - type Error = anyhow::Error; - - fn try_from(value: ProcInfo) -> anyhow::Result { - let pid = value.pid; - Ok(Self { pid }) - } -} - -#[repr(C)] -pub struct Metadata<'a> { - pub library_name: CharSlice<'a>, - pub library_version: CharSlice<'a>, - pub family: CharSlice<'a>, - /// Should include "service", "environment", etc - pub tags: Option<&'a ddcommon_ffi::Vec>, -} - -impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerMetadata { - type Error = anyhow::Error; - fn try_from(value: Metadata<'a>) -> anyhow::Result { - let library_name = value.library_name.try_to_utf8()?.to_string(); - let library_version = value.library_version.try_to_utf8()?.to_string(); - let family = value.family.try_to_utf8()?.to_string(); - let tags = value - .tags - .map(|tags| tags.iter().cloned().collect()) - .unwrap_or_default(); - Ok(Self::new(library_name, library_version, family, tags)) - } -} diff --git a/crashtracker-ffi/src/crash_info/metadata.rs b/crashtracker-ffi/src/crash_info/metadata.rs new file mode 100644 index 000000000..7f2af08de --- /dev/null +++ b/crashtracker-ffi/src/crash_info/metadata.rs @@ -0,0 +1,53 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use ddcommon::tag::Tag; +use ddcommon_ffi::{slice::AsBytes, CharSlice}; + +#[repr(C)] +pub struct Metadata<'a> { + pub library_name: CharSlice<'a>, + pub library_version: CharSlice<'a>, + pub family: CharSlice<'a>, + /// Should include "service", "environment", etc + pub tags: Option<&'a ddcommon_ffi::Vec>, +} + +impl<'a> TryFrom> for datadog_crashtracker::rfc5_crash_info::Metadata { + type Error = anyhow::Error; + fn try_from(value: Metadata<'a>) -> anyhow::Result { + let library_name = value.library_name.try_to_string()?; + let library_version = value.library_version.try_to_string()?; + let family = value.family.try_to_string()?; + let tags = if let Some(tags) = value.tags { + tags.into_iter().map(|t| t.to_string()).collect() + } else { + vec![] + }; + Ok(Self { + library_name, + library_version, + family, + tags, + }) + } +} + +impl<'a> TryFrom> for datadog_crashtracker::CrashtrackerMetadata { + type Error = anyhow::Error; + fn try_from(value: Metadata<'a>) -> anyhow::Result { + let library_name = value.library_name.try_to_string()?; + let library_version = value.library_version.try_to_string()?; + let family = value.family.try_to_string()?; + let tags = value + .tags + .map(|tags| tags.iter().cloned().collect()) + .unwrap_or_default(); + Ok(Self { + library_name, + library_version, + family, + tags, + }) + } +} diff --git a/crashtracker-ffi/src/crash_info/mod.rs b/crashtracker-ffi/src/crash_info/mod.rs index fddafd8c2..3d1a56f36 100644 --- a/crashtracker-ffi/src/crash_info/mod.rs +++ b/crashtracker-ffi/src/crash_info/mod.rs @@ -1,264 +1,26 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +mod api; +mod builder; mod datatypes; +mod metadata; +mod os_info; +mod proc_info; +mod sig_info; +mod span; +mod stackframe; +mod stacktrace; +mod thread_data; + +pub use api::*; +pub use builder::*; pub use datatypes::*; - -use crate::{option_from_char_slice, Result}; -use anyhow::Context; -use ddcommon::Endpoint; -use ddcommon_ffi::{slice::AsBytes, CharSlice, Slice, Timespec}; - -/// Create a new crashinfo, and returns an opaque reference to it. -/// # Safety -/// No safety issues. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_new() -> CrashInfoNewResult { - CrashInfoNewResult::Ok(CrashInfo::new(datadog_crashtracker::CrashInfo::new())) -} - -/// # Safety -/// The `crash_info` can be null, but if non-null it must point to a CrashInfo -/// made by this module, which has not previously been dropped. -#[no_mangle] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_drop(crashinfo: *mut CrashInfo) { - // Technically, this function has been designed so if it's double-dropped - // then it's okay, but it's not something that should be relied on. - if !crashinfo.is_null() { - drop((*crashinfo).take()) - } -} - -/// Best effort attempt to normalize all `ip` on the stacktrace. -/// `pid` must be the pid of the currently active process where the ips came from. -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -#[cfg(unix)] -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_normalize_ips( - crashinfo: *mut CrashInfo, - pid: u32, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - crashinfo.normalize_ips(pid) - })() - .context("ddog_crasht_CrashInfo_normalize_ips failed") - .into() -} - -/// Adds a "counter" variable, with the given value. Useful for determining if -/// "interesting" operations were occurring when the crash did. -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// `name` should be a valid reference to a utf8 encoded String. -/// The string is copied into the crashinfo, so it does not need to outlive this -/// call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_add_counter( - crashinfo: *mut CrashInfo, - name: CharSlice, - val: i64, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let name = name.to_utf8_lossy(); - crashinfo.add_counter(&name, val) - })() - .context("ddog_crasht_CrashInfo_add_counter failed") - .into() -} - -/// Adds the contents of "file" to the crashinfo -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// `name` should be a valid reference to a utf8 encoded String. -/// The string is copied into the crashinfo, so it does not need to outlive this -/// call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_add_file( - crashinfo: *mut CrashInfo, - filename: CharSlice, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let filename = filename.to_utf8_lossy(); - crashinfo.add_file(&filename) - })() - .context("ddog_crasht_CrashInfo_add_file failed") - .into() -} - -/// Adds the tag with given "key" and "value" to the crashinfo -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// `key` should be a valid reference to a utf8 encoded String. -/// `value` should be a valid reference to a utf8 encoded String. -/// The string is copied into the crashinfo, so it does not need to outlive this -/// call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_add_tag( - crashinfo: *mut CrashInfo, - key: CharSlice, - value: CharSlice, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let key = key.to_utf8_lossy().to_string(); - let value = value.to_utf8_lossy().to_string(); - crashinfo.add_tag(key, value) - })() - .context("ddog_crasht_CrashInfo_add_tag failed") - .into() -} - -/// Sets the crashinfo metadata -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// All references inside `metadata` must be valid. -/// Strings are copied into the crashinfo, and do not need to outlive this call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_metadata( - crashinfo: *mut CrashInfo, - metadata: Metadata, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let metadata = metadata.try_into()?; - crashinfo.set_metadata(metadata) - })() - .context("ddog_crasht_CrashInfo_set_metadata failed") - .into() -} - -/// Sets the crashinfo siginfo -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// All references inside `metadata` must be valid. -/// Strings are copied into the crashinfo, and do not need to outlive this call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_siginfo( - crashinfo: *mut CrashInfo, - siginfo: SigInfo, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let siginfo = siginfo.try_into()?; - crashinfo.set_siginfo(siginfo) - })() - .context("ddog_crasht_CrashInfo_set_siginfo failed") - .into() -} - -/// If `thread_id` is empty, sets `stacktrace` as the default stacktrace. -/// Otherwise, adds an additional stacktrace with id "thread_id". -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -/// All references inside `stacktraces` must be valid. -/// Strings are copied into the crashinfo, and do not need to outlive this call. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_stacktrace( - crashinfo: *mut CrashInfo, - thread_id: CharSlice, - stacktrace: Slice, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let thread_id = option_from_char_slice(thread_id)?; - let mut stacktrace_vec = Vec::with_capacity(stacktrace.len()); - for s in stacktrace.iter() { - stacktrace_vec.push(s.try_into()?) - } - crashinfo.set_stacktrace(thread_id, stacktrace_vec) - })() - .context("ddog_crasht_CrashInfo_set_stacktrace failed") - .into() -} - -/// Sets the timestamp to the given unix timestamp -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_timestamp( - crashinfo: *mut CrashInfo, - ts: Timespec, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - crashinfo.set_timestamp(ts.into()) - })() - .context("ddog_crasht_CrashInfo_set_timestamp_to_now failed") - .into() -} - -/// Sets the timestamp to the current time -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_timestamp_to_now( - crashinfo: *mut CrashInfo, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - crashinfo.set_timestamp_to_now() - })() - .context("ddog_crasht_CrashInfo_set_timestamp_to_now failed") - .into() -} - -/// Sets crashinfo procinfo -/// -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_set_procinfo( - crashinfo: *mut CrashInfo, - procinfo: ProcInfo, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let procinfo = procinfo.try_into()?; - crashinfo.set_procinfo(procinfo) - })() - .context("ddog_crasht_CrashInfo_set_procinfo failed") - .into() -} - -/// Exports `crashinfo` to the backend at `endpoint` -/// Note that we support the "file://" endpoint for local file output. -/// # Safety -/// `crashinfo` must be a valid pointer to a `CrashInfo` object. -#[no_mangle] -#[must_use] -pub unsafe extern "C" fn ddog_crasht_CrashInfo_upload_to_endpoint( - crashinfo: *mut CrashInfo, - endpoint: Option<&Endpoint>, -) -> Result { - (|| { - let crashinfo = crashinfo_ptr_to_inner(crashinfo)?; - let endpoint = endpoint.cloned(); - crashinfo.upload_to_endpoint(&endpoint) - })() - .context("ddog_crasht_CrashInfo_upload_to_endpoint failed") - .into() -} +pub use metadata::*; +pub use os_info::*; +pub use proc_info::*; +pub use sig_info::*; +pub use span::*; +pub use stackframe::*; +pub use stacktrace::*; +pub use thread_data::*; diff --git a/crashtracker-ffi/src/crash_info/os_info.rs b/crashtracker-ffi/src/crash_info/os_info.rs new file mode 100644 index 000000000..6c81032f3 --- /dev/null +++ b/crashtracker-ffi/src/crash_info/os_info.rs @@ -0,0 +1,41 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use ddcommon_ffi::{slice::AsBytes, CharSlice}; + +#[repr(C)] +pub struct OsInfo<'a> { + pub architecture: CharSlice<'a>, + pub bitness: CharSlice<'a>, + pub os_type: CharSlice<'a>, + pub version: CharSlice<'a>, +} + +impl<'a> TryFrom> for datadog_crashtracker::rfc5_crash_info::OsInfo { + type Error = anyhow::Error; + fn try_from(value: OsInfo<'a>) -> anyhow::Result { + let unknown = || "unknown".to_string(); + let architecture = value + .architecture + .try_to_string_option()? + .unwrap_or_else(unknown); + let bitness = value + .bitness + .try_to_string_option()? + .unwrap_or_else(unknown); + let os_type = value + .os_type + .try_to_string_option()? + .unwrap_or_else(unknown); + let version = value + .version + .try_to_string_option()? + .unwrap_or_else(unknown); + Ok(Self { + architecture, + bitness, + os_type, + version, + }) + } +} diff --git a/crashtracker-ffi/src/crash_info/proc_info.rs b/crashtracker-ffi/src/crash_info/proc_info.rs new file mode 100644 index 000000000..e815c0eea --- /dev/null +++ b/crashtracker-ffi/src/crash_info/proc_info.rs @@ -0,0 +1,14 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#[repr(C)] +pub struct ProcInfo { + pid: u32, +} + +impl TryFrom for datadog_crashtracker::rfc5_crash_info::ProcInfo { + type Error = anyhow::Error; + fn try_from(value: ProcInfo) -> anyhow::Result { + Ok(Self { pid: value.pid }) + } +} diff --git a/crashtracker-ffi/src/crash_info/sig_info.rs b/crashtracker-ffi/src/crash_info/sig_info.rs new file mode 100644 index 000000000..9352b7fe2 --- /dev/null +++ b/crashtracker-ffi/src/crash_info/sig_info.rs @@ -0,0 +1,27 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use datadog_crashtracker::rfc5_crash_info::{SiCodes, SignalNames}; +use ddcommon_ffi::{slice::AsBytes, CharSlice}; + +#[repr(C)] +pub struct SigInfo<'a> { + pub addr: CharSlice<'a>, + pub code: libc::c_int, + pub code_human_readable: SiCodes, + pub signo: libc::c_int, + pub signo_human_readable: SignalNames, +} + +impl<'a> TryFrom> for datadog_crashtracker::rfc5_crash_info::SigInfo { + type Error = anyhow::Error; + fn try_from(value: SigInfo<'a>) -> anyhow::Result { + Ok(Self { + si_addr: value.addr.try_to_string_option()?, + si_code: value.code, + si_code_human_readable: value.code_human_readable, + si_signo: value.signo, + si_signo_human_readable: value.signo_human_readable, + }) + } +} diff --git a/crashtracker-ffi/src/crash_info/span.rs b/crashtracker-ffi/src/crash_info/span.rs new file mode 100644 index 000000000..4f912e49e --- /dev/null +++ b/crashtracker-ffi/src/crash_info/span.rs @@ -0,0 +1,23 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Context; +use ddcommon_ffi::{slice::AsBytes, CharSlice}; +#[repr(C)] +pub struct Span<'a> { + pub id: CharSlice<'a>, + pub thread_name: CharSlice<'a>, +} + +impl<'a> TryFrom> for datadog_crashtracker::rfc5_crash_info::Span { + type Error = anyhow::Error; + fn try_from(value: Span<'a>) -> anyhow::Result { + Ok(Self { + id: value + .id + .try_to_string_option()? + .context("id cannot be empty")?, + thread_name: value.thread_name.try_to_string_option()?, + }) + } +} diff --git a/crashtracker-ffi/src/crash_info/stackframe.rs b/crashtracker-ffi/src/crash_info/stackframe.rs new file mode 100644 index 000000000..7e03b3fe4 --- /dev/null +++ b/crashtracker-ffi/src/crash_info/stackframe.rs @@ -0,0 +1,239 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use ::function_name::named; +use datadog_crashtracker::rfc5_crash_info::{BuildIdType, FileType, StackFrame}; +use ddcommon_ffi::{ + slice::AsBytes, wrap_with_void_ffi_result, CharSlice, Handle, Result, ToInner, VoidResult, +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// FFI API // +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Create a new StackFrame, and returns an opaque reference to it. +/// # Safety +/// No safety issues. +#[no_mangle] +#[must_use] +pub unsafe extern "C" fn ddog_crasht_StackFrame_new() -> Result> { + ddcommon_ffi::Result::Ok(StackFrame::new().into()) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame +/// made by this module, which has not previously been dropped. +#[no_mangle] +pub unsafe extern "C" fn ddog_crasht_StackFrame_drop(frame: *mut Handle) { + // Technically, this function has been designed so if it's double-dropped + // then it's okay, but it's not something that should be relied on. + if !frame.is_null() { + drop((*frame).take()) + } +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_ip( + mut frame: *mut Handle, + ip: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.ip = ip.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_module_base_address( + mut frame: *mut Handle, + module_base_address: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.module_base_address = module_base_address.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_sp( + mut frame: *mut Handle, + sp: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.sp = sp.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_symbol_address( + mut frame: *mut Handle, + symbol_address: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.symbol_address = symbol_address.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_build_id( + mut frame: *mut Handle, + build_id: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.build_id = build_id.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The BuildIdType must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_build_id_type( + mut frame: *mut Handle, + build_id_type: BuildIdType, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.build_id_type = Some(build_id_type); + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The FileType must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_file_type( + mut frame: *mut Handle, + file_type: FileType, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.file_type = Some(file_type); + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_path( + mut frame: *mut Handle, + path: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.path = path.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_relative_address( + mut frame: *mut Handle, + relative_address: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.relative_address = relative_address.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_column( + mut frame: *mut Handle, + column: u32, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.column = Some(column); + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_file( + mut frame: *mut Handle, + file: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.file = file.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The CharSlice must be valid. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_function( + mut frame: *mut Handle, + function: CharSlice, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.function = function.try_to_string_option()?; + }) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackFrame_with_line( + mut frame: *mut Handle, + line: u32, +) -> VoidResult { + wrap_with_void_ffi_result!({ + frame.to_inner_mut()?.line = Some(line); + }) +} diff --git a/crashtracker-ffi/src/crash_info/stacktrace.rs b/crashtracker-ffi/src/crash_info/stacktrace.rs new file mode 100644 index 000000000..967da961e --- /dev/null +++ b/crashtracker-ffi/src/crash_info/stacktrace.rs @@ -0,0 +1,66 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use ::function_name::named; +use datadog_crashtracker::rfc5_crash_info::{StackFrame, StackTrace}; +use ddcommon_ffi::{wrap_with_void_ffi_result, Handle, Result, ToInner, VoidResult}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// FFI API // +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Create a new StackTrace, and returns an opaque reference to it. +/// # Safety +/// No safety issues. +#[no_mangle] +#[must_use] +pub unsafe extern "C" fn ddog_crasht_StackTrace_new() -> Result> { + ddcommon_ffi::Result::Ok(StackTrace::new_incomplete().into()) +} + +/// # Safety +/// The `frame` can be null, but if non-null it must point to a Frame +/// made by this module, which has not previously been dropped. +#[no_mangle] +pub unsafe extern "C" fn ddog_crasht_StackTrace_drop(trace: *mut Handle) { + // Technically, this function has been designed so if it's double-dropped + // then it's okay, but it's not something that should be relied on. + if !trace.is_null() { + drop((*trace).take()) + } +} + +/// # Safety +/// The `stacktrace` can be null, but if non-null it must point to a StackTrace made by this module, +/// which has not previously been dropped. +/// The frame can be non-null, but if non-null it must point to a Frame made by this module, +/// which has not previously been dropped. +/// The frame is consumed, and does not need to be dropped after this operation. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackTrace_push_frame( + mut trace: *mut Handle, + mut frame: *mut Handle, + incomplete: bool, +) -> VoidResult { + wrap_with_void_ffi_result!({ + trace + .to_inner_mut()? + .push_frame(*frame.take()?, incomplete)?; + }) +} + +/// # Safety +/// The `stacktrace` can be null, but if non-null it must point to a StackTrace made by this module, +/// which has not previously been dropped. +#[no_mangle] +#[must_use] +#[named] +pub unsafe extern "C" fn ddog_crasht_StackTrace_set_complete( + mut trace: *mut Handle, +) -> VoidResult { + wrap_with_void_ffi_result!({ + trace.to_inner_mut()?.set_complete()?; + }) +} diff --git a/crashtracker-ffi/src/crash_info/thread_data.rs b/crashtracker-ffi/src/crash_info/thread_data.rs new file mode 100644 index 000000000..f451ea892 --- /dev/null +++ b/crashtracker-ffi/src/crash_info/thread_data.rs @@ -0,0 +1,36 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Context; +use ddcommon_ffi::{slice::AsBytes, CharSlice, Handle, ToInner}; + +use datadog_crashtracker::rfc5_crash_info::StackTrace; + +#[repr(C)] +pub struct ThreadData<'a> { + pub crashed: bool, + pub name: CharSlice<'a>, + pub stack: Handle, + pub state: CharSlice<'a>, +} + +impl<'a> TryFrom> for datadog_crashtracker::rfc5_crash_info::ThreadData { + type Error = anyhow::Error; + fn try_from(mut value: ThreadData<'a>) -> anyhow::Result { + let crashed = value.crashed; + let name = value + .name + .try_to_string_option()? + .context("Name cannot be empty")?; + // Safety: Handles should only be created using functions that leave them in a valid state. + let stack = unsafe { *value.stack.take()? }; + let state = value.state.try_to_string_option()?; + + Ok(Self { + crashed, + name, + stack, + state, + }) + } +} diff --git a/crashtracker-ffi/src/datatypes/mod.rs b/crashtracker-ffi/src/datatypes/mod.rs deleted file mode 100644 index 04eb263ab..000000000 --- a/crashtracker-ffi/src/datatypes/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use ddcommon_ffi::slice::{AsBytes, CharSlice}; -use ddcommon_ffi::Error; -use std::ops::Not; - -pub fn option_from_char_slice(s: CharSlice) -> anyhow::Result> { - let s = s.try_to_utf8()?.to_string(); - Ok(s.is_empty().not().then_some(s)) -} - -/// A generic result type for when a crashtracking operation may fail, -/// but there's nothing to return in the case of success. -#[repr(C)] -pub enum Result { - Ok( - /// Do not use the value of Ok. This value only exists to overcome - /// Rust -> C code generation. - bool, - ), - Err(Error), -} - -impl From> for Result { - fn from(value: anyhow::Result<()>) -> Self { - match value { - Ok(_) => Self::Ok(true), - Err(err) => Self::Err(err.into()), - } - } -} diff --git a/crashtracker-ffi/src/demangler/mod.rs b/crashtracker-ffi/src/demangler/mod.rs index 06755dcb6..7d6a58104 100644 --- a/crashtracker-ffi/src/demangler/mod.rs +++ b/crashtracker-ffi/src/demangler/mod.rs @@ -3,7 +3,8 @@ mod datatypes; pub use datatypes::*; -use ddcommon_ffi::{slice::AsBytes, CharSlice}; +use ::function_name::named; +use ddcommon_ffi::{slice::AsBytes, wrap_with_ffi_result, CharSlice, Result, StringWrapper}; use symbolic_common::Name; use symbolic_demangle::Demangle; @@ -15,17 +16,20 @@ use symbolic_demangle::Demangle; /// The string is copied into the result, and does not need to outlive this call #[no_mangle] #[must_use] +#[named] pub unsafe extern "C" fn ddog_crasht_demangle( name: CharSlice, options: DemangleOptions, -) -> StringWrapperResult { - let name = name.to_utf8_lossy(); - let name = Name::from(name); - let options = match options { - DemangleOptions::Complete => symbolic_demangle::DemangleOptions::complete(), - DemangleOptions::NameOnly => symbolic_demangle::DemangleOptions::name_only(), - }; - StringWrapperResult::Ok(name.demangle(options).unwrap_or_default().into()) +) -> Result { + wrap_with_ffi_result!({ + let name = name.to_utf8_lossy(); + let name = Name::from(name); + let options = match options { + DemangleOptions::Complete => symbolic_demangle::DemangleOptions::complete(), + DemangleOptions::NameOnly => symbolic_demangle::DemangleOptions::name_only(), + }; + anyhow::Ok(name.demangle(options).unwrap_or_default().into()) + }) } #[test] diff --git a/crashtracker-ffi/src/lib.rs b/crashtracker-ffi/src/lib.rs index 1dc4a8a8d..a9811d61c 100644 --- a/crashtracker-ffi/src/lib.rs +++ b/crashtracker-ffi/src/lib.rs @@ -4,7 +4,6 @@ #[cfg(all(unix, feature = "collector"))] mod collector; mod crash_info; -mod datatypes; #[cfg(feature = "demangler")] mod demangler; #[cfg(all(unix, feature = "receiver"))] @@ -13,7 +12,6 @@ mod receiver; #[cfg(all(unix, feature = "collector"))] pub use collector::*; pub use crash_info::*; -pub use datatypes::*; #[cfg(feature = "demangler")] pub use demangler::*; #[cfg(all(unix, feature = "receiver"))] diff --git a/crashtracker-ffi/src/receiver.rs b/crashtracker-ffi/src/receiver.rs index 8349ff0dd..741cc1a50 100644 --- a/crashtracker-ffi/src/receiver.rs +++ b/crashtracker-ffi/src/receiver.rs @@ -1,11 +1,11 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::Result; -use anyhow::Context; -use ddcommon_ffi::{slice::AsBytes, CharSlice}; +use ::function_name::named; +use ddcommon_ffi::{slice::AsBytes, wrap_with_void_ffi_result, CharSlice, VoidResult}; #[no_mangle] #[must_use] +#[named] /// Receives data from a crash collector via a pipe on `stdin`, formats it into /// `CrashInfo` json, and emits it to the endpoint/file defined in `config`. /// @@ -16,14 +16,13 @@ use ddcommon_ffi::{slice::AsBytes, CharSlice}; /// See comments in [crashtracker/lib.rs] for a full architecture description. /// # Safety /// No safety concerns -pub unsafe extern "C" fn ddog_crasht_receiver_entry_point_stdin() -> Result { - datadog_crashtracker::receiver_entry_point_stdin() - .context("ddog_crasht_receiver_entry_point_stdin failed") - .into() +pub unsafe extern "C" fn ddog_crasht_receiver_entry_point_stdin() -> VoidResult { + wrap_with_void_ffi_result!({ datadog_crashtracker::receiver_entry_point_stdin()? }) } #[no_mangle] #[must_use] +#[named] /// Receives data from a crash collector via a pipe on `stdin`, formats it into /// `CrashInfo` json, and emits it to the endpoint/file defined in `config`. /// @@ -37,11 +36,8 @@ pub unsafe extern "C" fn ddog_crasht_receiver_entry_point_stdin() -> Result { /// No safety concerns pub unsafe extern "C" fn ddog_crasht_receiver_entry_point_unix_socket( socket_path: CharSlice, -) -> Result { - (|| { - let socket_path = socket_path.try_to_utf8()?; - datadog_crashtracker::receiver_entry_point_unix_socket(socket_path) - })() - .context("ddog_crasht_receiver_entry_point_unix_socket failed") - .into() +) -> VoidResult { + wrap_with_void_ffi_result!({ + datadog_crashtracker::receiver_entry_point_unix_socket(socket_path.try_to_utf8()?)? + }) } diff --git a/crashtracker/libdatadog-crashtracking-receiver.c b/crashtracker/libdatadog-crashtracking-receiver.c index c8e3e265f..bc9e241fd 100644 --- a/crashtracker/libdatadog-crashtracking-receiver.c +++ b/crashtracker/libdatadog-crashtracking-receiver.c @@ -7,8 +7,8 @@ #include int main(void) { - ddog_crasht_Result new_result = ddog_crasht_receiver_entry_point_stdin(); - if (new_result.tag != DDOG_CRASHT_RESULT_OK) { + ddog_VoidResult new_result = ddog_crasht_receiver_entry_point_stdin(); + if (new_result.tag != DDOG_VOID_RESULT_OK) { ddog_CharSlice message = ddog_Error_message(&new_result.err); fprintf(stderr, "%.*s", (int)message.len, message.ptr); ddog_Error_drop(&new_result.err); diff --git a/crashtracker/src/crash_info/telemetry.rs b/crashtracker/src/crash_info/telemetry.rs index 37ae43b47..fa58e7c1f 100644 --- a/crashtracker/src/crash_info/telemetry.rs +++ b/crashtracker/src/crash_info/telemetry.rs @@ -149,6 +149,7 @@ impl TelemetryCrashUploader { seq_id: 1, application: &metadata.application, host: &metadata.host, + origin: Some("Crashtracker"), payload: &data::Payload::Logs(vec![data::Log { message, level: LogLevel::Error, @@ -165,6 +166,14 @@ impl TelemetryCrashUploader { http::header::CONTENT_TYPE, ddcommon::header::APPLICATION_JSON, ) + .header( + ddtelemetry::worker::http_client::header::API_VERSION, + ddtelemetry::data::ApiVersion::V2.to_str(), + ) + .header( + ddtelemetry::worker::http_client::header::REQUEST_TYPE, + "logs", + ) .body(serde_json::to_string(&payload)?.into())?; tokio::time::timeout( diff --git a/crashtracker/src/lib.rs b/crashtracker/src/lib.rs index 60324d017..fd8989512 100644 --- a/crashtracker/src/lib.rs +++ b/crashtracker/src/lib.rs @@ -49,10 +49,6 @@ mod collector; mod crash_info; #[cfg(all(unix, feature = "receiver"))] mod receiver; - -// TODO: For now, we have name conflicts with the `crash_info` module. -// Once that module is removed, those conflicts will go away -// Till then, keep things in two name spaces pub mod rfc5_crash_info; #[cfg(all(unix, any(feature = "collector", feature = "receiver")))] mod shared; diff --git a/crashtracker/src/receiver/entry_points.rs b/crashtracker/src/receiver/entry_points.rs new file mode 100644 index 000000000..8fc34d6d3 --- /dev/null +++ b/crashtracker/src/receiver/entry_points.rs @@ -0,0 +1,137 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::receive_report::{receive_report_from_stream, CrashReportStatus}; +use crate::{CrashInfo, CrashtrackerConfiguration, StacktraceCollection}; +use anyhow::Context; +use std::time::Duration; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + net::UnixListener, +}; + +/*----------------------------------------- +| Public API | +------------------------------------------*/ + +pub fn receiver_entry_point_stdin() -> anyhow::Result<()> { + let stream = BufReader::new(tokio::io::stdin()); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + rt.block_on(receiver_entry_point(receiver_timeout(), stream))?; + Ok(()) +} + +pub async fn async_receiver_entry_point_unix_socket( + socket_path: impl AsRef, + one_shot: bool, +) -> anyhow::Result<()> { + let listener = get_unix_socket(socket_path)?; + loop { + let (unix_stream, _) = listener.accept().await?; + let stream = BufReader::new(unix_stream); + let res = receiver_entry_point(receiver_timeout(), stream).await; + + if one_shot { + return res; + } + } +} + +pub fn receiver_entry_point_unix_socket(socket_path: impl AsRef) -> anyhow::Result<()> { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + rt.block_on(async_receiver_entry_point_unix_socket(socket_path, true))?; + Ok(()) + // Dropping the stream closes it, allowing the collector to exit if it was waiting. +} + +/*----------------------------------------- +| Helper Functions | +------------------------------------------*/ + +fn get_unix_socket(socket_path: impl AsRef) -> anyhow::Result { + fn path_bind(socket_path: impl AsRef) -> anyhow::Result { + let socket_path = socket_path.as_ref(); + if std::fs::metadata(socket_path).is_ok() { + std::fs::remove_file(socket_path).with_context(|| { + format!("could not delete previous socket at {:?}", socket_path) + })?; + } + Ok(UnixListener::bind(socket_path)?) + } + + #[cfg(target_os = "linux")] + let unix_listener = if socket_path.as_ref().starts_with(['.', '/']) { + path_bind(socket_path) + } else { + use std::os::linux::net::SocketAddrExt; + std::os::unix::net::SocketAddr::from_abstract_name(socket_path.as_ref()) + .and_then(|addr| { + std::os::unix::net::UnixListener::bind_addr(&addr) + .and_then(|listener| { + listener.set_nonblocking(true)?; + Ok(listener) + }) + .and_then(UnixListener::from_std) + }) + .map_err(anyhow::Error::msg) + }; + #[cfg(not(target_os = "linux"))] + let unix_listener = path_bind(socket_path); + unix_listener.context("Could not create the unix socket") +} + +/// Receives data from a crash collector via a stream, formats it into +/// `CrashInfo` json, and emits it to the endpoint/file defined in `config`. +/// +/// At a high-level, this exists because doing anything in a +/// signal handler is dangerous, so we fork a sidecar to do the stuff we aren't +/// allowed to do in the handler. +/// +/// See comments in [crashtracker/lib.rs] for a full architecture +/// description. +async fn receiver_entry_point( + timeout: Duration, + stream: impl AsyncBufReadExt + std::marker::Unpin, +) -> anyhow::Result<()> { + match receive_report_from_stream(timeout, stream).await? { + CrashReportStatus::NoCrash => Ok(()), + CrashReportStatus::CrashReport(config, mut crash_info) => { + resolve_frames(&config, &mut crash_info)?; + crash_info.async_upload_to_endpoint(&config.endpoint).await + } + CrashReportStatus::PartialCrashReport(config, mut crash_info, stdin_state) => { + eprintln!("Failed to fully receive crash. Exit state was: {stdin_state:?}"); + resolve_frames(&config, &mut crash_info)?; + crash_info.async_upload_to_endpoint(&config.endpoint).await + } + } +} + +fn receiver_timeout() -> Duration { + // https://github.com/DataDog/libdatadog/issues/717 + if let Ok(s) = std::env::var("DD_CRASHTRACKER_RECEIVER_TIMEOUT_MS") { + if let Ok(v) = s.parse() { + return Duration::from_millis(v); + } + } + // Default value + Duration::from_millis(4000) +} + +fn resolve_frames( + config: &CrashtrackerConfiguration, + crash_info: &mut CrashInfo, +) -> anyhow::Result<()> { + if config.resolve_frames == StacktraceCollection::EnabledWithSymbolsInReceiver { + let proc_info = crash_info + .proc_info + .as_ref() + .context("Unable to resolve frames: No PID specified")?; + crash_info.resolve_names_from_process(proc_info.pid)?; + } + Ok(()) +} diff --git a/crashtracker/src/receiver/mod.rs b/crashtracker/src/receiver/mod.rs new file mode 100644 index 000000000..b66433091 --- /dev/null +++ b/crashtracker/src/receiver/mod.rs @@ -0,0 +1,102 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 +#![cfg(unix)] + +mod entry_points; +pub use entry_points::{ + async_receiver_entry_point_unix_socket, receiver_entry_point_stdin, + receiver_entry_point_unix_socket, +}; +mod receive_report; + +#[cfg(test)] +mod tests { + use super::receive_report::*; + use crate::shared::constants::*; + use crate::{CrashtrackerConfiguration, SigInfo, StacktraceCollection}; + use std::time::Duration; + use tokio::io::{AsyncWriteExt, BufReader}; + use tokio::net::UnixStream; + + async fn to_socket( + target: &mut tokio::net::UnixStream, + msg: impl AsRef, + ) -> anyhow::Result { + let msg = msg.as_ref(); + let n = target.write(format!("{msg}\n").as_bytes()).await?; + target.flush().await?; + Ok(n) + } + + async fn send_report(delay: Duration, mut stream: UnixStream) -> anyhow::Result<()> { + let sender = &mut stream; + to_socket(sender, DD_CRASHTRACK_BEGIN_SIGINFO).await?; + to_socket( + sender, + serde_json::to_string(&SigInfo { + signame: Some("SIGSEGV".to_string()), + signum: 11, + faulting_address: None, + })?, + ) + .await?; + to_socket(sender, DD_CRASHTRACK_END_SIGINFO).await?; + + to_socket(sender, DD_CRASHTRACK_BEGIN_CONFIG).await?; + to_socket( + sender, + serde_json::to_string(&CrashtrackerConfiguration::new( + vec![], + false, + false, + None, + StacktraceCollection::Disabled, + 3000, + None, + )?)?, + ) + .await?; + to_socket(sender, DD_CRASHTRACK_END_CONFIG).await?; + tokio::time::sleep(delay).await; + to_socket(sender, DD_CRASHTRACK_DONE).await?; + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn test_receive_report_short_timeout() -> anyhow::Result<()> { + let (sender, receiver) = tokio::net::UnixStream::pair()?; + + let join_handle1 = tokio::spawn(receive_report_from_stream( + Duration::from_secs(1), + BufReader::new(receiver), + )); + let join_handle2 = tokio::spawn(send_report(Duration::from_secs(2), sender)); + + let crash_report = join_handle1.await??; + assert!(matches!( + crash_report, + CrashReportStatus::PartialCrashReport(_, _, _) + )); + let sender_error = join_handle2.await?.unwrap_err().to_string(); + assert_eq!(sender_error, "Broken pipe (os error 32)"); + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn test_receive_report_long_timeout() -> anyhow::Result<()> { + let (sender, receiver) = tokio::net::UnixStream::pair()?; + + let join_handle1 = tokio::spawn(receive_report_from_stream( + Duration::from_secs(2), + BufReader::new(receiver), + )); + let join_handle2 = tokio::spawn(send_report(Duration::from_secs(1), sender)); + + let crash_report = join_handle1.await??; + assert!(matches!(crash_report, CrashReportStatus::CrashReport(_, _))); + join_handle2.await??; + Ok(()) + } +} diff --git a/crashtracker/src/receiver.rs b/crashtracker/src/receiver/receive_report.rs similarity index 56% rename from crashtracker/src/receiver.rs rename to crashtracker/src/receiver/receive_report.rs index aaef56de7..06584175d 100644 --- a/crashtracker/src/receiver.rs +++ b/crashtracker/src/receiver/receive_report.rs @@ -1,138 +1,17 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -#![cfg(unix)] -use super::*; -use crate::shared::constants::*; +use crate::{shared::constants::*, CrashInfo, CrashtrackerConfiguration, StackFrame}; use anyhow::Context; use std::time::{Duration, Instant}; -use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio::net::UnixListener; - -pub fn resolve_frames( - config: &CrashtrackerConfiguration, - crash_info: &mut CrashInfo, -) -> anyhow::Result<()> { - if config.resolve_frames == StacktraceCollection::EnabledWithSymbolsInReceiver { - let proc_info = crash_info - .proc_info - .as_ref() - .context("Unable to resolve frames: No PID specified")?; - crash_info.resolve_names_from_process(proc_info.pid)?; - } - Ok(()) -} - -pub fn get_unix_socket(socket_path: impl AsRef) -> anyhow::Result { - fn path_bind(socket_path: impl AsRef) -> anyhow::Result { - let socket_path = socket_path.as_ref(); - if std::fs::metadata(socket_path).is_ok() { - std::fs::remove_file(socket_path).with_context(|| { - format!("could not delete previous socket at {:?}", socket_path) - })?; - } - Ok(UnixListener::bind(socket_path)?) - } - - #[cfg(target_os = "linux")] - let unix_listener = if socket_path.as_ref().starts_with(['.', '/']) { - path_bind(socket_path) - } else { - use std::os::linux::net::SocketAddrExt; - std::os::unix::net::SocketAddr::from_abstract_name(socket_path.as_ref()) - .and_then(|addr| { - std::os::unix::net::UnixListener::bind_addr(&addr) - .and_then(|listener| { - listener.set_nonblocking(true)?; - Ok(listener) - }) - .and_then(UnixListener::from_std) - }) - .map_err(anyhow::Error::msg) - }; - #[cfg(not(target_os = "linux"))] - let unix_listener = path_bind(socket_path); - unix_listener.context("Could not create the unix socket") -} - -pub fn receiver_entry_point_unix_socket(socket_path: impl AsRef) -> anyhow::Result<()> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - rt.block_on(async_receiver_entry_point_unix_socket(socket_path, true))?; - Ok(()) - // Dropping the stream closes it, allowing the collector to exit if it was waiting. -} - -pub fn receiver_timeout() -> Duration { - // https://github.com/DataDog/libdatadog/issues/717 - if let Ok(s) = std::env::var("DD_CRASHTRACKER_RECEIVER_TIMEOUT_MS") { - if let Ok(v) = s.parse() { - return Duration::from_millis(v); - } - } - // Default value - Duration::from_millis(4000) -} - -pub fn receiver_entry_point_stdin() -> anyhow::Result<()> { - let stream = BufReader::new(tokio::io::stdin()); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - rt.block_on(receiver_entry_point(receiver_timeout(), stream))?; - Ok(()) -} - -pub async fn async_receiver_entry_point_unix_socket( - socket_path: impl AsRef, - one_shot: bool, -) -> anyhow::Result<()> { - let listener = get_unix_socket(socket_path)?; - loop { - let (unix_stream, _) = listener.accept().await?; - let stream = BufReader::new(unix_stream); - let res = receiver_entry_point(receiver_timeout(), stream).await; - - if one_shot { - return res; - } - } -} - -/// Receives data from a crash collector via a stream, formats it into -/// `CrashInfo` json, and emits it to the endpoint/file defined in `config`. -/// -/// At a high-level, this exists because doing anything in a -/// signal handler is dangerous, so we fork a sidecar to do the stuff we aren't -/// allowed to do in the handler. -/// -/// See comments in [crashtracker/lib.rs] for a full architecture -/// description. -async fn receiver_entry_point( - timeout: Duration, - stream: impl AsyncBufReadExt + std::marker::Unpin, -) -> anyhow::Result<()> { - match receive_report(timeout, stream).await? { - CrashReportStatus::NoCrash => Ok(()), - CrashReportStatus::CrashReport(config, mut crash_info) => { - resolve_frames(&config, &mut crash_info)?; - crash_info.async_upload_to_endpoint(&config.endpoint).await - } - CrashReportStatus::PartialCrashReport(config, mut crash_info, stdin_state) => { - eprintln!("Failed to fully receive crash. Exit state was: {stdin_state:?}"); - resolve_frames(&config, &mut crash_info)?; - crash_info.async_upload_to_endpoint(&config.endpoint).await - } - } -} +use tokio::io::AsyncBufReadExt; /// The crashtracker collector sends data in blocks. /// This enum tracks which block we're currently in, and, for multi-line blocks, /// collects the partial data until the block is closed and it can be appended /// to the CrashReport. #[derive(Debug)] -enum StdinState { +pub(crate) enum StdinState { Config, Counters, Done, @@ -281,7 +160,7 @@ fn process_line( } #[derive(Debug)] -enum CrashReportStatus { +pub(crate) enum CrashReportStatus { NoCrash, CrashReport(CrashtrackerConfiguration, CrashInfo), PartialCrashReport(CrashtrackerConfiguration, CrashInfo, StdinState), @@ -294,7 +173,7 @@ enum CrashReportStatus { /// In the case where the parent failed to transfer a full crash-report /// (for instance if it crashed while calculating the crash-report), we return /// a PartialCrashReport. -async fn receive_report( +pub(crate) async fn receive_report_from_stream( timeout: Duration, stream: impl AsyncBufReadExt + std::marker::Unpin, ) -> anyhow::Result { @@ -373,92 +252,3 @@ async fn receive_report( )) } } - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::AsyncWriteExt; - use tokio::net::UnixStream; - - async fn to_socket( - target: &mut tokio::net::UnixStream, - msg: impl AsRef, - ) -> anyhow::Result { - let msg = msg.as_ref(); - let n = target.write(format!("{msg}\n").as_bytes()).await?; - target.flush().await?; - Ok(n) - } - - async fn send_report(delay: Duration, mut stream: UnixStream) -> anyhow::Result<()> { - let sender = &mut stream; - to_socket(sender, DD_CRASHTRACK_BEGIN_SIGINFO).await?; - to_socket( - sender, - serde_json::to_string(&SigInfo { - signame: Some("SIGSEGV".to_string()), - signum: 11, - faulting_address: None, - })?, - ) - .await?; - to_socket(sender, DD_CRASHTRACK_END_SIGINFO).await?; - - to_socket(sender, DD_CRASHTRACK_BEGIN_CONFIG).await?; - to_socket( - sender, - serde_json::to_string(&CrashtrackerConfiguration::new( - vec![], - false, - false, - None, - StacktraceCollection::Disabled, - 3000, - None, - )?)?, - ) - .await?; - to_socket(sender, DD_CRASHTRACK_END_CONFIG).await?; - tokio::time::sleep(delay).await; - to_socket(sender, DD_CRASHTRACK_DONE).await?; - Ok(()) - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_receive_report_short_timeout() -> anyhow::Result<()> { - let (sender, receiver) = tokio::net::UnixStream::pair()?; - - let join_handle1 = tokio::spawn(receive_report( - Duration::from_secs(1), - BufReader::new(receiver), - )); - let join_handle2 = tokio::spawn(send_report(Duration::from_secs(2), sender)); - - let crash_report = join_handle1.await??; - assert!(matches!( - crash_report, - CrashReportStatus::PartialCrashReport(_, _, _) - )); - let sender_error = join_handle2.await?.unwrap_err().to_string(); - assert_eq!(sender_error, "Broken pipe (os error 32)"); - Ok(()) - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn test_receive_report_long_timeout() -> anyhow::Result<()> { - let (sender, receiver) = tokio::net::UnixStream::pair()?; - - let join_handle1 = tokio::spawn(receive_report( - Duration::from_secs(2), - BufReader::new(receiver), - )); - let join_handle2 = tokio::spawn(send_report(Duration::from_secs(1), sender)); - - let crash_report = join_handle1.await??; - assert!(matches!(crash_report, CrashReportStatus::CrashReport(_, _))); - join_handle2.await??; - Ok(()) - } -} diff --git a/crashtracker/src/rfc5_crash_info/builder.rs b/crashtracker/src/rfc5_crash_info/builder.rs new file mode 100644 index 000000000..a6fec867f --- /dev/null +++ b/crashtracker/src/rfc5_crash_info/builder.rs @@ -0,0 +1,328 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use chrono::{DateTime, Utc}; +use error_data::ThreadData; +use stacktrace::StackTrace; +use std::io::{BufRead, BufReader}; +use unknown_value::UnknownValue; +use uuid::Uuid; + +use super::*; + +#[derive(Debug, Default)] +pub struct ErrorDataBuilder { + pub kind: Option, + pub message: Option, + pub stack: Option, + pub threads: Option>, +} + +impl ErrorDataBuilder { + pub fn build(self) -> anyhow::Result<(ErrorData, bool /* incomplete */)> { + let incomplete = self.stack.is_none(); + let is_crash = true; + let kind = self.kind.context("required field 'kind' missing")?; + let message = self.message; + let source_type = SourceType::Crashtracking; + let stack = self.stack.unwrap_or_else(StackTrace::missing); + let threads = self.threads.unwrap_or_default(); + Ok(( + ErrorData { + is_crash, + kind, + message, + source_type, + stack, + threads, + }, + incomplete, + )) + } + + pub fn new() -> Self { + Self::default() + } + + pub fn with_kind(&mut self, kind: ErrorKind) -> anyhow::Result<&mut Self> { + self.kind = Some(kind); + Ok(self) + } + + pub fn with_message(&mut self, message: String) -> anyhow::Result<&mut Self> { + self.message = Some(message); + Ok(self) + } + + pub fn with_stack(&mut self, stack: StackTrace) -> anyhow::Result<&mut Self> { + self.stack = Some(stack); + Ok(self) + } + + pub fn with_stack_frame( + &mut self, + frame: StackFrame, + incomplete: bool, + ) -> anyhow::Result<&mut Self> { + if let Some(stack) = &mut self.stack { + stack.push_frame(frame, incomplete)?; + } else { + self.stack = Some(StackTrace::from_frames(vec![frame], incomplete)); + } + Ok(self) + } + + pub fn with_stack_set_complete(&mut self) -> anyhow::Result<&mut Self> { + if let Some(stack) = &mut self.stack { + stack.set_complete()?; + } else { + anyhow::bail!("Can't set non-existant stack complete"); + } + Ok(self) + } + + pub fn with_threads(&mut self, threads: Vec) -> anyhow::Result<&mut Self> { + self.threads = Some(threads); + Ok(self) + } +} + +#[derive(Debug, Default)] +pub struct CrashInfoBuilder { + pub counters: Option>, + pub error: ErrorDataBuilder, + pub files: Option>>, + pub fingerprint: Option, + pub incomplete: Option, + pub log_messages: Option>, + pub metadata: Option, + pub os_info: Option, + pub proc_info: Option, + pub sig_info: Option, + pub span_ids: Option>, + pub timestamp: Option>, + pub trace_ids: Option>, + pub uuid: Option, +} + +impl CrashInfoBuilder { + pub fn build(self) -> anyhow::Result { + let counters = self.counters.unwrap_or_default(); + let data_schema_version = CrashInfo::current_schema_version().to_string(); + let (error, incomplete_error) = self.error.build()?; + let files = self.files.unwrap_or_default(); + let fingerprint = self.fingerprint; + let incomplete = incomplete_error; // TODO + let log_messages = self.log_messages.unwrap_or_default(); + let metadata = self.metadata.unwrap_or_else(Metadata::unknown_value); + let os_info = self.os_info.unwrap_or_else(OsInfo::unknown_value); + let proc_info = self.proc_info; + let sig_info = self.sig_info; + let span_ids = self.span_ids.unwrap_or_default(); + let timestamp = self.timestamp.unwrap_or_else(Utc::now).to_string(); + let trace_ids = self.trace_ids.unwrap_or_default(); + let uuid = self.uuid.unwrap_or_else(|| Uuid::new_v4().to_string()); + Ok(CrashInfo { + counters, + data_schema_version, + error, + files, + fingerprint, + incomplete, + log_messages, + metadata, + os_info, + proc_info, + sig_info, + span_ids, + timestamp, + trace_ids, + uuid, + }) + } + + pub fn new() -> Self { + Self::default() + } + + /// Inserts the given counter to the current set of counters in the builder. + pub fn with_counter(&mut self, name: String, value: i64) -> anyhow::Result<&mut Self> { + anyhow::ensure!(!name.is_empty(), "Empty counter name not allowed"); + if let Some(ref mut counters) = &mut self.counters { + counters.insert(name, value); + } else { + self.counters = Some(HashMap::from([(name, value)])); + } + Ok(self) + } + + pub fn with_counters(&mut self, counters: HashMap) -> anyhow::Result<&mut Self> { + self.counters = Some(counters); + Ok(self) + } + + pub fn with_kind(&mut self, kind: ErrorKind) -> anyhow::Result<&mut Self> { + self.error.with_kind(kind)?; + Ok(self) + } + + pub fn with_file(&mut self, filename: String) -> anyhow::Result<&mut Self> { + let file = File::open(&filename).with_context(|| format!("filename: {filename}"))?; + let lines: std::io::Result> = BufReader::new(file).lines().collect(); + self.with_file_and_contents(filename, lines?) + } + + /// Appends the given file to the current set of files in the builder. + pub fn with_file_and_contents( + &mut self, + filename: String, + contents: Vec, + ) -> anyhow::Result<&mut Self> { + if let Some(ref mut files) = &mut self.files { + files.insert(filename, contents); + } else { + self.files = Some(HashMap::from([(filename, contents)])); + } + Ok(self) + } + + /// Sets the current set of files in the builder. + pub fn with_files(&mut self, files: HashMap>) -> anyhow::Result<&mut Self> { + self.files = Some(files); + Ok(self) + } + + pub fn with_fingerprint(&mut self, fingerprint: String) -> anyhow::Result<&mut Self> { + anyhow::ensure!(!fingerprint.is_empty(), "Expect non-empty fingerprint"); + self.fingerprint = Some(fingerprint); + Ok(self) + } + + pub fn with_incomplete(&mut self, incomplete: bool) -> anyhow::Result<&mut Self> { + self.incomplete = Some(incomplete); + Ok(self) + } + + /// Appends the given message to the current set of messages in the builder. + pub fn with_log_message(&mut self, message: String) -> anyhow::Result<&mut Self> { + if let Some(ref mut messages) = &mut self.log_messages { + messages.push(message); + } else { + self.log_messages = Some(vec![message]); + } + Ok(self) + } + + pub fn with_log_messages(&mut self, log_messages: Vec) -> anyhow::Result<&mut Self> { + self.log_messages = Some(log_messages); + Ok(self) + } + + pub fn with_message(&mut self, message: String) -> anyhow::Result<&mut Self> { + self.error.with_message(message)?; + Ok(self) + } + + pub fn with_metadata(&mut self, metadata: Metadata) -> anyhow::Result<&mut Self> { + self.metadata = Some(metadata); + Ok(self) + } + + pub fn with_os_info(&mut self, os_info: OsInfo) -> anyhow::Result<&mut Self> { + self.os_info = Some(os_info); + Ok(self) + } + + pub fn with_os_info_this_machine(&mut self) -> anyhow::Result<&mut Self> { + self.with_os_info(::os_info::get().into()) + } + + pub fn with_proc_info(&mut self, proc_info: ProcInfo) -> anyhow::Result<&mut Self> { + self.proc_info = Some(proc_info); + Ok(self) + } + + pub fn with_sig_info(&mut self, sig_info: SigInfo) -> anyhow::Result<&mut Self> { + self.sig_info = Some(sig_info); + Ok(self) + } + + pub fn with_span_id(&mut self, span_id: Span) -> anyhow::Result<&mut Self> { + if let Some(ref mut span_ids) = &mut self.span_ids { + span_ids.push(span_id); + } else { + self.span_ids = Some(vec![span_id]); + } + Ok(self) + } + + pub fn with_span_ids(&mut self, span_ids: Vec) -> anyhow::Result<&mut Self> { + self.span_ids = Some(span_ids); + Ok(self) + } + + pub fn with_stack(&mut self, stack: StackTrace) -> anyhow::Result<&mut Self> { + self.error.with_stack(stack)?; + Ok(self) + } + + pub fn with_stack_frame( + &mut self, + frame: StackFrame, + incomplete: bool, + ) -> anyhow::Result<&mut Self> { + self.error.with_stack_frame(frame, incomplete)?; + Ok(self) + } + + pub fn with_stack_set_complete(&mut self) -> anyhow::Result<&mut Self> { + self.error.with_stack_set_complete()?; + Ok(self) + } + + pub fn with_thread(&mut self, thread: ThreadData) -> anyhow::Result<&mut Self> { + if let Some(ref mut threads) = &mut self.error.threads { + threads.push(thread); + } else { + self.error.threads = Some(vec![thread]); + } + Ok(self) + } + + pub fn with_threads(&mut self, threads: Vec) -> anyhow::Result<&mut Self> { + self.error.with_threads(threads)?; + Ok(self) + } + + pub fn with_timestamp(&mut self, timestamp: DateTime) -> anyhow::Result<&mut Self> { + self.timestamp = Some(timestamp); + Ok(self) + } + + pub fn with_timestamp_now(&mut self) -> anyhow::Result<&mut Self> { + self.with_timestamp(Utc::now()) + } + + pub fn with_trace_id(&mut self, trace_id: Span) -> anyhow::Result<&mut Self> { + if let Some(ref mut trace_ids) = &mut self.trace_ids { + trace_ids.push(trace_id); + } else { + self.trace_ids = Some(vec![trace_id]); + } + Ok(self) + } + + pub fn with_trace_ids(&mut self, trace_ids: Vec) -> anyhow::Result<&mut Self> { + self.trace_ids = Some(trace_ids); + Ok(self) + } + + pub fn with_uuid(&mut self, uuid: String) -> anyhow::Result<&mut Self> { + self.uuid = Some(uuid); + Ok(self) + } + + pub fn with_uuid_random(&mut self) -> anyhow::Result<&mut Self> { + self.with_uuid(Uuid::new_v4().to_string()) + } +} diff --git a/crashtracker/src/rfc5_crash_info/error_data.rs b/crashtracker/src/rfc5_crash_info/error_data.rs index 12671cc02..d6926d020 100644 --- a/crashtracker/src/rfc5_crash_info/error_data.rs +++ b/crashtracker/src/rfc5_crash_info/error_data.rs @@ -17,12 +17,41 @@ pub struct ErrorData { pub threads: Vec, } +#[cfg(unix)] +impl ErrorData { + pub fn normalize_ips(&mut self, pid: u32) -> anyhow::Result<()> { + let normalizer = blazesym::normalize::Normalizer::new(); + let pid = pid.into(); + // TODO, should we continue after error or just exit? + self.stack.normalize_ips(&normalizer, pid)?; + for thread in &mut self.threads { + thread.stack.normalize_ips(&normalizer, pid)?; + } + Ok(()) + } + + pub fn resolve_names(&mut self, pid: u32) -> anyhow::Result<()> { + let mut process = blazesym::symbolize::Process::new(pid.into()); + // https://github.com/libbpf/blazesym/issues/518 + process.map_files = false; + let src = blazesym::symbolize::Source::Process(process); + let symbolizer = blazesym::symbolize::Symbolizer::new(); + self.stack.resolve_names(&src, &symbolizer)?; + + for thread in &mut self.threads { + thread.stack.resolve_names(&src, &symbolizer)?; + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub enum SourceType { Crashtracking, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[repr(C)] pub enum ErrorKind { Panic, UnhandledException, diff --git a/crashtracker/src/rfc5_crash_info/metadata.rs b/crashtracker/src/rfc5_crash_info/metadata.rs index 28096fe59..343759dbc 100644 --- a/crashtracker/src/rfc5_crash_info/metadata.rs +++ b/crashtracker/src/rfc5_crash_info/metadata.rs @@ -3,6 +3,8 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use super::unknown_value::UnknownValue; + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct Metadata { pub library_name: String, @@ -13,6 +15,17 @@ pub struct Metadata { pub tags: Vec, } +impl UnknownValue for Metadata { + fn unknown_value() -> Self { + Self { + library_name: "unknown".to_string(), + library_version: "unknown".to_string(), + family: "unknown".to_string(), + tags: vec![], + } + } +} + impl From for Metadata { fn from(value: crate::crash_info::CrashtrackerMetadata) -> Self { let tags = value diff --git a/crashtracker/src/rfc5_crash_info/mod.rs b/crashtracker/src/rfc5_crash_info/mod.rs index d70a93f0f..68494d48a 100644 --- a/crashtracker/src/rfc5_crash_info/mod.rs +++ b/crashtracker/src/rfc5_crash_info/mod.rs @@ -1,6 +1,7 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +mod builder; mod error_data; mod metadata; mod os_info; @@ -10,18 +11,20 @@ mod spans; mod stacktrace; mod telemetry; mod test_utils; +mod unknown_value; -pub use error_data::{ErrorData, ErrorKind, SourceType}; +pub use builder::*; +use ddcommon::Endpoint; +pub use error_data::*; pub use metadata::Metadata; -pub use os_info::OsInfo; -pub use proc_info::ProcInfo; -pub use sig_info::{SiCodes, SigInfo, SignalNames}; -pub use spans::Span; -pub use stacktrace::{BuildIdType, FileType, StackFrame, StackTrace}; -pub use telemetry::TelemetryCrashUploader; +pub use os_info::*; +pub use proc_info::*; +pub use sig_info::*; +pub use spans::*; +pub use stacktrace::*; +pub use telemetry::*; use anyhow::Context; -use error_data::thread_data_from_additional_stacktraces; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, fs::File, path::Path}; @@ -41,9 +44,10 @@ pub struct CrashInfo { pub log_messages: Vec, pub metadata: Metadata, pub os_info: OsInfo, - pub proc_info: ProcInfo, #[serde(default, skip_serializing_if = "Option::is_none")] - pub sig_info: Option, + pub proc_info: Option, //TODO, update the schema + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sig_info: Option, //TODO, update the schema #[serde(default, skip_serializing_if = "Vec::is_empty")] pub span_ids: Vec, pub timestamp: String, @@ -52,10 +56,27 @@ pub struct CrashInfo { pub uuid: String, } +impl CrashInfo { + pub fn current_schema_version() -> String { + "1.0".to_string() + } +} + +#[cfg(unix)] +impl CrashInfo { + pub fn normalize_ips(&mut self, pid: u32) -> anyhow::Result<()> { + self.error.normalize_ips(pid) + } + + pub fn resolve_names(&mut self, pid: u32) -> anyhow::Result<()> { + self.error.resolve_names(pid) + } +} + impl From for CrashInfo { fn from(value: crate::crash_info::CrashInfo) -> Self { let counters = value.counters; - let data_schema_version = String::from("1.0"); + let data_schema_version = CrashInfo::current_schema_version(); let error = { let is_crash = true; let kind = ErrorKind::UnixSignal; @@ -78,7 +99,7 @@ impl From for CrashInfo { let log_messages = vec![]; let metadata = value.metadata.unwrap().into(); let os_info = value.os_info.into(); - let proc_info = value.proc_info.unwrap().into(); + let proc_info = value.proc_info.map(ProcInfo::from); let sig_info = value.siginfo.map(SigInfo::from); let span_ids = value .span_ids @@ -130,6 +151,36 @@ impl CrashInfo { .with_context(|| format!("Failed to write json to {}", path.display()))?; Ok(()) } + + pub fn upload_to_endpoint(&self, endpoint: &Option) -> anyhow::Result<()> { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async { self.async_upload_to_endpoint(endpoint).await }) + } + + pub async fn async_upload_to_endpoint( + &self, + endpoint: &Option, + ) -> anyhow::Result<()> { + // If we're debugging to a file, dump the actual crashinfo into a json + if let Some(endpoint) = endpoint { + if Some("file") == endpoint.url.scheme_str() { + let path = ddcommon::decode_uri_path_in_authority(&endpoint.url) + .context("crash output file path was not correctly formatted")?; + self.to_file(&path)?; + } + } + + self.upload_to_telemetry(endpoint).await + } + + async fn upload_to_telemetry(&self, endpoint: &Option) -> anyhow::Result<()> { + let uploader = TelemetryCrashUploader::new(&self.metadata, endpoint)?; + uploader.upload_to_telemetry(self).await?; + Ok(()) + } } #[cfg(test)] @@ -181,7 +232,7 @@ mod tests { log_messages: vec![], metadata: Metadata::test_instance(seed), os_info: ::os_info::Info::unknown().into(), - proc_info: ProcInfo::test_instance(seed), + proc_info: Some(ProcInfo::test_instance(seed)), sig_info: Some(SigInfo::test_instance(seed)), span_ids, timestamp: chrono::DateTime::from_timestamp(1568898000 /* Datadog IPO */, 0) diff --git a/crashtracker/src/rfc5_crash_info/os_info.rs b/crashtracker/src/rfc5_crash_info/os_info.rs index 56dbf5e22..ee9d16fa8 100644 --- a/crashtracker/src/rfc5_crash_info/os_info.rs +++ b/crashtracker/src/rfc5_crash_info/os_info.rs @@ -3,6 +3,8 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use super::unknown_value::UnknownValue; + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct OsInfo { pub architecture: String, @@ -11,6 +13,12 @@ pub struct OsInfo { pub version: String, } +impl UnknownValue for OsInfo { + fn unknown_value() -> Self { + os_info::Info::unknown().into() + } +} + impl From for OsInfo { fn from(value: os_info::Info) -> Self { let architecture = value.architecture().unwrap_or("unknown").to_string(); diff --git a/crashtracker/src/rfc5_crash_info/proc_info.rs b/crashtracker/src/rfc5_crash_info/proc_info.rs index 756a12ae1..0b95a497d 100644 --- a/crashtracker/src/rfc5_crash_info/proc_info.rs +++ b/crashtracker/src/rfc5_crash_info/proc_info.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ProcInfo { - pid: u32, + pub pid: u32, } impl From for ProcInfo { diff --git a/crashtracker/src/rfc5_crash_info/sig_info.rs b/crashtracker/src/rfc5_crash_info/sig_info.rs index e318d36d6..5b160a0d6 100644 --- a/crashtracker/src/rfc5_crash_info/sig_info.rs +++ b/crashtracker/src/rfc5_crash_info/sig_info.rs @@ -34,6 +34,7 @@ impl From for SigInfo { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[allow(clippy::upper_case_acronyms, non_camel_case_types)] +#[repr(C)] /// See https://man7.org/linux/man-pages/man7/signal.7.html pub enum SignalNames { SIGABRT, @@ -64,6 +65,7 @@ impl From for SignalNames { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[allow(clippy::upper_case_acronyms, non_camel_case_types)] +#[repr(C)] /// See https://man7.org/linux/man-pages/man2/sigaction.2.html pub enum SiCodes { BUS_ADRALN, diff --git a/crashtracker/src/rfc5_crash_info/stacktrace.rs b/crashtracker/src/rfc5_crash_info/stacktrace.rs index 02ef2e59c..09b9dcb98 100644 --- a/crashtracker/src/rfc5_crash_info/stacktrace.rs +++ b/crashtracker/src/rfc5_crash_info/stacktrace.rs @@ -1,15 +1,100 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use crate::NormalizedAddress; +#[cfg(unix)] +use blazesym::{ + helper::ElfResolver, + normalize::Normalizer, + symbolize::{Input, Source, Symbolized, Symbolizer, TranslateFileOffset}, + Pid, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::NormalizedAddress; - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct StackTrace { - format: String, - frames: Vec, + pub format: String, + pub frames: Vec, + pub incomplete: bool, +} + +const FORMAT_STRING: &str = "Datadog Crashtracker 1.0"; + +impl StackTrace { + pub fn empty() -> Self { + Self { + format: FORMAT_STRING.to_string(), + frames: vec![], + incomplete: false, + } + } + + pub fn from_frames(frames: Vec, incomplete: bool) -> Self { + Self { + format: FORMAT_STRING.to_string(), + frames, + incomplete, + } + } + + pub fn new_incomplete() -> Self { + Self { + format: FORMAT_STRING.to_string(), + frames: vec![], + incomplete: true, + } + } + + pub fn missing() -> Self { + Self { + format: FORMAT_STRING.to_string(), + frames: vec![], + incomplete: true, + } + } +} + +impl StackTrace { + pub fn set_complete(&mut self) -> anyhow::Result<()> { + self.incomplete = false; + Ok(()) + } + + pub fn push_frame(&mut self, frame: StackFrame, incomplete: bool) -> anyhow::Result<()> { + anyhow::ensure!( + self.incomplete, + "Can't push a new frame onto a complete stack" + ); + self.frames.push(frame); + self.incomplete = incomplete; + Ok(()) + } +} + +#[cfg(unix)] +impl StackTrace { + pub fn normalize_ips(&mut self, normalizer: &Normalizer, pid: Pid) -> anyhow::Result<()> { + for frame in &mut self.frames { + // TODO: Should this keep going on failure, and report at the end? + frame.normalize_ip(normalizer, pid)?; + } + Ok(()) + } + + pub fn resolve_names(&mut self, src: &Source, symbolizer: &Symbolizer) -> anyhow::Result<()> { + for frame in &mut self.frames { + // TODO: Should this keep going on failure, and report at the end? + frame.resolve_names(src, symbolizer)?; + } + Ok(()) + } +} + +impl Default for StackTrace { + fn default() -> Self { + Self::missing() + } } impl From> for StackTrace { @@ -63,7 +148,6 @@ impl From> for StackTrace { } } - let format = String::from("Datadog Crashtracker 1.0"); // Todo: this will under-estimate the cap needed if there are inlined functions. // Maybe not worth fixing this. let mut frames = Vec::with_capacity(value.len()); @@ -105,12 +189,11 @@ impl From> for StackTrace { } } } - - Self { format, frames } + Self::from_frames(frames, false) } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, Default)] pub struct StackFrame { // Absolute addresses #[serde(default, skip_serializing_if = "Option::is_none")] @@ -145,8 +228,64 @@ pub struct StackFrame { pub line: Option, } +impl StackFrame { + pub fn new() -> Self { + Self::default() + } +} + +#[cfg(unix)] +impl StackFrame { + pub fn normalize_ip(&mut self, normalizer: &Normalizer, pid: Pid) -> anyhow::Result<()> { + use anyhow::Context; + if let Some(ip) = &self.ip { + let ip = ip.trim_start_matches("0x"); + let ip = u64::from_str_radix(ip, 16)?; + let normed = normalizer.normalize_user_addrs(pid, &[ip])?; + anyhow::ensure!(normed.outputs.len() == 1); + let (file_offset, meta_idx) = normed.outputs[0]; + let meta = &normed.meta[meta_idx]; + let elf = meta.as_elf().context("Not elf")?; + let resolver = ElfResolver::open(&elf.path)?; + let virt_address = resolver + .file_offset_to_virt_offset(file_offset)? + .context("No matching segment found")?; + + self.build_id = elf.build_id.as_ref().map(|x| byte_slice_as_hex(x.as_ref())); + self.build_id_type = Some(BuildIdType::GNU); + self.file_type = Some(FileType::ELF); + self.path = Some(elf.path.to_string_lossy().to_string()); + self.relative_address = Some(format!("{virt_address:#018x}")); + } + Ok(()) + } + + pub fn resolve_names(&mut self, src: &Source, symbolizer: &Symbolizer) -> anyhow::Result<()> { + if let Some(ip) = &self.ip { + let ip = ip.trim_start_matches("0x"); + let ip = u64::from_str_radix(ip, 16)?; + let input = Input::AbsAddr(ip); + match symbolizer.symbolize_single(src, input)? { + Symbolized::Sym(s) => { + if let Some(c) = s.code_info { + self.column = c.column.map(u32::from); + self.file = Some(c.to_path().display().to_string()); + self.line = c.line; + } + self.function = Some(s.name.into_owned()); + } + Symbolized::Unknown(reason) => { + anyhow::bail!("Couldn't symbolize {ip}: {reason}"); + } + } + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[allow(clippy::upper_case_acronyms)] +#[repr(C)] pub enum BuildIdType { GNU, GO, @@ -157,6 +296,7 @@ pub enum BuildIdType { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[allow(clippy::upper_case_acronyms)] +#[repr(C)] pub enum FileType { APK, ELF, @@ -177,14 +317,22 @@ fn byte_vec_as_hex(bv: Option>) -> Option { } } +#[cfg(unix)] +fn byte_slice_as_hex(bv: &[u8]) -> String { + use std::fmt::Write; + + let mut s = String::new(); + for byte in bv { + let _ = write!(&mut s, "{byte:X}"); + } + s +} + #[cfg(test)] impl super::test_utils::TestInstance for StackTrace { fn test_instance(_seed: u64) -> Self { let frames = (0..10).map(StackFrame::test_instance).collect(); - Self { - format: "Datadog Crashtracker 1.0".to_string(), - frames, - } + Self::from_frames(frames, false) } } diff --git a/crashtracker/src/rfc5_crash_info/telemetry.rs b/crashtracker/src/rfc5_crash_info/telemetry.rs index 35c0d6244..8d6376d6e 100644 --- a/crashtracker/src/rfc5_crash_info/telemetry.rs +++ b/crashtracker/src/rfc5_crash_info/telemetry.rs @@ -136,6 +136,7 @@ impl TelemetryCrashUploader { is_sensitive: true, count: 1, }]), + origin: Some("Crashtracker"), }; let client = ddtelemetry::worker::http_client::from_config(&self.cfg); let req = request_builder(&self.cfg)? @@ -144,6 +145,14 @@ impl TelemetryCrashUploader { http::header::CONTENT_TYPE, ddcommon::header::APPLICATION_JSON, ) + .header( + ddtelemetry::worker::http_client::header::API_VERSION, + ddtelemetry::data::ApiVersion::V2.to_str(), + ) + .header( + ddtelemetry::worker::http_client::header::REQUEST_TYPE, + "logs", + ) .body(serde_json::to_string(&payload)?.into())?; tokio::time::timeout( diff --git a/crashtracker/src/rfc5_crash_info/unknown_value.rs b/crashtracker/src/rfc5_crash_info/unknown_value.rs new file mode 100644 index 000000000..ba2e42e82 --- /dev/null +++ b/crashtracker/src/rfc5_crash_info/unknown_value.rs @@ -0,0 +1,6 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub trait UnknownValue { + fn unknown_value() -> Self; +} diff --git a/crashtracker/src/shared/configuration.rs b/crashtracker/src/shared/configuration.rs index 4d7cddf88..8decbe594 100644 --- a/crashtracker/src/shared/configuration.rs +++ b/crashtracker/src/shared/configuration.rs @@ -31,7 +31,7 @@ pub struct CrashtrackerConfiguration { pub unix_socket_path: Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct CrashtrackerReceiverConfig { pub args: Vec, pub env: Vec<(String, String)>, diff --git a/data-pipeline-ffi/Cargo.toml b/data-pipeline-ffi/Cargo.toml index 8d5f4b6a0..1b90e71e4 100644 --- a/data-pipeline-ffi/Cargo.toml +++ b/data-pipeline-ffi/Cargo.toml @@ -21,8 +21,12 @@ cbindgen = ["build_common/cbindgen", "ddcommon-ffi/cbindgen"] [build-dependencies] build_common = { path = "../build-common" } +[dev-dependencies] +httpmock = "0.7.0" +rmp-serde = "1.1.1" +datadog-trace-utils = { path = "../trace-utils" } + [dependencies] data-pipeline = { path = "../data-pipeline" } ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false } -bytes = "1.4" -libc = "0.2.153" +tinybytes = { path = "../tinybytes" } diff --git a/data-pipeline-ffi/cbindgen.toml b/data-pipeline-ffi/cbindgen.toml index 5ccacbd2b..3135fbc30 100644 --- a/data-pipeline-ffi/cbindgen.toml +++ b/data-pipeline-ffi/cbindgen.toml @@ -20,6 +20,9 @@ renaming_overrides_prefixing = true "Slice_U8" = "ddog_Slice_U8" "Slice_CChar" = "ddog_Slice_CChar" "Error" = "ddog_Error" +"AgentResponse" = "ddog_AgentResponse" +"ExporterErrorCode" = "ddog_TraceExporterErrorCode" +"ExporterError" = "ddog_TraceExporterError" [export.mangle] rename_types = "PascalCase" diff --git a/data-pipeline-ffi/src/error.rs b/data-pipeline-ffi/src/error.rs new file mode 100644 index 000000000..4c3524bf0 --- /dev/null +++ b/data-pipeline-ffi/src/error.rs @@ -0,0 +1,176 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use data_pipeline::trace_exporter::error::{ + AgentErrorKind, BuilderErrorKind, NetworkErrorKind, TraceExporterError, +}; +use std::ffi::{c_char, CString}; +use std::fmt::Display; +use std::io::ErrorKind as IoErrorKind; + +/// Represent error codes that `Error` struct can hold +#[repr(C)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum ExporterErrorCode { + AddressInUse, + ConnectionAborted, + ConnectionRefused, + ConnectionReset, + HttpBodyFormat, + HttpBodyTooLong, + HttpClient, + HttpEmptyBody, + HttpParse, + HttpServer, + HttpUnknown, + HttpWrongStatus, + InvalidArgument, + InvalidData, + InvalidInput, + InvalidUrl, + IoError, + NetworkUnknown, + Serde, + TimedOut, +} + +impl Display for ExporterErrorCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::AddressInUse => write!(f, "Address already in use"), + Self::ConnectionAborted => write!(f, "Connection aborted"), + Self::ConnectionRefused => write!(f, "Connection refused"), + Self::ConnectionReset => write!(f, "Connection reset by peer"), + Self::HttpBodyFormat => write!(f, "Error parsing HTTP body"), + Self::HttpBodyTooLong => write!(f, "HTTP body too long"), + Self::HttpClient => write!(f, "HTTP error orgininated by client"), + Self::HttpEmptyBody => write!(f, "HTTP empty body"), + Self::HttpParse => write!(f, "Error while parsing HTTP message"), + Self::HttpServer => write!(f, "HTTP error orgininated by server"), + Self::HttpWrongStatus => write!(f, "HTTP wrong status number"), + Self::HttpUnknown => write!(f, "HTTP unknown error"), + Self::InvalidArgument => write!(f, "Invalid argument provided"), + Self::InvalidData => write!(f, "Invalid data payload"), + Self::InvalidInput => write!(f, "Invalid input"), + Self::InvalidUrl => write!(f, "Invalid URL"), + Self::IoError => write!(f, "Input/Output error"), + Self::NetworkUnknown => write!(f, "Unknown network error"), + Self::Serde => write!(f, "Serialization/Deserialization error"), + Self::TimedOut => write!(f, "Operation timed out"), + } + } +} + +/// Stucture that contains error information that `TraceExporter` API can return. +#[repr(C)] +#[derive(Debug, PartialEq)] +pub struct ExporterError { + pub code: ExporterErrorCode, + pub msg: *mut c_char, +} + +impl ExporterError { + pub fn new(code: ExporterErrorCode, msg: &str) -> Self { + Self { + code, + msg: CString::new(msg).unwrap_or_default().into_raw(), + } + } +} + +impl From for ExporterError { + fn from(value: TraceExporterError) -> Self { + let code = match &value { + TraceExporterError::Agent(e) => match e { + AgentErrorKind::EmptyResponse => ExporterErrorCode::HttpEmptyBody, + }, + TraceExporterError::Builder(e) => match e { + BuilderErrorKind::InvalidUri => ExporterErrorCode::InvalidUrl, + }, + TraceExporterError::Deserialization(_) => ExporterErrorCode::Serde, + TraceExporterError::Io(e) => match e.kind() { + IoErrorKind::InvalidData => ExporterErrorCode::InvalidData, + IoErrorKind::InvalidInput => ExporterErrorCode::InvalidInput, + IoErrorKind::ConnectionReset => ExporterErrorCode::ConnectionReset, + IoErrorKind::ConnectionAborted => ExporterErrorCode::ConnectionAborted, + IoErrorKind::ConnectionRefused => ExporterErrorCode::ConnectionRefused, + IoErrorKind::TimedOut => ExporterErrorCode::TimedOut, + IoErrorKind::AddrInUse => ExporterErrorCode::AddressInUse, + _ => ExporterErrorCode::IoError, + }, + TraceExporterError::Network(e) => match e.kind() { + NetworkErrorKind::Body => ExporterErrorCode::HttpBodyFormat, + NetworkErrorKind::Canceled => ExporterErrorCode::ConnectionAborted, + NetworkErrorKind::ConnectionClosed => ExporterErrorCode::ConnectionReset, + NetworkErrorKind::MessageTooLarge => ExporterErrorCode::HttpBodyTooLong, + NetworkErrorKind::Parse => ExporterErrorCode::HttpParse, + NetworkErrorKind::TimedOut => ExporterErrorCode::TimedOut, + NetworkErrorKind::Unknown => ExporterErrorCode::NetworkUnknown, + NetworkErrorKind::WrongStatus => ExporterErrorCode::HttpWrongStatus, + }, + TraceExporterError::Request(e) => { + let status: u16 = e.status().into(); + if (400..499).contains(&status) { + ExporterErrorCode::HttpClient + } else if status >= 500 { + ExporterErrorCode::HttpServer + } else { + ExporterErrorCode::HttpUnknown + } + } + TraceExporterError::Serde(_) => ExporterErrorCode::Serde, + }; + ExporterError::new(code, &value.to_string()) + } +} + +impl Drop for ExporterError { + fn drop(&mut self) { + if !self.msg.is_null() { + // SAFETY: `the caller must ensure that `ExporterError` has been created through its + // `new` method which ensures that `msg` property is originated from + // `Cstring::into_raw` call. Any other posibility could lead to UB. + unsafe { + drop(CString::from_raw(self.msg)); + self.msg = std::ptr::null_mut(); + } + } + } +} + +/// Frees `error` and all its contents. After being called error will not point to a valid memory +/// address so any further actions on it could lead to undefined behavior. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_error_free(error: Option>) { + if let Some(error) = error { + drop(error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn constructor_test() { + let code = ExporterErrorCode::InvalidArgument; + let error = Box::new(ExporterError::new(code, &code.to_string())); + + assert_eq!(error.code, ExporterErrorCode::InvalidArgument); + let msg = unsafe { CStr::from_ptr(error.msg).to_string_lossy() }; + assert_eq!(msg, ExporterErrorCode::InvalidArgument.to_string()); + } + + #[test] + fn destructor_test() { + let code = ExporterErrorCode::InvalidArgument; + let error = Box::new(ExporterError::new(code, &code.to_string())); + + assert_eq!(error.code, ExporterErrorCode::InvalidArgument); + let msg = unsafe { CStr::from_ptr(error.msg).to_string_lossy() }; + assert_eq!(msg, ExporterErrorCode::InvalidArgument.to_string()); + + unsafe { ddog_trace_exporter_error_free(Some(error)) }; + } +} diff --git a/data-pipeline-ffi/src/lib.rs b/data-pipeline-ffi/src/lib.rs index 54d99711e..f327fca85 100644 --- a/data-pipeline-ffi/src/lib.rs +++ b/data-pipeline-ffi/src/lib.rs @@ -1,4 +1,5 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +mod error; mod trace_exporter; diff --git a/data-pipeline-ffi/src/trace_exporter.rs b/data-pipeline-ffi/src/trace_exporter.rs index b3cdbe365..1a8497bd2 100644 --- a/data-pipeline-ffi/src/trace_exporter.rs +++ b/data-pipeline-ffi/src/trace_exporter.rs @@ -1,88 +1,265 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use crate::error::{ExporterError, ExporterErrorCode as ErrorCode}; +use data_pipeline::trace_exporter::agent_response::AgentResponse; use data_pipeline::trace_exporter::{ - ResponseCallback, TraceExporter, TraceExporterInputFormat, TraceExporterOutputFormat, + TraceExporter, TraceExporterInputFormat, TraceExporterOutputFormat, }; use ddcommon_ffi::{ - slice::{AsBytes, ByteSlice}, - CharSlice, MaybeError, + CharSlice, + {slice::AsBytes, slice::ByteSlice}, }; -use std::{ffi::c_char, ptr::NonNull, time::Duration}; +use std::{ptr::NonNull, time::Duration}; + +macro_rules! gen_error { + ($l:expr) => { + Some(Box::new(ExporterError::new($l, &$l.to_string()))) + }; +} + +#[inline] +fn sanitize_string(str: CharSlice) -> Result> { + match str.try_to_utf8() { + Ok(s) => Ok(s.to_string()), + Err(_) => Err(Box::new(ExporterError::new( + ErrorCode::InvalidInput, + &ErrorCode::InvalidInput.to_string(), + ))), + } +} + +/// The TraceExporterConfig object will hold the configuration properties for the TraceExporter. +/// Once the configuration is passed to the TraceExporter constructor the config is no longer +/// needed by the handle and it can be freed. +#[derive(Debug, Default, PartialEq)] +pub struct TraceExporterConfig { + url: Option, + tracer_version: Option, + language: Option, + language_version: Option, + language_interpreter: Option, + hostname: Option, + env: Option, + version: Option, + service: Option, + input_format: TraceExporterInputFormat, + output_format: TraceExporterOutputFormat, + compute_stats: bool, +} -/// Create a new TraceExporter instance. -/// -/// # Arguments -/// -/// * `out_handle` - The handle to write the TraceExporter instance in. -/// * `url` - The URL of the Datadog Agent to communicate with. -/// * `tracer_version` - The version of the client library. -/// * `language` - The language of the client library. -/// * `language_version` - The version of the language of the client library. -/// * `language_interpreter` - The interpreter of the language of the client library. -/// * `hostname` - The hostname of the application, used for stats aggregation -/// * `env` - The environment of the application, used for stats aggregation -/// * `version` - The version of the application, used for stats aggregation -/// * `service` - The service name of the application, used for stats aggregation -/// * `input_format` - The input format of the traces. Setting this to Proxy will send the trace -/// data to the Datadog Agent as is. -/// * `output_format` - The output format of the traces to send to the Datadog Agent. If using the -/// Proxy input format, this should be set to format if the trace data that will be passed through -/// as is. -/// * `agent_response_callback` - The callback into the client library that the TraceExporter uses -/// for updated Agent JSON responses. #[no_mangle] -pub unsafe extern "C" fn ddog_trace_exporter_new( - out_handle: NonNull>, +pub unsafe extern "C" fn ddog_trace_exporter_config_new( + out_handle: NonNull>, +) { + out_handle + .as_ptr() + .write(Box::::default()); +} + +/// Frees TraceExporterConfig handle internal resources. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_free(handle: Box) { + drop(handle); +} + +/// Sets traces destination. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_url( + config: Option<&mut TraceExporterConfig>, url: CharSlice, - tracer_version: CharSlice, - language: CharSlice, - language_version: CharSlice, - language_interpreter: CharSlice, +) -> Option> { + if let Some(handle) = config { + handle.url = match sanitize_string(url) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets tracer's version to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_tracer_version( + config: Option<&mut TraceExporterConfig>, + version: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.tracer_version = match sanitize_string(version) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets tracer's language to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_language( + config: Option<&mut TraceExporterConfig>, + lang: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.language = match sanitize_string(lang) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets tracer's language version to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_lang_version( + config: Option<&mut TraceExporterConfig>, + version: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.language_version = match sanitize_string(version) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets tracer's language interpreter to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_lang_interpreter( + config: Option<&mut TraceExporterConfig>, + interpreter: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.language_interpreter = match sanitize_string(interpreter) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets hostname information to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_hostname( + config: Option<&mut TraceExporterConfig>, hostname: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.hostname = match sanitize_string(hostname) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +/// Sets environmet information to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_env( + config: Option<&mut TraceExporterConfig>, env: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.env = match sanitize_string(env) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_version( + config: Option<&mut TraceExporterConfig>, version: CharSlice, - service: CharSlice, - input_format: TraceExporterInputFormat, - output_format: TraceExporterOutputFormat, - compute_stats: bool, - agent_response_callback: extern "C" fn(*const c_char), -) -> MaybeError { - let callback_wrapper = ResponseCallbackWrapper { - response_callback: agent_response_callback, - }; - // TODO - handle errors - https://datadoghq.atlassian.net/browse/APMSP-1095 - let mut builder = TraceExporter::builder() - .set_url(url.to_utf8_lossy().as_ref()) - .set_tracer_version(tracer_version.to_utf8_lossy().as_ref()) - .set_language(language.to_utf8_lossy().as_ref()) - .set_language_version(language_version.to_utf8_lossy().as_ref()) - .set_language_interpreter(language_interpreter.to_utf8_lossy().as_ref()) - .set_hostname(hostname.to_utf8_lossy().as_ref()) - .set_env(env.to_utf8_lossy().as_ref()) - .set_app_version(version.to_utf8_lossy().as_ref()) - .set_service(service.to_utf8_lossy().as_ref()) - .set_input_format(input_format) - .set_output_format(output_format) - .set_response_callback(Box::new(callback_wrapper)); - if compute_stats { - builder = builder.enable_stats(Duration::from_secs(10)) - // TODO: APMSP-1317 Enable peer tags aggregation and stats by span_kind based on agent - // configuration +) -> Option> { + if let Option::Some(handle) = config { + handle.version = match sanitize_string(version) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) } - let exporter = builder.build().unwrap(); - out_handle.as_ptr().write(Box::new(exporter)); - MaybeError::None } -struct ResponseCallbackWrapper { - response_callback: extern "C" fn(*const c_char), +/// Sets service name to be included in the headers request. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_service( + config: Option<&mut TraceExporterConfig>, + service: CharSlice, +) -> Option> { + if let Option::Some(handle) = config { + handle.service = match sanitize_string(service) { + Ok(s) => Some(s), + Err(e) => return Some(e), + }; + None + } else { + gen_error!(ErrorCode::InvalidArgument) + } } -impl ResponseCallback for ResponseCallbackWrapper { - fn call(&self, response: &str) { - let c_response = std::ffi::CString::new(response).unwrap(); - (self.response_callback)(c_response.as_ptr()); +/// Create a new TraceExporter instance. +/// +/// # Arguments +/// +/// * `out_handle` - The handle to write the TraceExporter instance in. +/// * `config` - The configuration used to set up the TraceExporter handle. +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_new( + out_handle: NonNull>, + config: Option<&TraceExporterConfig>, +) -> Option> { + if let Some(config) = config { + // let config = &*ptr; + let mut builder = TraceExporter::builder() + .set_url(config.url.as_ref().unwrap_or(&"".to_string())) + .set_tracer_version(config.tracer_version.as_ref().unwrap_or(&"".to_string())) + .set_language(config.language.as_ref().unwrap_or(&"".to_string())) + .set_language_version(config.language_version.as_ref().unwrap_or(&"".to_string())) + .set_language_interpreter( + config + .language_interpreter + .as_ref() + .unwrap_or(&"".to_string()), + ) + .set_hostname(config.hostname.as_ref().unwrap_or(&"".to_string())) + .set_env(config.env.as_ref().unwrap_or(&"".to_string())) + .set_app_version(config.version.as_ref().unwrap_or(&"".to_string())) + .set_service(config.service.as_ref().unwrap_or(&"".to_string())) + .set_input_format(config.input_format) + .set_output_format(config.output_format); + if config.compute_stats { + builder = builder.enable_stats(Duration::from_secs(10)) + // TODO: APMSP-1317 Enable peer tags aggregation and stats by span_kind based on agent + // configuration + } + + match builder.build() { + Ok(exporter) => { + out_handle.as_ptr().write(Box::new(exporter)); + None + } + Err(err) => Some(Box::new(ExporterError::from(err))), + } + } else { + gen_error!(ErrorCode::InvalidArgument) } } @@ -102,17 +279,453 @@ pub unsafe extern "C" fn ddog_trace_exporter_free(handle: Box) { /// /// * `handle` - The handle to the TraceExporter instance. /// * `trace` - The traces to send to the Datadog Agent in the input format used to create the -/// TraceExporter. +/// TraceExporter. The memory for the trace must be valid for the life of the call to this +/// function. /// * `trace_count` - The number of traces to send to the Datadog Agent. +/// * `response` - Optional parameter that will ontain the agent response information. #[no_mangle] pub unsafe extern "C" fn ddog_trace_exporter_send( - handle: &TraceExporter, + handle: Option<&TraceExporter>, trace: ByteSlice, trace_count: usize, -) -> MaybeError { - // TODO - handle errors - https://datadoghq.atlassian.net/browse/APMSP-1095 - handle - .send(trace.as_bytes(), trace_count) - .unwrap_or(String::from("")); - MaybeError::None + response: Option<&mut AgentResponse>, +) -> Option> { + let exporter = match handle { + Some(exp) => exp, + None => return gen_error!(ErrorCode::InvalidArgument), + }; + + // necessary that the trace be static for the life of the FFI function call as the caller + // currently owns the memory. + //APMSP-1621 - Properly fix this sharp-edge by allocating memory on the Rust side + let static_trace: ByteSlice<'static> = std::mem::transmute(trace); + match exporter.send( + tinybytes::Bytes::from_static(static_trace.as_slice()), + trace_count, + ) { + Ok(resp) => { + if let Some(result) = response { + *result = resp; + } + None + } + Err(e) => Some(Box::new(ExporterError::from(e))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::ddog_trace_exporter_error_free; + use crate::trace_exporter::AgentResponse; + use datadog_trace_utils::span_v04::Span; + use httpmock::prelude::*; + use httpmock::MockServer; + use std::{borrow::Borrow, mem::MaybeUninit}; + + #[test] + fn config_constructor_test() { + unsafe { + let mut config: MaybeUninit> = MaybeUninit::uninit(); + + ddog_trace_exporter_config_new(NonNull::new_unchecked(&mut config).cast()); + + let cfg = config.assume_init(); + assert_eq!(cfg.url, None); + assert_eq!(cfg.tracer_version, None); + assert_eq!(cfg.language, None); + assert_eq!(cfg.language_version, None); + assert_eq!(cfg.language_interpreter, None); + assert_eq!(cfg.env, None); + assert_eq!(cfg.hostname, None); + assert_eq!(cfg.version, None); + assert_eq!(cfg.service, None); + assert_eq!(cfg.input_format, TraceExporterInputFormat::V04); + assert_eq!(cfg.output_format, TraceExporterOutputFormat::V04); + assert!(!cfg.compute_stats); + + ddog_trace_exporter_config_free(cfg); + } + } + + #[test] + fn config_url_test() { + unsafe { + let error = + ddog_trace_exporter_config_set_url(None, CharSlice::from("http://localhost")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_url( + config.as_mut(), + CharSlice::from("http://localhost"), + ); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.url.as_ref().unwrap(), "http://localhost"); + } + } + + #[test] + fn config_tracer_version() { + unsafe { + let error = ddog_trace_exporter_config_set_tracer_version(None, CharSlice::from("1.0")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_tracer_version( + config.as_mut(), + CharSlice::from("1.0"), + ); + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.tracer_version.as_ref().unwrap(), "1.0"); + } + } + + #[test] + fn config_language() { + unsafe { + let error = ddog_trace_exporter_config_set_language(None, CharSlice::from("lang")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = + ddog_trace_exporter_config_set_language(config.as_mut(), CharSlice::from("lang")); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.language.as_ref().unwrap(), "lang"); + } + } + + #[test] + fn config_lang_version() { + unsafe { + let error = ddog_trace_exporter_config_set_lang_version(None, CharSlice::from("0.1")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_lang_version( + config.as_mut(), + CharSlice::from("0.1"), + ); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.language_version.as_ref().unwrap(), "0.1"); + } + } + + #[test] + fn config_lang_interpreter_test() { + unsafe { + let error = + ddog_trace_exporter_config_set_lang_interpreter(None, CharSlice::from("foo")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_lang_interpreter( + config.as_mut(), + CharSlice::from("foo"), + ); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.language_interpreter.as_ref().unwrap(), "foo"); + } + } + + #[test] + fn config_hostname_test() { + unsafe { + let error = ddog_trace_exporter_config_set_hostname(None, CharSlice::from("hostname")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_hostname( + config.as_mut(), + CharSlice::from("hostname"), + ); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.hostname.as_ref().unwrap(), "hostname"); + } + } + + #[test] + fn config_env_test() { + unsafe { + let error = ddog_trace_exporter_config_set_env(None, CharSlice::from("env-test")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = + ddog_trace_exporter_config_set_env(config.as_mut(), CharSlice::from("env-test")); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.env.as_ref().unwrap(), "env-test"); + } + } + + #[test] + fn config_version_test() { + unsafe { + let error = ddog_trace_exporter_config_set_version(None, CharSlice::from("1.2")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = + ddog_trace_exporter_config_set_version(config.as_mut(), CharSlice::from("1.2")); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.version.as_ref().unwrap(), "1.2"); + } + } + + #[test] + fn config_service_test() { + unsafe { + let error = ddog_trace_exporter_config_set_service(None, CharSlice::from("service")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + + ddog_trace_exporter_error_free(error); + + let mut config = Some(TraceExporterConfig::default()); + let error = + ddog_trace_exporter_config_set_service(config.as_mut(), CharSlice::from("service")); + + assert_eq!(error, None); + + let cfg = config.unwrap(); + assert_eq!(cfg.service.as_ref().unwrap(), "service"); + } + } + + #[test] + fn expoter_constructor_test() { + unsafe { + let mut config: MaybeUninit> = MaybeUninit::uninit(); + ddog_trace_exporter_config_new(NonNull::new_unchecked(&mut config).cast()); + + let mut cfg = config.assume_init(); + let error = ddog_trace_exporter_config_set_url( + Some(cfg.as_mut()), + CharSlice::from("http://localhost"), + ); + assert_eq!(error, None); + + let mut ptr: MaybeUninit> = MaybeUninit::uninit(); + + let ret = ddog_trace_exporter_new( + NonNull::new_unchecked(&mut ptr).cast(), + Some(cfg.borrow()), + ); + let exporter = ptr.assume_init(); + + assert_eq!(ret, None); + + ddog_trace_exporter_free(exporter); + ddog_trace_exporter_config_free(cfg); + } + } + + #[test] + fn expoter_constructor_error_test() { + unsafe { + let mut config: MaybeUninit> = MaybeUninit::uninit(); + ddog_trace_exporter_config_new(NonNull::new_unchecked(&mut config).cast()); + + let mut cfg = config.assume_init(); + let error = ddog_trace_exporter_config_set_service( + Some(cfg.as_mut()), + CharSlice::from("service"), + ); + assert_eq!(error, None); + + ddog_trace_exporter_error_free(error); + + let mut ptr: MaybeUninit> = MaybeUninit::uninit(); + + let ret = ddog_trace_exporter_new(NonNull::new_unchecked(&mut ptr).cast(), Some(&cfg)); + + let error = ret.as_ref().unwrap(); + assert_eq!(error.code, ErrorCode::InvalidUrl); + + ddog_trace_exporter_error_free(ret); + + ddog_trace_exporter_config_free(cfg); + } + } + + #[test] + fn exporter_send_test_arguments_test() { + unsafe { + let trace = ByteSlice::from(b"dummy contents" as &[u8]); + let mut resp = AgentResponse { rate: 0.0 }; + let ret = ddog_trace_exporter_send(None, trace, 0, Some(&mut resp)); + + assert!(ret.is_some()); + assert_eq!(ret.unwrap().code, ErrorCode::InvalidArgument); + } + } + + #[test] + #[cfg(not(any(target_arch = "arm", target_arch = "aarch64")))] + // TODO(APMSP-1632): investigate why test fails on ARM platforms due to a CharSlice constructor. + fn config_invalid_input_test() { + unsafe { + let mut config = Some(TraceExporterConfig::default()); + let invalid: [i8; 2] = [0x80u8 as i8, 0xFFu8 as i8]; + let error = + ddog_trace_exporter_config_set_service(config.as_mut(), CharSlice::new(&invalid)); + + assert_eq!(error.unwrap().code, ErrorCode::InvalidInput); + } + } + + #[test] + // Ignore because it seems, at least in the version we're currently using, miri can't emulate + // libc::socket function. + #[cfg_attr(miri, ignore)] + fn exporter_send_check_rate_test() { + unsafe { + let server = MockServer::start(); + + let _mock = server.mock(|when, then| { + when.method(POST) + .header("Content-type", "application/msgpack") + .path("/v0.4/traces"); + then.status(200).body( + r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:,env:": 0.8 + } + }"#, + ); + }); + + let cfg = TraceExporterConfig { + url: Some(server.url("/")), + tracer_version: Some("0.1".to_string()), + language: Some("lang".to_string()), + language_version: Some("0.1".to_string()), + language_interpreter: Some("interpreter".to_string()), + hostname: Some("hostname".to_string()), + env: Some("env-test".to_string()), + version: Some("1.0".to_string()), + service: Some("test-service".to_string()), + input_format: TraceExporterInputFormat::V04, + output_format: TraceExporterOutputFormat::V04, + compute_stats: false, + }; + + let mut ptr: MaybeUninit> = MaybeUninit::uninit(); + let mut ret = + ddog_trace_exporter_new(NonNull::new_unchecked(&mut ptr).cast(), Some(&cfg)); + + let exporter = ptr.assume_init(); + + assert_eq!(ret, None); + + let data = rmp_serde::to_vec_named::>>(&vec![vec![]]).unwrap(); + let traces = ByteSlice::new(&data); + let mut response = AgentResponse { rate: 0.0 }; + + ret = ddog_trace_exporter_send(Some(exporter.as_ref()), traces, 0, Some(&mut response)); + assert_eq!(ret, None); + assert_eq!(response.rate, 0.8); + + ddog_trace_exporter_free(exporter); + } + } + + #[test] + // Ignore because it seems, at least in the version we're currently using, miri can't emulate + // libc::socket function. + #[cfg_attr(miri, ignore)] + fn exporter_send_empty_array_test() { + // Test added due to ensure the exporter is able to send empty arrays because some tracers + // (.NET) ping the agent with the aforementioned data type. + unsafe { + let server = MockServer::start(); + + let mock_traces = server.mock(|when, then| { + when.method(POST) + .header("Content-type", "application/msgpack") + .path("/v0.4/traces"); + then.status(200).body( + r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:,env:": 0.8 + } + }"#, + ); + }); + + let cfg = TraceExporterConfig { + url: Some(server.url("/")), + tracer_version: Some("0.1".to_string()), + language: Some("lang".to_string()), + language_version: Some("0.1".to_string()), + language_interpreter: Some("interpreter".to_string()), + hostname: Some("hostname".to_string()), + env: Some("env-test".to_string()), + version: Some("1.0".to_string()), + service: Some("test-service".to_string()), + input_format: TraceExporterInputFormat::V04, + output_format: TraceExporterOutputFormat::V04, + compute_stats: false, + }; + + let mut ptr: MaybeUninit> = MaybeUninit::uninit(); + let mut ret = + ddog_trace_exporter_new(NonNull::new_unchecked(&mut ptr).cast(), Some(&cfg)); + + let exporter = ptr.assume_init(); + + assert_eq!(ret, None); + + let data = vec![0x90]; + let traces = ByteSlice::new(&data); + let mut response = AgentResponse { rate: 0.0 }; + + ret = ddog_trace_exporter_send(Some(exporter.as_ref()), traces, 0, Some(&mut response)); + mock_traces.assert(); + assert_eq!(ret, None); + assert_eq!(response.rate, 0.8); + + ddog_trace_exporter_free(exporter); + } + } } diff --git a/data-pipeline/benches/span_concentrator_bench.rs b/data-pipeline/benches/span_concentrator_bench.rs index cf370fee3..de8ee780c 100644 --- a/data-pipeline/benches/span_concentrator_bench.rs +++ b/data-pipeline/benches/span_concentrator_bench.rs @@ -7,28 +7,28 @@ use std::{ use criterion::{criterion_group, Criterion}; use data_pipeline::span_concentrator::SpanConcentrator; -use datadog_trace_protobuf::pb; +use datadog_trace_utils::span_v04::Span; fn get_bucket_start(now: SystemTime, n: u64) -> i64 { let start = now.duration_since(time::UNIX_EPOCH).unwrap() + Duration::from_secs(10 * n); start.as_nanos() as i64 } -fn get_span(now: SystemTime, trace_id: u64, span_id: u64) -> pb::Span { - let mut metrics = HashMap::from([("_dd.measured".to_string(), 1.0)]); +fn get_span(now: SystemTime, trace_id: u64, span_id: u64) -> Span { + let mut metrics = HashMap::from([("_dd.measured".into(), 1.0)]); if span_id == 1 { - metrics.insert("_dd.top_level".to_string(), 1.0); + metrics.insert("_dd.top_level".into(), 1.0); } - let mut meta = HashMap::from([("db_name".to_string(), "postgres".to_string())]); + let mut meta = HashMap::from([("db_name".into(), "postgres".into())]); if span_id % 3 == 0 { - meta.insert("bucket_s3".to_string(), "aws_bucket".to_string()); + meta.insert("bucket_s3".into(), "aws_bucket".into()); } - pb::Span { + Span { trace_id, span_id, - service: "test-service".to_string(), - name: "test-name".to_string(), - resource: format!("test-{trace_id}"), + service: "test-service".into(), + name: "test-name".into(), + resource: format!("test-{trace_id}").into(), error: (span_id % 2) as i32, metrics, meta, @@ -46,7 +46,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { Duration::from_secs(10), now, vec![], - vec!["db_name".to_string(), "bucket_s3".to_string()], + vec!["db_name".into(), "bucket_s3".into()], ); let mut spans = vec![]; for trace_id in 1..100 { diff --git a/data-pipeline/examples/send-traces-with-stats.rs b/data-pipeline/examples/send-traces-with-stats.rs index be58c1712..095c02528 100644 --- a/data-pipeline/examples/send-traces-with-stats.rs +++ b/data-pipeline/examples/send-traces-with-stats.rs @@ -55,6 +55,8 @@ fn main() { traces.push(trace); } let data = rmp_serde::to_vec_named(&traces).unwrap(); - exporter.send(&data, 100).unwrap(); + let data_as_bytes = tinybytes::Bytes::from(data); + + exporter.send(data_as_bytes, 100).unwrap(); exporter.shutdown(None).unwrap(); } diff --git a/data-pipeline/src/span_concentrator/aggregation.rs b/data-pipeline/src/span_concentrator/aggregation.rs index f551bb70b..04f693290 100644 --- a/data-pipeline/src/span_concentrator/aggregation.rs +++ b/data-pipeline/src/span_concentrator/aggregation.rs @@ -4,8 +4,9 @@ //! This includes the aggregation key to group spans together and the computation of stats from a //! span. use datadog_trace_protobuf::pb; -use datadog_trace_utils::trace_utils::has_top_level; -use ddcommon::tag::Tag; +use datadog_trace_utils::span_v04::{trace_utils, Span}; +use std::borrow::Borrow; +use std::borrow::Cow; use std::collections::HashMap; const TAG_STATUS_CODE: &str = "http.status_code"; @@ -15,65 +16,166 @@ const TAG_ORIGIN: &str = "_dd.origin"; /// This struct represent the key used to group spans together to compute stats. #[derive(Debug, Hash, PartialEq, Eq, Clone, Default)] -pub(super) struct AggregationKey { - resource_name: String, - service_name: String, - operation_name: String, - span_type: String, - span_kind: String, +pub(super) struct AggregationKey<'a> { + resource_name: Cow<'a, str>, + service_name: Cow<'a, str>, + operation_name: Cow<'a, str>, + span_type: Cow<'a, str>, + span_kind: Cow<'a, str>, http_status_code: u32, is_synthetics_request: bool, - peer_tags: Vec, + peer_tags: Vec<(Cow<'a, str>, Cow<'a, str>)>, is_trace_root: bool, } -impl AggregationKey { +/// Common representation of AggregationKey used to compare AggregationKey with different lifetimes +#[derive(Clone, Hash, PartialEq, Eq)] +pub(super) struct BorrowedAggregationKey<'a> { + resource_name: &'a str, + service_name: &'a str, + operation_name: &'a str, + span_type: &'a str, + span_kind: &'a str, + http_status_code: u32, + is_synthetics_request: bool, + peer_tags: Vec<(&'a str, &'a str)>, + is_trace_root: bool, +} + +/// Trait used to define a common type (`dyn BorrowableAggregationKey`) for all AggregationKey +/// regardless of lifetime. +/// This allows an hashmap with `AggregationKey<'static>` keys to lookup an entry with a +/// `AggregationKey<'a>`. +/// This is required because the `get_mut` method of Hashmap requires an input type `Q` such that +/// the key type `K` implements `Borrow`. Since `AggregationKey<'static>` cannot implement +/// `Borrow>` we use `dyn BorrowableAggregationKey` as a placeholder. +trait BorrowableAggregationKey { + fn borrowed_aggregation_key(&self) -> BorrowedAggregationKey; +} + +impl BorrowableAggregationKey for AggregationKey<'_> { + fn borrowed_aggregation_key(&self) -> BorrowedAggregationKey { + BorrowedAggregationKey { + resource_name: self.resource_name.borrow(), + service_name: self.service_name.borrow(), + operation_name: self.operation_name.borrow(), + span_type: self.span_type.borrow(), + span_kind: self.span_kind.borrow(), + http_status_code: self.http_status_code, + is_synthetics_request: self.is_synthetics_request, + peer_tags: self + .peer_tags + .iter() + .map(|(tag, value)| (tag.borrow(), value.borrow())) + .collect(), + is_trace_root: self.is_trace_root, + } + } +} + +impl BorrowableAggregationKey for BorrowedAggregationKey<'_> { + fn borrowed_aggregation_key(&self) -> BorrowedAggregationKey { + self.clone() + } +} + +impl<'a, 'b> Borrow for AggregationKey<'a> +where + 'a: 'b, +{ + fn borrow(&self) -> &(dyn BorrowableAggregationKey + 'b) { + self + } +} + +impl Eq for (dyn BorrowableAggregationKey + '_) {} + +impl PartialEq for (dyn BorrowableAggregationKey + '_) { + fn eq(&self, other: &dyn BorrowableAggregationKey) -> bool { + self.borrowed_aggregation_key() + .eq(&other.borrowed_aggregation_key()) + } +} + +impl std::hash::Hash for (dyn BorrowableAggregationKey + '_) { + fn hash(&self, state: &mut H) { + self.borrowed_aggregation_key().hash(state) + } +} + +impl<'a> AggregationKey<'a> { /// Return an AggregationKey matching the given span. /// /// If `peer_tags_keys` is not empty then the peer tags of the span will be included in the /// key. - pub(super) fn from_span(span: &pb::Span, peer_tag_keys: &[String]) -> Self { + pub(super) fn from_span(span: &'a Span, peer_tag_keys: &'a [String]) -> Self { let span_kind = span .meta .get(TAG_SPANKIND) - .map(|s| s.to_string()) + .map(|s| s.as_str()) .unwrap_or_default(); - let peer_tags = if client_or_producer(&span_kind) { + let peer_tags = if client_or_producer(span_kind) { get_peer_tags(span, peer_tag_keys) } else { vec![] }; Self { - resource_name: span.resource.clone(), - service_name: span.service.clone(), - operation_name: span.name.clone(), - span_type: span.r#type.clone(), - span_kind, + resource_name: span.resource.as_str().into(), + service_name: span.service.as_str().into(), + operation_name: span.name.as_str().into(), + span_type: span.r#type.as_str().into(), + span_kind: span_kind.into(), http_status_code: get_status_code(span), is_synthetics_request: span .meta .get(TAG_ORIGIN) - .is_some_and(|origin| origin.starts_with(TAG_SYNTHETICS)), + .is_some_and(|origin| origin.as_str().starts_with(TAG_SYNTHETICS)), is_trace_root: span.parent_id == 0, - peer_tags, + peer_tags: peer_tags + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + } + } + + /// Clone the fields of an AggregationKey to produce a static version of the key which is + /// not tied to the lifetime of a span. + pub(super) fn into_static_key(self) -> AggregationKey<'static> { + AggregationKey { + resource_name: Cow::Owned(self.resource_name.into_owned()), + service_name: Cow::Owned(self.service_name.into_owned()), + operation_name: Cow::Owned(self.operation_name.into_owned()), + span_type: Cow::Owned(self.span_type.into_owned()), + span_kind: Cow::Owned(self.span_kind.into_owned()), + http_status_code: self.http_status_code, + is_synthetics_request: self.is_synthetics_request, + is_trace_root: self.is_trace_root, + peer_tags: self + .peer_tags + .into_iter() + .map(|(key, value)| (Cow::from(key.into_owned()), Cow::from(value.into_owned()))) + .collect(), } } } -impl From for AggregationKey { +impl From for AggregationKey<'static> { fn from(value: pb::ClientGroupedStats) -> Self { Self { - resource_name: value.resource, - service_name: value.service, - operation_name: value.name, - span_type: value.r#type, - span_kind: value.span_kind, + resource_name: value.resource.into(), + service_name: value.service.into(), + operation_name: value.name.into(), + span_type: value.r#type.into(), + span_kind: value.span_kind.into(), http_status_code: value.http_status_code, is_synthetics_request: value.synthetics, peer_tags: value .peer_tags .into_iter() - .flat_map(|t| ddcommon::tag::parse_tags(&t).0) + .filter_map(|t| { + let (key, value) = t.split_once(':')?; + Some((key.to_string().into(), value.to_string().into())) + }) .collect(), is_trace_root: value.is_trace_root == 1, } @@ -81,11 +183,11 @@ impl From for AggregationKey { } /// Return the status code of a span based on the metrics and meta tags. -fn get_status_code(span: &pb::Span) -> u32 { +fn get_status_code(span: &Span) -> u32 { if let Some(status_code) = span.metrics.get(TAG_STATUS_CODE) { *status_code as u32 } else if let Some(status_code) = span.meta.get(TAG_STATUS_CODE) { - status_code.parse().unwrap_or(0) + status_code.as_str().parse().unwrap_or(0) } else { 0 } @@ -98,10 +200,10 @@ fn client_or_producer(span_kind: &str) -> bool { /// Parse the meta tags of a span and return a list of the peer tags based on the list of /// `peer_tag_keys` -fn get_peer_tags(span: &pb::Span, peer_tag_keys: &[String]) -> Vec { +fn get_peer_tags<'k, 'v>(span: &'v Span, peer_tag_keys: &'k [String]) -> Vec<(&'k str, &'v str)> { peer_tag_keys .iter() - .filter_map(|key| Tag::new(key, span.meta.get(key)?).ok()) + .filter_map(|key| Some((key.as_str(), span.meta.get(key.as_str())?.as_str()))) .collect() } @@ -118,7 +220,7 @@ pub(super) struct GroupedStats { impl GroupedStats { /// Update the stats of a GroupedStats by inserting a span. - fn insert(&mut self, value: &pb::Span) { + fn insert(&mut self, value: &Span) { self.hits += 1; self.duration += value.duration as u64; @@ -128,7 +230,7 @@ impl GroupedStats { } else { let _ = self.ok_summary.add(value.duration as f64); } - if has_top_level(value) { + if trace_utils::has_top_level(value) { self.top_level_hits += 1; } } @@ -138,7 +240,7 @@ impl GroupedStats { /// spans aggregated on their AggregationKey. #[derive(Debug, Clone)] pub(super) struct StatsBucket { - data: HashMap, + data: HashMap, GroupedStats>, start: u64, } @@ -153,8 +255,14 @@ impl StatsBucket { /// Insert a value as stats in the group corresponding to the aggregation key, if it does /// not exist it creates it. - pub(super) fn insert(&mut self, key: AggregationKey, value: &pb::Span) { - self.data.entry(key).or_default().insert(value); + pub(super) fn insert(&mut self, key: AggregationKey<'_>, value: &Span) { + if let Some(grouped_stats) = self.data.get_mut(&key as &dyn BorrowableAggregationKey) { + grouped_stats.insert(value); + } else { + let mut grouped_stats = GroupedStats::default(); + grouped_stats.insert(value); + self.data.insert(key.into_static_key(), grouped_stats); + } } /// Consume the bucket and return a ClientStatsBucket containing the bucket stats. @@ -177,11 +285,11 @@ impl StatsBucket { /// Create a ClientGroupedStats struct based on the given AggregationKey and GroupedStats fn encode_grouped_stats(key: AggregationKey, group: GroupedStats) -> pb::ClientGroupedStats { pb::ClientGroupedStats { - service: key.service_name, - name: key.operation_name, - resource: key.resource_name, + service: key.service_name.into_owned(), + name: key.operation_name.into_owned(), + resource: key.resource_name.into_owned(), http_status_code: key.http_status_code, - r#type: key.span_type, + r#type: key.span_type.into_owned(), db_type: String::new(), // db_type is not used yet (see proto definition) hits: group.hits, @@ -192,9 +300,13 @@ fn encode_grouped_stats(key: AggregationKey, group: GroupedStats) -> pb::ClientG error_summary: group.error_summary.encode_to_vec(), synthetics: key.is_synthetics_request, top_level_hits: group.top_level_hits, - span_kind: key.span_kind, + span_kind: key.span_kind.into_owned(), - peer_tags: key.peer_tags.into_iter().map(|t| t.to_string()).collect(), + peer_tags: key + .peer_tags + .into_iter() + .map(|(k, v)| format!("{k}:{v}")) + .collect(), is_trace_root: if key.is_trace_root { pb::Trilean::True.into() } else { @@ -206,93 +318,92 @@ fn encode_grouped_stats(key: AggregationKey, group: GroupedStats) -> pb::ClientG #[cfg(test)] mod tests { use super::*; - use ddcommon::tag; #[test] fn test_aggregation_key_from_span() { - let test_cases: Vec<(pb::Span, AggregationKey)> = vec![ + let test_cases: Vec<(Span, AggregationKey)> = vec![ // Root span ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), is_trace_root: true, ..Default::default() }, ), // Span with span kind ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, - meta: HashMap::from([("span.kind".to_string(), "client".to_string())]), + meta: HashMap::from([("span.kind".into(), "client".into())]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "client".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "client".into(), is_trace_root: true, ..Default::default() }, ), // Span with peer tags but peertags aggregation disabled ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "client".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), + ("span.kind".into(), "client".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "client".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "client".into(), is_trace_root: true, ..Default::default() }, ), // Span with multiple peer tags but peertags aggregation disabled ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "producer".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), - ("db.instance".to_string(), "dynamo.test.us1".to_string()), - ("db.system".to_string(), "dynamodb".to_string()), + ("span.kind".into(), "producer".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), + ("db.instance".into(), "dynamo.test.us1".into()), + ("db.system".into(), "dynamodb".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "producer".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "producer".into(), is_trace_root: true, ..Default::default() }, @@ -300,47 +411,44 @@ mod tests { // Span with multiple peer tags but peertags aggregation disabled and span kind is // server ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "server".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), - ("db.instance".to_string(), "dynamo.test.us1".to_string()), - ("db.system".to_string(), "dynamodb".to_string()), + ("span.kind".into(), "server".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), + ("db.instance".into(), "dynamo.test.us1".into()), + ("db.system".into(), "dynamodb".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "server".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "server".into(), is_trace_root: true, ..Default::default() }, ), // Span from synthetics ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, - meta: HashMap::from([( - "_dd.origin".to_string(), - "synthetics-browser".to_string(), - )]), + meta: HashMap::from([("_dd.origin".into(), "synthetics-browser".into())]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), is_synthetics_request: true, is_trace_root: true, ..Default::default() @@ -348,19 +456,19 @@ mod tests { ), // Span with status code in meta ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, - meta: HashMap::from([("http.status_code".to_string(), "418".to_string())]), + meta: HashMap::from([("http.status_code".into(), "418".into())]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), is_synthetics_request: false, is_trace_root: true, http_status_code: 418, @@ -369,19 +477,19 @@ mod tests { ), // Span with invalid status code in meta ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, - meta: HashMap::from([("http.status_code".to_string(), "x".to_string())]), + meta: HashMap::from([("http.status_code".into(), "x".into())]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), is_synthetics_request: false, is_trace_root: true, ..Default::default() @@ -389,19 +497,19 @@ mod tests { ), // Span with status code in metrics ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, - metrics: HashMap::from([("http.status_code".to_string(), 418.0)]), + metrics: HashMap::from([("http.status_code".into(), 418.0)]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), is_synthetics_request: false, is_trace_root: true, http_status_code: 418, @@ -416,56 +524,56 @@ mod tests { "db.system".to_string(), ]; - let test_cases_with_peer_tags: Vec<(pb::Span, AggregationKey)> = vec![ + let test_cases_with_peer_tags: Vec<(Span, AggregationKey)> = vec![ // Span with peer tags with peertags aggregation enabled ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "client".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), + ("span.kind".into(), "client".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "client".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "client".into(), is_trace_root: true, - peer_tags: vec![tag!("aws.s3.bucket", "bucket-a")], + peer_tags: vec![("aws.s3.bucket".into(), "bucket-a".into())], ..Default::default() }, ), // Span with multiple peer tags with peertags aggregation enabled ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "producer".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), - ("db.instance".to_string(), "dynamo.test.us1".to_string()), - ("db.system".to_string(), "dynamodb".to_string()), + ("span.kind".into(), "producer".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), + ("db.instance".into(), "dynamo.test.us1".into()), + ("db.system".into(), "dynamodb".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "producer".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "producer".into(), peer_tags: vec![ - tag!("aws.s3.bucket", "bucket-a"), - tag!("db.instance", "dynamo.test.us1"), - tag!("db.system", "dynamodb"), + ("aws.s3.bucket".into(), "bucket-a".into()), + ("db.instance".into(), "dynamo.test.us1".into()), + ("db.system".into(), "dynamodb".into()), ], is_trace_root: true, ..Default::default() @@ -474,25 +582,25 @@ mod tests { // Span with multiple peer tags with peertags aggregation enabled and span kind is // server ( - pb::Span { - service: "service".to_string(), - name: "op".to_string(), - resource: "res".to_string(), + Span { + service: "service".into(), + name: "op".into(), + resource: "res".into(), span_id: 1, parent_id: 0, meta: HashMap::from([ - ("span.kind".to_string(), "server".to_string()), - ("aws.s3.bucket".to_string(), "bucket-a".to_string()), - ("db.instance".to_string(), "dynamo.test.us1".to_string()), - ("db.system".to_string(), "dynamodb".to_string()), + ("span.kind".into(), "server".into()), + ("aws.s3.bucket".into(), "bucket-a".into()), + ("db.instance".into(), "dynamo.test.us1".into()), + ("db.system".into(), "dynamodb".into()), ]), ..Default::default() }, AggregationKey { - service_name: "service".to_string(), - operation_name: "op".to_string(), - resource_name: "res".to_string(), - span_kind: "server".to_string(), + service_name: "service".into(), + operation_name: "op".into(), + resource_name: "res".into(), + span_kind: "server".into(), is_trace_root: true, ..Default::default() }, @@ -505,7 +613,7 @@ mod tests { for (span, expected_key) in test_cases_with_peer_tags { assert_eq!( - AggregationKey::from_span(&span, &test_peer_tags), + AggregationKey::from_span(&span, test_peer_tags.as_slice()), expected_key ); } diff --git a/data-pipeline/src/span_concentrator/mod.rs b/data-pipeline/src/span_concentrator/mod.rs index 0a614bd11..866b8f8a5 100644 --- a/data-pipeline/src/span_concentrator/mod.rs +++ b/data-pipeline/src/span_concentrator/mod.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::time::{self, Duration, SystemTime}; use datadog_trace_protobuf::pb; -use datadog_trace_utils::trace_utils; +use datadog_trace_utils::span_v04::{trace_utils, Span}; use aggregation::{AggregationKey, StatsBucket}; @@ -25,16 +25,15 @@ fn align_timestamp(t: u64, bucket_size: u64) -> u64 { } /// Return true if the span has a span.kind that is eligible for stats computation -fn compute_stats_for_span_kind(span: &pb::Span, span_kinds_stats_computed: &[String]) -> bool { +fn compute_stats_for_span_kind(span: &Span, span_kinds_stats_computed: &[String]) -> bool { !span_kinds_stats_computed.is_empty() - && span - .meta - .get("span.kind") - .is_some_and(|span_kind| span_kinds_stats_computed.contains(&span_kind.to_lowercase())) + && span.meta.get("span.kind").is_some_and(|span_kind| { + span_kinds_stats_computed.contains(&span_kind.as_str().to_lowercase()) + }) } /// Return true if the span should be ignored for stats computation -fn should_ignore_span(span: &pb::Span, span_kinds_stats_computed: &[String]) -> bool { +fn should_ignore_span(span: &Span, span_kinds_stats_computed: &[String]) -> bool { !(trace_utils::has_top_level(span) || trace_utils::is_measured(span) || compute_stats_for_span_kind(span, span_kinds_stats_computed)) @@ -114,7 +113,7 @@ impl SpanConcentrator { /// Add a span into the concentrator, by computing stats if the span is elligible for stats /// computation. - pub fn add_span(&mut self, span: &pb::Span) { + pub fn add_span(&mut self, span: &Span) { // If the span is elligible for stats computation if !should_ignore_span(span, self.span_kinds_stats_computed.as_slice()) { let mut bucket_timestamp = @@ -125,7 +124,7 @@ impl SpanConcentrator { bucket_timestamp = self.oldest_timestamp; } - let agg_key = AggregationKey::from_span(span, &self.peer_tag_keys); + let agg_key = AggregationKey::from_span(span, self.peer_tag_keys.as_slice()); self.buckets .entry(bucket_timestamp) @@ -137,7 +136,7 @@ impl SpanConcentrator { /// Flush all stats bucket except for the `buffer_len` most recent. If `force` is true, flush /// all buckets. pub fn flush(&mut self, now: SystemTime, force: bool) -> Vec { - // TODO: Use drain filter from hashbrown to avoid removing current buckets + // TODO: Wait for HashMap::extract_if to be stabilized to avoid a full drain let now_timestamp = system_time_to_unix_duration(now).as_nanos() as u64; let buckets: Vec<(u64, StatsBucket)> = self.buckets.drain().collect(); self.oldest_timestamp = if force { diff --git a/data-pipeline/src/span_concentrator/tests.rs b/data-pipeline/src/span_concentrator/tests.rs index ba8b91739..342edfcd3 100644 --- a/data-pipeline/src/span_concentrator/tests.rs +++ b/data-pipeline/src/span_concentrator/tests.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use datadog_trace_utils::trace_utils::compute_top_level_span; +use datadog_trace_utils::span_v04::trace_utils::compute_top_level_span; use rand::{thread_rng, Rng}; const BUCKET_SIZE: u64 = Duration::from_secs(2).as_nanos() as u64; @@ -32,21 +32,21 @@ fn get_test_span( service: &str, resource: &str, error: i32, -) -> pb::Span { +) -> Span { let aligned_now = align_timestamp( system_time_to_unix_duration(now).as_nanos() as u64, BUCKET_SIZE, ); - pb::Span { + Span { span_id, parent_id, duration, start: get_timestamp_in_bucket(aligned_now, BUCKET_SIZE, offset) as i64 - duration, - service: service.to_string(), - name: "query".to_string(), - resource: resource.to_string(), + service: service.to_string().into(), + name: "query".into(), + resource: resource.to_string().into(), error, - r#type: "db".to_string(), + r#type: "db".into(), ..Default::default() } } @@ -63,16 +63,16 @@ fn get_test_span_with_meta( error: i32, meta: &[(&str, &str)], metrics: &[(&str, f64)], -) -> pb::Span { +) -> Span { let mut span = get_test_span( now, span_id, parent_id, duration, offset, service, resource, error, ); for (k, v) in meta { - span.meta.insert(k.to_string(), v.to_string()); + span.meta.insert(k.to_string().into(), v.to_string().into()); } span.metrics = HashMap::new(); for (k, v) in metrics { - span.metrics.insert(k.to_string(), *v); + span.metrics.insert(k.to_string().into(), *v); } span } @@ -647,7 +647,7 @@ fn test_ignore_partial_spans() { .get_mut(0) .unwrap() .metrics - .insert("_dd.partial_version".to_string(), 830604.0); + .insert("_dd.partial_version".into(), 830604.0); compute_top_level_span(spans.as_mut_slice()); let mut concentrator = SpanConcentrator::new( Duration::from_nanos(BUCKET_SIZE), @@ -877,121 +877,121 @@ fn test_peer_tags_aggregation() { #[test] fn test_compute_stats_for_span_kind() { - let test_cases: Vec<(pb::Span, bool)> = vec![ + let test_cases: Vec<(Span, bool)> = vec![ ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "server".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "server".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "consumer".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "consumer".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "client".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "client".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "producer".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "producer".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "internal".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "internal".into())]), ..Default::default() }, false, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "SERVER".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "SERVER".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "CONSUMER".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "CONSUMER".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "CLIENT".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "CLIENT".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "PRODUCER".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "PRODUCER".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "INTERNAL".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "INTERNAL".into())]), ..Default::default() }, false, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "SerVER".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "SerVER".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "ConSUMeR".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "ConSUMeR".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "CLiENT".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "CLiENT".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "PROducER".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "PROducER".into())]), ..Default::default() }, true, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "INtERNAL".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "INtERNAL".into())]), ..Default::default() }, false, ), ( - pb::Span { - meta: HashMap::from([("span.kind".to_string(), "".to_string())]), + Span { + meta: HashMap::from([("span.kind".into(), "".into())]), ..Default::default() }, false, ), ( - pb::Span { + Span { meta: HashMap::from([]), ..Default::default() }, diff --git a/data-pipeline/src/stats_exporter.rs b/data-pipeline/src/stats_exporter.rs index 7a7f74c68..fac027ef2 100644 --- a/data-pipeline/src/stats_exporter.rs +++ b/data-pipeline/src/stats_exporter.rs @@ -187,7 +187,7 @@ pub fn stats_url_from_agent_url(agent_url: &str) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - use datadog_trace_utils::trace_utils; + use datadog_trace_utils::span_v04::{trace_utils, Span}; use httpmock::prelude::*; use httpmock::MockServer; use time::Duration; @@ -229,8 +229,8 @@ mod tests { let mut trace = vec![]; for i in 1..100 { - trace.push(pb::Span { - service: "libdatadog-test".to_string(), + trace.push(Span { + service: "libdatadog-test".into(), duration: i, ..Default::default() }) diff --git a/data-pipeline/src/trace_exporter/agent_response.rs b/data-pipeline/src/trace_exporter/agent_response.rs new file mode 100644 index 000000000..11bda9289 --- /dev/null +++ b/data-pipeline/src/trace_exporter/agent_response.rs @@ -0,0 +1,122 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::Deserialize; +use serde_json::{Map, Value}; +use std::io::Error as IoError; +use std::io::ErrorKind as IoErrorKind; + +use crate::trace_exporter::error::TraceExporterError; +use std::{f64, str::FromStr}; + +/// `Rates` contains all service's sampling rates returned by the agent. +#[derive(Debug, Deserialize)] +pub struct Rates { + rate_by_service: Map, +} + +impl Rates { + /// Get the sampling rate for a service and evironment pair. + pub fn get(&self, service: &str, env: &str) -> Result { + for (id, value) in &self.rate_by_service { + let mut it = id + .split(',') + .filter_map(|pair| pair.split_once(':')) + .map(|(_, value)| value); + + let srv_pair = (it.next().unwrap_or(""), it.next().unwrap_or("")); + if srv_pair == (service, env) { + return value + .as_f64() + .ok_or(IoError::from(IoErrorKind::InvalidData)); + } + } + // Return default + if let Some(default) = self.rate_by_service.get("service:,env:") { + default + .as_f64() + .ok_or(IoError::from(IoErrorKind::InvalidData)) + } else { + Err(IoError::from(IoErrorKind::NotFound)) + } + } +} + +impl FromStr for Rates { + type Err = TraceExporterError; + fn from_str(s: &str) -> Result { + let obj: Rates = serde_json::from_str(s)?; + Ok(obj) + } +} + +/// `AgentResponse` structure holds agent response information upon successful request. +#[derive(Debug, PartialEq)] +#[repr(C)] +pub struct AgentResponse { + /// Sampling rate for the current service. + pub rate: f64, +} + +impl From for AgentResponse { + fn from(value: f64) -> Self { + AgentResponse { rate: value } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_test() { + let payload = r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:foo,env:prod": 0.3, + "service:,env:": 0.8 + } + }"#; + + let rates: Rates = payload.parse().unwrap(); + + assert_eq!(rates.rate_by_service.len(), 3); + assert_eq!(rates.get("foo", "staging").unwrap(), 1.0); + assert_eq!(rates.get("foo", "prod").unwrap(), 0.3); + assert_eq!(rates.get("bar", "bar-env").unwrap(), 0.8); + } + + #[test] + fn parse_invalid_data_test() { + let payload = r#"{ + "rate_by_service": { + "service:foo,env:staging": "", + "service:,env:": "" + } + }"#; + + let rates: Rates = payload.parse().unwrap(); + + assert_eq!(rates.rate_by_service.len(), 2); + assert!(rates + .get("foo", "staging") + .is_err_and(|e| e.kind() == IoErrorKind::InvalidData)); + assert!(rates + .get("bar", "staging") + .is_err_and(|e| e.kind() == IoErrorKind::InvalidData)); + } + + #[test] + fn parse_invalid_payload_test() { + let payload = r#"{ + "invalid": { + "service:foo,env:staging": "", + "service:,env:": "" + } + }"#; + + let res = payload.parse::(); + + assert!(res.is_err()); + } +} diff --git a/data-pipeline/src/trace_exporter/error.rs b/data-pipeline/src/trace_exporter/error.rs new file mode 100644 index 000000000..b7f13306e --- /dev/null +++ b/data-pipeline/src/trace_exporter/error.rs @@ -0,0 +1,178 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::trace_exporter::msgpack_decoder::v04::error::DecodeError; +use hyper::http::StatusCode; +use hyper::Error as HyperError; +use serde_json::error::Error as SerdeError; +use std::error::Error; +use std::fmt::{Debug, Display}; + +#[derive(Debug, PartialEq)] +pub enum AgentErrorKind { + EmptyResponse, +} + +impl Display for AgentErrorKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AgentErrorKind::EmptyResponse => write!(f, "Agent empty response"), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum BuilderErrorKind { + InvalidUri, +} + +impl Display for BuilderErrorKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BuilderErrorKind::InvalidUri => write!(f, "Invalid URI"), + } + } +} + +#[derive(Copy, Clone, Debug)] +pub enum NetworkErrorKind { + Body, + Canceled, + ConnectionClosed, + MessageTooLarge, + Parse, + TimedOut, + Unknown, + WrongStatus, +} + +#[derive(Debug)] +pub struct NetworkError { + kind: NetworkErrorKind, + source: HyperError, +} + +impl Error for NetworkError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(&self.source) + } +} + +impl NetworkError { + fn new(kind: NetworkErrorKind, source: HyperError) -> Self { + Self { kind, source } + } + + pub fn kind(&self) -> NetworkErrorKind { + self.kind + } +} + +impl Display for NetworkError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self.source().unwrap(), f) + } +} + +#[derive(Debug, PartialEq)] +pub struct RequestError { + code: StatusCode, + msg: String, +} + +impl Display for RequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + format_args!("Error code: {}, Response: {}", self.code, self.msg) + ) + } +} + +impl RequestError { + pub fn new(code: StatusCode, msg: &str) -> Self { + Self { + code, + msg: msg.to_owned(), + } + } + + pub fn status(&self) -> StatusCode { + self.code + } +} + +/// TraceExporterError holds different types of errors that occur when handling traces. +#[derive(Debug)] +pub enum TraceExporterError { + Agent(AgentErrorKind), + Builder(BuilderErrorKind), + Deserialization(DecodeError), + Io(std::io::Error), + Network(NetworkError), + Request(RequestError), + Serde(SerdeError), +} + +impl From for TraceExporterError { + fn from(value: SerdeError) -> Self { + TraceExporterError::Serde(value) + } +} + +impl From for TraceExporterError { + fn from(_value: hyper::http::uri::InvalidUri) -> Self { + TraceExporterError::Builder(BuilderErrorKind::InvalidUri) + } +} + +impl From for TraceExporterError { + fn from(err: HyperError) -> Self { + if err.is_parse() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::Parse, err)) + } else if err.is_canceled() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::Canceled, err)) + } else if err.is_connect() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::ConnectionClosed, err)) + } else if err.is_parse_too_large() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::MessageTooLarge, err)) + } else if err.is_incomplete_message() || err.is_body_write_aborted() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::Body, err)) + } else if err.is_parse_status() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::WrongStatus, err)) + } else if err.is_timeout() { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::TimedOut, err)) + } else { + TraceExporterError::Network(NetworkError::new(NetworkErrorKind::Unknown, err)) + } + } +} + +impl From for TraceExporterError { + fn from(err: DecodeError) -> Self { + TraceExporterError::Deserialization(err) + } +} + +impl From for TraceExporterError { + fn from(err: std::io::Error) -> Self { + TraceExporterError::Io(err) + } +} + +impl Display for TraceExporterError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TraceExporterError::Agent(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Builder(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Deserialization(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Io(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Network(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Request(e) => std::fmt::Display::fmt(e, f), + TraceExporterError::Serde(e) => std::fmt::Display::fmt(e, f), + } + } +} + +impl Error for TraceExporterError {} diff --git a/data-pipeline/src/trace_exporter.rs b/data-pipeline/src/trace_exporter/mod.rs similarity index 66% rename from data-pipeline/src/trace_exporter.rs rename to data-pipeline/src/trace_exporter/mod.rs index 90cdee829..fc843e6af 100644 --- a/data-pipeline/src/trace_exporter.rs +++ b/data-pipeline/src/trace_exporter/mod.rs @@ -1,12 +1,19 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +pub mod agent_response; +pub mod error; use crate::agent_info::{AgentInfoArc, AgentInfoFetcher}; +use crate::trace_exporter::error::{RequestError, TraceExporterError}; use crate::{ health_metrics, health_metrics::HealthMetric, span_concentrator::SpanConcentrator, stats_exporter, }; use arc_swap::{ArcSwap, ArcSwapOption}; use bytes::Bytes; +use datadog_trace_utils::span_v04::{ + trace_utils::{compute_top_level_span, has_top_level}, + Span, +}; use datadog_trace_utils::trace_utils::{self, SendData, TracerHeaderTags}; use datadog_trace_utils::tracer_payload::TraceCollection; use datadog_trace_utils::{msgpack_decoder, tracer_payload}; @@ -24,16 +31,15 @@ use std::{borrow::Borrow, collections::HashMap, str::FromStr, time}; use tokio::{runtime::Runtime, task::JoinHandle}; use tokio_util::sync::CancellationToken; +use self::agent_response::{AgentResponse, Rates}; + const DEFAULT_STATS_ELIGIBLE_SPAN_KINDS: [&str; 4] = ["client", "server", "producer", "consumer"]; const STATS_ENDPOINT: &str = "/v0.6/stats"; const INFO_ENDPOINT: &str = "/info"; // Keys used for sampling -#[allow(dead_code)] // TODO (APMSP-1583) these will be used with client side stats const SAMPLING_PRIORITY_KEY: &str = "_sampling_priority_v1"; -#[allow(dead_code)] // TODO (APMSP-1584) these will be used with client side stats const SAMPLING_SINGLE_SPAN_MECHANISM: &str = "_dd.span_sampling.mechanism"; -#[allow(dead_code)] // TODO (APMSP-1584) these will be used with client side stats const SAMPLING_ANALYTICS_RATE_KEY: &str = "_dd1.sr.eausr"; /// TraceExporterInputFormat represents the format of the input traces. @@ -99,14 +105,13 @@ fn add_path(url: &Uri, path: &str) -> Uri { Uri::from_parts(parts).unwrap() } -/* TODO (APMSP-1583) re-enable client side stats struct DroppedP0Counts { pub dropped_p0_traces: usize, pub dropped_p0_spans: usize, } -Remove spans and chunks only keeping the ones that may be sampled by the agent -fn drop_chunks(traces: &mut Vec>) -> DroppedP0Counts { +/// Remove spans and chunks only keeping the ones that may be sampled by the agent +fn drop_chunks(traces: &mut Vec>) -> DroppedP0Counts { let mut dropped_p0_traces = 0; let mut dropped_p0_spans = 0; traces.retain_mut(|chunk| { @@ -120,8 +125,8 @@ fn drop_chunks(traces: &mut Vec>) -> DroppedP0Counts { } // PrioritySampler and NoPrioritySampler let priority = span.metrics.get(SAMPLING_PRIORITY_KEY); - if has_top_level(span) && (priority.is_none() || priority.is_some_and(|p| *p > 0.0)) -{ // We send chunks with positive priority or no priority + if has_top_level(span) && (priority.is_none() || priority.is_some_and(|p| *p > 0.0)) { + // We send chunks with positive priority or no priority return true; } // SingleSpanSampler and AnalyzedSpansSampler @@ -153,7 +158,6 @@ fn drop_chunks(traces: &mut Vec>) -> DroppedP0Counts { dropped_p0_spans, } } - */ #[derive(Clone, Default, Debug)] pub struct TracerMetadata { @@ -193,6 +197,7 @@ impl<'a> From<&'a TracerMetadata> for HashMap<&'static str, String> { } } +#[derive(Debug)] enum StatsComputationStatus { /// Client-side stats has been disabled by the tracer Disabled, @@ -227,18 +232,17 @@ enum StatsComputationStatus { /// another task to send stats when a time bucket expire. When this feature is enabled the /// TraceExporter drops all spans that may not be sampled by the agent. #[allow(missing_docs)] +#[derive(Debug)] pub struct TraceExporter { endpoint: Endpoint, metadata: TracerMetadata, input_format: TraceExporterInputFormat, output_format: TraceExporterOutputFormat, // TODO - do something with the response callback - https://datadoghq.atlassian.net/browse/APMSP-1019 - _response_callback: Option>, runtime: Runtime, /// None if dogstatsd is disabled dogstatsd: Option, common_stats_tags: Vec, - #[allow(dead_code)] client_computed_top_level: bool, client_side_stats: ArcSwap, agent_info: AgentInfoArc, @@ -253,20 +257,32 @@ impl TraceExporter { /// Send msgpack serialized traces to the agent #[allow(missing_docs)] - pub fn send(&self, data: &[u8], trace_count: usize) -> Result { + pub fn send( + &self, + data: tinybytes::Bytes, + trace_count: usize, + ) -> Result { self.check_agent_info(); match self.input_format { - TraceExporterInputFormat::Proxy => self.send_proxy(data, trace_count), - TraceExporterInputFormat::V04 => { - self.send_deser_ser(tinybytes::Bytes::copy_from_slice(data)) - // TODO: APMSP-1582 - Refactor data-pipeline-ffi so we can leverage a type that - // implements tinybytes::UnderlyingBytes trait to avoid copying - } + TraceExporterInputFormat::Proxy => self.send_proxy(data.as_ref(), trace_count), + TraceExporterInputFormat::V04 => self.send_deser_ser(data), } + .and_then(|res| { + if res.is_empty() { + return Err(TraceExporterError::Agent( + error::AgentErrorKind::EmptyResponse, + )); + } + + let rates = res.parse::()?; + + let rate = rates.get(&self.metadata.service, &self.metadata.env)?; + Ok(AgentResponse::from(rate)) + }) } /// Safely shutdown the TraceExporter and all related tasks - pub fn shutdown(self, timeout: Option) -> Result<(), String> { + pub fn shutdown(self, timeout: Option) -> Result<(), TraceExporterError> { if let Some(timeout) = timeout { match self.runtime.block_on(async { tokio::time::timeout(timeout, async { @@ -287,7 +303,7 @@ impl TraceExporter { .await }) { Ok(()) => Ok(()), - Err(_) => Err("Shutdown timed out".to_string()), + Err(e) => Err(TraceExporterError::Io(e.into())), } } else { self.runtime.block_on(async { @@ -434,7 +450,7 @@ impl TraceExporter { } } - fn send_proxy(&self, data: &[u8], trace_count: usize) -> Result { + fn send_proxy(&self, data: &[u8], trace_count: usize) -> Result { self.send_data_to_url( data, trace_count, @@ -447,93 +463,91 @@ impl TraceExporter { data: &[u8], trace_count: usize, uri: Uri, - ) -> Result { - self.runtime - .block_on(async { - let mut req_builder = hyper::Request::builder() - .uri(uri) - .header( - hyper::header::USER_AGENT, - concat!("Tracer/", env!("CARGO_PKG_VERSION")), - ) - .method(Method::POST); - - let headers: HashMap<&'static str, String> = self.metadata.borrow().into(); - - for (key, value) in &headers { - req_builder = req_builder.header(*key, value); - } - req_builder = req_builder - .header("Content-type", "application/msgpack") - .header("X-Datadog-Trace-Count", trace_count.to_string().as_str()); - let req = req_builder - .body(Body::from(Bytes::copy_from_slice(data))) - .unwrap(); - - match hyper::Client::builder() - .build(connector::Connector::default()) - .request(req) - .await - { - Ok(response) => { - let response_status = response.status(); - if !response_status.is_success() { - let body_bytes = response.into_body().collect().await?.to_bytes(); - let response_body = - String::from_utf8(body_bytes.to_vec()).unwrap_or_default(); - let resp_tag_res = &Tag::new("response_code", response_status.as_str()); - match resp_tag_res { - Ok(resp_tag) => { - self.emit_metric( - HealthMetric::Count( - health_metrics::STAT_SEND_TRACES_ERRORS, - 1, - ), - Some(vec![&resp_tag]), - ); - } - Err(tag_err) => { - // This should really never happen as response_status is a - // `NonZeroU16`, but if the response status or tag requirements - // ever change in the future we still don't want to panic. - error!("Failed to serialize response_code to tag {}", tag_err) - } - } - anyhow::bail!("Agent did not accept traces: {response_body}"); - } - match response.into_body().collect().await { - Ok(body) => { - self.emit_metric( - HealthMetric::Count( - health_metrics::STAT_SEND_TRACES, - trace_count as i64, - ), - None, - ); - Ok(String::from_utf8_lossy(&body.to_bytes()).to_string()) - } - Err(err) => { + ) -> Result { + self.runtime.block_on(async { + let mut req_builder = hyper::Request::builder() + .uri(uri) + .header( + hyper::header::USER_AGENT, + concat!("Tracer/", env!("CARGO_PKG_VERSION")), + ) + .method(Method::POST); + + let headers: HashMap<&'static str, String> = self.metadata.borrow().into(); + + for (key, value) in &headers { + req_builder = req_builder.header(*key, value); + } + req_builder = req_builder + .header("Content-type", "application/msgpack") + .header("X-Datadog-Trace-Count", trace_count.to_string().as_str()); + let req = req_builder + .body(Body::from(Bytes::copy_from_slice(data))) + .unwrap(); + + match hyper::Client::builder() + .build(connector::Connector::default()) + .request(req) + .await + { + Ok(response) => { + let response_status = response.status(); + if !response_status.is_success() { + // TODO: remove unwrap + let body_bytes = response.into_body().collect().await.unwrap().to_bytes(); + let response_body = + String::from_utf8(body_bytes.to_vec()).unwrap_or_default(); + let resp_tag_res = &Tag::new("response_code", response_status.as_str()); + match resp_tag_res { + Ok(resp_tag) => { self.emit_metric( HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), - None, + Some(vec![&resp_tag]), ); - anyhow::bail!("Error reading agent response body: {err}"); + } + Err(tag_err) => { + // This should really never happen as response_status is a + // `NonZeroU16`, but if the response status or tag requirements + // ever change in the future we still don't want to panic. + error!("Failed to serialize response_code to tag {}", tag_err) } } + return Err(TraceExporterError::Request(RequestError::new( + response_status, + &response_body, + ))); + //anyhow::bail!("Agent did not accept traces: {response_body}"); } - Err(err) => { - self.emit_metric( - HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), - None, - ); - anyhow::bail!("Failed to send traces: {err}") + match response.into_body().collect().await { + Ok(body) => { + self.emit_metric( + HealthMetric::Count( + health_metrics::STAT_SEND_TRACES, + trace_count as i64, + ), + None, + ); + Ok(String::from_utf8_lossy(&body.to_bytes()).to_string()) + } + Err(err) => { + self.emit_metric( + HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), + None, + ); + Err(TraceExporterError::from(err)) + // anyhow::bail!("Error reading agent response body: {err}"); + } } } - }) - .or_else(|err| { - error!("Error sending traces: {err}"); - Ok(String::from("{}")) - }) + Err(err) => { + self.emit_metric( + HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), + None, + ); + Err(TraceExporterError::from(err)) + } + } + }) } /// Emit a health metric to dogstatsd @@ -551,27 +565,26 @@ impl TraceExporter { } } - // /// Add all spans from the given iterator into the stats concentrator - // /// # Panic - // /// Will panic if another thread panicked will holding the lock on `stats_concentrator` - // fn add_spans_to_stats<'a>(&self, spans: impl Iterator) { - // if let StatsComputationStatus::Enabled { - // stats_concentrator, - // cancellation_token: _, - // exporter_handle: _, - // } = &**self.client_side_stats.load() - // { - // let mut stats_concentrator = stats_concentrator.lock().unwrap(); - // for span in spans { - // stats_concentrator.add_span(span); - // } - // } - // } - - fn send_deser_ser(&self, data: tinybytes::Bytes) -> Result { - // let size = data.len(); + /// Add all spans from the given iterator into the stats concentrator + /// # Panic + /// Will panic if another thread panicked will holding the lock on `stats_concentrator` + fn add_spans_to_stats<'a>(&self, spans: impl Iterator) { + if let StatsComputationStatus::Enabled { + stats_concentrator, + cancellation_token: _, + exporter_handle: _, + } = &**self.client_side_stats.load() + { + let mut stats_concentrator = stats_concentrator.lock().unwrap(); + for span in spans { + stats_concentrator.add_span(span); + } + } + } + + fn send_deser_ser(&self, data: tinybytes::Bytes) -> Result { // TODO base on input format - let (traces, size) = match msgpack_decoder::v04::decoder::from_slice(data) { + let (mut traces, size) = match msgpack_decoder::v04::decoder::from_slice(data) { Ok(res) => res, Err(err) => { error!("Error deserializing trace from request body: {err}"); @@ -579,15 +592,10 @@ impl TraceExporter { HealthMetric::Count(health_metrics::STAT_DESER_TRACES_ERRORS, 1), None, ); - return Ok(String::from("{}")); + return Err(TraceExporterError::Deserialization(err)); } }; - if traces.is_empty() { - error!("No traces deserialized from the request body."); - return Ok(String::from("{}")); - } - let num_traces = traces.len(); self.emit_metric( @@ -595,24 +603,27 @@ impl TraceExporter { None, ); - let header_tags: TracerHeaderTags = self.metadata.borrow().into(); + let mut header_tags: TracerHeaderTags = self.metadata.borrow().into(); // Stats computation - // if let StatsComputationStatus::Enabled { .. } = &**self.client_side_stats.load() { - // if !self.client_computed_top_level { - // for chunk in traces.iter_mut() { - // compute_top_level_span(chunk); - // } - // } - // self.add_spans_to_stats(traces.iter().flat_map(|trace| trace.iter())); - // // Once stats have been computed we can drop all chunks that are not going to be - // // sampled by the agent - // let dropped_counts = drop_chunks(&mut traces); - // header_tags.client_computed_top_level = true; - // header_tags.client_computed_stats = true; - // header_tags.dropped_p0_traces = dropped_counts.dropped_p0_traces; - // header_tags.dropped_p0_spans = dropped_counts.dropped_p0_spans; - // } + if let StatsComputationStatus::Enabled { .. } = &**self.client_side_stats.load() { + if !self.client_computed_top_level { + for chunk in traces.iter_mut() { + compute_top_level_span(chunk); + } + } + self.add_spans_to_stats(traces.iter().flat_map(|trace| trace.iter())); + // Once stats have been computed we can drop all chunks that are not going to be + // sampled by the agent + let dropped_counts = drop_chunks(&mut traces); + + // Update the headers to indicate that stats have been computed and forward dropped + // traces counts + header_tags.client_computed_top_level = true; + header_tags.client_computed_stats = true; + header_tags.dropped_p0_traces = dropped_counts.dropped_p0_traces; + header_tags.dropped_p0_spans = dropped_counts.dropped_p0_spans; + } match self.output_format { TraceExporterOutputFormat::V04 => { @@ -631,17 +642,9 @@ impl TraceExporter { let send_data_result = send_data.send().await; match send_data_result.last_result { Ok(response) => { - self.emit_metric( - HealthMetric::Count( - health_metrics::STAT_SEND_TRACES, - num_traces as i64, - ), - None, - ); - match response.into_body().collect().await { - Ok(body) => { - Ok(String::from_utf8_lossy(&body.to_bytes()).to_string()) - } + let status = response.status(); + let body = match response.into_body().collect().await { + Ok(body) => String::from_utf8_lossy(&body.to_bytes()).to_string(), Err(err) => { error!("Error reading agent response body: {err}"); self.emit_metric( @@ -651,8 +654,27 @@ impl TraceExporter { ), None, ); - Ok(String::from("{}")) + return Err(TraceExporterError::from(err)); } + }; + + if status.is_success() { + self.emit_metric( + HealthMetric::Count( + health_metrics::STAT_SEND_TRACES, + num_traces as i64, + ), + None, + ); + Ok(body) + } else { + self.emit_metric( + HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), + None, + ); + Err(TraceExporterError::Request(RequestError::new( + status, &body, + ))) } } Err(err) => { @@ -661,7 +683,9 @@ impl TraceExporter { HealthMetric::Count(health_metrics::STAT_SEND_TRACES_ERRORS, 1), None, ); - Ok(String::from("{}")) + Err(TraceExporterError::Io(std::io::Error::from( + std::io::ErrorKind::Other, + ))) } } }) @@ -690,7 +714,6 @@ pub struct TraceExporterBuilder { git_commit_sha: String, input_format: TraceExporterInputFormat, output_format: TraceExporterOutputFormat, - response_callback: Option>, dogstatsd_url: Option, client_computed_stats: bool, client_computed_top_level: bool, @@ -793,12 +816,6 @@ impl TraceExporterBuilder { self } - #[allow(missing_docs)] - pub fn set_response_callback(mut self, response_callback: Box) -> Self { - self.response_callback = Some(response_callback); - self - } - /// Set the header indicating the tracer has computed the top-level tag pub fn set_client_computed_top_level(mut self) -> Self { self.client_computed_top_level = true; @@ -834,7 +851,7 @@ impl TraceExporterBuilder { } #[allow(missing_docs)] - pub fn build(self) -> anyhow::Result { + pub fn build(self) -> Result { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; @@ -887,7 +904,6 @@ impl TraceExporterBuilder { }, input_format: self.input_format, output_format: self.output_format, - _response_callback: self.response_callback, client_computed_top_level: self.client_computed_top_level, runtime, dogstatsd, @@ -907,17 +923,20 @@ pub trait ResponseCallback { #[cfg(test)] mod tests { + use self::error::AgentErrorKind; + use self::error::BuilderErrorKind; use super::*; use datadog_trace_utils::span_v04::Span; use httpmock::prelude::*; use httpmock::MockServer; - // use serde::Serialize; use std::collections::HashMap; use std::net; use std::time::Duration; use tinybytes::BytesString; use tokio::time::sleep; + const TRACER_TOP_LEVEL_KEY: &str = "_dd.top_level"; + #[cfg_attr(all(miri, target_os = "macos"), ignore)] #[test] fn new() { @@ -1028,119 +1047,119 @@ mod tests { assert!(hashmap.contains_key("datadog-client-computed-stats")); assert!(hashmap.contains_key("datadog-client-computed-top-level")); } - // - // #[test] - // fn test_drop_chunks() { - // let chunk_with_priority = vec![ - // pb::Span { - // span_id: 1, - // metrics: HashMap::from([ - // (SAMPLING_PRIORITY_KEY.to_string(), 1.0), - // ("_dd.top_level".to_string(), 1.0), - // ]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // ..Default::default() - // }, - // ]; - // let chunk_with_null_priority = vec![ - // pb::Span { - // span_id: 1, - // metrics: HashMap::from([ - // (SAMPLING_PRIORITY_KEY.to_string(), 0.0), - // ("_dd.top_level".to_string(), 1.0), - // ]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // ..Default::default() - // }, - // ]; - // let chunk_without_priority = vec![ - // pb::Span { - // span_id: 1, - // metrics: HashMap::from([("_dd.top_level".to_string(), 1.0)]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // ..Default::default() - // }, - // ]; - // let chunk_with_error = vec![ - // pb::Span { - // span_id: 1, - // error: 1, - // metrics: HashMap::from([ - // (SAMPLING_PRIORITY_KEY.to_string(), 0.0), - // ("_dd.top_level".to_string(), 1.0), - // ]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // ..Default::default() - // }, - // ]; - // let chunk_with_a_single_span = vec![ - // pb::Span { - // span_id: 1, - // metrics: HashMap::from([ - // (SAMPLING_PRIORITY_KEY.to_string(), 0.0), - // ("_dd.top_level".to_string(), 1.0), - // ]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // metrics: HashMap::from([(SAMPLING_SINGLE_SPAN_MECHANISM.to_string(), 8.0)]), - // ..Default::default() - // }, - // ]; - // let chunk_with_analyzed_span = vec![ - // pb::Span { - // span_id: 1, - // metrics: HashMap::from([ - // (SAMPLING_PRIORITY_KEY.to_string(), 0.0), - // ("_dd.top_level".to_string(), 1.0), - // ]), - // ..Default::default() - // }, - // pb::Span { - // span_id: 2, - // parent_id: 1, - // metrics: HashMap::from([(SAMPLING_ANALYTICS_RATE_KEY.to_string(), 1.0)]), - // ..Default::default() - // }, - // ]; - // - // let chunks_and_expected_sampled_spans = vec![ - // (chunk_with_priority, 2), - // (chunk_with_null_priority, 0), - // (chunk_without_priority, 2), - // (chunk_with_error, 2), - // (chunk_with_a_single_span, 1), - // (chunk_with_analyzed_span, 1), - // ]; - // - // for (chunk, expected_count) in chunks_and_expected_sampled_spans.into_iter() { - // let mut traces = vec![chunk]; - // drop_chunks(&mut traces); - // if expected_count == 0 { - // assert!(traces.is_empty()); - // } else { - // assert_eq!(traces[0].len(), expected_count); - // } - // } - // } + + #[test] + fn test_drop_chunks() { + let chunk_with_priority = vec![ + Span { + span_id: 1, + metrics: HashMap::from([ + (SAMPLING_PRIORITY_KEY.into(), 1.0), + (TRACER_TOP_LEVEL_KEY.into(), 1.0), + ]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + ..Default::default() + }, + ]; + let chunk_with_null_priority = vec![ + Span { + span_id: 1, + metrics: HashMap::from([ + (SAMPLING_PRIORITY_KEY.into(), 0.0), + (TRACER_TOP_LEVEL_KEY.into(), 1.0), + ]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + ..Default::default() + }, + ]; + let chunk_without_priority = vec![ + Span { + span_id: 1, + metrics: HashMap::from([(TRACER_TOP_LEVEL_KEY.into(), 1.0)]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + ..Default::default() + }, + ]; + let chunk_with_error = vec![ + Span { + span_id: 1, + error: 1, + metrics: HashMap::from([ + (SAMPLING_PRIORITY_KEY.into(), 0.0), + (TRACER_TOP_LEVEL_KEY.into(), 1.0), + ]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + ..Default::default() + }, + ]; + let chunk_with_a_single_span = vec![ + Span { + span_id: 1, + metrics: HashMap::from([ + (SAMPLING_PRIORITY_KEY.into(), 0.0), + (TRACER_TOP_LEVEL_KEY.into(), 1.0), + ]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + metrics: HashMap::from([(SAMPLING_SINGLE_SPAN_MECHANISM.into(), 8.0)]), + ..Default::default() + }, + ]; + let chunk_with_analyzed_span = vec![ + Span { + span_id: 1, + metrics: HashMap::from([ + (SAMPLING_PRIORITY_KEY.into(), 0.0), + (TRACER_TOP_LEVEL_KEY.into(), 1.0), + ]), + ..Default::default() + }, + Span { + span_id: 2, + parent_id: 1, + metrics: HashMap::from([(SAMPLING_ANALYTICS_RATE_KEY.into(), 1.0)]), + ..Default::default() + }, + ]; + + let chunks_and_expected_sampled_spans = vec![ + (chunk_with_priority, 2), + (chunk_with_null_priority, 0), + (chunk_without_priority, 2), + (chunk_with_error, 2), + (chunk_with_a_single_span, 1), + (chunk_with_analyzed_span, 1), + ]; + + for (chunk, expected_count) in chunks_and_expected_sampled_spans.into_iter() { + let mut traces = vec![chunk]; + drop_chunks(&mut traces); + if expected_count == 0 { + assert!(traces.is_empty()); + } else { + assert_eq!(traces[0].len(), expected_count); + } + } + } #[cfg_attr(miri, ignore)] #[test] @@ -1154,12 +1173,12 @@ mod tests { then.status(200).body(""); }); - // let mock_stats = server.mock(|when, then| { - // when.method(POST) - // .header("Content-type", "application/msgpack") - // .path("/v0.6/stats"); - // then.status(200).body(""); - // }); + let mock_stats = server.mock(|when, then| { + when.method(POST) + .header("Content-type", "application/msgpack") + .path("/v0.6/stats"); + then.status(200).body(""); + }); let mock_info = server.mock(|when, then| { when.method(GET).path("/info"); @@ -1172,6 +1191,8 @@ mod tests { let builder = TraceExporterBuilder::default(); let exporter = builder .set_url(&server.url("/")) + .set_service("test") + .set_env("staging") .set_tracer_version("v0.1") .set_language("nodejs") .set_language_version("1.0") @@ -1187,7 +1208,7 @@ mod tests { ..Default::default() }]; - let data = rmp_serde::to_vec_named(&vec![trace_chunk]).unwrap(); + let data = tinybytes::Bytes::from(rmp_serde::to_vec_named(&vec![trace_chunk]).unwrap()); // Wait for the info fetcher to get the config while mock_info.hits() == 0 { @@ -1196,14 +1217,16 @@ mod tests { }) } - exporter.send(data.as_slice(), 1).unwrap(); + let result = exporter.send(data, 1); + // Error received because server is returning an empty body. + assert!(result.is_err()); + exporter.shutdown(None).unwrap(); mock_traces.assert(); - //mock_stats.assert(); + mock_stats.assert(); } - /* TODO (APMSP-1583) Re-enable with client stats #[cfg_attr(miri, ignore)] #[test] fn test_shutdown_with_timeout() { @@ -1213,15 +1236,22 @@ mod tests { when.method(POST) .header("Content-type", "application/msgpack") .path("/v0.4/traces"); - then.status(200).body(""); + then.status(200).body( + r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:,env:": 0.8 + } + }"#, + ); }); - // let _mock_stats = server.mock(|when, then| { - // when.method(POST) - // .header("Content-type", "application/msgpack") - // .path("/v0.6/stats"); - // then.delay(Duration::from_secs(10)).status(200).body(""); - // }); + let _mock_stats = server.mock(|when, then| { + when.method(POST) + .header("Content-type", "application/msgpack") + .path("/v0.6/stats"); + then.delay(Duration::from_secs(10)).status(200).body(""); + }); let mock_info = server.mock(|when, then| { when.method(GET).path("/info"); @@ -1234,6 +1264,8 @@ mod tests { let builder = TraceExporterBuilder::default(); let exporter = builder .set_url(&server.url("/")) + .set_service("test") + .set_env("staging") .set_tracer_version("v0.1") .set_language("nodejs") .set_language_version("1.0") @@ -1245,11 +1277,15 @@ mod tests { .unwrap(); let trace_chunk = vec![Span { + service: "test".into(), + name: "test".into(), + resource: "test".into(), + r#type: "test".into(), duration: 10, ..Default::default() }]; - let data = rmp_serde::to_vec_named(&vec![trace_chunk]).unwrap(); + let bytes = tinybytes::Bytes::from(rmp_serde::to_vec_named(&vec![trace_chunk]).unwrap()); // Wait for the info fetcher to get the config while mock_info.hits() == 0 { @@ -1258,14 +1294,16 @@ mod tests { }) } - exporter.send(data.as_slice(), 1).unwrap(); + let result = exporter.send(bytes, 1).unwrap(); + + assert_eq!(result.rate, 0.8); + exporter .shutdown(Some(Duration::from_millis(500))) .unwrap_err(); // The shutdown should timeout mock_traces.assert(); } - */ fn read(socket: &net::UdpSocket) -> String { let mut buf = [0; 1_000]; @@ -1277,6 +1315,8 @@ mod tests { fn build_test_exporter(url: String, dogstatsd_url: String) -> TraceExporter { TraceExporterBuilder::default() .set_url(&url) + .set_service("test") + .set_env("staging") .set_dogstatsd_url(&dogstatsd_url) .set_tracer_version("v0.1") .set_language("nodejs") @@ -1296,7 +1336,7 @@ mod tests { let _mock_traces = fake_agent.mock(|_, then| { then.status(200) .header("content-type", "application/json") - .body("{}"); + .body(r#"{ "rate_by_service": { "service:test,env:staging": 1.0, "service:test,env:prod": 0.3 } }"#); }); let exporter = build_test_exporter( @@ -1314,8 +1354,11 @@ mod tests { ..Default::default() }], ]; - let bytes = rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"); - let _result = exporter.send(&bytes, 1).expect("failed to send trace"); + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + + let _result = exporter.send(bytes, 1).expect("failed to send trace"); assert_eq!( &format!( @@ -1346,9 +1389,10 @@ mod tests { stats_socket.local_addr().unwrap().to_string(), ); - let _result = exporter - .send(b"some_bad_payload", 1) - .expect("failed to send trace"); + let bad_payload = tinybytes::Bytes::copy_from_slice(b"some_bad_payload".as_ref()); + let result = exporter.send(bad_payload, 1); + + assert!(result.is_err()); assert_eq!( &format!( @@ -1381,8 +1425,12 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let bytes = rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"); - let _result = exporter.send(&bytes, 1).expect("failed to send trace"); + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + let result = exporter.send(bytes, 1); + + assert!(result.is_err()); assert_eq!( &format!( @@ -1402,4 +1450,220 @@ mod tests { &read(&stats_socket) ); } + + #[test] + #[cfg_attr(miri, ignore)] + fn agent_response_parse() { + let server = MockServer::start(); + let _agent = server.mock(|_, then| { + then.status(200) + .header("content-type", "application/json") + .body( + r#"{ + "rate_by_service": { + "service:test_service,env:testing":0.5, + "service:another_service,env:testing":1 + } + }"#, + ); + }); + + let exporter = TraceExporterBuilder::default() + .set_url(&server.url("/")) + .set_service("test_service") + .set_env("testing") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build() + .unwrap(); + + let traces: Vec> = vec![vec![Span { + name: BytesString::from_slice(b"test").unwrap(), + ..Default::default() + }]]; + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + let result = exporter.send(bytes, 1).unwrap(); + + assert_eq!(result, AgentResponse::from(0.5)); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agent_response_parse_default() { + let server = MockServer::start(); + let _agent = server.mock(|_, then| { + then.status(200) + .header("content-type", "application/json") + .body( + r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:,env:": 0.8 + } + }"#, + ); + }); + + let exporter = TraceExporterBuilder::default() + .set_url(&server.url("/")) + .set_service("foo") + .set_env("foo-env") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build() + .unwrap(); + + let traces: Vec> = vec![vec![Span { + name: BytesString::from_slice(b"test").unwrap(), + ..Default::default() + }]]; + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + let result = exporter.send(bytes, 1).unwrap(); + + assert_eq!(result, AgentResponse::from(0.8)); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agent_response_empty_array() { + let server = MockServer::start(); + let _agent = server.mock(|_, then| { + then.status(200) + .header("content-type", "application/json") + .body( + r#"{ + "rate_by_service": { + "service:foo,env:staging": 1.0, + "service:,env:": 0.8 + } + }"#, + ); + }); + + let exporter = TraceExporterBuilder::default() + .set_url(&server.url("/")) + .set_service("foo") + .set_env("foo-env") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build() + .unwrap(); + + let traces = vec![0x90]; + let bytes = tinybytes::Bytes::from(traces); + let result = exporter.send(bytes, 1).unwrap(); + + assert_eq!(result, AgentResponse::from(0.8)); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn builder_error() { + let exporter = TraceExporterBuilder::default() + .set_url("") + .set_service("foo") + .set_env("foo-env") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build(); + + assert!(exporter.is_err()); + + let err = match exporter.unwrap_err() { + TraceExporterError::Builder(e) => Some(e), + _ => None, + } + .unwrap(); + + assert_eq!(err, BuilderErrorKind::InvalidUri); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agent_response_error() { + let server = MockServer::start(); + let _agent = server.mock(|_, then| { + then.status(500) + .header("content-type", "application/json") + .body(r#"{ "error": "Unavailable" }"#); + }); + + let exporter = TraceExporterBuilder::default() + .set_url(&server.url("/")) + .set_service("foo") + .set_env("foo-env") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build() + .unwrap(); + + let traces: Vec> = vec![vec![Span { + name: BytesString::from_slice(b"test").unwrap(), + ..Default::default() + }]]; + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + let code = match exporter.send(bytes, 1).unwrap_err() { + TraceExporterError::Request(e) => Some(e.status()), + _ => None, + } + .unwrap(); + + assert_eq!(code, 500); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agent_empty_response_error() { + let server = MockServer::start(); + let _agent = server.mock(|_, then| { + then.status(200) + .header("content-type", "application/json") + .body(""); + }); + + let exporter = TraceExporterBuilder::default() + .set_url(&server.url("/")) + .set_service("foo") + .set_env("foo-env") + .set_tracer_version("v0.1") + .set_language("nodejs") + .set_language_version("1.0") + .set_language_interpreter("v8") + .build() + .unwrap(); + + let traces: Vec> = vec![vec![Span { + name: BytesString::from_slice(b"test").unwrap(), + ..Default::default() + }]]; + let bytes = tinybytes::Bytes::from( + rmp_serde::to_vec_named(&traces).expect("failed to serialize static trace"), + ); + let err = exporter.send(bytes, 1); + + assert!(err.is_err()); + assert_eq!( + match err.unwrap_err() { + TraceExporterError::Agent(e) => Some(e), + _ => None, + }, + Some(AgentErrorKind::EmptyResponse) + ); + } } diff --git a/ddcommon-ffi/src/endpoint.rs b/ddcommon-ffi/src/endpoint.rs index 89d974bfe..8bea28a7b 100644 --- a/ddcommon-ffi/src/endpoint.rs +++ b/ddcommon-ffi/src/endpoint.rs @@ -91,11 +91,17 @@ mod tests { ("file:/// file / with/weird chars 🤡", true), ("file://./", true), ("unix://./", true), + ("http://2001:db8:1::2:8126/", false), + ("http://[2001:db8:1::2]:8126/", true), ]; for (input, expected) in cases { - let actual = ddog_endpoint_from_url(CharSlice::from(input)).is_some(); + let ep = ddog_endpoint_from_url(CharSlice::from(input)); + let actual = ep.is_some(); assert_eq!(actual, expected); + if actual { + ddog_endpoint_drop(ep.unwrap()); + } } } diff --git a/ddcommon-ffi/src/handle.rs b/ddcommon-ffi/src/handle.rs new file mode 100644 index 000000000..c71cca5d5 --- /dev/null +++ b/ddcommon-ffi/src/handle.rs @@ -0,0 +1,64 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Context; + +/// Represents an object that should only be referred to by its handle. +/// Do not access its member for any reason, only use the C API functions on this struct. +#[repr(C)] +pub struct Handle { + // This may be null, but if not it will point to a valid . + inner: *mut T, +} + +pub trait ToInner { + /// # Safety + /// The Handle must hold a valid `inner` which has been allocated and not freed. + unsafe fn to_inner_mut(&mut self) -> anyhow::Result<&mut T>; + /// # Safety + /// The Handle must hold a valid `inner` [return OK(inner)], or null [returns Error]. + unsafe fn take(&mut self) -> anyhow::Result>; +} + +impl ToInner for *mut Handle { + unsafe fn to_inner_mut(&mut self) -> anyhow::Result<&mut T> { + self.as_mut().context("Null pointer")?.to_inner_mut() + } + + unsafe fn take(&mut self) -> anyhow::Result> { + self.as_mut().context("Null pointer")?.take() + } +} + +impl ToInner for Handle { + unsafe fn to_inner_mut(&mut self) -> anyhow::Result<&mut T> { + self.inner + .as_mut() + .context("inner pointer was null, indicates use after free") + } + + unsafe fn take(&mut self) -> anyhow::Result> { + // Leaving a null will help with double-free issues that can arise in C. + // Of course, it's best to never get there in the first place! + let raw = std::mem::replace(&mut self.inner, std::ptr::null_mut()); + anyhow::ensure!( + !raw.is_null(), + "inner pointer was null, indicates use after free" + ); + Ok(Box::from_raw(raw)) + } +} + +impl From for Handle { + fn from(value: T) -> Self { + Self { + inner: Box::into_raw(Box::new(value)), + } + } +} + +impl Drop for Handle { + fn drop(&mut self) { + drop(unsafe { self.take() }) + } +} diff --git a/ddcommon-ffi/src/lib.rs b/ddcommon-ffi/src/lib.rs index 5163ca1c0..98d59957f 100644 --- a/ddcommon-ffi/src/lib.rs +++ b/ddcommon-ffi/src/lib.rs @@ -5,15 +5,20 @@ mod error; pub mod array_queue; pub mod endpoint; +pub mod handle; pub mod option; +pub mod result; pub mod slice; pub mod string; pub mod tags; pub mod timespec; +pub mod utils; pub mod vec; pub use error::*; +pub use handle::*; pub use option::*; +pub use result::*; pub use slice::{CharSlice, Slice}; pub use string::*; pub use timespec::*; diff --git a/ddcommon-ffi/src/option.rs b/ddcommon-ffi/src/option.rs index 41ab5af46..78751b803 100644 --- a/ddcommon-ffi/src/option.rs +++ b/ddcommon-ffi/src/option.rs @@ -3,6 +3,7 @@ #[repr(C)] #[derive(Debug, PartialEq, Eq)] +#[must_use] pub enum Option { Some(T), None, @@ -19,6 +20,20 @@ impl Option { Option::None => None, } } + + pub fn as_mut(&mut self) -> Option<&mut T> { + match *self { + Option::Some(ref mut x) => Option::Some(x), + Option::None => Option::None, + } + } + + pub fn unwrap_none(self) { + match self { + Option::Some(_) => panic!("Called ffi::Option::unwrap_none but option was Some(_)"), + Option::None => {} + } + } } impl From> for std::option::Option { diff --git a/ddcommon-ffi/src/result.rs b/ddcommon-ffi/src/result.rs new file mode 100644 index 000000000..37c3a56b2 --- /dev/null +++ b/ddcommon-ffi/src/result.rs @@ -0,0 +1,51 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::Error; + +/// A generic result type for when an operation may fail, +/// but there's nothing to return in the case of success. +#[repr(C)] +pub enum VoidResult { + Ok( + /// Do not use the value of Ok. This value only exists to overcome + /// Rust -> C code generation. + bool, + ), + Err(Error), +} + +impl From> for VoidResult { + fn from(value: anyhow::Result<()>) -> Self { + match value { + Ok(_) => Self::Ok(true), + Err(err) => Self::Err(err.into()), + } + } +} + +/// A generic result type for when an operation may fail, +/// or may return in case of success. +#[repr(C)] +pub enum Result { + Ok(T), + Err(Error), +} + +impl Result { + pub fn unwrap(self) -> T { + match self { + Self::Ok(v) => v, + Self::Err(err) => panic!("{err}"), + } + } +} + +impl From> for Result { + fn from(value: anyhow::Result) -> Self { + match value { + Ok(v) => Self::Ok(v), + Err(err) => Self::Err(err.into()), + } + } +} diff --git a/ddcommon-ffi/src/slice.rs b/ddcommon-ffi/src/slice.rs index 4cc3ff417..8e84e21d9 100644 --- a/ddcommon-ffi/src/slice.rs +++ b/ddcommon-ffi/src/slice.rs @@ -72,6 +72,15 @@ pub trait AsBytes<'a> { std::str::from_utf8(self.as_bytes()) } + fn try_to_string(&self) -> Result { + Ok(self.try_to_utf8()?.to_string()) + } + + #[inline] + fn try_to_string_option(&self) -> Result, Utf8Error> { + Ok(Some(self.try_to_string()?).filter(|x| !x.is_empty())) + } + #[inline] fn to_utf8_lossy(&self) -> Cow<'a, str> { String::from_utf8_lossy(self.as_bytes()) diff --git a/ddcommon-ffi/src/utils.rs b/ddcommon-ffi/src/utils.rs new file mode 100644 index 000000000..94cd8f8eb --- /dev/null +++ b/ddcommon-ffi/src/utils.rs @@ -0,0 +1,30 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// Wraps a C-FFI function in standard form +/// Expects the function to return a result type that implements into and to be decorated with +/// #[named]. +#[macro_export] +macro_rules! wrap_with_ffi_result { + ($body:block) => {{ + use anyhow::Context; + (|| $body)() + .context(concat!(function_name!(), " failed")) + .into() + }}; +} + +/// Wraps a C-FFI function in standard form. +/// Expects the function to return a VoidResult and to be decorated with #[named]. +#[macro_export] +macro_rules! wrap_with_void_ffi_result { + ($body:block) => {{ + use anyhow::Context; + (|| { + $body; + anyhow::Ok(()) + })() + .context(concat!(function_name!(), " failed")) + .into() + }}; +} diff --git a/ddcommon/Cargo.toml b/ddcommon/Cargo.toml index c947a6f35..df68d7e9d 100644 --- a/ddcommon/Cargo.toml +++ b/ddcommon/Cargo.toml @@ -27,7 +27,7 @@ hyper = { version = "0.14", features = [ "backports", "deprecated" ], default-features = false } -cc = "=1.1.31" +cc = "1.1.31" hyper-util = "0.1.3" lazy_static = "1.4" log = { version = "0.4" } @@ -71,3 +71,6 @@ maplit = "1.0" [features] default = [] use_webpki_roots = ["hyper-rustls/webpki-roots"] +# Enable this feature to enable stubbing of cgroup +# php directly import this crate and uses functions gated by this feature for their test +cgroup_testing = [] diff --git a/ddcommon/src/entity_id/mod.rs b/ddcommon/src/entity_id/mod.rs index 5e5dea719..214134993 100644 --- a/ddcommon/src/entity_id/mod.rs +++ b/ddcommon/src/entity_id/mod.rs @@ -60,6 +60,15 @@ const EXTERNAL_ENV_ENVIRONMENT_VARIABLE: &str = "DD_EXTERNAL_ENV"; #[cfg(unix)] mod unix; +/// Set the path to cgroup file to mock it during tests +#[cfg(feature = "cgroup_testing")] +pub fn set_cgroup_file(_file: String) { + #[cfg(unix)] + { + unix::set_cgroup_file(_file) + } +} + /// Returns the `container_id` if available in the cgroup file, otherwise returns `None` pub fn get_container_id() -> Option<&'static str> { #[cfg(unix)] @@ -92,23 +101,3 @@ pub fn get_external_env() -> Option<&'static str> { } DD_EXTERNAL_ENV.as_deref() } - -/// Set the path to cgroup file to mock it during tests -/// # Safety -/// Must not be called in multi-threaded contexts -pub unsafe fn set_cgroup_file(_file: String) { - #[cfg(unix)] - { - unix::set_cgroup_file(_file) - } -} - -/// Set cgroup mount path to mock during tests -/// # Safety -/// Must not be called in multi-threaded contexts -pub unsafe fn set(_path: String) { - #[cfg(unix)] - { - unix::set_cgroup_mount_path(_path) - } -} diff --git a/ddcommon/src/entity_id/unix/mod.rs b/ddcommon/src/entity_id/unix/mod.rs index 7e1e687e4..1c2ae24ec 100644 --- a/ddcommon/src/entity_id/unix/mod.rs +++ b/ddcommon/src/entity_id/unix/mod.rs @@ -12,14 +12,13 @@ mod container_id; const DEFAULT_CGROUP_PATH: &str = "/proc/self/cgroup"; const DEFAULT_CGROUP_MOUNT_PATH: &str = "/sys/fs/cgroup"; +/// stores overridable cgroup path - used in end-to-end testing to "stub" cgroup values +#[cfg(feature = "cgroup_testing")] +static TESTING_CGROUP_PATH: std::sync::OnceLock = std::sync::OnceLock::new(); + /// the base controller used to identify the cgroup v1 mount point in the cgroupMounts map. const CGROUP_V1_BASE_CONTROLLER: &str = "memory"; -/// stores overridable cgroup path - used in end-to-end testing to "stub" cgroup values -static mut TESTING_CGROUP_PATH: Option = None; -/// stores overridable cgroup mount path -static mut TESTING_CGROUP_MOUNT_PATH: Option = None; - #[derive(Debug, Clone, PartialEq)] pub enum CgroupFileParsingError { ContainerIdNotFound, @@ -57,38 +56,24 @@ fn compute_entity_id( ) } -fn get_cgroup_path() -> &'static str { - // Safety: we assume set_cgroup_file is not called when it shouldn't - #[allow(static_mut_refs)] - unsafe { - TESTING_CGROUP_PATH - .as_deref() - .unwrap_or(DEFAULT_CGROUP_PATH) - } -} - -fn get_cgroup_mount_path() -> &'static str { - // Safety: we assume set_cgroup_file is not called when it shouldn't - #[allow(static_mut_refs)] - unsafe { - TESTING_CGROUP_MOUNT_PATH - .as_deref() - .unwrap_or(DEFAULT_CGROUP_MOUNT_PATH) - } +/// Set cgroup mount path to mock during tests +#[cfg(feature = "cgroup_testing")] +pub fn set_cgroup_file(path: String) { + let _ = TESTING_CGROUP_PATH.set(path); } -/// Set the path to cgroup file to mock it during tests -/// # Safety -/// Must not be called in multi-threaded contexts -pub unsafe fn set_cgroup_file(file: String) { - TESTING_CGROUP_PATH = Some(file) +fn get_cgroup_path() -> &'static str { + #[cfg(feature = "cgroup_testing")] + return TESTING_CGROUP_PATH + .get() + .map(std::ops::Deref::deref) + .unwrap_or(DEFAULT_CGROUP_PATH); + #[cfg(not(feature = "cgroup_testing"))] + return DEFAULT_CGROUP_PATH; } -/// Set cgroup mount path to mock during tests -/// # Safety -/// Must not be called in multi-threaded contexts -pub unsafe fn set_cgroup_mount_path(path: String) { - TESTING_CGROUP_MOUNT_PATH = Some(path) +fn get_cgroup_mount_path() -> &'static str { + DEFAULT_CGROUP_MOUNT_PATH } /// Returns the `container_id` if available in the cgroup file, otherwise returns `None` diff --git a/ddsketch/build.rs b/ddsketch/build.rs index 189e71071..8f7ddf126 100644 --- a/ddsketch/build.rs +++ b/ddsketch/build.rs @@ -36,6 +36,10 @@ fn main() -> Result<(), Box> { prepend_to_file(HEADER.as_bytes(), &output_path.join("pb.rs"))?; } + #[cfg(not(feature = "generate-protobuf"))] + { + println!("cargo:rerun-if-changed=build.rs"); + } Ok(()) } diff --git a/ddtelemetry-ffi/src/lib.rs b/ddtelemetry-ffi/src/lib.rs index d3ff23e91..bc8c6aac2 100644 --- a/ddtelemetry-ffi/src/lib.rs +++ b/ddtelemetry-ffi/src/lib.rs @@ -254,14 +254,16 @@ mod tests { ddog_telemetry_builder_with_bool_config_telemetry_debug_logging_enabled( &mut builder, true, - ); + ) + .unwrap_none(); let mut handle: MaybeUninit> = MaybeUninit::uninit(); - ddog_telemetry_builder_run(builder, NonNull::new(&mut handle).unwrap().cast()); + ddog_telemetry_builder_run(builder, NonNull::new(&mut handle).unwrap().cast()) + .unwrap_none(); let handle = handle.assume_init(); - ddog_telemetry_handle_start(&handle); - ddog_telemetry_handle_stop(&handle); + ddog_telemetry_handle_start(&handle).unwrap_none(); + ddog_telemetry_handle_stop(&handle).unwrap_none(); ddog_telemetry_handle_wait_for_shutdown(handle); } } @@ -297,13 +299,15 @@ mod tests { ddog_telemetry_builder_with_bool_config_telemetry_debug_logging_enabled( &mut builder, true, - ); + ) + .unwrap_none(); let mut handle: MaybeUninit> = MaybeUninit::uninit(); ddog_telemetry_builder_run_metric_logs( builder, NonNull::new(&mut handle).unwrap().cast(), - ); + ) + .unwrap_none(); let handle = handle.assume_init(); assert!(matches!( @@ -329,7 +333,7 @@ mod tests { true, MetricNamespace::Apm, ); - ddog_telemetry_handle_add_point(&handle, &context_key, 1.0); + ddog_telemetry_handle_add_point(&handle, &context_key, 1.0).unwrap_none(); assert_eq!(ddog_telemetry_handle_stop(&handle), MaybeError::None); ddog_telemetry_handle_wait_for_shutdown(handle); diff --git a/ddtelemetry/examples/tm-ping.rs b/ddtelemetry/examples/tm-ping.rs index c60244f17..cbea147f4 100644 --- a/ddtelemetry/examples/tm-ping.rs +++ b/ddtelemetry/examples/tm-ping.rs @@ -35,6 +35,7 @@ fn build_request<'a>( tracer_time: SystemTime::UNIX_EPOCH.elapsed().map_or(0, |d| d.as_secs()), runtime_id: "runtime_id", seq_id: seq_id(), + origin: Some("tm-ping"), application, host, payload, diff --git a/ddtelemetry/examples/tm-send-sketch.rs b/ddtelemetry/examples/tm-send-sketch.rs index 3585ad075..ee4bfe464 100644 --- a/ddtelemetry/examples/tm-send-sketch.rs +++ b/ddtelemetry/examples/tm-send-sketch.rs @@ -33,6 +33,7 @@ fn build_request<'a>( tracer_time: SystemTime::UNIX_EPOCH.elapsed().map_or(0, |d| d.as_secs()), runtime_id: "runtime_id", seq_id: seq_id(), + origin: Some("tm-send-sketch"), application, host, payload, diff --git a/ddtelemetry/src/data/common.rs b/ddtelemetry/src/data/common.rs index a9e7898f6..032e4e1c4 100644 --- a/ddtelemetry/src/data/common.rs +++ b/ddtelemetry/src/data/common.rs @@ -30,6 +30,8 @@ pub struct Telemetry<'a> { pub seq_id: u64, pub application: &'a Application, pub host: &'a Host, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin: Option<&'a str>, #[serde(flatten)] pub payload: &'a Payload, } diff --git a/ddtelemetry/src/data/payloads.rs b/ddtelemetry/src/data/payloads.rs index 3dccc035f..7acc34c1b 100644 --- a/ddtelemetry/src/data/payloads.rs +++ b/ddtelemetry/src/data/payloads.rs @@ -75,9 +75,9 @@ pub struct Log { #[serde(default)] pub stack_trace: Option, - #[serde(skip_serializing_if = "String::is_empty", default)] + #[serde(default)] pub tags: String, - #[serde(skip_serializing_if = "std::ops::Not::not", default)] + #[serde(default)] pub is_sensitive: bool, } diff --git a/ddtelemetry/src/worker/mod.rs b/ddtelemetry/src/worker/mod.rs index 9debd0cd5..0cbab21a0 100644 --- a/ddtelemetry/src/worker/mod.rs +++ b/ddtelemetry/src/worker/mod.rs @@ -100,7 +100,7 @@ pub enum LifecycleAction { #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct LogIdentifier { // Collisions? Never heard of them - indentifier: u64, + pub indentifier: u64, } // Holds the current state of the telemetry worker @@ -241,10 +241,10 @@ impl TelemetryWorker { Lifecycle(Start) => { if !self.data.started { self.deadlines - .schedule_event(LifecycleAction::FlushData) + .schedule_event(LifecycleAction::FlushMetricAggr) .unwrap(); self.deadlines - .schedule_event(LifecycleAction::FlushMetricAggr) + .schedule_event(LifecycleAction::FlushData) .unwrap(); self.data.started = true; } @@ -265,7 +265,7 @@ impl TelemetryWorker { .unwrap(); } Lifecycle(FlushData) => { - if !self.data.started { + if !(self.data.started || self.config.restartable) { return CONTINUE; } let batch = self.build_observability_batch(); @@ -296,7 +296,9 @@ impl TelemetryWorker { self.log_err(&e); } self.data.started = false; - self.deadlines.clear_pending(); + if !self.config.restartable { + self.deadlines.clear_pending(); + } return BREAK; } CollectStats(stats_sender) => { @@ -341,10 +343,11 @@ impl TelemetryWorker { Err(err) => self.log_err(&err), } self.deadlines - .schedule_event(LifecycleAction::FlushData) + .schedule_event(LifecycleAction::FlushMetricAggr) .unwrap(); + // flush data should be last to previously flushed metrics are sent self.deadlines - .schedule_event(LifecycleAction::FlushMetricAggr) + .schedule_event(LifecycleAction::FlushData) .unwrap(); self.data.started = true; } @@ -368,7 +371,7 @@ impl TelemetryWorker { .unwrap(); } Lifecycle(FlushData) => { - if !self.data.started { + if !(self.data.started || self.config.restartable) { return CONTINUE; } let mut batch = self.build_app_events_batch(); @@ -427,38 +430,35 @@ impl TelemetryWorker { let obsevability_events = self.build_observability_batch(); - future::join_all( - [ - Some(self.build_request(&data::Payload::MessageBatch(app_events))), - if obsevability_events.is_empty() { - None - } else { - Some( - self.build_request(&data::Payload::MessageBatch( - obsevability_events, - )), - ) - }, - ] - .into_iter() - .flatten() - .filter_map(|r| match r { - Ok(r) => Some(r), - Err(e) => { - self.log_err(&e); - None - } - }) - .map(|r| async { - if let Err(e) = self.send_request(r).await { - self.log_err(&e); + let mut payloads = vec![data::Payload::MessageBatch(app_events)]; + if !obsevability_events.is_empty() { + payloads.push(data::Payload::MessageBatch(obsevability_events)); + } + + let self_arc = Arc::new(tokio::sync::RwLock::new(&mut *self)); + let futures = payloads.into_iter().map(|payload| { + let self_arc = self_arc.clone(); + async move { + // This is different from the non-functional: + // match self_arc.read().await.send_payload(&payload).await { ... } + // presumably because the temp read guard would live till end of match + let res = { + let self_rguard = self_arc.read().await; + self_rguard.send_payload(&payload).await + }; + match res { + Ok(()) => self_arc.write().await.payload_sent_success(&payload), + Err(err) => self_arc.read().await.log_err(&err), } - }), - ) - .await; + } + }); + future::join_all(futures).await; self.data.started = false; - self.deadlines.clear_pending(); + if !self.config.restartable { + self.deadlines.clear_pending(); + } + return BREAK; } CollectStats(stats_sender) => { @@ -470,7 +470,7 @@ impl TelemetryWorker { } // Builds telemetry payloads containing lifecycle events - fn build_app_events_batch(&self) -> Vec { + fn build_app_events_batch(&mut self) -> Vec { let mut payloads = Vec::new(); if self.data.dependencies.flush_not_empty() { @@ -636,6 +636,7 @@ impl TelemetryWorker { runtime_id: &self.runtime_id, seq_id, host: &self.data.host, + origin: None, application: &self.data.app, payload, }; diff --git a/docs/RFCs/0006-crashtraker-incomplete-stacktraces.md b/docs/RFCs/0006-crashtraker-incomplete-stacktraces.md new file mode 100644 index 000000000..29df268e9 --- /dev/null +++ b/docs/RFCs/0006-crashtraker-incomplete-stacktraces.md @@ -0,0 +1,45 @@ +# RFC 0006: Crashtracker Structured Log Format (Version 1.1). Adds incomplete stacktraces. + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [IETF RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). + +## Summary + +This document describes version 1.1 of the crashinfo data format. + +## Motivation + +The `libdatadog` crashtracker detects program crashes. +It automatically collects information relevant to the characterizing and debugging the crash, including stack-traces, the crash-type (e.g. SIGSIGV, SIGBUS, etc) crash, the library version, etc. +In some cases, these stack traces may be incomplete. +This can occur intentionally, when deep traces are truncated for performance reasons, or unintentionally, if stack collection fails, e.g. because the stack is corrupted. +Having an (optional) flag on the stack trace allows the consumer of the crash report to know that frames may be missing from the trace. + +## Proposed format + +The format is an extension of the [1.0 json schema](0005-crashtracker-structured-log-format.md), with the following changes. +The updated schema is given in Appendix A. +Any field not listed as "Required" is optional. +Consumers MUST accept json with elided optional fields. + +### Fields + +- `data_schema_version`: **[required]** \*\*[UPDATED] + A string containing the semver ID of the crashtracker data schema ("1.1" for the current version). + +### Stacktraces + +A stacktrace consists of + +- `format`: **[required]** + An identifier describing the format of the stack trace. + Allows for extensibility to support different stack trace formats. + The format described below is identified using the string "Datadog Crashtracker 1.0" +- `frames`: **[required]** + An array of `StackFrame`, described below. + Note that each inlined function gets its own stack frame in this schema. +- `incomplete`: **[optional]** **[NEW]** + A boolean denoting whether the stacktrace may be missing frames, either due to intentional truncation, or an inability to fully collect a corrupted stack. + +## Appendix A: Json Schema + +[Available here](artifacts/0006-crashtracker-schema.json) diff --git a/docs/RFCs/artifacts/0006-crashtracker-schema.json b/docs/RFCs/artifacts/0006-crashtracker-schema.json new file mode 100644 index 000000000..0be82174b --- /dev/null +++ b/docs/RFCs/artifacts/0006-crashtracker-schema.json @@ -0,0 +1,452 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CrashInfo", + "type": "object", + "required": [ + "data_schema_version", + "error", + "incomplete", + "metadata", + "os_info", + "timestamp", + "uuid" + ], + "properties": { + "counters": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "int64" + } + }, + "data_schema_version": { + "type": "string" + }, + "error": { + "$ref": "#/definitions/ErrorData" + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "fingerprint": { + "type": [ + "string", + "null" + ] + }, + "incomplete": { + "type": "boolean" + }, + "log_messages": { + "type": "array", + "items": { + "type": "string" + } + }, + "metadata": { + "$ref": "#/definitions/Metadata" + }, + "os_info": { + "$ref": "#/definitions/OsInfo" + }, + "proc_info": { + "anyOf": [ + { + "$ref": "#/definitions/ProcInfo" + }, + { + "type": "null" + } + ] + }, + "sig_info": { + "anyOf": [ + { + "$ref": "#/definitions/SigInfo" + }, + { + "type": "null" + } + ] + }, + "span_ids": { + "type": "array", + "items": { + "$ref": "#/definitions/Span" + } + }, + "timestamp": { + "type": "string" + }, + "trace_ids": { + "type": "array", + "items": { + "$ref": "#/definitions/Span" + } + }, + "uuid": { + "type": "string" + } + }, + "definitions": { + "BuildIdType": { + "type": "string", + "enum": [ + "GNU", + "GO", + "PDB", + "PE", + "SHA1" + ] + }, + "ErrorData": { + "type": "object", + "required": [ + "is_crash", + "kind", + "source_type", + "stack" + ], + "properties": { + "is_crash": { + "type": "boolean" + }, + "kind": { + "$ref": "#/definitions/ErrorKind" + }, + "message": { + "type": [ + "string", + "null" + ] + }, + "source_type": { + "$ref": "#/definitions/SourceType" + }, + "stack": { + "$ref": "#/definitions/StackTrace" + }, + "threads": { + "type": "array", + "items": { + "$ref": "#/definitions/ThreadData" + } + } + } + }, + "ErrorKind": { + "type": "string", + "enum": [ + "Panic", + "UnhandledException", + "UnixSignal" + ] + }, + "FileType": { + "type": "string", + "enum": [ + "APK", + "ELF", + "PDB" + ] + }, + "Metadata": { + "type": "object", + "required": [ + "family", + "library_name", + "library_version" + ], + "properties": { + "family": { + "type": "string" + }, + "library_name": { + "type": "string" + }, + "library_version": { + "type": "string" + }, + "tags": { + "description": "A list of \"key:value\" tuples.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "OsInfo": { + "type": "object", + "required": [ + "architecture", + "bitness", + "os_type", + "version" + ], + "properties": { + "architecture": { + "type": "string" + }, + "bitness": { + "type": "string" + }, + "os_type": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "ProcInfo": { + "type": "object", + "required": [ + "pid" + ], + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "SiCodes": { + "description": "See https://man7.org/linux/man-pages/man2/sigaction.2.html", + "type": "string", + "enum": [ + "BUS_ADRALN", + "BUS_ADRERR", + "BUS_MCEERR_AO", + "BUS_MCEERR_AR", + "BUS_OBJERR", + "SEGV_ACCERR", + "SEGV_BNDERR", + "SEGV_MAPERR", + "SEGV_PKUERR", + "SI_ASYNCIO", + "SI_KERNEL", + "SI_MESGQ", + "SI_QUEUE", + "SI_SIGIO", + "SI_TIMER", + "SI_TKILL", + "SI_USER", + "SYS_SECCOMP" + ] + }, + "SigInfo": { + "type": "object", + "required": [ + "si_code", + "si_code_human_readable", + "si_signo", + "si_signo_human_readable" + ], + "properties": { + "si_addr": { + "type": [ + "string", + "null" + ] + }, + "si_code": { + "type": "integer", + "format": "int32" + }, + "si_code_human_readable": { + "$ref": "#/definitions/SiCodes" + }, + "si_signo": { + "type": "integer", + "format": "int32" + }, + "si_signo_human_readable": { + "$ref": "#/definitions/SignalNames" + } + } + }, + "SignalNames": { + "description": "See https://man7.org/linux/man-pages/man7/signal.7.html", + "type": "string", + "enum": [ + "SIGABRT", + "SIGBUS", + "SIGSEGV", + "SIGSYS" + ] + }, + "SourceType": { + "type": "string", + "enum": [ + "Crashtracking" + ] + }, + "Span": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "thread_name": { + "type": [ + "string", + "null" + ] + } + } + }, + "StackFrame": { + "type": "object", + "properties": { + "build_id": { + "type": [ + "string", + "null" + ] + }, + "build_id_type": { + "anyOf": [ + { + "$ref": "#/definitions/BuildIdType" + }, + { + "type": "null" + } + ] + }, + "column": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "file": { + "type": [ + "string", + "null" + ] + }, + "file_type": { + "anyOf": [ + { + "$ref": "#/definitions/FileType" + }, + { + "type": "null" + } + ] + }, + "function": { + "type": [ + "string", + "null" + ] + }, + "ip": { + "type": [ + "string", + "null" + ] + }, + "line": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "module_base_address": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "relative_address": { + "type": [ + "string", + "null" + ] + }, + "sp": { + "type": [ + "string", + "null" + ] + }, + "symbol_address": { + "type": [ + "string", + "null" + ] + } + } + }, + "StackTrace": { + "type": "object", + "required": [ + "format", + "frames", + "incomplete" + ], + "properties": { + "format": { + "type": "string" + }, + "frames": { + "type": "array", + "items": { + "$ref": "#/definitions/StackFrame" + } + }, + "incomplete": { + "type": "boolean" + } + } + }, + "ThreadData": { + "type": "object", + "required": [ + "crashed", + "name", + "stack" + ], + "properties": { + "crashed": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "stack": { + "$ref": "#/definitions/StackTrace" + }, + "state": { + "type": [ + "string", + "null" + ] + } + } + } + } +} diff --git a/dogstatsd-client/src/lib.rs b/dogstatsd-client/src/lib.rs index 1508280c0..ffa186aa8 100644 --- a/dogstatsd-client/src/lib.rs +++ b/dogstatsd-client/src/lib.rs @@ -68,6 +68,7 @@ pub enum DogStatsDAction<'a, T: AsRef, V: IntoIterator> { } /// A dogstatsd-client that flushes stats to a given endpoint. Use `new_flusher` to build one. +#[derive(Debug)] pub struct Client { client: StatsdClient, } diff --git a/dogstatsd/Cargo.toml b/dogstatsd/Cargo.toml index 51e1cb16a..bc3de340d 100644 --- a/dogstatsd/Cargo.toml +++ b/dogstatsd/Cargo.toml @@ -9,8 +9,8 @@ license.workspace = true bench = false [dependencies] -datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki-backport/", rev = "3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" } -ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki-backport/", rev = "3c5d87ab82dea4a1a98ef0c60fb3659ca35c2751" } +datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222" } +ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222" } hashbrown = { version = "0.14.3", default-features = false, features = ["inline-more"] } protobuf = { version = "3.5.0", default-features = false } ustr = { version = "1.0.0", default-features = false } diff --git a/dogstatsd/src/datadog.rs b/dogstatsd/src/datadog.rs index e73024458..95608fed8 100644 --- a/dogstatsd/src/datadog.rs +++ b/dogstatsd/src/datadog.rs @@ -8,6 +8,7 @@ use protobuf::Message; use reqwest; use serde::{Serialize, Serializer}; use serde_json; +use std::time::Duration; use tracing::{debug, error}; /// Interface for the `DogStatsD` metrics intake API. @@ -20,8 +21,13 @@ pub struct DdApi { impl DdApi { #[must_use] - pub fn new(api_key: String, site: String, https_proxy: Option) -> Self { - let client = match Self::build_client(https_proxy) { + pub fn new( + api_key: String, + site: String, + https_proxy: Option, + timeout: Duration, + ) -> Self { + let client = match Self::build_client(https_proxy, timeout) { Ok(client) => client, Err(e) => { error!("Unable to parse proxy URL, no proxy will be used. {:?}", e); @@ -104,8 +110,11 @@ impl DdApi { }; } - fn build_client(https_proxy: Option) -> Result { - let mut builder = reqwest::Client::builder(); + fn build_client( + https_proxy: Option, + timeout: Duration, + ) -> Result { + let mut builder = reqwest::Client::builder().timeout(timeout); if let Some(proxy) = https_proxy { builder = builder.proxy(reqwest::Proxy::https(proxy)?); } diff --git a/dogstatsd/src/flusher.rs b/dogstatsd/src/flusher.rs index 601d223b7..fe40f32c7 100644 --- a/dogstatsd/src/flusher.rs +++ b/dogstatsd/src/flusher.rs @@ -4,6 +4,8 @@ use crate::aggregator::Aggregator; use crate::datadog; use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tracing::debug; pub struct Flusher { dd_api: datadog::DdApi, @@ -23,8 +25,9 @@ impl Flusher { aggregator: Arc>, site: String, https_proxy: Option, + timeout: Duration, ) -> Self { - let dd_api = datadog::DdApi::new(api_key, site, https_proxy); + let dd_api = datadog::DdApi::new(api_key, site, https_proxy, timeout); Flusher { dd_api, aggregator } } @@ -36,6 +39,14 @@ impl Flusher { aggregator.consume_distributions(), ) }; + + let n_series = all_series.len(); + let n_distributions = all_distributions.len(); + + debug!("Flushing {n_series} series and {n_distributions} distributions"); + + // TODO: client timeout is for each invocation, so NxM times with N time series batches and + // M distro batches for a_batch in all_series { self.dd_api.ship_series(&a_batch).await; // TODO(astuyve) retry and do not panic diff --git a/dogstatsd/src/metric.rs b/dogstatsd/src/metric.rs index 5132f5934..2f89c8754 100644 --- a/dogstatsd/src/metric.rs +++ b/dogstatsd/src/metric.rs @@ -104,10 +104,10 @@ impl SortedTags { pub(crate) fn to_resources(&self) -> Vec { let mut resources = Vec::with_capacity(constants::MAX_TAGS); - for (name, kind) in &self.values { + for (kind, name) in &self.values { let resource = datadog::Resource { - name: name.as_str(), - kind: kind.as_str(), + name: name.as_str(), // val + kind: kind.as_str(), // key }; resources.push(resource); } @@ -450,6 +450,21 @@ mod tests { assert_eq!(id1, id2); } + + #[test] + #[cfg_attr(miri, ignore)] + fn resources_key_val_order(tags in metric_tags()) { + let sorted_tags = SortedTags { values: tags.into_iter() + .map(|(kind, name)| (Ustr::from(&kind), Ustr::from(&name))) + .collect() }; + + let resources = sorted_tags.to_resources(); + + for (i, resource) in resources.iter().enumerate() { + assert_eq!(resource.kind, sorted_tags.values[i].0); + assert_eq!(resource.name, sorted_tags.values[i].1); + } + } } #[test] diff --git a/dogstatsd/tests/integration_test.rs b/dogstatsd/tests/integration_test.rs index 7ffce5411..40ef9f059 100644 --- a/dogstatsd/tests/integration_test.rs +++ b/dogstatsd/tests/integration_test.rs @@ -42,6 +42,7 @@ async fn dogstatsd_server_ships_series() { Arc::clone(&metrics_aggr), mock_server.url(), None, + std::time::Duration::from_secs(5), ); let server_address = "127.0.0.1:18125"; diff --git a/examples/ffi/crashinfo.cpp b/examples/ffi/crashinfo.cpp index a5bd63857..b1541259a 100644 --- a/examples/ffi/crashinfo.cpp +++ b/examples/ffi/crashinfo.cpp @@ -15,129 +15,145 @@ extern "C" { #include static ddog_CharSlice to_slice_c_char(const char *s) { return {.ptr = s, .len = strlen(s)}; } -static ddog_CharSlice to_slice_c_char(const char *s, std::size_t size) { return {.ptr = s, .len = size}; } +static ddog_CharSlice to_slice_c_char(const char *s, std::size_t size) { + return {.ptr = s, .len = size}; +} static ddog_CharSlice to_slice_string(std::string const &s) { return {.ptr = s.data(), .len = s.length()}; } -template -static ddog_ByteSlice to_byte_slice(T const& c) { - return {.ptr = reinterpret_cast(c.data()), .len = c.size()}; -} - -struct Deleter { - void operator()(ddog_crasht_CrashInfo *object) { ddog_crasht_CrashInfo_drop(object); } -}; - void print_error(const char *s, const ddog_Error &err) { auto charslice = ddog_Error_message(&err); printf("%s (%.*s)\n", s, static_cast(charslice.len), charslice.ptr); } -void check_result(ddog_crasht_Result result, const char *msg) { - if (result.tag != DDOG_CRASHT_RESULT_OK) { - print_error(msg, result.err); - ddog_Error_drop(&result.err); - exit(EXIT_FAILURE); - } -} - -void add_stacktrace(std::unique_ptr &crashinfo) { - - // Collect things into vectors so they stay alive till the function exits - constexpr std::size_t nb_elements = 20; - std::vector> functions_and_filenames{nb_elements}; - for (uintptr_t i = 0; i < nb_elements; ++i) { - functions_and_filenames.push_back({"func_" + std::to_string(i), "/path/to/code/file_" + std::to_string(i)}); +#define CHECK_RESULT(typ, ok_tag) \ + void check_result(typ result, const char *msg) { \ + if (result.tag != ok_tag) { \ + print_error(msg, result.err); \ + ddog_Error_drop(&result.err); \ + exit(EXIT_FAILURE); \ + } \ } - std::vector names{nb_elements}; - for (auto i = 0; i < nb_elements; i++) { - auto const& [function_name, filename] = functions_and_filenames[i]; - - auto function_name_slice = to_slice_string(function_name); - auto res = ddog_crasht_demangle(function_name_slice, DDOG_CRASHT_DEMANGLE_OPTIONS_COMPLETE); - if (res.tag == DDOG_CRASHT_STRING_WRAPPER_RESULT_OK) - { - auto string_result = res.ok.message; - function_name_slice = to_slice_c_char((const char*)string_result.ptr, string_result.len); - } - - names.push_back({.colno = ddog_Option_U32_some(i), - .filename = to_slice_string(filename), - .lineno = ddog_Option_U32_some(2 * i + 3), - .name = function_name_slice}); +CHECK_RESULT(ddog_VoidResult, DDOG_VOID_RESULT_OK) +CHECK_RESULT(ddog_Vec_Tag_PushResult, DDOG_VEC_TAG_PUSH_RESULT_OK) + +#define EXTRACT_RESULT(typ, ok_tag) \ + struct typ##Deleter { \ + void operator()(ddog_crasht_Handle_##typ *object) { \ + ddog_crasht_##typ##_drop(object); \ + delete object; \ + } \ + }; \ + std::unique_ptr extract_result( \ + ddog_crasht_Result_Handle##typ result, const char *msg) { \ + if (result.tag != ok_tag) { \ + print_error(msg, result.err); \ + ddog_Error_drop(&result.err); \ + exit(EXIT_FAILURE); \ + } \ + std::unique_ptr rval{ \ + new ddog_crasht_Handle_##typ{result.ok}}; \ + return rval; \ } - std::vector trace; - for (uintptr_t i = 0; i < 20; ++i) { - ddog_crasht_StackFrame frame = {.ip = i, - .module_base_address = 0, - .names = {.ptr = &names[i], .len = 1}, - .sp = 0, - .symbol_address = 0}; - trace.push_back(frame); +EXTRACT_RESULT(CrashInfoBuilder, + DDOG_CRASHT_RESULT_HANDLE_CRASH_INFO_BUILDER_OK_HANDLE_CRASH_INFO_BUILDER) +EXTRACT_RESULT(CrashInfo, DDOG_CRASHT_RESULT_HANDLE_CRASH_INFO_OK_HANDLE_CRASH_INFO) +EXTRACT_RESULT(StackTrace, DDOG_CRASHT_RESULT_HANDLE_STACK_TRACE_OK_HANDLE_STACK_TRACE) +EXTRACT_RESULT(StackFrame, DDOG_CRASHT_RESULT_HANDLE_STACK_FRAME_OK_HANDLE_STACK_FRAME) + +void add_stacktrace(ddog_crasht_Handle_CrashInfoBuilder *builder) { + auto stacktrace = extract_result(ddog_crasht_StackTrace_new(), "failed to make new StackTrace"); + + for (uintptr_t i = 0; i < 10; ++i) { + auto new_frame = extract_result(ddog_crasht_StackFrame_new(), "failed to make StackFrame"); + std::string name = "func_" + std::to_string(i); + check_result(ddog_crasht_StackFrame_with_function(new_frame.get(), to_slice_string(name)), + "failed to add function"); + std::string filename = "/path/to/code/file_" + std::to_string(i); + check_result(ddog_crasht_StackFrame_with_file(new_frame.get(), to_slice_string(filename)), + "failed to add filename"); + check_result(ddog_crasht_StackFrame_with_line(new_frame.get(), i * 4 + 3), + "failed to add line"); + check_result(ddog_crasht_StackFrame_with_column(new_frame.get(), i * 3 + 7), + "failed to add line"); + + // This operation consumes the frame, so use .release here + check_result(ddog_crasht_StackTrace_push_frame(stacktrace.get(), new_frame.release(), true), + "failed to add stack frame"); } - std::vector build_id = {42}; - std::string filePath = "/usr/share/somewhere"; - // test with normalized - auto elfFrameWithNormalization = ddog_crasht_StackFrame{ - .ip = 42, - .module_base_address = 0, - .names = {.ptr = &names[0], .len = 1}, // just for the test - .normalized_ip = { - .file_offset = 1, - .build_id = to_byte_slice(build_id), - .path = to_slice_c_char(filePath.c_str(), filePath.size()), - .typ = DDOG_CRASHT_NORMALIZED_ADDRESS_TYPES_ELF, - }, - .sp = 0, - .symbol_address = 0, - }; - - trace.push_back(elfFrameWithNormalization); - - // Windows-kind of frame - auto dllFrameWithNormalization = ddog_crasht_StackFrame{ - .ip = 42, - .module_base_address = 0, - .names = {.ptr = &names[0], .len = 1}, // just for the test - .normalized_ip = { - .file_offset = 1, - .build_id = to_byte_slice(build_id), - .age = 21, - .path = to_slice_c_char(filePath.c_str(), filePath.size()), - .typ = DDOG_CRASHT_NORMALIZED_ADDRESS_TYPES_PDB, - }, - .sp = 0, - .symbol_address = 0, - }; - - trace.push_back(dllFrameWithNormalization); - - ddog_crasht_Slice_StackFrame trace_slice = {.ptr = trace.data(), .len = trace.size()}; - + // Windows style frame with normalization + auto pbd_frame = extract_result(ddog_crasht_StackFrame_new(), "failed to make StackFrame"); + check_result(ddog_crasht_StackFrame_with_ip(pbd_frame.get(), to_slice_c_char("0xDEADBEEF")), + "failed to add ip"); + check_result(ddog_crasht_StackFrame_with_module_base_address(pbd_frame.get(), + to_slice_c_char("0xABBAABBA")), + "failed to add module_base_address"); + check_result( + ddog_crasht_StackFrame_with_build_id(pbd_frame.get(), to_slice_c_char("abcdef12345")), + "failed to add build id"); + check_result( + ddog_crasht_StackFrame_with_build_id_type(pbd_frame.get(), DDOG_CRASHT_BUILD_ID_TYPE_PDB), + "failed to add build id type"); + check_result(ddog_crasht_StackFrame_with_file_type(pbd_frame.get(), DDOG_CRASHT_FILE_TYPE_PDB), + "failed to add file type"); + check_result(ddog_crasht_StackFrame_with_path( + pbd_frame.get(), to_slice_c_char("C:/Program Files/best_program_ever.exe")), + "failed to add path"); check_result( - ddog_crasht_CrashInfo_set_stacktrace(crashinfo.get(), to_slice_c_char(""), trace_slice), - "Failed to set stacktrace"); + ddog_crasht_StackFrame_with_relative_address(pbd_frame.get(), to_slice_c_char("0xBABEF00D")), + "failed to add relative address"); + // This operation consumes the frame, so use .release here + check_result(ddog_crasht_StackTrace_push_frame(stacktrace.get(), pbd_frame.release(), true), + "failed to add stack frame"); + + // ELF style frame with normalization + auto elf_frame = extract_result(ddog_crasht_StackFrame_new(), "failed to make StackFrame"); + check_result(ddog_crasht_StackFrame_with_ip(elf_frame.get(), to_slice_c_char("0xDEADBEEF")), + "failed to add ip"); + check_result(ddog_crasht_StackFrame_with_module_base_address(elf_frame.get(), + to_slice_c_char("0xABBAABBA")), + "failed to add module_base_address"); + check_result( + ddog_crasht_StackFrame_with_build_id(elf_frame.get(), to_slice_c_char("987654321fedcba0")), + "failed to add build id"); + check_result( + ddog_crasht_StackFrame_with_build_id_type(elf_frame.get(), DDOG_CRASHT_BUILD_ID_TYPE_GNU), + "failed to add build id type"); + check_result(ddog_crasht_StackFrame_with_file_type(elf_frame.get(), DDOG_CRASHT_FILE_TYPE_ELF), + "failed to add file type"); + check_result(ddog_crasht_StackFrame_with_path(elf_frame.get(), + to_slice_c_char("/usr/bin/awesome-gnu-utility.so")), + "failed to add path"); + check_result( + ddog_crasht_StackFrame_with_relative_address(elf_frame.get(), to_slice_c_char("0xBABEF00D")), + "failed to add relative address"); + // This operation consumes the frame, so use .release here + check_result(ddog_crasht_StackTrace_push_frame(stacktrace.get(), elf_frame.release(), true), + "failed to add stack frame"); + + check_result(ddog_crasht_StackTrace_set_complete(stacktrace.get()), + "unable to set stacktrace as complete"); + + // Now that all the frames are added to the stack, put the stack on the report + // This operation consumes the stack, so use .release here + check_result(ddog_crasht_CrashInfoBuilder_with_stack(builder, stacktrace.release()), + "failed to add stacktrace"); } int main(void) { - auto crashinfo_new_result = ddog_crasht_CrashInfo_new(); - if (crashinfo_new_result.tag != DDOG_CRASHT_CRASH_INFO_NEW_RESULT_OK) { - print_error("Failed to make new crashinfo: ", crashinfo_new_result.err); - ddog_Error_drop(&crashinfo_new_result.err); - exit(EXIT_FAILURE); - } - std::unique_ptr crashinfo{&crashinfo_new_result.ok}; + auto builder = extract_result(ddog_crasht_CrashInfoBuilder_new(), "failed to make builder"); + check_result(ddog_crasht_CrashInfoBuilder_with_counter(builder.get(), + to_slice_c_char("my_amazing_counter"), 3), + "Failed to add counter"); - check_result( - ddog_crasht_CrashInfo_add_counter(crashinfo.get(), to_slice_c_char("my_amazing_counter"), 3), - "Failed to add counter"); - - // TODO add some tags here auto tags = ddog_Vec_Tag_new(); + check_result( + ddog_Vec_Tag_push(&tags, to_slice_c_char("best-hockey-team"), to_slice_c_char("Habs")), + "failed to add tag"); const ddog_crasht_Metadata metadata = { .library_name = to_slice_c_char("libdatadog"), .library_version = to_slice_c_char("42"), @@ -145,35 +161,35 @@ int main(void) { .tags = &tags, }; - // TODO: We should set more tags that are expected by telemetry - check_result(ddog_crasht_CrashInfo_set_metadata(crashinfo.get(), metadata), + check_result(ddog_crasht_CrashInfoBuilder_with_metadata(builder.get(), metadata), "Failed to add metadata"); - check_result(ddog_crasht_CrashInfo_add_tag(crashinfo.get(), to_slice_c_char("best hockey team"), - to_slice_c_char("Habs")), - "Failed to add tag"); + ddog_Vec_Tag_drop(tags); // This API allows one to capture useful files (e.g. /proc/pid/maps) - // For testing purposes, use `/etc/hosts` which should exist on any reasonable - // UNIX system - check_result(ddog_crasht_CrashInfo_add_file(crashinfo.get(), to_slice_c_char("/etc/hosts")), + // For testing purposes, use `/etc/hosts` which should exist on any reasonable UNIX system + check_result(ddog_crasht_CrashInfoBuilder_with_file(builder.get(), to_slice_c_char("/etc/hosts")), "Failed to add file"); - add_stacktrace(crashinfo); + check_result(ddog_crasht_CrashInfoBuilder_with_kind(builder.get(), DDOG_CRASHT_ERROR_KIND_PANIC), + "Failed to set error kind"); + + add_stacktrace(builder.get()); - ddog_Timespec timestamp = {.seconds = 1568899800, .nanoseconds = 0}; // Datadog IPO at 2019-09-19T13:30:00Z = 1568899800 unix - check_result(ddog_crasht_CrashInfo_set_timestamp(crashinfo.get(), timestamp), + ddog_Timespec timestamp = {.seconds = 1568899800, .nanoseconds = 0}; + check_result(ddog_crasht_CrashInfoBuilder_with_timestamp(builder.get(), timestamp), "Failed to set timestamp"); - ddog_crasht_ProcInfo procinfo = { - .pid = 42 - }; - - check_result(ddog_crasht_CrashInfo_set_procinfo(crashinfo.get(), procinfo), + ddog_crasht_ProcInfo procinfo = {.pid = 42}; + check_result(ddog_crasht_CrashInfoBuilder_with_proc_info(builder.get(), procinfo), "Failed to set procinfo"); - auto endpoint = ddog_endpoint_from_filename(to_slice_c_char("/tmp/test")); + check_result(ddog_crasht_CrashInfoBuilder_with_os_info_this_machine(builder.get()), + "Failed to set os_info"); + auto crashinfo = extract_result(ddog_crasht_CrashInfoBuilder_build(builder.release()), + "failed to build CrashInfo"); + auto endpoint = ddog_endpoint_from_filename(to_slice_c_char("/tmp/test")); check_result(ddog_crasht_CrashInfo_upload_to_endpoint(crashinfo.get(), endpoint), "Failed to export to file"); ddog_endpoint_drop(endpoint); diff --git a/examples/ffi/crashtracking.c b/examples/ffi/crashtracking.c index 8555ab133..1cca8021e 100644 --- a/examples/ffi/crashtracking.c +++ b/examples/ffi/crashtracking.c @@ -12,8 +12,8 @@ void example_segfault_handler(int signal) { exit(-1); } -void handle_result(ddog_crasht_Result result) { - if (result.tag == DDOG_CRASHT_RESULT_ERR) { +void handle_result(ddog_VoidResult result) { + if (result.tag == DDOG_VOID_RESULT_ERR) { ddog_CharSlice message = ddog_Error_message(&result.err); fprintf(stderr, "%.*s\n", (int)message.len, message.ptr); ddog_Error_drop(&result.err); @@ -21,8 +21,8 @@ void handle_result(ddog_crasht_Result result) { } } -uintptr_t handle_uintptr_t_result(ddog_crasht_UsizeResult result) { - if (result.tag == DDOG_CRASHT_USIZE_RESULT_ERR) { +uintptr_t handle_uintptr_t_result(ddog_crasht_Result_Usize result) { + if (result.tag == DDOG_CRASHT_RESULT_USIZE_ERR_USIZE) { ddog_CharSlice message = ddog_Error_message(&result.err); fprintf(stderr, "%.*s\n", (int)message.len, message.ptr); ddog_Error_drop(&result.err); diff --git a/examples/ffi/trace_exporter.c b/examples/ffi/trace_exporter.c index 055d4748f..d2356853c 100644 --- a/examples/ffi/trace_exporter.c +++ b/examples/ffi/trace_exporter.c @@ -1,28 +1,26 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#include #include #include #include #include -#define TRY(expr) \ - { \ - ddog_MaybeError err = expr; \ - if (err.tag == DDOG_OPTION_ERROR_SOME_ERROR) { \ - ddog_CharSlice message = ddog_Error_message(&err.some); \ - fprintf(stderr, "ERROR: %.*s", (int)message.len, (char *)message.ptr); \ - return 1; \ - } \ - } - -void agent_response_callback(const char* response) -{ - printf("Agent response: %s\n", response); +enum { + SUCCESS, + ERROR_SEND, +}; + +void handle_error(ddog_TraceExporterError *err) { + fprintf(stderr, "Operation failed with error: %d, reason: %s\n", err->code, err->msg); + ddog_trace_exporter_error_free(err); } int main(int argc, char** argv) { + int error; + ddog_TraceExporter* trace_exporter; ddog_CharSlice url = DDOG_CHARSLICE_C("http://localhost:8126/"); ddog_CharSlice tracer_version = DDOG_CHARSLICE_C("v0.1"); @@ -33,33 +31,41 @@ int main(int argc, char** argv) ddog_CharSlice env = DDOG_CHARSLICE_C("staging"); ddog_CharSlice version = DDOG_CHARSLICE_C("1.0"); ddog_CharSlice service = DDOG_CHARSLICE_C("test_app"); - TRY(ddog_trace_exporter_new( - &trace_exporter, - url, - tracer_version, - language, - language_version, - language_interpreter, - hostname, - env, - version, - service, - DDOG_TRACE_EXPORTER_INPUT_FORMAT_PROXY, - DDOG_TRACE_EXPORTER_OUTPUT_FORMAT_V04, - true, - &agent_response_callback - )); - - if (trace_exporter == NULL) - { - printf("unable to build the trace exporter"); - return 1; - } + + + ddog_TraceExporterError *ret; + ddog_TraceExporterConfig *config; + + ddog_trace_exporter_config_new(&config); + ddog_trace_exporter_config_set_url(config, url); + ddog_trace_exporter_config_set_tracer_version(config, tracer_version); + ddog_trace_exporter_config_set_language(config, language); + + + ret = ddog_trace_exporter_new(&trace_exporter, config); + + assert(ret == NULL); + assert(trace_exporter != NULL); ddog_ByteSlice buffer = { .ptr = NULL, .len=0 }; - TRY(ddog_trace_exporter_send(trace_exporter, buffer, 0)); + ddog_AgentResponse response; + + ret = ddog_trace_exporter_send(trace_exporter, buffer, 0, &response); + + assert(ret->code == DDOG_TRACE_EXPORTER_ERROR_CODE_SERDE); + if (ret) { + error = ERROR_SEND; + handle_error(ret); + goto error; + } ddog_trace_exporter_free(trace_exporter); + ddog_trace_exporter_config_free(config); + + return SUCCESS; - return 0; +error: + if (trace_exporter) { ddog_trace_exporter_free(trace_exporter); } + if (config) { ddog_trace_exporter_config_free(config); } + return error; } diff --git a/profiling-ffi/src/exporter.rs b/profiling-ffi/src/exporter.rs index e1cc5db11..8f9ea79a3 100644 --- a/profiling-ffi/src/exporter.rs +++ b/profiling-ffi/src/exporter.rs @@ -590,7 +590,8 @@ mod tests { }; let timeout_milliseconds = 90; unsafe { - ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds); + ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds) + .unwrap_none(); } let build_result = unsafe { @@ -663,7 +664,8 @@ mod tests { }; let timeout_milliseconds = 90; unsafe { - ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds); + ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds) + .unwrap_none(); } let raw_internal_metadata = CharSlice::from( @@ -737,7 +739,8 @@ mod tests { }; let timeout_milliseconds = 90; unsafe { - ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds); + ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds) + .unwrap_none(); } let raw_internal_metadata = CharSlice::from("this is not a valid json string"); @@ -799,7 +802,8 @@ mod tests { }; let timeout_milliseconds = 90; unsafe { - ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds); + ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds) + .unwrap_none(); } let raw_info = CharSlice::from( @@ -915,7 +919,8 @@ mod tests { }; let timeout_milliseconds = 90; unsafe { - ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds); + ddog_prof_Exporter_set_timeout(Some(exporter.as_mut()), timeout_milliseconds) + .unwrap_none(); } let raw_info = CharSlice::from("this is not a valid json string"); diff --git a/remote-config/src/path.rs b/remote-config/src/path.rs index d890cbadb..4dee14910 100644 --- a/remote-config/src/path.rs +++ b/remote-config/src/path.rs @@ -21,6 +21,8 @@ pub enum RemoteConfigProduct { Asm, AsmDD, AsmFeatures, + AsmRaspLfi, + AsmRaspSsrf, LiveDebugger, } @@ -33,6 +35,8 @@ impl Display for RemoteConfigProduct { RemoteConfigProduct::AsmDD => "ASM_DD", RemoteConfigProduct::AsmData => "ASM_DATA", RemoteConfigProduct::AsmFeatures => "ASM_FEATURES", + RemoteConfigProduct::AsmRaspLfi => "ASM_RASP_LFI", + RemoteConfigProduct::AsmRaspSsrf => "ASM_RASP_SSRF", }; write!(f, "{}", str) } @@ -80,6 +84,8 @@ impl RemoteConfigPath { "ASM_DD" => RemoteConfigProduct::AsmDD, "ASM_DATA" => RemoteConfigProduct::AsmData, "ASM_FEATURES" => RemoteConfigProduct::AsmFeatures, + "ASM_RASP_LFI" => RemoteConfigProduct::AsmRaspLfi, + "ASM_RASP_SSRF" => RemoteConfigProduct::AsmRaspSsrf, product => anyhow::bail!("Unknown product {}", product), }, config_id: parts[parts.len() - 2], diff --git a/ruby/Rakefile b/ruby/Rakefile index 9e6e47075..c4b081f6a 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -11,7 +11,7 @@ require "rubygems/package" RSpec::Core::RakeTask.new(:spec) -LIB_VERSION_TO_PACKAGE = "14.1.0" +LIB_VERSION_TO_PACKAGE = "14.3.1" unless LIB_VERSION_TO_PACKAGE.start_with?(Libdatadog::LIB_VERSION) raise "`LIB_VERSION_TO_PACKAGE` setting in (#{LIB_VERSION_TO_PACKAGE}) does not match " \ "`LIB_VERSION` setting in (#{Libdatadog::LIB_VERSION})" @@ -20,22 +20,22 @@ end LIB_GITHUB_RELEASES = [ { file: "libdatadog-aarch64-alpine-linux-musl.tar.gz", - sha256: "fc6be3383d3a115804c43e2c66dd176c63f33b362d987d9b1211034e2b549c2d", + sha256: "57f83aff275628bb1af89c22bb4bd696726daf2a9e09b6cd0d966b29e65a7ad6", ruby_platform: "aarch64-linux-musl" }, { file: "libdatadog-aarch64-unknown-linux-gnu.tar.gz", - sha256: "1a9bc4d99d23f7baf403b6b7527f9b9d76bdb166dc34656150561dcb148cc90b", + sha256: "36db8d50ccabb71571158ea13835c0f1d05d30b32135385f97c16343cfb6ddd4", ruby_platform: "aarch64-linux" }, { file: "libdatadog-x86_64-alpine-linux-musl.tar.gz", - sha256: "8244831681332dfa939eefe6923fe6a8beaffff48cb336f836b55a438078add1", + sha256: "2f61fd21cf2f8147743e414b4a8c77250a17be3aecc42a69ffe54f0a603d5c92", ruby_platform: "x86_64-linux-musl" }, { file: "libdatadog-x86_64-unknown-linux-gnu.tar.gz", - sha256: "76fcb3bfe3b3971d77f6dd4968ffe6bd5f6a1ada82e2e990a78919107dc2ee40", + sha256: "f01f05600591063eba4faf388f54c155ab4e6302e5776c7855e3734955f7daf7", ruby_platform: "x86_64-linux" } ] diff --git a/ruby/lib/libdatadog/version.rb b/ruby/lib/libdatadog/version.rb index 18bdea371..57d77dbc1 100644 --- a/ruby/lib/libdatadog/version.rb +++ b/ruby/lib/libdatadog/version.rb @@ -2,7 +2,7 @@ module Libdatadog # Current libdatadog version - LIB_VERSION = "14.1.0" + LIB_VERSION = "14.3.1" GEM_MAJOR_VERSION = "1" GEM_MINOR_VERSION = "0" diff --git a/serverless/src/main.rs b/serverless/src/main.rs index a2e85ab20..95be1ead2 100644 --- a/serverless/src/main.rs +++ b/serverless/src/main.rs @@ -23,6 +23,7 @@ use dogstatsd::metric::EMPTY_TAGS; use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; +const DOGSTATSD_TIMEOUT_DURATION: Duration = Duration::from_secs(5); const DEFAULT_DOGSTATSD_PORT: u16 = 8125; const AGENT_HOST: &str = "0.0.0.0"; @@ -160,6 +161,7 @@ async fn start_dogstatsd( Arc::clone(&metrics_aggr), build_fqdn_metrics(dd_site), https_proxy, + DOGSTATSD_TIMEOUT_DURATION, ); Some(metrics_flusher) } diff --git a/sidecar-ffi/src/lib.rs b/sidecar-ffi/src/lib.rs index c5ce8c5f3..ebf359e3b 100644 --- a/sidecar-ffi/src/lib.rs +++ b/sidecar-ffi/src/lib.rs @@ -567,15 +567,17 @@ pub unsafe extern "C" fn ddog_sidecar_session_set_config( } else { LogMethod::File(String::from(log_path.to_utf8_lossy()).into()) }, - remote_config_products: slice::from_raw_parts( + remote_config_products: ffi::Slice::from_raw_parts( remote_config_products, remote_config_products_count ) + .as_slice() .to_vec(), - remote_config_capabilities: slice::from_raw_parts( + remote_config_capabilities: ffi::Slice::from_raw_parts( remote_config_capabilities, remote_config_capabilities_count ) + .as_slice() .to_vec(), }, )); diff --git a/sidecar-ffi/tests/sidecar.rs b/sidecar-ffi/tests/sidecar.rs index a857118a3..50b55a803 100644 --- a/sidecar-ffi/tests/sidecar.rs +++ b/sidecar-ffi/tests/sidecar.rs @@ -68,7 +68,7 @@ fn test_ddog_sidecar_connection() { } #[test] -#[ignore = "TODO: ci-flaky can't reproduce locally"] +#[cfg_attr(miri, ignore)] fn test_ddog_sidecar_register_app() { set_sidecar_per_process(); @@ -105,7 +105,8 @@ fn test_ddog_sidecar_register_app() { 0, null(), 0, - ); + ) + .unwrap_none(); let meta = ddog_sidecar_runtimeMeta_build( "language_name".into(), @@ -122,7 +123,8 @@ fn test_ddog_sidecar_register_app() { &queue_id, "dependency_name".into(), "dependency_version".into(), - ); + ) + .unwrap_none(); // ddog_sidecar_telemetry_addIntegration(&mut transport, instance_id, &queue_id, // integration_name, integration_version) TODO add ability to add configuration @@ -158,7 +160,8 @@ fn test_ddog_sidecar_register_app() { 0, null(), 0, - ); + ) + .unwrap_none(); //TODO: Shutdown the service // enough case: have C api that shutsdown telemetry worker diff --git a/sidecar/src/self_telemetry.rs b/sidecar/src/self_telemetry.rs index e980f6efc..9efaeee9e 100644 --- a/sidecar/src/self_telemetry.rs +++ b/sidecar/src/self_telemetry.rs @@ -32,7 +32,7 @@ struct MetricData<'a> { trace_chunks_sent: ContextKey, trace_chunks_dropped: ContextKey, } -impl<'a> MetricData<'a> { +impl MetricData<'_> { async fn send(&self, key: ContextKey, value: f64, tags: Vec) { let _ = self .worker diff --git a/sidecar/src/service/sidecar_server.rs b/sidecar/src/service/sidecar_server.rs index fd31fa9da..364584083 100644 --- a/sidecar/src/service/sidecar_server.rs +++ b/sidecar/src/service/sidecar_server.rs @@ -644,6 +644,13 @@ impl SidecarInterface for SidecarServer { } app.telemetry.send_msgs(actions).await.ok(); + + let mut extracted_actions: Vec = vec![]; + enqueued_data + .extract_telemetry_actions(&mut extracted_actions) + .await; + app.telemetry.send_msgs(extracted_actions).await.ok(); + // Ok, we dequeued all messages, now new enqueue_actions calls can handle it completer.complete((service_name, env_name)).await; } diff --git a/sidecar/src/setup/unix.rs b/sidecar/src/setup/unix.rs index b8337bed7..e3b2002dc 100644 --- a/sidecar/src/setup/unix.rs +++ b/sidecar/src/setup/unix.rs @@ -82,15 +82,20 @@ impl Liaison for SharedDirLiaison { } fn ipc_per_process() -> Self { - //TODO: implement per pid handling - Self::new_default_location() + static PROCESS_RANDOM_ID: std::sync::OnceLock = std::sync::OnceLock::new(); + let random_id = PROCESS_RANDOM_ID.get_or_init(rand::random); + + let pid = std::process::id(); + let liason_path = env::temp_dir().join(format!("libdatadog.{random_id}.{pid}")); + Self::new(liason_path) } } impl SharedDirLiaison { pub fn new>(base_dir: P) -> Self { let versioned_socket_basename = format!( - concat!("libdd.", crate::sidecar_version!(), "@{}.sock"), + "libdd.{}@{}.sock", + crate::sidecar_version!(), primary_sidecar_identifier() ); let base_dir = base_dir.as_ref(); diff --git a/sidecar/src/setup/windows.rs b/sidecar/src/setup/windows.rs index 9e252b9be..cd3b20374 100644 --- a/sidecar/src/setup/windows.rs +++ b/sidecar/src/setup/windows.rs @@ -143,10 +143,10 @@ impl Liaison for NamedPipeLiaison { } { INVALID_HANDLE_VALUE => { let error = io::Error::last_os_error(); - if error - .raw_os_error() - .map_or(true, |r| r as u32 == ERROR_ACCESS_DENIED) - { + if match error.raw_os_error() { + Some(code) => code as u32 == ERROR_ACCESS_DENIED, + None => true, + } { Ok(None) } else { Err(error) diff --git a/tinybytes/src/bytes_string.rs b/tinybytes/src/bytes_string.rs index 4b921b8bc..0fe0d22eb 100644 --- a/tinybytes/src/bytes_string.rs +++ b/tinybytes/src/bytes_string.rs @@ -106,6 +106,13 @@ impl BytesString { // SAFETY: We assume all BytesStrings are valid UTF-8. unsafe { std::str::from_utf8_unchecked(&self.bytes) } } + + /// Returns a `String` with a copy of the `BytesString`. + /// This is typically useful when you need to hold the content of a slice for a long time and + /// don't want to prevent the buffer from being dropped earlier. + pub fn copy_to_string(&self) -> String { + self.as_str().to_string() + } } impl Default for BytesString { @@ -218,14 +225,14 @@ mod tests { } #[test] - fn from_string() { + fn test_from_string() { let string = String::from("hello"); let bytes_string = BytesString::from(string); assert_eq!(bytes_string.as_str(), "hello") } #[test] - fn from_static_str() { + fn test_from_static_str() { let static_str = "hello"; let bytes_string = BytesString::from(static_str); assert_eq!(bytes_string.as_str(), "hello") @@ -238,8 +245,14 @@ mod tests { } #[test] - fn hash() { + fn test_hash() { let bytes_string = BytesString::from_slice(b"test hash").unwrap(); assert_eq!(calculate_hash(&bytes_string), calculate_hash(&"test hash")); } + + #[test] + fn test_copy_to_string() { + let bytes_string = BytesString::from("hello"); + assert_eq!(bytes_string.copy_to_string(), "hello") + } } diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index b6f744fc7..78958bf75 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -1,4 +1,4 @@ -ARG ALPINE_BASE_IMAGE="alpine:3.19.3" +ARG ALPINE_BASE_IMAGE="alpine:3.20.3" ARG CARGO_BUILD_INCREMENTAL="true" ARG CARGO_NET_RETRY="2" ARG BUILDER_IMAGE=debian_builder diff --git a/trace-mini-agent/Cargo.toml b/trace-mini-agent/Cargo.toml index 0de5d941d..381691fa5 100644 --- a/trace-mini-agent/Cargo.toml +++ b/trace-mini-agent/Cargo.toml @@ -15,7 +15,7 @@ anyhow = "1.0" hyper = { version = "0.14", default-features = false, features = ["server", "backports", "deprecated"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"]} async-trait = "0.1.64" -log = "0.4" +tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" ddcommon = { path = "../ddcommon" } diff --git a/trace-mini-agent/src/env_verifier.rs b/trace-mini-agent/src/env_verifier.rs index b00cb7e23..f06f30d6f 100644 --- a/trace-mini-agent/src/env_verifier.rs +++ b/trace-mini-agent/src/env_verifier.rs @@ -4,7 +4,6 @@ use async_trait::async_trait; use hyper::body::HttpBody; use hyper::{Body, Client, Method, Request, Response}; -use log::{debug, error}; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -12,6 +11,7 @@ use std::path::Path; use std::process; use std::sync::Arc; use std::time::{Duration, Instant}; +use tracing::{debug, error}; use datadog_trace_utils::trace_utils; diff --git a/trace-mini-agent/src/http_utils.rs b/trace-mini-agent/src/http_utils.rs index b5cf5763e..01db15c0a 100644 --- a/trace-mini-agent/src/http_utils.rs +++ b/trace-mini-agent/src/http_utils.rs @@ -6,8 +6,8 @@ use hyper::{ http::{self, HeaderMap}, Body, Response, StatusCode, }; -use log::{error, info}; use serde_json::json; +use tracing::{debug, error}; /// Does two things: /// 1. Logs the given message. A success status code (within 200-299) will cause an info log to be @@ -23,7 +23,7 @@ pub fn log_and_create_http_response( status: StatusCode, ) -> http::Result> { if status.is_success() { - info!("{message}"); + debug!("{message}"); } else { error!("{message}"); } @@ -46,7 +46,7 @@ pub fn log_and_create_traces_success_http_response( message: &str, status: StatusCode, ) -> http::Result> { - info!("{message}"); + debug!("{message}"); let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); Response::builder().status(status).body(Body::from(body)) } diff --git a/trace-mini-agent/src/mini_agent.rs b/trace-mini-agent/src/mini_agent.rs index e6e24b5bf..b6f548fdc 100644 --- a/trace-mini-agent/src/mini_agent.rs +++ b/trace-mini-agent/src/mini_agent.rs @@ -3,13 +3,13 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{http, Body, Method, Request, Response, Server, StatusCode}; -use log::{debug, error, info}; use serde_json::json; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; use tokio::sync::mpsc::{self, Receiver, Sender}; +use tracing::{debug, error}; use crate::http_utils::log_and_create_http_response; use crate::{config, env_verifier, stats_flusher, stats_processor, trace_flusher, trace_processor}; @@ -121,7 +121,7 @@ impl MiniAgent { let server = server_builder.serve(make_svc); - info!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); + debug!("Mini Agent started: listening on port {MINI_AGENT_PORT}"); debug!( "Time taken start the Mini Agent: {} ms", now.elapsed().as_millis() diff --git a/trace-mini-agent/src/stats_flusher.rs b/trace-mini-agent/src/stats_flusher.rs index 9d7bed76b..ac8d59a8d 100644 --- a/trace-mini-agent/src/stats_flusher.rs +++ b/trace-mini-agent/src/stats_flusher.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; -use log::{debug, error, info}; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; +use tracing::{debug, error}; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; @@ -61,7 +61,7 @@ impl StatsFlusher for ServerlessStatsFlusher { if stats.is_empty() { return; } - info!("Flushing {} stats", stats.len()); + debug!("Flushing {} stats", stats.len()); let stats_payload = stats_utils::construct_stats_payload(stats); @@ -82,7 +82,7 @@ impl StatsFlusher for ServerlessStatsFlusher { ) .await { - Ok(_) => info!("Successfully flushed stats"), + Ok(_) => debug!("Successfully flushed stats"), Err(e) => { error!("Error sending stats: {e:?}") } diff --git a/trace-mini-agent/src/stats_processor.rs b/trace-mini-agent/src/stats_processor.rs index 38539f5e4..28674eb18 100644 --- a/trace-mini-agent/src/stats_processor.rs +++ b/trace-mini-agent/src/stats_processor.rs @@ -6,8 +6,8 @@ use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; -use log::info; use tokio::sync::mpsc::Sender; +use tracing::debug; use datadog_trace_protobuf::pb; use datadog_trace_utils::stats_utils; @@ -38,7 +38,7 @@ impl StatsProcessor for ServerlessStatsProcessor { req: Request, tx: Sender, ) -> http::Result> { - info!("Recieved trace stats to process"); + debug!("Received trace stats to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( diff --git a/trace-mini-agent/src/trace_flusher.rs b/trace-mini-agent/src/trace_flusher.rs index 7987e7646..7004b3376 100644 --- a/trace-mini-agent/src/trace_flusher.rs +++ b/trace-mini-agent/src/trace_flusher.rs @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; -use log::{error, info}; use std::{sync::Arc, time}; use tokio::sync::{mpsc::Receiver, Mutex}; +use tracing::{debug, error}; use datadog_trace_utils::trace_utils; use datadog_trace_utils::trace_utils::SendData; @@ -53,7 +53,7 @@ impl TraceFlusher for ServerlessTraceFlusher { if traces.is_empty() { return; } - info!("Flushing {} traces", traces.len()); + debug!("Flushing {} traces", traces.len()); for traces in trace_utils::coalesce_send_data(traces) { match traces @@ -61,7 +61,7 @@ impl TraceFlusher for ServerlessTraceFlusher { .await .last_result { - Ok(_) => info!("Successfully flushed traces"), + Ok(_) => debug!("Successfully flushed traces"), Err(e) => { error!("Error sending trace: {e:?}") // TODO: Retries diff --git a/trace-mini-agent/src/trace_processor.rs b/trace-mini-agent/src/trace_processor.rs index 07360dfa2..14564e7d8 100644 --- a/trace-mini-agent/src/trace_processor.rs +++ b/trace-mini-agent/src/trace_processor.rs @@ -5,13 +5,13 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{http, Body, Request, Response, StatusCode}; -use log::info; use tokio::sync::mpsc::Sender; +use tracing::debug; use datadog_trace_obfuscation::obfuscate::obfuscate_span; use datadog_trace_protobuf::pb; -use datadog_trace_utils::trace_utils::SendData; use datadog_trace_utils::trace_utils::{self}; +use datadog_trace_utils::trace_utils::{EnvironmentType, SendData}; use datadog_trace_utils::tracer_payload::{TraceChunkProcessor, TraceCollection}; use crate::{ @@ -47,6 +47,13 @@ impl TraceChunkProcessor for ChunkProcessor { for span in chunk.spans.iter_mut() { trace_utils::enrich_span_with_mini_agent_metadata(span, &self.mini_agent_metadata); trace_utils::enrich_span_with_azure_function_metadata(span); + if let EnvironmentType::CloudFunction = &self.config.env_type { + trace_utils::enrich_span_with_google_cloud_function_metadata( + span, + &self.mini_agent_metadata, + self.config.app_name.clone(), + ); + } obfuscate_span(span, &self.config.obfuscation_config); } } @@ -63,7 +70,7 @@ impl TraceProcessor for ServerlessTraceProcessor { tx: Sender, mini_agent_metadata: Arc, ) -> http::Result> { - info!("Recieved traces to process"); + debug!("Received traces to process"); let (parts, body) = req.into_parts(); if let Some(response) = http_utils::verify_request_content_length( @@ -130,10 +137,10 @@ mod tests { trace_processor::{self, TraceProcessor}, }; use datadog_trace_protobuf::pb; + use datadog_trace_utils::test_utils::{create_test_gcp_json_span, create_test_gcp_span}; + use datadog_trace_utils::trace_utils::MiniAgentMetadata; use datadog_trace_utils::{ - test_utils::{create_test_json_span, create_test_span}, - trace_utils, - tracer_payload::TracerPayloadCollection, + test_utils::create_test_json_span, trace_utils, tracer_payload::TracerPayloadCollection, }; use ddcommon::Endpoint; @@ -167,6 +174,16 @@ mod tests { } } + fn create_test_metadata() -> MiniAgentMetadata { + MiniAgentMetadata { + azure_spring_app_hostname: Default::default(), + azure_spring_app_name: Default::default(), + gcp_project_id: Some("dummy_project_id".to_string()), + gcp_region: Some("dummy_region_west".to_string()), + version: Some("dummy_version".to_string()), + } + } + #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_process_trace() { @@ -196,7 +213,7 @@ mod tests { Arc::new(create_test_config()), request, tx, - Arc::new(trace_utils::MiniAgentMetadata::default()), + Arc::new(create_test_metadata()), ) .await; assert!(res.is_ok()); @@ -214,7 +231,7 @@ mod tests { chunks: vec![pb::TraceChunk { priority: i8::MIN as i32, origin: "".to_string(), - spans: vec![create_test_span(11, 222, 333, start, true)], + spans: vec![create_test_gcp_span(11, 222, 333, start, true)], tags: HashMap::new(), dropped_trace: false, }], @@ -230,7 +247,6 @@ mod tests { } else { None }; - assert_eq!(expected_tracer_payload, received_payload.unwrap()); } @@ -245,9 +261,9 @@ mod tests { let start = get_current_timestamp_nanos(); let json_trace = vec![ - create_test_json_span(11, 333, 222, start), - create_test_json_span(11, 222, 0, start), - create_test_json_span(11, 444, 333, start), + create_test_gcp_json_span(11, 333, 222, start), + create_test_gcp_json_span(11, 222, 0, start), + create_test_gcp_json_span(11, 444, 333, start), ]; let bytes = rmp_serde::to_vec(&vec![json_trace]).unwrap(); @@ -267,7 +283,7 @@ mod tests { Arc::new(create_test_config()), request, tx, - Arc::new(trace_utils::MiniAgentMetadata::default()), + Arc::new(create_test_metadata()), ) .await; assert!(res.is_ok()); @@ -286,9 +302,9 @@ mod tests { priority: i8::MIN as i32, origin: "".to_string(), spans: vec![ - create_test_span(11, 333, 222, start, false), - create_test_span(11, 222, 0, start, true), - create_test_span(11, 444, 333, start, false), + create_test_gcp_span(11, 333, 222, start, false), + create_test_gcp_span(11, 222, 0, start, true), + create_test_gcp_span(11, 444, 333, start, false), ], tags: HashMap::new(), dropped_trace: false, diff --git a/trace-protobuf/build.rs b/trace-protobuf/build.rs index f713e2eb0..dc28ade2c 100644 --- a/trace-protobuf/build.rs +++ b/trace-protobuf/build.rs @@ -19,6 +19,11 @@ fn main() -> Result<()> { // compiles the .proto files into rust structs generate_protobuf(); } + #[cfg(not(feature = "generate-protobuf"))] + { + println!("cargo:rerun-if-changed=build.rs"); + } + Ok(()) } diff --git a/trace-utils/src/msgpack_decoder/v04/decoder/mod.rs b/trace-utils/src/msgpack_decoder/v04/decoder/mod.rs index 3426c157b..d5e01f6c9 100644 --- a/trace-utils/src/msgpack_decoder/v04/decoder/mod.rs +++ b/trace-utils/src/msgpack_decoder/v04/decoder/mod.rs @@ -13,6 +13,9 @@ use rmp::{decode, decode::RmpRead, Marker}; use std::{collections::HashMap, f64}; use tinybytes::{Bytes, BytesString}; +// https://docs.rs/rmp/latest/rmp/enum.Marker.html#variant.Null (0xc0 == 192) +const NULL_MARKER: &u8 = &0xc0; + /// Decodes a slice of bytes into a vector of `TracerPayloadV04` objects. /// /// @@ -131,24 +134,44 @@ fn read_string_bytes(buf: &mut Bytes) -> Result { }) } +#[inline] +fn read_nullable_string_bytes(buf: &mut Bytes) -> Result { + if let Some(empty_string) = handle_null_marker(buf, BytesString::default) { + Ok(empty_string) + } else { + read_string_bytes(buf) + } +} + #[inline] // Safety: read_string_ref checks utf8 validity, so we don't do it again when creating the // BytesStrings. fn read_str_map_to_bytes_strings( - buf_wrapper: &mut Bytes, + buf: &mut Bytes, ) -> Result, DecodeError> { - let len = decode::read_map_len(unsafe { buf_wrapper.as_mut_slice() }) + let len = decode::read_map_len(unsafe { buf.as_mut_slice() }) .map_err(|_| DecodeError::InvalidFormat("Unable to get map len for str map".to_owned()))?; let mut map = HashMap::with_capacity(len.try_into().expect("Unable to cast map len to usize")); for _ in 0..len { - let key = read_string_bytes(buf_wrapper)?; - let value = read_string_bytes(buf_wrapper)?; + let key = read_string_bytes(buf)?; + let value = read_string_bytes(buf)?; map.insert(key, value); } Ok(map) } +#[inline] +fn read_nullable_str_map_to_bytes_strings( + buf: &mut Bytes, +) -> Result, DecodeError> { + if let Some(empty_map) = handle_null_marker(buf, HashMap::default) { + return Ok(empty_map); + } + + read_str_map_to_bytes_strings(buf) +} + #[inline] fn read_metric_pair(buf: &mut Bytes) -> Result<(BytesString, f64), DecodeError> { let key = read_string_bytes(buf)?; @@ -156,12 +179,23 @@ fn read_metric_pair(buf: &mut Bytes) -> Result<(BytesString, f64), DecodeError> Ok((key, v)) } +#[inline] fn read_metrics(buf: &mut Bytes) -> Result, DecodeError> { + if let Some(empty_map) = handle_null_marker(buf, HashMap::default) { + return Ok(empty_map); + } + let len = read_map_len(unsafe { buf.as_mut_slice() })?; + read_map(len, buf, read_metric_pair) } +#[inline] fn read_meta_struct(buf: &mut Bytes) -> Result>, DecodeError> { + if let Some(empty_map) = handle_null_marker(buf, HashMap::default) { + return Ok(empty_map); + } + fn read_meta_struct_pair(buf: &mut Bytes) -> Result<(BytesString, Vec), DecodeError> { let key = read_string_bytes(buf)?; let array_len = decode::read_array_len(unsafe { buf.as_mut_slice() }).map_err(|_| { @@ -208,6 +242,7 @@ fn read_meta_struct(buf: &mut Bytes) -> Result>, De /// * `K` - The type of the keys in the map. Must implement `std::hash::Hash` and `Eq`. /// * `V` - The type of the values in the map. /// * `F` - The type of the function used to read key-value pairs from the buffer. +#[inline] fn read_map( len: usize, buf: &mut Bytes, @@ -225,6 +260,7 @@ where Ok(map) } +#[inline] fn read_map_len(buf: &mut &[u8]) -> Result { match decode::read_marker(buf) .map_err(|_| DecodeError::InvalidFormat("Unable to read marker for map".to_owned()))? @@ -244,6 +280,23 @@ fn read_map_len(buf: &mut &[u8]) -> Result { } } +/// When you want to "peek" if the next value is a null marker, and only advance the buffer if it is +/// null and return the default value. If it is not null, you can continue to decode as expected. +#[inline] +fn handle_null_marker(buf: &mut Bytes, default: F) -> Option +where + F: FnOnce() -> T, +{ + let slice = unsafe { buf.as_mut_slice() }; + + if slice.first() == Some(NULL_MARKER) { + *slice = &slice[1..]; + Some(default()) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; @@ -269,27 +322,110 @@ mod tests { (key, rmp_serde::to_vec_named(&map).unwrap()) } + #[test] + fn test_empty_array() { + let encoded_data = vec![0x90]; + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); + let (_decoded_traces, decoded_size) = from_slice(bytes).expect("Decoding failed"); + + assert_eq!(0, decoded_size); + } #[test] - fn decoder_read_string_success() { - let expected_string = "test-service-name"; + fn test_decoder_size() { let span = Span { - name: BytesString::from_slice(expected_string.as_ref()).unwrap(), + name: BytesString::from_slice("span_name".as_ref()).unwrap(), ..Default::default() }; let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); let expected_size = encoded_data.len() - 1; // rmp_serde adds additional 0 byte encoded_data.extend_from_slice(&[0, 0, 0, 0]); // some garbage, to be ignored - let (decoded_traces, decoded_size) = + let (_decoded_traces, decoded_size) = from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); assert_eq!(expected_size, decoded_size); + } + + #[test] + fn test_decoder_read_string_success() { + let expected_string = "test-service-name"; + let span = Span { + name: BytesString::from_slice(expected_string.as_ref()).unwrap(), + ..Default::default() + }; + let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + encoded_data.extend_from_slice(&[0, 0, 0, 0]); // some garbage, to be ignored + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + assert_eq!(1, decoded_traces.len()); assert_eq!(1, decoded_traces[0].len()); let decoded_span = &decoded_traces[0][0]; assert_eq!(expected_string, decoded_span.name.as_str()); } + #[test] + fn test_decoder_read_null_string_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["name"] = json!(null); + let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + encoded_data.extend_from_slice(&[0, 0, 0, 0]); // some garbage, to be ignored + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + assert_eq!("", decoded_span.name.as_str()); + } + + #[test] + fn test_decoder_read_number_success() { + let span = create_test_json_span(1, 2, 0, 0); + let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + encoded_data.extend_from_slice(&[0, 0, 0, 0]); // some garbage, to be ignored + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + assert_eq!(1, decoded_span.trace_id); + } + + #[test] + fn test_decoder_read_null_number_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["trace_id"] = json!(null); + let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + encoded_data.extend_from_slice(&[0, 0, 0, 0]); // some garbage, to be ignored + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + assert_eq!(0, decoded_span.trace_id); + } + + #[test] + fn test_decoder_meta_struct_null_map_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["meta_struct"] = json!(null); + + let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + + assert!(decoded_span.meta_struct.is_empty()); + } + #[test] fn test_decoder_meta_struct_fixed_map_success() { let expected_meta_struct = HashMap::from([ @@ -366,6 +502,22 @@ mod tests { } } + #[test] + fn test_decoder_meta_null_map_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["meta"] = json!(null); + + let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + + assert!(decoded_span.meta.is_empty()); + } + #[test] fn test_decoder_meta_map_16_success() { let expected_meta: HashMap = (0..20) @@ -442,6 +594,20 @@ mod tests { } } + #[test] + fn test_decoder_metrics_null_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["metrics"] = json!(null); + let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + assert!(decoded_span.metrics.is_empty()); + } + #[test] fn test_decoder_span_link_success() { let expected_span_link = json!({ @@ -502,7 +668,22 @@ mod tests { } #[test] - #[cfg_attr(miri, ignore)] + fn test_decoder_null_span_link_success() { + let mut span = create_test_json_span(1, 2, 0, 0); + span["span_links"] = json!(null); + + let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let (decoded_traces, _) = + from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + + assert!(decoded_span.span_links.is_empty()); + } + + #[test] fn test_decoder_read_string_wrong_format() { let span = Span { service: BytesString::from_slice("my_service".as_ref()).unwrap(), @@ -511,8 +692,11 @@ mod tests { let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); // This changes the map size from 11 to 12 to trigger an InvalidMarkerRead error. encoded_data[2] = 0x8c; + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); - let result = from_slice(tinybytes::Bytes::from(encoded_data)); + let result = from_slice(bytes); assert_eq!( Err(DecodeError::InvalidFormat( "Expected at least bytes 1, but only got 0 (pos 0)".to_owned() @@ -522,7 +706,6 @@ mod tests { } #[test] - #[cfg_attr(miri, ignore)] fn test_decoder_read_string_utf8_error() { let invalid_seq = vec![0, 159, 146, 150]; let invalid_str = unsafe { String::from_utf8_unchecked(invalid_seq) }; @@ -532,8 +715,11 @@ mod tests { ..Default::default() }; let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); - let result = from_slice(tinybytes::Bytes::from(encoded_data)); + let result = from_slice(bytes); assert_eq!( Err(DecodeError::Utf8Error( "invalid utf-8 sequence of 1 bytes from index 1".to_owned() @@ -543,15 +729,18 @@ mod tests { } #[test] - #[cfg_attr(miri, ignore)] fn test_decoder_invalid_marker_for_trace_count_read() { let span = Span::default(); let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); // This changes the entire payload to a map with 12 keys in order to trigger an error when // reading the array len of traces encoded_data[0] = 0x8c; + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); + + let result = from_slice(bytes); - let result = from_slice(tinybytes::Bytes::from(encoded_data)); assert_eq!( Err(DecodeError::InvalidFormat( "Unable to read array len for trace count".to_string() @@ -561,7 +750,6 @@ mod tests { } #[test] - #[cfg_attr(miri, ignore)] fn test_decoder_invalid_marker_for_span_count_read() { let span = Span::default(); let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); @@ -569,7 +757,12 @@ mod tests { // reading the array len of spans encoded_data[1] = 0x8c; - let result = from_slice(tinybytes::Bytes::from(encoded_data)); + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); + + let result = from_slice(bytes); + assert_eq!( Err(DecodeError::InvalidFormat( "Unable to read array len for span count".to_owned() @@ -579,15 +772,18 @@ mod tests { } #[test] - #[cfg_attr(miri, ignore)] fn test_decoder_read_string_type_mismatch() { let span = Span::default(); let mut encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); // Modify the encoded data to cause a type mismatch by changing the marker for the `name` // field to an integer marker encoded_data[3] = 0x01; + let encoded_data = + unsafe { std::mem::transmute::<&'_ [u8], &'static [u8]>(encoded_data.as_ref()) }; + let bytes = tinybytes::Bytes::from_static(encoded_data); + + let result = from_slice(bytes); - let result = from_slice(tinybytes::Bytes::from(encoded_data)); assert_eq!( Err(DecodeError::InvalidType( "Type mismatch at marker FixPos(1)".to_owned() diff --git a/trace-utils/src/msgpack_decoder/v04/decoder/span.rs b/trace-utils/src/msgpack_decoder/v04/decoder/span.rs index 445337198..ccbb7d1e9 100644 --- a/trace-utils/src/msgpack_decoder/v04/decoder/span.rs +++ b/trace-utils/src/msgpack_decoder/v04/decoder/span.rs @@ -2,11 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use super::{ - read_meta_struct, read_metrics, read_str_map_to_bytes_strings, read_string_bytes, - read_string_ref, span_link::read_span_links, + read_meta_struct, read_metrics, read_nullable_str_map_to_bytes_strings, + read_nullable_string_bytes, read_string_ref, span_link::read_span_links, }; use crate::msgpack_decoder::v04::error::DecodeError; -use crate::msgpack_decoder::v04::number::read_number_bytes; +use crate::msgpack_decoder::v04::number::read_nullable_number_bytes; use crate::span_v04::{Span, SpanKey}; use tinybytes::Bytes; @@ -48,17 +48,17 @@ fn fill_span(span: &mut Span, buf: &mut Bytes) -> Result<(), DecodeError> { .map_err(|_| DecodeError::InvalidFormat("Invalid span key".to_owned()))?; match key { - SpanKey::Service => span.service = read_string_bytes(buf)?, - SpanKey::Name => span.name = read_string_bytes(buf)?, - SpanKey::Resource => span.resource = read_string_bytes(buf)?, - SpanKey::TraceId => span.trace_id = read_number_bytes(buf)?, - SpanKey::SpanId => span.span_id = read_number_bytes(buf)?, - SpanKey::ParentId => span.parent_id = read_number_bytes(buf)?, - SpanKey::Start => span.start = read_number_bytes(buf)?, - SpanKey::Duration => span.duration = read_number_bytes(buf)?, - SpanKey::Error => span.error = read_number_bytes(buf)?, - SpanKey::Type => span.r#type = read_string_bytes(buf)?, - SpanKey::Meta => span.meta = read_str_map_to_bytes_strings(buf)?, + SpanKey::Service => span.service = read_nullable_string_bytes(buf)?, + SpanKey::Name => span.name = read_nullable_string_bytes(buf)?, + SpanKey::Resource => span.resource = read_nullable_string_bytes(buf)?, + SpanKey::TraceId => span.trace_id = read_nullable_number_bytes(buf)?, + SpanKey::SpanId => span.span_id = read_nullable_number_bytes(buf)?, + SpanKey::ParentId => span.parent_id = read_nullable_number_bytes(buf)?, + SpanKey::Start => span.start = read_nullable_number_bytes(buf)?, + SpanKey::Duration => span.duration = read_nullable_number_bytes(buf)?, + SpanKey::Error => span.error = read_nullable_number_bytes(buf)?, + SpanKey::Type => span.r#type = read_nullable_string_bytes(buf)?, + SpanKey::Meta => span.meta = read_nullable_str_map_to_bytes_strings(buf)?, SpanKey::Metrics => span.metrics = read_metrics(buf)?, SpanKey::MetaStruct => span.meta_struct = read_meta_struct(buf)?, SpanKey::SpanLinks => span.span_links = read_span_links(buf)?, diff --git a/trace-utils/src/msgpack_decoder/v04/decoder/span_link.rs b/trace-utils/src/msgpack_decoder/v04/decoder/span_link.rs index 360e4522a..2e9c4ec5f 100644 --- a/trace-utils/src/msgpack_decoder/v04/decoder/span_link.rs +++ b/trace-utils/src/msgpack_decoder/v04/decoder/span_link.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::msgpack_decoder::v04::decoder::{ - read_str_map_to_bytes_strings, read_string_bytes, read_string_ref, + handle_null_marker, read_str_map_to_bytes_strings, read_string_bytes, read_string_ref, }; use crate::msgpack_decoder::v04::error::DecodeError; use crate::msgpack_decoder::v04::number::read_number_bytes; @@ -29,6 +29,10 @@ use tinybytes::Bytes; /// - Any `SpanLink` cannot be decoded. /// ``` pub(crate) fn read_span_links(buf: &mut Bytes) -> Result, DecodeError> { + if let Some(empty_vec) = handle_null_marker(buf, Vec::default) { + return Ok(empty_vec); + } + match rmp::decode::read_marker(unsafe { buf.as_mut_slice() }).map_err(|_| { DecodeError::InvalidFormat("Unable to read marker for span links".to_owned()) })? { diff --git a/trace-utils/src/msgpack_decoder/v04/number.rs b/trace-utils/src/msgpack_decoder/v04/number.rs index 4e0fce7f4..219591481 100644 --- a/trace-utils/src/msgpack_decoder/v04/number.rs +++ b/trace-utils/src/msgpack_decoder/v04/number.rs @@ -150,7 +150,7 @@ impl TryFrom for f64 { } } -pub fn read_number(buf: &mut &[u8]) -> Result { +fn read_number(buf: &mut &[u8], allow_null: bool) -> Result { match rmp::decode::read_marker(buf) .map_err(|_| DecodeError::InvalidFormat("Unable to read marker for number".to_owned()))? { @@ -186,6 +186,13 @@ pub fn read_number(buf: &mut &[u8]) -> Result { Marker::F64 => Ok(Number::Float( buf.read_data_f64().map_err(|_| DecodeError::IOError)?, )), + Marker::Null => { + if allow_null { + Ok(Number::Unsigned(0)) + } else { + Err(DecodeError::InvalidType("Invalid number type".to_owned())) + } + } _ => Err(DecodeError::InvalidType("Invalid number type".to_owned())), } } @@ -193,14 +200,99 @@ pub fn read_number(buf: &mut &[u8]) -> Result { pub fn read_number_bytes>( buf: &mut Bytes, ) -> Result { - read_number(unsafe { buf.as_mut_slice() })?.try_into() + read_number(unsafe { buf.as_mut_slice() }, false)?.try_into() +} + +pub fn read_nullable_number_bytes>( + buf: &mut Bytes, +) -> Result { + read_number(unsafe { buf.as_mut_slice() }, true)?.try_into() } #[cfg(test)] mod tests { use super::*; + use serde_json::json; use std::f64; + #[test] + fn test_decoding_not_nullable_bytes_to_unsigned() { + let mut buf = Vec::new(); + let expected_value = 42; + let val = json!(expected_value); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: u8 = read_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, expected_value); + } + + #[test] + fn test_decoding_not_nullable_bytes_to_signed() { + let mut buf = Vec::new(); + let expected_value = 42; + let val = json!(expected_value); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: i8 = read_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, expected_value); + } + + #[test] + fn test_decoding_not_nullable_bytes_to_float() { + let mut buf = Vec::new(); + let expected_value = 42.98; + let val = json!(expected_value); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: f64 = read_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, expected_value); + } + + #[test] + fn test_decoding_null_through_read_number_bytes_raises_exception() { + let mut buf = Vec::new(); + let val = json!(null); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: Result = read_number_bytes(&mut bytes); + assert!(matches!(result, Err(DecodeError::InvalidType(_)))); + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid type encountered: Invalid number type".to_owned() + ); + } + + #[test] + fn test_decoding_null_bytes_to_unsigned() { + let mut buf = Vec::new(); + let val = json!(null); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: u8 = read_nullable_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, 0); + } + + #[test] + fn test_decoding_null_bytes_to_signed() { + let mut buf = Vec::new(); + let val = json!(null); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: i8 = read_nullable_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, 0); + } + + #[test] + fn test_decoding_null_bytes_to_float() { + let mut buf = Vec::new(); + let val = json!(null); + rmp_serde::encode::write_named(&mut buf, &val).unwrap(); + let mut bytes = Bytes::from(buf.clone()); + let result: f64 = read_nullable_number_bytes(&mut bytes).unwrap(); + assert_eq!(result, 0.0); + } + #[test] fn test_i64_conversions() { let valid_max = i64::MAX; @@ -296,6 +388,66 @@ mod tests { ); } + #[test] + fn test_i8_null_conversions() { + let valid_signed_upper = i8::MAX; + let valid_unsigned_number = Number::Unsigned(valid_signed_upper as u64); + let zero_unsigned = Number::Unsigned(0u64); + let zero_signed = Number::Unsigned(0u64); + let valid_signed_number_upper = Number::Signed(valid_signed_upper as i64); + let valid_signed_lower = i8::MIN; + let valid_signed_number_lower = Number::Signed(valid_signed_lower as i64); + let invalid_float_number = Number::Float(4.14); + let invalid_unsigned = u8::MAX; + let invalid_unsigned_number = Number::Unsigned(invalid_unsigned as u64); + let invalid_signed_upper = i8::MAX as i64 + 1; + let invalid_signed_number_upper = Number::Signed(invalid_signed_upper); + let invalid_signed_lower = i8::MIN as i64 - 1; + let invalid_signed_number_lower = Number::Signed(invalid_signed_lower); + + assert_eq!( + valid_signed_upper, + TryInto::::try_into(valid_unsigned_number).unwrap() + ); + assert_eq!( + valid_signed_upper, + TryInto::::try_into(valid_signed_number_upper).unwrap() + ); + assert_eq!( + valid_signed_lower, + TryInto::::try_into(valid_signed_number_lower).unwrap() + ); + assert_eq!(0, TryInto::::try_into(zero_signed).unwrap()); + assert_eq!(0, TryInto::::try_into(zero_unsigned).unwrap()); + assert_eq!( + Err(DecodeError::InvalidConversion( + "Cannot convert float to int".to_owned() + )), + TryInto::::try_into(invalid_float_number) + ); + assert_eq!( + Err(DecodeError::InvalidConversion(format!( + "{} is out of bounds for conversion", + invalid_unsigned + ))), + TryInto::::try_into(invalid_unsigned_number) + ); + assert_eq!( + Err(DecodeError::InvalidConversion(format!( + "{} is out of bounds for conversion", + invalid_signed_upper + ))), + TryInto::::try_into(invalid_signed_number_upper) + ); + assert_eq!( + Err(DecodeError::InvalidConversion(format!( + "{} is out of bounds for conversion", + invalid_signed_lower + ))), + TryInto::::try_into(invalid_signed_number_lower) + ); + } + #[test] fn test_i8_conversions() { let valid_signed_upper = i8::MAX; diff --git a/trace-utils/src/send_data/mod.rs b/trace-utils/src/send_data/mod.rs index c61163077..e2ed00508 100644 --- a/trace-utils/src/send_data/mod.rs +++ b/trace-utils/src/send_data/mod.rs @@ -924,7 +924,8 @@ mod tests { mock.assert_hits_async(5).await; - assert!(res.last_result.is_err()); + assert!(res.last_result.is_ok()); + assert_eq!(res.last_result.unwrap().status(), 500); assert_eq!(res.errors_timeout, 0); assert_eq!(res.errors_network, 0); assert_eq!(res.errors_status_code, 1); diff --git a/trace-utils/src/send_data/send_data_result.rs b/trace-utils/src/send_data/send_data_result.rs index ff7589c23..6245a81eb 100644 --- a/trace-utils/src/send_data/send_data_result.rs +++ b/trace-utils/src/send_data/send_data_result.rs @@ -3,7 +3,6 @@ use crate::send_data::RequestResult; use anyhow::anyhow; -use hyper::body::HttpBody; use hyper::{Body, Response}; use std::collections::HashMap; @@ -73,15 +72,7 @@ impl SendDataResult { .or_default() += 1; self.chunks_dropped += chunks; self.requests_count += u64::from(attempts); - - let body = response.into_body().collect().await; - let response_body = String::from_utf8(body.unwrap_or_default().to_bytes().to_vec()) - .unwrap_or_default(); - self.last_result = Err(anyhow::format_err!( - "{} - Server did not accept traces: {}", - status_code, - response_body, - )); + self.last_result = Ok(response); } RequestResult::TimeoutError((attempts, chunks)) => { self.errors_timeout += 1; diff --git a/trace-utils/src/span_v04/mod.rs b/trace-utils/src/span_v04/mod.rs new file mode 100644 index 000000000..cf8d34ee0 --- /dev/null +++ b/trace-utils/src/span_v04/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +mod span; + +pub mod trace_utils; + +pub use span::{Span, SpanKey, SpanKeyParseError, SpanLink}; diff --git a/trace-utils/src/span_v04.rs b/trace-utils/src/span_v04/span.rs similarity index 100% rename from trace-utils/src/span_v04.rs rename to trace-utils/src/span_v04/span.rs diff --git a/trace-utils/src/span_v04/trace_utils.rs b/trace-utils/src/span_v04/trace_utils.rs new file mode 100644 index 000000000..b95e141cf --- /dev/null +++ b/trace-utils/src/span_v04/trace_utils.rs @@ -0,0 +1,172 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Trace-utils functionalities implementation for tinybytes based spans + +use std::collections::HashMap; +use tinybytes::BytesString; + +use super::Span; + +/// Span metric the mini agent must set for the backend to recognize top level span +const TOP_LEVEL_KEY: &str = "_top_level"; +/// Span metric the tracer sets to denote a top level span +const TRACER_TOP_LEVEL_KEY: &str = "_dd.top_level"; +const MEASURED_KEY: &str = "_dd.measured"; +const PARTIAL_VERSION_KEY: &str = "_dd.partial_version"; + +fn set_top_level_span(span: &mut Span, is_top_level: bool) { + if is_top_level { + span.metrics.insert(TOP_LEVEL_KEY.into(), 1.0); + } else { + span.metrics.remove(TOP_LEVEL_KEY); + } +} + +/// Updates all the spans top-level attribute. +/// A span is considered top-level if: +/// - it's a root span +/// - OR its parent is unknown (other part of the code, distributed trace) +/// - OR its parent belongs to another service (in that case it's a "local root" being the highest +/// ancestor of other spans belonging to this service and attached to it). +pub fn compute_top_level_span(trace: &mut [Span]) { + let mut span_id_to_service: HashMap = HashMap::new(); + for span in trace.iter() { + span_id_to_service.insert(span.span_id, span.service.clone()); + } + for span in trace.iter_mut() { + if span.parent_id == 0 { + set_top_level_span(span, true); + continue; + } + match span_id_to_service.get(&span.parent_id) { + Some(parent_span_service) => { + if !parent_span_service.eq(&span.service) { + // parent is not in the same service + set_top_level_span(span, true) + } + } + None => { + // span has no parent in chunk + set_top_level_span(span, true) + } + } + } +} + +/// Return true if the span has a top level key set +pub fn has_top_level(span: &Span) -> bool { + span.metrics + .get(TRACER_TOP_LEVEL_KEY) + .is_some_and(|v| *v == 1.0) + || span.metrics.get(TOP_LEVEL_KEY).is_some_and(|v| *v == 1.0) +} + +/// Returns true if a span should be measured (i.e., it should get trace metrics calculated). +pub fn is_measured(span: &Span) -> bool { + span.metrics.get(MEASURED_KEY).is_some_and(|v| *v == 1.0) +} + +/// Returns true if the span is a partial snapshot. +/// This kind of spans are partial images of long-running spans. +/// When incomplete, a partial snapshot has a metric _dd.partial_version which is a positive +/// integer. The metric usually increases each time a new version of the same span is sent by +/// the tracer +pub fn is_partial_snapshot(span: &Span) -> bool { + span.metrics + .get(PARTIAL_VERSION_KEY) + .is_some_and(|v| *v >= 0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_span( + trace_id: u64, + span_id: u64, + parent_id: u64, + start: i64, + is_top_level: bool, + ) -> Span { + let mut span = Span { + trace_id, + span_id, + service: "test-service".into(), + name: "test_name".into(), + resource: "test-resource".into(), + parent_id, + start, + duration: 5, + error: 0, + meta: HashMap::from([ + ("service".into(), "test-service".into()), + ("env".into(), "test-env".into()), + ("runtime-id".into(), "test-runtime-id-value".into()), + ]), + metrics: HashMap::new(), + r#type: "".into(), + meta_struct: HashMap::new(), + span_links: vec![], + }; + if is_top_level { + span.metrics.insert("_top_level".into(), 1.0); + span.meta + .insert("_dd.origin".into(), "cloudfunction".into()); + span.meta.insert("origin".into(), "cloudfunction".into()); + span.meta + .insert("functionname".into(), "dummy_function_name".into()); + span.r#type = "serverless".into(); + } + span + } + + #[test] + fn test_has_top_level() { + let top_level_span = create_test_span(123, 1234, 12, 1, true); + let not_top_level_span = create_test_span(123, 1234, 12, 1, false); + assert!(has_top_level(&top_level_span)); + assert!(!has_top_level(¬_top_level_span)); + } + + #[test] + fn test_is_measured() { + let mut measured_span = create_test_span(123, 1234, 12, 1, true); + measured_span.metrics.insert(MEASURED_KEY.into(), 1.0); + let not_measured_span = create_test_span(123, 1234, 12, 1, true); + assert!(is_measured(&measured_span)); + assert!(!is_measured(¬_measured_span)); + } + + #[test] + fn test_compute_top_level() { + let mut span_with_different_service = create_test_span(123, 5, 2, 1, false); + span_with_different_service.service = "another_service".into(); + let mut trace = vec![ + // Root span, should be marked as top-level + create_test_span(123, 1, 0, 1, false), + // Should not be marked as top-level + create_test_span(123, 2, 1, 1, false), + // No parent in local trace, should be marked as + // top-level + create_test_span(123, 4, 3, 1, false), + // Parent belongs to another service, should be marked + // as top-level + span_with_different_service, + ]; + + compute_top_level_span(trace.as_mut_slice()); + + let spans_marked_as_top_level: Vec = trace + .iter() + .filter_map(|span| { + if has_top_level(span) { + Some(span.span_id) + } else { + None + } + }) + .collect(); + assert_eq!(spans_marked_as_top_level, [1, 4, 5]) + } +} diff --git a/trace-utils/src/test_utils/mod.rs b/trace-utils/src/test_utils/mod.rs index 357260e77..47848cd82 100644 --- a/trace-utils/src/test_utils/mod.rs +++ b/trace-utils/src/test_utils/mod.rs @@ -118,6 +118,100 @@ pub fn create_test_span( span } +pub fn create_test_gcp_span( + trace_id: u64, + span_id: u64, + parent_id: u64, + start: i64, + is_top_level: bool, +) -> pb::Span { + let mut span = pb::Span { + trace_id, + span_id, + service: "test-service".to_string(), + name: "test_name".to_string(), + resource: "test-resource".to_string(), + parent_id, + start, + duration: 5, + error: 0, + meta: HashMap::from([ + ("service".to_string(), "test-service".to_string()), + ("env".to_string(), "test-env".to_string()), + ( + "runtime-id".to_string(), + "test-runtime-id-value".to_string(), + ), + ]), + metrics: HashMap::new(), + r#type: "".to_string(), + meta_struct: HashMap::new(), + span_links: vec![], + }; + span.meta.insert( + "_dd.mini_agent_version".to_string(), + "dummy_version".to_string(), + ); + span.meta.insert( + "gcrfx.project_id".to_string(), + "dummy_project_id".to_string(), + ); + span.meta.insert( + "gcrfx.location".to_string(), + "dummy_region_west".to_string(), + ); + span.meta.insert( + "gcrfx.resource_name".to_string(), + "projects/dummy_project_id/locations/dummy_region_west/functions/dummy_function_name" + .to_string(), + ); + if is_top_level { + span.meta.insert( + "functionname".to_string(), + "dummy_function_name".to_string(), + ); + span.metrics.insert("_top_level".to_string(), 1.0); + span.meta + .insert("_dd.origin".to_string(), "cloudfunction".to_string()); + span.meta + .insert("origin".to_string(), "cloudfunction".to_string()); + span.r#type = "serverless".to_string(); + } + span +} + +pub fn create_test_gcp_json_span( + trace_id: u64, + span_id: u64, + parent_id: u64, + start: i64, +) -> serde_json::Value { + json!( + { + "trace_id": trace_id, + "span_id": span_id, + "service": "test-service", + "name": "test_name", + "resource": "test-resource", + "parent_id": parent_id, + "start": start, + "duration": 5, + "error": 0, + "meta": { + "service": "test-service", + "env": "test-env", + "runtime-id": "test-runtime-id-value", + "gcrfx.project_id": "dummy_project_id", + "_dd.mini_agent_version": "dummy_version", + "gcrfx.resource_name": "projects/dummy_project_id/locations/dummy_region_west/functions/dummy_function_name", + "gcrfx.location": "dummy_region_west" + }, + "metrics": {}, + "meta_struct": {}, + } + ) +} + pub fn create_test_json_span( trace_id: u64, span_id: u64, diff --git a/trace-utils/src/trace_utils.rs b/trace-utils/src/trace_utils.rs index 1a8559556..6cbc20fe0 100644 --- a/trace-utils/src/trace_utils.rs +++ b/trace-utils/src/trace_utils.rs @@ -1,8 +1,16 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +pub use crate::send_data::send_data_result::SendDataResult; +pub use crate::send_data::SendData; +pub use crate::tracer_header_tags::TracerHeaderTags; +use crate::tracer_payload; +use crate::tracer_payload::{TraceCollection, TracerPayloadCollection}; use anyhow::anyhow; use bytes::buf::Reader; +use datadog_trace_normalization::normalizer; +use datadog_trace_protobuf::pb; +use ddcommon::azure_app_services; use hyper::body::HttpBody; use hyper::{body::Buf, Body}; use log::error; @@ -13,15 +21,6 @@ use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; use std::env; -pub use crate::send_data::send_data_result::SendDataResult; -pub use crate::send_data::SendData; -pub use crate::tracer_header_tags::TracerHeaderTags; -use crate::tracer_payload; -use crate::tracer_payload::{TraceCollection, TracerPayloadCollection}; -use datadog_trace_normalization::normalizer; -use datadog_trace_protobuf::pb; -use ddcommon::azure_app_services; - /// Span metric the mini agent must set for the backend to recognize top level span const TOP_LEVEL_KEY: &str = "_top_level"; /// Span metric the tracer sets to denote a top level span @@ -257,7 +256,7 @@ pub async fn get_v05_traces_from_request_body( Ok((body_size, traces)) } -// Tags gathered from a trace's root span +/// Tags gathered from a trace's root span #[derive(Default)] pub struct RootSpanTags<'a> { pub env: &'a str, @@ -425,13 +424,11 @@ pub fn has_top_level(span: &pb::Span) -> bool { } fn set_top_level_span(span: &mut pb::Span, is_top_level: bool) { - if !is_top_level { - if span.metrics.contains_key(TOP_LEVEL_KEY) { - span.metrics.remove(TOP_LEVEL_KEY); - } - return; + if is_top_level { + span.metrics.insert(TOP_LEVEL_KEY.to_string(), 1.0); + } else { + span.metrics.remove(TOP_LEVEL_KEY); } - span.metrics.insert(TOP_LEVEL_KEY.to_string(), 1.0); } pub fn set_serverless_root_span_tags( @@ -512,14 +509,6 @@ pub fn enrich_span_with_mini_agent_metadata( span.meta .insert("asa.name".to_string(), azure_spring_app_name.to_string()); } - if let Some(gcp_project_id) = &mini_agent_metadata.gcp_project_id { - span.meta - .insert("gcrfx.project_id".to_string(), gcp_project_id.to_string()); - } - if let Some(gcp_region) = &mini_agent_metadata.gcp_region { - span.meta - .insert("gcrfx.location".to_string(), gcp_region.to_string()); - } if let Some(mini_agent_version) = &mini_agent_metadata.version { span.meta.insert( "_dd.mini_agent_version".to_string(), @@ -528,6 +517,34 @@ pub fn enrich_span_with_mini_agent_metadata( } } +pub fn enrich_span_with_google_cloud_function_metadata( + span: &mut pb::Span, + mini_agent_metadata: &MiniAgentMetadata, + function: Option, +) { + let Some(region) = &mini_agent_metadata.gcp_region else { + todo!() + }; + let Some(project) = &mini_agent_metadata.gcp_project_id else { + todo!() + }; + if function.is_some() && !region.is_empty() && !project.is_empty() { + let resource_name = format!( + "projects/{}/locations/{}/functions/{}", + project, + region, + function.unwrap() + ); + + span.meta + .insert("gcrfx.location".to_string(), region.to_string()); + span.meta + .insert("gcrfx.project_id".to_string(), project.to_string()); + span.meta + .insert("gcrfx.resource_name".to_string(), resource_name.to_string()); + } +} + pub fn enrich_span_with_azure_function_metadata(span: &mut pb::Span) { if let Some(aas_metadata) = azure_app_services::get_function_metadata() { let aas_tags = [ @@ -651,7 +668,7 @@ pub fn collect_trace_chunks( } } -// Returns true if a span should be measured (i.e., it should get trace metrics calculated). +/// Returns true if a span should be measured (i.e., it should get trace metrics calculated). pub fn is_measured(span: &pb::Span) -> bool { span.metrics.get(MEASURED_KEY).is_some_and(|v| *v == 1.0) } @@ -669,32 +686,23 @@ pub fn is_partial_snapshot(span: &pb::Span) -> bool { #[cfg(test)] mod tests { + use super::*; + use crate::test_utils::create_test_span; + use ddcommon::Endpoint; use hyper::Request; use serde_json::json; - use std::collections::HashMap; - - use super::{get_root_span_index, set_serverless_root_span_tags}; - use crate::trace_utils::{has_top_level, TracerHeaderTags, MAX_PAYLOAD_SIZE}; - use crate::tracer_payload::TracerPayloadCollection; - use crate::{ - test_utils::create_test_span, - trace_utils::{self, SendData}, - }; - use datadog_trace_protobuf::pb::TraceChunk; - use datadog_trace_protobuf::pb::{Span, TracerPayload}; - use ddcommon::Endpoint; #[test] fn test_coalescing_does_not_exceed_max_size() { let dummy = SendData::new( MAX_PAYLOAD_SIZE / 5 + 1, - TracerPayloadCollection::V07(vec![TracerPayload { + TracerPayloadCollection::V07(vec![pb::TracerPayload { container_id: "".to_string(), language_name: "".to_string(), language_version: "".to_string(), tracer_version: "".to_string(), runtime_id: "".to_string(), - chunks: vec![TraceChunk { + chunks: vec![pb::TraceChunk { priority: 0, origin: "".to_string(), spans: vec![], @@ -709,7 +717,7 @@ mod tests { TracerHeaderTags::default(), &Endpoint::default(), ); - let coalesced = trace_utils::coalesce_send_data(vec![ + let coalesced = coalesce_send_data(vec![ dummy.clone(), dummy.clone(), dummy.clone(), @@ -744,7 +752,7 @@ mod tests { #[tokio::test] #[allow(clippy::type_complexity)] #[cfg_attr(all(miri, target_os = "macos"), ignore)] - async fn get_v05_traces_from_request_body() { + async fn test_get_v05_traces_from_request_body() { let data: ( Vec, Vec< @@ -793,12 +801,11 @@ mod tests { )]], ); let bytes = rmp_serde::to_vec(&data).unwrap(); - let res = - trace_utils::get_v05_traces_from_request_body(hyper::body::Body::from(bytes)).await; + let res = get_v05_traces_from_request_body(hyper::body::Body::from(bytes)).await; assert!(res.is_ok()); let (_, traces) = res.unwrap(); let span = traces[0][0].clone(); - let test_span = Span { + let test_span = pb::Span { service: "my-service".to_string(), name: "my-name".to_string(), resource: "my-resource".to_string(), @@ -842,7 +849,7 @@ mod tests { "meta": {}, "metrics": {}, }]), - vec![vec![Span { + vec![vec![pb::Span { service: "test-service".to_string(), name: "test-service-name".to_string(), resource: "test-service-resource".to_string(), @@ -869,7 +876,7 @@ mod tests { "duration": 5, "meta": {}, }]), - vec![vec![Span { + vec![vec![pb::Span { service: "".to_string(), name: "test-service-name".to_string(), resource: "test-service-resource".to_string(), @@ -893,7 +900,7 @@ mod tests { let request = Request::builder() .body(hyper::body::Body::from(bytes)) .unwrap(); - let res = trace_utils::get_traces_from_request_body(request.into_body()).await; + let res = get_traces_from_request_body(request.into_body()).await; assert!(res.is_ok()); assert_eq!(res.unwrap().1, output); } @@ -932,7 +939,7 @@ mod tests { set_serverless_root_span_tags( &mut span, Some("test_function".to_string()), - &trace_utils::EnvironmentType::AzureFunction, + &EnvironmentType::AzureFunction, ); assert_eq!( span.meta, @@ -957,7 +964,7 @@ mod tests { set_serverless_root_span_tags( &mut span, Some("test_function".to_string()), - &trace_utils::EnvironmentType::CloudFunction, + &EnvironmentType::CloudFunction, ); assert_eq!( span.meta, @@ -983,4 +990,45 @@ mod tests { assert!(has_top_level(&top_level_span)); assert!(!has_top_level(¬_top_level_span)); } + + #[test] + fn test_is_measured() { + let mut measured_span = create_test_span(123, 1234, 12, 1, true); + measured_span.metrics.insert(MEASURED_KEY.into(), 1.0); + let not_measured_span = create_test_span(123, 1234, 12, 1, true); + assert!(is_measured(&measured_span)); + assert!(!is_measured(¬_measured_span)); + } + + #[test] + fn test_compute_top_level() { + let mut span_with_different_service = create_test_span(123, 5, 2, 1, false); + span_with_different_service.service = "another_service".into(); + let mut trace = vec![ + // Root span, should be marked as top-level + create_test_span(123, 1, 0, 1, false), + // Should not be marked as top-level + create_test_span(123, 2, 1, 1, false), + // No parent in local trace, should be marked as + // top-level + create_test_span(123, 4, 3, 1, false), + // Parent belongs to another service, should be marked + // as top-level + span_with_different_service, + ]; + + compute_top_level_span(trace.as_mut_slice()); + + let spans_marked_as_top_level: Vec = trace + .iter() + .filter_map(|span| { + if has_top_level(span) { + Some(span.span_id) + } else { + None + } + }) + .collect(); + assert_eq!(spans_marked_as_top_level, [1, 4, 5]) + } }