More expressive error for EMSGSIZE from Jaeger reporter

Cargo.toml

@@ -39,6 +39,7 @@ governor = "0.6"
 hyper = { version = "0.14", default-features = false }
 indexmap = "2.0.0"
 ipnetwork = "0.20"
+libc = "0.2"
 once_cell = "1.5"
 tonic = { version = "0.11.0", default-features = false }
 opentelemetry-proto = "0.5.0"

foundations/Cargo.toml

@@ -120,6 +120,7 @@ tracing = [
     "dep:futures-util",
     "dep:tokio",
     "dep:serde",
+    "dep:libc",
 ]
 
 # Enables metrics functionality.
@@ -179,6 +180,7 @@ hyper = { workspace = true, optional = true, features = [
     "server",
 ] }
 indexmap = { workspace = true, optional = true, features = ["serde"] }
+libc = { workspace = true, optional = true }
 once_cell = { workspace = true, optional = true }
 opentelemetry-proto = { workspace = true, optional = true, features = ["gen-tonic-messages", "trace"] }
 parking_lot = { workspace = true, optional = true }

foundations/src/telemetry/tracing/output_jaeger_thrift_udp.rs

@@ -4,8 +4,9 @@ use crate::{BootstrapResult, ServiceInfo};
 use anyhow::bail;
 use cf_rustracing::tag::Tag;
 use cf_rustracing_jaeger::reporter::JaegerCompactReporter;
-use cf_rustracing_jaeger::span::SpanReceiver;
+use cf_rustracing_jaeger::span::{FinishedSpan, SpanReceiver};
 use futures_util::future::{BoxFuture, FutureExt as _};
+use std::io;
 use std::net::SocketAddr;
 use std::net::{Ipv4Addr, Ipv6Addr};
 
@@ -60,8 +61,65 @@ fn get_reporter_bind_addr(settings: &JaegerThriftUdpOutputSettings) -> Bootstrap
 async fn do_export(reporter: JaegerCompactReporter, mut span_rx: SpanReceiver) {
     while let Some(span) = span_rx.recv().await {
         // NOTE: we are limited with a UDP dgram size here, so doing batching is risky.
-        if let Err(err) = reporter.report(&[span][..]).await {
+        let spans = [span];
+        if let Err(err) = reporter.report(&spans).await {
+            #[cfg(feature = "logging")]
+            if is_msgsize_error(&err) {
+                span_too_large(&err, &spans[0]);
+                continue;
+            }
+
             reporter_error(err);
         }
     }
 }
+
+fn is_msgsize_error(err: &cf_rustracing::Error) -> bool {
+    err.concrete_cause::<io::Error>()
+        .and_then(io::Error::raw_os_error)
+        .map_or(false, |errno| errno == libc::EMSGSIZE)
+}
+
+#[cfg(feature = "logging")]
+fn span_too_large(err: &cf_rustracing::Error, span: &FinishedSpan) {
+    let tag_count = span.tags().len();
+    let tag_total: usize = span.tags().iter().map(tag_size).sum();
+    let top_tag = span.tags().iter().max_by_key(|t| tag_size(t));
+
+    let log_count = span.logs().len();
+    let log_total: usize = span.logs().iter().map(log_size).sum();
+
+    let top_tag = top_tag
+        .map(|t| format!(", top: {} @ {}", t.name(), tag_size(t)))
+        .unwrap_or_default();
+
+    crate::telemetry::log::error!(
+        "trace span exceeded thrift UDP message size limits";
+        "error" => %err,
+        "operation" => span.operation_name(),
+        "tags" => format!("count: {tag_count}, size: {tag_total}{top_tag}"),
+        "logs" => format!("count: {log_count}, size: {log_total}"),
+    );
+}
+
+/// Approximates the wire size of a span `Tag`. This is not exact and
+/// more closely resembles the non-compact thrift encoding, but should be
+/// sufficient to determine what causes the span to trigger EMSGSIZE.
+fn tag_size(tag: &Tag) -> usize {
+    use cf_rustracing::tag::TagValue;
+    let val_size = match tag.value() {
+        TagValue::String(s) => s.len(),
+        TagValue::Boolean(_) => 1,
+        TagValue::Integer(_) => 8,
+        TagValue::Float(_) => 8,
+    };
+    tag.name().len() + val_size
+}
+
+/// Approximates the wire size of a span `Log`, which is always stringified.
+fn log_size(log: &cf_rustracing::log::Log) -> usize {
+    log.fields()
+        .iter()
+        .map(|f| f.name().len() + f.value().len())
+        .sum()
+}