feat: populate metric labels from span context using metrics-tracing-context

Cargo.toml

@@ -98,6 +98,8 @@ bytes = "=1.11.1"
 log = "=0.4.29"
 metrics = "=0.24.2"
 metrics-exporter-prometheus = { version = "=0.17.2", optional = true }
+metrics-tracing-context = { version = "=0.18.1", optional = true }
+metrics-util = { version = "=0.20.0", optional = true }
 opentelemetry = "=0.31.0"
 opentelemetry_sdk = { version = "=0.31.0", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "=0.31.0", features = [
@@ -225,7 +227,7 @@ dev = []
 tools = ["dep:indicatif"]
 
 # Enable runtime metrics collection.
-metrics = ["dep:metrics-exporter-prometheus"]
+metrics = ["dep:metrics-exporter-prometheus", "dep:metrics-tracing-context", "dep:metrics-util"]
 
 # Enable runtime rocksdb metrics collection.
 rocks_metrics = ["metrics"]

config/stratus-follower.env.local

@@ -1,6 +1,7 @@
 RUST_LOG=info,stratus::eth::rpc::rpc_subscriptions::rx=off,stratus::eth::consensus::rx=off,stratus::eth::consensus=off,jsonrpsee-server=debug
 
 ADDRESS=0.0.0.0:3001
+METRICS_EXPORTER_ADDRESS=0.0.0.0:9001
 
 CHAIN_ID=2008
 EVMS=1

src/eth/rpc/server.rs

@@ -1124,7 +1124,13 @@ fn stratus_get_transaction_result(params: Params<'_>, ctx: Arc<RpcContext>, ext:
 fn eth_estimate_gas(params: Params<'_>, ctx: Arc<RpcContext>, ext: Extensions) -> Result<String, StratusError> {
     // enter span
     let _middleware_enter = ext.enter_middleware_span();
-    let _method_enter = info_span!("rpc::eth_estimateGas", tx_from = field::Empty, tx_to = field::Empty).entered();
+    let _method_enter = info_span!(
+        "rpc::eth_estimateGas",
+        tx_from = field::Empty,
+        tx_to = field::Empty,
+        point_in_time = field::Empty
+    )
+    .entered();
 
     // parse params
     let (_, call) = next_rpc_param::<CallInput>(params.sequence())?;
@@ -1134,6 +1140,7 @@ fn eth_estimate_gas(params: Params<'_>, ctx: Arc<RpcContext>, ext: Extensions) -
         s.rec_opt("tx_from", &call.from);
         s.rec_opt("tx_to", &call.to);
     });
+    Span::with(|s| s.rec_str("point_in_time", &PointInTime::Latest));
     tracing::info!("executing eth_estimateGas");
 
     // execute
@@ -1181,6 +1188,7 @@ fn rpc_call(params: Params<'_>, ctx: Arc<RpcContext>) -> Result<TransactionExecu
 
     // execute
     let point_in_time = ctx.server.storage.translate_to_point_in_time(filter)?;
+    Span::with(|s| s.rec_str("point_in_time", &point_in_time));
     if let Some(to_address) = call.to
         && !call.data.is_empty()
     {
@@ -1192,7 +1200,14 @@ fn rpc_call(params: Params<'_>, ctx: Arc<RpcContext>) -> Result<TransactionExecu
 fn eth_call(params: Params<'_>, ctx: Arc<RpcContext>, ext: Extensions) -> Result<String, StratusError> {
     // enter span
     let _middleware_enter = ext.enter_middleware_span();
-    let _method_enter = info_span!("rpc::eth_call", tx_from = field::Empty, tx_to = field::Empty, filter = field::Empty).entered();
+    let _method_enter = info_span!(
+        "rpc::eth_call",
+        tx_from = field::Empty,
+        tx_to = field::Empty,
+        filter = field::Empty,
+        point_in_time = field::Empty
+    )
+    .entered();
 
     match rpc_call(params, ctx) {
         // result is success

src/globals.rs

@@ -68,14 +68,12 @@ where
         // init tokio
         let tokio = common.init_tokio_runtime().expect("failed to init tokio runtime");
 
-        // init tracing
+        // init observability services
         tokio.block_on(async {
             common.tracing.init(&common.sentry).expect("failed to init tracing");
+            common.metrics.init().expect("failed to init metrics");
         });
 
-        // init observability services
-        common.metrics.init().expect("failed to init metrics");
-
         // init sentry
         let sentry_guard = common
             .sentry

src/infra/metrics/metrics_config.rs

@@ -1,8 +1,13 @@
 use std::net::SocketAddr;
-use std::stringify;
 
 use clap::Parser;
 use display_json::DebugAsJson;
+#[cfg(feature = "metrics")]
+use metrics_exporter_prometheus::PrometheusBuilder;
+#[cfg(feature = "metrics")]
+use metrics_tracing_context::TracingContextLayer as MetricsTracingContextLayer;
+#[cfg(feature = "metrics")]
+use metrics_util::layers::Layer as MetricsLayerExt;
 
 use crate::infra::metrics::metrics_for_consensus;
 use crate::infra::metrics::metrics_for_executor;
@@ -37,7 +42,7 @@ impl MetricsConfig {
         metrics.extend(metrics_for_kafka());
 
         // init metric exporter
-        init_metrics_exporter(self.metrics_exporter_address);
+        init_metrics_exporter(self.metrics_exporter_address)?;
 
         // init metric description (always after provider started)
         for metric in &metrics {
@@ -49,19 +54,32 @@ impl MetricsConfig {
 }
 
 #[cfg(feature = "metrics")]
-fn init_metrics_exporter(address: SocketAddr) {
+fn init_metrics_exporter(address: SocketAddr) -> anyhow::Result<()> {
     tracing::info!(%address, "creating prometheus metrics exporter");
-    if let Err(e) = metrics_exporter_prometheus::PrometheusBuilder::new()
+
+    let builder = PrometheusBuilder::new()
         .add_global_label("service", crate::infra::build_info::service_name())
         .add_global_label("version", crate::infra::build_info::version())
-        .with_http_listener(address)
-        .install()
-    {
-        tracing::error!(reason = ?e, %address, "failed to create metrics exporter");
-    }
+        .with_http_listener(address);
+
+    install_metrics_tracing_recorder(builder)?;
+
+    Ok(())
+}
+
+#[cfg(feature = "metrics")]
+fn install_metrics_tracing_recorder(builder: PrometheusBuilder) -> anyhow::Result<()> {
+    let (recorder, exporter) = builder.build()?;
+    tokio::spawn(exporter);
+
+    let recorder = MetricsTracingContextLayer::only_allow(["rpc_client", "rpc_method", "point_in_time"]).layer(recorder);
+    metrics::set_global_recorder(recorder)?;
+
+    Ok(())
 }
 
 #[cfg(not(feature = "metrics"))]
-fn init_metrics_exporter(_: SocketAddr) {
+fn init_metrics_exporter(_: SocketAddr) -> anyhow::Result<()> {
     tracing::info!("creating noop metrics exporter");
+    Ok(())
 }

src/infra/tracing/tracing_config.rs

@@ -10,6 +10,8 @@ use http::HeaderMap;
 use http::header::HeaderName;
 use http::header::HeaderValue;
 use itertools::Itertools;
+#[cfg(feature = "metrics")]
+use metrics_tracing_context::MetricsLayer as MetricsTracingFieldsLayer;
 use opentelemetry::KeyValue;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry_otlp::Protocol;
@@ -147,11 +149,15 @@ impl TracingConfig {
             }
         };
 
-        tracing_subscriber::registry()
-            .with(tracing_context_layer)
-            .with(stdout_layer)
-            .with(opentelemetry_layer)
-            .with(sentry_layer)
+        let registry = tracing_subscriber::registry().with(tracing_context_layer);
+
+        #[cfg(feature = "metrics")]
+        let registry = {
+            println!("tracing registry: enabling metrics tracing context recorder");
+            registry.with(MetricsTracingFieldsLayer::new())
+        };
+
+        registry.with(stdout_layer).with(opentelemetry_layer).with(sentry_layer)
     }
 }
 

supply-chain/config.toml

@@ -540,6 +540,10 @@ criteria = "safe-to-deploy"
 version = "1.0.0"
 criteria = "safe-to-deploy"
 
+[[exemptions.endian-type]]
+version = "0.1.2"
+criteria = "safe-to-deploy"
+
 [[exemptions.enum-ordinalize]]
 version = "4.3.0"
 criteria = "safe-to-deploy"
@@ -864,6 +868,10 @@ criteria = "safe-to-deploy"
 version = "1.1.22"
 criteria = "safe-to-deploy"
 
+[[exemptions.lockfree-object-pool]]
+version = "0.1.6"
+criteria = "safe-to-deploy"
+
 [[exemptions.lru-slab]]
 version = "0.1.2"
 criteria = "safe-to-deploy"
@@ -896,6 +904,10 @@ criteria = "safe-to-deploy"
 version = "0.17.2"
 criteria = "safe-to-deploy"
 
+[[exemptions.metrics-tracing-context]]
+version = "0.18.1"
+criteria = "safe-to-deploy"
+
 [[exemptions.metrics-util]]
 version = "0.20.0"
 criteria = "safe-to-deploy"
@@ -916,6 +928,10 @@ criteria = "safe-to-deploy"
 version = "0.4.0"
 criteria = "safe-to-deploy"
 
+[[exemptions.nibble_vec]]
+version = "0.1.0"
+criteria = "safe-to-deploy"
+
 [[exemptions.nom]]
 version = "8.0.0"
 criteria = "safe-to-deploy"
@@ -992,6 +1008,10 @@ criteria = "safe-to-deploy"
 version = "0.9.109"
 criteria = "safe-to-deploy"
 
+[[exemptions.ordered-float]]
+version = "4.6.0"
+criteria = "safe-to-deploy"
+
 [[exemptions.os_info]]
 version = "3.12.0"
 criteria = "safe-to-deploy"
@@ -1108,6 +1128,10 @@ criteria = "safe-to-deploy"
 version = "0.7.0"
 criteria = "safe-to-deploy"
 
+[[exemptions.radix_trie]]
+version = "0.2.1"
+criteria = "safe-to-deploy"
+
 [[exemptions.rand_xoshiro]]
 version = "0.7.0"
 criteria = "safe-to-deploy"