Skip to content

enhancement: add environment-backed exporter headers to direct APIs #901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ericevans-nv wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

enhancement: add environment-backed exporter headers to direct APIs #901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions crates/core/src/observability/otel.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use super::otel_signal::{
MetricMarkClassification, SignalRuntimeDiagnostics, classify_metric_mark,
MetricMarkClassification, SignalRuntimeDiagnostics, classify_metric_mark, resolve_header_env,
should_relog_runtime_diagnostic,
};
use super::{
Expand Down Expand Up @@ -237,6 +237,7 @@ pub struct OpenTelemetryConfig {
otel_type: OpenTelemetryType,
endpoint: String,
headers: HashMap<String, String>,
header_env: HashMap<String, String>,
resource_attributes: HashMap<String, String>,
service_name: String,
service_namespace: Option<String>,
Expand All @@ -260,6 +261,7 @@ impl OpenTelemetryConfig {
otel_type: OpenTelemetryType::Full,
endpoint: String::new(),
headers: HashMap::new(),
header_env: HashMap::new(),
resource_attributes: HashMap::new(),
service_name: "unknown_service".to_string(),
service_namespace: None,
Expand Down Expand Up @@ -331,6 +333,12 @@ impl OpenTelemetryConfig {
self
}

/// Maps an exporter header name to the environment variable supplying its value.
pub fn with_header_env(mut self, key: impl Into<String>, variable: impl Into<String>) -> Self {
self.header_env.insert(key.into(), variable.into());
self
}

#[cfg(test)]
pub(crate) fn header(&self, key: &str) -> Option<&str> {
self.headers.get(key).map(String::as_str)
Expand Down Expand Up @@ -540,7 +548,7 @@ impl OpenTelemetrySubscriber {
}

fn new_with_runtime_diagnostics(
config: OpenTelemetryConfig,
mut config: OpenTelemetryConfig,
diagnostic_field: Option<String>,
) -> Result<Self> {
if config.endpoint.trim().is_empty() {
Expand All @@ -559,6 +567,8 @@ impl OpenTelemetrySubscriber {
.map_err(OpenTelemetryError::InvalidMetadataPromotionPrefixes)?;
reject_global_header_environment()?;
validate_headers(&config.headers)?;
config.headers = resolve_header_env(&config.headers, &config.header_env)?;
validate_headers(&config.headers)?;
let runtime_diagnostics = SignalRuntimeDiagnostics::new(diagnostic_field);
let (provider, runtime) =
build_owned_tracer_provider(config.clone(), runtime_diagnostics.clone())?;
Expand Down
16 changes: 13 additions & 3 deletions crates/core/src/observability/otel_logs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ use super::otel::{
use super::otel_signal::{
MetricMarkClassification, SignalExporterRuntime, SignalRuntimeDiagnostics, build_grpc_metadata,
build_in_owned_runtime, classify_metric_mark, reject_signal_header_environment,
resolve_http_signal_endpoint, should_relog_runtime_diagnostic, signal_resource,
validate_signal_headers,
resolve_header_env, resolve_http_signal_endpoint, should_relog_runtime_diagnostic,
signal_resource, validate_signal_headers,
};

const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
Expand All @@ -51,6 +51,7 @@ const DEFAULT_SCHEDULED_DELAY: Duration = Duration::from_secs(1);
pub struct OpenTelemetryLogConfig {
endpoint: String,
headers: HashMap<String, String>,
header_env: HashMap<String, String>,
resource_attributes: HashMap<String, String>,
service_name: String,
service_namespace: Option<String>,
Expand All @@ -72,6 +73,7 @@ impl OpenTelemetryLogConfig {
Self {
endpoint: endpoint.into(),
headers: HashMap::new(),
header_env: HashMap::new(),
resource_attributes: HashMap::new(),
service_name: "unknown_service".to_string(),
service_namespace: None,
Expand Down Expand Up @@ -100,6 +102,12 @@ impl OpenTelemetryLogConfig {
self
}

/// Map an exporter header name to the environment variable supplying its value.
pub fn with_header_env(mut self, key: impl Into<String>, variable: impl Into<String>) -> Self {
self.header_env.insert(key.into(), variable.into());
self
}

/// Add an OpenTelemetry resource attribute.
pub fn with_resource_attribute(
mut self,
Expand Down Expand Up @@ -247,8 +255,10 @@ impl OpenTelemetryLogSubscriber {
Self::new_with_runtime_diagnostics(config)
}

fn new_with_runtime_diagnostics(config: OpenTelemetryLogConfig) -> Result<Self> {
fn new_with_runtime_diagnostics(mut config: OpenTelemetryLogConfig) -> Result<Self> {
config.validate()?;
config.headers = resolve_header_env(&config.headers, &config.header_env)?;
validate_signal_headers(&config.headers)?;
let minimum_severity = config.minimum_severity;
let completed_span_context_ttl = config.completed_span_context_ttl;
let instrumentation_scope = config.instrumentation_scope.clone();
Expand Down
16 changes: 13 additions & 3 deletions crates/core/src/observability/otel_metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ use super::otel::{OpenTelemetryError, OtlpTransport, Result, normalize_shutdown_
use super::otel_signal::{
MetricMarkClassification, SignalExporterRuntime, SignalRuntimeDiagnostics, build_grpc_metadata,
build_in_owned_runtime, classify_metric_mark, reject_signal_header_environment,
resolve_http_signal_endpoint, should_relog_runtime_diagnostic, signal_resource,
validate_signal_headers,
resolve_header_env, resolve_http_signal_endpoint, should_relog_runtime_diagnostic,
signal_resource, validate_signal_headers,
};

const DEFAULT_EXPORT_INTERVAL: Duration = Duration::from_secs(60);
Expand Down Expand Up @@ -97,6 +97,7 @@ impl std::str::FromStr for MetricTemporality {
pub struct OpenTelemetryMetricConfig {
endpoint: String,
headers: HashMap<String, String>,
header_env: HashMap<String, String>,
resource_attributes: HashMap<String, String>,
service_name: String,
service_namespace: Option<String>,
Expand All @@ -117,6 +118,7 @@ impl OpenTelemetryMetricConfig {
Self {
endpoint: endpoint.into(),
headers: HashMap::new(),
header_env: HashMap::new(),
resource_attributes: HashMap::new(),
service_name: "unknown_service".to_string(),
service_namespace: None,
Expand Down Expand Up @@ -144,6 +146,12 @@ impl OpenTelemetryMetricConfig {
self
}

/// Map an exporter header name to the environment variable supplying its value.
pub fn with_header_env(mut self, key: impl Into<String>, variable: impl Into<String>) -> Self {
self.header_env.insert(key.into(), variable.into());
self
}

/// Add an OpenTelemetry resource attribute.
pub fn with_resource_attribute(
mut self,
Expand Down Expand Up @@ -282,8 +290,10 @@ impl OpenTelemetryMetricSubscriber {
Self::new_with_runtime_diagnostics(config)
}

fn new_with_runtime_diagnostics(config: OpenTelemetryMetricConfig) -> Result<Self> {
fn new_with_runtime_diagnostics(mut config: OpenTelemetryMetricConfig) -> Result<Self> {
config.validate()?;
config.headers = resolve_header_env(&config.headers, &config.header_env)?;
validate_signal_headers(&config.headers)?;
let instrumentation_scope = config.instrumentation_scope.clone();
let max_instruments = config.max_instruments;
let cardinality_limit = config.cardinality_limit;
Expand Down
74 changes: 74 additions & 0 deletions crates/core/src/observability/otel_signal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,80 @@ pub(super) fn validate_signal_headers(headers: &HashMap<String, String>) -> Resu
Ok(())
}

pub(super) fn resolve_header_env(
headers: &HashMap<String, String>,
header_env: &HashMap<String, String>,
) -> Result<HashMap<String, String>> {
let mut normalized = HashSet::new();
for key in headers.keys() {
normalized.insert(key.to_ascii_lowercase());
}

for (key, variable) in header_env {
if !normalized.insert(key.to_ascii_lowercase()) {
return Err(OpenTelemetryError::InvalidHeader {
key: key.clone(),
message:
"header names must be unique across headers and header_env ignoring ASCII case"
.to_string(),
});
}
reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| {
OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: error.to_string(),
}
})?;
if variable.trim().is_empty()
|| variable.trim() != variable
|| variable.contains(['\0', '='])
{
return Err(OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: "header_env must name a nonblank environment variable without surrounding whitespace, '=' or NUL"
.to_string(),
});
}
}

let mut resolved = headers.clone();
for (key, variable) in header_env {
let value = match std::env::var(variable) {
Ok(value) => value,
Err(std::env::VarError::NotPresent) => {
return Err(OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: format!("environment variable {variable:?} is not set"),
});
}
Err(std::env::VarError::NotUnicode(_)) => {
return Err(OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: format!("environment variable {variable:?} is not valid Unicode"),
});
}
};
if value.trim().is_empty() || value.trim() != value {
return Err(OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: format!(
"environment variable {variable:?} must contain a nonblank value without surrounding whitespace"
),
});
}
reqwest::header::HeaderValue::from_str(&value).map_err(|_| {
OpenTelemetryError::InvalidHeader {
key: key.clone(),
message: format!(
"environment variable {variable:?} does not contain a valid header value"
),
}
})?;
resolved.insert(key.clone(), value);
}
Ok(resolved)
}

pub(super) fn reject_signal_header_environment(signal_variable: &'static str) -> Result<()> {
for variable in ["OTEL_EXPORTER_OTLP_HEADERS", signal_variable] {
if std::env::var_os(variable).is_some_and(|value| !value.is_empty()) {
Expand Down
8 changes: 4 additions & 4 deletions crates/core/src/observability/plugin_component.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2199,9 +2199,9 @@ fn resolve_signal_headers(
"OpenTelemetry {signal}.endpoints[{index}] header {key:?} cannot appear in both headers and header_env"
)));
}
let value = std::env::var(variable).map_err(|error| {
let value = std::env::var(variable).map_err(|_| {
PluginError::InvalidConfig(format!(
"OpenTelemetry {signal}.endpoints[{index}].header_env.{key} could not read environment variable {variable:?}: {error}"
"OpenTelemetry {signal}.endpoints[{index}].header_env.{key} could not read environment variable {variable:?}"
))
})?;
if value.trim().is_empty() || value.trim() != value {
Expand Down Expand Up @@ -3269,9 +3269,9 @@ fn apply_otel_environment_headers(
header_env: HashMap<String, String>,
) -> PluginResult<CoreOpenTelemetryConfig> {
for (key, variable) in header_env {
let value = std::env::var(&variable).map_err(|error| {
let value = std::env::var(&variable).map_err(|_| {
PluginError::InvalidConfig(format!(
"OpenTelemetry endpoints[{index}].header_env.{key} could not read environment variable {variable:?}: {error}"
"OpenTelemetry endpoints[{index}].header_env.{key} could not read environment variable {variable:?}"
))
})?;
if value.trim().is_empty() || value.trim() != value {
Expand Down
Loading
Loading