diff --git a/Makefile b/Makefile index fb3d3a91e4..54a73645c1 100644 --- a/Makefile +++ b/Makefile @@ -137,20 +137,13 @@ install-network-monitor: ## Installs network monitor binary # --- docker -------------------------------------------------------------------------------------- -.PHONY: compose-genesis -compose-genesis: ## Wipes node volumes and creates a fresh genesis block - docker compose $(COMPOSE_FILES) down --volumes --remove-orphans - docker volume rm -f miden-node_node-data - docker compose $(COMPOSE_FILES) --profile genesis run --rm genesis-store - docker compose $(COMPOSE_FILES) --profile genesis rm -f - .PHONY: compose-up compose-up: ## Starts all node components, telemetry, and monitor via docker compose docker compose $(COMPOSE_FILES) up -d .PHONY: compose-down compose-down: ## Stops and removes all containers via docker compose - docker compose $(COMPOSE_FILES) down + docker compose $(COMPOSE_FILES) down --remove-orphans .PHONY: compose-logs compose-logs: ## Follows logs for all components via docker compose diff --git a/bin/network-monitor/README.md b/bin/network-monitor/README.md index 51c87ce680..0e2554140f 100644 --- a/bin/network-monitor/README.md +++ b/bin/network-monitor/README.md @@ -23,7 +23,7 @@ miden-network-monitor --help # Common usage examples miden-network-monitor start --port 8080 --rpc.listen http://localhost:50051 miden-network-monitor start --remote-prover-urls http://prover1.com:50052,http://prover2.com:50053 -miden-network-monitor start --faucet-url http://localhost:8080 --enable-otel +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 miden-network-monitor start --faucet-url http://localhost:8080 ``` **Available Options:** @@ -41,7 +41,6 @@ miden-network-monitor start --faucet-url http://localhost:8080 --enable-otel - `--request-timeout`: Timeout for outgoing requests (default: `10s`) - `--stale-chain-tip-threshold`: Maximum time without a chain tip update before marking RPC as unhealthy (default: `1m`) - `--port, -p`: Web server port (default: `3000`) -- `--enable-otel`: Enable OpenTelemetry tracing - `--counter-increment-interval`: Interval at which to send the increment counter transaction (default: `30s`) - `--counter-pending-unhealthy-threshold`: Mark the Network Transactions card unhealthy when the gap between expected and observed counter values stays above this for several consecutive polls (default: `5`) - `--counter-latency-timeout`: Maximum time to wait for a counter update after submitting a transaction (default: `2m`) @@ -66,7 +65,6 @@ If command-line arguments are not provided, the application falls back to enviro - `MIDEN_MONITOR_REQUEST_TIMEOUT`: Timeout for outgoing requests - `MIDEN_MONITOR_STALE_CHAIN_TIP_THRESHOLD`: Maximum time without a chain tip update before marking RPC as unhealthy - `MIDEN_MONITOR_PORT`: Web server port -- `MIDEN_MONITOR_ENABLE_OTEL`: Enable OpenTelemetry tracing - `MIDEN_MONITOR_COUNTER_INCREMENT_INTERVAL`: Interval at which to send the increment counter transaction - `MIDEN_MONITOR_COUNTER_PENDING_UNHEALTHY_THRESHOLD`: Mark the Network Transactions card unhealthy when the gap between expected and observed counter values stays above this for several consecutive polls - `MIDEN_MONITOR_COUNTER_LATENCY_TIMEOUT`: Maximum time to wait for a counter update after submitting a transaction @@ -122,8 +120,7 @@ miden-network-monitor start \ --remote-prover-test-interval 2m \ --faucet-test-interval 2m \ --status-check-interval 3s \ - --port 8080 \ - --enable-otel + --port 8080 # Get help miden-network-monitor --help @@ -144,6 +141,9 @@ miden-network-monitor start Once running, the monitor will be available at `http://localhost:3000` (or the configured port). +OpenTelemetry tracing is enabled automatically when `OTEL_EXPORTER_OTLP_ENDPOINT` or +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is set. + ## Currently Supported Monitor The monitor application provides real-time status monitoring for the following Miden network components: diff --git a/bin/network-monitor/src/commands/start.rs b/bin/network-monitor/src/commands/start.rs index 9b9499daae..e5fbb0b0ca 100644 --- a/bin/network-monitor/src/commands/start.rs +++ b/bin/network-monitor/src/commands/start.rs @@ -28,11 +28,8 @@ use crate::monitor::tasks::Tasks; pub async fn start_monitor(config: MonitorConfig) -> Result<()> { info!("Loaded configuration: {:?}", config); - let _otel_guard = if config.enable_otel { - miden_node_utils::logging::setup_tracing(OpenTelemetry::Enabled)? - } else { - miden_node_utils::logging::setup_tracing(OpenTelemetry::Disabled)? - }; + let _otel_guard = + miden_node_utils::logging::setup_tracing(OpenTelemetry::from_env().with_name("monitor"))?; let mut tasks = Tasks::new(); diff --git a/bin/network-monitor/src/config.rs b/bin/network-monitor/src/config.rs index d569247b3f..91b7722a4e 100644 --- a/bin/network-monitor/src/config.rs +++ b/bin/network-monitor/src/config.rs @@ -94,16 +94,6 @@ pub struct MonitorConfig { )] pub port: u16, - /// Whether to enable OpenTelemetry. - #[arg( - long = "enable-otel", - env = "MIDEN_MONITOR_ENABLE_OTEL", - action = clap::ArgAction::SetTrue, - default_value_t = false, - help = "Whether to enable OpenTelemetry" - )] - pub enable_otel: bool, - /// Whether to disable the network transaction service checks (enabled by default). The network /// transaction service is a network account with a counter deployed at startup and incremented /// by sending a transaction to it. diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index fab6d0b0e0..e04db6375c 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -8,6 +8,7 @@ mod store; use clap::Subcommand; pub use lifecycle::{BootstrapCommand, MigrateCommand}; +use miden_node_utils::logging::OpenTelemetry; pub use modes::{FullNodeCommand, SequencerCommand}; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; @@ -51,13 +52,15 @@ pub enum Command { } impl Command { - pub(crate) fn open_telemetry(&self) -> miden_node_utils::logging::OpenTelemetry { + pub(crate) fn open_telemetry(&self) -> OpenTelemetry { match self { - Command::Sequencer(command) => command.runtime.open_telemetry(), - Command::Full(command) => command.runtime.open_telemetry(), - Command::Bootstrap(_) | Command::Migrate(_) => { - miden_node_utils::logging::OpenTelemetry::Disabled - }, + Command::Sequencer(_) => OpenTelemetry::from_env() + .with_name("node") + .with_attribute("miden.node.role", "sequencer"), + Command::Full(_) => OpenTelemetry::from_env() + .with_name("node") + .with_attribute("miden.node.role", "full"), + Command::Bootstrap(_) | Command::Migrate(_) => OpenTelemetry::Disabled, } } diff --git a/bin/node/src/commands/runtime.rs b/bin/node/src/commands/runtime.rs index 8931761fe4..4d9d28ec9e 100644 --- a/bin/node/src/commands/runtime.rs +++ b/bin/node/src/commands/runtime.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use miden_node_store::DatabaseOptions; use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; -use miden_node_utils::logging::OpenTelemetry; use super::ENV_DATA_DIRECTORY; use super::rpc::RpcOptions; @@ -18,31 +17,11 @@ pub struct RuntimeOptions { #[arg(long, env = ENV_DATA_DIRECTORY, value_name = "DIR")] pub data_directory: PathBuf, - /// Enables the exporting of traces for OpenTelemetry. - /// - /// This can be further configured using environment variables as defined in the official - /// OpenTelemetry documentation. See our operator manual for further details. - #[arg( - long = "enable-otel", - default_value_t = false, - env = "MIDEN_NODE_ENABLE_OTEL", - value_name = "BOOL" - )] - pub enable_otel: bool, - #[command(flatten)] pub rpc: RpcOptions, } impl RuntimeOptions { - pub fn open_telemetry(&self) -> OpenTelemetry { - if self.enable_otel { - OpenTelemetry::Enabled - } else { - OpenTelemetry::Disabled - } - } - pub(super) fn runtime_config(&self, store: &StoreOptions) -> RuntimeConfig { RuntimeConfig { data_directory: self.data_directory.clone(), diff --git a/bin/ntx-builder/src/commands/mod.rs b/bin/ntx-builder/src/commands/mod.rs index d943d19f30..fc32efeebb 100644 --- a/bin/ntx-builder/src/commands/mod.rs +++ b/bin/ntx-builder/src/commands/mod.rs @@ -6,13 +6,14 @@ use std::time::Duration; use anyhow::Context; use clap::Parser; use miden_node_utils::clap::duration_to_human_readable_string; +use miden_node_utils::fs::ensure_empty_directory; +use miden_node_utils::logging::OpenTelemetry; use miden_protocol::block::SignedBlock; use miden_protocol::utils::serde::Deserializable; use tokio::net::TcpListener; use tonic::metadata::AsciiMetadataValue; use url::Url; -const ENV_ENABLE_OTEL: &str = "MIDEN_NODE_ENABLE_OTEL"; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; const ENV_LISTEN: &str = "MIDEN_NODE_NTX_BUILDER_LISTEN"; const ENV_RPC_URL: &str = "MIDEN_NODE_NTX_BUILDER_RPC_URL"; @@ -121,13 +122,6 @@ pub enum NtxBuilderCommand { /// Directory for the ntx-builder's persistent database. #[arg(long = "data-directory", env = ENV_DATA_DIRECTORY, value_name = "DIR")] data_directory: PathBuf, - - /// Enables the exporting of traces for OpenTelemetry. - /// - /// This can be further configured using environment variables as defined in the official - /// OpenTelemetry documentation. See our operator manual for further details. - #[arg(long = "enable-otel", default_value_t = false, env = ENV_ENABLE_OTEL, value_name = "BOOL")] - enable_otel: bool, }, /// Bootstraps the ntx-builder database from a trusted genesis block. @@ -150,6 +144,7 @@ impl NtxBuilderCommand { match self { Self::Start { .. } => self.start().await, Self::Bootstrap { data_directory, genesis_block } => { + ensure_empty_directory(&data_directory)?; let database_filepath = data_directory.join("ntx-builder.sqlite3"); let genesis = read_genesis_block(&genesis_block)?; miden_ntx_builder::bootstrap(database_filepath, &genesis) @@ -172,7 +167,6 @@ impl NtxBuilderCommand { tx_expiration_delta, sqlite_connection_pool_size, data_directory, - enable_otel: _, } = self else { unreachable!("start is only called for the Start variant") @@ -206,11 +200,11 @@ impl NtxBuilderCommand { .context("failed while running ntx builder component") } - pub fn is_open_telemetry_enabled(&self) -> bool { + pub fn open_telemetry(&self) -> OpenTelemetry { match self { - Self::Start { enable_otel, .. } => *enable_otel, + Self::Start { .. } => OpenTelemetry::from_env().with_name("ntx-builder"), // Bootstrap is a one-shot command and does not set up a tracing pipeline. - Self::Bootstrap { .. } => false, + Self::Bootstrap { .. } => OpenTelemetry::Disabled, } } } diff --git a/bin/ntx-builder/src/main.rs b/bin/ntx-builder/src/main.rs index 6c71a1060b..d4136460a8 100644 --- a/bin/ntx-builder/src/main.rs +++ b/bin/ntx-builder/src/main.rs @@ -1,19 +1,11 @@ use clap::Parser; -use miden_node_utils::logging::OpenTelemetry; - mod commands; #[tokio::main] async fn main() -> anyhow::Result<()> { let command = commands::NtxBuilderCommand::parse(); - let otel = if command.is_open_telemetry_enabled() { - OpenTelemetry::Enabled - } else { - OpenTelemetry::Disabled - }; - - let _otel_guard = miden_node_utils::logging::setup_tracing(otel)?; + let _otel_guard = miden_node_utils::logging::setup_tracing(command.open_telemetry())?; command.handle().await } diff --git a/bin/remote-prover/src/main.rs b/bin/remote-prover/src/main.rs index 9de86e0892..b58ff2dd57 100644 --- a/bin/remote-prover/src/main.rs +++ b/bin/remote-prover/src/main.rs @@ -1,6 +1,5 @@ use anyhow::Context; use clap::Parser; -use miden_node_utils::logging::{OpenTelemetry, setup_tracing}; use tracing::info; mod server; @@ -9,11 +8,12 @@ const COMPONENT: &str = "miden-prover"; #[tokio::main] async fn main() -> anyhow::Result<()> { - let _otel_guard = setup_tracing(OpenTelemetry::Enabled)?; + let server = server::Server::parse(); + + let _otel_guard = miden_node_utils::logging::setup_tracing(server.open_telemetry())?; info!(target: COMPONENT, "Tracing initialized"); - let (handle, _port) = - server::Server::parse().spawn().await.context("failed to spawn server")?; + let (handle, _port) = server.spawn().await.context("failed to spawn server")?; handle.await.context("proof server panicked").flatten() } diff --git a/bin/remote-prover/src/server/mod.rs b/bin/remote-prover/src/server/mod.rs index 087820a194..8650e0c785 100644 --- a/bin/remote-prover/src/server/mod.rs +++ b/bin/remote-prover/src/server/mod.rs @@ -4,6 +4,7 @@ use anyhow::Context; use miden_node_proto::generated::remote_prover::api_server::ApiServer; use miden_node_proto::generated::remote_prover::worker_status_api_server::WorkerStatusApiServer; use miden_node_utils::cors::cors_for_grpc_web_layer; +use miden_node_utils::logging::OpenTelemetry; use miden_node_utils::panic::catch_panic_layer_fn; use miden_node_utils::tracing::grpc::grpc_trace_fn; use proof_kind::ProofKind; @@ -47,6 +48,12 @@ pub struct Server { } impl Server { + pub fn open_telemetry(&self) -> OpenTelemetry { + OpenTelemetry::from_env() + .with_name("remote-prover") + .with_attribute("miden.prover.kind", self.kind.as_str()) + } + /// Spawns the prover server, returning its handle and the port it is listening on. pub async fn spawn(&self) -> anyhow::Result<(JoinHandle>, u16)> { let listener = TcpListener::bind(format!("0.0.0.0:{}", self.port)) diff --git a/bin/remote-prover/src/server/proof_kind.rs b/bin/remote-prover/src/server/proof_kind.rs index 9971dda15c..5967835424 100644 --- a/bin/remote-prover/src/server/proof_kind.rs +++ b/bin/remote-prover/src/server/proof_kind.rs @@ -8,6 +8,16 @@ pub enum ProofKind { Block, } +impl ProofKind { + pub const fn as_str(self) -> &'static str { + match self { + ProofKind::Transaction => "transaction", + ProofKind::Batch => "batch", + ProofKind::Block => "block", + } + } +} + impl From for ProofKind { fn from(value: proto::ProofType) -> Self { match value { @@ -20,11 +30,7 @@ impl From for ProofKind { impl std::fmt::Display for ProofKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ProofKind::Transaction => write!(f, "transaction"), - ProofKind::Batch => write!(f, "batch"), - ProofKind::Block => write!(f, "block"), - } + f.write_str(self.as_str()) } } diff --git a/bin/validator/src/commands/mod.rs b/bin/validator/src/commands/mod.rs index 96fda9abfa..506403860a 100644 --- a/bin/validator/src/commands/mod.rs +++ b/bin/validator/src/commands/mod.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use clap::Parser; use miden_node_utils::clap::GrpcOptionsInternal; +use miden_node_utils::logging::OpenTelemetry; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::utils::serde::Deserializable; use miden_validator::ValidatorSigner; @@ -14,7 +15,6 @@ const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; const ENV_LISTEN: &str = "MIDEN_NODE_VALIDATOR_LISTEN"; const ENV_KEY: &str = "MIDEN_NODE_VALIDATOR_KEY"; const ENV_KMS_KEY_ID: &str = "MIDEN_NODE_VALIDATOR_KMS_KEY_ID"; -const ENV_ENABLE_OTEL: &str = "MIDEN_NODE_ENABLE_OTEL"; const ENV_GENESIS_CONFIG_FILE: &str = "MIDEN_NODE_VALIDATOR_GENESIS_CONFIG_FILE"; const ENV_SQLITE_CONNECTION_POOL_SIZE: &str = "MIDEN_NODE_VALIDATOR_SQLITE_CONNECTION_POOL_SIZE"; @@ -65,13 +65,6 @@ pub enum ValidatorCommand { #[arg(long = "listen", env = ENV_LISTEN, value_name = "LISTEN")] listen: std::net::SocketAddr, - /// Enables the exporting of traces for OpenTelemetry. - /// - /// This can be further configured using environment variables as defined in the official - /// OpenTelemetry documentation. See our operator manual for further details. - #[arg(long = "enable-otel", default_value_t = false, env = ENV_ENABLE_OTEL, value_name = "BOOL")] - enable_otel: bool, - #[command(flatten)] grpc_options: GrpcOptionsInternal, @@ -173,10 +166,10 @@ impl ValidatorCommand { } } - pub fn is_open_telemetry_enabled(&self) -> bool { + pub fn open_telemetry(&self) -> OpenTelemetry { match self { - Self::Start { enable_otel, .. } => *enable_otel, - Self::Bootstrap { .. } => false, + Self::Start { .. } => OpenTelemetry::from_env().with_name("validator"), + Self::Bootstrap { .. } => OpenTelemetry::Disabled, } } } diff --git a/bin/validator/src/main.rs b/bin/validator/src/main.rs index 6f27a578cc..b6a04d5a3d 100644 --- a/bin/validator/src/main.rs +++ b/bin/validator/src/main.rs @@ -1,6 +1,4 @@ use clap::Parser; -use miden_node_utils::logging::OpenTelemetry; - mod commands; // MAIN @@ -10,13 +8,7 @@ mod commands; async fn main() -> anyhow::Result<()> { let command = commands::ValidatorCommand::parse(); - let otel = if command.is_open_telemetry_enabled() { - OpenTelemetry::Enabled - } else { - OpenTelemetry::Disabled - }; - - let _otel_guard = miden_node_utils::logging::setup_tracing(otel)?; + let _otel_guard = miden_node_utils::logging::setup_tracing(command.open_telemetry())?; command.handle().await } diff --git a/compose/monitor.yml b/compose/monitor.yml index 2d8636aec9..70849a83d2 100644 --- a/compose/monitor.yml +++ b/compose/monitor.yml @@ -6,14 +6,10 @@ services: - miden-network-monitor - start environment: - - MIDEN_MONITOR_RPC_URL=http://localhost:57291 + - MIDEN_MONITOR_RPC_URL=http://sequencer:57291 - MIDEN_MONITOR_PORT=3001 - MIDEN_MONITOR_NETWORK_NAME=Localhost - MIDEN_MONITOR_DISABLE_NTX_SERVICE=true - - MIDEN_MONITOR_ENABLE_OTEL=true - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=monitor - extra_hosts: - - "localhost:host-gateway" ports: - "3001:3001" diff --git a/compose/telemetry.yml b/compose/telemetry.yml index b8302fff87..4e6f9a24e8 100644 --- a/compose/telemetry.yml +++ b/compose/telemetry.yml @@ -23,32 +23,15 @@ services: depends_on: - tempo - store: + sequencer: environment: - - MIDEN_NODE_ENABLE_OTEL=true - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=store + - OTEL_RESOURCE_ATTRIBUTES=service.instance.id=sequencer validator: environment: - - MIDEN_NODE_ENABLE_OTEL=true - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=validator - - block-producer: - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=block-producer - - rpc: - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=rpc ntx-builder: environment: - - MIDEN_NODE_ENABLE_OTEL=true - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=ntx-builder diff --git a/crates/utils/src/logging.rs b/crates/utils/src/logging.rs index 8e38ac7928..6cf2883275 100644 --- a/crates/utils/src/logging.rs +++ b/crates/utils/src/logging.rs @@ -2,7 +2,10 @@ use std::str::FromStr; use std::sync::OnceLock; use opentelemetry::trace::TracerProvider as _; +use opentelemetry::{KeyValue, Value}; +use opentelemetry_sdk::Resource; use opentelemetry_sdk::propagation::TraceContextPropagator; +use opentelemetry_sdk::resource::{EnvResourceDetector, TelemetryResourceDetector}; use opentelemetry_sdk::trace::SdkTracerProvider; use tracing::subscriber::Subscriber; use tracing_opentelemetry::OpenTelemetryLayer; @@ -17,16 +20,76 @@ use crate::tracing::OpenTelemetrySpanExt; /// pending spans before the program terminates. static TRACER_PROVIDER: OnceLock = OnceLock::new(); +/// Default OpenTelemetry resource attributes for this process. +#[derive(Clone, Default)] +pub struct ResourceConfig { + service_name: Option<&'static str>, + attributes: Vec<(&'static str, &'static str)>, +} + +impl ResourceConfig { + #[must_use] + pub fn with_name(mut self, service_name: &'static str) -> Self { + self.service_name = Some(service_name); + self + } + + #[must_use] + pub fn with_attribute(mut self, key: &'static str, value: &'static str) -> Self { + self.attributes.push((key, value)); + self + } +} + /// Configures [`setup_tracing`] to enable or disable the open-telemetry exporter. -#[derive(Clone, Copy)] +#[derive(Clone)] pub enum OpenTelemetry { - Enabled, + Enabled(ResourceConfig), Disabled, } impl OpenTelemetry { - fn is_enabled(self) -> bool { - matches!(self, OpenTelemetry::Enabled) + pub fn enabled() -> Self { + OpenTelemetry::Enabled(ResourceConfig::default()) + } + + pub fn from_env() -> Self { + if otlp_endpoint_configured() { + OpenTelemetry::enabled() + } else { + OpenTelemetry::Disabled + } + } + + #[must_use] + pub fn with_name(self, service_name: &'static str) -> Self { + match self { + OpenTelemetry::Enabled(config) => { + OpenTelemetry::Enabled(config.with_name(service_name)) + }, + OpenTelemetry::Disabled => OpenTelemetry::Disabled, + } + } + + #[must_use] + pub fn with_attribute(self, key: &'static str, value: &'static str) -> Self { + match self { + OpenTelemetry::Enabled(config) => { + OpenTelemetry::Enabled(config.with_attribute(key, value)) + }, + OpenTelemetry::Disabled => OpenTelemetry::Disabled, + } + } + + fn is_enabled(&self) -> bool { + matches!(self, OpenTelemetry::Enabled(_)) + } + + fn resource_config(self) -> Option { + match self { + OpenTelemetry::Enabled(config) => Some(config), + OpenTelemetry::Disabled => None, + } } } @@ -65,7 +128,10 @@ pub fn setup_tracing(otel: OpenTelemetry) -> anyhow::Result> { // `then_some`) to avoid crashing sync callers (with OpenTelemetry::Disabled set). Examples of // such callers are tests with logging enabled. let tracer_provider = if otel.is_enabled() { - let provider = init_tracer_provider()?; + let provider = init_tracer_provider( + otel.resource_config() + .expect("resource config is set when OpenTelemetry is enabled"), + )?; // Store the provider globally so the panic hook can flush it. SdkTracerProvider is // internally reference-counted, so cloning is cheap. @@ -112,16 +178,74 @@ pub fn setup_tracing(otel: OpenTelemetry) -> anyhow::Result> { Ok(tracer_provider.map(|tracer_provider| OtelGuard { tracer_provider })) } -fn init_tracer_provider() -> anyhow::Result { +fn init_tracer_provider(resource_config: ResourceConfig) -> anyhow::Result { let builder = opentelemetry_otlp::SpanExporter::builder().with_tonic(); let exporter = builder.build()?; + let resource = resource(resource_config); Ok(opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_resource(resource) .with_batch_exporter(exporter) .build()) } +fn resource(config: ResourceConfig) -> Resource { + let detected_resource = Resource::builder_empty() + .with_detector(Box::new(TelemetryResourceDetector)) + .with_detector(Box::new(EnvResourceDetector::new())) + .build(); + + resource_from_detected(config, &detected_resource, otel_service_name_override()) +} + +fn resource_from_detected( + config: ResourceConfig, + detected_resource: &Resource, + service_name_override: Option, +) -> Resource { + const SERVICE_NAME: &str = "service.name"; + const SERVICE_NAMESPACE: &str = "service.namespace"; + + let mut attributes = + std::collections::BTreeMap::from([(SERVICE_NAMESPACE.to_string(), Value::from("miden"))]); + + if let Some(service_name) = config.service_name { + attributes.insert(SERVICE_NAME.to_string(), Value::from(service_name)); + } + + for (key, value) in config.attributes { + attributes.insert(key.to_string(), Value::from(value)); + } + + // Environment resource attributes override defaults above, and OTEL_SERVICE_NAME overrides + // both. + for (key, value) in detected_resource { + attributes.insert(key.as_str().to_string(), value.clone()); + } + + if let Some(service_name) = service_name_override { + attributes.insert(SERVICE_NAME.to_string(), service_name); + } + + Resource::builder_empty() + .with_attributes(attributes.into_iter().map(|(key, value)| KeyValue::new(key, value))) + .build() +} + +fn otel_service_name_override() -> Option { + std::env::var("OTEL_SERVICE_NAME") + .ok() + .filter(|value| !value.is_empty()) + .map(Value::from) +} + +fn otlp_endpoint_configured() -> bool { + ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"] + .into_iter() + .any(|key| std::env::var(key).is_ok_and(|value| !value.trim().is_empty())) +} + /// Initializes tracing to a test exporter. /// /// Allows trace content to be inspected via the returned receiver. @@ -206,3 +330,78 @@ fn env_or_default_filter() -> Box + Send + Sync + 'static> { }, } } + +#[cfg(test)] +mod tests { + use opentelemetry::Key; + + use super::*; + + #[test] + fn resource_uses_configured_defaults() { + let detected_resource = Resource::builder_empty() + .with_attributes([KeyValue::new("telemetry.sdk.language", "rust")]) + .build(); + + let resource = resource_from_detected( + ResourceConfig::default() + .with_name("node") + .with_attribute("miden.node.role", "sequencer"), + &detected_resource, + None, + ); + + assert_eq!(resource_value(&resource, "service.name"), Some(Value::from("node")),); + assert_eq!(resource_value(&resource, "service.namespace"), Some(Value::from("miden")),); + assert_eq!(resource_value(&resource, "miden.node.role"), Some(Value::from("sequencer")),); + assert_eq!(resource_value(&resource, "telemetry.sdk.language"), Some(Value::from("rust")),); + } + + #[test] + fn resource_prefers_detected_attributes_over_configured_defaults() { + let detected_resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("service.name", "custom-node"), + KeyValue::new("service.namespace", "custom-namespace"), + KeyValue::new("miden.node.role", "custom-role"), + ]) + .build(); + + let resource = resource_from_detected( + ResourceConfig::default() + .with_name("node") + .with_attribute("miden.node.role", "sequencer"), + &detected_resource, + None, + ); + + assert_eq!(resource_value(&resource, "service.name"), Some(Value::from("custom-node")),); + assert_eq!( + resource_value(&resource, "service.namespace"), + Some(Value::from("custom-namespace")), + ); + assert_eq!(resource_value(&resource, "miden.node.role"), Some(Value::from("custom-role")),); + } + + #[test] + fn resource_prefers_explicit_service_name_override() { + let detected_resource = Resource::builder_empty() + .with_attributes([KeyValue::new("service.name", "resource-attribute-node")]) + .build(); + + let resource = resource_from_detected( + ResourceConfig::default().with_name("node"), + &detected_resource, + Some(Value::from("service-env-node")), + ); + + assert_eq!( + resource_value(&resource, "service.name"), + Some(Value::from("service-env-node")), + ); + } + + fn resource_value(resource: &Resource, key: &'static str) -> Option { + resource.get(&Key::from_static_str(key)) + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 60a76d13e2..567343e81d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,135 +1,136 @@ services: - genesis-validator: + bootstrap-validator: image: miden-validator pull_policy: if_not_present - profiles: - - genesis volumes: - node-data:/data entrypoint: ["/bin/sh", "-c"] command: - | set -e - mkdir -p /data/genesis /data/store /data/validator /data/accounts /data/ntx-builder - echo "Bootstrapping validator (creating genesis block)..." + if [ -f /data/validator/.bootstrapped ]; then + echo "Validator already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete validator bootstrap state..." + rm -rf /data/genesis /data/validator /data/accounts + mkdir -p /data/genesis /data/validator /data/accounts + + echo "Bootstrapping validator..." miden-validator bootstrap \ --data-directory /data/validator \ --genesis-block-directory /data/genesis \ --accounts-directory /data/accounts - genesis-store: + touch /data/validator/.bootstrapped + + bootstrap-node: image: miden-node pull_policy: if_not_present - profiles: - - genesis volumes: - node-data:/data entrypoint: ["/bin/sh", "-c"] depends_on: - genesis-validator: + bootstrap-validator: condition: service_completed_successfully command: - | set -e - echo "Bootstrapping store..." - miden-node store bootstrap \ - --data-directory /data/store \ + if [ -f /data/node/.bootstrapped ]; then + echo "Node already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete node bootstrap state..." + rm -rf /data/node + mkdir -p /data/node + + echo "Bootstrapping node..." + miden-node bootstrap \ + --data-directory /data/node \ --genesis-block /data/genesis/genesis.dat - store: + touch /data/node/.bootstrapped + + bootstrap-ntx-builder: + image: miden-ntx-builder + pull_policy: if_not_present + volumes: + - node-data:/data + entrypoint: ["/bin/sh", "-c"] + depends_on: + bootstrap-node: + condition: service_completed_successfully + command: + - | + set -e + if [ -f /data/ntx-builder/.bootstrapped ]; then + echo "Network transaction builder already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete network transaction builder bootstrap state..." + rm -rf /data/ntx-builder + mkdir -p /data/ntx-builder + + echo "Bootstrapping network transaction builder..." + miden-ntx-builder bootstrap \ + --data-directory /data/ntx-builder \ + --genesis-block /data/genesis/genesis.dat + + touch /data/ntx-builder/.bootstrapped + + sequencer: image: miden-node pull_policy: if_not_present volumes: - node-data:/data - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=store + depends_on: + bootstrap-node: + condition: service_completed_successfully + validator: + condition: service_started command: - miden-node - - store - - start - - --rpc.listen=0.0.0.0:50001 - - --block-producer.listen=0.0.0.0:50003 - - --data-directory=/data/store - - --account_tree.rocksdb.max_cache_size=4294967296 - - --account_tree.rocksdb.max_open_fds=512 - - --nullifier_tree.rocksdb.max_cache_size=4294967296 - - --nullifier_tree.rocksdb.max_open_fds=512 + - sequencer + - --rpc.listen=0.0.0.0:57291 + - --data-directory=/data/node + - --validator.url=http://validator:50101 + - --ntx-builder.url=http://ntx-builder:50301 ports: - - "50001:50001" - - "50003:50003" + - "57291:57291" validator: image: miden-validator pull_policy: if_not_present volumes: - node-data:/data - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=validator + depends_on: + bootstrap-validator: + condition: service_completed_successfully command: - miden-validator - start - --listen=0.0.0.0:50101 - --data-directory=/data/validator - ports: - - "50101:50101" - - block-producer: - image: miden-node - pull_policy: if_not_present - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=block-producer - command: - - miden-node - - block-producer - - start - - --listen=0.0.0.0:50201 - - --store.url=http://store:50003 - - --validator.url=http://validator:50101 - ports: - - "50201:50201" - - rpc: - image: miden-node - pull_policy: if_not_present - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=rpc - command: - - miden-node - - rpc - - start - - --listen=0.0.0.0:57291 - - --store.url=http://store:50001 - - --block-producer.url=http://block-producer:50201 - - --validator.url=http://validator:50101 - ports: - - "57291:57291" ntx-builder: image: miden-ntx-builder pull_policy: if_not_present volumes: - node-data:/data - environment: - - MIDEN_NODE_ENABLE_OTEL=true - - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 - - OTEL_SERVICE_NAME=ntx-builder + depends_on: + bootstrap-ntx-builder: + condition: service_completed_successfully + sequencer: + condition: service_started command: - miden-ntx-builder - start - --listen=0.0.0.0:50301 - - --store.url=http://store:50001 - - --block-producer.url=http://block-producer:50201 - - --validator.url=http://validator:50101 + - --rpc.url=http://sequencer:57291 - --data-directory=/data/ntx-builder - volumes: node-data: diff --git a/scripts/run-node.sh b/scripts/run-node.sh index 0dd92bd44f..930debdb32 100755 --- a/scripts/run-node.sh +++ b/scripts/run-node.sh @@ -3,8 +3,9 @@ set -euo pipefail # Configuration SKIP_BOOTSTRAP="${SKIP_BOOTSTRAP:-false}" +ENABLE_FULL_NODES="${ENABLE_FULL_NODES:-true}" EXTRA_ARGS="${EXTRA_ARGS:-}" -BINARY="${MIDEN_NODE_BIN:-./target/debug/miden-node}" +NODE_BINARY="${MIDEN_NODE_BIN:-./target/debug/miden-node}" VALIDATOR_BINARY="${MIDEN_VALIDATOR_BIN:-./target/debug/miden-validator}" NTX_BUILDER_BINARY="${MIDEN_NTX_BUILDER_BIN:-./target/debug/miden-ntx-builder}" KMS_KEY_ID="${KMS_KEY_ID:-}" @@ -14,27 +15,18 @@ if [[ -n "$KMS_KEY_ID" ]]; then fi GENESIS_CONFIG="crates/store/src/genesis/config/samples/01-simple.toml" -STORE_DIR="/tmp/store" -STORE_REPLICA_1_DIR="/tmp/store-replica-1" -STORE_REPLICA_2_DIR="/tmp/store-replica-2" +NODE_DIR="/tmp/node" +FULL_NODE_1_DIR="/tmp/full-node-1" +FULL_NODE_2_DIR="/tmp/full-node-2" VALIDATOR_DIR="/tmp/validator" NTX_BUILDER_DIR="/tmp/ntx-builder" ACCOUNTS_DIR="/tmp/accounts" -# Sequencer store. -STORE_RPC_PORT=50001 -STORE_BLOCK_PRODUCER_PORT=50003 - -# Replica stores expose only the RPC API (no block-producer endpoint). -STORE_REPLICA_1_RPC_PORT=50011 -STORE_REPLICA_2_RPC_PORT=50021 - VALIDATOR_PORT=50101 -BLOCK_PRODUCER_PORT=50201 NTX_BUILDER_PORT=50301 RPC_PORT=57291 -RPC_REPLICA_1_PORT=57292 -RPC_REPLICA_2_PORT=57293 +FULL_NODE_1_RPC_PORT=57292 +FULL_NODE_2_RPC_PORT=57293 PIDS=() @@ -43,34 +35,69 @@ cleanup() { for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true done - wait + wait "${PIDS[@]}" 2>/dev/null || true echo "All components stopped." } trap cleanup EXIT INT TERM -# --- Kill processes on required ports --- +kill_ports() { + local ports=("$VALIDATOR_PORT" "$NTX_BUILDER_PORT" "$RPC_PORT") -PORTS=(50001 50002 50003 50011 50021 50101 50201 50301 57291 57292 57293) -echo "=== Killing processes on required ports ===" -for port in "${PORTS[@]}"; do - pids=$(lsof -ti :"$port" 2>/dev/null || true) - if [[ -n "$pids" ]]; then - for pid in $pids; do - echo "Killing PID $pid on port $port" - kill -9 "$pid" 2>/dev/null || true - done + if [[ "$ENABLE_FULL_NODES" == "true" ]]; then + ports+=("$FULL_NODE_1_RPC_PORT" "$FULL_NODE_2_RPC_PORT") fi -done -sleep 1 + + echo "=== Killing processes on required ports ===" + for port in "${ports[@]}"; do + pids=$(lsof -ti :"$port" 2>/dev/null || true) + if [[ -n "$pids" ]]; then + for pid in $pids; do + echo "Killing PID $pid on port $port" + kill -9 "$pid" 2>/dev/null || true + done + fi + done + sleep 1 +} + +bootstrap_node_data_dir() { + local label="$1" + local data_dir="$2" + + echo "Bootstrapping $label..." + "$NODE_BINARY" bootstrap \ + --data-directory "$data_dir" \ + --genesis-block "$VALIDATOR_DIR/genesis.dat" +} + +bootstrap_ntx_builder() { + echo "Bootstrapping network transaction builder..." + + "$NTX_BUILDER_BINARY" bootstrap \ + --data-directory "$NTX_BUILDER_DIR" \ + --genesis-block "$VALIDATOR_DIR/genesis.dat" +} + +node_resource_attributes() { + local instance_id="$1" + + if [[ -n "${OTEL_RESOURCE_ATTRIBUTES:-}" ]]; then + printf "service.instance.id=%s,%s" "$instance_id" "$OTEL_RESOURCE_ATTRIBUTES" + else + printf "service.instance.id=%s" "$instance_id" + fi +} + +# --- Kill processes on required ports --- + +kill_ports # --- Bootstrap --- if [[ "$SKIP_BOOTSTRAP" != "true" ]]; then echo "=== Bootstrapping ===" - rm -rf "$VALIDATOR_DIR" "$ACCOUNTS_DIR" "$STORE_DIR" \ - "$STORE_REPLICA_1_DIR" "$STORE_REPLICA_2_DIR" "$NTX_BUILDER_DIR" - mkdir -p "$NTX_BUILDER_DIR" + rm -rf "$VALIDATOR_DIR" "$ACCOUNTS_DIR" "$NODE_DIR" "$FULL_NODE_1_DIR" "$FULL_NODE_2_DIR" "$NTX_BUILDER_DIR" echo "Bootstrapping validator..." KMS_BOOTSTRAP_ARGS=() @@ -78,27 +105,20 @@ if [[ "$SKIP_BOOTSTRAP" != "true" ]]; then KMS_BOOTSTRAP_ARGS+=(--validator.key.kms-id "$KMS_KEY_ID") fi - $VALIDATOR_BINARY bootstrap \ + "$VALIDATOR_BINARY" bootstrap \ --data-directory "$VALIDATOR_DIR" \ --genesis-block-directory "$VALIDATOR_DIR" \ --accounts-directory "$ACCOUNTS_DIR" \ --genesis-config-file "$GENESIS_CONFIG" \ "${KMS_BOOTSTRAP_ARGS[@]+"${KMS_BOOTSTRAP_ARGS[@]}"}" - echo "Bootstrapping store..." - $BINARY store bootstrap \ - --data-directory "$STORE_DIR" \ - --genesis-block "$VALIDATOR_DIR/genesis.dat" - - echo "Bootstrapping store replica 1..." - $BINARY store bootstrap \ - --data-directory "$STORE_REPLICA_1_DIR" \ - --genesis-block "$VALIDATOR_DIR/genesis.dat" + bootstrap_node_data_dir "sequencer node" "$NODE_DIR" + bootstrap_ntx_builder - echo "Bootstrapping store replica 2..." - $BINARY store bootstrap \ - --data-directory "$STORE_REPLICA_2_DIR" \ - --genesis-block "$VALIDATOR_DIR/genesis.dat" + if [[ "$ENABLE_FULL_NODES" == "true" ]]; then + bootstrap_node_data_dir "full node 1" "$FULL_NODE_1_DIR" + bootstrap_node_data_dir "full node 2" "$FULL_NODE_2_DIR" + fi else echo "=== Skipping bootstrap (SKIP_BOOTSTRAP=true) ===" fi @@ -107,92 +127,69 @@ fi echo "=== Starting components ===" -echo "Starting sequencer store..." -OTEL_SERVICE_NAME=miden-store-primary $BINARY store start \ - --rpc.listen "0.0.0.0:$STORE_RPC_PORT" \ - --block-producer.listen "0.0.0.0:$STORE_BLOCK_PRODUCER_PORT" \ - --data-directory "$STORE_DIR" \ - $EXTRA_ARGS & -PIDS+=($!) - KMS_START_ARGS=() if [[ -n "$KMS_KEY_ID" ]]; then KMS_START_ARGS+=(--key.kms-id "$KMS_KEY_ID") fi echo "Starting validator..." -OTEL_SERVICE_NAME=miden-validator $VALIDATOR_BINARY start --listen "0.0.0.0:$VALIDATOR_PORT" \ +"$VALIDATOR_BINARY" start --listen "0.0.0.0:$VALIDATOR_PORT" \ --data-directory "$VALIDATOR_DIR" \ $EXTRA_ARGS \ "${KMS_START_ARGS[@]+"${KMS_START_ARGS[@]}"}" & PIDS+=($!) -# Give store and validator a moment to bind their ports. +# Give the validator a moment to bind before the sequencer starts producing blocks. sleep 2 -# Replica 1 syncs from the primary store. -echo "Starting store replica 1 (upstream: primary store at 127.0.0.1:$STORE_RPC_PORT)..." -OTEL_SERVICE_NAME=miden-store-replica-1 $BINARY store start-replica \ - --rpc.listen "0.0.0.0:$STORE_REPLICA_1_RPC_PORT" \ - --upstream-store.url "http://127.0.0.1:$STORE_RPC_PORT" \ - --data-directory "$STORE_REPLICA_1_DIR" \ - $EXTRA_ARGS & -PIDS+=($!) - -# Replica 2 syncs from replica 1, proving replicas can act as upstreams. -echo "Starting store replica 2 (upstream: replica 1 at 127.0.0.1:$STORE_REPLICA_1_RPC_PORT)..." -OTEL_SERVICE_NAME=miden-store-replica-2 $BINARY store start-replica \ - --rpc.listen "0.0.0.0:$STORE_REPLICA_2_RPC_PORT" \ - --upstream-store.url "http://127.0.0.1:$STORE_REPLICA_1_RPC_PORT" \ - --data-directory "$STORE_REPLICA_2_DIR" \ - $EXTRA_ARGS & -PIDS+=($!) - -echo "Starting block producer..." -OTEL_SERVICE_NAME=miden-block-producer $BINARY block-producer start --listen "0.0.0.0:$BLOCK_PRODUCER_PORT" \ - --store.url "http://127.0.0.1:$STORE_BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ - $EXTRA_ARGS & -PIDS+=($!) - -echo "Starting RPC server (primary store)..." -OTEL_SERVICE_NAME=miden-rpc-primary $BINARY rpc start \ - --listen "0.0.0.0:$RPC_PORT" \ - --store.url "http://127.0.0.1:$STORE_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ - $EXTRA_ARGS & -PIDS+=($!) - -echo "Starting RPC server (replica 1)..." -OTEL_SERVICE_NAME=miden-rpc-replica-1 $BINARY rpc start \ - --listen "0.0.0.0:$RPC_REPLICA_1_PORT" \ - --store.url "http://127.0.0.1:$STORE_REPLICA_1_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ - $EXTRA_ARGS & -PIDS+=($!) - -echo "Starting RPC server (replica 2)..." -OTEL_SERVICE_NAME=miden-rpc-replica-2 $BINARY rpc start \ - --listen "0.0.0.0:$RPC_REPLICA_2_PORT" \ - --store.url "http://127.0.0.1:$STORE_REPLICA_2_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ +echo "Starting sequencer..." +OTEL_RESOURCE_ATTRIBUTES="$(node_resource_attributes sequencer)" \ +"$NODE_BINARY" sequencer \ + --rpc.listen "0.0.0.0:$RPC_PORT" \ + --data-directory "$NODE_DIR" \ --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ + --ntx-builder.url "http://127.0.0.1:$NTX_BUILDER_PORT" \ $EXTRA_ARGS & PIDS+=($!) echo "Starting network transaction builder..." -OTEL_SERVICE_NAME=miden-ntx-builder $NTX_BUILDER_BINARY start \ +"$NTX_BUILDER_BINARY" start \ --listen "0.0.0.0:$NTX_BUILDER_PORT" \ - --store.url "http://127.0.0.1:$STORE_RPC_PORT" \ - --block-producer.url "http://127.0.0.1:$BLOCK_PRODUCER_PORT" \ - --validator.url "http://127.0.0.1:$VALIDATOR_PORT" \ + --rpc.url "http://127.0.0.1:$RPC_PORT" \ --data-directory "$NTX_BUILDER_DIR" \ $EXTRA_ARGS & PIDS+=($!) +if [[ "$ENABLE_FULL_NODES" == "true" ]]; then + echo "Starting full node 1 (upstream: sequencer at 127.0.0.1:$RPC_PORT)..." + OTEL_RESOURCE_ATTRIBUTES="$(node_resource_attributes full-node-1)" \ + "$NODE_BINARY" full \ + --rpc.listen "0.0.0.0:$FULL_NODE_1_RPC_PORT" \ + --sync.block-source.url "http://127.0.0.1:$RPC_PORT" \ + --data-directory "$FULL_NODE_1_DIR" \ + $EXTRA_ARGS & + PIDS+=($!) + + # Give full node 1 a moment to bind before full node 2 uses it as an upstream. + sleep 2 + + echo "Starting full node 2 (upstream: full node 1 at 127.0.0.1:$FULL_NODE_1_RPC_PORT)..." + OTEL_RESOURCE_ATTRIBUTES="$(node_resource_attributes full-node-2)" \ + "$NODE_BINARY" full \ + --rpc.listen "0.0.0.0:$FULL_NODE_2_RPC_PORT" \ + --sync.block-source.url "http://127.0.0.1:$FULL_NODE_1_RPC_PORT" \ + --data-directory "$FULL_NODE_2_DIR" \ + $EXTRA_ARGS & + PIDS+=($!) +else + echo "=== Full nodes disabled (ENABLE_FULL_NODES=false) ===" +fi + echo "=== All components running. Ctrl+C to stop. ===" -echo "=== Block propagation chain: :$STORE_RPC_PORT -> :$STORE_REPLICA_1_RPC_PORT -> :$STORE_REPLICA_2_RPC_PORT ===" -echo "=== RPC endpoints: :$RPC_PORT, :$RPC_REPLICA_1_PORT, :$RPC_REPLICA_2_PORT ===" +if [[ "$ENABLE_FULL_NODES" == "true" ]]; then + echo "=== Block propagation chain: :$RPC_PORT -> :$FULL_NODE_1_RPC_PORT -> :$FULL_NODE_2_RPC_PORT ===" + echo "=== RPC endpoints: :$RPC_PORT, :$FULL_NODE_1_RPC_PORT, :$FULL_NODE_2_RPC_PORT ===" +else + echo "=== RPC endpoint: :$RPC_PORT ===" +fi wait