Add Zipkin tracing parity

nemo_retriever/helm/README.md

@@ -937,6 +937,91 @@ sanity check before opening Grafana.
 
 ---
 
+## Tracing and Zipkin
+
+Helm installs the chart-owned OpenTelemetry Collector and Zipkin backend on by
+default. This is intentional: the legacy 26.1.2 Helm chart shipped with a
+managed Zipkin deployment enabled, so the new chart keeps a default trace
+backend available for functional parity. Pod trace export is still opt-in:
+
+```yaml
+topology:
+  otel:
+    enabled: true
+  zipkin:
+    enabled: true
+
+service:
+  otel:
+    enabled: false
+
+nimOperator:
+  otel:
+    enabled: false
+```
+
+Because Zipkin is chart-owned by default, an upgrade with default values can
+create a Zipkin Deployment and Service. Set `topology.zipkin.enabled=false`
+before upgrading if your deployment uses an external backend or should not run
+chart-owned Zipkin.
+
+Set `service.otel.enabled=true` and `nimOperator.otel.enabled=true` to have
+retriever service pods and chart-managed NIMs emit OTLP to the chart's
+OpenTelemetry Collector. Once pod trace export is enabled, the collector exports
+traces to the chart-owned Zipkin service. Open a job and read the Zipkin lookup
+key from either the JSON body or the `x-trace-id` response header:
+
+```bash
+kubectl port-forward svc/tracing-smoke-nemo-retriever 7670:80
+
+curl -s -D headers.txt -o job.json \
+  -X POST http://localhost:7670/v1/ingest/job \
+  -H 'content-type: application/json' \
+  -d '{"expected_documents":1}'
+
+TRACE_ID=$(jq -r .trace_id job.json)
+grep -i x-trace-id headers.txt
+```
+
+Port-forward Zipkin and query the trace directly:
+
+```bash
+kubectl port-forward svc/tracing-smoke-nemo-retriever-zipkin 9411:9411
+curl "http://localhost:9411/api/v2/trace/${TRACE_ID}"
+```
+
+Common opt-out and override knobs:
+
+```yaml
+topology:
+  zipkin:
+    enabled: false                 # do not deploy chart-owned Zipkin
+    exporter:
+      enabled: false               # keep Zipkin deployed, but do not export traces to it
+      endpoint: http://external-zipkin:9411/api/v2/spans
+
+service:
+  otel:
+    enabled: true                  # inject service pod instrumentation env
+
+nimOperator:
+  otel:
+    enabled: true                  # inject inherited NIM OTLP env
+  page_elements:
+    otel:
+      enabled: false               # per-NIM opt-out
+  ocr:
+    otel:
+      env:
+        TRITON_OTEL_RATE: "10"     # per-NIM Triton OTel override
+```
+
+Set `topology.zipkin.exporter.endpoint` when you run your own Zipkin-compatible
+collector. Set `topology.otel.enabled=false` to disable the chart-owned collector
+and all chart-rendered collector wiring.
+
+---
+
 ## OpenShift deployment { #openshift-deployment }
 
 The chart defaults target generic Kubernetes clusters that allow fixed numeric

nemo_retriever/helm/templates/_helpers.tpl

@@ -200,6 +200,285 @@ nemo-retriever.role.configMapName
 {{- printf "%s-config" (include "nemo-retriever.role.fullname" .) -}}
 {{- end -}}
 
+
+{{/*
+=============================================================================
+Tracing helpers
+=============================================================================
+*/}}
+
+{{- define "nemo-retriever.suffixedFullname" -}}
+{{- $base := include "nemo-retriever.fullname" .context -}}
+{{- $suffix := .suffix -}}
+{{- $maxBaseLen := int (sub 63 (len $suffix)) -}}
+{{- printf "%s%s" ($base | trunc $maxBaseLen | trimSuffix "-") $suffix | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "nemo-retriever.zipkin.fullname" -}}
+{{- include "nemo-retriever.suffixedFullname" (dict "context" . "suffix" "-zipkin") -}}
+{{- end -}}
+
+{{- define "nemo-retriever.otel.fullname" -}}
+{{- include "nemo-retriever.suffixedFullname" (dict "context" . "suffix" "-otel") -}}
+{{- end -}}
+
+{{- define "nemo-retriever.otel.configMapName" -}}
+{{- include "nemo-retriever.suffixedFullname" (dict "context" . "suffix" "-otel-config") -}}
+{{- end -}}
+
+{{- define "nemo-retriever.otel.ports" -}}
+{{- $otel := .Values.topology.otel | default dict -}}
+{{- if not (kindIs "map" $otel) -}}
+{{- fail "topology.otel must be a map" -}}
+{{- end -}}
+{{- $ports := get $otel "ports" -}}
+{{- if not (kindIs "map" $ports) -}}
+{{- fail "topology.otel.ports must be a map when topology.otel.enabled=true" -}}
+{{- end -}}
+{{- range $portName := list "otlpGrpc" "otlpHttp" "prometheus" -}}
+{{- if not (get $ports $portName) -}}
+{{- fail (printf "topology.otel.ports.%s is required when topology.otel.enabled=true" $portName) -}}
+{{- end -}}
+{{- end -}}
+{{- toYaml $ports -}}
+{{- end -}}
+
+{{- define "nemo-retriever.zipkin.image" -}}
+{{- $zipkin := .Values.topology.zipkin | default dict -}}
+{{- if not (kindIs "map" $zipkin) -}}
+{{- fail "topology.zipkin must be a map" -}}
+{{- end -}}
+{{- $image := get $zipkin "image" -}}
+{{- if not (kindIs "map" $image) -}}
+{{- fail "topology.zipkin.image must be a map when topology.zipkin.enabled=true" -}}
+{{- end -}}
+{{- range $fieldName := list "repository" "tag" "pullPolicy" -}}
+{{- if not (get $image $fieldName) -}}
+{{- fail (printf "topology.zipkin.image.%s is required when topology.zipkin.enabled=true" $fieldName) -}}
+{{- end -}}
+{{- end -}}
+{{- toYaml $image -}}
+{{- end -}}
+
+{{- define "nemo-retriever.zipkin.port" -}}
+{{- $zipkin := .Values.topology.zipkin | default dict -}}
+{{- if not (kindIs "map" $zipkin) -}}
+{{- fail "topology.zipkin must be a map" -}}
+{{- end -}}
+{{- $port := get $zipkin "port" -}}
+{{- if not $port -}}
+{{- fail "topology.zipkin.port is required when topology.zipkin.enabled=true" -}}
+{{- end -}}
+{{- $port -}}
+{{- end -}}
+
+
+{{- define "nemo-retriever.zipkin.endpoint" -}}
+{{- $zipkin := .Values.topology.zipkin | default dict -}}
+{{- if not (kindIs "map" $zipkin) -}}
+{{- fail "topology.zipkin must be a map" -}}
+{{- end -}}
+{{- $exporter := get $zipkin "exporter" | default dict -}}
+{{- if not (kindIs "map" $exporter) -}}
+{{- fail "topology.zipkin.exporter must be a map" -}}
+{{- end -}}
+{{- $endpoint := get $exporter "endpoint" -}}
+{{- if $endpoint -}}
+{{- tpl $endpoint . -}}
+{{- else -}}
+{{- printf "http://%s:%v/api/v2/spans" (include "nemo-retriever.zipkin.fullname" .) (include "nemo-retriever.zipkin.port" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "nemo-retriever.otel.config" -}}
+{{- $otel := .Values.topology.otel | default dict -}}
+{{- if not (kindIs "map" $otel) -}}
+{{- fail "topology.otel must be a map" -}}
+{{- end -}}
+{{- $configValue := get $otel "config" -}}
+{{- if not (kindIs "map" $configValue) -}}
+{{- fail "topology.otel.config must be a map when topology.otel.enabled=true" -}}
+{{- end -}}
+{{- $config := deepCopy $configValue -}}
+{{- $zipkin := .Values.topology.zipkin | default dict -}}
+{{- if not (kindIs "map" $zipkin) -}}
+{{- fail "topology.zipkin must be a map" -}}
+{{- end -}}
+{{- $zipkinExporterConfig := get $zipkin "exporter" | default dict -}}
+{{- if not (kindIs "map" $zipkinExporterConfig) -}}
+{{- fail "topology.zipkin.exporter must be a map" -}}
+{{- end -}}
+{{- $zipkinInjectionEnabled := and (get $zipkinExporterConfig "enabled") (or (get $zipkin "enabled") (get $zipkinExporterConfig "endpoint")) -}}
+{{- if $zipkinInjectionEnabled -}}
+{{- $service := get $config "service" | default dict -}}
+{{- $pipelines := get $service "pipelines" | default dict -}}
+{{- $traces := get $pipelines "traces" -}}
+{{- if not $traces -}}
+{{- fail "topology.zipkin.exporter.enabled requires topology.otel.config.service.pipelines.traces with non-empty receivers; provide that traces pipeline or set topology.zipkin.exporter.enabled=false" -}}
+{{- end -}}
+{{- $traceReceivers := get $traces "receivers" -}}
+{{- if not $traceReceivers -}}
+{{- fail "topology.zipkin.exporter.enabled requires topology.otel.config.service.pipelines.traces with non-empty receivers; provide that traces pipeline or set topology.zipkin.exporter.enabled=false" -}}
+{{- end -}}
+{{- $receivers := get $config "receivers" | default dict -}}
+{{- range $receiverName := $traceReceivers -}}
+{{- if or (not (hasKey $receivers $receiverName)) (eq (get $receivers $receiverName) nil) -}}
+{{- fail (printf "topology.otel.config.service.pipelines.traces trace receiver %q is missing or null in topology.otel.config.receivers; fix topology.otel.config or set topology.zipkin.exporter.enabled=false" $receiverName) -}}
+{{- end -}}
+{{- end -}}
+{{- $processors := get $config "processors" | default dict -}}
+{{- $traceProcessors := get $traces "processors" | default list -}}
+{{- range $processorName := $traceProcessors -}}
+{{- if or (not (hasKey $processors $processorName)) (eq (get $processors $processorName) nil) -}}
+{{- fail (printf "topology.otel.config.service.pipelines.traces trace processor %q is missing or null in topology.otel.config.processors; fix topology.otel.config or set topology.zipkin.exporter.enabled=false" $processorName) -}}
+{{- end -}}
+{{- end -}}
+{{- $exporters := get $config "exporters" | default dict -}}
+{{- $zipkinExporter := get $exporters "zipkin" | default dict -}}
+{{- if not (kindIs "map" $zipkinExporter) -}}
+{{- fail "topology.otel.config.exporters.zipkin must be a map; fix topology.otel.config or set topology.zipkin.exporter.enabled=false" -}}
+{{- end -}}
+{{- $zipkinExporter = mergeOverwrite (deepCopy $zipkinExporter) (dict "endpoint" (include "nemo-retriever.zipkin.endpoint" .)) -}}
+{{- $_ := set $exporters "zipkin" $zipkinExporter -}}
+{{- $_ := set $config "exporters" $exporters -}}
+{{- $traceExporters := get $traces "exporters" | default list -}}
+{{- if not (has "zipkin" $traceExporters) -}}
+{{- $traceExporters = append $traceExporters "zipkin" -}}
+{{- end -}}
+{{- range $exporterName := $traceExporters -}}
+{{- if or (not (hasKey $exporters $exporterName)) (eq (get $exporters $exporterName) nil) -}}
+{{- fail (printf "topology.otel.config.service.pipelines.traces trace exporter %q is missing or null in topology.otel.config.exporters; fix topology.otel.config or set topology.zipkin.exporter.enabled=false" $exporterName) -}}
+{{- end -}}
+{{- end -}}
+{{- $_ := set $traces "exporters" $traceExporters -}}
+{{- $_ := set $pipelines "traces" $traces -}}
+{{- $_ := set $service "pipelines" $pipelines -}}
+{{- $_ := set $config "service" $service -}}
+{{- end -}}
+{{- toYaml $config -}}
+{{- end -}}
+
+{{- define "nemo-retriever.serviceOtelEnv" -}}
+{{- $root := .context -}}
+{{- $role := .role -}}
+{{- $serviceOtel := $root.Values.service.otel | default dict -}}
+{{- $topologyOtel := $root.Values.topology.otel | default dict -}}
+{{- if not (kindIs "map" $serviceOtel) -}}
+{{- fail "service.otel must be a map" -}}
+{{- end -}}
+{{- if not (kindIs "map" $topologyOtel) -}}
+{{- fail "topology.otel must be a map" -}}
+{{- end -}}
+{{- if and (get $topologyOtel "enabled") (get $serviceOtel "enabled") -}}
+{{- $serviceOtelEnv := get $serviceOtel "env" | default dict -}}
+{{- if not (kindIs "map" $serviceOtelEnv) -}}
+{{- fail "service.otel.env must be a map" -}}
+{{- end -}}
+{{- $userEnvNames := dict -}}
+{{- range $env := $root.Values.service.env -}}
+{{- if and $env (hasKey $env "name") -}}
+{{- $_ := set $userEnvNames $env.name true -}}
+{{- end -}}
+{{- end -}}
+{{- $otelPorts := include "nemo-retriever.otel.ports" $root | fromYaml -}}
+{{- $defaults := dict
+  "OTEL_EXPORTER_OTLP_ENDPOINT" (printf "http://%s:%v" (include "nemo-retriever.otel.fullname" $root) (get $otelPorts "otlpGrpc"))
+  "OTEL_SERVICE_NAME" (default "nemo-retriever-service" (get $serviceOtel "serviceName"))
+  "OTEL_TRACES_EXPORTER" "otlp"
+  "OTEL_METRICS_EXPORTER" "otlp"
+  "OTEL_LOGS_EXPORTER" "none"
+  "OTEL_PROPAGATORS" "tracecontext,baggage"
+  "OTEL_RESOURCE_ATTRIBUTES" (printf "service.namespace=nemo-retriever,service.role=%s" $role)
+  "OTEL_PYTHON_EXCLUDED_URLS" "health"
+-}}
+{{- $otelEnv := mergeOverwrite (deepCopy $defaults) (deepCopy $serviceOtelEnv) -}}
+{{- range $name, $value := $otelEnv }}
+{{- if not (hasKey $userEnvNames $name) }}
+- name: {{ $name }}
+  value: {{ $value | quote }}
+{{- end }}
+{{- end }}
+{{- end -}}
+{{- end -}}
+
+{{- define "nemo-retriever.nimServiceOtelEnv" -}}
+{{- $root := .context -}}
+{{- $key := .key -}}
+{{- $name := .name -}}
+{{- $nim := index $root.Values.nimOperator $key -}}
+{{- $chartOtel := $root.Values.nimOperator.otel | default dict -}}
+{{- $nimOtel := $nim.otel | default dict -}}
+{{- $topologyOtel := $root.Values.topology.otel | default dict -}}
+{{- if not (kindIs "map" $chartOtel) -}}
+{{- fail "nimOperator.otel must be a map" -}}
+{{- end -}}
+{{- if not (kindIs "map" $topologyOtel) -}}
+{{- fail "topology.otel must be a map" -}}
+{{- end -}}
+{{- if not (kindIs "map" $nimOtel) -}}
+{{- fail (printf "nimOperator.%s.otel must be a map" $key) -}}
+{{- end -}}
+{{- $enabled := get $chartOtel "enabled" -}}
+{{- if and (hasKey $nimOtel "enabled") (not (eq (get $nimOtel "enabled") nil)) -}}
+{{- $enabled = get $nimOtel "enabled" -}}
+{{- end -}}
+{{- if and (get $topologyOtel "enabled") $enabled -}}
+{{- $chartNimOtelEnv := get $chartOtel "env" | default dict -}}
+{{- $nimOtelEnv := get $nimOtel "env" | default dict -}}
+{{- if not (kindIs "map" $chartNimOtelEnv) -}}
+{{- fail "nimOperator.otel.env must be a map" -}}
+{{- end -}}
+{{- if not (kindIs "map" $nimOtelEnv) -}}
+{{- fail (printf "nimOperator.%s.otel.env must be a map" $key) -}}
+{{- end -}}
+{{- $existingEnvNames := dict -}}
+{{- $existingEnvValues := dict -}}
+{{- range $env := $nim.env -}}
+{{- if and $env (hasKey $env "name") -}}
+{{- $_ := set $existingEnvNames $env.name true -}}
+{{- if hasKey $env "value" -}}
+{{- $_ := set $existingEnvValues $env.name $env.value -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- $otelPorts := include "nemo-retriever.otel.ports" $root | fromYaml -}}
+{{- $defaultEndpoint := printf "http://%s:%v" (include "nemo-retriever.otel.fullname" $root) (get $otelPorts "otlpHttp") -}}
+{{- $chartEndpoint := default $defaultEndpoint (get $chartOtel "endpoint") -}}
+{{- $endpoint := default $chartEndpoint (get $nimOtel "endpoint") -}}
+{{- $tritonPath := default "/v1/traces" (get $chartOtel "tritonPath") -}}
+{{- $serviceName := default $name (get $nimOtel "serviceName") -}}
+{{- $defaults := dict
+  "NIM_ENABLE_OTEL" "true"
+  "NIM_OTEL_SERVICE_NAME" $serviceName
+  "NIM_OTEL_TRACES_EXPORTER" "otlp"
+  "NIM_OTEL_METRICS_EXPORTER" "console"
+  "NIM_OTEL_EXPORTER_OTLP_ENDPOINT" $endpoint
+  "TRITON_OTEL_URL" (printf "%s%s" $endpoint $tritonPath)
+  "TRITON_OTEL_RATE" "1"
+-}}
+{{- $explicitTritonUrl := or (hasKey $existingEnvNames "TRITON_OTEL_URL") (hasKey $chartNimOtelEnv "TRITON_OTEL_URL") (hasKey $nimOtelEnv "TRITON_OTEL_URL") -}}
+{{- $existingEndpointWithoutLiteral := and (hasKey $existingEnvNames "NIM_OTEL_EXPORTER_OTLP_ENDPOINT") (not (hasKey $existingEnvValues "NIM_OTEL_EXPORTER_OTLP_ENDPOINT")) -}}
+{{- $otelEnv := mergeOverwrite (deepCopy $defaults) (deepCopy $chartNimOtelEnv) (deepCopy $nimOtelEnv) -}}
+{{- $finalEndpoint := get $otelEnv "NIM_OTEL_EXPORTER_OTLP_ENDPOINT" -}}
+{{- if hasKey $existingEnvValues "NIM_OTEL_EXPORTER_OTLP_ENDPOINT" -}}
+{{- $finalEndpoint = get $existingEnvValues "NIM_OTEL_EXPORTER_OTLP_ENDPOINT" -}}
+{{- end -}}
+{{- if not $explicitTritonUrl -}}
+{{- if $existingEndpointWithoutLiteral -}}
+{{- $_ := unset $otelEnv "TRITON_OTEL_URL" -}}
+{{- else -}}
+{{- $_ := set $otelEnv "TRITON_OTEL_URL" (printf "%s%s" $finalEndpoint $tritonPath) -}}
+{{- end -}}
+{{- end -}}
+{{- range $envName, $envValue := $otelEnv }}
+{{- if not (hasKey $existingEnvNames $envName) }}
+- name: {{ $envName }}
+  value: {{ $envValue | quote }}
+{{- end }}
+{{- end }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 =============================================================================
 NIMService GPU resources

nemo_retriever/helm/templates/configmap-otel.yaml

@@ -1,13 +1,16 @@
-{{- if .Values.topology.otel.enabled }}
-{{- $fullname := include "nemo-retriever.fullname" . -}}
-{{- $otelName := printf "%s-otel" $fullname -}}
+{{- $otel := .Values.topology.otel | default dict -}}
+{{- if not (kindIs "map" $otel) -}}
+{{- fail "topology.otel must be a map" -}}
+{{- end -}}
+{{- if get $otel "enabled" }}
+{{- $otelConfigName := include "nemo-retriever.otel.configMapName" . -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ $otelName }}-config
+  name: {{ $otelConfigName }}
   labels:
     {{- include "nemo-retriever.role.labels" (dict "context" $ "role" "otel") | nindent 4 }}
 data:
   config.yaml: |
-    {{- toYaml .Values.topology.otel.config | nindent 4 }}
+    {{- include "nemo-retriever.otel.config" . | nindent 4 }}
 {{- end }}