Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions pkg/monitors/network/dns.go
Original file line number Diff line number Diff line change
Expand Up @@ -2410,22 +2410,30 @@ func (m *DNSMonitor) updateFailureTracking(clusterOK, externalOK bool, status *t
m.externalTrendDetector.Observe(m.externalSuccessTracker.GetSuccessRate(), now)
}

// Toggle ClusterDNSDown True/False based on the consecutive-failure count.
if m.clusterFailureCount >= m.config.FailureCountThreshold {
status.AddCondition(types.NewCondition(
"ClusterDNSDown",
types.ConditionTrue,
"RepeatedClusterDNSFailures",
fmt.Sprintf("Cluster DNS has failed %d consecutive times (threshold: %d)",
m.clusterFailureCount, m.config.FailureCountThreshold),
))
} else if m.clusterFailureCount == 0 {
status.AddCondition(types.NewCondition(
"ClusterDNSDown",
types.ConditionFalse,
"ClusterDNSResolved",
"Cluster DNS resolution is healthy",
))
// Toggle ClusterDNSDown True/False based on the consecutive-failure count — but
// ONLY when this monitor is actually checking cluster domains. With clusterDomains: []
// (the hostNetwork default: the agent cannot resolve ClusterIP cluster records via
// Cilium), checkDNSDomains trivially returns true, which would make this emit an
// unconditional ClusterDNSDown=False and MASK a real ClusterDNSDown=True reported by
// the pod-network cluster-dns-pod monitor (which owns this condition when the in-agent
// check is disabled). So skip the emission entirely when no cluster domains are set.
if len(m.config.ClusterDomains) > 0 {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Clear stale ClusterDNSDown when cluster checks are disabled

When clusterDomains: [] and the pod monitor is not enabled/running, this guard stops the DNS monitor from writing ClusterDNSDown at all. Existing NodeDoctorClusterDNSDown=True values are preserved on startup because retiredDNSConditionTypes does not include it (pkg/exporters/kubernetes/condition_manager.go:237), and the Kubernetes exporter only updates conditions present in each status, so an upgrade or config reload from a failing cluster-domain check to disabled cluster checks leaves the old True condition latched indefinitely instead of clearing/removing it. Please either remove/clear the condition when disabling the in-agent check or ensure another owner is configured before suppressing both True and False writes.

Useful? React with 👍 / 👎.

if m.clusterFailureCount >= m.config.FailureCountThreshold {
status.AddCondition(types.NewCondition(
"ClusterDNSDown",
types.ConditionTrue,
"RepeatedClusterDNSFailures",
fmt.Sprintf("Cluster DNS has failed %d consecutive times (threshold: %d)",
m.clusterFailureCount, m.config.FailureCountThreshold),
))
} else if m.clusterFailureCount == 0 {
status.AddCondition(types.NewCondition(
"ClusterDNSDown",
types.ConditionFalse,
"ClusterDNSResolved",
"Cluster DNS resolution is healthy",
))
}
}

// Toggle ExternalDNSDown True/False based on the consecutive-failure count.
Expand Down
82 changes: 82 additions & 0 deletions pkg/monitors/network/dns_clusterdown_guard_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package network

import (
"testing"

"github.com/supporttools/node-doctor/pkg/types"
)

// newFailureTrackingMonitor builds a minimal DNSMonitor sufficient to exercise
// updateFailureTracking with the given cluster domains.
func newFailureTrackingMonitor(clusterDomains []string) *DNSMonitor {
return &DNSMonitor{
config: &DNSMonitorConfig{
ClusterDomains: clusterDomains,
FailureCountThreshold: 3,
SuccessRateTracking: &SuccessRateConfig{Enabled: false, WindowSize: 10},
},
clusterSuccessTracker: NewRingBuffer(10),
externalSuccessTracker: NewRingBuffer(10),
}
}

func hasConditionType(s *types.Status, condType string) bool {
for _, c := range s.Conditions {
if c.Type == condType {
return true
}
}
return false
}

// TestClusterDNSDownNotEmittedWhenClusterDomainsEmpty verifies the fix for the
// two-monitor conflict: when clusterDomains is empty (in-agent cluster check disabled
// because a hostNetwork agent can't resolve ClusterIP cluster records), the DNS monitor
// must NOT emit ClusterDNSDown — otherwise its unconditional False would mask the
// pod-network cluster-dns-pod monitor's True.
func TestClusterDNSDownNotEmittedWhenClusterDomainsEmpty(t *testing.T) {
m := newFailureTrackingMonitor(nil) // no cluster domains

// Even a "healthy" cluster result must not emit ClusterDNSDown.
s := types.NewStatus("test-dns")
m.updateFailureTracking(true, true, s)
if hasConditionType(s, "ClusterDNSDown") {
t.Errorf("ClusterDNSDown must NOT be emitted when clusterDomains is empty, got conditions: %+v", s.Conditions)
}

// Even repeated cluster "failures" must not emit ClusterDNSDown when disabled.
for i := 0; i < 5; i++ {
s = types.NewStatus("test-dns")
m.updateFailureTracking(false, true, s)
}
if hasConditionType(s, "ClusterDNSDown") {
t.Errorf("ClusterDNSDown must NOT be emitted when clusterDomains is empty even after failures, got: %+v", s.Conditions)
}

// ExternalDNSDown is unaffected — still emitted (False on healthy external).
if !hasConditionType(s, "ExternalDNSDown") {
t.Errorf("ExternalDNSDown should still be emitted regardless of clusterDomains")
}
}

// TestClusterDNSDownEmittedWhenClusterDomainsSet verifies the normal path still works:
// with cluster domains configured, the DNS monitor owns ClusterDNSDown as before.
func TestClusterDNSDownEmittedWhenClusterDomainsSet(t *testing.T) {
m := newFailureTrackingMonitor([]string{"kubernetes.default.svc.cluster.local"})

// Healthy -> ClusterDNSDown=False.
s := types.NewStatus("test-dns")
m.updateFailureTracking(true, true, s)
if !hasCondition(s, "ClusterDNSDown", types.ConditionFalse, "ClusterDNSResolved") {
t.Errorf("expected ClusterDNSDown=False when clusterDomains set and healthy, got: %+v", s.Conditions)
}

// Repeated failures past threshold -> ClusterDNSDown=True.
for i := 0; i < 3; i++ {
s = types.NewStatus("test-dns")
m.updateFailureTracking(false, true, s)
}
if !hasCondition(s, "ClusterDNSDown", types.ConditionTrue, "RepeatedClusterDNSFailures") {
t.Errorf("expected ClusterDNSDown=True after threshold failures, got: %+v", s.Conditions)
}
}
Loading