feat: ambient reconcile host-level health check rules at runtime

cni/pkg/cmd/root.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -127,6 +128,7 @@ var rootCmd = &cobra.Command{
 					DNSCapture:                 cfg.InstallConfig.AmbientDNSCapture,
 					EnableIPv6:                 cfg.InstallConfig.AmbientIPv6,
 					ReconcilePodRulesOnStartup: cfg.InstallConfig.AmbientReconcilePodRulesOnStartup,
+					ReconcileHostRulesInterval: parseReconcileHostRulesInterval(cfg.InstallConfig.AmbientReconcileHostRulesInterval),
 					NativeNftables:             cfg.InstallConfig.NativeNftables,
 					ForceIptablesBinary:        cfg.InstallConfig.ForceIptablesBinary,
 				})
@@ -266,6 +268,31 @@ func init() {
 		"A set of field selectors in label=value format that will be added to the pod list filters")
 }
 
+// defaultReconcileHostRulesInterval is the default interval of the periodic host rules
+// reconciliation, matching kube-proxy's --iptables-sync-period default.
+const defaultReconcileHostRulesInterval = 30 * time.Second
+
+// parseReconcileHostRulesInterval parses the host rules reconcile interval:
+// an empty or invalid value logs a warning and falls back to the default;
+// <= 0 disables the feature.
+func parseReconcileHostRulesInterval(raw string) time.Duration {
+	if raw == "" {
+		return defaultReconcileHostRulesInterval
+	}
+	interval, err := time.ParseDuration(raw)
+	if err != nil {
+		log.Warnf("invalid %s value %q, falling back to default %v: %v",
+			constants.AmbientReconcileHostRulesInterval, raw, defaultReconcileHostRulesInterval, err)
+		return defaultReconcileHostRulesInterval
+	}
+	if interval <= 0 {
+		log.Infof("host rules periodic reconcile is disabled (%s=%q)",
+			constants.AmbientReconcileHostRulesInterval, raw)
+		return 0
+	}
+	return interval
+}
+
 func registerStringParameter(name, value, usage string) {
 	rootCmd.Flags().String(name, value, usage)
 	registerEnvironment(name, value, usage)
@@ -333,6 +360,7 @@ func constructConfig() (*config.Config, error) {
 		AmbientIPv6:                       viper.GetBool(constants.AmbientIPv6),
 		AmbientDisableSafeUpgrade:         viper.GetBool(constants.AmbientDisableSafeUpgrade),
 		AmbientReconcilePodRulesOnStartup: viper.GetBool(constants.AmbientReconcilePodRulesOnStartup),
+		AmbientReconcileHostRulesInterval: viper.GetString(constants.AmbientReconcileHostRulesInterval),
 		EnableAmbientDetectionRetry:       viper.GetBool(constants.EnableAmbientDetectionRetry),
 
 		NativeNftables:      viper.GetBool(constants.NativeNftables),

cni/pkg/cmd/root_test.go

@@ -0,0 +1,43 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"istio.io/istio/pkg/test/util/assert"
+)
+
+func TestParseReconcileHostRulesInterval(t *testing.T) {
+	cases := []struct {
+		name     string
+		raw      string
+		expected time.Duration
+	}{
+		{"empty falls back to default", "", defaultReconcileHostRulesInterval},
+		{"valid interval", "1m30s", 90 * time.Second},
+		{"zero disables", "0", 0},
+		{"zero seconds disables", "0s", 0},
+		{"negative disables", "-5s", 0},
+		{"invalid falls back to default", "banana", defaultReconcileHostRulesInterval},
+		{"missing unit falls back to default", "30", defaultReconcileHostRulesInterval},
+	}
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, parseReconcileHostRulesInterval(tt.raw))
+		})
+	}
+}

cni/pkg/config/config.go

@@ -160,6 +160,10 @@ type InstallConfig struct {
 	// Whether reconciliation of iptables at post startup is enabled for Ambient workloads
 	AmbientReconcilePodRulesOnStartup bool
 
+	// The interval of the periodic runtime reconciliation of the ambient host-level
+	// health check rules (duration string, e.g. "30s"; "0" disables it)
+	AmbientReconcileHostRulesInterval string
+
 	// Whether to retry checking if a pod is ambient in the cni plugin when there are errors
 	EnableAmbientDetectionRetry bool
 
@@ -251,6 +255,7 @@ func (c InstallConfig) String() string {
 	b.WriteString("AmbientIPv6: " + fmt.Sprint(c.AmbientIPv6) + "\n")
 	b.WriteString("AmbientDisableSafeUpgrade: " + fmt.Sprint(c.AmbientDisableSafeUpgrade) + "\n")
 	b.WriteString("AmbientReconcilePodRulesOnStartup: " + fmt.Sprint(c.AmbientReconcilePodRulesOnStartup) + "\n")
+	b.WriteString("AmbientReconcileHostRulesInterval: " + c.AmbientReconcileHostRulesInterval + "\n")
 	b.WriteString("EnableAmbientDetectionRetry: " + fmt.Sprint(c.EnableAmbientDetectionRetry) + "\n")
 
 	b.WriteString("NativeNftables: " + fmt.Sprint(c.NativeNftables) + "\n")

cni/pkg/constants/constants.go

@@ -47,6 +47,7 @@ const (
 	AmbientIPv6                       = "ambient-ipv6"
 	AmbientDisableSafeUpgrade         = "ambient-disable-safe-upgrade"
 	AmbientReconcilePodRulesOnStartup = "ambient-reconcile-pod-rules-on-startup"
+	AmbientReconcileHostRulesInterval = "ambient-reconcile-host-rules-interval"
 	EnableAmbientDetectionRetry       = "enable-ambient-detection-retry"
 
 	NativeNftables = "native-nftables"

cni/pkg/iptables/iptables.go

@@ -18,6 +18,9 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
 
 	"istio.io/istio/cni/pkg/config"
 	"istio.io/istio/cni/pkg/ipset"
@@ -585,6 +588,109 @@ func (cfg *IptablesConfigurator) CreateHostRulesForHealthChecks() error {
 	})
 }
 
+// EnsureHostRules verifies that the health check rules in the host netns are still in
+// place and re-installs them when they have drifted (flushed/overwritten by an external
+// actor). When everything is in place it performs no writes at all. It is idempotent
+// and meant to be invoked from a periodic reconcile loop (istio/istio#60607).
+//
+// Notes:
+//   - This deliberately does NOT reuse the cfg.Reconcile cleanup path: that path sets up
+//     DROP guardrails in the filter table INPUT/FORWARD/OUTPUT chains. Inside a pod netns
+//     they prevent traffic from escaping the mesh while rules are rebuilt, but in the
+//     host netns they would drop the entire node's traffic, so they must never be
+//     enabled here.
+//   - The repair reuses the same retried DeleteHostRules + CreateHostRulesForHealthChecks
+//     delete-then-create sequence as the startup path, instead of simply replaying
+//     iptables-restore: the restore payload creates chains with -N, which fails when the
+//     chain already exists (so partial residue would never converge), and replaying
+//     would also duplicate the POSTROUTING jump.
+func (cfg *IptablesConfigurator) EnsureHostRules() (bool, error) {
+	builder := cfg.AppendHostRules()
+
+	converged := false
+	err := util.RunAsHost(func() error {
+		var err error
+		converged, err = cfg.hostRulesConverged(builder)
+		return err
+	})
+	if err != nil {
+		// The state could not be read; that is not evidence that the (possibly
+		// healthy) rules are gone. Skip the repair instead of deleting live rules
+		// based on a failed read - the loop will verify again on the next tick.
+		return false, fmt.Errorf("failed to verify host iptables rules, skipping repair: %w", err)
+	}
+	if converged {
+		return false, nil
+	}
+
+	log.WithLabels("component", "host").
+		Warn("detected drift in host-level iptables rules (external flush/overwrite?), re-applying them")
+	// Once the leftovers are deleted the re-create must succeed, otherwise the node
+	// would be left without probe SNAT rules until the next tick; retry the sequence
+	// like the startup path does, but bounded well below the reconcile interval.
+	err = backoff.Retry(func() error {
+		cfg.DeleteHostRules()
+		return cfg.CreateHostRulesForHealthChecks()
+	}, backoff.WithMaxRetries(backoff.NewConstantBackOff(2*time.Second), 5))
+	if err != nil {
+		return true, err
+	}
+	return true, nil
+}
+
+// hostRulesConverged reports whether every host-level rule this configurator manages is
+// still present, using the same read-only probes as VerifyIptablesState (an
+// iptables-save snapshot for chain existence and ISTIO_* chain rule counts, plus a
+// per-rule `iptables -C`). It intentionally diverges from VerifyIptablesState in two
+// ways that matter for a periodic runtime loop:
+//   - a failed iptables-save is returned as an error instead of being reported as
+//     drift, so a transient read failure can never trigger a destructive repair;
+//   - ISTIO_* chains that are not part of the expected host state (e.g. residue from
+//     other istio components or versions) are ignored, since the delete+create repair
+//     cannot remove chains it does not own and treating them as drift would make the
+//     loop re-apply the rules on every tick without ever converging.
+//
+// The IPv6 state is only verified when IPv6 is enabled, since no v6 rules are managed
+// otherwise.
+func (cfg *IptablesConfigurator) hostRulesConverged(ruleBuilder *builder.IptablesRuleBuilder) (bool, error) {
+	log := log.WithLabels("component", "host")
+	type ipFamily struct {
+		ver      *dep.IptablesVersion
+		expected string
+		checks   [][]string
+	}
+	families := []ipFamily{
+		{&cfg.iptV, ruleBuilder.BuildV4Restore(), ruleBuilder.BuildCheckV4()},
+	}
+	if cfg.cfg.EnableIPv6 {
+		families = append(families, ipFamily{&cfg.ipt6V, ruleBuilder.BuildV6Restore(), ruleBuilder.BuildCheckV6()})
+	}
+	for _, family := range families {
+		output, err := cfg.ext.Run(log, true, iptablesconstants.IPTablesSave, family.ver, nil)
+		if err != nil {
+			return false, fmt.Errorf("%s failed: %w", family.ver.CmdToString(iptablesconstants.IPTablesSave), err)
+		}
+		currentState := ruleBuilder.GetStateFromSave(output.String())
+		expectedState := ruleBuilder.GetStateFromSave(family.expected)
+		for table, chains := range expectedState {
+			for chain, rules := range chains {
+				currentRules, ok := currentState[table][chain]
+				if !ok || (strings.HasPrefix(chain, "ISTIO_") && len(rules) != len(currentRules)) {
+					log.Debugf("host chain %s (table: %s) is missing or has an unexpected number of rules", chain, table)
+					return false, nil
+				}
+			}
+		}
+		for _, cmd := range family.checks {
+			if _, err := cfg.ext.Run(log, true, iptablesconstants.IPTables, family.ver, nil, cmd...); err != nil {
+				log.Debugf("host rule check %v failed", cmd)
+				return false, nil
+			}
+		}
+	}
+	return true, nil
+}
+
 func (cfg *IptablesConfigurator) DeleteHostRules() {
 	log.Debug("Attempting to delete hostside iptables rules (if they exist)")
 	builder := cfg.AppendHostRules()

cni/pkg/iptables/iptables_e2e_linux_test.go

@@ -30,6 +30,7 @@ import (
 	"istio.io/istio/cni/pkg/scopes"
 	"istio.io/istio/pkg/test/util/assert"
 	iptablescapture "istio.io/istio/tools/istio-iptables/pkg/capture"
+	iptablesconstants "istio.io/istio/tools/istio-iptables/pkg/constants"
 	dep "istio.io/istio/tools/istio-iptables/pkg/dependencies"
 )
 
@@ -255,6 +256,82 @@ func TestIptablesHostCleanRoundTrip(t *testing.T) {
 	}
 }
 
+// TestEnsureHostRulesRepairsExternalFlushE2E reproduces istio/istio#60607 against a
+// real kernel: after an external actor flushes the ISTIO_POSTRT chain, EnsureHostRules
+// must detect the drift and repair the host rules back to a converged state.
+func TestEnsureHostRulesRepairsExternalFlushE2E(t *testing.T) {
+	setup(t)
+
+	ext := &dep.RealDependencies{
+		UsePodScopedXtablesLock: false,
+		NetworkNamespace:        "",
+	}
+	iptVer, err := ext.DetectIptablesVersion(false)
+	if err != nil {
+		t.Fatalf("Can't detect iptables version: %v", err)
+	}
+	ipt6Ver, err := ext.DetectIptablesVersion(true)
+	if err != nil {
+		t.Fatalf("Can't detect ip6tables version")
+	}
+
+	cfg := constructTestConfig()
+
+	deps := &dep.RealDependencies{}
+	set, err := createHostsideProbeIpset(true)
+	if err != nil {
+		t.Fatalf("failed to create hostside probe ipset: %v", err)
+	}
+	defer func() {
+		assert.NoError(t, set.DestroySet())
+	}()
+
+	iptConfigurator, _, err := NewIptablesConfigurator(cfg, cfg, deps, deps, RealNlDeps())
+	if err != nil {
+		t.Fatalf("failed to setup iptables configurator: %v", err)
+	}
+	builder := iptConfigurator.AppendHostRules()
+	defer func() {
+		iptConfigurator.DeleteHostRules()
+	}()
+
+	assert.NoError(t, iptConfigurator.CreateHostRulesForHealthChecks())
+
+	// When converged, EnsureHostRules must be a read-only no-op
+	repaired, err := iptConfigurator.EnsureHostRules()
+	assert.NoError(t, err)
+	assert.Equal(t, repaired, false)
+
+	// Simulate an external flush (the issue's reproduction step): empty the
+	// ISTIO_POSTRT chain but keep the chain and the jump
+	if _, err := ext.Run(log, false, iptablesconstants.IPTables, &iptVer, nil,
+		"-t", "nat", "-F", ChainHostPostrouting); err != nil {
+		t.Fatalf("failed to simulate external flush: %v", err)
+	}
+	_, deltaExists := iptablescapture.VerifyIptablesState(log, iptConfigurator.ext, builder, &iptVer, &ipt6Ver)
+	assert.Equal(t, deltaExists, true)
+
+	// EnsureHostRules must detect the drift and repair it
+	repaired, err = iptConfigurator.EnsureHostRules()
+	assert.NoError(t, err)
+	assert.Equal(t, repaired, true)
+
+	residueExists, deltaExists := iptablescapture.VerifyIptablesState(log, iptConfigurator.ext, builder, &iptVer, &ipt6Ver)
+	assert.Equal(t, residueExists, true)
+	assert.Equal(t, deltaExists, false)
+
+	// A more thorough breakage: remove the chain and the jump altogether,
+	// which must also be repaired
+	iptConfigurator.DeleteHostRules()
+	repaired, err = iptConfigurator.EnsureHostRules()
+	assert.NoError(t, err)
+	assert.Equal(t, repaired, true)
+
+	residueExists, deltaExists = iptablescapture.VerifyIptablesState(log, iptConfigurator.ext, builder, &iptVer, &ipt6Ver)
+	assert.Equal(t, residueExists, true)
+	assert.Equal(t, deltaExists, false)
+}
+
 var initialized = &sync.Once{}
 
 func setup(t *testing.T) {