[9.5] (backport #15427) Override fleet.ssl.certificate_authorities from env vars in container mode

changelog/fragments/1783541510-container-fleet-ca-override.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Override fleet.ssl.certificate_authorities from env vars in container mode
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

internal/pkg/agent/cmd/container.go

@@ -904,6 +904,20 @@ func containerCfgOverrides(cfg *config.Config) error {
 	uCfg["fleet.ssl.certificate"] = envWithDefault("", "ELASTIC_AGENT_CERT")
 	uCfg["fleet.ssl.key"] = envWithDefault("", "ELASTIC_AGENT_CERT_KEY")
 
+	for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} {
+		// Override with first explicitly defined env var, even if value is empty
+		// ucfg Merge won't allow an empty array to overwrite, so we have to
+		// edit cfg instead of uCfg here
+		if caEnv, ok := os.LookupEnv(key); ok {
+			if caChild, err := ucfg.NewFrom(cli.StringToSlice(caEnv)); err != nil {
+				return fmt.Errorf("failed to create CA config for container override: %w", err)
+			} else if err := cfg.Agent.SetChild("fleet.ssl.certificate_authorities", -1, caChild, ucfg.PathSep(".")); err != nil {
+				return fmt.Errorf("failed to write CA config for container override: %w", err)
+			}
+			break
+		}
+	}
+
 	return cfg.Merge(uCfg)
 }
 
@@ -1185,6 +1199,18 @@ func shouldFleetEnroll(setupCfg setupConfig) (bool, error) {
 		return false, fmt.Errorf("failed to override cert key: %w", err)
 	}
 
+	for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} {
+		if caEnv, ok := os.LookupEnv(key); ok {
+			// Override with first explicitly defined env var, even if value is empty
+			if caChild, err := ucfg.NewFrom(cli.StringToSlice(caEnv)); err != nil {
+				return false, fmt.Errorf("failed to create CA config for container override: %w", err)
+			} else if err := cfg.Agent.SetChild("fleet.ssl.certificate_authorities", -1, caChild, ucfg.PathSep(".")); err != nil {
+				return false, fmt.Errorf("failed to write CA config for container override: %w", err)
+			}
+			break
+		}
+	}
+
 	storedConfig, err := configuration.NewFromConfig(cfg)
 	if err != nil {
 		return false, fmt.Errorf("failed to read from disk store: %w", err)

internal/pkg/agent/cmd/container_test.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
+	"fmt"
 	"io"
 	"io/fs"
 	"net/http"
@@ -1358,3 +1359,119 @@ func TestContainerEnvOverridesFleetTLSPaths(t *testing.T) {
 	require.Equal(t, envCertPath, loaded.Fleet.Client.Transport.TLS.Certificate.Certificate)
 	require.Equal(t, envKeyPath, loaded.Fleet.Client.Transport.TLS.Certificate.Key)
 }
+
+// Regression test for #15408: FLEET_CA/KIBANA_CA/ELASTICSEARCH_CA overrides fleet.enc CA
+func TestContainerEnvOverridesFleetCA(t *testing.T) {
+	rootPair, _, err := certutil.NewRSARootAndChildCerts()
+	require.NoError(t, err)
+	certDir := t.TempDir()
+	envFleetCAPath := filepath.Join(certDir, "fleet-ca.crt")
+	require.NoError(t, os.WriteFile(envFleetCAPath, rootPair.Cert, 0o644))
+	storedCAPath := filepath.Join(certDir, "stored-ca.crt")
+	require.NoError(t, os.WriteFile(storedCAPath, rootPair.Cert, 0o644))
+	kibanaCAPath := filepath.Join(certDir, "kibana-ca.crt")
+	require.NoError(t, os.WriteFile(kibanaCAPath, rootPair.Cert, 0o644))
+	esCAPath := filepath.Join(certDir, "es-ca.crt")
+	require.NoError(t, os.WriteFile(esCAPath, rootPair.Cert, 0o644))
+
+	origCfg := paths.Config()
+	t.Cleanup(func() { paths.SetConfig(origCfg) })
+	paths.SetConfig(t.TempDir())
+	for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} {
+		if orig, ok := os.LookupEnv(key); ok {
+			t.Cleanup(func() { os.Setenv(key, orig) })
+		} else {
+			t.Cleanup(func() { os.Unsetenv(key) })
+		}
+		os.Unsetenv(key)
+	}
+
+	ctx := t.Context()
+	require.NoError(t, secret.CreateAgentSecret(ctx, vault.WithUnprivileged(true)))
+	require.NoError(t, os.WriteFile(paths.ConfigFile(), []byte("fleet:\n  enabled: true\n"), 0o644))
+	store, err := storage.NewEncryptedDiskStore(ctx, paths.AgentConfigFile())
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		env        map[string]string
+		fleetEncCA string
+		wantCAs    []string
+		wantErr    bool
+	}{
+		{
+			name:       "valid fleet.enc CAs with no overrides",
+			fleetEncCA: storedCAPath,
+			wantCAs:    []string{storedCAPath},
+		},
+		{
+			name:       "invalid fleet.enc CAs with no overrides",
+			fleetEncCA: "/nonexistent/ca.crt",
+			wantErr:    true,
+		},
+		{
+			name:       "set FLEET_CA should override valid fleet.enc CA",
+			fleetEncCA: storedCAPath,
+			env:        map[string]string{"FLEET_CA": envFleetCAPath},
+			wantCAs:    []string{envFleetCAPath},
+		},
+		{
+			name:       "set FLEET_CA should override invalid fleet.enc CA",
+			fleetEncCA: "/nonexistent/ca.crt",
+			env:        map[string]string{"FLEET_CA": envFleetCAPath},
+			wantCAs:    []string{envFleetCAPath},
+		},
+		{
+			name:       "explicitly empty FLEET_CA should override valid fleet.enc",
+			fleetEncCA: storedCAPath,
+			env:        map[string]string{"FLEET_CA": ""},
+			wantCAs:    []string{},
+		},
+		{
+			name:       "explicitly empty FLEET_CA should override invalid fleet.enc",
+			fleetEncCA: "/nonexistent/ca.crt",
+			env:        map[string]string{"FLEET_CA": ""},
+			wantCAs:    []string{},
+		},
+		{
+			name:       "KIBANA_CA should override fleet.enc CAs when FLEET_CA is unset",
+			fleetEncCA: "/nonexistent/ca.crt",
+			env:        map[string]string{"KIBANA_CA": kibanaCAPath},
+			wantCAs:    []string{kibanaCAPath},
+		},
+		{
+			name:       "ELASTICSEARCH_CA should override fleet.enc CAs when FLEET_CA and KIBANA_CA are unset",
+			fleetEncCA: "/nonexistent/ca.crt",
+			env:        map[string]string{"ELASTICSEARCH_CA": esCAPath},
+			wantCAs:    []string{esCAPath},
+		},
+		{
+			name:       "explicitly empty FLEET_CA should take precedence over set KIBANA_CA and ELASTICSEARCH_CA",
+			fleetEncCA: "/nonexistent/ca.crt",
+			env:        map[string]string{"FLEET_CA": "", "KIBANA_CA": kibanaCAPath, "ELASTICSEARCH_CA": esCAPath},
+			wantCAs:    []string{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.NoError(t, store.Save(strings.NewReader(fmt.Sprintf(`fleet:
+  ssl:
+    certificate_authorities:
+      - %q`, tc.fleetEncCA))))
+			for k, v := range tc.env {
+				t.Setenv(k, v)
+			}
+
+			_, err := shouldFleetEnroll(setupConfig{Fleet: fleetConfig{Enroll: true}})
+			if tc.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				loaded, err := configuration.LoadConfig(ctx, containerCfgOverrides)
+				require.NoError(t, err)
+				require.Equal(t, tc.wantCAs, loaded.Fleet.Client.Transport.TLS.CAs)
+			}
+		})
+	}
+}

testing/integration/ess/container_cmd_test.go

@@ -745,7 +745,7 @@ func addLogIntegration(t *testing.T, info *define.Info, policyID, logFilePath st
 
 	// Call Kibana to create the policy.
 	// Docs: https://www.elastic.co/guide/en/fleet/current/fleet-api-docs.html#create-integration-policy-api
-	resp, err := info.KibanaClient.Connection.Send(
+	resp, err := info.KibanaClient.Send(
 		http.MethodPost,
 		"/api/fleet/package_policies",
 		nil,
@@ -754,6 +754,7 @@ func addLogIntegration(t *testing.T, info *define.Info, policyID, logFilePath st
 	if err != nil {
 		t.Fatalf("could not execute request to Kibana/Fleet: %s", err)
 	}
+	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		// On error dump the whole request response so we can easily spot
 		// what went wrong.
@@ -894,10 +895,10 @@ func TestContainerCMDEnrollByPolicyName(t *testing.T) {
 	env := []string{
 		"FLEET_ENROLL=1",
 		"FLEET_URL=" + fleetURL,
-		"KIBANA_FLEET_HOST=" + info.KibanaClient.Connection.URL,
+		"KIBANA_FLEET_HOST=" + info.KibanaClient.URL,
 		"FLEET_TOKEN_POLICY_NAME=" + resp.Name,
-		"KIBANA_FLEET_USERNAME=" + info.KibanaClient.Connection.Username,
-		"KIBANA_FLEET_PASSWORD=" + info.KibanaClient.Connection.Password,
+		"KIBANA_FLEET_USERNAME=" + info.KibanaClient.Username,
+		"KIBANA_FLEET_PASSWORD=" + info.KibanaClient.Password,
 		"STATE_PATH=" + agentFixture.WorkDir(),
 	}
 	cmd, agentOutput := prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, env)
@@ -1082,3 +1083,123 @@ func TestContainerCMDTLSCertOverride(t *testing.T) {
 		err, agentOutput,
 	)
 }
+
+// Regression test for #15408: FLEET_CA/KIBANA_CA/ELASTICSEARCH_CA overrides fleet.enc CA
+func TestContainerCMDFleetCAOverride(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Stack: &define.Stack{},
+		Local: false,
+		Sudo:  true,
+		OS: []define.OS{
+			{Type: define.Linux},
+		},
+		Group: "container",
+	})
+
+	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+	defer cancel()
+
+	agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
+	require.NoError(t, err)
+
+	require.NoError(t, agentFixture.Prepare(ctx))
+	require.NoError(t, err)
+
+	certDir := t.TempDir()
+
+	caPath1 := filepath.Join(certDir, "fleet-ca-1.crt")
+	rootPair1, _, err := certutil.NewRSARootAndChildCerts()
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(caPath1, rootPair1.Cert, 0o644))
+
+	caPath2 := filepath.Join(certDir, "fleet-ca-2.crt")
+	rootPair2, _, err := certutil.NewRSARootAndChildCerts()
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(caPath2, rootPair2.Cert, 0o644))
+
+	fleetURL, err := fleettools.DefaultURL(ctx, info.KibanaClient)
+	require.NoErrorf(t, err, "could not get Fleet URL")
+
+	_, enrollmentToken := createPolicy(
+		t, ctx, agentFixture, info,
+		fmt.Sprintf("%s-%s", t.Name(), uuid.Must(uuid.NewV4()).String()),
+		"")
+
+	statePath := agentFixture.WorkDir()
+	env := []string{
+		"FLEET_ENROLL=1",
+		"FLEET_URL=" + fleetURL,
+		"FLEET_ENROLLMENT_TOKEN=" + enrollmentToken,
+		"FLEET_INSECURE=1",
+		"STATE_PATH=" + statePath,
+	}
+
+	cmd, agentOutput := prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA=" + caPath1}, env...))
+	t.Logf(">> running binary with: %v", cmd.Args)
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("error running container cmd: %s", err)
+	}
+
+	require.Eventuallyf(t, func() bool {
+		err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath})))
+		return err == nil
+	},
+		2*time.Minute, time.Second,
+		"Elastic-Agent did not report healthy after first enrollment. Agent status error: \"%v\", Agent logs\n%s",
+		err, agentOutput,
+	)
+
+	require.FileExists(t, filepath.Join(statePath, "fleet.enc"),
+		"fleet.enc must exist for regression test")
+
+	_ = cmd.Process.Kill()
+	_ = cmd.Wait()
+	cmd.Process = nil
+
+	require.NoError(t, os.Remove(caPath1))
+
+	// Agent should crash on the stale fleet.enc caPath1 when no CA env var is set to override it
+	cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, env)
+	t.Logf(">> running binary with: %v", cmd.Args)
+	require.Error(t, cmd.Run(), "agent should exit with an error when the stored CA path is stale and no CA env var is set")
+	require.Contains(t, agentOutput.String(), "Error: failed to read from disk store",
+		"agent should fail to load the stale stored CA path when no CA env var is set")
+	cmd.Process = nil
+
+	// Agent should start healthy with FLEET_CA's caPath1 overriding fleet.enc's now stale caPath2
+	cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA=" + caPath2}, env...))
+	t.Logf(">> running binary with: %v", cmd.Args)
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("error running container cmd: %s", err)
+	}
+
+	require.Eventuallyf(t, func() bool {
+		err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath})))
+		return err == nil
+	},
+		2*time.Minute, time.Second,
+		"Elastic-Agent did not report healthy after restart with FLEET_CA overriding the stale stored path. Agent status error: \"%v\", Agent logs\n%s",
+		err, agentOutput,
+	)
+
+	_ = cmd.Process.Kill()
+	_ = cmd.Wait()
+	cmd.Process = nil
+
+	// Agent should start healthy with FLEET_CA's empty var overriding fleet.enc's now stale caPath2
+	require.NoError(t, os.Remove(caPath2))
+	cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA="}, env...))
+	t.Logf(">> running binary with: %v", cmd.Args)
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("error running container cmd: %s", err)
+	}
+
+	require.Eventuallyf(t, func() bool {
+		err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath})))
+		return err == nil
+	},
+		2*time.Minute, time.Second,
+		"Elastic-Agent did not report healthy after restart with an explicitly empty FLEET_CA overriding the stale stored path. Agent status error: \"%v\", Agent logs\n%s",
+		err, agentOutput,
+	)
+}