diff --git a/changelog/fragments/1783541510-container-fleet-ca-override.yaml b/changelog/fragments/1783541510-container-fleet-ca-override.yaml new file mode 100644 index 00000000000..9a39b2b79d7 --- /dev/null +++ b/changelog/fragments/1783541510-container-fleet-ca-override.yaml @@ -0,0 +1,45 @@ +# REQUIRED +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# REQUIRED for all kinds +# Change summary; a 80ish characters long description of the change. +summary: Override fleet.ssl.certificate_authorities from env vars in container mode + +# REQUIRED for breaking-change, deprecation, known-issue +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# description: + +# REQUIRED for breaking-change, deprecation, known-issue +# impact: + +# REQUIRED for breaking-change, deprecation, known-issue +# action: + +# REQUIRED for all kinds +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# AUTOMATED +# OPTIONAL to manually add other PR URLs +# PR URL: A link the PR that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +# pr: https://github.com/owner/repo/1234 + +# AUTOMATED +# OPTIONAL to manually add other issue URLs +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +# issue: https://github.com/owner/repo/1234 diff --git a/internal/pkg/agent/cmd/container.go b/internal/pkg/agent/cmd/container.go index 90a9df7ee17..baaa9c7ebad 100644 --- a/internal/pkg/agent/cmd/container.go +++ b/internal/pkg/agent/cmd/container.go @@ -904,6 +904,20 @@ func containerCfgOverrides(cfg *config.Config) error { uCfg["fleet.ssl.certificate"] = envWithDefault("", "ELASTIC_AGENT_CERT") uCfg["fleet.ssl.key"] = envWithDefault("", "ELASTIC_AGENT_CERT_KEY") + for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} { + // Override with first explicitly defined env var, even if value is empty + // ucfg Merge won't allow an empty array to overwrite, so we have to + // edit cfg instead of uCfg here + if caEnv, ok := os.LookupEnv(key); ok { + if caChild, err := ucfg.NewFrom(cli.StringToSlice(caEnv)); err != nil { + return fmt.Errorf("failed to create CA config for container override: %w", err) + } else if err := cfg.Agent.SetChild("fleet.ssl.certificate_authorities", -1, caChild, ucfg.PathSep(".")); err != nil { + return fmt.Errorf("failed to write CA config for container override: %w", err) + } + break + } + } + return cfg.Merge(uCfg) } @@ -1185,6 +1199,18 @@ func shouldFleetEnroll(setupCfg setupConfig) (bool, error) { return false, fmt.Errorf("failed to override cert key: %w", err) } + for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} { + if caEnv, ok := os.LookupEnv(key); ok { + // Override with first explicitly defined env var, even if value is empty + if caChild, err := ucfg.NewFrom(cli.StringToSlice(caEnv)); err != nil { + return false, fmt.Errorf("failed to create CA config for container override: %w", err) + } else if err := cfg.Agent.SetChild("fleet.ssl.certificate_authorities", -1, caChild, ucfg.PathSep(".")); err != nil { + return false, fmt.Errorf("failed to write CA config for container override: %w", err) + } + break + } + } + storedConfig, err := configuration.NewFromConfig(cfg) if err != nil { return false, fmt.Errorf("failed to read from disk store: %w", err) diff --git a/internal/pkg/agent/cmd/container_test.go b/internal/pkg/agent/cmd/container_test.go index d4c6ec5ea4b..e922376753d 100644 --- a/internal/pkg/agent/cmd/container_test.go +++ b/internal/pkg/agent/cmd/container_test.go @@ -8,6 +8,7 @@ import ( "context" "encoding/base64" "errors" + "fmt" "io" "io/fs" "net/http" @@ -1358,3 +1359,119 @@ func TestContainerEnvOverridesFleetTLSPaths(t *testing.T) { require.Equal(t, envCertPath, loaded.Fleet.Client.Transport.TLS.Certificate.Certificate) require.Equal(t, envKeyPath, loaded.Fleet.Client.Transport.TLS.Certificate.Key) } + +// Regression test for #15408: FLEET_CA/KIBANA_CA/ELASTICSEARCH_CA overrides fleet.enc CA +func TestContainerEnvOverridesFleetCA(t *testing.T) { + rootPair, _, err := certutil.NewRSARootAndChildCerts() + require.NoError(t, err) + certDir := t.TempDir() + envFleetCAPath := filepath.Join(certDir, "fleet-ca.crt") + require.NoError(t, os.WriteFile(envFleetCAPath, rootPair.Cert, 0o644)) + storedCAPath := filepath.Join(certDir, "stored-ca.crt") + require.NoError(t, os.WriteFile(storedCAPath, rootPair.Cert, 0o644)) + kibanaCAPath := filepath.Join(certDir, "kibana-ca.crt") + require.NoError(t, os.WriteFile(kibanaCAPath, rootPair.Cert, 0o644)) + esCAPath := filepath.Join(certDir, "es-ca.crt") + require.NoError(t, os.WriteFile(esCAPath, rootPair.Cert, 0o644)) + + origCfg := paths.Config() + t.Cleanup(func() { paths.SetConfig(origCfg) }) + paths.SetConfig(t.TempDir()) + for _, key := range []string{"FLEET_CA", "KIBANA_CA", "ELASTICSEARCH_CA"} { + if orig, ok := os.LookupEnv(key); ok { + t.Cleanup(func() { os.Setenv(key, orig) }) + } else { + t.Cleanup(func() { os.Unsetenv(key) }) + } + os.Unsetenv(key) + } + + ctx := t.Context() + require.NoError(t, secret.CreateAgentSecret(ctx, vault.WithUnprivileged(true))) + require.NoError(t, os.WriteFile(paths.ConfigFile(), []byte("fleet:\n enabled: true\n"), 0o644)) + store, err := storage.NewEncryptedDiskStore(ctx, paths.AgentConfigFile()) + require.NoError(t, err) + + cases := []struct { + name string + env map[string]string + fleetEncCA string + wantCAs []string + wantErr bool + }{ + { + name: "valid fleet.enc CAs with no overrides", + fleetEncCA: storedCAPath, + wantCAs: []string{storedCAPath}, + }, + { + name: "invalid fleet.enc CAs with no overrides", + fleetEncCA: "/nonexistent/ca.crt", + wantErr: true, + }, + { + name: "set FLEET_CA should override valid fleet.enc CA", + fleetEncCA: storedCAPath, + env: map[string]string{"FLEET_CA": envFleetCAPath}, + wantCAs: []string{envFleetCAPath}, + }, + { + name: "set FLEET_CA should override invalid fleet.enc CA", + fleetEncCA: "/nonexistent/ca.crt", + env: map[string]string{"FLEET_CA": envFleetCAPath}, + wantCAs: []string{envFleetCAPath}, + }, + { + name: "explicitly empty FLEET_CA should override valid fleet.enc", + fleetEncCA: storedCAPath, + env: map[string]string{"FLEET_CA": ""}, + wantCAs: []string{}, + }, + { + name: "explicitly empty FLEET_CA should override invalid fleet.enc", + fleetEncCA: "/nonexistent/ca.crt", + env: map[string]string{"FLEET_CA": ""}, + wantCAs: []string{}, + }, + { + name: "KIBANA_CA should override fleet.enc CAs when FLEET_CA is unset", + fleetEncCA: "/nonexistent/ca.crt", + env: map[string]string{"KIBANA_CA": kibanaCAPath}, + wantCAs: []string{kibanaCAPath}, + }, + { + name: "ELASTICSEARCH_CA should override fleet.enc CAs when FLEET_CA and KIBANA_CA are unset", + fleetEncCA: "/nonexistent/ca.crt", + env: map[string]string{"ELASTICSEARCH_CA": esCAPath}, + wantCAs: []string{esCAPath}, + }, + { + name: "explicitly empty FLEET_CA should take precedence over set KIBANA_CA and ELASTICSEARCH_CA", + fleetEncCA: "/nonexistent/ca.crt", + env: map[string]string{"FLEET_CA": "", "KIBANA_CA": kibanaCAPath, "ELASTICSEARCH_CA": esCAPath}, + wantCAs: []string{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, store.Save(strings.NewReader(fmt.Sprintf(`fleet: + ssl: + certificate_authorities: + - %q`, tc.fleetEncCA)))) + for k, v := range tc.env { + t.Setenv(k, v) + } + + _, err := shouldFleetEnroll(setupConfig{Fleet: fleetConfig{Enroll: true}}) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + loaded, err := configuration.LoadConfig(ctx, containerCfgOverrides) + require.NoError(t, err) + require.Equal(t, tc.wantCAs, loaded.Fleet.Client.Transport.TLS.CAs) + } + }) + } +} diff --git a/testing/integration/ess/container_cmd_test.go b/testing/integration/ess/container_cmd_test.go index ed1b692f414..de5f6007196 100644 --- a/testing/integration/ess/container_cmd_test.go +++ b/testing/integration/ess/container_cmd_test.go @@ -745,7 +745,7 @@ func addLogIntegration(t *testing.T, info *define.Info, policyID, logFilePath st // Call Kibana to create the policy. // Docs: https://www.elastic.co/guide/en/fleet/current/fleet-api-docs.html#create-integration-policy-api - resp, err := info.KibanaClient.Connection.Send( + resp, err := info.KibanaClient.Send( http.MethodPost, "/api/fleet/package_policies", nil, @@ -754,6 +754,7 @@ func addLogIntegration(t *testing.T, info *define.Info, policyID, logFilePath st if err != nil { t.Fatalf("could not execute request to Kibana/Fleet: %s", err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { // On error dump the whole request response so we can easily spot // what went wrong. @@ -894,10 +895,10 @@ func TestContainerCMDEnrollByPolicyName(t *testing.T) { env := []string{ "FLEET_ENROLL=1", "FLEET_URL=" + fleetURL, - "KIBANA_FLEET_HOST=" + info.KibanaClient.Connection.URL, + "KIBANA_FLEET_HOST=" + info.KibanaClient.URL, "FLEET_TOKEN_POLICY_NAME=" + resp.Name, - "KIBANA_FLEET_USERNAME=" + info.KibanaClient.Connection.Username, - "KIBANA_FLEET_PASSWORD=" + info.KibanaClient.Connection.Password, + "KIBANA_FLEET_USERNAME=" + info.KibanaClient.Username, + "KIBANA_FLEET_PASSWORD=" + info.KibanaClient.Password, "STATE_PATH=" + agentFixture.WorkDir(), } cmd, agentOutput := prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, env) @@ -1082,3 +1083,123 @@ func TestContainerCMDTLSCertOverride(t *testing.T) { err, agentOutput, ) } + +// Regression test for #15408: FLEET_CA/KIBANA_CA/ELASTICSEARCH_CA overrides fleet.enc CA +func TestContainerCMDFleetCAOverride(t *testing.T) { + info := define.Require(t, define.Requirements{ + Stack: &define.Stack{}, + Local: false, + Sudo: true, + OS: []define.OS{ + {Type: define.Linux}, + }, + Group: "container", + }) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + + agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) + require.NoError(t, err) + + require.NoError(t, agentFixture.Prepare(ctx)) + require.NoError(t, err) + + certDir := t.TempDir() + + caPath1 := filepath.Join(certDir, "fleet-ca-1.crt") + rootPair1, _, err := certutil.NewRSARootAndChildCerts() + require.NoError(t, err) + require.NoError(t, os.WriteFile(caPath1, rootPair1.Cert, 0o644)) + + caPath2 := filepath.Join(certDir, "fleet-ca-2.crt") + rootPair2, _, err := certutil.NewRSARootAndChildCerts() + require.NoError(t, err) + require.NoError(t, os.WriteFile(caPath2, rootPair2.Cert, 0o644)) + + fleetURL, err := fleettools.DefaultURL(ctx, info.KibanaClient) + require.NoErrorf(t, err, "could not get Fleet URL") + + _, enrollmentToken := createPolicy( + t, ctx, agentFixture, info, + fmt.Sprintf("%s-%s", t.Name(), uuid.Must(uuid.NewV4()).String()), + "") + + statePath := agentFixture.WorkDir() + env := []string{ + "FLEET_ENROLL=1", + "FLEET_URL=" + fleetURL, + "FLEET_ENROLLMENT_TOKEN=" + enrollmentToken, + "FLEET_INSECURE=1", + "STATE_PATH=" + statePath, + } + + cmd, agentOutput := prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA=" + caPath1}, env...)) + t.Logf(">> running binary with: %v", cmd.Args) + if err := cmd.Start(); err != nil { + t.Fatalf("error running container cmd: %s", err) + } + + require.Eventuallyf(t, func() bool { + err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath}))) + return err == nil + }, + 2*time.Minute, time.Second, + "Elastic-Agent did not report healthy after first enrollment. Agent status error: \"%v\", Agent logs\n%s", + err, agentOutput, + ) + + require.FileExists(t, filepath.Join(statePath, "fleet.enc"), + "fleet.enc must exist for regression test") + + _ = cmd.Process.Kill() + _ = cmd.Wait() + cmd.Process = nil + + require.NoError(t, os.Remove(caPath1)) + + // Agent should crash on the stale fleet.enc caPath1 when no CA env var is set to override it + cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, env) + t.Logf(">> running binary with: %v", cmd.Args) + require.Error(t, cmd.Run(), "agent should exit with an error when the stored CA path is stale and no CA env var is set") + require.Contains(t, agentOutput.String(), "Error: failed to read from disk store", + "agent should fail to load the stale stored CA path when no CA env var is set") + cmd.Process = nil + + // Agent should start healthy with FLEET_CA's caPath1 overriding fleet.enc's now stale caPath2 + cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA=" + caPath2}, env...)) + t.Logf(">> running binary with: %v", cmd.Args) + if err := cmd.Start(); err != nil { + t.Fatalf("error running container cmd: %s", err) + } + + require.Eventuallyf(t, func() bool { + err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath}))) + return err == nil + }, + 2*time.Minute, time.Second, + "Elastic-Agent did not report healthy after restart with FLEET_CA overriding the stale stored path. Agent status error: \"%v\", Agent logs\n%s", + err, agentOutput, + ) + + _ = cmd.Process.Kill() + _ = cmd.Wait() + cmd.Process = nil + + // Agent should start healthy with FLEET_CA's empty var overriding fleet.enc's now stale caPath2 + require.NoError(t, os.Remove(caPath2)) + cmd, agentOutput = prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, append([]string{"FLEET_CA="}, env...)) + t.Logf(">> running binary with: %v", cmd.Args) + if err := cmd.Start(); err != nil { + t.Fatalf("error running container cmd: %s", err) + } + + require.Eventuallyf(t, func() bool { + err = agentFixture.IsHealthy(ctx, atesting.WithCmdOptions(withEnv([]string{"STATE_PATH=" + statePath}))) + return err == nil + }, + 2*time.Minute, time.Second, + "Elastic-Agent did not report healthy after restart with an explicitly empty FLEET_CA overriding the stale stored path. Agent status error: \"%v\", Agent logs\n%s", + err, agentOutput, + ) +}