From 020978b0fab70391860d5815e55b89705a556ffc Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 15 Apr 2026 20:00:40 +0500 Subject: [PATCH 1/2] config: add allow_incompatible_key_usage TLS option Let's Encrypt announced they will stop issuing certificates with the TLS Client Authentication Extended Key Usage (EKU) in 2026. Modern Go TLS rejects peer certificates that do not carry the expected EKU, causing failures in components that use the same LE certificate for both client and server roles (e.g. Alertmanager cluster/gossip mTLS). Add AllowIncompatibleKeyUsage to TLSConfig. When true the built-in EKU check is bypassed while the full certificate chain trust, expiry, and (on client connections) hostname verification are preserved. Implementation: setting InsecureSkipVerify=true prevents Go's TLS stack from running its EKU assertion, while a VerifyPeerCertificate callback re-implements the remaining checks with x509.ExtKeyUsageAny so the EKU field is accepted regardless of its value. The new bool serialises as allow_incompatible_key_usage in both YAML and JSON config files. Fixes prometheus/alertmanager#5151 Signed-off-by: Ali --- config/http_config.go | 49 +++++++++++++++++++ config/http_config_test.go | 97 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/config/http_config.go b/config/http_config.go index 55cc5b07..b8772390 100644 --- a/config/http_config.go +++ b/config/http_config.go @@ -1192,6 +1192,45 @@ func NewTLSConfigWithContext(ctx context.Context, cfg *TLSConfig, optFuncs ...TL } } + if cfg.AllowIncompatibleKeyUsage && !cfg.InsecureSkipVerify { + // Go's TLS library always checks Extended Key Usage (EKU) as part of + // its built-in peer certificate verification. To skip only the EKU + // check while preserving chain trust, hostname, and expiry validation, + // we must set InsecureSkipVerify=true (which disables all built-in + // verification) and then re-implement the verification ourselves via + // VerifyPeerCertificate — omitting the KeyUsages constraint. + serverName := tlsConfig.ServerName + tlsConfig.InsecureSkipVerify = true //nolint:gosec // EKU-only bypass; chain+hostname still verified below. + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return nil + } + certs := make([]*x509.Certificate, len(rawCerts)) + for i, raw := range rawCerts { + cert, err := x509.ParseCertificate(raw) + if err != nil { + return fmt.Errorf("tls: failed to parse peer certificate: %w", err) + } + certs[i] = cert + } + opts := x509.VerifyOptions{ + // Access tlsConfig.RootCAs at call time so callers who + // modify RootCAs after NewTLSConfig returns are respected. + Roots: tlsConfig.RootCAs, + Intermediates: x509.NewCertPool(), + DNSName: serverName, + // ExtKeyUsageAny bypasses EKU checking. An empty slice + // defaults to ExtKeyUsageServerAuth per the x509 package docs. + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + for _, cert := range certs[1:] { + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err + } + } + return tlsConfig, nil } @@ -1222,6 +1261,16 @@ type TLSConfig struct { ServerName string `yaml:"server_name,omitempty" json:"server_name,omitempty"` // Disable target certificate validation. InsecureSkipVerify bool `yaml:"insecure_skip_verify" json:"insecure_skip_verify"` + // AllowIncompatibleKeyUsage disables the Extended Key Usage (EKU) check on + // peer certificates while still verifying the certificate chain, expiry, + // and (for client connections) the server hostname. + // + // This is useful when connecting to services that use a certificate without + // the expected EKU — for example Let's Encrypt certificates after they + // dropped TLS Client Authentication support in 2026. The full certificate + // chain trust and hostname verification still apply; only the EKU assertion + // is skipped. + AllowIncompatibleKeyUsage bool `yaml:"allow_incompatible_key_usage,omitempty" json:"allow_incompatible_key_usage,omitempty"` // Minimum TLS version. MinVersion TLSVersion `yaml:"min_version,omitempty" json:"min_version,omitempty"` // Maximum TLS version. diff --git a/config/http_config_test.go b/config/http_config_test.go index 9968d37a..554f03bb 100644 --- a/config/http_config_test.go +++ b/config/http_config_test.go @@ -15,13 +15,18 @@ package config import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "encoding/base64" "encoding/json" "errors" "fmt" "io" + "math/big" "net" "net/http" "net/http/httptest" @@ -2321,3 +2326,95 @@ func TestMultipleHeaders(t *testing.T) { _, err = client.Get(ts.URL) require.NoErrorf(t, err, "can't fetch URL: %v", err) } + +// TestTLSConfigAllowIncompatibleKeyUsage verifies that when +// AllowIncompatibleKeyUsage is set, a TLS connection to a server whose +// certificate lacks the expected Extended Key Usage (e.g. a Let's Encrypt +// certificate used for mutual TLS after LE dropped clientAuth EKU support) +// succeeds where it would otherwise fail. +func TestTLSConfigAllowIncompatibleKeyUsage(t *testing.T) { + // Generate a self-signed CA + server cert that has ONLY serverAuth EKU + // (no clientAuth). This simulates a Let's Encrypt-style cert. + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + require.NoError(t, err) + caCert, err := x509.ParseCertificate(caDER) + require.NoError(t, err) + + srvKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + srvTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + // clientAuth only — no serverAuth. This triggers the EKU mismatch a + // TLS client sees when the peer cert lacks ExtKeyUsageServerAuth, which + // is the scenario for LE certs used in mutual-TLS gossip rings. + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + srvDER, err := x509.CreateCertificate(rand.Reader, srvTemplate, caCert, &srvKey.PublicKey, caKey) + require.NoError(t, err) + + srvTLSCert := tls.Certificate{ + Certificate: [][]byte{srvDER}, + PrivateKey: srvKey, + } + caPool := x509.NewCertPool() + caPool.AddCert(caCert) + + // Start a TLS test server using the serverAuth-only certificate. + serverTLSCfg := &tls.Config{Certificates: []tls.Certificate{srvTLSCert}} + listener, err := tls.Listen("tcp", "127.0.0.1:0", serverTLSCfg) + require.NoError(t, err) + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })} + go srv.Serve(listener) //nolint:errcheck + defer srv.Close() + addr := "https://" + listener.Addr().String() + + t.Run("without AllowIncompatibleKeyUsage fails on EKU mismatch", func(t *testing.T) { + cfg := HTTPClientConfig{ + TLSConfig: TLSConfig{ + // CA is trusted but EKU check will reject the server cert. + InsecureSkipVerify: false, + }, + } + // Use the generated CA so hostname+chain pass; only EKU should fail. + tlsCfg, err := NewTLSConfig(&cfg.TLSConfig) + require.NoError(t, err) + tlsCfg.RootCAs = caPool + client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} + _, err = client.Get(addr) + require.Error(t, err, "expected EKU error without AllowIncompatibleKeyUsage") + require.Contains(t, err.Error(), "incompatible key usage") + }) + + t.Run("with AllowIncompatibleKeyUsage succeeds", func(t *testing.T) { + cfg := TLSConfig{ + AllowIncompatibleKeyUsage: true, + // ServerName override for IP-addressed connection. + } + tlsCfg, err := NewTLSConfig(&cfg) + require.NoError(t, err) + require.True(t, tlsCfg.InsecureSkipVerify, "InsecureSkipVerify should be true internally") + require.NotNil(t, tlsCfg.VerifyPeerCertificate, "VerifyPeerCertificate should be set") + tlsCfg.RootCAs = caPool + client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} + resp, err := client.Get(addr) + require.NoError(t, err, "connection should succeed when AllowIncompatibleKeyUsage=true") + resp.Body.Close() + }) +} From b29a90ca7e6398e4fcb2eb2f52fecb7ac83e657b Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 15 Apr 2026 20:09:01 +0500 Subject: [PATCH 2/2] =?UTF-8?q?config:=20fix=20testifylint=20=E2=80=94=20u?= =?UTF-8?q?se=20f-variants=20with=20message=20args?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ali --- config/http_config_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/http_config_test.go b/config/http_config_test.go index 554f03bb..8d132bb9 100644 --- a/config/http_config_test.go +++ b/config/http_config_test.go @@ -2398,23 +2398,22 @@ func TestTLSConfigAllowIncompatibleKeyUsage(t *testing.T) { tlsCfg.RootCAs = caPool client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} _, err = client.Get(addr) - require.Error(t, err, "expected EKU error without AllowIncompatibleKeyUsage") + require.Errorf(t, err, "expected EKU error without AllowIncompatibleKeyUsage") require.Contains(t, err.Error(), "incompatible key usage") }) t.Run("with AllowIncompatibleKeyUsage succeeds", func(t *testing.T) { cfg := TLSConfig{ AllowIncompatibleKeyUsage: true, - // ServerName override for IP-addressed connection. } tlsCfg, err := NewTLSConfig(&cfg) require.NoError(t, err) - require.True(t, tlsCfg.InsecureSkipVerify, "InsecureSkipVerify should be true internally") - require.NotNil(t, tlsCfg.VerifyPeerCertificate, "VerifyPeerCertificate should be set") + require.Truef(t, tlsCfg.InsecureSkipVerify, "InsecureSkipVerify should be true internally") + require.NotNilf(t, tlsCfg.VerifyPeerCertificate, "VerifyPeerCertificate should be set") tlsCfg.RootCAs = caPool client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} resp, err := client.Get(addr) - require.NoError(t, err, "connection should succeed when AllowIncompatibleKeyUsage=true") + require.NoErrorf(t, err, "connection should succeed when AllowIncompatibleKeyUsage=true") resp.Body.Close() }) }