[Agentless] Protect API keys on agentless HTTP transports

tracer/src/Datadog.Trace/Agent/Transports/ApiKeyHttpTransportException.cs

@@ -0,0 +1,23 @@
+// <copyright file="ApiKeyHttpTransportException.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+
+namespace Datadog.Trace.Agent.Transports;
+
+internal sealed class ApiKeyHttpTransportException : InvalidOperationException
+{
+    public ApiKeyHttpTransportException(string message)
+        : base(message)
+    {
+    }
+
+    public ApiKeyHttpTransportException(string message, Exception innerException)
+        : base(message, innerException)
+    {
+    }
+}

tracer/src/Datadog.Trace/Agent/Transports/ApiKeyHttpTransportGuard.cs

@@ -0,0 +1,51 @@
+// <copyright file="ApiKeyHttpTransportGuard.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+
+namespace Datadog.Trace.Agent.Transports;
+
+internal static class ApiKeyHttpTransportGuard
+{
+    internal const string ApiKeyHeaderName = "DD-API-KEY";
+
+    public static bool IsPlaintextLoopback(Uri endpoint)
+        => string.Equals(endpoint.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && endpoint.IsLoopback;
+
+    public static void RejectLateApiKeyHeader(string headerName)
+    {
+        if (string.Equals(headerName, ApiKeyHeaderName, StringComparison.OrdinalIgnoreCase))
+        {
+            throw new ApiKeyHttpTransportException("DD-API-KEY must be configured when constructing the request factory.");
+        }
+    }
+
+    public static void EnsureSafeEndpoint(Uri endpoint)
+    {
+        if (string.Equals(endpoint.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase) ||
+            IsPlaintextLoopback(endpoint))
+        {
+            return;
+        }
+
+        throw new ApiKeyHttpTransportException(
+            "Refusing to send DD-API-KEY unless the endpoint uses HTTPS or loopback HTTP.");
+    }
+
+    public static void EnsureSafe(Uri endpoint, bool isProxyDisabled, bool redirectsDisabled)
+    {
+        EnsureSafeEndpoint(endpoint);
+
+        if (redirectsDisabled && (!IsPlaintextLoopback(endpoint) || isProxyDisabled))
+        {
+            return;
+        }
+
+        throw new ApiKeyHttpTransportException(
+            "Refusing to send DD-API-KEY unless automatic redirects are disabled and the endpoint uses HTTPS or direct loopback HTTP.");
+    }
+}

tracer/src/Datadog.Trace/Agent/Transports/ApiWebRequest.cs

@@ -24,17 +24,24 @@ internal sealed class ApiWebRequest : IApiRequest
 
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<ApiWebRequest>();
         private readonly HttpWebRequest _request;
+        private readonly bool _hasApiKeyHeader;
 
         private byte[] _boundarySeparatorInBytes;
         private byte[] _boundaryTrailerInBytes;
 
-        public ApiWebRequest(HttpWebRequest request)
+        public ApiWebRequest(HttpWebRequest request, bool hasApiKeyHeader)
         {
             _request = request;
+            _hasApiKeyHeader = hasApiKeyHeader;
+            if (hasApiKeyHeader)
+            {
+                ConfigureApiKeyTransport();
+            }
         }
 
         public void AddHeader(string name, string value)
         {
+            ApiKeyHttpTransportGuard.RejectLateApiKeyHeader(name);
             _request.Headers.Add(name, value);
         }
 
@@ -194,6 +201,51 @@ private void ResetRequest(string method, string contentType, string contentEncod
             {
                 _request.Headers.Set(HttpRequestHeader.ContentEncoding, contentEncoding);
             }
+
+            ValidateApiKeyTransport();
+        }
+
+        private void ConfigureApiKeyTransport()
+        {
+            try
+            {
+                _request.AllowAutoRedirect = false;
+                var endpoint = _request.RequestUri;
+                if (ApiKeyHttpTransportGuard.IsPlaintextLoopback(endpoint))
+                {
+                    // Proxy bypass checks are mutable and racy. Enforce a direct connection instead.
+                    _request.Proxy = null;
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new ApiKeyHttpTransportException("Unable to configure a safe HTTP transport for DD-API-KEY.", ex);
+            }
+        }
+
+        private void ValidateApiKeyTransport()
+        {
+            if (!_hasApiKeyHeader)
+            {
+                return;
+            }
+
+            try
+            {
+                var endpoint = _request.RequestUri;
+                ApiKeyHttpTransportGuard.EnsureSafe(
+                    endpoint,
+                    isProxyDisabled: !ApiKeyHttpTransportGuard.IsPlaintextLoopback(endpoint) || _request.Proxy is null,
+                    redirectsDisabled: !_request.AllowAutoRedirect);
+            }
+            catch (ApiKeyHttpTransportException)
+            {
+                throw;
+            }
+            catch (Exception ex)
+            {
+                throw new ApiKeyHttpTransportException("Unable to verify a safe HTTP transport for DD-API-KEY.", ex);
+            }
         }
 
         private async Task<IApiResponse> FinishAndGetResponse()

tracer/src/Datadog.Trace/Agent/Transports/ApiWebRequestFactory.cs

@@ -53,12 +53,17 @@ public IApiRequest Create(Uri endpoint)
                 request.Timeout = (int)_timeout.Value.TotalMilliseconds;
             }
 
+            var hasApiKeyHeader = false;
             foreach (var pair in _defaultHeaders)
             {
                 request.Headers.Add(pair.Key, pair.Value);
+                if (string.Equals(pair.Key, ApiKeyHttpTransportGuard.ApiKeyHeaderName, StringComparison.OrdinalIgnoreCase))
+                {
+                    hasApiKeyHeader = true;
+                }
             }
 
-            return new ApiWebRequest(request);
+            return new ApiWebRequest(request, hasApiKeyHeader);
         }
 
         public void SetProxy(WebProxy proxy, NetworkCredential credential)

tracer/src/Datadog.Trace/Agent/Transports/HttpClientRequest.cs

@@ -24,20 +24,23 @@ internal sealed class HttpClientRequest : IApiRequest
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<HttpClientRequest>();
 
         private readonly HttpClient _client;
+        private readonly HttpClientHandler _apiKeyProtectedHandler;
         private readonly HttpRequestMessage _postRequest;
         private readonly HttpRequestMessage _getRequest;
         private readonly Uri _uri;
 
-        public HttpClientRequest(HttpClient client, Uri endpoint)
+        public HttpClientRequest(HttpClient client, HttpClientHandler apiKeyProtectedHandler, Uri endpoint)
         {
             _client = client;
+            _apiKeyProtectedHandler = apiKeyProtectedHandler;
             _postRequest = new HttpRequestMessage(HttpMethod.Post, endpoint);
             _getRequest = new HttpRequestMessage(HttpMethod.Get, endpoint);
             _uri = endpoint;
         }
 
         public void AddHeader(string name, string value)
         {
+            ApiKeyHttpTransportGuard.RejectLateApiKeyHeader(name);
             _postRequest.Headers.Add(name, value);
             _getRequest.Headers.Add(name, value);
         }
@@ -46,7 +49,7 @@ public async Task<IApiResponse> GetAsync()
         {
             _getRequest.Content = null;
 
-            return new HttpClientResponse(await _client.SendAsync(_getRequest).ConfigureAwait(false));
+            return new HttpClientResponse(await SendAsync(_getRequest).ConfigureAwait(false));
         }
 
         public Task<IApiResponse> PostAsync(ArraySegment<byte> bytes, string contentType)
@@ -65,7 +68,7 @@ public async Task<IApiResponse> PostAsync(ArraySegment<byte> bytes, string conte
 
                 _postRequest.Content = content;
 
-                var response = await _client.SendAsync(_postRequest).ConfigureAwait(false);
+                var response = await SendAsync(_postRequest).ConfigureAwait(false);
 
                 return new HttpClientResponse(response);
             }
@@ -91,7 +94,7 @@ public async Task<IApiResponse> PostAsJsonAsync<T>(T payload, MultipartCompressi
 
             _postRequest.Content = content;
 
-            var response = await _client.SendAsync(_postRequest).ConfigureAwait(false);
+            var response = await SendAsync(_postRequest).ConfigureAwait(false);
             return new HttpClientResponse(response);
         }
 
@@ -114,7 +117,7 @@ public async Task<IApiResponse> PostAsync(Func<Stream, Task> writeToRequestStrea
             }
 
             _postRequest.Content = content;
-            var response = await _client.SendAsync(_postRequest).ConfigureAwait(false);
+            var response = await SendAsync(_postRequest).ConfigureAwait(false);
 
             return new HttpClientResponse(response);
         }
@@ -175,9 +178,35 @@ public async Task<IApiResponse> PostAsync(MultipartFormItem[] items, MultipartCo
                 _postRequest.Content = formDataContent;
             }
 
-            var response = await _client.SendAsync(_postRequest).ConfigureAwait(false);
+            var response = await SendAsync(_postRequest).ConfigureAwait(false);
             return new HttpClientResponse(response);
         }
+
+        private Task<HttpResponseMessage> SendAsync(HttpRequestMessage request)
+        {
+            if (_apiKeyProtectedHandler is not null)
+            {
+                ApiKeyHttpTransportGuard.EnsureSafe(
+                    _uri,
+                    isProxyDisabled: IsProxyDisabledForEndpoint(),
+                    redirectsDisabled: AreRedirectsDisabled());
+            }
+
+            return _client.SendAsync(request);
+        }
+
+        private bool AreRedirectsDisabled()
+            => !_apiKeyProtectedHandler.AllowAutoRedirect;
+
+        private bool IsProxyDisabledForEndpoint()
+        {
+            if (!ApiKeyHttpTransportGuard.IsPlaintextLoopback(_uri))
+            {
+                return true;
+            }
+
+            return !_apiKeyProtectedHandler.UseProxy;
+        }
     }
 }
 #endif

tracer/src/Datadog.Trace/Agent/Transports/HttpClientRequestFactory.cs

@@ -20,13 +20,45 @@ internal sealed class HttpClientRequestFactory : IApiRequestFactory
     {
         private readonly HttpClient _client;
         private readonly HttpMessageHandler _handler;
+        private readonly HttpClientHandler _apiKeyProtectedHandler;
+        private readonly bool _disableProxyForPlaintextLoopback;
+        private readonly bool _hasApiKeyHeader;
         private readonly Uri _baseEndpoint;
 
-        public HttpClientRequestFactory(Uri baseEndpoint, KeyValuePair<string, string>[] defaultHeaders, HttpMessageHandler handler = null, TimeSpan? timeout = null)
+        public HttpClientRequestFactory(
+            Uri baseEndpoint,
+            KeyValuePair<string, string>[] defaultHeaders,
+            HttpMessageHandler handler = null,
+            TimeSpan? timeout = null,
+            DecompressionMethods automaticDecompression = DecompressionMethods.None)
         {
-            _handler = handler ?? new HttpClientHandler();
-            _client = new HttpClient(_handler);
             _baseEndpoint = baseEndpoint;
+            foreach (var pair in defaultHeaders)
+            {
+                if (string.Equals(pair.Key, ApiKeyHttpTransportGuard.ApiKeyHeaderName, StringComparison.OrdinalIgnoreCase))
+                {
+                    _hasApiKeyHeader = true;
+                }
+            }
+
+            if (_hasApiKeyHeader && handler is not null)
+            {
+                throw new ApiKeyHttpTransportException("Caller-provided HTTP handlers are not supported for protected DD-API-KEY transport.");
+            }
+
+            _handler = handler ?? new HttpClientHandler { AutomaticDecompression = automaticDecompression };
+            _disableProxyForPlaintextLoopback = _hasApiKeyHeader && ApiKeyHttpTransportGuard.IsPlaintextLoopback(baseEndpoint);
+            if (_hasApiKeyHeader)
+            {
+                _apiKeyProtectedHandler = (HttpClientHandler)_handler;
+                _apiKeyProtectedHandler.AllowAutoRedirect = false;
+                if (_disableProxyForPlaintextLoopback)
+                {
+                    _apiKeyProtectedHandler.UseProxy = false;
+                }
+            }
+
+            _client = new HttpClient(_handler);
             if (timeout.HasValue)
             {
                 _client.Timeout = timeout.Value;
@@ -54,11 +86,16 @@ public string Info(Uri endpoint)
 
         public IApiRequest Create(Uri endpoint)
         {
-            return new HttpClientRequest(_client, endpoint);
+            return new HttpClientRequest(_client, _apiKeyProtectedHandler, endpoint);
         }
 
         public void SetProxy(WebProxy proxy, NetworkCredential credential)
         {
+            if (_disableProxyForPlaintextLoopback)
+            {
+                return;
+            }
+
             if (_handler is HttpClientHandler handler)
             {
                 handler.Proxy = proxy;

tracer/src/Datadog.Trace/Agent/Transports/HttpStreamRequest.cs

@@ -35,6 +35,7 @@ public HttpStreamRequest(DatadogHttpClient client, Uri uri, IStreamFactory strea
 
         public void AddHeader(string name, string value)
         {
+            ApiKeyHttpTransportGuard.RejectLateApiKeyHeader(name);
             _headers.Add(name, value);
         }