Skip to main content

nautilus_network/http/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! HTTP client implementation with rate limiting and timeout support.
17
18use std::{borrow::Cow, collections::HashMap, str::FromStr, sync::Arc, time::Duration};
19
20use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
21use bytes::Bytes;
22use http::{
23    Method,
24    header::{HeaderMap, HeaderName, HeaderValue},
25};
26use http_body_util::Full;
27use nautilus_core::{collections::into_ustr_vec, string::secret::SecretString};
28use nautilus_cryptography::providers::install_cryptographic_provider;
29use url::Url;
30use ustr::Ustr;
31
32use super::{
33    HttpClientError, HttpResponse, HttpResponseStream, HttpStatus,
34    stream::{check_deadline, read_chunk, response_error},
35};
36use crate::ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota};
37
38/// Default maximum idle connections per host.
39#[cfg(not(all(feature = "simulation", madsim)))]
40const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 32;
41
42/// Default idle connection timeout in seconds.
43#[cfg(not(all(feature = "simulation", madsim)))]
44const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 60;
45
46/// Default HTTP/2 keep-alive interval in seconds.
47#[cfg(not(all(feature = "simulation", madsim)))]
48const DEFAULT_HTTP2_KEEP_ALIVE_SECS: u64 = 30;
49
50/// Default HTTP/2 stream flow-control window in bytes (16 MiB).
51#[cfg(not(all(feature = "simulation", madsim)))]
52const DEFAULT_HTTP2_STREAM_WINDOW_BYTES: u32 = 16 * 1024 * 1024;
53
54/// Default HTTP/2 connection flow-control window in bytes (32 MiB).
55#[cfg(not(all(feature = "simulation", madsim)))]
56const DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES: u32 = 32 * 1024 * 1024;
57
58/// Environment variable that enables adaptive HTTP/2 flow-control windows when set to `true`.
59#[cfg(not(all(feature = "simulation", madsim)))]
60const NAUTILUS_HTTP2_ADAPTIVE_WINDOW: &str = "NAUTILUS_HTTP2_ADAPTIVE_WINDOW";
61
62/// Default maximum HTTP response body size in bytes (100 MiB).
63///
64/// Bounds peak memory per response so a hostile or malfunctioning endpoint
65/// cannot exhaust memory by streaming an arbitrarily large body. Mirrors the
66/// caps already enforced on the WebSocket and raw-socket paths.
67const DEFAULT_MAX_RESPONSE_BYTES: usize = 100 * 1024 * 1024;
68
69const GZIP: &str = "gzip";
70const GZIP_DECODE_CHUNK_BYTES: usize = 64 * 1024;
71
72#[cfg(all(feature = "simulation", madsim))]
73pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "simulated request deadline elapsed";
74#[cfg(not(all(feature = "simulation", madsim)))]
75pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "request deadline elapsed";
76
77/// Controls whether an HTTP client follows redirects.
78#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
79pub enum HttpRedirectPolicy {
80    /// Follow up to ten redirects.
81    #[default]
82    Follow,
83    /// Reject every redirect response.
84    Reject,
85}
86
87/// An asynchronous HTTP client with rate limiting, timeouts, and custom headers.
88///
89/// The client uses Hyper for normal I/O and supports default and per-key quotas. Multiple
90/// clients can share the same rate limiter when their requests consume one quota budget.
91/// With `simulation` and `cfg(madsim)`, plaintext HTTP/1.1 uses simulated byte streams;
92/// HTTPS, client certificates, explicit proxies, and redirect following are unsupported.
93///
94/// Request URLs in transport errors and response-stream diagnostics are limited to their origin
95/// (scheme, host, and port). Userinfo, paths, queries, and fragments are excluded. Set
96/// `redact_url(true)` on the builder to omit the origin too. This policy does not change requests
97/// or diagnostic messages from lower layers. It controls the URL context added by this client.
98#[derive(Clone, Debug)]
99pub struct HttpClient {
100    pub(crate) client: InnerHttpClient,
101    pub(crate) rate_limiters: Arc<[Arc<RateLimiter<Ustr, MonotonicClock>>]>,
102}
103
104#[bon::bon]
105impl HttpClient {
106    /// Returns a builder for a new [`HttpClient`] instance.
107    ///
108    /// Set `rate_limiters` to share quota state across clients. When omitted, the client creates
109    /// one rate limiter from `default_quota` and `keyed_quotas`. An explicit empty vector disables
110    /// rate limiting. Each request awaits every configured limiter with the same keys. A limiter
111    /// without a default quota ignores keys it does not own, allowing independent scopes such as
112    /// per-IP and per-account limits to apply to one request.
113    ///
114    /// # Errors
115    ///
116    /// Returns an error if:
117    /// - Shared rate limiters are combined with quota configuration.
118    /// - The timeout exceeds the monotonic clock range.
119    /// - The proxy URL is malformed.
120    /// - `NAUTILUS_HTTP2_ADAPTIVE_WINDOW` is set to a value other than `true` or `false`.
121    /// - The certificate directory does not contain a matching client certificate and private key.
122    /// - Building the underlying HTTP transport fails.
123    #[allow(
124        clippy::needless_pass_by_value,
125        reason = "owned proxy URLs are part of the public builder API"
126    )]
127    #[builder(finish_fn = build)]
128    pub fn builder(
129        #[builder(default)] headers: HashMap<String, String>,
130        #[builder(default)] header_keys: Vec<String>,
131        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
132        default_quota: Option<Quota>,
133        timeout_secs: Option<u64>,
134        proxy_url: Option<String>,
135        rate_limiters: Option<Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>>,
136        #[builder(default)] redirect_policy: HttpRedirectPolicy,
137        /// Whether to honor ambient proxy configuration when `proxy_url` is not set.
138        #[builder(default = true)]
139        use_system_proxy: bool,
140        /// Whether to omit the request origin added by this client to transport errors and
141        /// response-stream diagnostics.
142        /// Defaults to `false`.
143        /// Userinfo, paths, queries, and fragments are always excluded from the added URL context.
144        /// Diagnostic messages from lower layers are unchanged. TLS certificate errors and
145        /// dependency logs can still include the hostname.
146        #[builder(default)]
147        redact_url: bool,
148        /// Maximum size in bytes of a buffered response body after `Content-Encoding` decoding.
149        /// Defaults to the response body limit (100 MiB). Raise it for endpoints whose compressed
150        /// payloads legitimately decode beyond that limit; the wire body stays bounded by the
151        /// response body limit either way.
152        #[builder(default = DEFAULT_MAX_RESPONSE_BYTES)]
153        max_decompressed_bytes: usize,
154        /// Directory containing a matching client certificate and private key.
155        /// Uses bundled roots plus certificates from this directory instead of system trust.
156        certs_dir: Option<String>,
157    ) -> Result<Self, HttpClientError> {
158        let rate_limiters = if let Some(rate_limiters) = rate_limiters {
159            if default_quota.is_some() || !keyed_quotas.is_empty() {
160                return Err(HttpClientError::Error(
161                    "Cannot combine shared rate limiters with quota configuration".to_string(),
162                ));
163            }
164
165            rate_limiters
166        } else {
167            let keyed_quotas = keyed_quotas
168                .into_iter()
169                .map(|(key, quota)| (Ustr::from(&key), quota))
170                .collect();
171            vec![Arc::new(RateLimiter::new_with_quota(
172                default_quota,
173                keyed_quotas,
174            ))]
175        };
176
177        Self::build(
178            headers,
179            header_keys,
180            timeout_secs,
181            proxy_url.as_deref(),
182            rate_limiters,
183            redirect_policy,
184            use_system_proxy,
185            redact_url,
186            max_decompressed_bytes,
187            certs_dir.as_deref(),
188        )
189    }
190
191    #[expect(
192        clippy::too_many_arguments,
193        reason = "client construction consumes the builder settings"
194    )]
195    fn build(
196        headers: HashMap<String, String>,
197        header_keys: Vec<String>,
198        timeout_secs: Option<u64>,
199        proxy_url: Option<&str>,
200        rate_limiters: Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>,
201        redirect_policy: HttpRedirectPolicy,
202        use_system_proxy: bool,
203        redact_url: bool,
204        max_decompressed_bytes: usize,
205        certs_dir: Option<&str>,
206    ) -> Result<Self, HttpClientError> {
207        let timeout = timeout_secs.map(Duration::from_secs);
208        request_deadline(timeout)?;
209        install_cryptographic_provider();
210
211        let mut header_map = HeaderMap::new();
212
213        for (key, value) in headers {
214            let header_name = HeaderName::from_str(&key)
215                .map_err(|e| HttpClientError::Error(format!("Invalid header name '{key}': {e}")))?;
216
217            let mut header_value = HeaderValue::from_str(&value).map_err(|e| {
218                HttpClientError::Error(format!("Invalid header value for '{key}': {e}"))
219            })?;
220
221            header_value.set_sensitive(true);
222            header_map.insert(header_name, header_value);
223        }
224
225        #[cfg(all(feature = "simulation", madsim))]
226        let simulation = super::simulation::Client::new(redirect_policy, proxy_url, certs_dir)?;
227
228        #[cfg(not(all(feature = "simulation", madsim)))]
229        let client = super::transport::Client::new(
230            proxy_url,
231            use_system_proxy,
232            certs_dir,
233            header_map.get(http::header::USER_AGENT).cloned(),
234            super::transport::Settings {
235                pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST,
236                pool_idle_timeout: Duration::from_secs(DEFAULT_POOL_IDLE_TIMEOUT_SECS),
237                keep_alive_interval: Some(Duration::from_secs(DEFAULT_HTTP2_KEEP_ALIVE_SECS)),
238                stream_window: Some(DEFAULT_HTTP2_STREAM_WINDOW_BYTES),
239                connection_window: Some(DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES),
240                adaptive_window: http2_adaptive_window()?,
241            },
242        )?;
243
244        #[cfg(all(feature = "simulation", madsim))]
245        let _ = use_system_proxy;
246
247        // Pre-intern header keys as HeaderName. An invalid key is an error: a silent drop would
248        // make response extraction read nothing.
249        let response_headers = header_keys
250            .into_iter()
251            .map(|key| match HeaderName::from_str(&key) {
252                Ok(name) => Ok((key, name)),
253                Err(e) => Err(HttpClientError::Error(format!(
254                    "Invalid header key '{key}': {e}"
255                ))),
256            })
257            .collect::<Result<Vec<_>, _>>()?;
258
259        let client = InnerHttpClient {
260            #[cfg(not(all(feature = "simulation", madsim)))]
261            client,
262            headers: header_map,
263            timeout,
264            redact_url,
265            #[cfg(not(all(feature = "simulation", madsim)))]
266            redirect_policy,
267            #[cfg(all(feature = "simulation", madsim))]
268            simulation,
269            response_headers: Arc::from(response_headers),
270            max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
271            max_decompressed_bytes,
272        };
273
274        Ok(Self {
275            client,
276            rate_limiters: rate_limiters.into(),
277        })
278    }
279
280    /// Sends an HTTP request.
281    ///
282    /// # Errors
283    ///
284    /// Returns an error if the timeout exceeds the monotonic clock range, the request cannot be sent,
285    /// or it times out.
286    ///
287    /// # Examples
288    ///
289    /// If requesting `/foo/bar`, pass rate-limit keys `["foo/bar", "foo"]`.
290    #[expect(clippy::too_many_arguments)]
291    pub async fn request(
292        &self,
293        method: Method,
294        url: String,
295        params: Option<&HashMap<String, Vec<String>>>,
296        headers: Option<HashMap<String, String>>,
297        body: Option<Vec<u8>>,
298        timeout_secs: Option<u64>,
299        keys: Option<Vec<String>>,
300    ) -> Result<HttpResponse, HttpClientError> {
301        let keys = keys.map(into_ustr_vec);
302
303        self.request_with_ustr_keys(method, url, params, headers, body, timeout_secs, keys)
304            .await
305    }
306
307    /// Sends an HTTP request whose body contains secret material.
308    ///
309    /// The body retains its zeroizing owner until the transport releases the last byte buffer.
310    /// Transport, TLS, and operating-system layers may make additional plaintext copies that this
311    /// client cannot zeroize.
312    ///
313    /// # Errors
314    ///
315    /// Returns an error if unable to send the request or if it times out.
316    #[expect(clippy::too_many_arguments)]
317    pub async fn request_with_secret_body(
318        &self,
319        method: Method,
320        url: String,
321        params: Option<&HashMap<String, Vec<String>>>,
322        headers: Option<HashMap<String, String>>,
323        body: SecretString,
324        timeout_secs: Option<u64>,
325        keys: Option<Vec<String>>,
326    ) -> Result<HttpResponse, HttpClientError> {
327        let keys = keys.map(into_ustr_vec);
328        self.await_rate_limits(keys.as_deref()).await;
329
330        self.client
331            .send_request_with_secret_body(method, url, params, headers, body, timeout_secs)
332            .await
333    }
334
335    /// Sends an HTTP request with serializable query parameters.
336    ///
337    /// This method accepts any type implementing `Serialize` for query parameters,
338    /// which are URL-encoded directly into the query string without an intermediate `HashMap`.
339    ///
340    /// # Errors
341    ///
342    /// Returns an error if unable to send request or times out.
343    #[expect(clippy::too_many_arguments)]
344    pub async fn request_with_params<P: serde::Serialize>(
345        &self,
346        method: Method,
347        url: String,
348        params: Option<&P>,
349        headers: Option<HashMap<String, String>>,
350        body: Option<Vec<u8>>,
351        timeout_secs: Option<u64>,
352        keys: Option<Vec<String>>,
353    ) -> Result<HttpResponse, HttpClientError> {
354        let keys = keys.map(into_ustr_vec);
355        self.await_rate_limits(keys.as_deref()).await;
356
357        self.client
358            .send_request_with_query(method, url, params, headers, body, timeout_secs)
359            .await
360    }
361
362    /// Sends a GET request and returns its response body as a stream.
363    ///
364    /// Returns raw body chunks without content decoding or adding `Accept-Encoding`.
365    /// Applies default headers and the client timeout. No rate-limit keys are supplied, so no
366    /// quota is consumed. One absolute deadline covers response headers and the whole body,
367    /// including time spent processing chunks. Streaming has no total body size limit; callers
368    /// must process or discard each chunk without accumulating an unbounded body.
369    /// Dropping the response releases the unfinished exchange, including its simulated driver.
370    ///
371    /// # Errors
372    ///
373    /// Returns an error if request preparation, connection, or response headers fail or time out.
374    pub async fn get_stream(&self, url: String) -> Result<HttpResponseStream, HttpClientError> {
375        self.await_rate_limits(None).await;
376        self.client
377            .send_stream_internal::<[(String, String); 0]>(
378                Method::GET,
379                &url,
380                None,
381                None,
382                None,
383                None,
384            )
385            .await
386    }
387
388    /// Sends an HTTP request using pre-interned rate limiter keys.
389    ///
390    /// # Errors
391    ///
392    /// Returns an error if unable to send the request or the request times out.
393    #[expect(clippy::too_many_arguments)]
394    pub async fn request_with_ustr_keys(
395        &self,
396        method: Method,
397        url: String,
398        params: Option<&HashMap<String, Vec<String>>>,
399        headers: Option<HashMap<String, String>>,
400        body: Option<Vec<u8>>,
401        timeout_secs: Option<u64>,
402        keys: Option<Vec<Ustr>>,
403    ) -> Result<HttpResponse, HttpClientError> {
404        self.await_rate_limits(keys.as_deref()).await;
405
406        self.client
407            .send_request(method, url, params, headers, body, timeout_secs)
408            .await
409    }
410
411    /// Acquires quota for the supplied keys before a caller prepares a request.
412    ///
413    /// Waits without a timeout and consumes quota atomically across the configured limiters.
414    /// Dropping a pending future consumes no quota. Callers must bound the wait when needed.
415    /// The next request must omit rate-limit keys to avoid consuming quota twice.
416    /// Passing no keys acquires no quota.
417    pub async fn await_rate_limits(&self, keys: Option<&[Ustr]>) {
418        RateLimiter::await_limiters_ready(&self.rate_limiters, keys).await;
419    }
420
421    /// Sends an HTTP GET request.
422    ///
423    /// # Errors
424    ///
425    /// Returns an error if unable to send request or times out.
426    pub async fn get(
427        &self,
428        url: String,
429        params: Option<&HashMap<String, Vec<String>>>,
430        headers: Option<HashMap<String, String>>,
431        timeout_secs: Option<u64>,
432        keys: Option<Vec<String>>,
433    ) -> Result<HttpResponse, HttpClientError> {
434        self.request(Method::GET, url, params, headers, None, timeout_secs, keys)
435            .await
436    }
437
438    /// Sends an HTTP POST request.
439    ///
440    /// # Errors
441    ///
442    /// Returns an error if unable to send request or times out.
443    pub async fn post(
444        &self,
445        url: String,
446        params: Option<&HashMap<String, Vec<String>>>,
447        headers: Option<HashMap<String, String>>,
448        body: Option<Vec<u8>>,
449        timeout_secs: Option<u64>,
450        keys: Option<Vec<String>>,
451    ) -> Result<HttpResponse, HttpClientError> {
452        self.request(Method::POST, url, params, headers, body, timeout_secs, keys)
453            .await
454    }
455
456    /// Sends an HTTP PATCH request.
457    ///
458    /// # Errors
459    ///
460    /// Returns an error if unable to send request or times out.
461    pub async fn patch(
462        &self,
463        url: String,
464        params: Option<&HashMap<String, Vec<String>>>,
465        headers: Option<HashMap<String, String>>,
466        body: Option<Vec<u8>>,
467        timeout_secs: Option<u64>,
468        keys: Option<Vec<String>>,
469    ) -> Result<HttpResponse, HttpClientError> {
470        self.request(
471            Method::PATCH,
472            url,
473            params,
474            headers,
475            body,
476            timeout_secs,
477            keys,
478        )
479        .await
480    }
481
482    /// Sends an HTTP DELETE request.
483    ///
484    /// # Errors
485    ///
486    /// Returns an error if unable to send request or times out.
487    pub async fn delete(
488        &self,
489        url: String,
490        params: Option<&HashMap<String, Vec<String>>>,
491        headers: Option<HashMap<String, String>>,
492        timeout_secs: Option<u64>,
493        keys: Option<Vec<String>>,
494    ) -> Result<HttpResponse, HttpClientError> {
495        self.request(
496            Method::DELETE,
497            url,
498            params,
499            headers,
500            None,
501            timeout_secs,
502            keys,
503        )
504        .await
505    }
506}
507
508/// Internal implementation backing [`HttpClient`].
509///
510/// The underlying Hyper client reuses pooled connections and is cheap to clone. Responses
511/// retain only configured header fields, and bodies larger than `max_response_bytes` are rejected.
512/// Decoded (decompressed) bodies are bounded separately by `max_decompressed_bytes`.
513#[derive(Clone, Debug)]
514pub struct InnerHttpClient {
515    #[cfg(all(feature = "simulation", madsim))]
516    simulation: super::simulation::Client,
517    #[cfg(not(all(feature = "simulation", madsim)))]
518    client: super::transport::Client,
519    headers: HeaderMap,
520    timeout: Option<Duration>,
521    redact_url: bool,
522    #[cfg(not(all(feature = "simulation", madsim)))]
523    redirect_policy: HttpRedirectPolicy,
524    pub(crate) response_headers: Arc<[(String, HeaderName)]>,
525    pub(crate) max_response_bytes: usize,
526    pub(crate) max_decompressed_bytes: usize,
527}
528
529impl InnerHttpClient {
530    /// Sends an HTTP request and returns an [`HttpResponse`].
531    ///
532    /// # Errors
533    ///
534    /// Returns an error if unable to send request or times out.
535    pub async fn send_request(
536        &self,
537        method: Method,
538        url: String,
539        params: Option<&HashMap<String, Vec<String>>>,
540        headers: Option<HashMap<String, String>>,
541        body: Option<Vec<u8>>,
542        timeout_secs: Option<u64>,
543    ) -> Result<HttpResponse, HttpClientError> {
544        self.send_request_with_body(
545            method,
546            url,
547            params,
548            headers,
549            body.map(RequestBody::Plain),
550            timeout_secs,
551        )
552        .await
553    }
554
555    async fn send_request_with_secret_body(
556        &self,
557        method: Method,
558        url: String,
559        params: Option<&HashMap<String, Vec<String>>>,
560        headers: Option<HashMap<String, String>>,
561        body: SecretString,
562        timeout_secs: Option<u64>,
563    ) -> Result<HttpResponse, HttpClientError> {
564        self.send_request_with_body(
565            method,
566            url,
567            params,
568            headers,
569            Some(RequestBody::Secret(body)),
570            timeout_secs,
571        )
572        .await
573    }
574
575    async fn send_request_with_body(
576        &self,
577        method: Method,
578        url: String,
579        params: Option<&HashMap<String, Vec<String>>>,
580        headers: Option<HashMap<String, String>>,
581        body: Option<RequestBody>,
582        timeout_secs: Option<u64>,
583    ) -> Result<HttpResponse, HttpClientError> {
584        let full_url = encode_url_params(&url, params)?;
585        self.send_request_internal(
586            method,
587            full_url.as_ref(),
588            None::<&()>,
589            headers,
590            body,
591            timeout_secs,
592        )
593        .await
594    }
595
596    /// Sends an HTTP request with URL-encoded serializable query parameters.
597    ///
598    /// This method accepts any type implementing `Serialize` for query parameters,
599    /// avoiding `HashMap` conversion overhead.
600    ///
601    /// # Errors
602    ///
603    /// Returns an error if unable to send request or times out.
604    pub async fn send_request_with_query<Q: serde::Serialize>(
605        &self,
606        method: Method,
607        url: String,
608        query: Option<&Q>,
609        headers: Option<HashMap<String, String>>,
610        body: Option<Vec<u8>>,
611        timeout_secs: Option<u64>,
612    ) -> Result<HttpResponse, HttpClientError> {
613        self.send_request_internal(
614            method,
615            &url,
616            query,
617            headers,
618            body.map(RequestBody::Plain),
619            timeout_secs,
620        )
621        .await
622    }
623
624    /// Internal implementation for sending HTTP requests.
625    ///
626    /// # Errors
627    ///
628    /// Returns an error if unable to send request or times out.
629    async fn send_request_internal<Q: serde::Serialize>(
630        &self,
631        method: Method,
632        url: &str,
633        query: Option<&Q>,
634        headers: Option<HashMap<String, String>>,
635        body: Option<RequestBody>,
636        timeout_secs: Option<u64>,
637    ) -> Result<HttpResponse, HttpClientError> {
638        let mut headers = headers.unwrap_or_default();
639        let has_accept_encoding = headers
640            .keys()
641            .any(|key| key.eq_ignore_ascii_case(http::header::ACCEPT_ENCODING.as_str()))
642            || self.headers.contains_key(http::header::ACCEPT_ENCODING);
643
644        if !has_accept_encoding {
645            headers.insert(http::header::ACCEPT_ENCODING.to_string(), GZIP.to_string());
646        }
647
648        let is_head = method == Method::HEAD;
649        let stream = self
650            .send_stream_internal(method, url, query, Some(headers), body, timeout_secs)
651            .await?;
652        let result = self
653            .consume_response(stream.response, stream.deadline, is_head)
654            .await;
655        result.map_err(|e| response_error(e, stream.origin.as_deref()))
656    }
657
658    async fn send_stream_internal<Q: serde::Serialize>(
659        &self,
660        method: Method,
661        url: &str,
662        query: Option<&Q>,
663        headers: Option<HashMap<String, String>>,
664        body: Option<RequestBody>,
665        timeout_secs: Option<u64>,
666    ) -> Result<HttpResponseStream, HttpClientError> {
667        let mut url =
668            Url::parse(url).map_err(|e| HttpClientError::from(format!("URL parse error: {e}")))?;
669        if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
670            return Err(HttpClientError::Error(
671                "unsupported HTTP URL scheme or hostname".into(),
672            ));
673        }
674
675        let mut header_map = self.headers.clone();
676
677        if let Ok(username) = percent_encoding::percent_decode_str(url.username()).decode_utf8() {
678            let password = url.password().and_then(|password| {
679                percent_encoding::percent_decode_str(password)
680                    .decode_utf8()
681                    .ok()
682            });
683
684            if !username.is_empty() || password.is_some() {
685                let mut value = HeaderValue::from_str(&format!(
686                    "Basic {}",
687                    BASE64.encode(format!(
688                        "{username}:{}",
689                        password.as_deref().unwrap_or_default()
690                    ))
691                ))
692                .map_err(|e| HttpClientError::Error(e.to_string()))?;
693                value.set_sensitive(true);
694                header_map.insert(http::header::AUTHORIZATION, value);
695                let _ = url.set_username("");
696                let _ = url.set_password(None);
697            }
698        }
699
700        let extra_header_count = headers.as_ref().map_or(0, HashMap::len);
701
702        if let Some(headers) = headers {
703            for (key, value) in headers {
704                let key = HeaderName::from_bytes(key.as_bytes())
705                    .map_err(|e| HttpClientError::from(format!("Invalid header name: {e}")))?;
706                let value = HeaderValue::from_str(&value)
707                    .map_err(|e| HttpClientError::from(format!("Invalid header value: {e}")))?;
708
709                if header_map.insert(key.clone(), value).is_some() {
710                    log::trace!("Replaced duplicate request header '{key}'");
711                }
712            }
713        }
714
715        if let Some(query) = query {
716            {
717                let mut pairs = url.query_pairs_mut();
718                let serializer = serde_urlencoded::Serializer::new(&mut pairs);
719                query
720                    .serialize(serializer)
721                    .map_err(|e| HttpClientError::Error(e.to_string()))?;
722            }
723
724            if url.query() == Some("") {
725                url.set_query(None);
726            }
727        }
728
729        if !header_map.contains_key(http::header::ACCEPT) {
730            header_map.insert(http::header::ACCEPT, HeaderValue::from_static("*/*"));
731        }
732
733        let body = body.map(RequestBody::into_bytes).unwrap_or_default();
734        let body_len = body.len();
735        let query_len = url.query().map_or(0, str::len);
736        let mut request = http::Request::new(Full::new(body));
737        *request.method_mut() = method;
738        *request.uri_mut() = url[..url::Position::AfterQuery]
739            .parse()
740            .map_err(|_| HttpClientError::Error("invalid HTTP request target".into()))?;
741        *request.headers_mut() = header_map;
742        log::trace!(
743            "Sending HTTP request: method={} extra_headers={extra_header_count} \
744             query_bytes={query_len} body_bytes={body_len}",
745            request.method(),
746        );
747
748        let origin = (!self.redact_url).then(|| url.origin().ascii_serialization());
749
750        let duration = timeout_secs.map(Duration::from_secs).or(self.timeout);
751        let deadline = request_deadline(duration)?;
752
753        let operation = async {
754            #[cfg(all(feature = "simulation", madsim))]
755            let (response, connection) = self.simulation.send(request, &url).await?;
756            #[cfg(not(all(feature = "simulation", madsim)))]
757            let response = self.client.send(request, self.redirect_policy).await?;
758            Ok(HttpResponseStream {
759                response,
760                deadline,
761                origin: origin.clone(),
762                #[cfg(all(feature = "simulation", madsim))]
763                _connection: connection,
764            })
765        };
766
767        let result = match deadline {
768            Some(deadline) => tokio::select! {
769                biased;
770                () = crate::dst::time::sleep_until(deadline) => Err(HttpClientError::TimeoutError(REQUEST_TIMEOUT_MESSAGE.into())),
771                result = operation => result,
772            },
773            None => operation.await,
774        };
775
776        result.map_err(|e| response_error(e, origin.as_deref()))
777    }
778
779    async fn consume_response<B>(
780        &self,
781        response: http::Response<B>,
782        deadline: Option<crate::dst::time::Instant>,
783        is_head: bool,
784    ) -> Result<HttpResponse, HttpClientError>
785    where
786        B: http_body::Body<Data = Bytes> + Unpin,
787        B::Error: std::error::Error + 'static,
788    {
789        let (parts, mut body) = response.into_parts();
790        let mut headers =
791            HashMap::with_capacity(self.response_headers.len().min(parts.headers.len()));
792        for (key, name) in self.response_headers.iter() {
793            if let Some(value) = parts
794                .headers
795                .get(name)
796                .and_then(|value| value.to_str().ok())
797            {
798                headers.insert(key.clone(), value.to_owned());
799            }
800        }
801
802        let max = self.max_response_bytes;
803
804        if let Some(len) = body.size_hint().exact()
805            && len > max as u64
806        {
807            return Err(HttpClientError::Error(format!(
808                "HTTP response body of {len} bytes exceeds maximum of {max} bytes",
809            )));
810        }
811
812        let mut buf = bytes::BytesMut::new();
813        while let Some(chunk) = read_chunk(&mut body, deadline).await? {
814            if chunk.len() > max - buf.len() {
815                return Err(HttpClientError::Error(format!(
816                    "HTTP response body exceeds maximum of {max} bytes",
817                )));
818            }
819
820            buf.extend_from_slice(&chunk);
821        }
822
823        log::trace!(
824            "Received HTTP response: status={} headers={} body_bytes={}",
825            parts.status,
826            parts.headers.len(),
827            buf.len()
828        );
829
830        let bodyless = is_head
831            || parts.status.is_informational()
832            || matches!(
833                parts.status,
834                http::StatusCode::NO_CONTENT
835                    | http::StatusCode::RESET_CONTENT
836                    | http::StatusCode::NOT_MODIFIED
837            );
838
839        let (body, decoded) = if bodyless {
840            (buf.freeze(), false)
841        } else {
842            decode_content_encoding(&parts.headers, buf, self.max_decompressed_bytes, deadline)
843                .await?
844        };
845
846        if decoded {
847            headers.retain(|key, _| {
848                !key.eq_ignore_ascii_case("content-encoding")
849                    && !key.eq_ignore_ascii_case("content-length")
850            });
851        }
852
853        Ok(HttpResponse {
854            status: HttpStatus::new(parts.status),
855            headers,
856            body,
857        })
858    }
859}
860
861enum RequestBody {
862    Plain(Vec<u8>),
863    Secret(SecretString),
864}
865
866fn request_deadline(
867    timeout: Option<Duration>,
868) -> Result<Option<crate::dst::time::Instant>, HttpClientError> {
869    timeout
870        .map(|duration| {
871            crate::dst::time::Instant::now()
872                .checked_add(duration)
873                .ok_or_else(|| {
874                    HttpClientError::Error("HTTP request timeout exceeds clock range".into())
875                })
876        })
877        .transpose()
878}
879
880async fn decode_content_encoding(
881    headers: &HeaderMap,
882    body: bytes::BytesMut,
883    max: usize,
884    deadline: Option<crate::dst::time::Instant>,
885) -> Result<(Bytes, bool), HttpClientError> {
886    let mut encodings = headers.get_all(http::header::CONTENT_ENCODING).iter();
887
888    let Some(encoding) = encodings.next() else {
889        return Ok((body.freeze(), false));
890    };
891
892    if encodings.next().is_some() {
893        return Err(HttpClientError::Error(
894            "unsupported multiple HTTP content-encoding fields".into(),
895        ));
896    }
897
898    let encoding = encoding
899        .to_str()
900        .map_err(|_| HttpClientError::Error("invalid HTTP content-encoding value".into()))?;
901    let encoding = encoding.trim();
902    if encoding.is_empty() || encoding.eq_ignore_ascii_case("identity") {
903        return Ok((body.freeze(), false));
904    }
905
906    if !encoding.eq_ignore_ascii_case(GZIP) {
907        return Err(HttpClientError::Error(format!(
908            "unsupported HTTP content-encoding '{encoding}'"
909        )));
910    }
911
912    check_deadline(deadline)?;
913    let body = body.freeze();
914    let mut out = Vec::new();
915
916    let mut reader = std::io::Read::take(
917        flate2::bufread::MultiGzDecoder::new(GzipInput {
918            bytes: body.as_ref(),
919            remaining: GZIP_DECODE_CHUNK_BYTES,
920        }),
921        (max as u64).saturating_add(1),
922    );
923
924    loop {
925        check_deadline(deadline)?;
926        let mut chunk = std::io::Read::take(&mut reader, GZIP_DECODE_CHUNK_BYTES as u64);
927        let result = std::io::Read::read_to_end(&mut chunk, &mut out);
928        check_deadline(deadline)?;
929
930        let complete = match result {
931            Ok(count) => count < GZIP_DECODE_CHUNK_BYTES,
932            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => false,
933            Err(e) => {
934                return Err(HttpClientError::Error(format!(
935                    "failed to decode gzip response body: {e}"
936                )));
937            }
938        };
939
940        if out.len() > max {
941            return Err(HttpClientError::Error(format!(
942                "decompressed HTTP response body exceeds maximum of {max} bytes"
943            )));
944        }
945
946        if complete {
947            break;
948        }
949
950        crate::dst::task::yield_now().await;
951        reader.get_mut().get_mut().remaining = GZIP_DECODE_CHUNK_BYTES;
952    }
953
954    Ok((Bytes::from(out), true))
955}
956
957struct GzipInput<'a> {
958    bytes: &'a [u8],
959    remaining: usize,
960}
961
962impl std::io::Read for GzipInput<'_> {
963    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
964        if buf.is_empty() {
965            return Ok(0);
966        }
967
968        let mut available = std::io::BufRead::fill_buf(self)?;
969        let count = std::io::Read::read(&mut available, buf)?;
970        std::io::BufRead::consume(self, count);
971        Ok(count)
972    }
973}
974
975impl std::io::BufRead for GzipInput<'_> {
976    fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
977        if self.bytes.is_empty() {
978            return Ok(self.bytes);
979        }
980
981        if self.remaining == 0 {
982            return Err(std::io::ErrorKind::WouldBlock.into());
983        }
984
985        Ok(&self.bytes[..self.bytes.len().min(self.remaining)])
986    }
987
988    fn consume(&mut self, count: usize) {
989        self.bytes = &self.bytes[count..];
990        self.remaining -= count;
991    }
992}
993
994impl RequestBody {
995    fn into_bytes(self) -> Bytes {
996        match self {
997            Self::Plain(body) => body.into(),
998            Self::Secret(body) => Bytes::from_owner(SecretBody(body)),
999        }
1000    }
1001}
1002
1003struct SecretBody(SecretString);
1004
1005impl AsRef<[u8]> for SecretBody {
1006    fn as_ref(&self) -> &[u8] {
1007        self.0.expose_secret().as_bytes()
1008    }
1009}
1010
1011#[cfg(not(all(feature = "simulation", madsim)))]
1012fn http2_adaptive_window() -> Result<bool, HttpClientError> {
1013    let Some(value) = std::env::var_os(NAUTILUS_HTTP2_ADAPTIVE_WINDOW) else {
1014        return Ok(false);
1015    };
1016
1017    value.to_str().and_then(|v| v.parse().ok()).ok_or_else(|| {
1018        HttpClientError::ClientBuildError(format!(
1019            "{NAUTILUS_HTTP2_ADAPTIVE_WINDOW} must be 'true' or 'false', was '{}'",
1020            value.display()
1021        ))
1022    })
1023}
1024
1025/// Encodes URL parameters into the query string.
1026///
1027/// Returns `Cow::Borrowed` when no parameters need appending (zero-alloc fast path).
1028/// Parameters can have multiple values per key (for doseq=True behavior).
1029/// Preserves existing query strings in the URL by appending with '&' instead of '?'.
1030/// The query is inserted before any fragment, which is preserved unchanged.
1031fn encode_url_params<'a>(
1032    url: &'a str,
1033    params: Option<&HashMap<String, Vec<String>>>,
1034) -> Result<Cow<'a, str>, HttpClientError> {
1035    let Some(params) = params else {
1036        return Ok(Cow::Borrowed(url));
1037    };
1038
1039    let pairs: Vec<(&str, &str)> = params
1040        .iter()
1041        .flat_map(|(key, values)| {
1042            values
1043                .iter()
1044                .map(move |value| (key.as_str(), value.as_str()))
1045        })
1046        .collect();
1047
1048    if pairs.is_empty() {
1049        return Ok(Cow::Borrowed(url));
1050    }
1051
1052    let query_string = serde_urlencoded::to_string(pairs)
1053        .map_err(|e| HttpClientError::Error(format!("Failed to encode params: {e}")))?;
1054
1055    // The first literal '#' starts the fragment per RFC 3986 section 3.5.
1056    // A data '#' in an earlier component must be percent-encoded as "%23".
1057    let (base, fragment) = match url.split_once('#') {
1058        Some((base, fragment)) => (base, Some(fragment)),
1059        None => (url, None),
1060    };
1061
1062    let separator = if base.contains('?') { '&' } else { '?' };
1063
1064    Ok(Cow::Owned(match fragment {
1065        Some(fragment) => format!("{base}{separator}{query_string}#{fragment}"),
1066        None => format!("{base}{separator}{query_string}"),
1067    }))
1068}
1069
1070#[cfg(test)]
1071mod encode_url_params_tests {
1072    use std::{borrow::Cow, collections::HashMap};
1073
1074    use rstest::rstest;
1075
1076    use super::encode_url_params;
1077
1078    fn params(pairs: &[(&str, &str)]) -> HashMap<String, Vec<String>> {
1079        let mut map: HashMap<String, Vec<String>> = HashMap::new();
1080        for (key, value) in pairs {
1081            map.entry((*key).to_string())
1082                .or_default()
1083                .push((*value).to_string());
1084        }
1085
1086        map
1087    }
1088
1089    #[rstest]
1090    #[case("https://x/y", "https://x/y?a=b")]
1091    #[case("https://x/y?old=1", "https://x/y?old=1&a=b")]
1092    #[case("https://x/y#frag", "https://x/y?a=b#frag")]
1093    #[case("https://x/y?old=1#frag", "https://x/y?old=1&a=b#frag")]
1094    #[case(
1095        "https://x/y#section?display=full",
1096        "https://x/y?a=b#section?display=full"
1097    )]
1098    #[case("https://x/y#", "https://x/y?a=b#")]
1099    fn test_query_is_inserted_before_the_fragment(#[case] url: &str, #[case] expected: &str) {
1100        let params = params(&[("a", "b")]);
1101
1102        assert_eq!(encode_url_params(url, Some(&params)).unwrap(), expected);
1103    }
1104
1105    #[rstest]
1106    fn test_url_is_borrowed_when_no_params_are_supplied() {
1107        assert!(matches!(
1108            encode_url_params("https://x/y#frag", None).unwrap(),
1109            Cow::Borrowed("https://x/y#frag")
1110        ));
1111    }
1112
1113    #[rstest]
1114    fn test_url_is_borrowed_when_params_are_empty() {
1115        let params = HashMap::new();
1116
1117        assert!(matches!(
1118            encode_url_params("https://x/y#frag", Some(&params)).unwrap(),
1119            Cow::Borrowed("https://x/y#frag")
1120        ));
1121    }
1122}
1123
1124#[cfg(test)]
1125#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
1126#[cfg(not(all(feature = "simulation", madsim)))]
1127mod tests {
1128    use std::net::SocketAddr;
1129
1130    use axum::{
1131        Router,
1132        body::to_bytes,
1133        extract::Request,
1134        response::IntoResponse,
1135        routing::{any, delete, get, patch, post},
1136        serve,
1137    };
1138    use http::status::StatusCode;
1139    use log::Level;
1140    use rstest::rstest;
1141    use tokio::{
1142        io::{AsyncReadExt, AsyncWriteExt},
1143        sync::oneshot,
1144    };
1145
1146    use super::*;
1147    use crate::logging::tests::capture_logs;
1148
1149    #[rstest]
1150    fn test_debug_redacts_custom_default_headers() {
1151        let client = HttpClient::builder()
1152            .use_system_proxy(false)
1153            .headers(HashMap::from([(
1154                "kalshi-access-key".into(),
1155                "HEADER_DEBUG_SECRET".into(),
1156            )]))
1157            .build()
1158            .unwrap();
1159        let header = &client.client.headers["kalshi-access-key"];
1160        let debug = format!("{client:?}");
1161
1162        assert_eq!(header.to_str().unwrap(), "HEADER_DEBUG_SECRET");
1163        assert!(header.is_sensitive());
1164        assert!(!debug.contains("HEADER_DEBUG_SECRET"), "{debug}");
1165    }
1166
1167    #[rstest]
1168    #[case("ftp://127.0.0.1:1/resource")]
1169    #[case("file:///resource")]
1170    #[tokio::test]
1171    async fn test_request_rejects_unsupported_url(#[case] url: &str) {
1172        let client = HttpClient::builder()
1173            .use_system_proxy(false)
1174            .build()
1175            .unwrap();
1176
1177        let error = client
1178            .request(Method::GET, url.to_string(), None, None, None, None, None)
1179            .await
1180            .unwrap_err();
1181
1182        assert!(
1183            matches!(error, HttpClientError::Error(ref message) if message == "unsupported HTTP URL scheme or hostname")
1184        );
1185    }
1186
1187    #[rstest]
1188    #[case::username_only("user%2F17@", "Basic dXNlci8xNzo=")]
1189    #[case::password_only(":secret%2F29@", "Basic OnNlY3JldC8yOQ==")]
1190    #[tokio::test]
1191    async fn test_request_url_credentials_allow_missing_username_or_password(
1192        #[case] userinfo: &str,
1193        #[case] expected: &str,
1194    ) {
1195        let addr = start_test_server().await.unwrap();
1196        let client = HttpClient::builder()
1197            .use_system_proxy(false)
1198            .build()
1199            .unwrap();
1200
1201        let response = client
1202            .request(
1203                Method::GET,
1204                format!("http://{userinfo}{addr}/headers"),
1205                None,
1206                None,
1207                None,
1208                None,
1209                None,
1210            )
1211            .await
1212            .unwrap();
1213
1214        assert_eq!(response.status.as_u16(), 200);
1215        assert_eq!(response.headers, HashMap::new());
1216        assert_eq!(
1217            response.body.as_ref(),
1218            format!("{expected}\n*/*").as_bytes()
1219        );
1220    }
1221
1222    #[rstest]
1223    #[case::default(None, "*/*")]
1224    #[case::explicit(Some("application/octet-stream"), "application/octet-stream")]
1225    #[tokio::test]
1226    async fn test_request_accept_header_preserves_explicit_value(
1227        #[case] accept: Option<&str>,
1228        #[case] expected: &str,
1229    ) {
1230        let addr = start_test_server().await.unwrap();
1231        let client = HttpClient::builder()
1232            .use_system_proxy(false)
1233            .build()
1234            .unwrap();
1235        let headers =
1236            accept.map(|value| HashMap::from([("accept".to_string(), value.to_string())]));
1237
1238        let response = client
1239            .request(
1240                Method::GET,
1241                format!("http://{addr}/headers"),
1242                None,
1243                headers,
1244                None,
1245                None,
1246                None,
1247            )
1248            .await
1249            .unwrap();
1250
1251        assert_eq!(response.status.as_u16(), 200);
1252        assert_eq!(response.headers, HashMap::new());
1253        assert_eq!(
1254            response.body.as_ref(),
1255            format!("absent\n{expected}").as_bytes()
1256        );
1257    }
1258
1259    async fn capture_request(request: Request) -> impl IntoResponse {
1260        let (parts, body) = request.into_parts();
1261        let body = to_bytes(body, usize::MAX).await.unwrap();
1262        let default_header = parts.headers.get("x-default").unwrap().to_str().unwrap();
1263        let request_header = parts.headers.get("x-request").unwrap().to_str().unwrap();
1264        let query = parts.uri.query().unwrap_or_default();
1265        let body = String::from_utf8(body.to_vec()).unwrap();
1266        let capture = format!(
1267            "{}\n{}\n{query}\n{default_header}\n{request_header}\n{body}",
1268            parts.method,
1269            parts.uri.path(),
1270        );
1271
1272        ([("x-response-id", "response-42")], capture)
1273    }
1274
1275    fn create_router() -> Router {
1276        Router::new()
1277            .route("/get", get(|| async { "hello-world!" }))
1278            .route("/post", post(|body: Bytes| async move { body }))
1279            .route("/patch", patch(|body: Bytes| async move { body }))
1280            .route("/delete", delete(|| async { StatusCode::OK }))
1281            .route("/capture", any(capture_request))
1282            .route(
1283                "/headers",
1284                get(|request: Request| async move {
1285                    let headers = request.headers();
1286                    format!(
1287                        "{}\n{}",
1288                        headers
1289                            .get(http::header::AUTHORIZATION)
1290                            .map_or("absent", |v| v.to_str().unwrap()),
1291                        headers
1292                            .get(http::header::ACCEPT)
1293                            .map_or("absent", |v| v.to_str().unwrap()),
1294                    )
1295                }),
1296            )
1297            .route("/notfound", get(|| async { StatusCode::NOT_FOUND }))
1298            .route(
1299                "/redirect",
1300                get(|| async { (StatusCode::TEMPORARY_REDIRECT, [("location", "/get")]) }),
1301            )
1302            .route(
1303                "/slow",
1304                get(|| async {
1305                    tokio::time::sleep(Duration::from_secs(2)).await;
1306                    "Eventually responded"
1307                }),
1308            )
1309            .route(
1310                "/large",
1311                // Returns a 1 MiB body to exercise the response size cap.
1312                get(|| async { "x".repeat(1024 * 1024) }),
1313            )
1314    }
1315
1316    async fn start_test_server() -> Result<SocketAddr, Box<dyn std::error::Error + Send + Sync>> {
1317        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1318        let addr = listener.local_addr().unwrap();
1319        tokio::spawn(async move {
1320            serve(listener, create_router()).await.unwrap();
1321        });
1322
1323        Ok(addr)
1324    }
1325
1326    async fn spawn_connection_dropper() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1327        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1328        let addr = listener.local_addr().unwrap();
1329
1330        let task = tokio::spawn(async move {
1331            loop {
1332                let (stream, _) = listener.accept().await.unwrap();
1333                drop(stream);
1334            }
1335        });
1336
1337        (addr, task)
1338    }
1339
1340    async fn spawn_chunked_response_server() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1341        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1342        let addr = listener.local_addr().unwrap();
1343        let task = tokio::spawn(async move {
1344            let (mut stream, _) = listener.accept().await.unwrap();
1345            let mut request = Vec::new();
1346            let mut chunk = [0u8; 1024];
1347            loop {
1348                let read = stream.read(&mut chunk).await.unwrap();
1349                if read == 0 {
1350                    break;
1351                }
1352
1353                request.extend_from_slice(&chunk[..read]);
1354                if request.windows(4).any(|window| window == b"\r\n\r\n") {
1355                    break;
1356                }
1357            }
1358
1359            stream
1360                .write_all(
1361                    b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n\
1362                      5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n",
1363                )
1364                .await
1365                .unwrap();
1366        });
1367
1368        (addr, task)
1369    }
1370
1371    async fn spawn_rejecting_connect_proxy() -> (SocketAddr, oneshot::Receiver<String>) {
1372        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1373        let addr = listener.local_addr().unwrap();
1374        let (request_tx, request_rx) = oneshot::channel();
1375
1376        tokio::spawn(async move {
1377            let (mut stream, _) = listener.accept().await.unwrap();
1378            let mut request = Vec::new();
1379            let mut chunk = [0u8; 1024];
1380            loop {
1381                let read = stream.read(&mut chunk).await.unwrap();
1382                if read == 0 {
1383                    break;
1384                }
1385
1386                request.extend_from_slice(&chunk[..read]);
1387                if request.windows(4).any(|window| window == b"\r\n\r\n") {
1388                    break;
1389                }
1390            }
1391
1392            request_tx
1393                .send(String::from_utf8(request).unwrap())
1394                .unwrap();
1395            stream
1396                .write_all(
1397                    b"HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n",
1398                )
1399                .await
1400                .unwrap();
1401        });
1402
1403        (addr, request_rx)
1404    }
1405
1406    fn inner_client() -> InnerHttpClient {
1407        HttpClient::builder()
1408            .use_system_proxy(false)
1409            .build()
1410            .unwrap()
1411            .client
1412    }
1413
1414    #[tokio::test(start_paused = true)]
1415    async fn test_body_ready_at_deadline_is_rejected() {
1416        let client = inner_client();
1417        let response = http::Response::new(Full::new(Bytes::from_static(b"ready")));
1418        let result = client
1419            .consume_response(response, Some(crate::dst::time::Instant::now()), false)
1420            .await;
1421        assert!(
1422            matches!(result, Err(HttpClientError::TimeoutError(message)) if message == REQUEST_TIMEOUT_MESSAGE)
1423        );
1424    }
1425
1426    #[tokio::test]
1427    async fn test_get() {
1428        let addr = start_test_server().await.unwrap();
1429        let url = format!("http://{addr}");
1430
1431        let client = inner_client();
1432        let response = client
1433            .send_request(Method::GET, format!("{url}/get"), None, None, None, None)
1434            .await
1435            .unwrap();
1436
1437        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1438        assert_eq!(response.headers, HashMap::new());
1439        assert_eq!(response.body.as_ref(), b"hello-world!");
1440    }
1441
1442    #[rstest]
1443    #[tokio::test]
1444    async fn test_request_preserves_wire_semantics_and_extracts_response_headers(
1445        #[values(false, true)] redact_url: bool,
1446    ) {
1447        let addr = start_test_server().await.unwrap();
1448        let mut default_headers = HashMap::new();
1449        default_headers.insert("x-default".to_string(), "default-a".to_string());
1450        let client = HttpClient::builder()
1451            .redact_url(redact_url)
1452            .headers(default_headers)
1453            .header_keys(vec!["x-response-id".to_string()])
1454            .build()
1455            .unwrap();
1456        let mut params = HashMap::new();
1457        params.insert(
1458            "tag".to_string(),
1459            vec!["A B".to_string(), "C/D".to_string()],
1460        );
1461        let mut request_headers = HashMap::new();
1462        request_headers.insert("x-request".to_string(), "request-b".to_string());
1463
1464        let response = client
1465            .request(
1466                Method::PUT,
1467                format!("http://{addr}/capture?existing=seed"),
1468                Some(&params),
1469                Some(request_headers),
1470                Some(b"payload-c".to_vec()),
1471                None,
1472                None,
1473            )
1474            .await
1475            .unwrap();
1476
1477        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1478        assert_eq!(
1479            response.headers,
1480            HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1481        );
1482        assert_eq!(
1483            response.body.as_ref(),
1484            b"PUT\n/capture\nexisting=seed&tag=A+B&tag=C%2FD\ndefault-a\nrequest-b\npayload-c"
1485        );
1486    }
1487
1488    #[rstest]
1489    #[tokio::test]
1490    async fn test_request_with_secret_body_preserves_wire_body(
1491        #[values(false, true)] redact_url: bool,
1492    ) {
1493        let addr = start_test_server().await.unwrap();
1494        let client = HttpClient::builder()
1495            .redact_url(redact_url)
1496            .headers(HashMap::from([(
1497                "x-default".to_string(),
1498                "default-secret".to_string(),
1499            )]))
1500            .build()
1501            .unwrap();
1502        let headers = HashMap::from([("x-request".to_string(), "request-secret".to_string())]);
1503
1504        let response = client
1505            .request_with_secret_body(
1506                Method::POST,
1507                format!("http://{addr}/capture"),
1508                None,
1509                Some(headers),
1510                SecretString::from("credential-body"),
1511                None,
1512                None,
1513            )
1514            .await
1515            .unwrap();
1516
1517        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1518        assert_eq!(response.headers, HashMap::new());
1519        assert_eq!(
1520            response.body.as_ref(),
1521            b"POST\n/capture\n\ndefault-secret\nrequest-secret\ncredential-body"
1522        );
1523    }
1524
1525    #[rstest]
1526    #[tokio::test]
1527    async fn test_request_with_params_serializes_query_fields(
1528        #[values(false, true)] redact_url: bool,
1529    ) {
1530        #[derive(serde::Serialize)]
1531        struct Query<'a> {
1532            symbol: &'a str,
1533            limit: u32,
1534        }
1535
1536        let addr = start_test_server().await.unwrap();
1537        let mut default_headers = HashMap::new();
1538        default_headers.insert("x-default".to_string(), "default-d".to_string());
1539        let client = HttpClient::builder()
1540            .redact_url(redact_url)
1541            .headers(default_headers)
1542            .header_keys(vec!["x-response-id".to_string()])
1543            .build()
1544            .unwrap();
1545        let mut request_headers = HashMap::new();
1546        request_headers.insert("x-request".to_string(), "request-e".to_string());
1547
1548        let params = Query {
1549            symbol: "BTC/USDT",
1550            limit: 37,
1551        };
1552
1553        let response = client
1554            .request_with_params(
1555                Method::GET,
1556                format!("http://{addr}/capture"),
1557                Some(&params),
1558                Some(request_headers),
1559                None,
1560                None,
1561                None,
1562            )
1563            .await
1564            .unwrap();
1565
1566        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1567        assert_eq!(
1568            response.headers,
1569            HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1570        );
1571        assert_eq!(
1572            response.body.as_ref(),
1573            b"GET\n/capture\nsymbol=BTC%2FUSDT&limit=37\ndefault-d\nrequest-e\n"
1574        );
1575    }
1576
1577    #[rstest]
1578    #[tokio::test]
1579    async fn test_request_with_params_preserves_secret_query_fields(
1580        #[values(false, true)] redact_url: bool,
1581    ) {
1582        #[derive(serde::Serialize)]
1583        struct Query<'a> {
1584            auth: &'a str,
1585            market_id: i16,
1586        }
1587
1588        let addr = start_test_server().await.unwrap();
1589        let client = HttpClient::builder()
1590            .redact_url(redact_url)
1591            .headers(HashMap::from([(
1592                "x-default".to_string(),
1593                "default-f".to_string(),
1594            )]))
1595            .build()
1596            .unwrap();
1597        let headers = HashMap::from([("x-request".to_string(), "request-g".to_string())]);
1598
1599        let params = Query {
1600            auth: "token/42",
1601            market_id: 7,
1602        };
1603
1604        let response = client
1605            .request_with_params(
1606                Method::GET,
1607                format!("http://{addr}/capture"),
1608                Some(&params),
1609                Some(headers),
1610                None,
1611                None,
1612                None,
1613            )
1614            .await
1615            .unwrap();
1616
1617        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1618        assert_eq!(
1619            response.body.as_ref(),
1620            b"GET\n/capture\nauth=token%2F42&market_id=7\ndefault-f\nrequest-g\n"
1621        );
1622    }
1623
1624    #[rstest]
1625    #[case::empty_at_zero_cap(b"", 0)]
1626    #[case::at_cap(b"body-37", 7)]
1627    #[case::below_cap(b"body-37", 8)]
1628    #[tokio::test]
1629    async fn test_declared_response_body_at_or_below_cap_is_returned(
1630        #[case] bytes: &'static [u8],
1631        #[case] max_response_bytes: usize,
1632    ) {
1633        let client = InnerHttpClient {
1634            max_response_bytes,
1635            ..inner_client()
1636        };
1637
1638        let response = http::Response::new(Full::new(Bytes::from_static(bytes)));
1639
1640        let response = client
1641            .consume_response(response, None, false)
1642            .await
1643            .unwrap();
1644
1645        assert_eq!(response.status.as_u16(), 200);
1646        assert_eq!(response.headers, HashMap::new());
1647        assert_eq!(response.body.as_ref(), bytes);
1648    }
1649
1650    #[tokio::test]
1651    async fn test_response_body_within_cap_is_returned() {
1652        let addr = start_test_server().await.unwrap();
1653        let url = format!("http://{addr}");
1654
1655        // Cap above the 1 MiB payload: body should be returned intact.
1656        let client = InnerHttpClient {
1657            max_response_bytes: 4 * 1024 * 1024,
1658            ..inner_client()
1659        };
1660
1661        let response = client
1662            .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1663            .await
1664            .unwrap();
1665
1666        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1667        assert_eq!(response.headers, HashMap::new());
1668        assert_eq!(response.body.as_ref(), vec![b'x'; 1024 * 1024]);
1669    }
1670
1671    #[tokio::test]
1672    async fn test_response_body_exceeding_cap_is_rejected() {
1673        let addr = start_test_server().await.unwrap();
1674        let url = format!("http://{addr}");
1675
1676        // Cap below the 1 MiB payload: the request must fail rather than buffer it.
1677        let client = InnerHttpClient {
1678            max_response_bytes: 16 * 1024,
1679            ..inner_client()
1680        };
1681
1682        let result = client
1683            .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1684            .await;
1685
1686        let err = result.expect_err("oversized response body should be rejected");
1687
1688        let HttpClientError::Error(message) = err else {
1689            panic!("expected HTTP error, was {err:?}");
1690        };
1691
1692        assert_eq!(
1693            message,
1694            "HTTP response body of 1048576 bytes exceeds maximum of 16384 bytes"
1695        );
1696    }
1697
1698    #[rstest]
1699    #[case::at_cap(11)]
1700    #[case::below_cap(12)]
1701    #[tokio::test]
1702    async fn test_chunked_response_body_at_or_below_cap_is_returned(
1703        #[case] max_response_bytes: usize,
1704    ) {
1705        let (addr, server_task) = spawn_chunked_response_server().await;
1706
1707        let client = InnerHttpClient {
1708            max_response_bytes,
1709            ..inner_client()
1710        };
1711
1712        let response = client
1713            .send_request(
1714                Method::GET,
1715                format!("http://{addr}"),
1716                None,
1717                None,
1718                None,
1719                None,
1720            )
1721            .await
1722            .unwrap();
1723        server_task.await.unwrap();
1724
1725        assert_eq!(response.status.as_u16(), 200);
1726        assert_eq!(response.headers, HashMap::new());
1727        assert_eq!(response.body.as_ref(), b"firstsecond");
1728    }
1729
1730    #[tokio::test]
1731    async fn test_chunked_response_body_exceeding_cap_is_rejected() {
1732        let (addr, server_task) = spawn_chunked_response_server().await;
1733        let max_response_bytes = 8;
1734
1735        let client = InnerHttpClient {
1736            max_response_bytes,
1737            ..inner_client()
1738        };
1739
1740        let error = client
1741            .send_request(
1742                Method::GET,
1743                format!("http://{addr}"),
1744                None,
1745                None,
1746                None,
1747                None,
1748            )
1749            .await
1750            .expect_err("chunked response body should be rejected");
1751        server_task.await.unwrap();
1752
1753        let HttpClientError::Error(message) = error else {
1754            panic!("expected HTTP error, was {error:?}");
1755        };
1756
1757        assert_eq!(
1758            message,
1759            format!("HTTP response body exceeds maximum of {max_response_bytes} bytes")
1760        );
1761    }
1762
1763    #[tokio::test]
1764    async fn test_post() {
1765        let addr = start_test_server().await.unwrap();
1766        let url = format!("http://{addr}");
1767
1768        let client = inner_client();
1769        let response = client
1770            .send_request(Method::POST, format!("{url}/post"), None, None, None, None)
1771            .await
1772            .unwrap();
1773
1774        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1775        assert_eq!(response.headers, HashMap::new());
1776        assert_eq!(response.body.as_ref(), b"");
1777    }
1778
1779    #[tokio::test]
1780    async fn test_post_with_body() {
1781        let addr = start_test_server().await.unwrap();
1782        let url = format!("http://{addr}");
1783
1784        let client = inner_client();
1785
1786        let mut body = HashMap::new();
1787        body.insert(
1788            "key1".to_string(),
1789            serde_json::Value::String("value1".to_string()),
1790        );
1791        body.insert(
1792            "key2".to_string(),
1793            serde_json::Value::String("value2".to_string()),
1794        );
1795
1796        let body_string = serde_json::to_string(&body).unwrap();
1797        let body_bytes = body_string.into_bytes();
1798
1799        let response = client
1800            .send_request(
1801                Method::POST,
1802                format!("{url}/post"),
1803                None,
1804                None,
1805                Some(body_bytes.clone()),
1806                None,
1807            )
1808            .await
1809            .unwrap();
1810
1811        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1812        assert_eq!(response.headers, HashMap::new());
1813        assert_eq!(response.body.as_ref(), body_bytes);
1814    }
1815
1816    #[tokio::test]
1817    async fn test_patch() {
1818        let addr = start_test_server().await.unwrap();
1819        let url = format!("http://{addr}");
1820
1821        let client = inner_client();
1822        let response = client
1823            .send_request(
1824                Method::PATCH,
1825                format!("{url}/patch"),
1826                None,
1827                None,
1828                None,
1829                None,
1830            )
1831            .await
1832            .unwrap();
1833
1834        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1835        assert_eq!(response.headers, HashMap::new());
1836        assert_eq!(response.body.as_ref(), b"");
1837    }
1838
1839    #[tokio::test]
1840    async fn test_delete() {
1841        let addr = start_test_server().await.unwrap();
1842        let url = format!("http://{addr}");
1843
1844        let client = inner_client();
1845        let response = client
1846            .send_request(
1847                Method::DELETE,
1848                format!("{url}/delete"),
1849                None,
1850                None,
1851                None,
1852                None,
1853            )
1854            .await
1855            .unwrap();
1856
1857        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1858        assert_eq!(response.headers, HashMap::new());
1859        assert_eq!(response.body.as_ref(), b"");
1860    }
1861
1862    #[tokio::test]
1863    async fn test_not_found() {
1864        let addr = start_test_server().await.unwrap();
1865        let url = format!("http://{addr}/notfound");
1866        let client = inner_client();
1867
1868        let response = client
1869            .send_request(Method::GET, url, None, None, None, None)
1870            .await
1871            .unwrap();
1872
1873        assert!(response.status.is_client_error());
1874        assert_eq!(response.status.as_u16(), 404);
1875        assert_eq!(response.headers, HashMap::new());
1876        assert_eq!(response.body.as_ref(), b"");
1877    }
1878
1879    #[tokio::test]
1880    async fn test_timeout() {
1881        let addr = start_test_server().await.unwrap();
1882        let url = format!("http://{addr}/slow");
1883        let client = inner_client();
1884
1885        // We'll set a 1-second timeout for a route that sleeps 2 seconds
1886        let result = client
1887            .send_request(Method::GET, url, None, None, None, Some(1))
1888            .await;
1889
1890        assert!(
1891            matches!(&result, Err(HttpClientError::TimeoutError(_))),
1892            "Expected a timeout error, was: {result:?}"
1893        );
1894    }
1895
1896    #[rstest]
1897    fn test_http_client_without_proxy() {
1898        // Create client with no proxy
1899        let result = HttpClient::builder().build();
1900
1901        assert!(result.is_ok());
1902    }
1903
1904    #[rstest]
1905    fn test_http_client_rejects_timeout_overflow() {
1906        let result = HttpClient::builder().timeout_secs(u64::MAX).build();
1907
1908        assert!(matches!(
1909            result,
1910            Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1911        ));
1912    }
1913
1914    #[tokio::test]
1915    async fn test_http_request_rejects_timeout_override_overflow() {
1916        let client = HttpClient::builder()
1917            .use_system_proxy(false)
1918            .timeout_secs(1)
1919            .build()
1920            .unwrap();
1921        let result = client
1922            .request(
1923                Method::GET,
1924                "http://127.0.0.1:1/".to_string(),
1925                None,
1926                None,
1927                None,
1928                Some(u64::MAX),
1929                None,
1930            )
1931            .await;
1932
1933        assert!(matches!(
1934            result,
1935            Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1936        ));
1937    }
1938
1939    #[rstest]
1940    fn test_http_client_builder_preserves_empty_rate_limiters() {
1941        let client = HttpClient::builder()
1942            .rate_limiters(Vec::new())
1943            .build()
1944            .unwrap();
1945
1946        assert!(client.rate_limiters.is_empty());
1947    }
1948
1949    #[rstest]
1950    fn test_http_client_builder_rejects_shared_rate_limiters_with_quotas() {
1951        let quota = Quota::with_period(Duration::from_secs(1)).unwrap();
1952        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
1953        let result = HttpClient::builder()
1954            .default_quota(quota)
1955            .rate_limiters(vec![rate_limiter])
1956            .build();
1957
1958        assert_eq!(
1959            result.unwrap_err().to_string(),
1960            "HTTP error occurred: Cannot combine shared rate limiters with quota configuration"
1961        );
1962    }
1963
1964    #[tokio::test]
1965    async fn test_http_client_without_proxy_requests_directly() {
1966        let addr = start_test_server().await.unwrap();
1967        let client = HttpClient::builder().timeout_secs(2).build().unwrap();
1968        let response = client
1969            .request(
1970                Method::GET,
1971                format!("http://{addr}/get"),
1972                None,
1973                None,
1974                None,
1975                None,
1976                None,
1977            )
1978            .await
1979            .expect("direct request");
1980
1981        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1982        assert_eq!(response.body.as_ref(), b"hello-world!");
1983    }
1984
1985    #[tokio::test]
1986    async fn test_http_client_redirect_policy() {
1987        let addr = start_test_server().await.unwrap();
1988        let follow = HttpClient::builder().timeout_secs(2).build().unwrap();
1989        let reject = HttpClient::builder()
1990            .timeout_secs(2)
1991            .redirect_policy(HttpRedirectPolicy::Reject)
1992            .build()
1993            .unwrap();
1994
1995        let followed = follow
1996            .request(
1997                Method::GET,
1998                format!("http://{addr}/redirect"),
1999                None,
2000                None,
2001                None,
2002                None,
2003                None,
2004            )
2005            .await
2006            .unwrap();
2007        let rejected = reject
2008            .request(
2009                Method::GET,
2010                format!("http://{addr}/redirect"),
2011                None,
2012                None,
2013                None,
2014                None,
2015                None,
2016            )
2017            .await
2018            .unwrap();
2019
2020        assert_eq!(followed.status.as_u16(), StatusCode::OK.as_u16());
2021        assert_eq!(followed.body.as_ref(), b"hello-world!");
2022        assert_eq!(
2023            rejected.status.as_u16(),
2024            StatusCode::TEMPORARY_REDIRECT.as_u16()
2025        );
2026        assert!(rejected.body.is_empty());
2027    }
2028
2029    #[rstest]
2030    #[tokio::test]
2031    async fn test_http_client_url_policy_preserves_response(
2032        #[values(false, true)] redact_url: bool,
2033    ) {
2034        let addr = start_test_server().await.unwrap();
2035        let client = HttpClient::builder()
2036            .redact_url(redact_url)
2037            .timeout_secs(2)
2038            .build()
2039            .unwrap();
2040        let response = client
2041            .request(
2042                Method::GET,
2043                format!("http://{addr}/get"),
2044                None,
2045                None,
2046                None,
2047                None,
2048                None,
2049            )
2050            .await
2051            .expect("direct request with URL redaction");
2052
2053        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2054        assert_eq!(response.body.as_ref(), b"hello-world!");
2055    }
2056
2057    #[rstest]
2058    #[case::request("request")]
2059    #[case::preinterned_keys("keys")]
2060    #[case::get("get")]
2061    #[case::post("post")]
2062    #[case::patch("patch")]
2063    #[case::delete("delete")]
2064    #[case::stream("stream")]
2065    #[tokio::test]
2066    async fn test_http_client_url_policy_redacts_transport_errors(
2067        #[case] request: &str,
2068        #[values(false, true)] redact_url: bool,
2069    ) {
2070        const USERINFO_SECRET: &str = "transport-userinfo-secret";
2071        const PATH_SECRET: &str = "transport-path-secret";
2072        const QUERY_SECRET: &str = "transport-query-secret";
2073        let (addr, drop_task) = spawn_connection_dropper().await;
2074        let url = format!(
2075            "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2076        );
2077        let client = HttpClient::builder()
2078            .redact_url(redact_url)
2079            .timeout_secs(1)
2080            .build()
2081            .unwrap();
2082
2083        let error = match request {
2084            "request" => client
2085                .request(Method::GET, url.clone(), None, None, None, None, None)
2086                .await
2087                .unwrap_err(),
2088            "keys" => client
2089                .request_with_ustr_keys(
2090                    Method::GET,
2091                    url.clone(),
2092                    None,
2093                    None,
2094                    None,
2095                    None,
2096                    Some(vec![Ustr::from("policy")]),
2097                )
2098                .await
2099                .unwrap_err(),
2100            "get" => client
2101                .get(url.clone(), None, None, None, None)
2102                .await
2103                .unwrap_err(),
2104            "post" => client
2105                .post(url.clone(), None, None, None, None, None)
2106                .await
2107                .unwrap_err(),
2108            "patch" => client
2109                .patch(url.clone(), None, None, None, None, None)
2110                .await
2111                .unwrap_err(),
2112            "delete" => client
2113                .delete(url.clone(), None, None, None, None)
2114                .await
2115                .unwrap_err(),
2116            "stream" => client.get_stream(url.clone()).await.unwrap_err(),
2117            _ => panic!("unknown request shape"),
2118        };
2119
2120        drop_task.abort();
2121        let task_error = drop_task
2122            .await
2123            .expect_err("connection dropper should be cancelled");
2124
2125        assert!(task_error.is_cancelled());
2126        assert!(
2127            matches!(error, HttpClientError::TransportError(_)),
2128            "{error:?}"
2129        );
2130
2131        for rendered in [error.to_string(), format!("{error:?}")] {
2132            assert!(!rendered.contains(USERINFO_SECRET));
2133            assert!(!rendered.contains(PATH_SECRET));
2134            assert!(!rendered.contains(QUERY_SECRET));
2135            assert!(!rendered.contains(&url));
2136            assert_eq!(
2137                rendered.contains(&format!("http://{addr}")),
2138                !redact_url,
2139                "{rendered}"
2140            );
2141            assert_eq!(rendered.contains("for origin ("), !redact_url, "{rendered}");
2142        }
2143    }
2144
2145    #[tokio::test]
2146    async fn test_request_with_params_redact_url_removes_origin_from_error() {
2147        const QUERY_SECRET: &str = "transport-query-secret";
2148        #[derive(serde::Serialize)]
2149        struct Query<'a> {
2150            auth: &'a str,
2151        }
2152
2153        let (addr, drop_task) = spawn_connection_dropper().await;
2154        let url = format!("http://{addr}/trades");
2155        let params = Query { auth: QUERY_SECRET };
2156        let client = HttpClient::builder()
2157            .redact_url(true)
2158            .timeout_secs(1)
2159            .build()
2160            .unwrap();
2161
2162        let error = client
2163            .request_with_params(Method::GET, url, Some(&params), None, None, None, None)
2164            .await
2165            .expect_err("a dropped connection should fail");
2166        drop_task.abort();
2167        let task_error = drop_task
2168            .await
2169            .expect_err("connection dropper should be cancelled");
2170
2171        assert!(task_error.is_cancelled());
2172
2173        for rendered in [error.to_string(), format!("{error:?}")] {
2174            assert!(!rendered.contains(&addr.to_string()));
2175            assert!(!rendered.contains("auth="));
2176            assert!(!rendered.contains(QUERY_SECRET));
2177        }
2178    }
2179
2180    #[tokio::test]
2181    async fn test_request_removes_query_string_from_transport_error_by_default() {
2182        const QUERY_SECRET: &str = "default-query-secret";
2183        const FRAGMENT_MARKER: &str = "default-fragment-marker";
2184        let (addr, drop_task) = spawn_connection_dropper().await;
2185        let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}#{FRAGMENT_MARKER}");
2186        let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2187
2188        let error = client
2189            .request(Method::GET, url, None, None, None, None, None)
2190            .await
2191            .expect_err("a dropped connection should fail");
2192        drop_task.abort();
2193        let task_error = drop_task
2194            .await
2195            .expect_err("connection dropper should be cancelled");
2196
2197        assert!(task_error.is_cancelled());
2198
2199        for rendered in [error.to_string(), format!("{error:?}")] {
2200            assert!(
2201                rendered.contains(&format!("for origin (http://{addr})")),
2202                "default error omitted the origin: {rendered}"
2203            );
2204            assert!(!rendered.contains("/trades"), "{rendered}");
2205            assert!(!rendered.contains("api_key="), "{rendered}");
2206            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2207            assert!(!rendered.contains(FRAGMENT_MARKER), "{rendered}");
2208        }
2209    }
2210
2211    #[rstest]
2212    #[tokio::test]
2213    async fn test_request_with_secret_body_url_policy_redacts_transport_error(
2214        #[values(false, true)] redact_url: bool,
2215    ) {
2216        const QUERY_SECRET: &str = "secret-body-query-secret";
2217        let (addr, drop_task) = spawn_connection_dropper().await;
2218        let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}");
2219        let client = HttpClient::builder()
2220            .redact_url(redact_url)
2221            .timeout_secs(1)
2222            .build()
2223            .unwrap();
2224
2225        let error = client
2226            .request_with_secret_body(
2227                Method::POST,
2228                url,
2229                None,
2230                None,
2231                SecretString::from("credential-body"),
2232                None,
2233                None,
2234            )
2235            .await
2236            .expect_err("a dropped connection should fail");
2237        drop_task.abort();
2238        let task_error = drop_task
2239            .await
2240            .expect_err("connection dropper should be cancelled");
2241
2242        assert!(task_error.is_cancelled());
2243
2244        for rendered in [error.to_string(), format!("{error:?}")] {
2245            assert_eq!(
2246                rendered.contains(&format!("for origin (http://{addr})")),
2247                !redact_url,
2248                "{rendered}"
2249            );
2250            assert_eq!(
2251                rendered.contains(&addr.to_string()),
2252                !redact_url,
2253                "{rendered}"
2254            );
2255            assert!(!rendered.contains("/trades"), "{rendered}");
2256            assert!(!rendered.contains("credential-body"), "{rendered}");
2257            assert!(!rendered.contains("api_key="), "{rendered}");
2258            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2259        }
2260    }
2261
2262    #[tokio::test]
2263    async fn test_request_with_params_removes_query_from_transport_error_by_default() {
2264        const QUERY_SECRET: &str = "default-params-query-secret";
2265        #[derive(serde::Serialize)]
2266        struct Query<'a> {
2267            auth: &'a str,
2268        }
2269
2270        let (addr, drop_task) = spawn_connection_dropper().await;
2271        let url = format!("http://{addr}/trades");
2272        let params = Query { auth: QUERY_SECRET };
2273        let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2274
2275        let error = client
2276            .request_with_params(Method::GET, url, Some(&params), None, None, None, None)
2277            .await
2278            .expect_err("a dropped connection should fail");
2279        drop_task.abort();
2280        let task_error = drop_task
2281            .await
2282            .expect_err("connection dropper should be cancelled");
2283
2284        assert!(task_error.is_cancelled());
2285
2286        for rendered in [error.to_string(), format!("{error:?}")] {
2287            assert!(
2288                rendered.contains(&format!("for origin (http://{addr})")),
2289                "default error omitted the origin: {rendered}"
2290            );
2291            assert!(!rendered.contains("/trades"), "{rendered}");
2292            assert!(!rendered.contains("auth="), "{rendered}");
2293            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2294        }
2295    }
2296
2297    #[rstest]
2298    #[tokio::test]
2299    async fn test_http_client_url_policy_removes_endpoint_from_trace_logs(
2300        #[values(false, true)] redact_url: bool,
2301    ) {
2302        const USERINFO_SECRET: &str = "trace-userinfo-secret";
2303        const PATH_SECRET: &str = "trace-path-secret";
2304        const QUERY_SECRET: &str = "trace-query-secret";
2305        let capture = capture_logs().await;
2306        let addr = start_test_server().await.unwrap();
2307        let url = format!(
2308            "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2309        );
2310        let client = HttpClient::builder()
2311            .redact_url(redact_url)
2312            .timeout_secs(2)
2313            .build()
2314            .unwrap();
2315
2316        let response = client
2317            .request(Method::GET, url.clone(), None, None, None, None, None)
2318            .await
2319            .expect("credentialized endpoint should return an HTTP response");
2320        let messages = capture.messages();
2321
2322        assert_eq!(response.status.as_u16(), StatusCode::NOT_FOUND.as_u16());
2323        assert!(messages.iter().any(|(level, message)| {
2324            *level == Level::Trace && message.starts_with("Sending HTTP request: method=GET")
2325        }));
2326        assert!(messages.iter().any(|(level, message)| {
2327            *level == Level::Trace
2328                && message.starts_with("Received HTTP response: status=404 Not Found")
2329        }));
2330
2331        for (_, message) in messages {
2332            assert!(!message.contains(USERINFO_SECRET));
2333            assert!(!message.contains(PATH_SECRET));
2334            assert!(!message.contains(QUERY_SECRET));
2335            assert!(!message.contains(&url));
2336        }
2337    }
2338
2339    #[rstest]
2340    #[case::without_user_agent(None)]
2341    #[case::with_user_agent(Some("NautilusTrader/proxy-test-73"))]
2342    #[tokio::test]
2343    async fn test_http_client_uses_connect_and_proxy_authorization_for_https(
2344        #[case] user_agent: Option<&str>,
2345    ) {
2346        const USERNAME: &str = "proxytest";
2347        const PASSWORD: &str = "fixture42";
2348        let mut headers =
2349            HashMap::from([("authorization".into(), "Bearer origin-secret-19".into())]);
2350
2351        if let Some(user_agent) = user_agent {
2352            headers.insert("user-agent".into(), user_agent.into());
2353        }
2354
2355        let (proxy_addr, request_rx) = spawn_rejecting_connect_proxy().await;
2356        let client = HttpClient::builder()
2357            .headers(headers)
2358            .timeout_secs(2)
2359            .proxy_url(format!("http://{USERNAME}:{PASSWORD}@{proxy_addr}"))
2360            .build()
2361            .unwrap();
2362        let error = client
2363            .request(
2364                Method::GET,
2365                "https://fixture.example.test/path".to_string(),
2366                None,
2367                None,
2368                None,
2369                None,
2370                None,
2371            )
2372            .await
2373            .expect_err("proxy should reject CONNECT");
2374        let request = request_rx.await.expect("captured CONNECT request");
2375        let mut lines = request.split("\r\n");
2376        let request_line = lines.next().expect("CONNECT request line");
2377
2378        let headers: HashMap<_, _> = lines
2379            .filter_map(|line| {
2380                let (name, value) = line.split_once(':')?;
2381                Some((name.to_ascii_lowercase(), value.trim()))
2382            })
2383            .collect();
2384
2385        let expected_auth = format!("Basic {}", BASE64.encode(format!("{USERNAME}:{PASSWORD}")));
2386        let mut expected_headers = HashMap::from([
2387            ("host".into(), "fixture.example.test:443"),
2388            ("proxy-authorization".into(), expected_auth.as_str()),
2389        ]);
2390
2391        if let Some(user_agent) = user_agent {
2392            expected_headers.insert("user-agent".into(), user_agent);
2393        }
2394
2395        assert_eq!(request_line, "CONNECT fixture.example.test:443 HTTP/1.1");
2396        assert_eq!(headers, expected_headers);
2397        assert!(!error.to_string().contains(PASSWORD));
2398        assert!(!error.to_string().contains(&BASE64.encode(PASSWORD)));
2399        assert!(!error.to_string().contains(&expected_auth));
2400    }
2401
2402    #[rstest]
2403    #[case::cross_origin("http://127.0.0.1:2/final", None, false)]
2404    #[case::cross_origin_override("http://127.0.0.1:2/final", Some("Basic b3ZlcnJpZGU="), false)]
2405    #[case::same_origin("http://127.0.0.1:1/final", None, true)]
2406    #[case::same_origin_override("http://127.0.0.1:1/final", Some("Basic b3ZlcnJpZGU="), true)]
2407    #[case::https_connect("https://127.0.0.1:2/final", None, false)]
2408    #[tokio::test]
2409    async fn test_proxy_authentication_per_redirect_hop(
2410        #[case] destination: &'static str,
2411        #[case] initial_auth: Option<&'static str>,
2412        #[case] same_origin: bool,
2413    ) {
2414        const PROXY_AUTH: &str = "Basic cHJveHl0ZXN0OmZpeHR1cmU0Mg==";
2415
2416        let expected_initial_auth = initial_auth.unwrap_or(PROXY_AUTH);
2417        let expected_redirect_auth = if same_origin {
2418            expected_initial_auth
2419        } else {
2420            PROXY_AUTH
2421        };
2422
2423        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2424        let addr = listener.local_addr().unwrap();
2425        let (request_tx, mut request_rx) = tokio::sync::mpsc::channel(2);
2426        let router = Router::new().fallback(any(move |request: Request| {
2427            let request_tx = request_tx.clone();
2428
2429            async move {
2430                let (parts, _) = request.into_parts();
2431                let first = parts.uri.path() == "/start";
2432
2433                let expected_auth = if first {
2434                    expected_initial_auth
2435                } else {
2436                    expected_redirect_auth
2437                };
2438
2439                let authenticated = parts.headers.get(http::header::PROXY_AUTHORIZATION)
2440                    == Some(&http::HeaderValue::from_static(expected_auth));
2441                let connect = parts.method == http::Method::CONNECT;
2442                request_tx.send(parts).await.unwrap();
2443
2444                if !authenticated || connect {
2445                    return StatusCode::PROXY_AUTHENTICATION_REQUIRED.into_response();
2446                }
2447
2448                if first {
2449                    return (StatusCode::TEMPORARY_REDIRECT, [("location", destination)])
2450                        .into_response();
2451                }
2452
2453                "accepted".into_response()
2454            }
2455        }));
2456
2457        let server = tokio::spawn(async move { serve(listener, router).await.unwrap() });
2458        let origin_headers = HashMap::from([
2459            ("authorization".to_string(), "Bearer origin-19".to_string()),
2460            ("cookie".to_string(), "session=origin-23".to_string()),
2461            ("cookie2".to_string(), "legacy=origin-29".to_string()),
2462            (
2463                "www-authenticate".to_string(),
2464                "Basic realm=origin-31".to_string(),
2465            ),
2466        ]);
2467        let mut headers = origin_headers.clone();
2468        if let Some(initial_auth) = initial_auth {
2469            headers.insert("proxy-authorization".to_string(), initial_auth.to_string());
2470        }
2471
2472        let client = HttpClient::builder()
2473            .headers(headers)
2474            .timeout_secs(2)
2475            .use_system_proxy(false)
2476            .proxy_url(format!("http://proxytest:fixture42@{addr}"))
2477            .build()
2478            .unwrap();
2479        let result = client
2480            .request(
2481                Method::GET,
2482                "http://127.0.0.1:1/start".to_string(),
2483                None,
2484                None,
2485                None,
2486                None,
2487                None,
2488            )
2489            .await;
2490        let first = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2491            .await
2492            .unwrap()
2493            .unwrap();
2494        let redirected = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2495            .await
2496            .unwrap()
2497            .unwrap();
2498        server.abort();
2499        let connect = destination.starts_with("https:");
2500
2501        assert_eq!(first.method, http::Method::GET);
2502        assert_eq!(first.uri.to_string(), "http://127.0.0.1:1/start");
2503        assert_eq!(
2504            first.headers[http::header::PROXY_AUTHORIZATION],
2505            expected_initial_auth
2506        );
2507        assert_eq!(
2508            redirected.headers[http::header::PROXY_AUTHORIZATION],
2509            expected_redirect_auth
2510        );
2511
2512        for (name, value) in origin_headers {
2513            assert_eq!(first.headers.get(&name).unwrap(), value.as_str());
2514            assert_eq!(
2515                redirected
2516                    .headers
2517                    .get(&name)
2518                    .map(|value| value.to_str().unwrap()),
2519                same_origin.then_some(value.as_str())
2520            );
2521        }
2522
2523        if connect {
2524            assert_eq!(redirected.method, http::Method::CONNECT);
2525            assert_eq!(redirected.uri.to_string(), "127.0.0.1:2");
2526            assert!(result.is_err());
2527        } else {
2528            let response = result.unwrap();
2529            assert_eq!(redirected.method, http::Method::GET);
2530            assert_eq!(redirected.uri.to_string(), destination);
2531            assert_eq!(response.status.as_u16(), 200);
2532            assert_eq!(response.body.as_ref(), b"accepted");
2533        }
2534    }
2535
2536    #[tokio::test]
2537    async fn test_http_client_unreachable_proxy_error_redacts_credentials() {
2538        const USERNAME: &str = "proxy-user";
2539        const SECRET: &str = "unreachable-proxy-secret";
2540        let (proxy_addr, drop_task) = spawn_connection_dropper().await;
2541        let client = HttpClient::builder()
2542            .timeout_secs(1)
2543            .proxy_url(format!("http://{USERNAME}:{SECRET}@{proxy_addr}"))
2544            .build()
2545            .unwrap();
2546        let error = client
2547            .request(
2548                Method::GET,
2549                "https://fixture.example.test/".to_string(),
2550                None,
2551                None,
2552                None,
2553                None,
2554                None,
2555            )
2556            .await
2557            .expect_err("unreachable proxy should fail");
2558        drop_task.abort();
2559        let task_error = drop_task
2560            .await
2561            .expect_err("connection dropper should be cancelled");
2562
2563        assert!(task_error.is_cancelled());
2564        assert!(!error.to_string().contains(SECRET));
2565        assert!(!error.to_string().contains(&BASE64.encode(SECRET)));
2566        assert!(
2567            !error
2568                .to_string()
2569                .contains(&BASE64.encode(format!("{USERNAME}:{SECRET}")))
2570        );
2571    }
2572
2573    #[rstest]
2574    fn test_http_client_with_valid_proxy() {
2575        // Create client with a valid proxy URL
2576        let result = HttpClient::builder()
2577            .proxy_url("http://proxy.example.com:8080".to_string())
2578            .build();
2579
2580        assert!(result.is_ok());
2581    }
2582
2583    #[rstest]
2584    fn test_http_client_with_socks5_proxy() {
2585        // Create client with a SOCKS5 proxy URL
2586        let result = HttpClient::builder()
2587            .proxy_url("socks5://127.0.0.1:1080".to_string())
2588            .build();
2589
2590        assert!(result.is_ok());
2591    }
2592
2593    #[rstest]
2594    fn test_http_client_with_malformed_proxy() {
2595        // Proxy parsing accepts scheme-less hostnames.
2596        // It only fails on obviously malformed URLs like "://invalid" or "http://".
2597        // More subtle issues (like "not-a-valid-url") are caught when connecting.
2598        let result = HttpClient::builder()
2599            .proxy_url("://invalid".to_string())
2600            .build();
2601
2602        assert!(result.is_err());
2603        assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2604    }
2605
2606    #[rstest]
2607    fn test_http_client_invalid_proxy_error_redacts_credentials() {
2608        const SECRET: &str = "unique-proxy-secret";
2609        let result = HttpClient::builder()
2610            .proxy_url(format!("http://proxytest:{SECRET}@[::1"))
2611            .build();
2612        let error = result.expect_err("malformed proxy URL should fail");
2613
2614        assert_eq!(
2615            error.to_string(),
2616            "Invalid proxy URL: proxy URL is malformed"
2617        );
2618        assert!(!error.to_string().contains(SECRET));
2619    }
2620
2621    #[rstest]
2622    fn test_http_client_with_empty_proxy_string() {
2623        // Create client with an empty proxy URL string
2624        let result = HttpClient::builder().proxy_url(String::new()).build();
2625
2626        assert!(result.is_err());
2627        assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2628    }
2629
2630    #[tokio::test]
2631    async fn test_http_client_get() {
2632        let addr = start_test_server().await.unwrap();
2633        let url = format!("http://{addr}/get");
2634
2635        let client = HttpClient::builder().build().unwrap();
2636        let response = client.get(url, None, None, None, None).await.unwrap();
2637
2638        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2639        assert_eq!(response.headers, HashMap::new());
2640        assert_eq!(response.body.as_ref(), b"hello-world!");
2641    }
2642
2643    #[tokio::test]
2644    async fn test_http_client_post() {
2645        let addr = start_test_server().await.unwrap();
2646        let url = format!("http://{addr}/post");
2647
2648        let client = HttpClient::builder().build().unwrap();
2649        let response = client
2650            .post(url, None, None, Some(b"post-body-73".to_vec()), None, None)
2651            .await
2652            .unwrap();
2653
2654        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2655        assert_eq!(response.headers, HashMap::new());
2656        assert_eq!(response.body.as_ref(), b"post-body-73");
2657    }
2658
2659    #[tokio::test]
2660    async fn test_http_client_patch() {
2661        let addr = start_test_server().await.unwrap();
2662        let url = format!("http://{addr}/patch");
2663
2664        let client = HttpClient::builder().build().unwrap();
2665        let response = client
2666            .patch(url, None, None, Some(b"patch-body-91".to_vec()), None, None)
2667            .await
2668            .unwrap();
2669
2670        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2671        assert_eq!(response.headers, HashMap::new());
2672        assert_eq!(response.body.as_ref(), b"patch-body-91");
2673    }
2674
2675    #[tokio::test]
2676    async fn test_http_client_delete() {
2677        let addr = start_test_server().await.unwrap();
2678        let url = format!("http://{addr}/delete");
2679
2680        let client = HttpClient::builder().build().unwrap();
2681        let response = client.delete(url, None, None, None, None).await.unwrap();
2682
2683        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2684        assert_eq!(response.headers, HashMap::new());
2685        assert_eq!(response.body.as_ref(), b"");
2686    }
2687}
2688
2689#[cfg(test)]
2690mod rate_limit_tests {
2691    use std::{num::NonZeroU32, sync::Arc, time::Duration};
2692
2693    #[cfg(all(feature = "simulation", madsim))]
2694    use madsim::task as test_task;
2695    #[cfg(not(all(feature = "simulation", madsim)))]
2696    use tokio::task as test_task;
2697    use ustr::Ustr;
2698
2699    use super::HttpClient;
2700    use crate::ratelimiter::{RateLimiter, quota::Quota};
2701
2702    #[tokio::test]
2703    async fn test_http_client_awaits_multiple_rate_limiters() {
2704        let quota = Quota::per_minute(NonZeroU32::MIN);
2705        let request_key = Ustr::from("scope:request");
2706        let order_key = Ustr::from("scope:order");
2707
2708        let request_limiter = Arc::new(RateLimiter::new_with_quota(
2709            None,
2710            vec![(request_key, quota)],
2711        ));
2712        let order_limiter = Arc::new(RateLimiter::new_with_quota(None, vec![(order_key, quota)]));
2713        let client = HttpClient::builder()
2714            .rate_limiters(vec![
2715                Arc::clone(&request_limiter),
2716                Arc::clone(&order_limiter),
2717            ])
2718            .build()
2719            .unwrap();
2720
2721        client
2722            .await_rate_limits(Some(&[request_key, order_key]))
2723            .await;
2724
2725        assert!(request_limiter.check_key(&request_key).is_err());
2726        assert!(order_limiter.check_key(&order_key).is_err());
2727    }
2728
2729    #[cfg_attr(
2730        not(all(feature = "simulation", madsim)),
2731        tokio::test(start_paused = true)
2732    )]
2733    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2734    async fn test_http_client_reserves_multiple_rate_limits_together() {
2735        let global_key = Ustr::from("scope:global");
2736        let order_key = Ustr::from("scope:order");
2737
2738        let global_limiter = Arc::new(RateLimiter::new_with_quota(
2739            None,
2740            vec![(
2741                global_key,
2742                Quota::with_period(Duration::from_secs(1)).unwrap(),
2743            )],
2744        ));
2745
2746        let order_limiter = Arc::new(RateLimiter::new_with_quota(
2747            None,
2748            vec![(
2749                order_key,
2750                Quota::with_period(Duration::from_secs(10)).unwrap(),
2751            )],
2752        ));
2753        order_limiter.check_key(&order_key).unwrap();
2754
2755        let client = HttpClient::builder()
2756            .rate_limiters(vec![
2757                Arc::clone(&global_limiter),
2758                Arc::clone(&order_limiter),
2759            ])
2760            .build()
2761            .unwrap();
2762
2763        let request = test_task::spawn(async move {
2764            client
2765                .await_rate_limits(Some(&[global_key, order_key]))
2766                .await;
2767        });
2768
2769        test_task::yield_now().await;
2770
2771        global_limiter.check_key(&global_key).unwrap();
2772        assert!(!request.is_finished());
2773
2774        advance_test_clock(Duration::from_millis(9_999)).await;
2775        global_limiter.until_key_ready(&global_key).await;
2776        advance_test_clock(Duration::from_millis(1)).await;
2777        test_task::yield_now().await;
2778        assert!(!request.is_finished());
2779
2780        advance_test_clock(Duration::from_millis(998)).await;
2781        test_task::yield_now().await;
2782        assert!(!request.is_finished());
2783
2784        advance_test_clock(Duration::from_millis(1)).await;
2785        request.await.unwrap();
2786
2787        assert!(global_limiter.check_key(&global_key).is_err());
2788        assert!(order_limiter.check_key(&order_key).is_err());
2789    }
2790
2791    #[cfg(all(feature = "simulation", madsim))]
2792    async fn advance_test_clock(duration: Duration) {
2793        madsim::time::advance(duration);
2794        test_task::yield_now().await;
2795    }
2796
2797    #[cfg(not(all(feature = "simulation", madsim)))]
2798    async fn advance_test_clock(duration: Duration) {
2799        tokio::time::advance(duration).await;
2800    }
2801}
2802
2803#[cfg(test)]
2804pub(super) mod content_encoding_tests {
2805    use std::{collections::HashMap, io::Write, time::Duration};
2806
2807    use bytes::Bytes;
2808    use flate2::{Compression, write::GzEncoder};
2809    use http_body_util::Full;
2810    use rstest::rstest;
2811
2812    use super::{HttpClient, HttpClientError, REQUEST_TIMEOUT_MESSAGE};
2813    use crate::dst::time;
2814
2815    #[rstest]
2816    #[case::large_body(false, false)]
2817    #[case::large_body_deadline(true, false)]
2818    #[case::empty_members(false, true)]
2819    #[case::empty_members_deadline(true, true)]
2820    #[cfg_attr(
2821        not(all(feature = "simulation", madsim)),
2822        tokio::test(start_paused = true)
2823    )]
2824    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2825    async fn gzip_decoding_yields_and_obeys_deadline(
2826        #[case] expires: bool,
2827        #[case] empty_members: bool,
2828    ) {
2829        let (payload, compressed) = if empty_members {
2830            (Vec::new(), gzip(&[]).repeat(4_000))
2831        } else {
2832            let payload = vec![b'x'; 2 * 1024 * 1024];
2833            let compressed = gzip(&payload);
2834            (payload, compressed)
2835        };
2836
2837        let body = Bytes::from(compressed);
2838        let body = Full::new(body);
2839        let response = http::Response::builder()
2840            .header("Content-Encoding", "gzip")
2841            .body(body)
2842            .unwrap();
2843        let client = HttpClient::builder()
2844            .use_system_proxy(false)
2845            .build()
2846            .unwrap()
2847            .client;
2848        let deadline = expires.then(|| time::Instant::now() + Duration::from_secs(1));
2849        let mut request = Box::pin(client.consume_response(response, deadline, false));
2850
2851        assert!(futures_util::poll!(request.as_mut()).is_pending());
2852
2853        if expires {
2854            #[cfg(not(all(feature = "simulation", madsim)))]
2855            tokio::time::advance(Duration::from_secs(2)).await;
2856            #[cfg(all(feature = "simulation", madsim))]
2857            madsim::time::advance(Duration::from_secs(2));
2858        }
2859
2860        let result = request.await;
2861        if expires {
2862            let HttpClientError::TimeoutError(message) = result.unwrap_err() else {
2863                panic!("expected decoding timeout");
2864            };
2865
2866            assert_eq!(message, REQUEST_TIMEOUT_MESSAGE);
2867        } else {
2868            let response = result.unwrap();
2869            assert_eq!(response.status.as_u16(), 200);
2870            assert_eq!(response.headers, HashMap::new());
2871            assert_eq!(response.body.as_ref(), payload);
2872        }
2873    }
2874
2875    #[rstest]
2876    #[case::stored_blocks(false)]
2877    #[case::header_fields(true)]
2878    #[cfg_attr(not(all(feature = "simulation", madsim)), tokio::test)]
2879    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2880    async fn gzip_decoding_resumes_at_input_boundaries(#[case] metadata: bool) {
2881        let payload = vec![b'x'; 128 * 1024];
2882
2883        let mut encoder = if metadata {
2884            flate2::GzBuilder::new()
2885                .filename(vec![b'f'; 32 * 1024])
2886                .comment(vec![b'c'; 32 * 1024])
2887                .write(Vec::new(), Compression::default())
2888        } else {
2889            GzEncoder::new(Vec::new(), Compression::none())
2890        };
2891
2892        encoder.write_all(&payload).unwrap();
2893        let compressed = encoder.finish().unwrap();
2894        let response = http::Response::builder()
2895            .header("Content-Encoding", "gzip")
2896            .body(Full::new(Bytes::from(compressed)))
2897            .unwrap();
2898        let client = HttpClient::builder()
2899            .use_system_proxy(false)
2900            .build()
2901            .unwrap()
2902            .client;
2903
2904        let response = client
2905            .consume_response(response, None, false)
2906            .await
2907            .unwrap();
2908
2909        assert_eq!(response.status.as_u16(), 200);
2910        assert_eq!(response.headers, HashMap::new());
2911        assert_eq!(response.body.as_ref(), payload);
2912    }
2913
2914    pub(in crate::http) fn gzip(data: &[u8]) -> Vec<u8> {
2915        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
2916        encoder.write_all(data).unwrap();
2917        encoder.finish().unwrap()
2918    }
2919}