Skip to main content

nautilus_network/http/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! HTTP client implementation with rate limiting and timeout support.
17
18use std::{borrow::Cow, collections::HashMap, str::FromStr, sync::Arc, time::Duration};
19
20use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
21use bytes::Bytes;
22use http::{
23    Method,
24    header::{HeaderMap, HeaderName, HeaderValue},
25};
26use http_body_util::Full;
27use nautilus_core::{collections::into_ustr_vec, string::secret::SecretString};
28use nautilus_cryptography::providers::install_cryptographic_provider;
29use url::Url;
30use ustr::Ustr;
31
32use super::{
33    HttpClientError, HttpResponse, HttpResponseStream, HttpStatus,
34    stream::{check_deadline, read_chunk, response_error},
35};
36use crate::ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota};
37
38/// Default maximum idle connections per host.
39#[cfg(not(all(feature = "simulation", madsim)))]
40const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 32;
41
42/// Default idle connection timeout in seconds.
43#[cfg(not(all(feature = "simulation", madsim)))]
44const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 60;
45
46/// Default HTTP/2 keep-alive interval in seconds.
47#[cfg(not(all(feature = "simulation", madsim)))]
48const DEFAULT_HTTP2_KEEP_ALIVE_SECS: u64 = 30;
49
50/// Default HTTP/2 stream flow-control window in bytes (16 MiB).
51#[cfg(not(all(feature = "simulation", madsim)))]
52const DEFAULT_HTTP2_STREAM_WINDOW_BYTES: u32 = 16 * 1024 * 1024;
53
54/// Default HTTP/2 connection flow-control window in bytes (32 MiB).
55#[cfg(not(all(feature = "simulation", madsim)))]
56const DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES: u32 = 32 * 1024 * 1024;
57
58/// Environment variable that enables adaptive HTTP/2 flow-control windows when set to `true`.
59#[cfg(not(all(feature = "simulation", madsim)))]
60const NAUTILUS_HTTP2_ADAPTIVE_WINDOW: &str = "NAUTILUS_HTTP2_ADAPTIVE_WINDOW";
61
62/// Default maximum HTTP response body size in bytes (100 MiB).
63///
64/// Bounds peak memory per response so a hostile or malfunctioning endpoint
65/// cannot exhaust memory by streaming an arbitrarily large body. Mirrors the
66/// caps already enforced on the WebSocket and raw-socket paths.
67const DEFAULT_MAX_RESPONSE_BYTES: usize = 100 * 1024 * 1024;
68
69const GZIP: &str = "gzip";
70const GZIP_DECODE_CHUNK_BYTES: usize = 64 * 1024;
71
72#[cfg(all(feature = "simulation", madsim))]
73pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "simulated request deadline elapsed";
74#[cfg(not(all(feature = "simulation", madsim)))]
75pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "request deadline elapsed";
76
77/// Controls whether an HTTP client follows redirects.
78#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
79pub enum HttpRedirectPolicy {
80    /// Follow up to ten redirects.
81    ///
82    /// A redirect to another origin keeps only the `Accept`, `Accept-Encoding`, and `User-Agent`
83    /// request headers. The request fails when a redirect goes from HTTPS to HTTP, or when a
84    /// request with a body is redirected to another origin.
85    #[default]
86    Follow,
87    /// Reject every redirect response.
88    Reject,
89}
90
91/// An asynchronous HTTP client with rate limiting, timeouts, and custom headers.
92///
93/// The client uses Hyper for normal I/O and supports default and per-key quotas. Multiple
94/// clients can share the same rate limiter when their requests consume one quota budget.
95/// With `simulation` and `cfg(madsim)`, plaintext HTTP/1.1 uses simulated byte streams;
96/// HTTPS, client certificates, explicit proxies, and redirect following are unsupported.
97///
98/// Request URLs in transport errors and response-stream diagnostics are limited to their origin
99/// (scheme, host, and port). Userinfo, paths, queries, and fragments are excluded. Set
100/// `redact_url(true)` on the builder to omit the origin too. This policy does not change requests
101/// or diagnostic messages from lower layers. It controls the URL context added by this client.
102#[derive(Clone, Debug)]
103pub struct HttpClient {
104    pub(crate) client: InnerHttpClient,
105    pub(crate) rate_limiters: Arc<[Arc<RateLimiter<Ustr, MonotonicClock>>]>,
106}
107
108#[bon::bon]
109impl HttpClient {
110    /// Returns a builder for a new [`HttpClient`] instance.
111    ///
112    /// Set `rate_limiters` to share quota state across clients. When omitted, the client creates
113    /// one rate limiter from `default_quota` and `keyed_quotas`. An explicit empty vector disables
114    /// rate limiting. Each request awaits every configured limiter with the same keys. A limiter
115    /// without a default quota ignores keys it does not own, allowing independent scopes such as
116    /// per-IP and per-account limits to apply to one request.
117    ///
118    /// # Errors
119    ///
120    /// Returns an error if:
121    /// - Shared rate limiters are combined with quota configuration.
122    /// - The timeout exceeds the monotonic clock range.
123    /// - The proxy URL is malformed.
124    /// - `NAUTILUS_HTTP2_ADAPTIVE_WINDOW` is set to a value other than `true` or `false`.
125    /// - The certificate directory does not contain a matching client certificate and private key.
126    /// - Building the underlying HTTP transport fails.
127    #[allow(
128        clippy::needless_pass_by_value,
129        reason = "owned proxy URLs are part of the public builder API"
130    )]
131    #[builder(finish_fn = build)]
132    pub fn builder(
133        #[builder(default)] headers: HashMap<String, String>,
134        #[builder(default)] header_keys: Vec<String>,
135        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
136        default_quota: Option<Quota>,
137        timeout_secs: Option<u64>,
138        proxy_url: Option<String>,
139        rate_limiters: Option<Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>>,
140        #[builder(default)] redirect_policy: HttpRedirectPolicy,
141        /// Whether to honor ambient proxy configuration when `proxy_url` is not set.
142        #[builder(default = true)]
143        use_system_proxy: bool,
144        /// Whether to omit the request origin added by this client to transport errors and
145        /// response-stream diagnostics.
146        /// Defaults to `false`.
147        /// Userinfo, paths, queries, and fragments are always excluded from the added URL context.
148        /// Diagnostic messages from lower layers are unchanged. TLS certificate errors and
149        /// dependency logs can still include the hostname.
150        #[builder(default)]
151        redact_url: bool,
152        /// Maximum size in bytes of a buffered response body after `Content-Encoding` decoding.
153        /// Defaults to the response body limit (100 MiB). Raise it for endpoints whose compressed
154        /// payloads legitimately decode beyond that limit; the wire body stays bounded by the
155        /// response body limit either way.
156        #[builder(default = DEFAULT_MAX_RESPONSE_BYTES)]
157        max_decompressed_bytes: usize,
158        /// Directory containing a matching client certificate and private key.
159        /// Uses bundled roots plus certificates from this directory instead of system trust.
160        certs_dir: Option<String>,
161    ) -> Result<Self, HttpClientError> {
162        let rate_limiters = if let Some(rate_limiters) = rate_limiters {
163            if default_quota.is_some() || !keyed_quotas.is_empty() {
164                return Err(HttpClientError::Error(
165                    "Cannot combine shared rate limiters with quota configuration".to_string(),
166                ));
167            }
168
169            rate_limiters
170        } else {
171            let keyed_quotas = keyed_quotas
172                .into_iter()
173                .map(|(key, quota)| (Ustr::from(&key), quota))
174                .collect();
175            vec![Arc::new(RateLimiter::new_with_quota(
176                default_quota,
177                keyed_quotas,
178            ))]
179        };
180
181        Self::build(
182            headers,
183            header_keys,
184            timeout_secs,
185            proxy_url.as_deref(),
186            rate_limiters,
187            redirect_policy,
188            use_system_proxy,
189            redact_url,
190            max_decompressed_bytes,
191            certs_dir.as_deref(),
192        )
193    }
194
195    #[expect(
196        clippy::too_many_arguments,
197        reason = "client construction consumes the builder settings"
198    )]
199    fn build(
200        headers: HashMap<String, String>,
201        header_keys: Vec<String>,
202        timeout_secs: Option<u64>,
203        proxy_url: Option<&str>,
204        rate_limiters: Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>,
205        redirect_policy: HttpRedirectPolicy,
206        use_system_proxy: bool,
207        redact_url: bool,
208        max_decompressed_bytes: usize,
209        certs_dir: Option<&str>,
210    ) -> Result<Self, HttpClientError> {
211        let timeout = timeout_secs.map(Duration::from_secs);
212        request_deadline(timeout)?;
213        install_cryptographic_provider();
214
215        let mut header_map = HeaderMap::new();
216
217        for (key, value) in headers {
218            let header_name = HeaderName::from_str(&key)
219                .map_err(|e| HttpClientError::Error(format!("Invalid header name '{key}': {e}")))?;
220
221            let mut header_value = HeaderValue::from_str(&value).map_err(|e| {
222                HttpClientError::Error(format!("Invalid header value for '{key}': {e}"))
223            })?;
224
225            header_value.set_sensitive(true);
226            header_map.insert(header_name, header_value);
227        }
228
229        #[cfg(all(feature = "simulation", madsim))]
230        let simulation = super::simulation::Client::new(redirect_policy, proxy_url, certs_dir)?;
231
232        #[cfg(not(all(feature = "simulation", madsim)))]
233        let client = super::transport::Client::new(
234            proxy_url,
235            use_system_proxy,
236            certs_dir,
237            header_map.get(http::header::USER_AGENT).cloned(),
238            super::transport::Settings {
239                pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST,
240                pool_idle_timeout: Duration::from_secs(DEFAULT_POOL_IDLE_TIMEOUT_SECS),
241                keep_alive_interval: Some(Duration::from_secs(DEFAULT_HTTP2_KEEP_ALIVE_SECS)),
242                stream_window: Some(DEFAULT_HTTP2_STREAM_WINDOW_BYTES),
243                connection_window: Some(DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES),
244                adaptive_window: http2_adaptive_window()?,
245            },
246        )?;
247
248        #[cfg(all(feature = "simulation", madsim))]
249        let _ = use_system_proxy;
250
251        // Pre-intern header keys as HeaderName. An invalid key is an error: a silent drop would
252        // make response extraction read nothing.
253        let response_headers = header_keys
254            .into_iter()
255            .map(|key| match HeaderName::from_str(&key) {
256                Ok(name) => Ok((key, name)),
257                Err(e) => Err(HttpClientError::Error(format!(
258                    "Invalid header key '{key}': {e}"
259                ))),
260            })
261            .collect::<Result<Vec<_>, _>>()?;
262
263        let client = InnerHttpClient {
264            #[cfg(not(all(feature = "simulation", madsim)))]
265            client,
266            headers: header_map,
267            timeout,
268            redact_url,
269            #[cfg(not(all(feature = "simulation", madsim)))]
270            redirect_policy,
271            #[cfg(all(feature = "simulation", madsim))]
272            simulation,
273            response_headers: Arc::from(response_headers),
274            max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
275            max_decompressed_bytes,
276        };
277
278        Ok(Self {
279            client,
280            rate_limiters: rate_limiters.into(),
281        })
282    }
283
284    /// Sends an HTTP request.
285    ///
286    /// # Errors
287    ///
288    /// Returns an error if the timeout exceeds the monotonic clock range, the request cannot be sent,
289    /// or it times out.
290    ///
291    /// # Examples
292    ///
293    /// If requesting `/foo/bar`, pass rate-limit keys `["foo/bar", "foo"]`.
294    #[expect(clippy::too_many_arguments)]
295    pub async fn request(
296        &self,
297        method: Method,
298        url: String,
299        params: Option<&HashMap<String, Vec<String>>>,
300        headers: Option<HashMap<String, String>>,
301        body: Option<Vec<u8>>,
302        timeout_secs: Option<u64>,
303        keys: Option<Vec<String>>,
304    ) -> Result<HttpResponse, HttpClientError> {
305        let keys = keys.map(into_ustr_vec);
306
307        self.request_with_ustr_keys(method, url, params, headers, body, timeout_secs, keys)
308            .await
309    }
310
311    /// Sends an HTTP request whose body contains secret material.
312    ///
313    /// The body retains its zeroizing owner until the transport releases the last byte buffer.
314    /// Transport, TLS, and operating-system layers may make additional plaintext copies that this
315    /// client cannot zeroize.
316    ///
317    /// # Errors
318    ///
319    /// Returns an error if unable to send the request or if it times out.
320    #[expect(clippy::too_many_arguments)]
321    pub async fn request_with_secret_body(
322        &self,
323        method: Method,
324        url: String,
325        params: Option<&HashMap<String, Vec<String>>>,
326        headers: Option<HashMap<String, String>>,
327        body: SecretString,
328        timeout_secs: Option<u64>,
329        keys: Option<Vec<String>>,
330    ) -> Result<HttpResponse, HttpClientError> {
331        let keys = keys.map(into_ustr_vec);
332        self.await_rate_limits(keys.as_deref()).await;
333
334        self.client
335            .send_request_with_secret_body(method, url, params, headers, body, timeout_secs)
336            .await
337    }
338
339    /// Sends an HTTP request with serializable query parameters.
340    ///
341    /// This method accepts any type implementing `Serialize` for query parameters,
342    /// which are URL-encoded directly into the query string without an intermediate `HashMap`.
343    ///
344    /// # Errors
345    ///
346    /// Returns an error if unable to send request or times out.
347    #[expect(clippy::too_many_arguments)]
348    pub async fn request_with_params<P: serde::Serialize>(
349        &self,
350        method: Method,
351        url: String,
352        params: Option<&P>,
353        headers: Option<HashMap<String, String>>,
354        body: Option<Vec<u8>>,
355        timeout_secs: Option<u64>,
356        keys: Option<Vec<String>>,
357    ) -> Result<HttpResponse, HttpClientError> {
358        let keys = keys.map(into_ustr_vec);
359        self.await_rate_limits(keys.as_deref()).await;
360
361        self.client
362            .send_request_with_query(method, url, params, headers, body, timeout_secs)
363            .await
364    }
365
366    /// Sends a GET request and returns its response body as a stream.
367    ///
368    /// Returns raw body chunks without content decoding or adding `Accept-Encoding`.
369    /// Applies default headers and the client timeout. No rate-limit keys are supplied, so no
370    /// quota is consumed. One absolute deadline covers response headers and the whole body,
371    /// including time spent processing chunks. Streaming has no total body size limit; callers
372    /// must process or discard each chunk without accumulating an unbounded body.
373    /// Dropping the response releases the unfinished exchange, including its simulated driver.
374    ///
375    /// # Errors
376    ///
377    /// Returns an error if request preparation, connection, or response headers fail or time out.
378    pub async fn get_stream(&self, url: String) -> Result<HttpResponseStream, HttpClientError> {
379        self.await_rate_limits(None).await;
380        self.client
381            .send_stream_internal::<[(String, String); 0]>(
382                Method::GET,
383                &url,
384                None,
385                None,
386                None,
387                None,
388            )
389            .await
390    }
391
392    /// Sends an HTTP request using pre-interned rate limiter keys.
393    ///
394    /// # Errors
395    ///
396    /// Returns an error if unable to send the request or the request times out.
397    #[expect(clippy::too_many_arguments)]
398    pub async fn request_with_ustr_keys(
399        &self,
400        method: Method,
401        url: String,
402        params: Option<&HashMap<String, Vec<String>>>,
403        headers: Option<HashMap<String, String>>,
404        body: Option<Vec<u8>>,
405        timeout_secs: Option<u64>,
406        keys: Option<Vec<Ustr>>,
407    ) -> Result<HttpResponse, HttpClientError> {
408        self.await_rate_limits(keys.as_deref()).await;
409
410        self.client
411            .send_request(method, url, params, headers, body, timeout_secs)
412            .await
413    }
414
415    /// Acquires quota for the supplied keys before a caller prepares a request.
416    ///
417    /// Waits without a timeout and consumes quota atomically across the configured limiters.
418    /// Dropping a pending future consumes no quota. Callers must bound the wait when needed.
419    /// The next request must omit rate-limit keys to avoid consuming quota twice.
420    /// Passing no keys acquires no quota.
421    pub async fn await_rate_limits(&self, keys: Option<&[Ustr]>) {
422        RateLimiter::await_limiters_ready(&self.rate_limiters, keys).await;
423    }
424
425    /// Sends an HTTP GET request.
426    ///
427    /// # Errors
428    ///
429    /// Returns an error if unable to send request or times out.
430    pub async fn get(
431        &self,
432        url: String,
433        params: Option<&HashMap<String, Vec<String>>>,
434        headers: Option<HashMap<String, String>>,
435        timeout_secs: Option<u64>,
436        keys: Option<Vec<String>>,
437    ) -> Result<HttpResponse, HttpClientError> {
438        self.request(Method::GET, url, params, headers, None, timeout_secs, keys)
439            .await
440    }
441
442    /// Sends an HTTP POST request.
443    ///
444    /// # Errors
445    ///
446    /// Returns an error if unable to send request or times out.
447    pub async fn post(
448        &self,
449        url: String,
450        params: Option<&HashMap<String, Vec<String>>>,
451        headers: Option<HashMap<String, String>>,
452        body: Option<Vec<u8>>,
453        timeout_secs: Option<u64>,
454        keys: Option<Vec<String>>,
455    ) -> Result<HttpResponse, HttpClientError> {
456        self.request(Method::POST, url, params, headers, body, timeout_secs, keys)
457            .await
458    }
459
460    /// Sends an HTTP PATCH request.
461    ///
462    /// # Errors
463    ///
464    /// Returns an error if unable to send request or times out.
465    pub async fn patch(
466        &self,
467        url: String,
468        params: Option<&HashMap<String, Vec<String>>>,
469        headers: Option<HashMap<String, String>>,
470        body: Option<Vec<u8>>,
471        timeout_secs: Option<u64>,
472        keys: Option<Vec<String>>,
473    ) -> Result<HttpResponse, HttpClientError> {
474        self.request(
475            Method::PATCH,
476            url,
477            params,
478            headers,
479            body,
480            timeout_secs,
481            keys,
482        )
483        .await
484    }
485
486    /// Sends an HTTP DELETE request.
487    ///
488    /// # Errors
489    ///
490    /// Returns an error if unable to send request or times out.
491    pub async fn delete(
492        &self,
493        url: String,
494        params: Option<&HashMap<String, Vec<String>>>,
495        headers: Option<HashMap<String, String>>,
496        timeout_secs: Option<u64>,
497        keys: Option<Vec<String>>,
498    ) -> Result<HttpResponse, HttpClientError> {
499        self.request(
500            Method::DELETE,
501            url,
502            params,
503            headers,
504            None,
505            timeout_secs,
506            keys,
507        )
508        .await
509    }
510}
511
512/// Internal implementation backing [`HttpClient`].
513///
514/// The underlying Hyper client reuses pooled connections and is cheap to clone. Responses
515/// retain only configured header fields, and bodies larger than `max_response_bytes` are rejected.
516/// Decoded (decompressed) bodies are bounded separately by `max_decompressed_bytes`.
517#[derive(Clone, Debug)]
518pub struct InnerHttpClient {
519    #[cfg(all(feature = "simulation", madsim))]
520    simulation: super::simulation::Client,
521    #[cfg(not(all(feature = "simulation", madsim)))]
522    client: super::transport::Client,
523    headers: HeaderMap,
524    timeout: Option<Duration>,
525    redact_url: bool,
526    #[cfg(not(all(feature = "simulation", madsim)))]
527    redirect_policy: HttpRedirectPolicy,
528    pub(crate) response_headers: Arc<[(String, HeaderName)]>,
529    pub(crate) max_response_bytes: usize,
530    pub(crate) max_decompressed_bytes: usize,
531}
532
533impl InnerHttpClient {
534    /// Sends an HTTP request and returns an [`HttpResponse`].
535    ///
536    /// # Errors
537    ///
538    /// Returns an error if unable to send request or times out.
539    pub async fn send_request(
540        &self,
541        method: Method,
542        url: String,
543        params: Option<&HashMap<String, Vec<String>>>,
544        headers: Option<HashMap<String, String>>,
545        body: Option<Vec<u8>>,
546        timeout_secs: Option<u64>,
547    ) -> Result<HttpResponse, HttpClientError> {
548        self.send_request_with_body(
549            method,
550            url,
551            params,
552            headers,
553            body.map(RequestBody::Plain),
554            timeout_secs,
555        )
556        .await
557    }
558
559    async fn send_request_with_secret_body(
560        &self,
561        method: Method,
562        url: String,
563        params: Option<&HashMap<String, Vec<String>>>,
564        headers: Option<HashMap<String, String>>,
565        body: SecretString,
566        timeout_secs: Option<u64>,
567    ) -> Result<HttpResponse, HttpClientError> {
568        self.send_request_with_body(
569            method,
570            url,
571            params,
572            headers,
573            Some(RequestBody::Secret(body)),
574            timeout_secs,
575        )
576        .await
577    }
578
579    async fn send_request_with_body(
580        &self,
581        method: Method,
582        url: String,
583        params: Option<&HashMap<String, Vec<String>>>,
584        headers: Option<HashMap<String, String>>,
585        body: Option<RequestBody>,
586        timeout_secs: Option<u64>,
587    ) -> Result<HttpResponse, HttpClientError> {
588        let full_url = encode_url_params(&url, params)?;
589        self.send_request_internal(
590            method,
591            full_url.as_ref(),
592            None::<&()>,
593            headers,
594            body,
595            timeout_secs,
596        )
597        .await
598    }
599
600    /// Sends an HTTP request with URL-encoded serializable query parameters.
601    ///
602    /// This method accepts any type implementing `Serialize` for query parameters,
603    /// avoiding `HashMap` conversion overhead.
604    ///
605    /// # Errors
606    ///
607    /// Returns an error if unable to send request or times out.
608    pub async fn send_request_with_query<Q: serde::Serialize>(
609        &self,
610        method: Method,
611        url: String,
612        query: Option<&Q>,
613        headers: Option<HashMap<String, String>>,
614        body: Option<Vec<u8>>,
615        timeout_secs: Option<u64>,
616    ) -> Result<HttpResponse, HttpClientError> {
617        self.send_request_internal(
618            method,
619            &url,
620            query,
621            headers,
622            body.map(RequestBody::Plain),
623            timeout_secs,
624        )
625        .await
626    }
627
628    /// Internal implementation for sending HTTP requests.
629    ///
630    /// # Errors
631    ///
632    /// Returns an error if unable to send request or times out.
633    async fn send_request_internal<Q: serde::Serialize>(
634        &self,
635        method: Method,
636        url: &str,
637        query: Option<&Q>,
638        headers: Option<HashMap<String, String>>,
639        body: Option<RequestBody>,
640        timeout_secs: Option<u64>,
641    ) -> Result<HttpResponse, HttpClientError> {
642        let mut headers = headers.unwrap_or_default();
643        let has_accept_encoding = headers
644            .keys()
645            .any(|key| key.eq_ignore_ascii_case(http::header::ACCEPT_ENCODING.as_str()))
646            || self.headers.contains_key(http::header::ACCEPT_ENCODING);
647
648        if !has_accept_encoding {
649            headers.insert(http::header::ACCEPT_ENCODING.to_string(), GZIP.to_string());
650        }
651
652        let is_head = method == Method::HEAD;
653        let stream = self
654            .send_stream_internal(method, url, query, Some(headers), body, timeout_secs)
655            .await?;
656        let result = self
657            .consume_response(stream.response, stream.deadline, is_head)
658            .await;
659        result.map_err(|e| response_error(e, stream.origin.as_deref()))
660    }
661
662    async fn send_stream_internal<Q: serde::Serialize>(
663        &self,
664        method: Method,
665        url: &str,
666        query: Option<&Q>,
667        headers: Option<HashMap<String, String>>,
668        body: Option<RequestBody>,
669        timeout_secs: Option<u64>,
670    ) -> Result<HttpResponseStream, HttpClientError> {
671        let mut url =
672            Url::parse(url).map_err(|e| HttpClientError::from(format!("URL parse error: {e}")))?;
673        if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
674            return Err(HttpClientError::Error(
675                "unsupported HTTP URL scheme or hostname".into(),
676            ));
677        }
678
679        let mut header_map = self.headers.clone();
680
681        if let Ok(username) = percent_encoding::percent_decode_str(url.username()).decode_utf8() {
682            let password = url.password().and_then(|password| {
683                percent_encoding::percent_decode_str(password)
684                    .decode_utf8()
685                    .ok()
686            });
687
688            if !username.is_empty() || password.is_some() {
689                let mut value = HeaderValue::from_str(&format!(
690                    "Basic {}",
691                    BASE64.encode(format!(
692                        "{username}:{}",
693                        password.as_deref().unwrap_or_default()
694                    ))
695                ))
696                .map_err(|e| HttpClientError::Error(e.to_string()))?;
697                value.set_sensitive(true);
698                header_map.insert(http::header::AUTHORIZATION, value);
699                let _ = url.set_username("");
700                let _ = url.set_password(None);
701            }
702        }
703
704        let extra_header_count = headers.as_ref().map_or(0, HashMap::len);
705
706        if let Some(headers) = headers {
707            for (key, value) in headers {
708                let key = HeaderName::from_bytes(key.as_bytes())
709                    .map_err(|e| HttpClientError::from(format!("Invalid header name: {e}")))?;
710                let value = HeaderValue::from_str(&value)
711                    .map_err(|e| HttpClientError::from(format!("Invalid header value: {e}")))?;
712
713                if header_map.insert(key.clone(), value).is_some() {
714                    log::trace!("Replaced duplicate request header '{key}'");
715                }
716            }
717        }
718
719        if let Some(query) = query {
720            {
721                let mut pairs = url.query_pairs_mut();
722                let serializer = serde_urlencoded::Serializer::new(&mut pairs);
723                query
724                    .serialize(serializer)
725                    .map_err(|e| HttpClientError::Error(e.to_string()))?;
726            }
727
728            if url.query() == Some("") {
729                url.set_query(None);
730            }
731        }
732
733        if !header_map.contains_key(http::header::ACCEPT) {
734            header_map.insert(http::header::ACCEPT, HeaderValue::from_static("*/*"));
735        }
736
737        let body = body.map_or_default(RequestBody::into_bytes);
738        let body_len = body.len();
739        let query_len = url.query().map_or(0, str::len);
740        let mut request = http::Request::new(Full::new(body));
741        *request.method_mut() = method;
742        *request.uri_mut() = url[..url::Position::AfterQuery]
743            .parse()
744            .map_err(|_| HttpClientError::Error("invalid HTTP request target".into()))?;
745        *request.headers_mut() = header_map;
746        log::trace!(
747            "Sending HTTP request: method={} extra_headers={extra_header_count} \
748             query_bytes={query_len} body_bytes={body_len}",
749            request.method(),
750        );
751
752        let origin = (!self.redact_url).then(|| url.origin().ascii_serialization());
753
754        let duration = timeout_secs.map(Duration::from_secs).or(self.timeout);
755        let deadline = request_deadline(duration)?;
756
757        let operation = async {
758            #[cfg(all(feature = "simulation", madsim))]
759            let (response, connection) = self.simulation.send(request, &url).await?;
760            #[cfg(not(all(feature = "simulation", madsim)))]
761            let response = self.client.send(request, self.redirect_policy).await?;
762            Ok(HttpResponseStream {
763                response,
764                deadline,
765                origin: origin.clone(),
766                #[cfg(all(feature = "simulation", madsim))]
767                _connection: connection,
768            })
769        };
770
771        let result = match deadline {
772            Some(deadline) => tokio::select! {
773                biased;
774                () = crate::dst::time::sleep_until(deadline) => Err(HttpClientError::TimeoutError(REQUEST_TIMEOUT_MESSAGE.into())),
775                result = operation => result,
776            },
777            None => operation.await,
778        };
779
780        result.map_err(|e| response_error(e, origin.as_deref()))
781    }
782
783    async fn consume_response<B>(
784        &self,
785        response: http::Response<B>,
786        deadline: Option<crate::dst::time::Instant>,
787        is_head: bool,
788    ) -> Result<HttpResponse, HttpClientError>
789    where
790        B: http_body::Body<Data = Bytes> + Unpin,
791        B::Error: std::error::Error + 'static,
792    {
793        let (parts, mut body) = response.into_parts();
794        let mut headers =
795            HashMap::with_capacity(self.response_headers.len().min(parts.headers.len()));
796        for (key, name) in self.response_headers.iter() {
797            if let Some(value) = parts
798                .headers
799                .get(name)
800                .and_then(|value| value.to_str().ok())
801            {
802                headers.insert(key.clone(), value.to_owned());
803            }
804        }
805
806        let max = self.max_response_bytes;
807
808        if let Some(len) = body.size_hint().exact()
809            && len > max as u64
810        {
811            return Err(HttpClientError::Error(format!(
812                "HTTP response body of {len} bytes exceeds maximum of {max} bytes",
813            )));
814        }
815
816        let mut buf = bytes::BytesMut::new();
817        while let Some(chunk) = read_chunk(&mut body, deadline).await? {
818            if chunk.len() > max - buf.len() {
819                return Err(HttpClientError::Error(format!(
820                    "HTTP response body exceeds maximum of {max} bytes",
821                )));
822            }
823
824            buf.extend_from_slice(&chunk);
825        }
826
827        log::trace!(
828            "Received HTTP response: status={} headers={} body_bytes={}",
829            parts.status,
830            parts.headers.len(),
831            buf.len()
832        );
833
834        let bodyless = is_head
835            || parts.status.is_informational()
836            || matches!(
837                parts.status,
838                http::StatusCode::NO_CONTENT
839                    | http::StatusCode::RESET_CONTENT
840                    | http::StatusCode::NOT_MODIFIED
841            );
842
843        let (body, decoded) = if bodyless {
844            (buf.freeze(), false)
845        } else {
846            decode_content_encoding(&parts.headers, buf, self.max_decompressed_bytes, deadline)
847                .await?
848        };
849
850        if decoded {
851            headers.retain(|key, _| {
852                !key.eq_ignore_ascii_case("content-encoding")
853                    && !key.eq_ignore_ascii_case("content-length")
854            });
855        }
856
857        Ok(HttpResponse {
858            status: HttpStatus::new(parts.status),
859            headers,
860            body,
861        })
862    }
863}
864
865enum RequestBody {
866    Plain(Vec<u8>),
867    Secret(SecretString),
868}
869
870fn request_deadline(
871    timeout: Option<Duration>,
872) -> Result<Option<crate::dst::time::Instant>, HttpClientError> {
873    timeout
874        .map(|duration| {
875            crate::dst::time::Instant::now()
876                .checked_add(duration)
877                .ok_or_else(|| {
878                    HttpClientError::Error("HTTP request timeout exceeds clock range".into())
879                })
880        })
881        .transpose()
882}
883
884async fn decode_content_encoding(
885    headers: &HeaderMap,
886    body: bytes::BytesMut,
887    max: usize,
888    deadline: Option<crate::dst::time::Instant>,
889) -> Result<(Bytes, bool), HttpClientError> {
890    let mut encodings = headers.get_all(http::header::CONTENT_ENCODING).iter();
891
892    let Some(encoding) = encodings.next() else {
893        return Ok((body.freeze(), false));
894    };
895
896    if encodings.next().is_some() {
897        return Err(HttpClientError::Error(
898            "unsupported multiple HTTP content-encoding fields".into(),
899        ));
900    }
901
902    let encoding = encoding
903        .to_str()
904        .map_err(|_| HttpClientError::Error("invalid HTTP content-encoding value".into()))?;
905    let encoding = encoding.trim();
906    if encoding.is_empty() || encoding.eq_ignore_ascii_case("identity") {
907        return Ok((body.freeze(), false));
908    }
909
910    if !encoding.eq_ignore_ascii_case(GZIP) {
911        return Err(HttpClientError::Error(format!(
912            "unsupported HTTP content-encoding '{encoding}'"
913        )));
914    }
915
916    check_deadline(deadline)?;
917    let body = body.freeze();
918    let mut out = Vec::new();
919
920    let mut reader = std::io::Read::take(
921        flate2::bufread::MultiGzDecoder::new(GzipInput {
922            bytes: body.as_ref(),
923            remaining: GZIP_DECODE_CHUNK_BYTES,
924        }),
925        (max as u64).saturating_add(1),
926    );
927
928    loop {
929        check_deadline(deadline)?;
930        let mut chunk = std::io::Read::take(&mut reader, GZIP_DECODE_CHUNK_BYTES as u64);
931        let result = std::io::Read::read_to_end(&mut chunk, &mut out);
932        check_deadline(deadline)?;
933
934        let complete = match result {
935            Ok(count) => count < GZIP_DECODE_CHUNK_BYTES,
936            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => false,
937            Err(e) => {
938                return Err(HttpClientError::Error(format!(
939                    "failed to decode gzip response body: {e}"
940                )));
941            }
942        };
943
944        if out.len() > max {
945            return Err(HttpClientError::Error(format!(
946                "decompressed HTTP response body exceeds maximum of {max} bytes"
947            )));
948        }
949
950        if complete {
951            break;
952        }
953
954        crate::dst::task::yield_now().await;
955        reader.get_mut().get_mut().remaining = GZIP_DECODE_CHUNK_BYTES;
956    }
957
958    Ok((Bytes::from(out), true))
959}
960
961struct GzipInput<'a> {
962    bytes: &'a [u8],
963    remaining: usize,
964}
965
966impl std::io::Read for GzipInput<'_> {
967    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
968        if buf.is_empty() {
969            return Ok(0);
970        }
971
972        let mut available = std::io::BufRead::fill_buf(self)?;
973        let count = std::io::Read::read(&mut available, buf)?;
974        std::io::BufRead::consume(self, count);
975        Ok(count)
976    }
977}
978
979impl std::io::BufRead for GzipInput<'_> {
980    fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
981        if self.bytes.is_empty() {
982            return Ok(self.bytes);
983        }
984
985        if self.remaining == 0 {
986            return Err(std::io::ErrorKind::WouldBlock.into());
987        }
988
989        Ok(&self.bytes[..self.bytes.len().min(self.remaining)])
990    }
991
992    fn consume(&mut self, count: usize) {
993        self.bytes = &self.bytes[count..];
994        self.remaining -= count;
995    }
996}
997
998impl RequestBody {
999    fn into_bytes(self) -> Bytes {
1000        match self {
1001            Self::Plain(body) => body.into(),
1002            Self::Secret(body) => Bytes::from_owner(SecretBody(body)),
1003        }
1004    }
1005}
1006
1007struct SecretBody(SecretString);
1008
1009impl AsRef<[u8]> for SecretBody {
1010    fn as_ref(&self) -> &[u8] {
1011        self.0.expose_secret().as_bytes()
1012    }
1013}
1014
1015#[cfg(not(all(feature = "simulation", madsim)))]
1016fn http2_adaptive_window() -> Result<bool, HttpClientError> {
1017    let Some(value) = std::env::var_os(NAUTILUS_HTTP2_ADAPTIVE_WINDOW) else {
1018        return Ok(false);
1019    };
1020
1021    value.to_str().and_then(|v| v.parse().ok()).ok_or_else(|| {
1022        HttpClientError::ClientBuildError(format!(
1023            "{NAUTILUS_HTTP2_ADAPTIVE_WINDOW} must be 'true' or 'false', was '{}'",
1024            value.display()
1025        ))
1026    })
1027}
1028
1029/// Encodes URL parameters into the query string.
1030///
1031/// Returns `Cow::Borrowed` when no parameters need appending (zero-alloc fast path).
1032/// Parameters can have multiple values per key (for doseq=True behavior).
1033/// Preserves existing query strings in the URL by appending with '&' instead of '?'.
1034/// The query is inserted before any fragment, which is preserved unchanged.
1035fn encode_url_params<'a>(
1036    url: &'a str,
1037    params: Option<&HashMap<String, Vec<String>>>,
1038) -> Result<Cow<'a, str>, HttpClientError> {
1039    let Some(params) = params else {
1040        return Ok(Cow::Borrowed(url));
1041    };
1042
1043    let pairs: Vec<(&str, &str)> = params
1044        .iter()
1045        .flat_map(|(key, values)| {
1046            values
1047                .iter()
1048                .map(move |value| (key.as_str(), value.as_str()))
1049        })
1050        .collect();
1051
1052    if pairs.is_empty() {
1053        return Ok(Cow::Borrowed(url));
1054    }
1055
1056    let query_string = serde_urlencoded::to_string(pairs)
1057        .map_err(|e| HttpClientError::Error(format!("Failed to encode params: {e}")))?;
1058
1059    // The first literal '#' starts the fragment per RFC 3986 section 3.5.
1060    // A data '#' in an earlier component must be percent-encoded as "%23".
1061    let (base, fragment) = match url.split_once('#') {
1062        Some((base, fragment)) => (base, Some(fragment)),
1063        None => (url, None),
1064    };
1065
1066    let separator = if base.contains('?') { '&' } else { '?' };
1067
1068    Ok(Cow::Owned(match fragment {
1069        Some(fragment) => format!("{base}{separator}{query_string}#{fragment}"),
1070        None => format!("{base}{separator}{query_string}"),
1071    }))
1072}
1073
1074#[cfg(test)]
1075mod encode_url_params_tests {
1076    use std::{borrow::Cow, collections::HashMap};
1077
1078    use rstest::rstest;
1079
1080    use super::encode_url_params;
1081
1082    fn params(pairs: &[(&str, &str)]) -> HashMap<String, Vec<String>> {
1083        let mut map: HashMap<String, Vec<String>> = HashMap::new();
1084        for (key, value) in pairs {
1085            map.entry((*key).to_string())
1086                .or_default()
1087                .push((*value).to_string());
1088        }
1089
1090        map
1091    }
1092
1093    #[rstest]
1094    #[case("https://x/y", "https://x/y?a=b")]
1095    #[case("https://x/y?old=1", "https://x/y?old=1&a=b")]
1096    #[case("https://x/y#frag", "https://x/y?a=b#frag")]
1097    #[case("https://x/y?old=1#frag", "https://x/y?old=1&a=b#frag")]
1098    #[case(
1099        "https://x/y#section?display=full",
1100        "https://x/y?a=b#section?display=full"
1101    )]
1102    #[case("https://x/y#", "https://x/y?a=b#")]
1103    fn test_query_is_inserted_before_the_fragment(#[case] url: &str, #[case] expected: &str) {
1104        let params = params(&[("a", "b")]);
1105
1106        assert_eq!(encode_url_params(url, Some(&params)).unwrap(), expected);
1107    }
1108
1109    #[rstest]
1110    fn test_url_is_borrowed_when_no_params_are_supplied() {
1111        assert!(matches!(
1112            encode_url_params("https://x/y#frag", None).unwrap(),
1113            Cow::Borrowed("https://x/y#frag")
1114        ));
1115    }
1116
1117    #[rstest]
1118    fn test_url_is_borrowed_when_params_are_empty() {
1119        let params = HashMap::new();
1120
1121        assert!(matches!(
1122            encode_url_params("https://x/y#frag", Some(&params)).unwrap(),
1123            Cow::Borrowed("https://x/y#frag")
1124        ));
1125    }
1126}
1127
1128#[cfg(test)]
1129#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
1130#[cfg(not(all(feature = "simulation", madsim)))]
1131mod tests {
1132    use std::net::SocketAddr;
1133
1134    use axum::{
1135        Router,
1136        body::to_bytes,
1137        extract::Request,
1138        response::IntoResponse,
1139        routing::{any, delete, get, patch, post},
1140        serve,
1141    };
1142    use http::status::StatusCode;
1143    use log::Level;
1144    use rstest::rstest;
1145    use tokio::{
1146        io::{AsyncReadExt, AsyncWriteExt},
1147        sync::oneshot,
1148    };
1149
1150    use super::*;
1151    use crate::logging::tests::capture_logs;
1152
1153    #[rstest]
1154    fn test_debug_redacts_custom_default_headers() {
1155        let client = HttpClient::builder()
1156            .use_system_proxy(false)
1157            .headers(HashMap::from([(
1158                "kalshi-access-key".into(),
1159                "HEADER_DEBUG_SECRET".into(),
1160            )]))
1161            .build()
1162            .unwrap();
1163        let header = &client.client.headers["kalshi-access-key"];
1164        let debug = format!("{client:?}");
1165
1166        assert_eq!(header.to_str().unwrap(), "HEADER_DEBUG_SECRET");
1167        assert!(header.is_sensitive());
1168        assert!(!debug.contains("HEADER_DEBUG_SECRET"), "{debug}");
1169    }
1170
1171    #[rstest]
1172    #[case("ftp://127.0.0.1:1/resource")]
1173    #[case("file:///resource")]
1174    #[tokio::test]
1175    async fn test_request_rejects_unsupported_url(#[case] url: &str) {
1176        let client = HttpClient::builder()
1177            .use_system_proxy(false)
1178            .build()
1179            .unwrap();
1180
1181        let error = client
1182            .request(Method::GET, url.to_string(), None, None, None, None, None)
1183            .await
1184            .unwrap_err();
1185
1186        assert!(
1187            matches!(error, HttpClientError::Error(ref message) if message == "unsupported HTTP URL scheme or hostname")
1188        );
1189    }
1190
1191    #[rstest]
1192    #[case::username_only("user%2F17@", "Basic dXNlci8xNzo=")]
1193    #[case::password_only(":secret%2F29@", "Basic OnNlY3JldC8yOQ==")]
1194    #[tokio::test]
1195    async fn test_request_url_credentials_allow_missing_username_or_password(
1196        #[case] userinfo: &str,
1197        #[case] expected: &str,
1198    ) {
1199        let addr = start_test_server().await.unwrap();
1200        let client = HttpClient::builder()
1201            .use_system_proxy(false)
1202            .build()
1203            .unwrap();
1204
1205        let response = client
1206            .request(
1207                Method::GET,
1208                format!("http://{userinfo}{addr}/headers"),
1209                None,
1210                None,
1211                None,
1212                None,
1213                None,
1214            )
1215            .await
1216            .unwrap();
1217
1218        assert_eq!(response.status.as_u16(), 200);
1219        assert_eq!(response.headers, HashMap::new());
1220        assert_eq!(
1221            response.body.as_ref(),
1222            format!("{expected}\n*/*").as_bytes()
1223        );
1224    }
1225
1226    #[rstest]
1227    #[case::default(None, "*/*")]
1228    #[case::explicit(Some("application/octet-stream"), "application/octet-stream")]
1229    #[tokio::test]
1230    async fn test_request_accept_header_preserves_explicit_value(
1231        #[case] accept: Option<&str>,
1232        #[case] expected: &str,
1233    ) {
1234        let addr = start_test_server().await.unwrap();
1235        let client = HttpClient::builder()
1236            .use_system_proxy(false)
1237            .build()
1238            .unwrap();
1239        let headers =
1240            accept.map(|value| HashMap::from([("accept".to_string(), value.to_string())]));
1241
1242        let response = client
1243            .request(
1244                Method::GET,
1245                format!("http://{addr}/headers"),
1246                None,
1247                headers,
1248                None,
1249                None,
1250                None,
1251            )
1252            .await
1253            .unwrap();
1254
1255        assert_eq!(response.status.as_u16(), 200);
1256        assert_eq!(response.headers, HashMap::new());
1257        assert_eq!(
1258            response.body.as_ref(),
1259            format!("absent\n{expected}").as_bytes()
1260        );
1261    }
1262
1263    async fn capture_request(request: Request) -> impl IntoResponse {
1264        let (parts, body) = request.into_parts();
1265        let body = to_bytes(body, usize::MAX).await.unwrap();
1266        let default_header = parts.headers.get("x-default").unwrap().to_str().unwrap();
1267        let request_header = parts.headers.get("x-request").unwrap().to_str().unwrap();
1268        let query = parts.uri.query().unwrap_or_default();
1269        let body = String::from_utf8(body.to_vec()).unwrap();
1270        let capture = format!(
1271            "{}\n{}\n{query}\n{default_header}\n{request_header}\n{body}",
1272            parts.method,
1273            parts.uri.path(),
1274        );
1275
1276        ([("x-response-id", "response-42")], capture)
1277    }
1278
1279    fn create_router() -> Router {
1280        Router::new()
1281            .route("/get", get(|| async { "hello-world!" }))
1282            .route("/post", post(|body: Bytes| async move { body }))
1283            .route("/patch", patch(|body: Bytes| async move { body }))
1284            .route("/delete", delete(|| async { StatusCode::OK }))
1285            .route("/capture", any(capture_request))
1286            .route(
1287                "/headers",
1288                get(|request: Request| async move {
1289                    let headers = request.headers();
1290                    format!(
1291                        "{}\n{}",
1292                        headers
1293                            .get(http::header::AUTHORIZATION)
1294                            .map_or("absent", |v| v.to_str().unwrap()),
1295                        headers
1296                            .get(http::header::ACCEPT)
1297                            .map_or("absent", |v| v.to_str().unwrap()),
1298                    )
1299                }),
1300            )
1301            .route("/notfound", get(|| async { StatusCode::NOT_FOUND }))
1302            .route(
1303                "/redirect",
1304                get(|| async { (StatusCode::TEMPORARY_REDIRECT, [("location", "/get")]) }),
1305            )
1306            .route(
1307                "/slow",
1308                get(|| async {
1309                    tokio::time::sleep(Duration::from_secs(2)).await;
1310                    "Eventually responded"
1311                }),
1312            )
1313            .route(
1314                "/large",
1315                // Returns a 1 MiB body to exercise the response size cap.
1316                get(|| async { "x".repeat(1024 * 1024) }),
1317            )
1318    }
1319
1320    async fn start_test_server() -> Result<SocketAddr, Box<dyn std::error::Error + Send + Sync>> {
1321        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1322        let addr = listener.local_addr().unwrap();
1323        tokio::spawn(async move {
1324            serve(listener, create_router()).await.unwrap();
1325        });
1326
1327        Ok(addr)
1328    }
1329
1330    async fn spawn_connection_dropper() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1331        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1332        let addr = listener.local_addr().unwrap();
1333
1334        let task = tokio::spawn(async move {
1335            loop {
1336                let (stream, _) = listener.accept().await.unwrap();
1337                drop(stream);
1338            }
1339        });
1340
1341        (addr, task)
1342    }
1343
1344    async fn spawn_chunked_response_server() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1345        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1346        let addr = listener.local_addr().unwrap();
1347        let task = tokio::spawn(async move {
1348            let (mut stream, _) = listener.accept().await.unwrap();
1349            let mut request = Vec::new();
1350            let mut chunk = [0u8; 1024];
1351            loop {
1352                let read = stream.read(&mut chunk).await.unwrap();
1353                if read == 0 {
1354                    break;
1355                }
1356
1357                request.extend_from_slice(&chunk[..read]);
1358                if request.windows(4).any(|window| window == b"\r\n\r\n") {
1359                    break;
1360                }
1361            }
1362
1363            stream
1364                .write_all(
1365                    b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n\
1366                      5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n",
1367                )
1368                .await
1369                .unwrap();
1370        });
1371
1372        (addr, task)
1373    }
1374
1375    async fn spawn_rejecting_connect_proxy() -> (SocketAddr, oneshot::Receiver<String>) {
1376        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1377        let addr = listener.local_addr().unwrap();
1378        let (request_tx, request_rx) = oneshot::channel();
1379
1380        tokio::spawn(async move {
1381            let (mut stream, _) = listener.accept().await.unwrap();
1382            let mut request = Vec::new();
1383            let mut chunk = [0u8; 1024];
1384            loop {
1385                let read = stream.read(&mut chunk).await.unwrap();
1386                if read == 0 {
1387                    break;
1388                }
1389
1390                request.extend_from_slice(&chunk[..read]);
1391                if request.windows(4).any(|window| window == b"\r\n\r\n") {
1392                    break;
1393                }
1394            }
1395
1396            request_tx
1397                .send(String::from_utf8(request).unwrap())
1398                .unwrap();
1399            stream
1400                .write_all(
1401                    b"HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n",
1402                )
1403                .await
1404                .unwrap();
1405        });
1406
1407        (addr, request_rx)
1408    }
1409
1410    fn inner_client() -> InnerHttpClient {
1411        HttpClient::builder()
1412            .use_system_proxy(false)
1413            .build()
1414            .unwrap()
1415            .client
1416    }
1417
1418    #[tokio::test(start_paused = true)]
1419    async fn test_body_ready_at_deadline_is_rejected() {
1420        let client = inner_client();
1421        let response = http::Response::new(Full::new(Bytes::from_static(b"ready")));
1422        let result = client
1423            .consume_response(response, Some(crate::dst::time::Instant::now()), false)
1424            .await;
1425        assert!(
1426            matches!(result, Err(HttpClientError::TimeoutError(message)) if message == REQUEST_TIMEOUT_MESSAGE)
1427        );
1428    }
1429
1430    #[tokio::test]
1431    async fn test_get() {
1432        let addr = start_test_server().await.unwrap();
1433        let url = format!("http://{addr}");
1434
1435        let client = inner_client();
1436        let response = client
1437            .send_request(Method::GET, format!("{url}/get"), None, None, None, None)
1438            .await
1439            .unwrap();
1440
1441        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1442        assert_eq!(response.headers, HashMap::new());
1443        assert_eq!(response.body.as_ref(), b"hello-world!");
1444    }
1445
1446    #[rstest]
1447    #[tokio::test]
1448    async fn test_request_preserves_wire_semantics_and_extracts_response_headers(
1449        #[values(false, true)] redact_url: bool,
1450    ) {
1451        let addr = start_test_server().await.unwrap();
1452        let mut default_headers = HashMap::new();
1453        default_headers.insert("x-default".to_string(), "default-a".to_string());
1454        let client = HttpClient::builder()
1455            .redact_url(redact_url)
1456            .headers(default_headers)
1457            .header_keys(vec!["x-response-id".to_string()])
1458            .build()
1459            .unwrap();
1460        let mut params = HashMap::new();
1461        params.insert(
1462            "tag".to_string(),
1463            vec!["A B".to_string(), "C/D".to_string()],
1464        );
1465        let mut request_headers = HashMap::new();
1466        request_headers.insert("x-request".to_string(), "request-b".to_string());
1467
1468        let response = client
1469            .request(
1470                Method::PUT,
1471                format!("http://{addr}/capture?existing=seed"),
1472                Some(&params),
1473                Some(request_headers),
1474                Some(b"payload-c".to_vec()),
1475                None,
1476                None,
1477            )
1478            .await
1479            .unwrap();
1480
1481        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1482        assert_eq!(
1483            response.headers,
1484            HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1485        );
1486        assert_eq!(
1487            response.body.as_ref(),
1488            b"PUT\n/capture\nexisting=seed&tag=A+B&tag=C%2FD\ndefault-a\nrequest-b\npayload-c"
1489        );
1490    }
1491
1492    #[rstest]
1493    #[tokio::test]
1494    async fn test_request_with_secret_body_preserves_wire_body(
1495        #[values(false, true)] redact_url: bool,
1496    ) {
1497        let addr = start_test_server().await.unwrap();
1498        let client = HttpClient::builder()
1499            .redact_url(redact_url)
1500            .headers(HashMap::from([(
1501                "x-default".to_string(),
1502                "default-secret".to_string(),
1503            )]))
1504            .build()
1505            .unwrap();
1506        let headers = HashMap::from([("x-request".to_string(), "request-secret".to_string())]);
1507
1508        let response = client
1509            .request_with_secret_body(
1510                Method::POST,
1511                format!("http://{addr}/capture"),
1512                None,
1513                Some(headers),
1514                SecretString::from("credential-body"),
1515                None,
1516                None,
1517            )
1518            .await
1519            .unwrap();
1520
1521        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1522        assert_eq!(response.headers, HashMap::new());
1523        assert_eq!(
1524            response.body.as_ref(),
1525            b"POST\n/capture\n\ndefault-secret\nrequest-secret\ncredential-body"
1526        );
1527    }
1528
1529    #[rstest]
1530    #[tokio::test]
1531    async fn test_request_with_params_serializes_query_fields(
1532        #[values(false, true)] redact_url: bool,
1533    ) {
1534        #[derive(serde::Serialize)]
1535        struct Query<'a> {
1536            symbol: &'a str,
1537            limit: u32,
1538        }
1539
1540        let addr = start_test_server().await.unwrap();
1541        let mut default_headers = HashMap::new();
1542        default_headers.insert("x-default".to_string(), "default-d".to_string());
1543        let client = HttpClient::builder()
1544            .redact_url(redact_url)
1545            .headers(default_headers)
1546            .header_keys(vec!["x-response-id".to_string()])
1547            .build()
1548            .unwrap();
1549        let mut request_headers = HashMap::new();
1550        request_headers.insert("x-request".to_string(), "request-e".to_string());
1551
1552        let params = Query {
1553            symbol: "BTC/USDT",
1554            limit: 37,
1555        };
1556
1557        let response = client
1558            .request_with_params(
1559                Method::GET,
1560                format!("http://{addr}/capture"),
1561                Some(&params),
1562                Some(request_headers),
1563                None,
1564                None,
1565                None,
1566            )
1567            .await
1568            .unwrap();
1569
1570        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1571        assert_eq!(
1572            response.headers,
1573            HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1574        );
1575        assert_eq!(
1576            response.body.as_ref(),
1577            b"GET\n/capture\nsymbol=BTC%2FUSDT&limit=37\ndefault-d\nrequest-e\n"
1578        );
1579    }
1580
1581    #[rstest]
1582    #[tokio::test]
1583    async fn test_request_with_params_preserves_secret_query_fields(
1584        #[values(false, true)] redact_url: bool,
1585    ) {
1586        #[derive(serde::Serialize)]
1587        struct Query<'a> {
1588            auth: &'a str,
1589            market_id: i16,
1590        }
1591
1592        let addr = start_test_server().await.unwrap();
1593        let client = HttpClient::builder()
1594            .redact_url(redact_url)
1595            .headers(HashMap::from([(
1596                "x-default".to_string(),
1597                "default-f".to_string(),
1598            )]))
1599            .build()
1600            .unwrap();
1601        let headers = HashMap::from([("x-request".to_string(), "request-g".to_string())]);
1602
1603        let params = Query {
1604            auth: "token/42",
1605            market_id: 7,
1606        };
1607
1608        let response = client
1609            .request_with_params(
1610                Method::GET,
1611                format!("http://{addr}/capture"),
1612                Some(&params),
1613                Some(headers),
1614                None,
1615                None,
1616                None,
1617            )
1618            .await
1619            .unwrap();
1620
1621        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1622        assert_eq!(
1623            response.body.as_ref(),
1624            b"GET\n/capture\nauth=token%2F42&market_id=7\ndefault-f\nrequest-g\n"
1625        );
1626    }
1627
1628    #[rstest]
1629    #[case::empty_at_zero_cap(b"", 0)]
1630    #[case::at_cap(b"body-37", 7)]
1631    #[case::below_cap(b"body-37", 8)]
1632    #[tokio::test]
1633    async fn test_declared_response_body_at_or_below_cap_is_returned(
1634        #[case] bytes: &'static [u8],
1635        #[case] max_response_bytes: usize,
1636    ) {
1637        let client = InnerHttpClient {
1638            max_response_bytes,
1639            ..inner_client()
1640        };
1641
1642        let response = http::Response::new(Full::new(Bytes::from_static(bytes)));
1643
1644        let response = client
1645            .consume_response(response, None, false)
1646            .await
1647            .unwrap();
1648
1649        assert_eq!(response.status.as_u16(), 200);
1650        assert_eq!(response.headers, HashMap::new());
1651        assert_eq!(response.body.as_ref(), bytes);
1652    }
1653
1654    #[tokio::test]
1655    async fn test_response_body_within_cap_is_returned() {
1656        let addr = start_test_server().await.unwrap();
1657        let url = format!("http://{addr}");
1658
1659        // Cap above the 1 MiB payload: body should be returned intact.
1660        let client = InnerHttpClient {
1661            max_response_bytes: 4 * 1024 * 1024,
1662            ..inner_client()
1663        };
1664
1665        let response = client
1666            .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1667            .await
1668            .unwrap();
1669
1670        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1671        assert_eq!(response.headers, HashMap::new());
1672        assert_eq!(response.body.as_ref(), vec![b'x'; 1024 * 1024]);
1673    }
1674
1675    #[tokio::test]
1676    async fn test_response_body_exceeding_cap_is_rejected() {
1677        let addr = start_test_server().await.unwrap();
1678        let url = format!("http://{addr}");
1679
1680        // Cap below the 1 MiB payload: the request must fail rather than buffer it.
1681        let client = InnerHttpClient {
1682            max_response_bytes: 16 * 1024,
1683            ..inner_client()
1684        };
1685
1686        let result = client
1687            .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1688            .await;
1689
1690        let err = result.expect_err("oversized response body should be rejected");
1691
1692        let HttpClientError::Error(message) = err else {
1693            panic!("expected HTTP error, was {err:?}");
1694        };
1695
1696        assert_eq!(
1697            message,
1698            "HTTP response body of 1048576 bytes exceeds maximum of 16384 bytes"
1699        );
1700    }
1701
1702    #[rstest]
1703    #[case::at_cap(11)]
1704    #[case::below_cap(12)]
1705    #[tokio::test]
1706    async fn test_chunked_response_body_at_or_below_cap_is_returned(
1707        #[case] max_response_bytes: usize,
1708    ) {
1709        let (addr, server_task) = spawn_chunked_response_server().await;
1710
1711        let client = InnerHttpClient {
1712            max_response_bytes,
1713            ..inner_client()
1714        };
1715
1716        let response = client
1717            .send_request(
1718                Method::GET,
1719                format!("http://{addr}"),
1720                None,
1721                None,
1722                None,
1723                None,
1724            )
1725            .await
1726            .unwrap();
1727        server_task.await.unwrap();
1728
1729        assert_eq!(response.status.as_u16(), 200);
1730        assert_eq!(response.headers, HashMap::new());
1731        assert_eq!(response.body.as_ref(), b"firstsecond");
1732    }
1733
1734    #[tokio::test]
1735    async fn test_chunked_response_body_exceeding_cap_is_rejected() {
1736        let (addr, server_task) = spawn_chunked_response_server().await;
1737        let max_response_bytes = 8;
1738
1739        let client = InnerHttpClient {
1740            max_response_bytes,
1741            ..inner_client()
1742        };
1743
1744        let error = client
1745            .send_request(
1746                Method::GET,
1747                format!("http://{addr}"),
1748                None,
1749                None,
1750                None,
1751                None,
1752            )
1753            .await
1754            .expect_err("chunked response body should be rejected");
1755        server_task.await.unwrap();
1756
1757        let HttpClientError::Error(message) = error else {
1758            panic!("expected HTTP error, was {error:?}");
1759        };
1760
1761        assert_eq!(
1762            message,
1763            format!("HTTP response body exceeds maximum of {max_response_bytes} bytes")
1764        );
1765    }
1766
1767    #[tokio::test]
1768    async fn test_post() {
1769        let addr = start_test_server().await.unwrap();
1770        let url = format!("http://{addr}");
1771
1772        let client = inner_client();
1773        let response = client
1774            .send_request(Method::POST, format!("{url}/post"), None, None, None, None)
1775            .await
1776            .unwrap();
1777
1778        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1779        assert_eq!(response.headers, HashMap::new());
1780        assert_eq!(response.body.as_ref(), b"");
1781    }
1782
1783    #[tokio::test]
1784    async fn test_post_with_body() {
1785        let addr = start_test_server().await.unwrap();
1786        let url = format!("http://{addr}");
1787
1788        let client = inner_client();
1789
1790        let mut body = HashMap::new();
1791        body.insert(
1792            "key1".to_string(),
1793            serde_json::Value::String("value1".to_string()),
1794        );
1795        body.insert(
1796            "key2".to_string(),
1797            serde_json::Value::String("value2".to_string()),
1798        );
1799
1800        let body_string = serde_json::to_string(&body).unwrap();
1801        let body_bytes = body_string.into_bytes();
1802
1803        let response = client
1804            .send_request(
1805                Method::POST,
1806                format!("{url}/post"),
1807                None,
1808                None,
1809                Some(body_bytes.clone()),
1810                None,
1811            )
1812            .await
1813            .unwrap();
1814
1815        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1816        assert_eq!(response.headers, HashMap::new());
1817        assert_eq!(response.body.as_ref(), body_bytes);
1818    }
1819
1820    #[tokio::test]
1821    async fn test_patch() {
1822        let addr = start_test_server().await.unwrap();
1823        let url = format!("http://{addr}");
1824
1825        let client = inner_client();
1826        let response = client
1827            .send_request(
1828                Method::PATCH,
1829                format!("{url}/patch"),
1830                None,
1831                None,
1832                None,
1833                None,
1834            )
1835            .await
1836            .unwrap();
1837
1838        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1839        assert_eq!(response.headers, HashMap::new());
1840        assert_eq!(response.body.as_ref(), b"");
1841    }
1842
1843    #[tokio::test]
1844    async fn test_delete() {
1845        let addr = start_test_server().await.unwrap();
1846        let url = format!("http://{addr}");
1847
1848        let client = inner_client();
1849        let response = client
1850            .send_request(
1851                Method::DELETE,
1852                format!("{url}/delete"),
1853                None,
1854                None,
1855                None,
1856                None,
1857            )
1858            .await
1859            .unwrap();
1860
1861        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1862        assert_eq!(response.headers, HashMap::new());
1863        assert_eq!(response.body.as_ref(), b"");
1864    }
1865
1866    #[tokio::test]
1867    async fn test_not_found() {
1868        let addr = start_test_server().await.unwrap();
1869        let url = format!("http://{addr}/notfound");
1870        let client = inner_client();
1871
1872        let response = client
1873            .send_request(Method::GET, url, None, None, None, None)
1874            .await
1875            .unwrap();
1876
1877        assert!(response.status.is_client_error());
1878        assert_eq!(response.status.as_u16(), 404);
1879        assert_eq!(response.headers, HashMap::new());
1880        assert_eq!(response.body.as_ref(), b"");
1881    }
1882
1883    #[tokio::test]
1884    async fn test_timeout() {
1885        let addr = start_test_server().await.unwrap();
1886        let url = format!("http://{addr}/slow");
1887        let client = inner_client();
1888
1889        // We'll set a 1-second timeout for a route that sleeps 2 seconds
1890        let result = client
1891            .send_request(Method::GET, url, None, None, None, Some(1))
1892            .await;
1893
1894        assert!(
1895            matches!(&result, Err(HttpClientError::TimeoutError(_))),
1896            "Expected a timeout error, was: {result:?}"
1897        );
1898    }
1899
1900    #[rstest]
1901    fn test_http_client_without_proxy() {
1902        // Create client with no proxy
1903        let result = HttpClient::builder().build();
1904
1905        assert!(result.is_ok());
1906    }
1907
1908    #[rstest]
1909    fn test_http_client_rejects_timeout_overflow() {
1910        let result = HttpClient::builder().timeout_secs(u64::MAX).build();
1911
1912        assert!(matches!(
1913            result,
1914            Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1915        ));
1916    }
1917
1918    #[tokio::test]
1919    async fn test_http_request_rejects_timeout_override_overflow() {
1920        let client = HttpClient::builder()
1921            .use_system_proxy(false)
1922            .timeout_secs(1)
1923            .build()
1924            .unwrap();
1925        let result = client
1926            .request(
1927                Method::GET,
1928                "http://127.0.0.1:1/".to_string(),
1929                None,
1930                None,
1931                None,
1932                Some(u64::MAX),
1933                None,
1934            )
1935            .await;
1936
1937        assert!(matches!(
1938            result,
1939            Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1940        ));
1941    }
1942
1943    #[rstest]
1944    fn test_http_client_builder_preserves_empty_rate_limiters() {
1945        let client = HttpClient::builder()
1946            .rate_limiters(Vec::new())
1947            .build()
1948            .unwrap();
1949
1950        assert!(client.rate_limiters.is_empty());
1951    }
1952
1953    #[rstest]
1954    fn test_http_client_builder_rejects_shared_rate_limiters_with_quotas() {
1955        let quota = Quota::with_period(Duration::from_secs(1)).unwrap();
1956        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
1957        let result = HttpClient::builder()
1958            .default_quota(quota)
1959            .rate_limiters(vec![rate_limiter])
1960            .build();
1961
1962        assert_eq!(
1963            result.unwrap_err().to_string(),
1964            "HTTP error occurred: Cannot combine shared rate limiters with quota configuration"
1965        );
1966    }
1967
1968    #[tokio::test]
1969    async fn test_http_client_without_proxy_requests_directly() {
1970        let addr = start_test_server().await.unwrap();
1971        let client = HttpClient::builder().timeout_secs(2).build().unwrap();
1972        let response = client
1973            .request(
1974                Method::GET,
1975                format!("http://{addr}/get"),
1976                None,
1977                None,
1978                None,
1979                None,
1980                None,
1981            )
1982            .await
1983            .expect("direct request");
1984
1985        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1986        assert_eq!(response.body.as_ref(), b"hello-world!");
1987    }
1988
1989    #[tokio::test]
1990    async fn test_http_client_redirect_policy() {
1991        let addr = start_test_server().await.unwrap();
1992        let follow = HttpClient::builder().timeout_secs(2).build().unwrap();
1993        let reject = HttpClient::builder()
1994            .timeout_secs(2)
1995            .redirect_policy(HttpRedirectPolicy::Reject)
1996            .build()
1997            .unwrap();
1998
1999        let followed = follow
2000            .request(
2001                Method::GET,
2002                format!("http://{addr}/redirect"),
2003                None,
2004                None,
2005                None,
2006                None,
2007                None,
2008            )
2009            .await
2010            .unwrap();
2011        let rejected = reject
2012            .request(
2013                Method::GET,
2014                format!("http://{addr}/redirect"),
2015                None,
2016                None,
2017                None,
2018                None,
2019                None,
2020            )
2021            .await
2022            .unwrap();
2023
2024        assert_eq!(followed.status.as_u16(), StatusCode::OK.as_u16());
2025        assert_eq!(followed.body.as_ref(), b"hello-world!");
2026        assert_eq!(
2027            rejected.status.as_u16(),
2028            StatusCode::TEMPORARY_REDIRECT.as_u16()
2029        );
2030        assert!(rejected.body.is_empty());
2031    }
2032
2033    #[rstest]
2034    #[tokio::test]
2035    async fn test_http_client_url_policy_preserves_response(
2036        #[values(false, true)] redact_url: bool,
2037    ) {
2038        let addr = start_test_server().await.unwrap();
2039        let client = HttpClient::builder()
2040            .redact_url(redact_url)
2041            .timeout_secs(2)
2042            .build()
2043            .unwrap();
2044        let response = client
2045            .request(
2046                Method::GET,
2047                format!("http://{addr}/get"),
2048                None,
2049                None,
2050                None,
2051                None,
2052                None,
2053            )
2054            .await
2055            .expect("direct request with URL redaction");
2056
2057        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2058        assert_eq!(response.body.as_ref(), b"hello-world!");
2059    }
2060
2061    #[rstest]
2062    #[case::request("request")]
2063    #[case::preinterned_keys("keys")]
2064    #[case::get("get")]
2065    #[case::post("post")]
2066    #[case::patch("patch")]
2067    #[case::delete("delete")]
2068    #[case::stream("stream")]
2069    #[tokio::test]
2070    async fn test_http_client_url_policy_redacts_transport_errors(
2071        #[case] request: &str,
2072        #[values(false, true)] redact_url: bool,
2073    ) {
2074        const USERINFO_SECRET: &str = "transport-userinfo-secret";
2075        const PATH_SECRET: &str = "transport-path-secret";
2076        const QUERY_SECRET: &str = "transport-query-secret";
2077        let (addr, drop_task) = spawn_connection_dropper().await;
2078        let url = format!(
2079            "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2080        );
2081        let client = HttpClient::builder()
2082            .redact_url(redact_url)
2083            .timeout_secs(1)
2084            .build()
2085            .unwrap();
2086
2087        let error = match request {
2088            "request" => client
2089                .request(Method::GET, url.clone(), None, None, None, None, None)
2090                .await
2091                .unwrap_err(),
2092            "keys" => client
2093                .request_with_ustr_keys(
2094                    Method::GET,
2095                    url.clone(),
2096                    None,
2097                    None,
2098                    None,
2099                    None,
2100                    Some(vec![Ustr::from("policy")]),
2101                )
2102                .await
2103                .unwrap_err(),
2104            "get" => client
2105                .get(url.clone(), None, None, None, None)
2106                .await
2107                .unwrap_err(),
2108            "post" => client
2109                .post(url.clone(), None, None, None, None, None)
2110                .await
2111                .unwrap_err(),
2112            "patch" => client
2113                .patch(url.clone(), None, None, None, None, None)
2114                .await
2115                .unwrap_err(),
2116            "delete" => client
2117                .delete(url.clone(), None, None, None, None)
2118                .await
2119                .unwrap_err(),
2120            "stream" => client.get_stream(url.clone()).await.unwrap_err(),
2121            _ => panic!("unknown request shape"),
2122        };
2123
2124        drop_task.abort();
2125        let task_error = drop_task
2126            .await
2127            .expect_err("connection dropper should be cancelled");
2128
2129        assert!(task_error.is_cancelled());
2130        assert!(
2131            matches!(error, HttpClientError::TransportError(_)),
2132            "{error:?}"
2133        );
2134
2135        for rendered in [error.to_string(), format!("{error:?}")] {
2136            assert!(!rendered.contains(USERINFO_SECRET));
2137            assert!(!rendered.contains(PATH_SECRET));
2138            assert!(!rendered.contains(QUERY_SECRET));
2139            assert!(!rendered.contains(&url));
2140            assert_eq!(
2141                rendered.contains(&format!("http://{addr}")),
2142                !redact_url,
2143                "{rendered}"
2144            );
2145            assert_eq!(rendered.contains("for origin ("), !redact_url, "{rendered}");
2146        }
2147    }
2148
2149    #[tokio::test]
2150    async fn test_request_with_params_redact_url_removes_origin_from_error() {
2151        const QUERY_SECRET: &str = "transport-query-secret";
2152        #[derive(serde::Serialize)]
2153        struct Query<'a> {
2154            auth: &'a str,
2155        }
2156
2157        let (addr, drop_task) = spawn_connection_dropper().await;
2158        let url = format!("http://{addr}/trades");
2159        let params = Query { auth: QUERY_SECRET };
2160        let client = HttpClient::builder()
2161            .redact_url(true)
2162            .timeout_secs(1)
2163            .build()
2164            .unwrap();
2165
2166        let error = client
2167            .request_with_params(Method::GET, url, Some(&params), None, None, None, None)
2168            .await
2169            .expect_err("a dropped connection should fail");
2170        drop_task.abort();
2171        let task_error = drop_task
2172            .await
2173            .expect_err("connection dropper should be cancelled");
2174
2175        assert!(task_error.is_cancelled());
2176
2177        for rendered in [error.to_string(), format!("{error:?}")] {
2178            assert!(!rendered.contains(&addr.to_string()));
2179            assert!(!rendered.contains("auth="));
2180            assert!(!rendered.contains(QUERY_SECRET));
2181        }
2182    }
2183
2184    #[tokio::test]
2185    async fn test_request_removes_query_string_from_transport_error_by_default() {
2186        const QUERY_SECRET: &str = "default-query-secret";
2187        const FRAGMENT_MARKER: &str = "default-fragment-marker";
2188        let (addr, drop_task) = spawn_connection_dropper().await;
2189        let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}#{FRAGMENT_MARKER}");
2190        let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2191
2192        let error = client
2193            .request(Method::GET, url, None, None, None, None, None)
2194            .await
2195            .expect_err("a dropped connection should fail");
2196        drop_task.abort();
2197        let task_error = drop_task
2198            .await
2199            .expect_err("connection dropper should be cancelled");
2200
2201        assert!(task_error.is_cancelled());
2202
2203        for rendered in [error.to_string(), format!("{error:?}")] {
2204            assert!(
2205                rendered.contains(&format!("for origin (http://{addr})")),
2206                "default error omitted the origin: {rendered}"
2207            );
2208            assert!(!rendered.contains("/trades"), "{rendered}");
2209            assert!(!rendered.contains("api_key="), "{rendered}");
2210            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2211            assert!(!rendered.contains(FRAGMENT_MARKER), "{rendered}");
2212        }
2213    }
2214
2215    #[rstest]
2216    #[tokio::test]
2217    async fn test_request_with_secret_body_url_policy_redacts_transport_error(
2218        #[values(false, true)] redact_url: bool,
2219    ) {
2220        const QUERY_SECRET: &str = "secret-body-query-secret";
2221        let (addr, drop_task) = spawn_connection_dropper().await;
2222        let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}");
2223        let client = HttpClient::builder()
2224            .redact_url(redact_url)
2225            .timeout_secs(1)
2226            .build()
2227            .unwrap();
2228
2229        let error = client
2230            .request_with_secret_body(
2231                Method::POST,
2232                url,
2233                None,
2234                None,
2235                SecretString::from("credential-body"),
2236                None,
2237                None,
2238            )
2239            .await
2240            .expect_err("a dropped connection should fail");
2241        drop_task.abort();
2242        let task_error = drop_task
2243            .await
2244            .expect_err("connection dropper should be cancelled");
2245
2246        assert!(task_error.is_cancelled());
2247
2248        for rendered in [error.to_string(), format!("{error:?}")] {
2249            assert_eq!(
2250                rendered.contains(&format!("for origin (http://{addr})")),
2251                !redact_url,
2252                "{rendered}"
2253            );
2254            assert_eq!(
2255                rendered.contains(&addr.to_string()),
2256                !redact_url,
2257                "{rendered}"
2258            );
2259            assert!(!rendered.contains("/trades"), "{rendered}");
2260            assert!(!rendered.contains("credential-body"), "{rendered}");
2261            assert!(!rendered.contains("api_key="), "{rendered}");
2262            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2263        }
2264    }
2265
2266    #[tokio::test]
2267    async fn test_request_with_params_removes_query_from_transport_error_by_default() {
2268        const QUERY_SECRET: &str = "default-params-query-secret";
2269        #[derive(serde::Serialize)]
2270        struct Query<'a> {
2271            auth: &'a str,
2272        }
2273
2274        let (addr, drop_task) = spawn_connection_dropper().await;
2275        let url = format!("http://{addr}/trades");
2276        let params = Query { auth: QUERY_SECRET };
2277        let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2278
2279        let error = client
2280            .request_with_params(Method::GET, url, Some(&params), None, None, None, None)
2281            .await
2282            .expect_err("a dropped connection should fail");
2283        drop_task.abort();
2284        let task_error = drop_task
2285            .await
2286            .expect_err("connection dropper should be cancelled");
2287
2288        assert!(task_error.is_cancelled());
2289
2290        for rendered in [error.to_string(), format!("{error:?}")] {
2291            assert!(
2292                rendered.contains(&format!("for origin (http://{addr})")),
2293                "default error omitted the origin: {rendered}"
2294            );
2295            assert!(!rendered.contains("/trades"), "{rendered}");
2296            assert!(!rendered.contains("auth="), "{rendered}");
2297            assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2298        }
2299    }
2300
2301    #[rstest]
2302    #[tokio::test]
2303    async fn test_http_client_url_policy_removes_endpoint_from_trace_logs(
2304        #[values(false, true)] redact_url: bool,
2305    ) {
2306        const USERINFO_SECRET: &str = "trace-userinfo-secret";
2307        const PATH_SECRET: &str = "trace-path-secret";
2308        const QUERY_SECRET: &str = "trace-query-secret";
2309        let capture = capture_logs().await;
2310        let addr = start_test_server().await.unwrap();
2311        let url = format!(
2312            "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2313        );
2314        let client = HttpClient::builder()
2315            .redact_url(redact_url)
2316            .timeout_secs(2)
2317            .build()
2318            .unwrap();
2319
2320        let response = client
2321            .request(Method::GET, url.clone(), None, None, None, None, None)
2322            .await
2323            .expect("credentialized endpoint should return an HTTP response");
2324        let messages = capture.messages();
2325
2326        assert_eq!(response.status.as_u16(), StatusCode::NOT_FOUND.as_u16());
2327        assert!(messages.iter().any(|(level, message)| {
2328            *level == Level::Trace && message.starts_with("Sending HTTP request: method=GET")
2329        }));
2330        assert!(messages.iter().any(|(level, message)| {
2331            *level == Level::Trace
2332                && message.starts_with("Received HTTP response: status=404 Not Found")
2333        }));
2334
2335        for (_, message) in messages {
2336            assert!(!message.contains(USERINFO_SECRET));
2337            assert!(!message.contains(PATH_SECRET));
2338            assert!(!message.contains(QUERY_SECRET));
2339            assert!(!message.contains(&url));
2340        }
2341    }
2342
2343    #[rstest]
2344    #[case::without_user_agent(None)]
2345    #[case::with_user_agent(Some("NautilusTrader/proxy-test-73"))]
2346    #[tokio::test]
2347    async fn test_http_client_uses_connect_and_proxy_authorization_for_https(
2348        #[case] user_agent: Option<&str>,
2349    ) {
2350        const USERNAME: &str = "proxytest";
2351        const PASSWORD: &str = "fixture42";
2352        let mut headers =
2353            HashMap::from([("authorization".into(), "Bearer origin-secret-19".into())]);
2354
2355        if let Some(user_agent) = user_agent {
2356            headers.insert("user-agent".into(), user_agent.into());
2357        }
2358
2359        let (proxy_addr, request_rx) = spawn_rejecting_connect_proxy().await;
2360        let client = HttpClient::builder()
2361            .headers(headers)
2362            .timeout_secs(2)
2363            .proxy_url(format!("http://{USERNAME}:{PASSWORD}@{proxy_addr}"))
2364            .build()
2365            .unwrap();
2366        let error = client
2367            .request(
2368                Method::GET,
2369                "https://fixture.example.test/path".to_string(),
2370                None,
2371                None,
2372                None,
2373                None,
2374                None,
2375            )
2376            .await
2377            .expect_err("proxy should reject CONNECT");
2378        let request = request_rx.await.expect("captured CONNECT request");
2379        let mut lines = request.split("\r\n");
2380        let request_line = lines.next().expect("CONNECT request line");
2381
2382        let headers: HashMap<_, _> = lines
2383            .filter_map(|line| {
2384                let (name, value) = line.split_once(':')?;
2385                Some((name.to_ascii_lowercase(), value.trim()))
2386            })
2387            .collect();
2388
2389        let expected_auth = format!("Basic {}", BASE64.encode(format!("{USERNAME}:{PASSWORD}")));
2390        let mut expected_headers = HashMap::from([
2391            ("host".into(), "fixture.example.test:443"),
2392            ("proxy-authorization".into(), expected_auth.as_str()),
2393        ]);
2394
2395        if let Some(user_agent) = user_agent {
2396            expected_headers.insert("user-agent".into(), user_agent);
2397        }
2398
2399        assert_eq!(request_line, "CONNECT fixture.example.test:443 HTTP/1.1");
2400        assert_eq!(headers, expected_headers);
2401        assert!(!error.to_string().contains(PASSWORD));
2402        assert!(!error.to_string().contains(&BASE64.encode(PASSWORD)));
2403        assert!(!error.to_string().contains(&expected_auth));
2404    }
2405
2406    #[rstest]
2407    #[case::cross_origin("http://127.0.0.1:2/final", None, false)]
2408    #[case::cross_origin_override("http://127.0.0.1:2/final", Some("Basic b3ZlcnJpZGU="), false)]
2409    #[case::same_origin("http://127.0.0.1:1/final", None, true)]
2410    #[case::same_origin_override("http://127.0.0.1:1/final", Some("Basic b3ZlcnJpZGU="), true)]
2411    #[case::https_connect("https://127.0.0.1:2/final", None, false)]
2412    #[tokio::test]
2413    async fn test_proxy_authentication_per_redirect_hop(
2414        #[case] destination: &'static str,
2415        #[case] initial_auth: Option<&'static str>,
2416        #[case] same_origin: bool,
2417    ) {
2418        const PROXY_AUTH: &str = "Basic cHJveHl0ZXN0OmZpeHR1cmU0Mg==";
2419
2420        let expected_initial_auth = initial_auth.unwrap_or(PROXY_AUTH);
2421        let expected_redirect_auth = if same_origin {
2422            expected_initial_auth
2423        } else {
2424            PROXY_AUTH
2425        };
2426
2427        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2428        let addr = listener.local_addr().unwrap();
2429        let (request_tx, mut request_rx) = tokio::sync::mpsc::channel(2);
2430        let router = Router::new().fallback(any(move |request: Request| {
2431            let request_tx = request_tx.clone();
2432
2433            async move {
2434                let (parts, _) = request.into_parts();
2435                let first = parts.uri.path() == "/start";
2436
2437                let expected_auth = if first {
2438                    expected_initial_auth
2439                } else {
2440                    expected_redirect_auth
2441                };
2442
2443                let authenticated = parts.headers.get(http::header::PROXY_AUTHORIZATION)
2444                    == Some(&http::HeaderValue::from_static(expected_auth));
2445                let connect = parts.method == http::Method::CONNECT;
2446                request_tx.send(parts).await.unwrap();
2447
2448                if !authenticated || connect {
2449                    return StatusCode::PROXY_AUTHENTICATION_REQUIRED.into_response();
2450                }
2451
2452                if first {
2453                    return (StatusCode::TEMPORARY_REDIRECT, [("location", destination)])
2454                        .into_response();
2455                }
2456
2457                "accepted".into_response()
2458            }
2459        }));
2460
2461        let server = tokio::spawn(async move { serve(listener, router).await.unwrap() });
2462        let origin_headers = HashMap::from([
2463            ("authorization".to_string(), "Bearer origin-19".to_string()),
2464            ("cookie".to_string(), "session=origin-23".to_string()),
2465            ("cookie2".to_string(), "legacy=origin-29".to_string()),
2466            (
2467                "www-authenticate".to_string(),
2468                "Basic realm=origin-31".to_string(),
2469            ),
2470        ]);
2471        let mut headers = origin_headers.clone();
2472        if let Some(initial_auth) = initial_auth {
2473            headers.insert("proxy-authorization".to_string(), initial_auth.to_string());
2474        }
2475
2476        let client = HttpClient::builder()
2477            .headers(headers)
2478            .timeout_secs(2)
2479            .use_system_proxy(false)
2480            .proxy_url(format!("http://proxytest:fixture42@{addr}"))
2481            .build()
2482            .unwrap();
2483        let result = client
2484            .request(
2485                Method::GET,
2486                "http://127.0.0.1:1/start".to_string(),
2487                None,
2488                None,
2489                None,
2490                None,
2491                None,
2492            )
2493            .await;
2494        let first = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2495            .await
2496            .unwrap()
2497            .unwrap();
2498        let redirected = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2499            .await
2500            .unwrap()
2501            .unwrap();
2502        server.abort();
2503        let connect = destination.starts_with("https:");
2504
2505        assert_eq!(first.method, http::Method::GET);
2506        assert_eq!(first.uri.to_string(), "http://127.0.0.1:1/start");
2507        assert_eq!(
2508            first.headers[http::header::PROXY_AUTHORIZATION],
2509            expected_initial_auth
2510        );
2511        assert_eq!(
2512            redirected.headers[http::header::PROXY_AUTHORIZATION],
2513            expected_redirect_auth
2514        );
2515
2516        for (name, value) in origin_headers {
2517            assert_eq!(first.headers.get(&name).unwrap(), value.as_str());
2518            assert_eq!(
2519                redirected
2520                    .headers
2521                    .get(&name)
2522                    .map(|value| value.to_str().unwrap()),
2523                same_origin.then_some(value.as_str())
2524            );
2525        }
2526
2527        if connect {
2528            assert_eq!(redirected.method, http::Method::CONNECT);
2529            assert_eq!(redirected.uri.to_string(), "127.0.0.1:2");
2530            assert!(result.is_err());
2531        } else {
2532            let response = result.unwrap();
2533            assert_eq!(redirected.method, http::Method::GET);
2534            assert_eq!(redirected.uri.to_string(), destination);
2535            assert_eq!(response.status.as_u16(), 200);
2536            assert_eq!(response.body.as_ref(), b"accepted");
2537        }
2538    }
2539
2540    #[tokio::test]
2541    async fn test_http_client_unreachable_proxy_error_redacts_credentials() {
2542        const USERNAME: &str = "proxy-user";
2543        const SECRET: &str = "unreachable-proxy-secret";
2544        let (proxy_addr, drop_task) = spawn_connection_dropper().await;
2545        let client = HttpClient::builder()
2546            .timeout_secs(1)
2547            .proxy_url(format!("http://{USERNAME}:{SECRET}@{proxy_addr}"))
2548            .build()
2549            .unwrap();
2550        let error = client
2551            .request(
2552                Method::GET,
2553                "https://fixture.example.test/".to_string(),
2554                None,
2555                None,
2556                None,
2557                None,
2558                None,
2559            )
2560            .await
2561            .expect_err("unreachable proxy should fail");
2562        drop_task.abort();
2563        let task_error = drop_task
2564            .await
2565            .expect_err("connection dropper should be cancelled");
2566
2567        assert!(task_error.is_cancelled());
2568        assert!(!error.to_string().contains(SECRET));
2569        assert!(!error.to_string().contains(&BASE64.encode(SECRET)));
2570        assert!(
2571            !error
2572                .to_string()
2573                .contains(&BASE64.encode(format!("{USERNAME}:{SECRET}")))
2574        );
2575    }
2576
2577    #[rstest]
2578    fn test_http_client_with_valid_proxy() {
2579        // Create client with a valid proxy URL
2580        let result = HttpClient::builder()
2581            .proxy_url("http://proxy.example.com:8080".to_string())
2582            .build();
2583
2584        assert!(result.is_ok());
2585    }
2586
2587    #[rstest]
2588    fn test_http_client_with_socks5_proxy() {
2589        // Create client with a SOCKS5 proxy URL
2590        let result = HttpClient::builder()
2591            .proxy_url("socks5://127.0.0.1:1080".to_string())
2592            .build();
2593
2594        assert!(result.is_ok());
2595    }
2596
2597    #[rstest]
2598    fn test_http_client_with_malformed_proxy() {
2599        // Proxy parsing accepts scheme-less hostnames.
2600        // It only fails on obviously malformed URLs like "://invalid" or "http://".
2601        // More subtle issues (like "not-a-valid-url") are caught when connecting.
2602        let result = HttpClient::builder()
2603            .proxy_url("://invalid".to_string())
2604            .build();
2605
2606        assert!(result.is_err());
2607        assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2608    }
2609
2610    #[rstest]
2611    fn test_http_client_invalid_proxy_error_redacts_credentials() {
2612        const SECRET: &str = "unique-proxy-secret";
2613        let result = HttpClient::builder()
2614            .proxy_url(format!("http://proxytest:{SECRET}@[::1"))
2615            .build();
2616        let error = result.expect_err("malformed proxy URL should fail");
2617
2618        assert_eq!(
2619            error.to_string(),
2620            "Invalid proxy URL: proxy URL is malformed"
2621        );
2622        assert!(!error.to_string().contains(SECRET));
2623    }
2624
2625    #[rstest]
2626    fn test_http_client_with_empty_proxy_string() {
2627        // Create client with an empty proxy URL string
2628        let result = HttpClient::builder().proxy_url(String::new()).build();
2629
2630        assert!(result.is_err());
2631        assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2632    }
2633
2634    #[tokio::test]
2635    async fn test_http_client_get() {
2636        let addr = start_test_server().await.unwrap();
2637        let url = format!("http://{addr}/get");
2638
2639        let client = HttpClient::builder().build().unwrap();
2640        let response = client.get(url, None, None, None, None).await.unwrap();
2641
2642        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2643        assert_eq!(response.headers, HashMap::new());
2644        assert_eq!(response.body.as_ref(), b"hello-world!");
2645    }
2646
2647    #[tokio::test]
2648    async fn test_http_client_post() {
2649        let addr = start_test_server().await.unwrap();
2650        let url = format!("http://{addr}/post");
2651
2652        let client = HttpClient::builder().build().unwrap();
2653        let response = client
2654            .post(url, None, None, Some(b"post-body-73".to_vec()), None, None)
2655            .await
2656            .unwrap();
2657
2658        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2659        assert_eq!(response.headers, HashMap::new());
2660        assert_eq!(response.body.as_ref(), b"post-body-73");
2661    }
2662
2663    #[tokio::test]
2664    async fn test_http_client_patch() {
2665        let addr = start_test_server().await.unwrap();
2666        let url = format!("http://{addr}/patch");
2667
2668        let client = HttpClient::builder().build().unwrap();
2669        let response = client
2670            .patch(url, None, None, Some(b"patch-body-91".to_vec()), None, None)
2671            .await
2672            .unwrap();
2673
2674        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2675        assert_eq!(response.headers, HashMap::new());
2676        assert_eq!(response.body.as_ref(), b"patch-body-91");
2677    }
2678
2679    #[tokio::test]
2680    async fn test_http_client_delete() {
2681        let addr = start_test_server().await.unwrap();
2682        let url = format!("http://{addr}/delete");
2683
2684        let client = HttpClient::builder().build().unwrap();
2685        let response = client.delete(url, None, None, None, None).await.unwrap();
2686
2687        assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2688        assert_eq!(response.headers, HashMap::new());
2689        assert_eq!(response.body.as_ref(), b"");
2690    }
2691}
2692
2693#[cfg(test)]
2694mod rate_limit_tests {
2695    use std::{num::NonZeroU32, sync::Arc, time::Duration};
2696
2697    #[cfg(all(feature = "simulation", madsim))]
2698    use madsim::task as test_task;
2699    #[cfg(not(all(feature = "simulation", madsim)))]
2700    use tokio::task as test_task;
2701    use ustr::Ustr;
2702
2703    use super::HttpClient;
2704    use crate::ratelimiter::{RateLimiter, quota::Quota};
2705
2706    #[tokio::test]
2707    async fn test_http_client_awaits_multiple_rate_limiters() {
2708        let quota = Quota::per_minute(NonZeroU32::MIN);
2709        let request_key = Ustr::from("scope:request");
2710        let order_key = Ustr::from("scope:order");
2711
2712        let request_limiter = Arc::new(RateLimiter::new_with_quota(
2713            None,
2714            vec![(request_key, quota)],
2715        ));
2716        let order_limiter = Arc::new(RateLimiter::new_with_quota(None, vec![(order_key, quota)]));
2717        let client = HttpClient::builder()
2718            .rate_limiters(vec![
2719                Arc::clone(&request_limiter),
2720                Arc::clone(&order_limiter),
2721            ])
2722            .build()
2723            .unwrap();
2724
2725        client
2726            .await_rate_limits(Some(&[request_key, order_key]))
2727            .await;
2728
2729        assert!(request_limiter.check_key(&request_key).is_err());
2730        assert!(order_limiter.check_key(&order_key).is_err());
2731    }
2732
2733    #[cfg_attr(
2734        not(all(feature = "simulation", madsim)),
2735        tokio::test(start_paused = true)
2736    )]
2737    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2738    async fn test_http_client_reserves_multiple_rate_limits_together() {
2739        let global_key = Ustr::from("scope:global");
2740        let order_key = Ustr::from("scope:order");
2741
2742        let global_limiter = Arc::new(RateLimiter::new_with_quota(
2743            None,
2744            vec![(
2745                global_key,
2746                Quota::with_period(Duration::from_secs(1)).unwrap(),
2747            )],
2748        ));
2749
2750        let order_limiter = Arc::new(RateLimiter::new_with_quota(
2751            None,
2752            vec![(
2753                order_key,
2754                Quota::with_period(Duration::from_secs(10)).unwrap(),
2755            )],
2756        ));
2757        order_limiter.check_key(&order_key).unwrap();
2758
2759        let client = HttpClient::builder()
2760            .rate_limiters(vec![
2761                Arc::clone(&global_limiter),
2762                Arc::clone(&order_limiter),
2763            ])
2764            .build()
2765            .unwrap();
2766
2767        let request = test_task::spawn(async move {
2768            client
2769                .await_rate_limits(Some(&[global_key, order_key]))
2770                .await;
2771        });
2772
2773        test_task::yield_now().await;
2774
2775        global_limiter.check_key(&global_key).unwrap();
2776        assert!(!request.is_finished());
2777
2778        advance_test_clock(Duration::from_millis(9_999)).await;
2779        global_limiter.until_key_ready(&global_key).await;
2780        advance_test_clock(Duration::from_millis(1)).await;
2781        test_task::yield_now().await;
2782        assert!(!request.is_finished());
2783
2784        advance_test_clock(Duration::from_millis(998)).await;
2785        test_task::yield_now().await;
2786        assert!(!request.is_finished());
2787
2788        advance_test_clock(Duration::from_millis(1)).await;
2789        request.await.unwrap();
2790
2791        assert!(global_limiter.check_key(&global_key).is_err());
2792        assert!(order_limiter.check_key(&order_key).is_err());
2793    }
2794
2795    #[cfg(all(feature = "simulation", madsim))]
2796    async fn advance_test_clock(duration: Duration) {
2797        madsim::time::advance(duration);
2798        test_task::yield_now().await;
2799    }
2800
2801    #[cfg(not(all(feature = "simulation", madsim)))]
2802    async fn advance_test_clock(duration: Duration) {
2803        tokio::time::advance(duration).await;
2804    }
2805}
2806
2807#[cfg(test)]
2808pub(super) mod content_encoding_tests {
2809    use std::{collections::HashMap, io::Write, time::Duration};
2810
2811    use bytes::Bytes;
2812    use flate2::{Compression, write::GzEncoder};
2813    use http_body_util::Full;
2814    use rstest::rstest;
2815
2816    use super::{HttpClient, HttpClientError, REQUEST_TIMEOUT_MESSAGE};
2817    use crate::dst::time;
2818
2819    #[rstest]
2820    #[case::large_body(false, false)]
2821    #[case::large_body_deadline(true, false)]
2822    #[case::empty_members(false, true)]
2823    #[case::empty_members_deadline(true, true)]
2824    #[cfg_attr(
2825        not(all(feature = "simulation", madsim)),
2826        tokio::test(start_paused = true)
2827    )]
2828    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2829    async fn gzip_decoding_yields_and_obeys_deadline(
2830        #[case] expires: bool,
2831        #[case] empty_members: bool,
2832    ) {
2833        let (payload, compressed) = if empty_members {
2834            (Vec::new(), gzip(&[]).repeat(4_000))
2835        } else {
2836            let payload = vec![b'x'; 2 * 1024 * 1024];
2837            let compressed = gzip(&payload);
2838            (payload, compressed)
2839        };
2840
2841        let body = Bytes::from(compressed);
2842        let body = Full::new(body);
2843        let response = http::Response::builder()
2844            .header("Content-Encoding", "gzip")
2845            .body(body)
2846            .unwrap();
2847        let client = HttpClient::builder()
2848            .use_system_proxy(false)
2849            .build()
2850            .unwrap()
2851            .client;
2852        let deadline = expires.then(|| time::Instant::now() + Duration::from_secs(1));
2853        let mut request = Box::pin(client.consume_response(response, deadline, false));
2854
2855        assert!(futures_util::poll!(request.as_mut()).is_pending());
2856
2857        if expires {
2858            #[cfg(not(all(feature = "simulation", madsim)))]
2859            tokio::time::advance(Duration::from_secs(2)).await;
2860            #[cfg(all(feature = "simulation", madsim))]
2861            madsim::time::advance(Duration::from_secs(2));
2862        }
2863
2864        let result = request.await;
2865        if expires {
2866            let HttpClientError::TimeoutError(message) = result.unwrap_err() else {
2867                panic!("expected decoding timeout");
2868            };
2869
2870            assert_eq!(message, REQUEST_TIMEOUT_MESSAGE);
2871        } else {
2872            let response = result.unwrap();
2873            assert_eq!(response.status.as_u16(), 200);
2874            assert_eq!(response.headers, HashMap::new());
2875            assert_eq!(response.body.as_ref(), payload);
2876        }
2877    }
2878
2879    #[rstest]
2880    #[case::stored_blocks(false)]
2881    #[case::header_fields(true)]
2882    #[cfg_attr(not(all(feature = "simulation", madsim)), tokio::test)]
2883    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2884    async fn gzip_decoding_resumes_at_input_boundaries(#[case] metadata: bool) {
2885        let payload = vec![b'x'; 128 * 1024];
2886
2887        let mut encoder = if metadata {
2888            flate2::GzBuilder::new()
2889                .filename(vec![b'f'; 32 * 1024])
2890                .comment(vec![b'c'; 32 * 1024])
2891                .write(Vec::new(), Compression::default())
2892        } else {
2893            GzEncoder::new(Vec::new(), Compression::none())
2894        };
2895
2896        encoder.write_all(&payload).unwrap();
2897        let compressed = encoder.finish().unwrap();
2898        let response = http::Response::builder()
2899            .header("Content-Encoding", "gzip")
2900            .body(Full::new(Bytes::from(compressed)))
2901            .unwrap();
2902        let client = HttpClient::builder()
2903            .use_system_proxy(false)
2904            .build()
2905            .unwrap()
2906            .client;
2907
2908        let response = client
2909            .consume_response(response, None, false)
2910            .await
2911            .unwrap();
2912
2913        assert_eq!(response.status.as_u16(), 200);
2914        assert_eq!(response.headers, HashMap::new());
2915        assert_eq!(response.body.as_ref(), payload);
2916    }
2917
2918    pub(in crate::http) fn gzip(data: &[u8]) -> Vec<u8> {
2919        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
2920        encoder.write_all(data).unwrap();
2921        encoder.finish().unwrap()
2922    }
2923}