1use std::{borrow::Cow, collections::HashMap, str::FromStr, sync::Arc, time::Duration};
19
20use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
21use bytes::Bytes;
22use http::{
23 Method,
24 header::{HeaderMap, HeaderName, HeaderValue},
25};
26use http_body_util::Full;
27use nautilus_core::{collections::into_ustr_vec, string::secret::SecretString};
28use nautilus_cryptography::providers::install_cryptographic_provider;
29use url::Url;
30use ustr::Ustr;
31
32use super::{
33 HttpClientError, HttpResponse, HttpResponseStream, HttpStatus,
34 stream::{check_deadline, read_chunk, response_error},
35};
36use crate::ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota};
37
38#[cfg(not(all(feature = "simulation", madsim)))]
40const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 32;
41
42#[cfg(not(all(feature = "simulation", madsim)))]
44const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 60;
45
46#[cfg(not(all(feature = "simulation", madsim)))]
48const DEFAULT_HTTP2_KEEP_ALIVE_SECS: u64 = 30;
49
50#[cfg(not(all(feature = "simulation", madsim)))]
52const DEFAULT_HTTP2_STREAM_WINDOW_BYTES: u32 = 16 * 1024 * 1024;
53
54#[cfg(not(all(feature = "simulation", madsim)))]
56const DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES: u32 = 32 * 1024 * 1024;
57
58#[cfg(not(all(feature = "simulation", madsim)))]
60const NAUTILUS_HTTP2_ADAPTIVE_WINDOW: &str = "NAUTILUS_HTTP2_ADAPTIVE_WINDOW";
61
62const DEFAULT_MAX_RESPONSE_BYTES: usize = 100 * 1024 * 1024;
68
69const GZIP: &str = "gzip";
70const GZIP_DECODE_CHUNK_BYTES: usize = 64 * 1024;
71
72#[cfg(all(feature = "simulation", madsim))]
73pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "simulated request deadline elapsed";
74#[cfg(not(all(feature = "simulation", madsim)))]
75pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "request deadline elapsed";
76
77#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
79pub enum HttpRedirectPolicy {
80 #[default]
82 Follow,
83 Reject,
85}
86
87#[derive(Clone, Debug)]
99pub struct HttpClient {
100 pub(crate) client: InnerHttpClient,
101 pub(crate) rate_limiters: Arc<[Arc<RateLimiter<Ustr, MonotonicClock>>]>,
102}
103
104#[bon::bon]
105impl HttpClient {
106 #[allow(
124 clippy::needless_pass_by_value,
125 reason = "owned proxy URLs are part of the public builder API"
126 )]
127 #[builder(finish_fn = build)]
128 pub fn builder(
129 #[builder(default)] headers: HashMap<String, String>,
130 #[builder(default)] header_keys: Vec<String>,
131 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
132 default_quota: Option<Quota>,
133 timeout_secs: Option<u64>,
134 proxy_url: Option<String>,
135 rate_limiters: Option<Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>>,
136 #[builder(default)] redirect_policy: HttpRedirectPolicy,
137 #[builder(default = true)]
139 use_system_proxy: bool,
140 #[builder(default)]
147 redact_url: bool,
148 #[builder(default = DEFAULT_MAX_RESPONSE_BYTES)]
153 max_decompressed_bytes: usize,
154 certs_dir: Option<String>,
157 ) -> Result<Self, HttpClientError> {
158 let rate_limiters = if let Some(rate_limiters) = rate_limiters {
159 if default_quota.is_some() || !keyed_quotas.is_empty() {
160 return Err(HttpClientError::Error(
161 "Cannot combine shared rate limiters with quota configuration".to_string(),
162 ));
163 }
164
165 rate_limiters
166 } else {
167 let keyed_quotas = keyed_quotas
168 .into_iter()
169 .map(|(key, quota)| (Ustr::from(&key), quota))
170 .collect();
171 vec![Arc::new(RateLimiter::new_with_quota(
172 default_quota,
173 keyed_quotas,
174 ))]
175 };
176
177 Self::build(
178 headers,
179 header_keys,
180 timeout_secs,
181 proxy_url.as_deref(),
182 rate_limiters,
183 redirect_policy,
184 use_system_proxy,
185 redact_url,
186 max_decompressed_bytes,
187 certs_dir.as_deref(),
188 )
189 }
190
191 #[expect(
192 clippy::too_many_arguments,
193 reason = "client construction consumes the builder settings"
194 )]
195 fn build(
196 headers: HashMap<String, String>,
197 header_keys: Vec<String>,
198 timeout_secs: Option<u64>,
199 proxy_url: Option<&str>,
200 rate_limiters: Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>,
201 redirect_policy: HttpRedirectPolicy,
202 use_system_proxy: bool,
203 redact_url: bool,
204 max_decompressed_bytes: usize,
205 certs_dir: Option<&str>,
206 ) -> Result<Self, HttpClientError> {
207 let timeout = timeout_secs.map(Duration::from_secs);
208 request_deadline(timeout)?;
209 install_cryptographic_provider();
210
211 let mut header_map = HeaderMap::new();
212
213 for (key, value) in headers {
214 let header_name = HeaderName::from_str(&key)
215 .map_err(|e| HttpClientError::Error(format!("Invalid header name '{key}': {e}")))?;
216
217 let mut header_value = HeaderValue::from_str(&value).map_err(|e| {
218 HttpClientError::Error(format!("Invalid header value for '{key}': {e}"))
219 })?;
220
221 header_value.set_sensitive(true);
222 header_map.insert(header_name, header_value);
223 }
224
225 #[cfg(all(feature = "simulation", madsim))]
226 let simulation = super::simulation::Client::new(redirect_policy, proxy_url, certs_dir)?;
227
228 #[cfg(not(all(feature = "simulation", madsim)))]
229 let client = super::transport::Client::new(
230 proxy_url,
231 use_system_proxy,
232 certs_dir,
233 header_map.get(http::header::USER_AGENT).cloned(),
234 super::transport::Settings {
235 pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST,
236 pool_idle_timeout: Duration::from_secs(DEFAULT_POOL_IDLE_TIMEOUT_SECS),
237 keep_alive_interval: Some(Duration::from_secs(DEFAULT_HTTP2_KEEP_ALIVE_SECS)),
238 stream_window: Some(DEFAULT_HTTP2_STREAM_WINDOW_BYTES),
239 connection_window: Some(DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES),
240 adaptive_window: http2_adaptive_window()?,
241 },
242 )?;
243
244 #[cfg(all(feature = "simulation", madsim))]
245 let _ = use_system_proxy;
246
247 let response_headers = header_keys
250 .into_iter()
251 .map(|key| match HeaderName::from_str(&key) {
252 Ok(name) => Ok((key, name)),
253 Err(e) => Err(HttpClientError::Error(format!(
254 "Invalid header key '{key}': {e}"
255 ))),
256 })
257 .collect::<Result<Vec<_>, _>>()?;
258
259 let client = InnerHttpClient {
260 #[cfg(not(all(feature = "simulation", madsim)))]
261 client,
262 headers: header_map,
263 timeout,
264 redact_url,
265 #[cfg(not(all(feature = "simulation", madsim)))]
266 redirect_policy,
267 #[cfg(all(feature = "simulation", madsim))]
268 simulation,
269 response_headers: Arc::from(response_headers),
270 max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
271 max_decompressed_bytes,
272 };
273
274 Ok(Self {
275 client,
276 rate_limiters: rate_limiters.into(),
277 })
278 }
279
280 #[expect(clippy::too_many_arguments)]
291 pub async fn request(
292 &self,
293 method: Method,
294 url: String,
295 params: Option<&HashMap<String, Vec<String>>>,
296 headers: Option<HashMap<String, String>>,
297 body: Option<Vec<u8>>,
298 timeout_secs: Option<u64>,
299 keys: Option<Vec<String>>,
300 ) -> Result<HttpResponse, HttpClientError> {
301 let keys = keys.map(into_ustr_vec);
302
303 self.request_with_ustr_keys(method, url, params, headers, body, timeout_secs, keys)
304 .await
305 }
306
307 #[expect(clippy::too_many_arguments)]
317 pub async fn request_with_secret_body(
318 &self,
319 method: Method,
320 url: String,
321 params: Option<&HashMap<String, Vec<String>>>,
322 headers: Option<HashMap<String, String>>,
323 body: SecretString,
324 timeout_secs: Option<u64>,
325 keys: Option<Vec<String>>,
326 ) -> Result<HttpResponse, HttpClientError> {
327 let keys = keys.map(into_ustr_vec);
328 self.await_rate_limits(keys.as_deref()).await;
329
330 self.client
331 .send_request_with_secret_body(method, url, params, headers, body, timeout_secs)
332 .await
333 }
334
335 #[expect(clippy::too_many_arguments)]
344 pub async fn request_with_params<P: serde::Serialize>(
345 &self,
346 method: Method,
347 url: String,
348 params: Option<&P>,
349 headers: Option<HashMap<String, String>>,
350 body: Option<Vec<u8>>,
351 timeout_secs: Option<u64>,
352 keys: Option<Vec<String>>,
353 ) -> Result<HttpResponse, HttpClientError> {
354 let keys = keys.map(into_ustr_vec);
355 self.await_rate_limits(keys.as_deref()).await;
356
357 self.client
358 .send_request_with_query(method, url, params, headers, body, timeout_secs)
359 .await
360 }
361
362 pub async fn get_stream(&self, url: String) -> Result<HttpResponseStream, HttpClientError> {
375 self.await_rate_limits(None).await;
376 self.client
377 .send_stream_internal::<[(String, String); 0]>(
378 Method::GET,
379 &url,
380 None,
381 None,
382 None,
383 None,
384 )
385 .await
386 }
387
388 #[expect(clippy::too_many_arguments)]
394 pub async fn request_with_ustr_keys(
395 &self,
396 method: Method,
397 url: String,
398 params: Option<&HashMap<String, Vec<String>>>,
399 headers: Option<HashMap<String, String>>,
400 body: Option<Vec<u8>>,
401 timeout_secs: Option<u64>,
402 keys: Option<Vec<Ustr>>,
403 ) -> Result<HttpResponse, HttpClientError> {
404 self.await_rate_limits(keys.as_deref()).await;
405
406 self.client
407 .send_request(method, url, params, headers, body, timeout_secs)
408 .await
409 }
410
411 pub async fn await_rate_limits(&self, keys: Option<&[Ustr]>) {
418 RateLimiter::await_limiters_ready(&self.rate_limiters, keys).await;
419 }
420
421 pub async fn get(
427 &self,
428 url: String,
429 params: Option<&HashMap<String, Vec<String>>>,
430 headers: Option<HashMap<String, String>>,
431 timeout_secs: Option<u64>,
432 keys: Option<Vec<String>>,
433 ) -> Result<HttpResponse, HttpClientError> {
434 self.request(Method::GET, url, params, headers, None, timeout_secs, keys)
435 .await
436 }
437
438 pub async fn post(
444 &self,
445 url: String,
446 params: Option<&HashMap<String, Vec<String>>>,
447 headers: Option<HashMap<String, String>>,
448 body: Option<Vec<u8>>,
449 timeout_secs: Option<u64>,
450 keys: Option<Vec<String>>,
451 ) -> Result<HttpResponse, HttpClientError> {
452 self.request(Method::POST, url, params, headers, body, timeout_secs, keys)
453 .await
454 }
455
456 pub async fn patch(
462 &self,
463 url: String,
464 params: Option<&HashMap<String, Vec<String>>>,
465 headers: Option<HashMap<String, String>>,
466 body: Option<Vec<u8>>,
467 timeout_secs: Option<u64>,
468 keys: Option<Vec<String>>,
469 ) -> Result<HttpResponse, HttpClientError> {
470 self.request(
471 Method::PATCH,
472 url,
473 params,
474 headers,
475 body,
476 timeout_secs,
477 keys,
478 )
479 .await
480 }
481
482 pub async fn delete(
488 &self,
489 url: String,
490 params: Option<&HashMap<String, Vec<String>>>,
491 headers: Option<HashMap<String, String>>,
492 timeout_secs: Option<u64>,
493 keys: Option<Vec<String>>,
494 ) -> Result<HttpResponse, HttpClientError> {
495 self.request(
496 Method::DELETE,
497 url,
498 params,
499 headers,
500 None,
501 timeout_secs,
502 keys,
503 )
504 .await
505 }
506}
507
508#[derive(Clone, Debug)]
514pub struct InnerHttpClient {
515 #[cfg(all(feature = "simulation", madsim))]
516 simulation: super::simulation::Client,
517 #[cfg(not(all(feature = "simulation", madsim)))]
518 client: super::transport::Client,
519 headers: HeaderMap,
520 timeout: Option<Duration>,
521 redact_url: bool,
522 #[cfg(not(all(feature = "simulation", madsim)))]
523 redirect_policy: HttpRedirectPolicy,
524 pub(crate) response_headers: Arc<[(String, HeaderName)]>,
525 pub(crate) max_response_bytes: usize,
526 pub(crate) max_decompressed_bytes: usize,
527}
528
529impl InnerHttpClient {
530 pub async fn send_request(
536 &self,
537 method: Method,
538 url: String,
539 params: Option<&HashMap<String, Vec<String>>>,
540 headers: Option<HashMap<String, String>>,
541 body: Option<Vec<u8>>,
542 timeout_secs: Option<u64>,
543 ) -> Result<HttpResponse, HttpClientError> {
544 self.send_request_with_body(
545 method,
546 url,
547 params,
548 headers,
549 body.map(RequestBody::Plain),
550 timeout_secs,
551 )
552 .await
553 }
554
555 async fn send_request_with_secret_body(
556 &self,
557 method: Method,
558 url: String,
559 params: Option<&HashMap<String, Vec<String>>>,
560 headers: Option<HashMap<String, String>>,
561 body: SecretString,
562 timeout_secs: Option<u64>,
563 ) -> Result<HttpResponse, HttpClientError> {
564 self.send_request_with_body(
565 method,
566 url,
567 params,
568 headers,
569 Some(RequestBody::Secret(body)),
570 timeout_secs,
571 )
572 .await
573 }
574
575 async fn send_request_with_body(
576 &self,
577 method: Method,
578 url: String,
579 params: Option<&HashMap<String, Vec<String>>>,
580 headers: Option<HashMap<String, String>>,
581 body: Option<RequestBody>,
582 timeout_secs: Option<u64>,
583 ) -> Result<HttpResponse, HttpClientError> {
584 let full_url = encode_url_params(&url, params)?;
585 self.send_request_internal(
586 method,
587 full_url.as_ref(),
588 None::<&()>,
589 headers,
590 body,
591 timeout_secs,
592 )
593 .await
594 }
595
596 pub async fn send_request_with_query<Q: serde::Serialize>(
605 &self,
606 method: Method,
607 url: String,
608 query: Option<&Q>,
609 headers: Option<HashMap<String, String>>,
610 body: Option<Vec<u8>>,
611 timeout_secs: Option<u64>,
612 ) -> Result<HttpResponse, HttpClientError> {
613 self.send_request_internal(
614 method,
615 &url,
616 query,
617 headers,
618 body.map(RequestBody::Plain),
619 timeout_secs,
620 )
621 .await
622 }
623
624 async fn send_request_internal<Q: serde::Serialize>(
630 &self,
631 method: Method,
632 url: &str,
633 query: Option<&Q>,
634 headers: Option<HashMap<String, String>>,
635 body: Option<RequestBody>,
636 timeout_secs: Option<u64>,
637 ) -> Result<HttpResponse, HttpClientError> {
638 let mut headers = headers.unwrap_or_default();
639 let has_accept_encoding = headers
640 .keys()
641 .any(|key| key.eq_ignore_ascii_case(http::header::ACCEPT_ENCODING.as_str()))
642 || self.headers.contains_key(http::header::ACCEPT_ENCODING);
643
644 if !has_accept_encoding {
645 headers.insert(http::header::ACCEPT_ENCODING.to_string(), GZIP.to_string());
646 }
647
648 let is_head = method == Method::HEAD;
649 let stream = self
650 .send_stream_internal(method, url, query, Some(headers), body, timeout_secs)
651 .await?;
652 let result = self
653 .consume_response(stream.response, stream.deadline, is_head)
654 .await;
655 result.map_err(|e| response_error(e, stream.origin.as_deref()))
656 }
657
658 async fn send_stream_internal<Q: serde::Serialize>(
659 &self,
660 method: Method,
661 url: &str,
662 query: Option<&Q>,
663 headers: Option<HashMap<String, String>>,
664 body: Option<RequestBody>,
665 timeout_secs: Option<u64>,
666 ) -> Result<HttpResponseStream, HttpClientError> {
667 let mut url =
668 Url::parse(url).map_err(|e| HttpClientError::from(format!("URL parse error: {e}")))?;
669 if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
670 return Err(HttpClientError::Error(
671 "unsupported HTTP URL scheme or hostname".into(),
672 ));
673 }
674
675 let mut header_map = self.headers.clone();
676
677 if let Ok(username) = percent_encoding::percent_decode_str(url.username()).decode_utf8() {
678 let password = url.password().and_then(|password| {
679 percent_encoding::percent_decode_str(password)
680 .decode_utf8()
681 .ok()
682 });
683
684 if !username.is_empty() || password.is_some() {
685 let mut value = HeaderValue::from_str(&format!(
686 "Basic {}",
687 BASE64.encode(format!(
688 "{username}:{}",
689 password.as_deref().unwrap_or_default()
690 ))
691 ))
692 .map_err(|e| HttpClientError::Error(e.to_string()))?;
693 value.set_sensitive(true);
694 header_map.insert(http::header::AUTHORIZATION, value);
695 let _ = url.set_username("");
696 let _ = url.set_password(None);
697 }
698 }
699
700 let extra_header_count = headers.as_ref().map_or(0, HashMap::len);
701
702 if let Some(headers) = headers {
703 for (key, value) in headers {
704 let key = HeaderName::from_bytes(key.as_bytes())
705 .map_err(|e| HttpClientError::from(format!("Invalid header name: {e}")))?;
706 let value = HeaderValue::from_str(&value)
707 .map_err(|e| HttpClientError::from(format!("Invalid header value: {e}")))?;
708
709 if header_map.insert(key.clone(), value).is_some() {
710 log::trace!("Replaced duplicate request header '{key}'");
711 }
712 }
713 }
714
715 if let Some(query) = query {
716 {
717 let mut pairs = url.query_pairs_mut();
718 let serializer = serde_urlencoded::Serializer::new(&mut pairs);
719 query
720 .serialize(serializer)
721 .map_err(|e| HttpClientError::Error(e.to_string()))?;
722 }
723
724 if url.query() == Some("") {
725 url.set_query(None);
726 }
727 }
728
729 if !header_map.contains_key(http::header::ACCEPT) {
730 header_map.insert(http::header::ACCEPT, HeaderValue::from_static("*/*"));
731 }
732
733 let body = body.map(RequestBody::into_bytes).unwrap_or_default();
734 let body_len = body.len();
735 let query_len = url.query().map_or(0, str::len);
736 let mut request = http::Request::new(Full::new(body));
737 *request.method_mut() = method;
738 *request.uri_mut() = url[..url::Position::AfterQuery]
739 .parse()
740 .map_err(|_| HttpClientError::Error("invalid HTTP request target".into()))?;
741 *request.headers_mut() = header_map;
742 log::trace!(
743 "Sending HTTP request: method={} extra_headers={extra_header_count} \
744 query_bytes={query_len} body_bytes={body_len}",
745 request.method(),
746 );
747
748 let origin = (!self.redact_url).then(|| url.origin().ascii_serialization());
749
750 let duration = timeout_secs.map(Duration::from_secs).or(self.timeout);
751 let deadline = request_deadline(duration)?;
752
753 let operation = async {
754 #[cfg(all(feature = "simulation", madsim))]
755 let (response, connection) = self.simulation.send(request, &url).await?;
756 #[cfg(not(all(feature = "simulation", madsim)))]
757 let response = self.client.send(request, self.redirect_policy).await?;
758 Ok(HttpResponseStream {
759 response,
760 deadline,
761 origin: origin.clone(),
762 #[cfg(all(feature = "simulation", madsim))]
763 _connection: connection,
764 })
765 };
766
767 let result = match deadline {
768 Some(deadline) => tokio::select! {
769 biased;
770 () = crate::dst::time::sleep_until(deadline) => Err(HttpClientError::TimeoutError(REQUEST_TIMEOUT_MESSAGE.into())),
771 result = operation => result,
772 },
773 None => operation.await,
774 };
775
776 result.map_err(|e| response_error(e, origin.as_deref()))
777 }
778
779 async fn consume_response<B>(
780 &self,
781 response: http::Response<B>,
782 deadline: Option<crate::dst::time::Instant>,
783 is_head: bool,
784 ) -> Result<HttpResponse, HttpClientError>
785 where
786 B: http_body::Body<Data = Bytes> + Unpin,
787 B::Error: std::error::Error + 'static,
788 {
789 let (parts, mut body) = response.into_parts();
790 let mut headers =
791 HashMap::with_capacity(self.response_headers.len().min(parts.headers.len()));
792 for (key, name) in self.response_headers.iter() {
793 if let Some(value) = parts
794 .headers
795 .get(name)
796 .and_then(|value| value.to_str().ok())
797 {
798 headers.insert(key.clone(), value.to_owned());
799 }
800 }
801
802 let max = self.max_response_bytes;
803
804 if let Some(len) = body.size_hint().exact()
805 && len > max as u64
806 {
807 return Err(HttpClientError::Error(format!(
808 "HTTP response body of {len} bytes exceeds maximum of {max} bytes",
809 )));
810 }
811
812 let mut buf = bytes::BytesMut::new();
813 while let Some(chunk) = read_chunk(&mut body, deadline).await? {
814 if chunk.len() > max - buf.len() {
815 return Err(HttpClientError::Error(format!(
816 "HTTP response body exceeds maximum of {max} bytes",
817 )));
818 }
819
820 buf.extend_from_slice(&chunk);
821 }
822
823 log::trace!(
824 "Received HTTP response: status={} headers={} body_bytes={}",
825 parts.status,
826 parts.headers.len(),
827 buf.len()
828 );
829
830 let bodyless = is_head
831 || parts.status.is_informational()
832 || matches!(
833 parts.status,
834 http::StatusCode::NO_CONTENT
835 | http::StatusCode::RESET_CONTENT
836 | http::StatusCode::NOT_MODIFIED
837 );
838
839 let (body, decoded) = if bodyless {
840 (buf.freeze(), false)
841 } else {
842 decode_content_encoding(&parts.headers, buf, self.max_decompressed_bytes, deadline)
843 .await?
844 };
845
846 if decoded {
847 headers.retain(|key, _| {
848 !key.eq_ignore_ascii_case("content-encoding")
849 && !key.eq_ignore_ascii_case("content-length")
850 });
851 }
852
853 Ok(HttpResponse {
854 status: HttpStatus::new(parts.status),
855 headers,
856 body,
857 })
858 }
859}
860
861enum RequestBody {
862 Plain(Vec<u8>),
863 Secret(SecretString),
864}
865
866fn request_deadline(
867 timeout: Option<Duration>,
868) -> Result<Option<crate::dst::time::Instant>, HttpClientError> {
869 timeout
870 .map(|duration| {
871 crate::dst::time::Instant::now()
872 .checked_add(duration)
873 .ok_or_else(|| {
874 HttpClientError::Error("HTTP request timeout exceeds clock range".into())
875 })
876 })
877 .transpose()
878}
879
880async fn decode_content_encoding(
881 headers: &HeaderMap,
882 body: bytes::BytesMut,
883 max: usize,
884 deadline: Option<crate::dst::time::Instant>,
885) -> Result<(Bytes, bool), HttpClientError> {
886 let mut encodings = headers.get_all(http::header::CONTENT_ENCODING).iter();
887
888 let Some(encoding) = encodings.next() else {
889 return Ok((body.freeze(), false));
890 };
891
892 if encodings.next().is_some() {
893 return Err(HttpClientError::Error(
894 "unsupported multiple HTTP content-encoding fields".into(),
895 ));
896 }
897
898 let encoding = encoding
899 .to_str()
900 .map_err(|_| HttpClientError::Error("invalid HTTP content-encoding value".into()))?;
901 let encoding = encoding.trim();
902 if encoding.is_empty() || encoding.eq_ignore_ascii_case("identity") {
903 return Ok((body.freeze(), false));
904 }
905
906 if !encoding.eq_ignore_ascii_case(GZIP) {
907 return Err(HttpClientError::Error(format!(
908 "unsupported HTTP content-encoding '{encoding}'"
909 )));
910 }
911
912 check_deadline(deadline)?;
913 let body = body.freeze();
914 let mut out = Vec::new();
915
916 let mut reader = std::io::Read::take(
917 flate2::bufread::MultiGzDecoder::new(GzipInput {
918 bytes: body.as_ref(),
919 remaining: GZIP_DECODE_CHUNK_BYTES,
920 }),
921 (max as u64).saturating_add(1),
922 );
923
924 loop {
925 check_deadline(deadline)?;
926 let mut chunk = std::io::Read::take(&mut reader, GZIP_DECODE_CHUNK_BYTES as u64);
927 let result = std::io::Read::read_to_end(&mut chunk, &mut out);
928 check_deadline(deadline)?;
929
930 let complete = match result {
931 Ok(count) => count < GZIP_DECODE_CHUNK_BYTES,
932 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => false,
933 Err(e) => {
934 return Err(HttpClientError::Error(format!(
935 "failed to decode gzip response body: {e}"
936 )));
937 }
938 };
939
940 if out.len() > max {
941 return Err(HttpClientError::Error(format!(
942 "decompressed HTTP response body exceeds maximum of {max} bytes"
943 )));
944 }
945
946 if complete {
947 break;
948 }
949
950 crate::dst::task::yield_now().await;
951 reader.get_mut().get_mut().remaining = GZIP_DECODE_CHUNK_BYTES;
952 }
953
954 Ok((Bytes::from(out), true))
955}
956
957struct GzipInput<'a> {
958 bytes: &'a [u8],
959 remaining: usize,
960}
961
962impl std::io::Read for GzipInput<'_> {
963 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
964 if buf.is_empty() {
965 return Ok(0);
966 }
967
968 let mut available = std::io::BufRead::fill_buf(self)?;
969 let count = std::io::Read::read(&mut available, buf)?;
970 std::io::BufRead::consume(self, count);
971 Ok(count)
972 }
973}
974
975impl std::io::BufRead for GzipInput<'_> {
976 fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
977 if self.bytes.is_empty() {
978 return Ok(self.bytes);
979 }
980
981 if self.remaining == 0 {
982 return Err(std::io::ErrorKind::WouldBlock.into());
983 }
984
985 Ok(&self.bytes[..self.bytes.len().min(self.remaining)])
986 }
987
988 fn consume(&mut self, count: usize) {
989 self.bytes = &self.bytes[count..];
990 self.remaining -= count;
991 }
992}
993
994impl RequestBody {
995 fn into_bytes(self) -> Bytes {
996 match self {
997 Self::Plain(body) => body.into(),
998 Self::Secret(body) => Bytes::from_owner(SecretBody(body)),
999 }
1000 }
1001}
1002
1003struct SecretBody(SecretString);
1004
1005impl AsRef<[u8]> for SecretBody {
1006 fn as_ref(&self) -> &[u8] {
1007 self.0.expose_secret().as_bytes()
1008 }
1009}
1010
1011#[cfg(not(all(feature = "simulation", madsim)))]
1012fn http2_adaptive_window() -> Result<bool, HttpClientError> {
1013 let Some(value) = std::env::var_os(NAUTILUS_HTTP2_ADAPTIVE_WINDOW) else {
1014 return Ok(false);
1015 };
1016
1017 value.to_str().and_then(|v| v.parse().ok()).ok_or_else(|| {
1018 HttpClientError::ClientBuildError(format!(
1019 "{NAUTILUS_HTTP2_ADAPTIVE_WINDOW} must be 'true' or 'false', was '{}'",
1020 value.display()
1021 ))
1022 })
1023}
1024
1025fn encode_url_params<'a>(
1032 url: &'a str,
1033 params: Option<&HashMap<String, Vec<String>>>,
1034) -> Result<Cow<'a, str>, HttpClientError> {
1035 let Some(params) = params else {
1036 return Ok(Cow::Borrowed(url));
1037 };
1038
1039 let pairs: Vec<(&str, &str)> = params
1040 .iter()
1041 .flat_map(|(key, values)| {
1042 values
1043 .iter()
1044 .map(move |value| (key.as_str(), value.as_str()))
1045 })
1046 .collect();
1047
1048 if pairs.is_empty() {
1049 return Ok(Cow::Borrowed(url));
1050 }
1051
1052 let query_string = serde_urlencoded::to_string(pairs)
1053 .map_err(|e| HttpClientError::Error(format!("Failed to encode params: {e}")))?;
1054
1055 let (base, fragment) = match url.split_once('#') {
1058 Some((base, fragment)) => (base, Some(fragment)),
1059 None => (url, None),
1060 };
1061
1062 let separator = if base.contains('?') { '&' } else { '?' };
1063
1064 Ok(Cow::Owned(match fragment {
1065 Some(fragment) => format!("{base}{separator}{query_string}#{fragment}"),
1066 None => format!("{base}{separator}{query_string}"),
1067 }))
1068}
1069
1070#[cfg(test)]
1071mod encode_url_params_tests {
1072 use std::{borrow::Cow, collections::HashMap};
1073
1074 use rstest::rstest;
1075
1076 use super::encode_url_params;
1077
1078 fn params(pairs: &[(&str, &str)]) -> HashMap<String, Vec<String>> {
1079 let mut map: HashMap<String, Vec<String>> = HashMap::new();
1080 for (key, value) in pairs {
1081 map.entry((*key).to_string())
1082 .or_default()
1083 .push((*value).to_string());
1084 }
1085
1086 map
1087 }
1088
1089 #[rstest]
1090 #[case("https://x/y", "https://x/y?a=b")]
1091 #[case("https://x/y?old=1", "https://x/y?old=1&a=b")]
1092 #[case("https://x/y#frag", "https://x/y?a=b#frag")]
1093 #[case("https://x/y?old=1#frag", "https://x/y?old=1&a=b#frag")]
1094 #[case(
1095 "https://x/y#section?display=full",
1096 "https://x/y?a=b#section?display=full"
1097 )]
1098 #[case("https://x/y#", "https://x/y?a=b#")]
1099 fn test_query_is_inserted_before_the_fragment(#[case] url: &str, #[case] expected: &str) {
1100 let params = params(&[("a", "b")]);
1101
1102 assert_eq!(encode_url_params(url, Some(¶ms)).unwrap(), expected);
1103 }
1104
1105 #[rstest]
1106 fn test_url_is_borrowed_when_no_params_are_supplied() {
1107 assert!(matches!(
1108 encode_url_params("https://x/y#frag", None).unwrap(),
1109 Cow::Borrowed("https://x/y#frag")
1110 ));
1111 }
1112
1113 #[rstest]
1114 fn test_url_is_borrowed_when_params_are_empty() {
1115 let params = HashMap::new();
1116
1117 assert!(matches!(
1118 encode_url_params("https://x/y#frag", Some(¶ms)).unwrap(),
1119 Cow::Borrowed("https://x/y#frag")
1120 ));
1121 }
1122}
1123
1124#[cfg(test)]
1125#[cfg(target_os = "linux")] #[cfg(not(all(feature = "simulation", madsim)))]
1127mod tests {
1128 use std::net::SocketAddr;
1129
1130 use axum::{
1131 Router,
1132 body::to_bytes,
1133 extract::Request,
1134 response::IntoResponse,
1135 routing::{any, delete, get, patch, post},
1136 serve,
1137 };
1138 use http::status::StatusCode;
1139 use log::Level;
1140 use rstest::rstest;
1141 use tokio::{
1142 io::{AsyncReadExt, AsyncWriteExt},
1143 sync::oneshot,
1144 };
1145
1146 use super::*;
1147 use crate::logging::tests::capture_logs;
1148
1149 #[rstest]
1150 fn test_debug_redacts_custom_default_headers() {
1151 let client = HttpClient::builder()
1152 .use_system_proxy(false)
1153 .headers(HashMap::from([(
1154 "kalshi-access-key".into(),
1155 "HEADER_DEBUG_SECRET".into(),
1156 )]))
1157 .build()
1158 .unwrap();
1159 let header = &client.client.headers["kalshi-access-key"];
1160 let debug = format!("{client:?}");
1161
1162 assert_eq!(header.to_str().unwrap(), "HEADER_DEBUG_SECRET");
1163 assert!(header.is_sensitive());
1164 assert!(!debug.contains("HEADER_DEBUG_SECRET"), "{debug}");
1165 }
1166
1167 #[rstest]
1168 #[case("ftp://127.0.0.1:1/resource")]
1169 #[case("file:///resource")]
1170 #[tokio::test]
1171 async fn test_request_rejects_unsupported_url(#[case] url: &str) {
1172 let client = HttpClient::builder()
1173 .use_system_proxy(false)
1174 .build()
1175 .unwrap();
1176
1177 let error = client
1178 .request(Method::GET, url.to_string(), None, None, None, None, None)
1179 .await
1180 .unwrap_err();
1181
1182 assert!(
1183 matches!(error, HttpClientError::Error(ref message) if message == "unsupported HTTP URL scheme or hostname")
1184 );
1185 }
1186
1187 #[rstest]
1188 #[case::username_only("user%2F17@", "Basic dXNlci8xNzo=")]
1189 #[case::password_only(":secret%2F29@", "Basic OnNlY3JldC8yOQ==")]
1190 #[tokio::test]
1191 async fn test_request_url_credentials_allow_missing_username_or_password(
1192 #[case] userinfo: &str,
1193 #[case] expected: &str,
1194 ) {
1195 let addr = start_test_server().await.unwrap();
1196 let client = HttpClient::builder()
1197 .use_system_proxy(false)
1198 .build()
1199 .unwrap();
1200
1201 let response = client
1202 .request(
1203 Method::GET,
1204 format!("http://{userinfo}{addr}/headers"),
1205 None,
1206 None,
1207 None,
1208 None,
1209 None,
1210 )
1211 .await
1212 .unwrap();
1213
1214 assert_eq!(response.status.as_u16(), 200);
1215 assert_eq!(response.headers, HashMap::new());
1216 assert_eq!(
1217 response.body.as_ref(),
1218 format!("{expected}\n*/*").as_bytes()
1219 );
1220 }
1221
1222 #[rstest]
1223 #[case::default(None, "*/*")]
1224 #[case::explicit(Some("application/octet-stream"), "application/octet-stream")]
1225 #[tokio::test]
1226 async fn test_request_accept_header_preserves_explicit_value(
1227 #[case] accept: Option<&str>,
1228 #[case] expected: &str,
1229 ) {
1230 let addr = start_test_server().await.unwrap();
1231 let client = HttpClient::builder()
1232 .use_system_proxy(false)
1233 .build()
1234 .unwrap();
1235 let headers =
1236 accept.map(|value| HashMap::from([("accept".to_string(), value.to_string())]));
1237
1238 let response = client
1239 .request(
1240 Method::GET,
1241 format!("http://{addr}/headers"),
1242 None,
1243 headers,
1244 None,
1245 None,
1246 None,
1247 )
1248 .await
1249 .unwrap();
1250
1251 assert_eq!(response.status.as_u16(), 200);
1252 assert_eq!(response.headers, HashMap::new());
1253 assert_eq!(
1254 response.body.as_ref(),
1255 format!("absent\n{expected}").as_bytes()
1256 );
1257 }
1258
1259 async fn capture_request(request: Request) -> impl IntoResponse {
1260 let (parts, body) = request.into_parts();
1261 let body = to_bytes(body, usize::MAX).await.unwrap();
1262 let default_header = parts.headers.get("x-default").unwrap().to_str().unwrap();
1263 let request_header = parts.headers.get("x-request").unwrap().to_str().unwrap();
1264 let query = parts.uri.query().unwrap_or_default();
1265 let body = String::from_utf8(body.to_vec()).unwrap();
1266 let capture = format!(
1267 "{}\n{}\n{query}\n{default_header}\n{request_header}\n{body}",
1268 parts.method,
1269 parts.uri.path(),
1270 );
1271
1272 ([("x-response-id", "response-42")], capture)
1273 }
1274
1275 fn create_router() -> Router {
1276 Router::new()
1277 .route("/get", get(|| async { "hello-world!" }))
1278 .route("/post", post(|body: Bytes| async move { body }))
1279 .route("/patch", patch(|body: Bytes| async move { body }))
1280 .route("/delete", delete(|| async { StatusCode::OK }))
1281 .route("/capture", any(capture_request))
1282 .route(
1283 "/headers",
1284 get(|request: Request| async move {
1285 let headers = request.headers();
1286 format!(
1287 "{}\n{}",
1288 headers
1289 .get(http::header::AUTHORIZATION)
1290 .map_or("absent", |v| v.to_str().unwrap()),
1291 headers
1292 .get(http::header::ACCEPT)
1293 .map_or("absent", |v| v.to_str().unwrap()),
1294 )
1295 }),
1296 )
1297 .route("/notfound", get(|| async { StatusCode::NOT_FOUND }))
1298 .route(
1299 "/redirect",
1300 get(|| async { (StatusCode::TEMPORARY_REDIRECT, [("location", "/get")]) }),
1301 )
1302 .route(
1303 "/slow",
1304 get(|| async {
1305 tokio::time::sleep(Duration::from_secs(2)).await;
1306 "Eventually responded"
1307 }),
1308 )
1309 .route(
1310 "/large",
1311 get(|| async { "x".repeat(1024 * 1024) }),
1313 )
1314 }
1315
1316 async fn start_test_server() -> Result<SocketAddr, Box<dyn std::error::Error + Send + Sync>> {
1317 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1318 let addr = listener.local_addr().unwrap();
1319 tokio::spawn(async move {
1320 serve(listener, create_router()).await.unwrap();
1321 });
1322
1323 Ok(addr)
1324 }
1325
1326 async fn spawn_connection_dropper() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1327 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1328 let addr = listener.local_addr().unwrap();
1329
1330 let task = tokio::spawn(async move {
1331 loop {
1332 let (stream, _) = listener.accept().await.unwrap();
1333 drop(stream);
1334 }
1335 });
1336
1337 (addr, task)
1338 }
1339
1340 async fn spawn_chunked_response_server() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1341 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1342 let addr = listener.local_addr().unwrap();
1343 let task = tokio::spawn(async move {
1344 let (mut stream, _) = listener.accept().await.unwrap();
1345 let mut request = Vec::new();
1346 let mut chunk = [0u8; 1024];
1347 loop {
1348 let read = stream.read(&mut chunk).await.unwrap();
1349 if read == 0 {
1350 break;
1351 }
1352
1353 request.extend_from_slice(&chunk[..read]);
1354 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1355 break;
1356 }
1357 }
1358
1359 stream
1360 .write_all(
1361 b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n\
1362 5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n",
1363 )
1364 .await
1365 .unwrap();
1366 });
1367
1368 (addr, task)
1369 }
1370
1371 async fn spawn_rejecting_connect_proxy() -> (SocketAddr, oneshot::Receiver<String>) {
1372 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1373 let addr = listener.local_addr().unwrap();
1374 let (request_tx, request_rx) = oneshot::channel();
1375
1376 tokio::spawn(async move {
1377 let (mut stream, _) = listener.accept().await.unwrap();
1378 let mut request = Vec::new();
1379 let mut chunk = [0u8; 1024];
1380 loop {
1381 let read = stream.read(&mut chunk).await.unwrap();
1382 if read == 0 {
1383 break;
1384 }
1385
1386 request.extend_from_slice(&chunk[..read]);
1387 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1388 break;
1389 }
1390 }
1391
1392 request_tx
1393 .send(String::from_utf8(request).unwrap())
1394 .unwrap();
1395 stream
1396 .write_all(
1397 b"HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n",
1398 )
1399 .await
1400 .unwrap();
1401 });
1402
1403 (addr, request_rx)
1404 }
1405
1406 fn inner_client() -> InnerHttpClient {
1407 HttpClient::builder()
1408 .use_system_proxy(false)
1409 .build()
1410 .unwrap()
1411 .client
1412 }
1413
1414 #[tokio::test(start_paused = true)]
1415 async fn test_body_ready_at_deadline_is_rejected() {
1416 let client = inner_client();
1417 let response = http::Response::new(Full::new(Bytes::from_static(b"ready")));
1418 let result = client
1419 .consume_response(response, Some(crate::dst::time::Instant::now()), false)
1420 .await;
1421 assert!(
1422 matches!(result, Err(HttpClientError::TimeoutError(message)) if message == REQUEST_TIMEOUT_MESSAGE)
1423 );
1424 }
1425
1426 #[tokio::test]
1427 async fn test_get() {
1428 let addr = start_test_server().await.unwrap();
1429 let url = format!("http://{addr}");
1430
1431 let client = inner_client();
1432 let response = client
1433 .send_request(Method::GET, format!("{url}/get"), None, None, None, None)
1434 .await
1435 .unwrap();
1436
1437 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1438 assert_eq!(response.headers, HashMap::new());
1439 assert_eq!(response.body.as_ref(), b"hello-world!");
1440 }
1441
1442 #[rstest]
1443 #[tokio::test]
1444 async fn test_request_preserves_wire_semantics_and_extracts_response_headers(
1445 #[values(false, true)] redact_url: bool,
1446 ) {
1447 let addr = start_test_server().await.unwrap();
1448 let mut default_headers = HashMap::new();
1449 default_headers.insert("x-default".to_string(), "default-a".to_string());
1450 let client = HttpClient::builder()
1451 .redact_url(redact_url)
1452 .headers(default_headers)
1453 .header_keys(vec!["x-response-id".to_string()])
1454 .build()
1455 .unwrap();
1456 let mut params = HashMap::new();
1457 params.insert(
1458 "tag".to_string(),
1459 vec!["A B".to_string(), "C/D".to_string()],
1460 );
1461 let mut request_headers = HashMap::new();
1462 request_headers.insert("x-request".to_string(), "request-b".to_string());
1463
1464 let response = client
1465 .request(
1466 Method::PUT,
1467 format!("http://{addr}/capture?existing=seed"),
1468 Some(¶ms),
1469 Some(request_headers),
1470 Some(b"payload-c".to_vec()),
1471 None,
1472 None,
1473 )
1474 .await
1475 .unwrap();
1476
1477 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1478 assert_eq!(
1479 response.headers,
1480 HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1481 );
1482 assert_eq!(
1483 response.body.as_ref(),
1484 b"PUT\n/capture\nexisting=seed&tag=A+B&tag=C%2FD\ndefault-a\nrequest-b\npayload-c"
1485 );
1486 }
1487
1488 #[rstest]
1489 #[tokio::test]
1490 async fn test_request_with_secret_body_preserves_wire_body(
1491 #[values(false, true)] redact_url: bool,
1492 ) {
1493 let addr = start_test_server().await.unwrap();
1494 let client = HttpClient::builder()
1495 .redact_url(redact_url)
1496 .headers(HashMap::from([(
1497 "x-default".to_string(),
1498 "default-secret".to_string(),
1499 )]))
1500 .build()
1501 .unwrap();
1502 let headers = HashMap::from([("x-request".to_string(), "request-secret".to_string())]);
1503
1504 let response = client
1505 .request_with_secret_body(
1506 Method::POST,
1507 format!("http://{addr}/capture"),
1508 None,
1509 Some(headers),
1510 SecretString::from("credential-body"),
1511 None,
1512 None,
1513 )
1514 .await
1515 .unwrap();
1516
1517 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1518 assert_eq!(response.headers, HashMap::new());
1519 assert_eq!(
1520 response.body.as_ref(),
1521 b"POST\n/capture\n\ndefault-secret\nrequest-secret\ncredential-body"
1522 );
1523 }
1524
1525 #[rstest]
1526 #[tokio::test]
1527 async fn test_request_with_params_serializes_query_fields(
1528 #[values(false, true)] redact_url: bool,
1529 ) {
1530 #[derive(serde::Serialize)]
1531 struct Query<'a> {
1532 symbol: &'a str,
1533 limit: u32,
1534 }
1535
1536 let addr = start_test_server().await.unwrap();
1537 let mut default_headers = HashMap::new();
1538 default_headers.insert("x-default".to_string(), "default-d".to_string());
1539 let client = HttpClient::builder()
1540 .redact_url(redact_url)
1541 .headers(default_headers)
1542 .header_keys(vec!["x-response-id".to_string()])
1543 .build()
1544 .unwrap();
1545 let mut request_headers = HashMap::new();
1546 request_headers.insert("x-request".to_string(), "request-e".to_string());
1547
1548 let params = Query {
1549 symbol: "BTC/USDT",
1550 limit: 37,
1551 };
1552
1553 let response = client
1554 .request_with_params(
1555 Method::GET,
1556 format!("http://{addr}/capture"),
1557 Some(¶ms),
1558 Some(request_headers),
1559 None,
1560 None,
1561 None,
1562 )
1563 .await
1564 .unwrap();
1565
1566 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1567 assert_eq!(
1568 response.headers,
1569 HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1570 );
1571 assert_eq!(
1572 response.body.as_ref(),
1573 b"GET\n/capture\nsymbol=BTC%2FUSDT&limit=37\ndefault-d\nrequest-e\n"
1574 );
1575 }
1576
1577 #[rstest]
1578 #[tokio::test]
1579 async fn test_request_with_params_preserves_secret_query_fields(
1580 #[values(false, true)] redact_url: bool,
1581 ) {
1582 #[derive(serde::Serialize)]
1583 struct Query<'a> {
1584 auth: &'a str,
1585 market_id: i16,
1586 }
1587
1588 let addr = start_test_server().await.unwrap();
1589 let client = HttpClient::builder()
1590 .redact_url(redact_url)
1591 .headers(HashMap::from([(
1592 "x-default".to_string(),
1593 "default-f".to_string(),
1594 )]))
1595 .build()
1596 .unwrap();
1597 let headers = HashMap::from([("x-request".to_string(), "request-g".to_string())]);
1598
1599 let params = Query {
1600 auth: "token/42",
1601 market_id: 7,
1602 };
1603
1604 let response = client
1605 .request_with_params(
1606 Method::GET,
1607 format!("http://{addr}/capture"),
1608 Some(¶ms),
1609 Some(headers),
1610 None,
1611 None,
1612 None,
1613 )
1614 .await
1615 .unwrap();
1616
1617 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1618 assert_eq!(
1619 response.body.as_ref(),
1620 b"GET\n/capture\nauth=token%2F42&market_id=7\ndefault-f\nrequest-g\n"
1621 );
1622 }
1623
1624 #[rstest]
1625 #[case::empty_at_zero_cap(b"", 0)]
1626 #[case::at_cap(b"body-37", 7)]
1627 #[case::below_cap(b"body-37", 8)]
1628 #[tokio::test]
1629 async fn test_declared_response_body_at_or_below_cap_is_returned(
1630 #[case] bytes: &'static [u8],
1631 #[case] max_response_bytes: usize,
1632 ) {
1633 let client = InnerHttpClient {
1634 max_response_bytes,
1635 ..inner_client()
1636 };
1637
1638 let response = http::Response::new(Full::new(Bytes::from_static(bytes)));
1639
1640 let response = client
1641 .consume_response(response, None, false)
1642 .await
1643 .unwrap();
1644
1645 assert_eq!(response.status.as_u16(), 200);
1646 assert_eq!(response.headers, HashMap::new());
1647 assert_eq!(response.body.as_ref(), bytes);
1648 }
1649
1650 #[tokio::test]
1651 async fn test_response_body_within_cap_is_returned() {
1652 let addr = start_test_server().await.unwrap();
1653 let url = format!("http://{addr}");
1654
1655 let client = InnerHttpClient {
1657 max_response_bytes: 4 * 1024 * 1024,
1658 ..inner_client()
1659 };
1660
1661 let response = client
1662 .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1663 .await
1664 .unwrap();
1665
1666 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1667 assert_eq!(response.headers, HashMap::new());
1668 assert_eq!(response.body.as_ref(), vec![b'x'; 1024 * 1024]);
1669 }
1670
1671 #[tokio::test]
1672 async fn test_response_body_exceeding_cap_is_rejected() {
1673 let addr = start_test_server().await.unwrap();
1674 let url = format!("http://{addr}");
1675
1676 let client = InnerHttpClient {
1678 max_response_bytes: 16 * 1024,
1679 ..inner_client()
1680 };
1681
1682 let result = client
1683 .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1684 .await;
1685
1686 let err = result.expect_err("oversized response body should be rejected");
1687
1688 let HttpClientError::Error(message) = err else {
1689 panic!("expected HTTP error, was {err:?}");
1690 };
1691
1692 assert_eq!(
1693 message,
1694 "HTTP response body of 1048576 bytes exceeds maximum of 16384 bytes"
1695 );
1696 }
1697
1698 #[rstest]
1699 #[case::at_cap(11)]
1700 #[case::below_cap(12)]
1701 #[tokio::test]
1702 async fn test_chunked_response_body_at_or_below_cap_is_returned(
1703 #[case] max_response_bytes: usize,
1704 ) {
1705 let (addr, server_task) = spawn_chunked_response_server().await;
1706
1707 let client = InnerHttpClient {
1708 max_response_bytes,
1709 ..inner_client()
1710 };
1711
1712 let response = client
1713 .send_request(
1714 Method::GET,
1715 format!("http://{addr}"),
1716 None,
1717 None,
1718 None,
1719 None,
1720 )
1721 .await
1722 .unwrap();
1723 server_task.await.unwrap();
1724
1725 assert_eq!(response.status.as_u16(), 200);
1726 assert_eq!(response.headers, HashMap::new());
1727 assert_eq!(response.body.as_ref(), b"firstsecond");
1728 }
1729
1730 #[tokio::test]
1731 async fn test_chunked_response_body_exceeding_cap_is_rejected() {
1732 let (addr, server_task) = spawn_chunked_response_server().await;
1733 let max_response_bytes = 8;
1734
1735 let client = InnerHttpClient {
1736 max_response_bytes,
1737 ..inner_client()
1738 };
1739
1740 let error = client
1741 .send_request(
1742 Method::GET,
1743 format!("http://{addr}"),
1744 None,
1745 None,
1746 None,
1747 None,
1748 )
1749 .await
1750 .expect_err("chunked response body should be rejected");
1751 server_task.await.unwrap();
1752
1753 let HttpClientError::Error(message) = error else {
1754 panic!("expected HTTP error, was {error:?}");
1755 };
1756
1757 assert_eq!(
1758 message,
1759 format!("HTTP response body exceeds maximum of {max_response_bytes} bytes")
1760 );
1761 }
1762
1763 #[tokio::test]
1764 async fn test_post() {
1765 let addr = start_test_server().await.unwrap();
1766 let url = format!("http://{addr}");
1767
1768 let client = inner_client();
1769 let response = client
1770 .send_request(Method::POST, format!("{url}/post"), None, None, None, None)
1771 .await
1772 .unwrap();
1773
1774 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1775 assert_eq!(response.headers, HashMap::new());
1776 assert_eq!(response.body.as_ref(), b"");
1777 }
1778
1779 #[tokio::test]
1780 async fn test_post_with_body() {
1781 let addr = start_test_server().await.unwrap();
1782 let url = format!("http://{addr}");
1783
1784 let client = inner_client();
1785
1786 let mut body = HashMap::new();
1787 body.insert(
1788 "key1".to_string(),
1789 serde_json::Value::String("value1".to_string()),
1790 );
1791 body.insert(
1792 "key2".to_string(),
1793 serde_json::Value::String("value2".to_string()),
1794 );
1795
1796 let body_string = serde_json::to_string(&body).unwrap();
1797 let body_bytes = body_string.into_bytes();
1798
1799 let response = client
1800 .send_request(
1801 Method::POST,
1802 format!("{url}/post"),
1803 None,
1804 None,
1805 Some(body_bytes.clone()),
1806 None,
1807 )
1808 .await
1809 .unwrap();
1810
1811 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1812 assert_eq!(response.headers, HashMap::new());
1813 assert_eq!(response.body.as_ref(), body_bytes);
1814 }
1815
1816 #[tokio::test]
1817 async fn test_patch() {
1818 let addr = start_test_server().await.unwrap();
1819 let url = format!("http://{addr}");
1820
1821 let client = inner_client();
1822 let response = client
1823 .send_request(
1824 Method::PATCH,
1825 format!("{url}/patch"),
1826 None,
1827 None,
1828 None,
1829 None,
1830 )
1831 .await
1832 .unwrap();
1833
1834 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1835 assert_eq!(response.headers, HashMap::new());
1836 assert_eq!(response.body.as_ref(), b"");
1837 }
1838
1839 #[tokio::test]
1840 async fn test_delete() {
1841 let addr = start_test_server().await.unwrap();
1842 let url = format!("http://{addr}");
1843
1844 let client = inner_client();
1845 let response = client
1846 .send_request(
1847 Method::DELETE,
1848 format!("{url}/delete"),
1849 None,
1850 None,
1851 None,
1852 None,
1853 )
1854 .await
1855 .unwrap();
1856
1857 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1858 assert_eq!(response.headers, HashMap::new());
1859 assert_eq!(response.body.as_ref(), b"");
1860 }
1861
1862 #[tokio::test]
1863 async fn test_not_found() {
1864 let addr = start_test_server().await.unwrap();
1865 let url = format!("http://{addr}/notfound");
1866 let client = inner_client();
1867
1868 let response = client
1869 .send_request(Method::GET, url, None, None, None, None)
1870 .await
1871 .unwrap();
1872
1873 assert!(response.status.is_client_error());
1874 assert_eq!(response.status.as_u16(), 404);
1875 assert_eq!(response.headers, HashMap::new());
1876 assert_eq!(response.body.as_ref(), b"");
1877 }
1878
1879 #[tokio::test]
1880 async fn test_timeout() {
1881 let addr = start_test_server().await.unwrap();
1882 let url = format!("http://{addr}/slow");
1883 let client = inner_client();
1884
1885 let result = client
1887 .send_request(Method::GET, url, None, None, None, Some(1))
1888 .await;
1889
1890 assert!(
1891 matches!(&result, Err(HttpClientError::TimeoutError(_))),
1892 "Expected a timeout error, was: {result:?}"
1893 );
1894 }
1895
1896 #[rstest]
1897 fn test_http_client_without_proxy() {
1898 let result = HttpClient::builder().build();
1900
1901 assert!(result.is_ok());
1902 }
1903
1904 #[rstest]
1905 fn test_http_client_rejects_timeout_overflow() {
1906 let result = HttpClient::builder().timeout_secs(u64::MAX).build();
1907
1908 assert!(matches!(
1909 result,
1910 Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1911 ));
1912 }
1913
1914 #[tokio::test]
1915 async fn test_http_request_rejects_timeout_override_overflow() {
1916 let client = HttpClient::builder()
1917 .use_system_proxy(false)
1918 .timeout_secs(1)
1919 .build()
1920 .unwrap();
1921 let result = client
1922 .request(
1923 Method::GET,
1924 "http://127.0.0.1:1/".to_string(),
1925 None,
1926 None,
1927 None,
1928 Some(u64::MAX),
1929 None,
1930 )
1931 .await;
1932
1933 assert!(matches!(
1934 result,
1935 Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1936 ));
1937 }
1938
1939 #[rstest]
1940 fn test_http_client_builder_preserves_empty_rate_limiters() {
1941 let client = HttpClient::builder()
1942 .rate_limiters(Vec::new())
1943 .build()
1944 .unwrap();
1945
1946 assert!(client.rate_limiters.is_empty());
1947 }
1948
1949 #[rstest]
1950 fn test_http_client_builder_rejects_shared_rate_limiters_with_quotas() {
1951 let quota = Quota::with_period(Duration::from_secs(1)).unwrap();
1952 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
1953 let result = HttpClient::builder()
1954 .default_quota(quota)
1955 .rate_limiters(vec![rate_limiter])
1956 .build();
1957
1958 assert_eq!(
1959 result.unwrap_err().to_string(),
1960 "HTTP error occurred: Cannot combine shared rate limiters with quota configuration"
1961 );
1962 }
1963
1964 #[tokio::test]
1965 async fn test_http_client_without_proxy_requests_directly() {
1966 let addr = start_test_server().await.unwrap();
1967 let client = HttpClient::builder().timeout_secs(2).build().unwrap();
1968 let response = client
1969 .request(
1970 Method::GET,
1971 format!("http://{addr}/get"),
1972 None,
1973 None,
1974 None,
1975 None,
1976 None,
1977 )
1978 .await
1979 .expect("direct request");
1980
1981 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1982 assert_eq!(response.body.as_ref(), b"hello-world!");
1983 }
1984
1985 #[tokio::test]
1986 async fn test_http_client_redirect_policy() {
1987 let addr = start_test_server().await.unwrap();
1988 let follow = HttpClient::builder().timeout_secs(2).build().unwrap();
1989 let reject = HttpClient::builder()
1990 .timeout_secs(2)
1991 .redirect_policy(HttpRedirectPolicy::Reject)
1992 .build()
1993 .unwrap();
1994
1995 let followed = follow
1996 .request(
1997 Method::GET,
1998 format!("http://{addr}/redirect"),
1999 None,
2000 None,
2001 None,
2002 None,
2003 None,
2004 )
2005 .await
2006 .unwrap();
2007 let rejected = reject
2008 .request(
2009 Method::GET,
2010 format!("http://{addr}/redirect"),
2011 None,
2012 None,
2013 None,
2014 None,
2015 None,
2016 )
2017 .await
2018 .unwrap();
2019
2020 assert_eq!(followed.status.as_u16(), StatusCode::OK.as_u16());
2021 assert_eq!(followed.body.as_ref(), b"hello-world!");
2022 assert_eq!(
2023 rejected.status.as_u16(),
2024 StatusCode::TEMPORARY_REDIRECT.as_u16()
2025 );
2026 assert!(rejected.body.is_empty());
2027 }
2028
2029 #[rstest]
2030 #[tokio::test]
2031 async fn test_http_client_url_policy_preserves_response(
2032 #[values(false, true)] redact_url: bool,
2033 ) {
2034 let addr = start_test_server().await.unwrap();
2035 let client = HttpClient::builder()
2036 .redact_url(redact_url)
2037 .timeout_secs(2)
2038 .build()
2039 .unwrap();
2040 let response = client
2041 .request(
2042 Method::GET,
2043 format!("http://{addr}/get"),
2044 None,
2045 None,
2046 None,
2047 None,
2048 None,
2049 )
2050 .await
2051 .expect("direct request with URL redaction");
2052
2053 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2054 assert_eq!(response.body.as_ref(), b"hello-world!");
2055 }
2056
2057 #[rstest]
2058 #[case::request("request")]
2059 #[case::preinterned_keys("keys")]
2060 #[case::get("get")]
2061 #[case::post("post")]
2062 #[case::patch("patch")]
2063 #[case::delete("delete")]
2064 #[case::stream("stream")]
2065 #[tokio::test]
2066 async fn test_http_client_url_policy_redacts_transport_errors(
2067 #[case] request: &str,
2068 #[values(false, true)] redact_url: bool,
2069 ) {
2070 const USERINFO_SECRET: &str = "transport-userinfo-secret";
2071 const PATH_SECRET: &str = "transport-path-secret";
2072 const QUERY_SECRET: &str = "transport-query-secret";
2073 let (addr, drop_task) = spawn_connection_dropper().await;
2074 let url = format!(
2075 "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2076 );
2077 let client = HttpClient::builder()
2078 .redact_url(redact_url)
2079 .timeout_secs(1)
2080 .build()
2081 .unwrap();
2082
2083 let error = match request {
2084 "request" => client
2085 .request(Method::GET, url.clone(), None, None, None, None, None)
2086 .await
2087 .unwrap_err(),
2088 "keys" => client
2089 .request_with_ustr_keys(
2090 Method::GET,
2091 url.clone(),
2092 None,
2093 None,
2094 None,
2095 None,
2096 Some(vec![Ustr::from("policy")]),
2097 )
2098 .await
2099 .unwrap_err(),
2100 "get" => client
2101 .get(url.clone(), None, None, None, None)
2102 .await
2103 .unwrap_err(),
2104 "post" => client
2105 .post(url.clone(), None, None, None, None, None)
2106 .await
2107 .unwrap_err(),
2108 "patch" => client
2109 .patch(url.clone(), None, None, None, None, None)
2110 .await
2111 .unwrap_err(),
2112 "delete" => client
2113 .delete(url.clone(), None, None, None, None)
2114 .await
2115 .unwrap_err(),
2116 "stream" => client.get_stream(url.clone()).await.unwrap_err(),
2117 _ => panic!("unknown request shape"),
2118 };
2119
2120 drop_task.abort();
2121 let task_error = drop_task
2122 .await
2123 .expect_err("connection dropper should be cancelled");
2124
2125 assert!(task_error.is_cancelled());
2126 assert!(
2127 matches!(error, HttpClientError::TransportError(_)),
2128 "{error:?}"
2129 );
2130
2131 for rendered in [error.to_string(), format!("{error:?}")] {
2132 assert!(!rendered.contains(USERINFO_SECRET));
2133 assert!(!rendered.contains(PATH_SECRET));
2134 assert!(!rendered.contains(QUERY_SECRET));
2135 assert!(!rendered.contains(&url));
2136 assert_eq!(
2137 rendered.contains(&format!("http://{addr}")),
2138 !redact_url,
2139 "{rendered}"
2140 );
2141 assert_eq!(rendered.contains("for origin ("), !redact_url, "{rendered}");
2142 }
2143 }
2144
2145 #[tokio::test]
2146 async fn test_request_with_params_redact_url_removes_origin_from_error() {
2147 const QUERY_SECRET: &str = "transport-query-secret";
2148 #[derive(serde::Serialize)]
2149 struct Query<'a> {
2150 auth: &'a str,
2151 }
2152
2153 let (addr, drop_task) = spawn_connection_dropper().await;
2154 let url = format!("http://{addr}/trades");
2155 let params = Query { auth: QUERY_SECRET };
2156 let client = HttpClient::builder()
2157 .redact_url(true)
2158 .timeout_secs(1)
2159 .build()
2160 .unwrap();
2161
2162 let error = client
2163 .request_with_params(Method::GET, url, Some(¶ms), None, None, None, None)
2164 .await
2165 .expect_err("a dropped connection should fail");
2166 drop_task.abort();
2167 let task_error = drop_task
2168 .await
2169 .expect_err("connection dropper should be cancelled");
2170
2171 assert!(task_error.is_cancelled());
2172
2173 for rendered in [error.to_string(), format!("{error:?}")] {
2174 assert!(!rendered.contains(&addr.to_string()));
2175 assert!(!rendered.contains("auth="));
2176 assert!(!rendered.contains(QUERY_SECRET));
2177 }
2178 }
2179
2180 #[tokio::test]
2181 async fn test_request_removes_query_string_from_transport_error_by_default() {
2182 const QUERY_SECRET: &str = "default-query-secret";
2183 const FRAGMENT_MARKER: &str = "default-fragment-marker";
2184 let (addr, drop_task) = spawn_connection_dropper().await;
2185 let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}#{FRAGMENT_MARKER}");
2186 let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2187
2188 let error = client
2189 .request(Method::GET, url, None, None, None, None, None)
2190 .await
2191 .expect_err("a dropped connection should fail");
2192 drop_task.abort();
2193 let task_error = drop_task
2194 .await
2195 .expect_err("connection dropper should be cancelled");
2196
2197 assert!(task_error.is_cancelled());
2198
2199 for rendered in [error.to_string(), format!("{error:?}")] {
2200 assert!(
2201 rendered.contains(&format!("for origin (http://{addr})")),
2202 "default error omitted the origin: {rendered}"
2203 );
2204 assert!(!rendered.contains("/trades"), "{rendered}");
2205 assert!(!rendered.contains("api_key="), "{rendered}");
2206 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2207 assert!(!rendered.contains(FRAGMENT_MARKER), "{rendered}");
2208 }
2209 }
2210
2211 #[rstest]
2212 #[tokio::test]
2213 async fn test_request_with_secret_body_url_policy_redacts_transport_error(
2214 #[values(false, true)] redact_url: bool,
2215 ) {
2216 const QUERY_SECRET: &str = "secret-body-query-secret";
2217 let (addr, drop_task) = spawn_connection_dropper().await;
2218 let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}");
2219 let client = HttpClient::builder()
2220 .redact_url(redact_url)
2221 .timeout_secs(1)
2222 .build()
2223 .unwrap();
2224
2225 let error = client
2226 .request_with_secret_body(
2227 Method::POST,
2228 url,
2229 None,
2230 None,
2231 SecretString::from("credential-body"),
2232 None,
2233 None,
2234 )
2235 .await
2236 .expect_err("a dropped connection should fail");
2237 drop_task.abort();
2238 let task_error = drop_task
2239 .await
2240 .expect_err("connection dropper should be cancelled");
2241
2242 assert!(task_error.is_cancelled());
2243
2244 for rendered in [error.to_string(), format!("{error:?}")] {
2245 assert_eq!(
2246 rendered.contains(&format!("for origin (http://{addr})")),
2247 !redact_url,
2248 "{rendered}"
2249 );
2250 assert_eq!(
2251 rendered.contains(&addr.to_string()),
2252 !redact_url,
2253 "{rendered}"
2254 );
2255 assert!(!rendered.contains("/trades"), "{rendered}");
2256 assert!(!rendered.contains("credential-body"), "{rendered}");
2257 assert!(!rendered.contains("api_key="), "{rendered}");
2258 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2259 }
2260 }
2261
2262 #[tokio::test]
2263 async fn test_request_with_params_removes_query_from_transport_error_by_default() {
2264 const QUERY_SECRET: &str = "default-params-query-secret";
2265 #[derive(serde::Serialize)]
2266 struct Query<'a> {
2267 auth: &'a str,
2268 }
2269
2270 let (addr, drop_task) = spawn_connection_dropper().await;
2271 let url = format!("http://{addr}/trades");
2272 let params = Query { auth: QUERY_SECRET };
2273 let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2274
2275 let error = client
2276 .request_with_params(Method::GET, url, Some(¶ms), None, None, None, None)
2277 .await
2278 .expect_err("a dropped connection should fail");
2279 drop_task.abort();
2280 let task_error = drop_task
2281 .await
2282 .expect_err("connection dropper should be cancelled");
2283
2284 assert!(task_error.is_cancelled());
2285
2286 for rendered in [error.to_string(), format!("{error:?}")] {
2287 assert!(
2288 rendered.contains(&format!("for origin (http://{addr})")),
2289 "default error omitted the origin: {rendered}"
2290 );
2291 assert!(!rendered.contains("/trades"), "{rendered}");
2292 assert!(!rendered.contains("auth="), "{rendered}");
2293 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2294 }
2295 }
2296
2297 #[rstest]
2298 #[tokio::test]
2299 async fn test_http_client_url_policy_removes_endpoint_from_trace_logs(
2300 #[values(false, true)] redact_url: bool,
2301 ) {
2302 const USERINFO_SECRET: &str = "trace-userinfo-secret";
2303 const PATH_SECRET: &str = "trace-path-secret";
2304 const QUERY_SECRET: &str = "trace-query-secret";
2305 let capture = capture_logs().await;
2306 let addr = start_test_server().await.unwrap();
2307 let url = format!(
2308 "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2309 );
2310 let client = HttpClient::builder()
2311 .redact_url(redact_url)
2312 .timeout_secs(2)
2313 .build()
2314 .unwrap();
2315
2316 let response = client
2317 .request(Method::GET, url.clone(), None, None, None, None, None)
2318 .await
2319 .expect("credentialized endpoint should return an HTTP response");
2320 let messages = capture.messages();
2321
2322 assert_eq!(response.status.as_u16(), StatusCode::NOT_FOUND.as_u16());
2323 assert!(messages.iter().any(|(level, message)| {
2324 *level == Level::Trace && message.starts_with("Sending HTTP request: method=GET")
2325 }));
2326 assert!(messages.iter().any(|(level, message)| {
2327 *level == Level::Trace
2328 && message.starts_with("Received HTTP response: status=404 Not Found")
2329 }));
2330
2331 for (_, message) in messages {
2332 assert!(!message.contains(USERINFO_SECRET));
2333 assert!(!message.contains(PATH_SECRET));
2334 assert!(!message.contains(QUERY_SECRET));
2335 assert!(!message.contains(&url));
2336 }
2337 }
2338
2339 #[rstest]
2340 #[case::without_user_agent(None)]
2341 #[case::with_user_agent(Some("NautilusTrader/proxy-test-73"))]
2342 #[tokio::test]
2343 async fn test_http_client_uses_connect_and_proxy_authorization_for_https(
2344 #[case] user_agent: Option<&str>,
2345 ) {
2346 const USERNAME: &str = "proxytest";
2347 const PASSWORD: &str = "fixture42";
2348 let mut headers =
2349 HashMap::from([("authorization".into(), "Bearer origin-secret-19".into())]);
2350
2351 if let Some(user_agent) = user_agent {
2352 headers.insert("user-agent".into(), user_agent.into());
2353 }
2354
2355 let (proxy_addr, request_rx) = spawn_rejecting_connect_proxy().await;
2356 let client = HttpClient::builder()
2357 .headers(headers)
2358 .timeout_secs(2)
2359 .proxy_url(format!("http://{USERNAME}:{PASSWORD}@{proxy_addr}"))
2360 .build()
2361 .unwrap();
2362 let error = client
2363 .request(
2364 Method::GET,
2365 "https://fixture.example.test/path".to_string(),
2366 None,
2367 None,
2368 None,
2369 None,
2370 None,
2371 )
2372 .await
2373 .expect_err("proxy should reject CONNECT");
2374 let request = request_rx.await.expect("captured CONNECT request");
2375 let mut lines = request.split("\r\n");
2376 let request_line = lines.next().expect("CONNECT request line");
2377
2378 let headers: HashMap<_, _> = lines
2379 .filter_map(|line| {
2380 let (name, value) = line.split_once(':')?;
2381 Some((name.to_ascii_lowercase(), value.trim()))
2382 })
2383 .collect();
2384
2385 let expected_auth = format!("Basic {}", BASE64.encode(format!("{USERNAME}:{PASSWORD}")));
2386 let mut expected_headers = HashMap::from([
2387 ("host".into(), "fixture.example.test:443"),
2388 ("proxy-authorization".into(), expected_auth.as_str()),
2389 ]);
2390
2391 if let Some(user_agent) = user_agent {
2392 expected_headers.insert("user-agent".into(), user_agent);
2393 }
2394
2395 assert_eq!(request_line, "CONNECT fixture.example.test:443 HTTP/1.1");
2396 assert_eq!(headers, expected_headers);
2397 assert!(!error.to_string().contains(PASSWORD));
2398 assert!(!error.to_string().contains(&BASE64.encode(PASSWORD)));
2399 assert!(!error.to_string().contains(&expected_auth));
2400 }
2401
2402 #[rstest]
2403 #[case::cross_origin("http://127.0.0.1:2/final", None, false)]
2404 #[case::cross_origin_override("http://127.0.0.1:2/final", Some("Basic b3ZlcnJpZGU="), false)]
2405 #[case::same_origin("http://127.0.0.1:1/final", None, true)]
2406 #[case::same_origin_override("http://127.0.0.1:1/final", Some("Basic b3ZlcnJpZGU="), true)]
2407 #[case::https_connect("https://127.0.0.1:2/final", None, false)]
2408 #[tokio::test]
2409 async fn test_proxy_authentication_per_redirect_hop(
2410 #[case] destination: &'static str,
2411 #[case] initial_auth: Option<&'static str>,
2412 #[case] same_origin: bool,
2413 ) {
2414 const PROXY_AUTH: &str = "Basic cHJveHl0ZXN0OmZpeHR1cmU0Mg==";
2415
2416 let expected_initial_auth = initial_auth.unwrap_or(PROXY_AUTH);
2417 let expected_redirect_auth = if same_origin {
2418 expected_initial_auth
2419 } else {
2420 PROXY_AUTH
2421 };
2422
2423 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2424 let addr = listener.local_addr().unwrap();
2425 let (request_tx, mut request_rx) = tokio::sync::mpsc::channel(2);
2426 let router = Router::new().fallback(any(move |request: Request| {
2427 let request_tx = request_tx.clone();
2428
2429 async move {
2430 let (parts, _) = request.into_parts();
2431 let first = parts.uri.path() == "/start";
2432
2433 let expected_auth = if first {
2434 expected_initial_auth
2435 } else {
2436 expected_redirect_auth
2437 };
2438
2439 let authenticated = parts.headers.get(http::header::PROXY_AUTHORIZATION)
2440 == Some(&http::HeaderValue::from_static(expected_auth));
2441 let connect = parts.method == http::Method::CONNECT;
2442 request_tx.send(parts).await.unwrap();
2443
2444 if !authenticated || connect {
2445 return StatusCode::PROXY_AUTHENTICATION_REQUIRED.into_response();
2446 }
2447
2448 if first {
2449 return (StatusCode::TEMPORARY_REDIRECT, [("location", destination)])
2450 .into_response();
2451 }
2452
2453 "accepted".into_response()
2454 }
2455 }));
2456
2457 let server = tokio::spawn(async move { serve(listener, router).await.unwrap() });
2458 let origin_headers = HashMap::from([
2459 ("authorization".to_string(), "Bearer origin-19".to_string()),
2460 ("cookie".to_string(), "session=origin-23".to_string()),
2461 ("cookie2".to_string(), "legacy=origin-29".to_string()),
2462 (
2463 "www-authenticate".to_string(),
2464 "Basic realm=origin-31".to_string(),
2465 ),
2466 ]);
2467 let mut headers = origin_headers.clone();
2468 if let Some(initial_auth) = initial_auth {
2469 headers.insert("proxy-authorization".to_string(), initial_auth.to_string());
2470 }
2471
2472 let client = HttpClient::builder()
2473 .headers(headers)
2474 .timeout_secs(2)
2475 .use_system_proxy(false)
2476 .proxy_url(format!("http://proxytest:fixture42@{addr}"))
2477 .build()
2478 .unwrap();
2479 let result = client
2480 .request(
2481 Method::GET,
2482 "http://127.0.0.1:1/start".to_string(),
2483 None,
2484 None,
2485 None,
2486 None,
2487 None,
2488 )
2489 .await;
2490 let first = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2491 .await
2492 .unwrap()
2493 .unwrap();
2494 let redirected = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2495 .await
2496 .unwrap()
2497 .unwrap();
2498 server.abort();
2499 let connect = destination.starts_with("https:");
2500
2501 assert_eq!(first.method, http::Method::GET);
2502 assert_eq!(first.uri.to_string(), "http://127.0.0.1:1/start");
2503 assert_eq!(
2504 first.headers[http::header::PROXY_AUTHORIZATION],
2505 expected_initial_auth
2506 );
2507 assert_eq!(
2508 redirected.headers[http::header::PROXY_AUTHORIZATION],
2509 expected_redirect_auth
2510 );
2511
2512 for (name, value) in origin_headers {
2513 assert_eq!(first.headers.get(&name).unwrap(), value.as_str());
2514 assert_eq!(
2515 redirected
2516 .headers
2517 .get(&name)
2518 .map(|value| value.to_str().unwrap()),
2519 same_origin.then_some(value.as_str())
2520 );
2521 }
2522
2523 if connect {
2524 assert_eq!(redirected.method, http::Method::CONNECT);
2525 assert_eq!(redirected.uri.to_string(), "127.0.0.1:2");
2526 assert!(result.is_err());
2527 } else {
2528 let response = result.unwrap();
2529 assert_eq!(redirected.method, http::Method::GET);
2530 assert_eq!(redirected.uri.to_string(), destination);
2531 assert_eq!(response.status.as_u16(), 200);
2532 assert_eq!(response.body.as_ref(), b"accepted");
2533 }
2534 }
2535
2536 #[tokio::test]
2537 async fn test_http_client_unreachable_proxy_error_redacts_credentials() {
2538 const USERNAME: &str = "proxy-user";
2539 const SECRET: &str = "unreachable-proxy-secret";
2540 let (proxy_addr, drop_task) = spawn_connection_dropper().await;
2541 let client = HttpClient::builder()
2542 .timeout_secs(1)
2543 .proxy_url(format!("http://{USERNAME}:{SECRET}@{proxy_addr}"))
2544 .build()
2545 .unwrap();
2546 let error = client
2547 .request(
2548 Method::GET,
2549 "https://fixture.example.test/".to_string(),
2550 None,
2551 None,
2552 None,
2553 None,
2554 None,
2555 )
2556 .await
2557 .expect_err("unreachable proxy should fail");
2558 drop_task.abort();
2559 let task_error = drop_task
2560 .await
2561 .expect_err("connection dropper should be cancelled");
2562
2563 assert!(task_error.is_cancelled());
2564 assert!(!error.to_string().contains(SECRET));
2565 assert!(!error.to_string().contains(&BASE64.encode(SECRET)));
2566 assert!(
2567 !error
2568 .to_string()
2569 .contains(&BASE64.encode(format!("{USERNAME}:{SECRET}")))
2570 );
2571 }
2572
2573 #[rstest]
2574 fn test_http_client_with_valid_proxy() {
2575 let result = HttpClient::builder()
2577 .proxy_url("http://proxy.example.com:8080".to_string())
2578 .build();
2579
2580 assert!(result.is_ok());
2581 }
2582
2583 #[rstest]
2584 fn test_http_client_with_socks5_proxy() {
2585 let result = HttpClient::builder()
2587 .proxy_url("socks5://127.0.0.1:1080".to_string())
2588 .build();
2589
2590 assert!(result.is_ok());
2591 }
2592
2593 #[rstest]
2594 fn test_http_client_with_malformed_proxy() {
2595 let result = HttpClient::builder()
2599 .proxy_url("://invalid".to_string())
2600 .build();
2601
2602 assert!(result.is_err());
2603 assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2604 }
2605
2606 #[rstest]
2607 fn test_http_client_invalid_proxy_error_redacts_credentials() {
2608 const SECRET: &str = "unique-proxy-secret";
2609 let result = HttpClient::builder()
2610 .proxy_url(format!("http://proxytest:{SECRET}@[::1"))
2611 .build();
2612 let error = result.expect_err("malformed proxy URL should fail");
2613
2614 assert_eq!(
2615 error.to_string(),
2616 "Invalid proxy URL: proxy URL is malformed"
2617 );
2618 assert!(!error.to_string().contains(SECRET));
2619 }
2620
2621 #[rstest]
2622 fn test_http_client_with_empty_proxy_string() {
2623 let result = HttpClient::builder().proxy_url(String::new()).build();
2625
2626 assert!(result.is_err());
2627 assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2628 }
2629
2630 #[tokio::test]
2631 async fn test_http_client_get() {
2632 let addr = start_test_server().await.unwrap();
2633 let url = format!("http://{addr}/get");
2634
2635 let client = HttpClient::builder().build().unwrap();
2636 let response = client.get(url, None, None, None, None).await.unwrap();
2637
2638 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2639 assert_eq!(response.headers, HashMap::new());
2640 assert_eq!(response.body.as_ref(), b"hello-world!");
2641 }
2642
2643 #[tokio::test]
2644 async fn test_http_client_post() {
2645 let addr = start_test_server().await.unwrap();
2646 let url = format!("http://{addr}/post");
2647
2648 let client = HttpClient::builder().build().unwrap();
2649 let response = client
2650 .post(url, None, None, Some(b"post-body-73".to_vec()), None, None)
2651 .await
2652 .unwrap();
2653
2654 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2655 assert_eq!(response.headers, HashMap::new());
2656 assert_eq!(response.body.as_ref(), b"post-body-73");
2657 }
2658
2659 #[tokio::test]
2660 async fn test_http_client_patch() {
2661 let addr = start_test_server().await.unwrap();
2662 let url = format!("http://{addr}/patch");
2663
2664 let client = HttpClient::builder().build().unwrap();
2665 let response = client
2666 .patch(url, None, None, Some(b"patch-body-91".to_vec()), None, None)
2667 .await
2668 .unwrap();
2669
2670 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2671 assert_eq!(response.headers, HashMap::new());
2672 assert_eq!(response.body.as_ref(), b"patch-body-91");
2673 }
2674
2675 #[tokio::test]
2676 async fn test_http_client_delete() {
2677 let addr = start_test_server().await.unwrap();
2678 let url = format!("http://{addr}/delete");
2679
2680 let client = HttpClient::builder().build().unwrap();
2681 let response = client.delete(url, None, None, None, None).await.unwrap();
2682
2683 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2684 assert_eq!(response.headers, HashMap::new());
2685 assert_eq!(response.body.as_ref(), b"");
2686 }
2687}
2688
2689#[cfg(test)]
2690mod rate_limit_tests {
2691 use std::{num::NonZeroU32, sync::Arc, time::Duration};
2692
2693 #[cfg(all(feature = "simulation", madsim))]
2694 use madsim::task as test_task;
2695 #[cfg(not(all(feature = "simulation", madsim)))]
2696 use tokio::task as test_task;
2697 use ustr::Ustr;
2698
2699 use super::HttpClient;
2700 use crate::ratelimiter::{RateLimiter, quota::Quota};
2701
2702 #[tokio::test]
2703 async fn test_http_client_awaits_multiple_rate_limiters() {
2704 let quota = Quota::per_minute(NonZeroU32::MIN);
2705 let request_key = Ustr::from("scope:request");
2706 let order_key = Ustr::from("scope:order");
2707
2708 let request_limiter = Arc::new(RateLimiter::new_with_quota(
2709 None,
2710 vec![(request_key, quota)],
2711 ));
2712 let order_limiter = Arc::new(RateLimiter::new_with_quota(None, vec![(order_key, quota)]));
2713 let client = HttpClient::builder()
2714 .rate_limiters(vec![
2715 Arc::clone(&request_limiter),
2716 Arc::clone(&order_limiter),
2717 ])
2718 .build()
2719 .unwrap();
2720
2721 client
2722 .await_rate_limits(Some(&[request_key, order_key]))
2723 .await;
2724
2725 assert!(request_limiter.check_key(&request_key).is_err());
2726 assert!(order_limiter.check_key(&order_key).is_err());
2727 }
2728
2729 #[cfg_attr(
2730 not(all(feature = "simulation", madsim)),
2731 tokio::test(start_paused = true)
2732 )]
2733 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2734 async fn test_http_client_reserves_multiple_rate_limits_together() {
2735 let global_key = Ustr::from("scope:global");
2736 let order_key = Ustr::from("scope:order");
2737
2738 let global_limiter = Arc::new(RateLimiter::new_with_quota(
2739 None,
2740 vec![(
2741 global_key,
2742 Quota::with_period(Duration::from_secs(1)).unwrap(),
2743 )],
2744 ));
2745
2746 let order_limiter = Arc::new(RateLimiter::new_with_quota(
2747 None,
2748 vec![(
2749 order_key,
2750 Quota::with_period(Duration::from_secs(10)).unwrap(),
2751 )],
2752 ));
2753 order_limiter.check_key(&order_key).unwrap();
2754
2755 let client = HttpClient::builder()
2756 .rate_limiters(vec![
2757 Arc::clone(&global_limiter),
2758 Arc::clone(&order_limiter),
2759 ])
2760 .build()
2761 .unwrap();
2762
2763 let request = test_task::spawn(async move {
2764 client
2765 .await_rate_limits(Some(&[global_key, order_key]))
2766 .await;
2767 });
2768
2769 test_task::yield_now().await;
2770
2771 global_limiter.check_key(&global_key).unwrap();
2772 assert!(!request.is_finished());
2773
2774 advance_test_clock(Duration::from_millis(9_999)).await;
2775 global_limiter.until_key_ready(&global_key).await;
2776 advance_test_clock(Duration::from_millis(1)).await;
2777 test_task::yield_now().await;
2778 assert!(!request.is_finished());
2779
2780 advance_test_clock(Duration::from_millis(998)).await;
2781 test_task::yield_now().await;
2782 assert!(!request.is_finished());
2783
2784 advance_test_clock(Duration::from_millis(1)).await;
2785 request.await.unwrap();
2786
2787 assert!(global_limiter.check_key(&global_key).is_err());
2788 assert!(order_limiter.check_key(&order_key).is_err());
2789 }
2790
2791 #[cfg(all(feature = "simulation", madsim))]
2792 async fn advance_test_clock(duration: Duration) {
2793 madsim::time::advance(duration);
2794 test_task::yield_now().await;
2795 }
2796
2797 #[cfg(not(all(feature = "simulation", madsim)))]
2798 async fn advance_test_clock(duration: Duration) {
2799 tokio::time::advance(duration).await;
2800 }
2801}
2802
2803#[cfg(test)]
2804pub(super) mod content_encoding_tests {
2805 use std::{collections::HashMap, io::Write, time::Duration};
2806
2807 use bytes::Bytes;
2808 use flate2::{Compression, write::GzEncoder};
2809 use http_body_util::Full;
2810 use rstest::rstest;
2811
2812 use super::{HttpClient, HttpClientError, REQUEST_TIMEOUT_MESSAGE};
2813 use crate::dst::time;
2814
2815 #[rstest]
2816 #[case::large_body(false, false)]
2817 #[case::large_body_deadline(true, false)]
2818 #[case::empty_members(false, true)]
2819 #[case::empty_members_deadline(true, true)]
2820 #[cfg_attr(
2821 not(all(feature = "simulation", madsim)),
2822 tokio::test(start_paused = true)
2823 )]
2824 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2825 async fn gzip_decoding_yields_and_obeys_deadline(
2826 #[case] expires: bool,
2827 #[case] empty_members: bool,
2828 ) {
2829 let (payload, compressed) = if empty_members {
2830 (Vec::new(), gzip(&[]).repeat(4_000))
2831 } else {
2832 let payload = vec![b'x'; 2 * 1024 * 1024];
2833 let compressed = gzip(&payload);
2834 (payload, compressed)
2835 };
2836
2837 let body = Bytes::from(compressed);
2838 let body = Full::new(body);
2839 let response = http::Response::builder()
2840 .header("Content-Encoding", "gzip")
2841 .body(body)
2842 .unwrap();
2843 let client = HttpClient::builder()
2844 .use_system_proxy(false)
2845 .build()
2846 .unwrap()
2847 .client;
2848 let deadline = expires.then(|| time::Instant::now() + Duration::from_secs(1));
2849 let mut request = Box::pin(client.consume_response(response, deadline, false));
2850
2851 assert!(futures_util::poll!(request.as_mut()).is_pending());
2852
2853 if expires {
2854 #[cfg(not(all(feature = "simulation", madsim)))]
2855 tokio::time::advance(Duration::from_secs(2)).await;
2856 #[cfg(all(feature = "simulation", madsim))]
2857 madsim::time::advance(Duration::from_secs(2));
2858 }
2859
2860 let result = request.await;
2861 if expires {
2862 let HttpClientError::TimeoutError(message) = result.unwrap_err() else {
2863 panic!("expected decoding timeout");
2864 };
2865
2866 assert_eq!(message, REQUEST_TIMEOUT_MESSAGE);
2867 } else {
2868 let response = result.unwrap();
2869 assert_eq!(response.status.as_u16(), 200);
2870 assert_eq!(response.headers, HashMap::new());
2871 assert_eq!(response.body.as_ref(), payload);
2872 }
2873 }
2874
2875 #[rstest]
2876 #[case::stored_blocks(false)]
2877 #[case::header_fields(true)]
2878 #[cfg_attr(not(all(feature = "simulation", madsim)), tokio::test)]
2879 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2880 async fn gzip_decoding_resumes_at_input_boundaries(#[case] metadata: bool) {
2881 let payload = vec![b'x'; 128 * 1024];
2882
2883 let mut encoder = if metadata {
2884 flate2::GzBuilder::new()
2885 .filename(vec![b'f'; 32 * 1024])
2886 .comment(vec![b'c'; 32 * 1024])
2887 .write(Vec::new(), Compression::default())
2888 } else {
2889 GzEncoder::new(Vec::new(), Compression::none())
2890 };
2891
2892 encoder.write_all(&payload).unwrap();
2893 let compressed = encoder.finish().unwrap();
2894 let response = http::Response::builder()
2895 .header("Content-Encoding", "gzip")
2896 .body(Full::new(Bytes::from(compressed)))
2897 .unwrap();
2898 let client = HttpClient::builder()
2899 .use_system_proxy(false)
2900 .build()
2901 .unwrap()
2902 .client;
2903
2904 let response = client
2905 .consume_response(response, None, false)
2906 .await
2907 .unwrap();
2908
2909 assert_eq!(response.status.as_u16(), 200);
2910 assert_eq!(response.headers, HashMap::new());
2911 assert_eq!(response.body.as_ref(), payload);
2912 }
2913
2914 pub(in crate::http) fn gzip(data: &[u8]) -> Vec<u8> {
2915 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
2916 encoder.write_all(data).unwrap();
2917 encoder.finish().unwrap()
2918 }
2919}