1use std::{borrow::Cow, collections::HashMap, str::FromStr, sync::Arc, time::Duration};
19
20use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
21use bytes::Bytes;
22use http::{
23 Method,
24 header::{HeaderMap, HeaderName, HeaderValue},
25};
26use http_body_util::Full;
27use nautilus_core::{collections::into_ustr_vec, string::secret::SecretString};
28use nautilus_cryptography::providers::install_cryptographic_provider;
29use url::Url;
30use ustr::Ustr;
31
32use super::{
33 HttpClientError, HttpResponse, HttpResponseStream, HttpStatus,
34 stream::{check_deadline, read_chunk, response_error},
35};
36use crate::ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota};
37
38#[cfg(not(all(feature = "simulation", madsim)))]
40const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 32;
41
42#[cfg(not(all(feature = "simulation", madsim)))]
44const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 60;
45
46#[cfg(not(all(feature = "simulation", madsim)))]
48const DEFAULT_HTTP2_KEEP_ALIVE_SECS: u64 = 30;
49
50#[cfg(not(all(feature = "simulation", madsim)))]
52const DEFAULT_HTTP2_STREAM_WINDOW_BYTES: u32 = 16 * 1024 * 1024;
53
54#[cfg(not(all(feature = "simulation", madsim)))]
56const DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES: u32 = 32 * 1024 * 1024;
57
58#[cfg(not(all(feature = "simulation", madsim)))]
60const NAUTILUS_HTTP2_ADAPTIVE_WINDOW: &str = "NAUTILUS_HTTP2_ADAPTIVE_WINDOW";
61
62const DEFAULT_MAX_RESPONSE_BYTES: usize = 100 * 1024 * 1024;
68
69const GZIP: &str = "gzip";
70const GZIP_DECODE_CHUNK_BYTES: usize = 64 * 1024;
71
72#[cfg(all(feature = "simulation", madsim))]
73pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "simulated request deadline elapsed";
74#[cfg(not(all(feature = "simulation", madsim)))]
75pub(super) const REQUEST_TIMEOUT_MESSAGE: &str = "request deadline elapsed";
76
77#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
79pub enum HttpRedirectPolicy {
80 #[default]
86 Follow,
87 Reject,
89}
90
91#[derive(Clone, Debug)]
103pub struct HttpClient {
104 pub(crate) client: InnerHttpClient,
105 pub(crate) rate_limiters: Arc<[Arc<RateLimiter<Ustr, MonotonicClock>>]>,
106}
107
108#[bon::bon]
109impl HttpClient {
110 #[allow(
128 clippy::needless_pass_by_value,
129 reason = "owned proxy URLs are part of the public builder API"
130 )]
131 #[builder(finish_fn = build)]
132 pub fn builder(
133 #[builder(default)] headers: HashMap<String, String>,
134 #[builder(default)] header_keys: Vec<String>,
135 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
136 default_quota: Option<Quota>,
137 timeout_secs: Option<u64>,
138 proxy_url: Option<String>,
139 rate_limiters: Option<Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>>,
140 #[builder(default)] redirect_policy: HttpRedirectPolicy,
141 #[builder(default = true)]
143 use_system_proxy: bool,
144 #[builder(default)]
151 redact_url: bool,
152 #[builder(default = DEFAULT_MAX_RESPONSE_BYTES)]
157 max_decompressed_bytes: usize,
158 certs_dir: Option<String>,
161 ) -> Result<Self, HttpClientError> {
162 let rate_limiters = if let Some(rate_limiters) = rate_limiters {
163 if default_quota.is_some() || !keyed_quotas.is_empty() {
164 return Err(HttpClientError::Error(
165 "Cannot combine shared rate limiters with quota configuration".to_string(),
166 ));
167 }
168
169 rate_limiters
170 } else {
171 let keyed_quotas = keyed_quotas
172 .into_iter()
173 .map(|(key, quota)| (Ustr::from(&key), quota))
174 .collect();
175 vec![Arc::new(RateLimiter::new_with_quota(
176 default_quota,
177 keyed_quotas,
178 ))]
179 };
180
181 Self::build(
182 headers,
183 header_keys,
184 timeout_secs,
185 proxy_url.as_deref(),
186 rate_limiters,
187 redirect_policy,
188 use_system_proxy,
189 redact_url,
190 max_decompressed_bytes,
191 certs_dir.as_deref(),
192 )
193 }
194
195 #[expect(
196 clippy::too_many_arguments,
197 reason = "client construction consumes the builder settings"
198 )]
199 fn build(
200 headers: HashMap<String, String>,
201 header_keys: Vec<String>,
202 timeout_secs: Option<u64>,
203 proxy_url: Option<&str>,
204 rate_limiters: Vec<Arc<RateLimiter<Ustr, MonotonicClock>>>,
205 redirect_policy: HttpRedirectPolicy,
206 use_system_proxy: bool,
207 redact_url: bool,
208 max_decompressed_bytes: usize,
209 certs_dir: Option<&str>,
210 ) -> Result<Self, HttpClientError> {
211 let timeout = timeout_secs.map(Duration::from_secs);
212 request_deadline(timeout)?;
213 install_cryptographic_provider();
214
215 let mut header_map = HeaderMap::new();
216
217 for (key, value) in headers {
218 let header_name = HeaderName::from_str(&key)
219 .map_err(|e| HttpClientError::Error(format!("Invalid header name '{key}': {e}")))?;
220
221 let mut header_value = HeaderValue::from_str(&value).map_err(|e| {
222 HttpClientError::Error(format!("Invalid header value for '{key}': {e}"))
223 })?;
224
225 header_value.set_sensitive(true);
226 header_map.insert(header_name, header_value);
227 }
228
229 #[cfg(all(feature = "simulation", madsim))]
230 let simulation = super::simulation::Client::new(redirect_policy, proxy_url, certs_dir)?;
231
232 #[cfg(not(all(feature = "simulation", madsim)))]
233 let client = super::transport::Client::new(
234 proxy_url,
235 use_system_proxy,
236 certs_dir,
237 header_map.get(http::header::USER_AGENT).cloned(),
238 super::transport::Settings {
239 pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST,
240 pool_idle_timeout: Duration::from_secs(DEFAULT_POOL_IDLE_TIMEOUT_SECS),
241 keep_alive_interval: Some(Duration::from_secs(DEFAULT_HTTP2_KEEP_ALIVE_SECS)),
242 stream_window: Some(DEFAULT_HTTP2_STREAM_WINDOW_BYTES),
243 connection_window: Some(DEFAULT_HTTP2_CONNECTION_WINDOW_BYTES),
244 adaptive_window: http2_adaptive_window()?,
245 },
246 )?;
247
248 #[cfg(all(feature = "simulation", madsim))]
249 let _ = use_system_proxy;
250
251 let response_headers = header_keys
254 .into_iter()
255 .map(|key| match HeaderName::from_str(&key) {
256 Ok(name) => Ok((key, name)),
257 Err(e) => Err(HttpClientError::Error(format!(
258 "Invalid header key '{key}': {e}"
259 ))),
260 })
261 .collect::<Result<Vec<_>, _>>()?;
262
263 let client = InnerHttpClient {
264 #[cfg(not(all(feature = "simulation", madsim)))]
265 client,
266 headers: header_map,
267 timeout,
268 redact_url,
269 #[cfg(not(all(feature = "simulation", madsim)))]
270 redirect_policy,
271 #[cfg(all(feature = "simulation", madsim))]
272 simulation,
273 response_headers: Arc::from(response_headers),
274 max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
275 max_decompressed_bytes,
276 };
277
278 Ok(Self {
279 client,
280 rate_limiters: rate_limiters.into(),
281 })
282 }
283
284 #[expect(clippy::too_many_arguments)]
295 pub async fn request(
296 &self,
297 method: Method,
298 url: String,
299 params: Option<&HashMap<String, Vec<String>>>,
300 headers: Option<HashMap<String, String>>,
301 body: Option<Vec<u8>>,
302 timeout_secs: Option<u64>,
303 keys: Option<Vec<String>>,
304 ) -> Result<HttpResponse, HttpClientError> {
305 let keys = keys.map(into_ustr_vec);
306
307 self.request_with_ustr_keys(method, url, params, headers, body, timeout_secs, keys)
308 .await
309 }
310
311 #[expect(clippy::too_many_arguments)]
321 pub async fn request_with_secret_body(
322 &self,
323 method: Method,
324 url: String,
325 params: Option<&HashMap<String, Vec<String>>>,
326 headers: Option<HashMap<String, String>>,
327 body: SecretString,
328 timeout_secs: Option<u64>,
329 keys: Option<Vec<String>>,
330 ) -> Result<HttpResponse, HttpClientError> {
331 let keys = keys.map(into_ustr_vec);
332 self.await_rate_limits(keys.as_deref()).await;
333
334 self.client
335 .send_request_with_secret_body(method, url, params, headers, body, timeout_secs)
336 .await
337 }
338
339 #[expect(clippy::too_many_arguments)]
348 pub async fn request_with_params<P: serde::Serialize>(
349 &self,
350 method: Method,
351 url: String,
352 params: Option<&P>,
353 headers: Option<HashMap<String, String>>,
354 body: Option<Vec<u8>>,
355 timeout_secs: Option<u64>,
356 keys: Option<Vec<String>>,
357 ) -> Result<HttpResponse, HttpClientError> {
358 let keys = keys.map(into_ustr_vec);
359 self.await_rate_limits(keys.as_deref()).await;
360
361 self.client
362 .send_request_with_query(method, url, params, headers, body, timeout_secs)
363 .await
364 }
365
366 pub async fn get_stream(&self, url: String) -> Result<HttpResponseStream, HttpClientError> {
379 self.await_rate_limits(None).await;
380 self.client
381 .send_stream_internal::<[(String, String); 0]>(
382 Method::GET,
383 &url,
384 None,
385 None,
386 None,
387 None,
388 )
389 .await
390 }
391
392 #[expect(clippy::too_many_arguments)]
398 pub async fn request_with_ustr_keys(
399 &self,
400 method: Method,
401 url: String,
402 params: Option<&HashMap<String, Vec<String>>>,
403 headers: Option<HashMap<String, String>>,
404 body: Option<Vec<u8>>,
405 timeout_secs: Option<u64>,
406 keys: Option<Vec<Ustr>>,
407 ) -> Result<HttpResponse, HttpClientError> {
408 self.await_rate_limits(keys.as_deref()).await;
409
410 self.client
411 .send_request(method, url, params, headers, body, timeout_secs)
412 .await
413 }
414
415 pub async fn await_rate_limits(&self, keys: Option<&[Ustr]>) {
422 RateLimiter::await_limiters_ready(&self.rate_limiters, keys).await;
423 }
424
425 pub async fn get(
431 &self,
432 url: String,
433 params: Option<&HashMap<String, Vec<String>>>,
434 headers: Option<HashMap<String, String>>,
435 timeout_secs: Option<u64>,
436 keys: Option<Vec<String>>,
437 ) -> Result<HttpResponse, HttpClientError> {
438 self.request(Method::GET, url, params, headers, None, timeout_secs, keys)
439 .await
440 }
441
442 pub async fn post(
448 &self,
449 url: String,
450 params: Option<&HashMap<String, Vec<String>>>,
451 headers: Option<HashMap<String, String>>,
452 body: Option<Vec<u8>>,
453 timeout_secs: Option<u64>,
454 keys: Option<Vec<String>>,
455 ) -> Result<HttpResponse, HttpClientError> {
456 self.request(Method::POST, url, params, headers, body, timeout_secs, keys)
457 .await
458 }
459
460 pub async fn patch(
466 &self,
467 url: String,
468 params: Option<&HashMap<String, Vec<String>>>,
469 headers: Option<HashMap<String, String>>,
470 body: Option<Vec<u8>>,
471 timeout_secs: Option<u64>,
472 keys: Option<Vec<String>>,
473 ) -> Result<HttpResponse, HttpClientError> {
474 self.request(
475 Method::PATCH,
476 url,
477 params,
478 headers,
479 body,
480 timeout_secs,
481 keys,
482 )
483 .await
484 }
485
486 pub async fn delete(
492 &self,
493 url: String,
494 params: Option<&HashMap<String, Vec<String>>>,
495 headers: Option<HashMap<String, String>>,
496 timeout_secs: Option<u64>,
497 keys: Option<Vec<String>>,
498 ) -> Result<HttpResponse, HttpClientError> {
499 self.request(
500 Method::DELETE,
501 url,
502 params,
503 headers,
504 None,
505 timeout_secs,
506 keys,
507 )
508 .await
509 }
510}
511
512#[derive(Clone, Debug)]
518pub struct InnerHttpClient {
519 #[cfg(all(feature = "simulation", madsim))]
520 simulation: super::simulation::Client,
521 #[cfg(not(all(feature = "simulation", madsim)))]
522 client: super::transport::Client,
523 headers: HeaderMap,
524 timeout: Option<Duration>,
525 redact_url: bool,
526 #[cfg(not(all(feature = "simulation", madsim)))]
527 redirect_policy: HttpRedirectPolicy,
528 pub(crate) response_headers: Arc<[(String, HeaderName)]>,
529 pub(crate) max_response_bytes: usize,
530 pub(crate) max_decompressed_bytes: usize,
531}
532
533impl InnerHttpClient {
534 pub async fn send_request(
540 &self,
541 method: Method,
542 url: String,
543 params: Option<&HashMap<String, Vec<String>>>,
544 headers: Option<HashMap<String, String>>,
545 body: Option<Vec<u8>>,
546 timeout_secs: Option<u64>,
547 ) -> Result<HttpResponse, HttpClientError> {
548 self.send_request_with_body(
549 method,
550 url,
551 params,
552 headers,
553 body.map(RequestBody::Plain),
554 timeout_secs,
555 )
556 .await
557 }
558
559 async fn send_request_with_secret_body(
560 &self,
561 method: Method,
562 url: String,
563 params: Option<&HashMap<String, Vec<String>>>,
564 headers: Option<HashMap<String, String>>,
565 body: SecretString,
566 timeout_secs: Option<u64>,
567 ) -> Result<HttpResponse, HttpClientError> {
568 self.send_request_with_body(
569 method,
570 url,
571 params,
572 headers,
573 Some(RequestBody::Secret(body)),
574 timeout_secs,
575 )
576 .await
577 }
578
579 async fn send_request_with_body(
580 &self,
581 method: Method,
582 url: String,
583 params: Option<&HashMap<String, Vec<String>>>,
584 headers: Option<HashMap<String, String>>,
585 body: Option<RequestBody>,
586 timeout_secs: Option<u64>,
587 ) -> Result<HttpResponse, HttpClientError> {
588 let full_url = encode_url_params(&url, params)?;
589 self.send_request_internal(
590 method,
591 full_url.as_ref(),
592 None::<&()>,
593 headers,
594 body,
595 timeout_secs,
596 )
597 .await
598 }
599
600 pub async fn send_request_with_query<Q: serde::Serialize>(
609 &self,
610 method: Method,
611 url: String,
612 query: Option<&Q>,
613 headers: Option<HashMap<String, String>>,
614 body: Option<Vec<u8>>,
615 timeout_secs: Option<u64>,
616 ) -> Result<HttpResponse, HttpClientError> {
617 self.send_request_internal(
618 method,
619 &url,
620 query,
621 headers,
622 body.map(RequestBody::Plain),
623 timeout_secs,
624 )
625 .await
626 }
627
628 async fn send_request_internal<Q: serde::Serialize>(
634 &self,
635 method: Method,
636 url: &str,
637 query: Option<&Q>,
638 headers: Option<HashMap<String, String>>,
639 body: Option<RequestBody>,
640 timeout_secs: Option<u64>,
641 ) -> Result<HttpResponse, HttpClientError> {
642 let mut headers = headers.unwrap_or_default();
643 let has_accept_encoding = headers
644 .keys()
645 .any(|key| key.eq_ignore_ascii_case(http::header::ACCEPT_ENCODING.as_str()))
646 || self.headers.contains_key(http::header::ACCEPT_ENCODING);
647
648 if !has_accept_encoding {
649 headers.insert(http::header::ACCEPT_ENCODING.to_string(), GZIP.to_string());
650 }
651
652 let is_head = method == Method::HEAD;
653 let stream = self
654 .send_stream_internal(method, url, query, Some(headers), body, timeout_secs)
655 .await?;
656 let result = self
657 .consume_response(stream.response, stream.deadline, is_head)
658 .await;
659 result.map_err(|e| response_error(e, stream.origin.as_deref()))
660 }
661
662 async fn send_stream_internal<Q: serde::Serialize>(
663 &self,
664 method: Method,
665 url: &str,
666 query: Option<&Q>,
667 headers: Option<HashMap<String, String>>,
668 body: Option<RequestBody>,
669 timeout_secs: Option<u64>,
670 ) -> Result<HttpResponseStream, HttpClientError> {
671 let mut url =
672 Url::parse(url).map_err(|e| HttpClientError::from(format!("URL parse error: {e}")))?;
673 if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
674 return Err(HttpClientError::Error(
675 "unsupported HTTP URL scheme or hostname".into(),
676 ));
677 }
678
679 let mut header_map = self.headers.clone();
680
681 if let Ok(username) = percent_encoding::percent_decode_str(url.username()).decode_utf8() {
682 let password = url.password().and_then(|password| {
683 percent_encoding::percent_decode_str(password)
684 .decode_utf8()
685 .ok()
686 });
687
688 if !username.is_empty() || password.is_some() {
689 let mut value = HeaderValue::from_str(&format!(
690 "Basic {}",
691 BASE64.encode(format!(
692 "{username}:{}",
693 password.as_deref().unwrap_or_default()
694 ))
695 ))
696 .map_err(|e| HttpClientError::Error(e.to_string()))?;
697 value.set_sensitive(true);
698 header_map.insert(http::header::AUTHORIZATION, value);
699 let _ = url.set_username("");
700 let _ = url.set_password(None);
701 }
702 }
703
704 let extra_header_count = headers.as_ref().map_or(0, HashMap::len);
705
706 if let Some(headers) = headers {
707 for (key, value) in headers {
708 let key = HeaderName::from_bytes(key.as_bytes())
709 .map_err(|e| HttpClientError::from(format!("Invalid header name: {e}")))?;
710 let value = HeaderValue::from_str(&value)
711 .map_err(|e| HttpClientError::from(format!("Invalid header value: {e}")))?;
712
713 if header_map.insert(key.clone(), value).is_some() {
714 log::trace!("Replaced duplicate request header '{key}'");
715 }
716 }
717 }
718
719 if let Some(query) = query {
720 {
721 let mut pairs = url.query_pairs_mut();
722 let serializer = serde_urlencoded::Serializer::new(&mut pairs);
723 query
724 .serialize(serializer)
725 .map_err(|e| HttpClientError::Error(e.to_string()))?;
726 }
727
728 if url.query() == Some("") {
729 url.set_query(None);
730 }
731 }
732
733 if !header_map.contains_key(http::header::ACCEPT) {
734 header_map.insert(http::header::ACCEPT, HeaderValue::from_static("*/*"));
735 }
736
737 let body = body.map_or_default(RequestBody::into_bytes);
738 let body_len = body.len();
739 let query_len = url.query().map_or(0, str::len);
740 let mut request = http::Request::new(Full::new(body));
741 *request.method_mut() = method;
742 *request.uri_mut() = url[..url::Position::AfterQuery]
743 .parse()
744 .map_err(|_| HttpClientError::Error("invalid HTTP request target".into()))?;
745 *request.headers_mut() = header_map;
746 log::trace!(
747 "Sending HTTP request: method={} extra_headers={extra_header_count} \
748 query_bytes={query_len} body_bytes={body_len}",
749 request.method(),
750 );
751
752 let origin = (!self.redact_url).then(|| url.origin().ascii_serialization());
753
754 let duration = timeout_secs.map(Duration::from_secs).or(self.timeout);
755 let deadline = request_deadline(duration)?;
756
757 let operation = async {
758 #[cfg(all(feature = "simulation", madsim))]
759 let (response, connection) = self.simulation.send(request, &url).await?;
760 #[cfg(not(all(feature = "simulation", madsim)))]
761 let response = self.client.send(request, self.redirect_policy).await?;
762 Ok(HttpResponseStream {
763 response,
764 deadline,
765 origin: origin.clone(),
766 #[cfg(all(feature = "simulation", madsim))]
767 _connection: connection,
768 })
769 };
770
771 let result = match deadline {
772 Some(deadline) => tokio::select! {
773 biased;
774 () = crate::dst::time::sleep_until(deadline) => Err(HttpClientError::TimeoutError(REQUEST_TIMEOUT_MESSAGE.into())),
775 result = operation => result,
776 },
777 None => operation.await,
778 };
779
780 result.map_err(|e| response_error(e, origin.as_deref()))
781 }
782
783 async fn consume_response<B>(
784 &self,
785 response: http::Response<B>,
786 deadline: Option<crate::dst::time::Instant>,
787 is_head: bool,
788 ) -> Result<HttpResponse, HttpClientError>
789 where
790 B: http_body::Body<Data = Bytes> + Unpin,
791 B::Error: std::error::Error + 'static,
792 {
793 let (parts, mut body) = response.into_parts();
794 let mut headers =
795 HashMap::with_capacity(self.response_headers.len().min(parts.headers.len()));
796 for (key, name) in self.response_headers.iter() {
797 if let Some(value) = parts
798 .headers
799 .get(name)
800 .and_then(|value| value.to_str().ok())
801 {
802 headers.insert(key.clone(), value.to_owned());
803 }
804 }
805
806 let max = self.max_response_bytes;
807
808 if let Some(len) = body.size_hint().exact()
809 && len > max as u64
810 {
811 return Err(HttpClientError::Error(format!(
812 "HTTP response body of {len} bytes exceeds maximum of {max} bytes",
813 )));
814 }
815
816 let mut buf = bytes::BytesMut::new();
817 while let Some(chunk) = read_chunk(&mut body, deadline).await? {
818 if chunk.len() > max - buf.len() {
819 return Err(HttpClientError::Error(format!(
820 "HTTP response body exceeds maximum of {max} bytes",
821 )));
822 }
823
824 buf.extend_from_slice(&chunk);
825 }
826
827 log::trace!(
828 "Received HTTP response: status={} headers={} body_bytes={}",
829 parts.status,
830 parts.headers.len(),
831 buf.len()
832 );
833
834 let bodyless = is_head
835 || parts.status.is_informational()
836 || matches!(
837 parts.status,
838 http::StatusCode::NO_CONTENT
839 | http::StatusCode::RESET_CONTENT
840 | http::StatusCode::NOT_MODIFIED
841 );
842
843 let (body, decoded) = if bodyless {
844 (buf.freeze(), false)
845 } else {
846 decode_content_encoding(&parts.headers, buf, self.max_decompressed_bytes, deadline)
847 .await?
848 };
849
850 if decoded {
851 headers.retain(|key, _| {
852 !key.eq_ignore_ascii_case("content-encoding")
853 && !key.eq_ignore_ascii_case("content-length")
854 });
855 }
856
857 Ok(HttpResponse {
858 status: HttpStatus::new(parts.status),
859 headers,
860 body,
861 })
862 }
863}
864
865enum RequestBody {
866 Plain(Vec<u8>),
867 Secret(SecretString),
868}
869
870fn request_deadline(
871 timeout: Option<Duration>,
872) -> Result<Option<crate::dst::time::Instant>, HttpClientError> {
873 timeout
874 .map(|duration| {
875 crate::dst::time::Instant::now()
876 .checked_add(duration)
877 .ok_or_else(|| {
878 HttpClientError::Error("HTTP request timeout exceeds clock range".into())
879 })
880 })
881 .transpose()
882}
883
884async fn decode_content_encoding(
885 headers: &HeaderMap,
886 body: bytes::BytesMut,
887 max: usize,
888 deadline: Option<crate::dst::time::Instant>,
889) -> Result<(Bytes, bool), HttpClientError> {
890 let mut encodings = headers.get_all(http::header::CONTENT_ENCODING).iter();
891
892 let Some(encoding) = encodings.next() else {
893 return Ok((body.freeze(), false));
894 };
895
896 if encodings.next().is_some() {
897 return Err(HttpClientError::Error(
898 "unsupported multiple HTTP content-encoding fields".into(),
899 ));
900 }
901
902 let encoding = encoding
903 .to_str()
904 .map_err(|_| HttpClientError::Error("invalid HTTP content-encoding value".into()))?;
905 let encoding = encoding.trim();
906 if encoding.is_empty() || encoding.eq_ignore_ascii_case("identity") {
907 return Ok((body.freeze(), false));
908 }
909
910 if !encoding.eq_ignore_ascii_case(GZIP) {
911 return Err(HttpClientError::Error(format!(
912 "unsupported HTTP content-encoding '{encoding}'"
913 )));
914 }
915
916 check_deadline(deadline)?;
917 let body = body.freeze();
918 let mut out = Vec::new();
919
920 let mut reader = std::io::Read::take(
921 flate2::bufread::MultiGzDecoder::new(GzipInput {
922 bytes: body.as_ref(),
923 remaining: GZIP_DECODE_CHUNK_BYTES,
924 }),
925 (max as u64).saturating_add(1),
926 );
927
928 loop {
929 check_deadline(deadline)?;
930 let mut chunk = std::io::Read::take(&mut reader, GZIP_DECODE_CHUNK_BYTES as u64);
931 let result = std::io::Read::read_to_end(&mut chunk, &mut out);
932 check_deadline(deadline)?;
933
934 let complete = match result {
935 Ok(count) => count < GZIP_DECODE_CHUNK_BYTES,
936 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => false,
937 Err(e) => {
938 return Err(HttpClientError::Error(format!(
939 "failed to decode gzip response body: {e}"
940 )));
941 }
942 };
943
944 if out.len() > max {
945 return Err(HttpClientError::Error(format!(
946 "decompressed HTTP response body exceeds maximum of {max} bytes"
947 )));
948 }
949
950 if complete {
951 break;
952 }
953
954 crate::dst::task::yield_now().await;
955 reader.get_mut().get_mut().remaining = GZIP_DECODE_CHUNK_BYTES;
956 }
957
958 Ok((Bytes::from(out), true))
959}
960
961struct GzipInput<'a> {
962 bytes: &'a [u8],
963 remaining: usize,
964}
965
966impl std::io::Read for GzipInput<'_> {
967 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
968 if buf.is_empty() {
969 return Ok(0);
970 }
971
972 let mut available = std::io::BufRead::fill_buf(self)?;
973 let count = std::io::Read::read(&mut available, buf)?;
974 std::io::BufRead::consume(self, count);
975 Ok(count)
976 }
977}
978
979impl std::io::BufRead for GzipInput<'_> {
980 fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
981 if self.bytes.is_empty() {
982 return Ok(self.bytes);
983 }
984
985 if self.remaining == 0 {
986 return Err(std::io::ErrorKind::WouldBlock.into());
987 }
988
989 Ok(&self.bytes[..self.bytes.len().min(self.remaining)])
990 }
991
992 fn consume(&mut self, count: usize) {
993 self.bytes = &self.bytes[count..];
994 self.remaining -= count;
995 }
996}
997
998impl RequestBody {
999 fn into_bytes(self) -> Bytes {
1000 match self {
1001 Self::Plain(body) => body.into(),
1002 Self::Secret(body) => Bytes::from_owner(SecretBody(body)),
1003 }
1004 }
1005}
1006
1007struct SecretBody(SecretString);
1008
1009impl AsRef<[u8]> for SecretBody {
1010 fn as_ref(&self) -> &[u8] {
1011 self.0.expose_secret().as_bytes()
1012 }
1013}
1014
1015#[cfg(not(all(feature = "simulation", madsim)))]
1016fn http2_adaptive_window() -> Result<bool, HttpClientError> {
1017 let Some(value) = std::env::var_os(NAUTILUS_HTTP2_ADAPTIVE_WINDOW) else {
1018 return Ok(false);
1019 };
1020
1021 value.to_str().and_then(|v| v.parse().ok()).ok_or_else(|| {
1022 HttpClientError::ClientBuildError(format!(
1023 "{NAUTILUS_HTTP2_ADAPTIVE_WINDOW} must be 'true' or 'false', was '{}'",
1024 value.display()
1025 ))
1026 })
1027}
1028
1029fn encode_url_params<'a>(
1036 url: &'a str,
1037 params: Option<&HashMap<String, Vec<String>>>,
1038) -> Result<Cow<'a, str>, HttpClientError> {
1039 let Some(params) = params else {
1040 return Ok(Cow::Borrowed(url));
1041 };
1042
1043 let pairs: Vec<(&str, &str)> = params
1044 .iter()
1045 .flat_map(|(key, values)| {
1046 values
1047 .iter()
1048 .map(move |value| (key.as_str(), value.as_str()))
1049 })
1050 .collect();
1051
1052 if pairs.is_empty() {
1053 return Ok(Cow::Borrowed(url));
1054 }
1055
1056 let query_string = serde_urlencoded::to_string(pairs)
1057 .map_err(|e| HttpClientError::Error(format!("Failed to encode params: {e}")))?;
1058
1059 let (base, fragment) = match url.split_once('#') {
1062 Some((base, fragment)) => (base, Some(fragment)),
1063 None => (url, None),
1064 };
1065
1066 let separator = if base.contains('?') { '&' } else { '?' };
1067
1068 Ok(Cow::Owned(match fragment {
1069 Some(fragment) => format!("{base}{separator}{query_string}#{fragment}"),
1070 None => format!("{base}{separator}{query_string}"),
1071 }))
1072}
1073
1074#[cfg(test)]
1075mod encode_url_params_tests {
1076 use std::{borrow::Cow, collections::HashMap};
1077
1078 use rstest::rstest;
1079
1080 use super::encode_url_params;
1081
1082 fn params(pairs: &[(&str, &str)]) -> HashMap<String, Vec<String>> {
1083 let mut map: HashMap<String, Vec<String>> = HashMap::new();
1084 for (key, value) in pairs {
1085 map.entry((*key).to_string())
1086 .or_default()
1087 .push((*value).to_string());
1088 }
1089
1090 map
1091 }
1092
1093 #[rstest]
1094 #[case("https://x/y", "https://x/y?a=b")]
1095 #[case("https://x/y?old=1", "https://x/y?old=1&a=b")]
1096 #[case("https://x/y#frag", "https://x/y?a=b#frag")]
1097 #[case("https://x/y?old=1#frag", "https://x/y?old=1&a=b#frag")]
1098 #[case(
1099 "https://x/y#section?display=full",
1100 "https://x/y?a=b#section?display=full"
1101 )]
1102 #[case("https://x/y#", "https://x/y?a=b#")]
1103 fn test_query_is_inserted_before_the_fragment(#[case] url: &str, #[case] expected: &str) {
1104 let params = params(&[("a", "b")]);
1105
1106 assert_eq!(encode_url_params(url, Some(¶ms)).unwrap(), expected);
1107 }
1108
1109 #[rstest]
1110 fn test_url_is_borrowed_when_no_params_are_supplied() {
1111 assert!(matches!(
1112 encode_url_params("https://x/y#frag", None).unwrap(),
1113 Cow::Borrowed("https://x/y#frag")
1114 ));
1115 }
1116
1117 #[rstest]
1118 fn test_url_is_borrowed_when_params_are_empty() {
1119 let params = HashMap::new();
1120
1121 assert!(matches!(
1122 encode_url_params("https://x/y#frag", Some(¶ms)).unwrap(),
1123 Cow::Borrowed("https://x/y#frag")
1124 ));
1125 }
1126}
1127
1128#[cfg(test)]
1129#[cfg(target_os = "linux")] #[cfg(not(all(feature = "simulation", madsim)))]
1131mod tests {
1132 use std::net::SocketAddr;
1133
1134 use axum::{
1135 Router,
1136 body::to_bytes,
1137 extract::Request,
1138 response::IntoResponse,
1139 routing::{any, delete, get, patch, post},
1140 serve,
1141 };
1142 use http::status::StatusCode;
1143 use log::Level;
1144 use rstest::rstest;
1145 use tokio::{
1146 io::{AsyncReadExt, AsyncWriteExt},
1147 sync::oneshot,
1148 };
1149
1150 use super::*;
1151 use crate::logging::tests::capture_logs;
1152
1153 #[rstest]
1154 fn test_debug_redacts_custom_default_headers() {
1155 let client = HttpClient::builder()
1156 .use_system_proxy(false)
1157 .headers(HashMap::from([(
1158 "kalshi-access-key".into(),
1159 "HEADER_DEBUG_SECRET".into(),
1160 )]))
1161 .build()
1162 .unwrap();
1163 let header = &client.client.headers["kalshi-access-key"];
1164 let debug = format!("{client:?}");
1165
1166 assert_eq!(header.to_str().unwrap(), "HEADER_DEBUG_SECRET");
1167 assert!(header.is_sensitive());
1168 assert!(!debug.contains("HEADER_DEBUG_SECRET"), "{debug}");
1169 }
1170
1171 #[rstest]
1172 #[case("ftp://127.0.0.1:1/resource")]
1173 #[case("file:///resource")]
1174 #[tokio::test]
1175 async fn test_request_rejects_unsupported_url(#[case] url: &str) {
1176 let client = HttpClient::builder()
1177 .use_system_proxy(false)
1178 .build()
1179 .unwrap();
1180
1181 let error = client
1182 .request(Method::GET, url.to_string(), None, None, None, None, None)
1183 .await
1184 .unwrap_err();
1185
1186 assert!(
1187 matches!(error, HttpClientError::Error(ref message) if message == "unsupported HTTP URL scheme or hostname")
1188 );
1189 }
1190
1191 #[rstest]
1192 #[case::username_only("user%2F17@", "Basic dXNlci8xNzo=")]
1193 #[case::password_only(":secret%2F29@", "Basic OnNlY3JldC8yOQ==")]
1194 #[tokio::test]
1195 async fn test_request_url_credentials_allow_missing_username_or_password(
1196 #[case] userinfo: &str,
1197 #[case] expected: &str,
1198 ) {
1199 let addr = start_test_server().await.unwrap();
1200 let client = HttpClient::builder()
1201 .use_system_proxy(false)
1202 .build()
1203 .unwrap();
1204
1205 let response = client
1206 .request(
1207 Method::GET,
1208 format!("http://{userinfo}{addr}/headers"),
1209 None,
1210 None,
1211 None,
1212 None,
1213 None,
1214 )
1215 .await
1216 .unwrap();
1217
1218 assert_eq!(response.status.as_u16(), 200);
1219 assert_eq!(response.headers, HashMap::new());
1220 assert_eq!(
1221 response.body.as_ref(),
1222 format!("{expected}\n*/*").as_bytes()
1223 );
1224 }
1225
1226 #[rstest]
1227 #[case::default(None, "*/*")]
1228 #[case::explicit(Some("application/octet-stream"), "application/octet-stream")]
1229 #[tokio::test]
1230 async fn test_request_accept_header_preserves_explicit_value(
1231 #[case] accept: Option<&str>,
1232 #[case] expected: &str,
1233 ) {
1234 let addr = start_test_server().await.unwrap();
1235 let client = HttpClient::builder()
1236 .use_system_proxy(false)
1237 .build()
1238 .unwrap();
1239 let headers =
1240 accept.map(|value| HashMap::from([("accept".to_string(), value.to_string())]));
1241
1242 let response = client
1243 .request(
1244 Method::GET,
1245 format!("http://{addr}/headers"),
1246 None,
1247 headers,
1248 None,
1249 None,
1250 None,
1251 )
1252 .await
1253 .unwrap();
1254
1255 assert_eq!(response.status.as_u16(), 200);
1256 assert_eq!(response.headers, HashMap::new());
1257 assert_eq!(
1258 response.body.as_ref(),
1259 format!("absent\n{expected}").as_bytes()
1260 );
1261 }
1262
1263 async fn capture_request(request: Request) -> impl IntoResponse {
1264 let (parts, body) = request.into_parts();
1265 let body = to_bytes(body, usize::MAX).await.unwrap();
1266 let default_header = parts.headers.get("x-default").unwrap().to_str().unwrap();
1267 let request_header = parts.headers.get("x-request").unwrap().to_str().unwrap();
1268 let query = parts.uri.query().unwrap_or_default();
1269 let body = String::from_utf8(body.to_vec()).unwrap();
1270 let capture = format!(
1271 "{}\n{}\n{query}\n{default_header}\n{request_header}\n{body}",
1272 parts.method,
1273 parts.uri.path(),
1274 );
1275
1276 ([("x-response-id", "response-42")], capture)
1277 }
1278
1279 fn create_router() -> Router {
1280 Router::new()
1281 .route("/get", get(|| async { "hello-world!" }))
1282 .route("/post", post(|body: Bytes| async move { body }))
1283 .route("/patch", patch(|body: Bytes| async move { body }))
1284 .route("/delete", delete(|| async { StatusCode::OK }))
1285 .route("/capture", any(capture_request))
1286 .route(
1287 "/headers",
1288 get(|request: Request| async move {
1289 let headers = request.headers();
1290 format!(
1291 "{}\n{}",
1292 headers
1293 .get(http::header::AUTHORIZATION)
1294 .map_or("absent", |v| v.to_str().unwrap()),
1295 headers
1296 .get(http::header::ACCEPT)
1297 .map_or("absent", |v| v.to_str().unwrap()),
1298 )
1299 }),
1300 )
1301 .route("/notfound", get(|| async { StatusCode::NOT_FOUND }))
1302 .route(
1303 "/redirect",
1304 get(|| async { (StatusCode::TEMPORARY_REDIRECT, [("location", "/get")]) }),
1305 )
1306 .route(
1307 "/slow",
1308 get(|| async {
1309 tokio::time::sleep(Duration::from_secs(2)).await;
1310 "Eventually responded"
1311 }),
1312 )
1313 .route(
1314 "/large",
1315 get(|| async { "x".repeat(1024 * 1024) }),
1317 )
1318 }
1319
1320 async fn start_test_server() -> Result<SocketAddr, Box<dyn std::error::Error + Send + Sync>> {
1321 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1322 let addr = listener.local_addr().unwrap();
1323 tokio::spawn(async move {
1324 serve(listener, create_router()).await.unwrap();
1325 });
1326
1327 Ok(addr)
1328 }
1329
1330 async fn spawn_connection_dropper() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1331 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1332 let addr = listener.local_addr().unwrap();
1333
1334 let task = tokio::spawn(async move {
1335 loop {
1336 let (stream, _) = listener.accept().await.unwrap();
1337 drop(stream);
1338 }
1339 });
1340
1341 (addr, task)
1342 }
1343
1344 async fn spawn_chunked_response_server() -> (SocketAddr, tokio::task::JoinHandle<()>) {
1345 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1346 let addr = listener.local_addr().unwrap();
1347 let task = tokio::spawn(async move {
1348 let (mut stream, _) = listener.accept().await.unwrap();
1349 let mut request = Vec::new();
1350 let mut chunk = [0u8; 1024];
1351 loop {
1352 let read = stream.read(&mut chunk).await.unwrap();
1353 if read == 0 {
1354 break;
1355 }
1356
1357 request.extend_from_slice(&chunk[..read]);
1358 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1359 break;
1360 }
1361 }
1362
1363 stream
1364 .write_all(
1365 b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n\
1366 5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n",
1367 )
1368 .await
1369 .unwrap();
1370 });
1371
1372 (addr, task)
1373 }
1374
1375 async fn spawn_rejecting_connect_proxy() -> (SocketAddr, oneshot::Receiver<String>) {
1376 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1377 let addr = listener.local_addr().unwrap();
1378 let (request_tx, request_rx) = oneshot::channel();
1379
1380 tokio::spawn(async move {
1381 let (mut stream, _) = listener.accept().await.unwrap();
1382 let mut request = Vec::new();
1383 let mut chunk = [0u8; 1024];
1384 loop {
1385 let read = stream.read(&mut chunk).await.unwrap();
1386 if read == 0 {
1387 break;
1388 }
1389
1390 request.extend_from_slice(&chunk[..read]);
1391 if request.windows(4).any(|window| window == b"\r\n\r\n") {
1392 break;
1393 }
1394 }
1395
1396 request_tx
1397 .send(String::from_utf8(request).unwrap())
1398 .unwrap();
1399 stream
1400 .write_all(
1401 b"HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n",
1402 )
1403 .await
1404 .unwrap();
1405 });
1406
1407 (addr, request_rx)
1408 }
1409
1410 fn inner_client() -> InnerHttpClient {
1411 HttpClient::builder()
1412 .use_system_proxy(false)
1413 .build()
1414 .unwrap()
1415 .client
1416 }
1417
1418 #[tokio::test(start_paused = true)]
1419 async fn test_body_ready_at_deadline_is_rejected() {
1420 let client = inner_client();
1421 let response = http::Response::new(Full::new(Bytes::from_static(b"ready")));
1422 let result = client
1423 .consume_response(response, Some(crate::dst::time::Instant::now()), false)
1424 .await;
1425 assert!(
1426 matches!(result, Err(HttpClientError::TimeoutError(message)) if message == REQUEST_TIMEOUT_MESSAGE)
1427 );
1428 }
1429
1430 #[tokio::test]
1431 async fn test_get() {
1432 let addr = start_test_server().await.unwrap();
1433 let url = format!("http://{addr}");
1434
1435 let client = inner_client();
1436 let response = client
1437 .send_request(Method::GET, format!("{url}/get"), None, None, None, None)
1438 .await
1439 .unwrap();
1440
1441 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1442 assert_eq!(response.headers, HashMap::new());
1443 assert_eq!(response.body.as_ref(), b"hello-world!");
1444 }
1445
1446 #[rstest]
1447 #[tokio::test]
1448 async fn test_request_preserves_wire_semantics_and_extracts_response_headers(
1449 #[values(false, true)] redact_url: bool,
1450 ) {
1451 let addr = start_test_server().await.unwrap();
1452 let mut default_headers = HashMap::new();
1453 default_headers.insert("x-default".to_string(), "default-a".to_string());
1454 let client = HttpClient::builder()
1455 .redact_url(redact_url)
1456 .headers(default_headers)
1457 .header_keys(vec!["x-response-id".to_string()])
1458 .build()
1459 .unwrap();
1460 let mut params = HashMap::new();
1461 params.insert(
1462 "tag".to_string(),
1463 vec!["A B".to_string(), "C/D".to_string()],
1464 );
1465 let mut request_headers = HashMap::new();
1466 request_headers.insert("x-request".to_string(), "request-b".to_string());
1467
1468 let response = client
1469 .request(
1470 Method::PUT,
1471 format!("http://{addr}/capture?existing=seed"),
1472 Some(¶ms),
1473 Some(request_headers),
1474 Some(b"payload-c".to_vec()),
1475 None,
1476 None,
1477 )
1478 .await
1479 .unwrap();
1480
1481 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1482 assert_eq!(
1483 response.headers,
1484 HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1485 );
1486 assert_eq!(
1487 response.body.as_ref(),
1488 b"PUT\n/capture\nexisting=seed&tag=A+B&tag=C%2FD\ndefault-a\nrequest-b\npayload-c"
1489 );
1490 }
1491
1492 #[rstest]
1493 #[tokio::test]
1494 async fn test_request_with_secret_body_preserves_wire_body(
1495 #[values(false, true)] redact_url: bool,
1496 ) {
1497 let addr = start_test_server().await.unwrap();
1498 let client = HttpClient::builder()
1499 .redact_url(redact_url)
1500 .headers(HashMap::from([(
1501 "x-default".to_string(),
1502 "default-secret".to_string(),
1503 )]))
1504 .build()
1505 .unwrap();
1506 let headers = HashMap::from([("x-request".to_string(), "request-secret".to_string())]);
1507
1508 let response = client
1509 .request_with_secret_body(
1510 Method::POST,
1511 format!("http://{addr}/capture"),
1512 None,
1513 Some(headers),
1514 SecretString::from("credential-body"),
1515 None,
1516 None,
1517 )
1518 .await
1519 .unwrap();
1520
1521 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1522 assert_eq!(response.headers, HashMap::new());
1523 assert_eq!(
1524 response.body.as_ref(),
1525 b"POST\n/capture\n\ndefault-secret\nrequest-secret\ncredential-body"
1526 );
1527 }
1528
1529 #[rstest]
1530 #[tokio::test]
1531 async fn test_request_with_params_serializes_query_fields(
1532 #[values(false, true)] redact_url: bool,
1533 ) {
1534 #[derive(serde::Serialize)]
1535 struct Query<'a> {
1536 symbol: &'a str,
1537 limit: u32,
1538 }
1539
1540 let addr = start_test_server().await.unwrap();
1541 let mut default_headers = HashMap::new();
1542 default_headers.insert("x-default".to_string(), "default-d".to_string());
1543 let client = HttpClient::builder()
1544 .redact_url(redact_url)
1545 .headers(default_headers)
1546 .header_keys(vec!["x-response-id".to_string()])
1547 .build()
1548 .unwrap();
1549 let mut request_headers = HashMap::new();
1550 request_headers.insert("x-request".to_string(), "request-e".to_string());
1551
1552 let params = Query {
1553 symbol: "BTC/USDT",
1554 limit: 37,
1555 };
1556
1557 let response = client
1558 .request_with_params(
1559 Method::GET,
1560 format!("http://{addr}/capture"),
1561 Some(¶ms),
1562 Some(request_headers),
1563 None,
1564 None,
1565 None,
1566 )
1567 .await
1568 .unwrap();
1569
1570 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1571 assert_eq!(
1572 response.headers,
1573 HashMap::from([("x-response-id".to_string(), "response-42".to_string())])
1574 );
1575 assert_eq!(
1576 response.body.as_ref(),
1577 b"GET\n/capture\nsymbol=BTC%2FUSDT&limit=37\ndefault-d\nrequest-e\n"
1578 );
1579 }
1580
1581 #[rstest]
1582 #[tokio::test]
1583 async fn test_request_with_params_preserves_secret_query_fields(
1584 #[values(false, true)] redact_url: bool,
1585 ) {
1586 #[derive(serde::Serialize)]
1587 struct Query<'a> {
1588 auth: &'a str,
1589 market_id: i16,
1590 }
1591
1592 let addr = start_test_server().await.unwrap();
1593 let client = HttpClient::builder()
1594 .redact_url(redact_url)
1595 .headers(HashMap::from([(
1596 "x-default".to_string(),
1597 "default-f".to_string(),
1598 )]))
1599 .build()
1600 .unwrap();
1601 let headers = HashMap::from([("x-request".to_string(), "request-g".to_string())]);
1602
1603 let params = Query {
1604 auth: "token/42",
1605 market_id: 7,
1606 };
1607
1608 let response = client
1609 .request_with_params(
1610 Method::GET,
1611 format!("http://{addr}/capture"),
1612 Some(¶ms),
1613 Some(headers),
1614 None,
1615 None,
1616 None,
1617 )
1618 .await
1619 .unwrap();
1620
1621 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1622 assert_eq!(
1623 response.body.as_ref(),
1624 b"GET\n/capture\nauth=token%2F42&market_id=7\ndefault-f\nrequest-g\n"
1625 );
1626 }
1627
1628 #[rstest]
1629 #[case::empty_at_zero_cap(b"", 0)]
1630 #[case::at_cap(b"body-37", 7)]
1631 #[case::below_cap(b"body-37", 8)]
1632 #[tokio::test]
1633 async fn test_declared_response_body_at_or_below_cap_is_returned(
1634 #[case] bytes: &'static [u8],
1635 #[case] max_response_bytes: usize,
1636 ) {
1637 let client = InnerHttpClient {
1638 max_response_bytes,
1639 ..inner_client()
1640 };
1641
1642 let response = http::Response::new(Full::new(Bytes::from_static(bytes)));
1643
1644 let response = client
1645 .consume_response(response, None, false)
1646 .await
1647 .unwrap();
1648
1649 assert_eq!(response.status.as_u16(), 200);
1650 assert_eq!(response.headers, HashMap::new());
1651 assert_eq!(response.body.as_ref(), bytes);
1652 }
1653
1654 #[tokio::test]
1655 async fn test_response_body_within_cap_is_returned() {
1656 let addr = start_test_server().await.unwrap();
1657 let url = format!("http://{addr}");
1658
1659 let client = InnerHttpClient {
1661 max_response_bytes: 4 * 1024 * 1024,
1662 ..inner_client()
1663 };
1664
1665 let response = client
1666 .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1667 .await
1668 .unwrap();
1669
1670 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1671 assert_eq!(response.headers, HashMap::new());
1672 assert_eq!(response.body.as_ref(), vec![b'x'; 1024 * 1024]);
1673 }
1674
1675 #[tokio::test]
1676 async fn test_response_body_exceeding_cap_is_rejected() {
1677 let addr = start_test_server().await.unwrap();
1678 let url = format!("http://{addr}");
1679
1680 let client = InnerHttpClient {
1682 max_response_bytes: 16 * 1024,
1683 ..inner_client()
1684 };
1685
1686 let result = client
1687 .send_request(Method::GET, format!("{url}/large"), None, None, None, None)
1688 .await;
1689
1690 let err = result.expect_err("oversized response body should be rejected");
1691
1692 let HttpClientError::Error(message) = err else {
1693 panic!("expected HTTP error, was {err:?}");
1694 };
1695
1696 assert_eq!(
1697 message,
1698 "HTTP response body of 1048576 bytes exceeds maximum of 16384 bytes"
1699 );
1700 }
1701
1702 #[rstest]
1703 #[case::at_cap(11)]
1704 #[case::below_cap(12)]
1705 #[tokio::test]
1706 async fn test_chunked_response_body_at_or_below_cap_is_returned(
1707 #[case] max_response_bytes: usize,
1708 ) {
1709 let (addr, server_task) = spawn_chunked_response_server().await;
1710
1711 let client = InnerHttpClient {
1712 max_response_bytes,
1713 ..inner_client()
1714 };
1715
1716 let response = client
1717 .send_request(
1718 Method::GET,
1719 format!("http://{addr}"),
1720 None,
1721 None,
1722 None,
1723 None,
1724 )
1725 .await
1726 .unwrap();
1727 server_task.await.unwrap();
1728
1729 assert_eq!(response.status.as_u16(), 200);
1730 assert_eq!(response.headers, HashMap::new());
1731 assert_eq!(response.body.as_ref(), b"firstsecond");
1732 }
1733
1734 #[tokio::test]
1735 async fn test_chunked_response_body_exceeding_cap_is_rejected() {
1736 let (addr, server_task) = spawn_chunked_response_server().await;
1737 let max_response_bytes = 8;
1738
1739 let client = InnerHttpClient {
1740 max_response_bytes,
1741 ..inner_client()
1742 };
1743
1744 let error = client
1745 .send_request(
1746 Method::GET,
1747 format!("http://{addr}"),
1748 None,
1749 None,
1750 None,
1751 None,
1752 )
1753 .await
1754 .expect_err("chunked response body should be rejected");
1755 server_task.await.unwrap();
1756
1757 let HttpClientError::Error(message) = error else {
1758 panic!("expected HTTP error, was {error:?}");
1759 };
1760
1761 assert_eq!(
1762 message,
1763 format!("HTTP response body exceeds maximum of {max_response_bytes} bytes")
1764 );
1765 }
1766
1767 #[tokio::test]
1768 async fn test_post() {
1769 let addr = start_test_server().await.unwrap();
1770 let url = format!("http://{addr}");
1771
1772 let client = inner_client();
1773 let response = client
1774 .send_request(Method::POST, format!("{url}/post"), None, None, None, None)
1775 .await
1776 .unwrap();
1777
1778 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1779 assert_eq!(response.headers, HashMap::new());
1780 assert_eq!(response.body.as_ref(), b"");
1781 }
1782
1783 #[tokio::test]
1784 async fn test_post_with_body() {
1785 let addr = start_test_server().await.unwrap();
1786 let url = format!("http://{addr}");
1787
1788 let client = inner_client();
1789
1790 let mut body = HashMap::new();
1791 body.insert(
1792 "key1".to_string(),
1793 serde_json::Value::String("value1".to_string()),
1794 );
1795 body.insert(
1796 "key2".to_string(),
1797 serde_json::Value::String("value2".to_string()),
1798 );
1799
1800 let body_string = serde_json::to_string(&body).unwrap();
1801 let body_bytes = body_string.into_bytes();
1802
1803 let response = client
1804 .send_request(
1805 Method::POST,
1806 format!("{url}/post"),
1807 None,
1808 None,
1809 Some(body_bytes.clone()),
1810 None,
1811 )
1812 .await
1813 .unwrap();
1814
1815 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1816 assert_eq!(response.headers, HashMap::new());
1817 assert_eq!(response.body.as_ref(), body_bytes);
1818 }
1819
1820 #[tokio::test]
1821 async fn test_patch() {
1822 let addr = start_test_server().await.unwrap();
1823 let url = format!("http://{addr}");
1824
1825 let client = inner_client();
1826 let response = client
1827 .send_request(
1828 Method::PATCH,
1829 format!("{url}/patch"),
1830 None,
1831 None,
1832 None,
1833 None,
1834 )
1835 .await
1836 .unwrap();
1837
1838 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1839 assert_eq!(response.headers, HashMap::new());
1840 assert_eq!(response.body.as_ref(), b"");
1841 }
1842
1843 #[tokio::test]
1844 async fn test_delete() {
1845 let addr = start_test_server().await.unwrap();
1846 let url = format!("http://{addr}");
1847
1848 let client = inner_client();
1849 let response = client
1850 .send_request(
1851 Method::DELETE,
1852 format!("{url}/delete"),
1853 None,
1854 None,
1855 None,
1856 None,
1857 )
1858 .await
1859 .unwrap();
1860
1861 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1862 assert_eq!(response.headers, HashMap::new());
1863 assert_eq!(response.body.as_ref(), b"");
1864 }
1865
1866 #[tokio::test]
1867 async fn test_not_found() {
1868 let addr = start_test_server().await.unwrap();
1869 let url = format!("http://{addr}/notfound");
1870 let client = inner_client();
1871
1872 let response = client
1873 .send_request(Method::GET, url, None, None, None, None)
1874 .await
1875 .unwrap();
1876
1877 assert!(response.status.is_client_error());
1878 assert_eq!(response.status.as_u16(), 404);
1879 assert_eq!(response.headers, HashMap::new());
1880 assert_eq!(response.body.as_ref(), b"");
1881 }
1882
1883 #[tokio::test]
1884 async fn test_timeout() {
1885 let addr = start_test_server().await.unwrap();
1886 let url = format!("http://{addr}/slow");
1887 let client = inner_client();
1888
1889 let result = client
1891 .send_request(Method::GET, url, None, None, None, Some(1))
1892 .await;
1893
1894 assert!(
1895 matches!(&result, Err(HttpClientError::TimeoutError(_))),
1896 "Expected a timeout error, was: {result:?}"
1897 );
1898 }
1899
1900 #[rstest]
1901 fn test_http_client_without_proxy() {
1902 let result = HttpClient::builder().build();
1904
1905 assert!(result.is_ok());
1906 }
1907
1908 #[rstest]
1909 fn test_http_client_rejects_timeout_overflow() {
1910 let result = HttpClient::builder().timeout_secs(u64::MAX).build();
1911
1912 assert!(matches!(
1913 result,
1914 Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1915 ));
1916 }
1917
1918 #[tokio::test]
1919 async fn test_http_request_rejects_timeout_override_overflow() {
1920 let client = HttpClient::builder()
1921 .use_system_proxy(false)
1922 .timeout_secs(1)
1923 .build()
1924 .unwrap();
1925 let result = client
1926 .request(
1927 Method::GET,
1928 "http://127.0.0.1:1/".to_string(),
1929 None,
1930 None,
1931 None,
1932 Some(u64::MAX),
1933 None,
1934 )
1935 .await;
1936
1937 assert!(matches!(
1938 result,
1939 Err(HttpClientError::Error(message)) if message == "HTTP request timeout exceeds clock range"
1940 ));
1941 }
1942
1943 #[rstest]
1944 fn test_http_client_builder_preserves_empty_rate_limiters() {
1945 let client = HttpClient::builder()
1946 .rate_limiters(Vec::new())
1947 .build()
1948 .unwrap();
1949
1950 assert!(client.rate_limiters.is_empty());
1951 }
1952
1953 #[rstest]
1954 fn test_http_client_builder_rejects_shared_rate_limiters_with_quotas() {
1955 let quota = Quota::with_period(Duration::from_secs(1)).unwrap();
1956 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
1957 let result = HttpClient::builder()
1958 .default_quota(quota)
1959 .rate_limiters(vec![rate_limiter])
1960 .build();
1961
1962 assert_eq!(
1963 result.unwrap_err().to_string(),
1964 "HTTP error occurred: Cannot combine shared rate limiters with quota configuration"
1965 );
1966 }
1967
1968 #[tokio::test]
1969 async fn test_http_client_without_proxy_requests_directly() {
1970 let addr = start_test_server().await.unwrap();
1971 let client = HttpClient::builder().timeout_secs(2).build().unwrap();
1972 let response = client
1973 .request(
1974 Method::GET,
1975 format!("http://{addr}/get"),
1976 None,
1977 None,
1978 None,
1979 None,
1980 None,
1981 )
1982 .await
1983 .expect("direct request");
1984
1985 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
1986 assert_eq!(response.body.as_ref(), b"hello-world!");
1987 }
1988
1989 #[tokio::test]
1990 async fn test_http_client_redirect_policy() {
1991 let addr = start_test_server().await.unwrap();
1992 let follow = HttpClient::builder().timeout_secs(2).build().unwrap();
1993 let reject = HttpClient::builder()
1994 .timeout_secs(2)
1995 .redirect_policy(HttpRedirectPolicy::Reject)
1996 .build()
1997 .unwrap();
1998
1999 let followed = follow
2000 .request(
2001 Method::GET,
2002 format!("http://{addr}/redirect"),
2003 None,
2004 None,
2005 None,
2006 None,
2007 None,
2008 )
2009 .await
2010 .unwrap();
2011 let rejected = reject
2012 .request(
2013 Method::GET,
2014 format!("http://{addr}/redirect"),
2015 None,
2016 None,
2017 None,
2018 None,
2019 None,
2020 )
2021 .await
2022 .unwrap();
2023
2024 assert_eq!(followed.status.as_u16(), StatusCode::OK.as_u16());
2025 assert_eq!(followed.body.as_ref(), b"hello-world!");
2026 assert_eq!(
2027 rejected.status.as_u16(),
2028 StatusCode::TEMPORARY_REDIRECT.as_u16()
2029 );
2030 assert!(rejected.body.is_empty());
2031 }
2032
2033 #[rstest]
2034 #[tokio::test]
2035 async fn test_http_client_url_policy_preserves_response(
2036 #[values(false, true)] redact_url: bool,
2037 ) {
2038 let addr = start_test_server().await.unwrap();
2039 let client = HttpClient::builder()
2040 .redact_url(redact_url)
2041 .timeout_secs(2)
2042 .build()
2043 .unwrap();
2044 let response = client
2045 .request(
2046 Method::GET,
2047 format!("http://{addr}/get"),
2048 None,
2049 None,
2050 None,
2051 None,
2052 None,
2053 )
2054 .await
2055 .expect("direct request with URL redaction");
2056
2057 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2058 assert_eq!(response.body.as_ref(), b"hello-world!");
2059 }
2060
2061 #[rstest]
2062 #[case::request("request")]
2063 #[case::preinterned_keys("keys")]
2064 #[case::get("get")]
2065 #[case::post("post")]
2066 #[case::patch("patch")]
2067 #[case::delete("delete")]
2068 #[case::stream("stream")]
2069 #[tokio::test]
2070 async fn test_http_client_url_policy_redacts_transport_errors(
2071 #[case] request: &str,
2072 #[values(false, true)] redact_url: bool,
2073 ) {
2074 const USERINFO_SECRET: &str = "transport-userinfo-secret";
2075 const PATH_SECRET: &str = "transport-path-secret";
2076 const QUERY_SECRET: &str = "transport-query-secret";
2077 let (addr, drop_task) = spawn_connection_dropper().await;
2078 let url = format!(
2079 "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2080 );
2081 let client = HttpClient::builder()
2082 .redact_url(redact_url)
2083 .timeout_secs(1)
2084 .build()
2085 .unwrap();
2086
2087 let error = match request {
2088 "request" => client
2089 .request(Method::GET, url.clone(), None, None, None, None, None)
2090 .await
2091 .unwrap_err(),
2092 "keys" => client
2093 .request_with_ustr_keys(
2094 Method::GET,
2095 url.clone(),
2096 None,
2097 None,
2098 None,
2099 None,
2100 Some(vec![Ustr::from("policy")]),
2101 )
2102 .await
2103 .unwrap_err(),
2104 "get" => client
2105 .get(url.clone(), None, None, None, None)
2106 .await
2107 .unwrap_err(),
2108 "post" => client
2109 .post(url.clone(), None, None, None, None, None)
2110 .await
2111 .unwrap_err(),
2112 "patch" => client
2113 .patch(url.clone(), None, None, None, None, None)
2114 .await
2115 .unwrap_err(),
2116 "delete" => client
2117 .delete(url.clone(), None, None, None, None)
2118 .await
2119 .unwrap_err(),
2120 "stream" => client.get_stream(url.clone()).await.unwrap_err(),
2121 _ => panic!("unknown request shape"),
2122 };
2123
2124 drop_task.abort();
2125 let task_error = drop_task
2126 .await
2127 .expect_err("connection dropper should be cancelled");
2128
2129 assert!(task_error.is_cancelled());
2130 assert!(
2131 matches!(error, HttpClientError::TransportError(_)),
2132 "{error:?}"
2133 );
2134
2135 for rendered in [error.to_string(), format!("{error:?}")] {
2136 assert!(!rendered.contains(USERINFO_SECRET));
2137 assert!(!rendered.contains(PATH_SECRET));
2138 assert!(!rendered.contains(QUERY_SECRET));
2139 assert!(!rendered.contains(&url));
2140 assert_eq!(
2141 rendered.contains(&format!("http://{addr}")),
2142 !redact_url,
2143 "{rendered}"
2144 );
2145 assert_eq!(rendered.contains("for origin ("), !redact_url, "{rendered}");
2146 }
2147 }
2148
2149 #[tokio::test]
2150 async fn test_request_with_params_redact_url_removes_origin_from_error() {
2151 const QUERY_SECRET: &str = "transport-query-secret";
2152 #[derive(serde::Serialize)]
2153 struct Query<'a> {
2154 auth: &'a str,
2155 }
2156
2157 let (addr, drop_task) = spawn_connection_dropper().await;
2158 let url = format!("http://{addr}/trades");
2159 let params = Query { auth: QUERY_SECRET };
2160 let client = HttpClient::builder()
2161 .redact_url(true)
2162 .timeout_secs(1)
2163 .build()
2164 .unwrap();
2165
2166 let error = client
2167 .request_with_params(Method::GET, url, Some(¶ms), None, None, None, None)
2168 .await
2169 .expect_err("a dropped connection should fail");
2170 drop_task.abort();
2171 let task_error = drop_task
2172 .await
2173 .expect_err("connection dropper should be cancelled");
2174
2175 assert!(task_error.is_cancelled());
2176
2177 for rendered in [error.to_string(), format!("{error:?}")] {
2178 assert!(!rendered.contains(&addr.to_string()));
2179 assert!(!rendered.contains("auth="));
2180 assert!(!rendered.contains(QUERY_SECRET));
2181 }
2182 }
2183
2184 #[tokio::test]
2185 async fn test_request_removes_query_string_from_transport_error_by_default() {
2186 const QUERY_SECRET: &str = "default-query-secret";
2187 const FRAGMENT_MARKER: &str = "default-fragment-marker";
2188 let (addr, drop_task) = spawn_connection_dropper().await;
2189 let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}#{FRAGMENT_MARKER}");
2190 let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2191
2192 let error = client
2193 .request(Method::GET, url, None, None, None, None, None)
2194 .await
2195 .expect_err("a dropped connection should fail");
2196 drop_task.abort();
2197 let task_error = drop_task
2198 .await
2199 .expect_err("connection dropper should be cancelled");
2200
2201 assert!(task_error.is_cancelled());
2202
2203 for rendered in [error.to_string(), format!("{error:?}")] {
2204 assert!(
2205 rendered.contains(&format!("for origin (http://{addr})")),
2206 "default error omitted the origin: {rendered}"
2207 );
2208 assert!(!rendered.contains("/trades"), "{rendered}");
2209 assert!(!rendered.contains("api_key="), "{rendered}");
2210 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2211 assert!(!rendered.contains(FRAGMENT_MARKER), "{rendered}");
2212 }
2213 }
2214
2215 #[rstest]
2216 #[tokio::test]
2217 async fn test_request_with_secret_body_url_policy_redacts_transport_error(
2218 #[values(false, true)] redact_url: bool,
2219 ) {
2220 const QUERY_SECRET: &str = "secret-body-query-secret";
2221 let (addr, drop_task) = spawn_connection_dropper().await;
2222 let url = format!("http://{addr}/trades?api_key={QUERY_SECRET}");
2223 let client = HttpClient::builder()
2224 .redact_url(redact_url)
2225 .timeout_secs(1)
2226 .build()
2227 .unwrap();
2228
2229 let error = client
2230 .request_with_secret_body(
2231 Method::POST,
2232 url,
2233 None,
2234 None,
2235 SecretString::from("credential-body"),
2236 None,
2237 None,
2238 )
2239 .await
2240 .expect_err("a dropped connection should fail");
2241 drop_task.abort();
2242 let task_error = drop_task
2243 .await
2244 .expect_err("connection dropper should be cancelled");
2245
2246 assert!(task_error.is_cancelled());
2247
2248 for rendered in [error.to_string(), format!("{error:?}")] {
2249 assert_eq!(
2250 rendered.contains(&format!("for origin (http://{addr})")),
2251 !redact_url,
2252 "{rendered}"
2253 );
2254 assert_eq!(
2255 rendered.contains(&addr.to_string()),
2256 !redact_url,
2257 "{rendered}"
2258 );
2259 assert!(!rendered.contains("/trades"), "{rendered}");
2260 assert!(!rendered.contains("credential-body"), "{rendered}");
2261 assert!(!rendered.contains("api_key="), "{rendered}");
2262 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2263 }
2264 }
2265
2266 #[tokio::test]
2267 async fn test_request_with_params_removes_query_from_transport_error_by_default() {
2268 const QUERY_SECRET: &str = "default-params-query-secret";
2269 #[derive(serde::Serialize)]
2270 struct Query<'a> {
2271 auth: &'a str,
2272 }
2273
2274 let (addr, drop_task) = spawn_connection_dropper().await;
2275 let url = format!("http://{addr}/trades");
2276 let params = Query { auth: QUERY_SECRET };
2277 let client = HttpClient::builder().timeout_secs(1).build().unwrap();
2278
2279 let error = client
2280 .request_with_params(Method::GET, url, Some(¶ms), None, None, None, None)
2281 .await
2282 .expect_err("a dropped connection should fail");
2283 drop_task.abort();
2284 let task_error = drop_task
2285 .await
2286 .expect_err("connection dropper should be cancelled");
2287
2288 assert!(task_error.is_cancelled());
2289
2290 for rendered in [error.to_string(), format!("{error:?}")] {
2291 assert!(
2292 rendered.contains(&format!("for origin (http://{addr})")),
2293 "default error omitted the origin: {rendered}"
2294 );
2295 assert!(!rendered.contains("/trades"), "{rendered}");
2296 assert!(!rendered.contains("auth="), "{rendered}");
2297 assert!(!rendered.contains(QUERY_SECRET), "{rendered}");
2298 }
2299 }
2300
2301 #[rstest]
2302 #[tokio::test]
2303 async fn test_http_client_url_policy_removes_endpoint_from_trace_logs(
2304 #[values(false, true)] redact_url: bool,
2305 ) {
2306 const USERINFO_SECRET: &str = "trace-userinfo-secret";
2307 const PATH_SECRET: &str = "trace-path-secret";
2308 const QUERY_SECRET: &str = "trace-query-secret";
2309 let capture = capture_logs().await;
2310 let addr = start_test_server().await.unwrap();
2311 let url = format!(
2312 "http://rpc-user:{USERINFO_SECRET}@{addr}/{PATH_SECRET}?api_key={QUERY_SECRET}"
2313 );
2314 let client = HttpClient::builder()
2315 .redact_url(redact_url)
2316 .timeout_secs(2)
2317 .build()
2318 .unwrap();
2319
2320 let response = client
2321 .request(Method::GET, url.clone(), None, None, None, None, None)
2322 .await
2323 .expect("credentialized endpoint should return an HTTP response");
2324 let messages = capture.messages();
2325
2326 assert_eq!(response.status.as_u16(), StatusCode::NOT_FOUND.as_u16());
2327 assert!(messages.iter().any(|(level, message)| {
2328 *level == Level::Trace && message.starts_with("Sending HTTP request: method=GET")
2329 }));
2330 assert!(messages.iter().any(|(level, message)| {
2331 *level == Level::Trace
2332 && message.starts_with("Received HTTP response: status=404 Not Found")
2333 }));
2334
2335 for (_, message) in messages {
2336 assert!(!message.contains(USERINFO_SECRET));
2337 assert!(!message.contains(PATH_SECRET));
2338 assert!(!message.contains(QUERY_SECRET));
2339 assert!(!message.contains(&url));
2340 }
2341 }
2342
2343 #[rstest]
2344 #[case::without_user_agent(None)]
2345 #[case::with_user_agent(Some("NautilusTrader/proxy-test-73"))]
2346 #[tokio::test]
2347 async fn test_http_client_uses_connect_and_proxy_authorization_for_https(
2348 #[case] user_agent: Option<&str>,
2349 ) {
2350 const USERNAME: &str = "proxytest";
2351 const PASSWORD: &str = "fixture42";
2352 let mut headers =
2353 HashMap::from([("authorization".into(), "Bearer origin-secret-19".into())]);
2354
2355 if let Some(user_agent) = user_agent {
2356 headers.insert("user-agent".into(), user_agent.into());
2357 }
2358
2359 let (proxy_addr, request_rx) = spawn_rejecting_connect_proxy().await;
2360 let client = HttpClient::builder()
2361 .headers(headers)
2362 .timeout_secs(2)
2363 .proxy_url(format!("http://{USERNAME}:{PASSWORD}@{proxy_addr}"))
2364 .build()
2365 .unwrap();
2366 let error = client
2367 .request(
2368 Method::GET,
2369 "https://fixture.example.test/path".to_string(),
2370 None,
2371 None,
2372 None,
2373 None,
2374 None,
2375 )
2376 .await
2377 .expect_err("proxy should reject CONNECT");
2378 let request = request_rx.await.expect("captured CONNECT request");
2379 let mut lines = request.split("\r\n");
2380 let request_line = lines.next().expect("CONNECT request line");
2381
2382 let headers: HashMap<_, _> = lines
2383 .filter_map(|line| {
2384 let (name, value) = line.split_once(':')?;
2385 Some((name.to_ascii_lowercase(), value.trim()))
2386 })
2387 .collect();
2388
2389 let expected_auth = format!("Basic {}", BASE64.encode(format!("{USERNAME}:{PASSWORD}")));
2390 let mut expected_headers = HashMap::from([
2391 ("host".into(), "fixture.example.test:443"),
2392 ("proxy-authorization".into(), expected_auth.as_str()),
2393 ]);
2394
2395 if let Some(user_agent) = user_agent {
2396 expected_headers.insert("user-agent".into(), user_agent);
2397 }
2398
2399 assert_eq!(request_line, "CONNECT fixture.example.test:443 HTTP/1.1");
2400 assert_eq!(headers, expected_headers);
2401 assert!(!error.to_string().contains(PASSWORD));
2402 assert!(!error.to_string().contains(&BASE64.encode(PASSWORD)));
2403 assert!(!error.to_string().contains(&expected_auth));
2404 }
2405
2406 #[rstest]
2407 #[case::cross_origin("http://127.0.0.1:2/final", None, false)]
2408 #[case::cross_origin_override("http://127.0.0.1:2/final", Some("Basic b3ZlcnJpZGU="), false)]
2409 #[case::same_origin("http://127.0.0.1:1/final", None, true)]
2410 #[case::same_origin_override("http://127.0.0.1:1/final", Some("Basic b3ZlcnJpZGU="), true)]
2411 #[case::https_connect("https://127.0.0.1:2/final", None, false)]
2412 #[tokio::test]
2413 async fn test_proxy_authentication_per_redirect_hop(
2414 #[case] destination: &'static str,
2415 #[case] initial_auth: Option<&'static str>,
2416 #[case] same_origin: bool,
2417 ) {
2418 const PROXY_AUTH: &str = "Basic cHJveHl0ZXN0OmZpeHR1cmU0Mg==";
2419
2420 let expected_initial_auth = initial_auth.unwrap_or(PROXY_AUTH);
2421 let expected_redirect_auth = if same_origin {
2422 expected_initial_auth
2423 } else {
2424 PROXY_AUTH
2425 };
2426
2427 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2428 let addr = listener.local_addr().unwrap();
2429 let (request_tx, mut request_rx) = tokio::sync::mpsc::channel(2);
2430 let router = Router::new().fallback(any(move |request: Request| {
2431 let request_tx = request_tx.clone();
2432
2433 async move {
2434 let (parts, _) = request.into_parts();
2435 let first = parts.uri.path() == "/start";
2436
2437 let expected_auth = if first {
2438 expected_initial_auth
2439 } else {
2440 expected_redirect_auth
2441 };
2442
2443 let authenticated = parts.headers.get(http::header::PROXY_AUTHORIZATION)
2444 == Some(&http::HeaderValue::from_static(expected_auth));
2445 let connect = parts.method == http::Method::CONNECT;
2446 request_tx.send(parts).await.unwrap();
2447
2448 if !authenticated || connect {
2449 return StatusCode::PROXY_AUTHENTICATION_REQUIRED.into_response();
2450 }
2451
2452 if first {
2453 return (StatusCode::TEMPORARY_REDIRECT, [("location", destination)])
2454 .into_response();
2455 }
2456
2457 "accepted".into_response()
2458 }
2459 }));
2460
2461 let server = tokio::spawn(async move { serve(listener, router).await.unwrap() });
2462 let origin_headers = HashMap::from([
2463 ("authorization".to_string(), "Bearer origin-19".to_string()),
2464 ("cookie".to_string(), "session=origin-23".to_string()),
2465 ("cookie2".to_string(), "legacy=origin-29".to_string()),
2466 (
2467 "www-authenticate".to_string(),
2468 "Basic realm=origin-31".to_string(),
2469 ),
2470 ]);
2471 let mut headers = origin_headers.clone();
2472 if let Some(initial_auth) = initial_auth {
2473 headers.insert("proxy-authorization".to_string(), initial_auth.to_string());
2474 }
2475
2476 let client = HttpClient::builder()
2477 .headers(headers)
2478 .timeout_secs(2)
2479 .use_system_proxy(false)
2480 .proxy_url(format!("http://proxytest:fixture42@{addr}"))
2481 .build()
2482 .unwrap();
2483 let result = client
2484 .request(
2485 Method::GET,
2486 "http://127.0.0.1:1/start".to_string(),
2487 None,
2488 None,
2489 None,
2490 None,
2491 None,
2492 )
2493 .await;
2494 let first = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2495 .await
2496 .unwrap()
2497 .unwrap();
2498 let redirected = tokio::time::timeout(Duration::from_secs(2), request_rx.recv())
2499 .await
2500 .unwrap()
2501 .unwrap();
2502 server.abort();
2503 let connect = destination.starts_with("https:");
2504
2505 assert_eq!(first.method, http::Method::GET);
2506 assert_eq!(first.uri.to_string(), "http://127.0.0.1:1/start");
2507 assert_eq!(
2508 first.headers[http::header::PROXY_AUTHORIZATION],
2509 expected_initial_auth
2510 );
2511 assert_eq!(
2512 redirected.headers[http::header::PROXY_AUTHORIZATION],
2513 expected_redirect_auth
2514 );
2515
2516 for (name, value) in origin_headers {
2517 assert_eq!(first.headers.get(&name).unwrap(), value.as_str());
2518 assert_eq!(
2519 redirected
2520 .headers
2521 .get(&name)
2522 .map(|value| value.to_str().unwrap()),
2523 same_origin.then_some(value.as_str())
2524 );
2525 }
2526
2527 if connect {
2528 assert_eq!(redirected.method, http::Method::CONNECT);
2529 assert_eq!(redirected.uri.to_string(), "127.0.0.1:2");
2530 assert!(result.is_err());
2531 } else {
2532 let response = result.unwrap();
2533 assert_eq!(redirected.method, http::Method::GET);
2534 assert_eq!(redirected.uri.to_string(), destination);
2535 assert_eq!(response.status.as_u16(), 200);
2536 assert_eq!(response.body.as_ref(), b"accepted");
2537 }
2538 }
2539
2540 #[tokio::test]
2541 async fn test_http_client_unreachable_proxy_error_redacts_credentials() {
2542 const USERNAME: &str = "proxy-user";
2543 const SECRET: &str = "unreachable-proxy-secret";
2544 let (proxy_addr, drop_task) = spawn_connection_dropper().await;
2545 let client = HttpClient::builder()
2546 .timeout_secs(1)
2547 .proxy_url(format!("http://{USERNAME}:{SECRET}@{proxy_addr}"))
2548 .build()
2549 .unwrap();
2550 let error = client
2551 .request(
2552 Method::GET,
2553 "https://fixture.example.test/".to_string(),
2554 None,
2555 None,
2556 None,
2557 None,
2558 None,
2559 )
2560 .await
2561 .expect_err("unreachable proxy should fail");
2562 drop_task.abort();
2563 let task_error = drop_task
2564 .await
2565 .expect_err("connection dropper should be cancelled");
2566
2567 assert!(task_error.is_cancelled());
2568 assert!(!error.to_string().contains(SECRET));
2569 assert!(!error.to_string().contains(&BASE64.encode(SECRET)));
2570 assert!(
2571 !error
2572 .to_string()
2573 .contains(&BASE64.encode(format!("{USERNAME}:{SECRET}")))
2574 );
2575 }
2576
2577 #[rstest]
2578 fn test_http_client_with_valid_proxy() {
2579 let result = HttpClient::builder()
2581 .proxy_url("http://proxy.example.com:8080".to_string())
2582 .build();
2583
2584 assert!(result.is_ok());
2585 }
2586
2587 #[rstest]
2588 fn test_http_client_with_socks5_proxy() {
2589 let result = HttpClient::builder()
2591 .proxy_url("socks5://127.0.0.1:1080".to_string())
2592 .build();
2593
2594 assert!(result.is_ok());
2595 }
2596
2597 #[rstest]
2598 fn test_http_client_with_malformed_proxy() {
2599 let result = HttpClient::builder()
2603 .proxy_url("://invalid".to_string())
2604 .build();
2605
2606 assert!(result.is_err());
2607 assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2608 }
2609
2610 #[rstest]
2611 fn test_http_client_invalid_proxy_error_redacts_credentials() {
2612 const SECRET: &str = "unique-proxy-secret";
2613 let result = HttpClient::builder()
2614 .proxy_url(format!("http://proxytest:{SECRET}@[::1"))
2615 .build();
2616 let error = result.expect_err("malformed proxy URL should fail");
2617
2618 assert_eq!(
2619 error.to_string(),
2620 "Invalid proxy URL: proxy URL is malformed"
2621 );
2622 assert!(!error.to_string().contains(SECRET));
2623 }
2624
2625 #[rstest]
2626 fn test_http_client_with_empty_proxy_string() {
2627 let result = HttpClient::builder().proxy_url(String::new()).build();
2629
2630 assert!(result.is_err());
2631 assert!(matches!(result, Err(HttpClientError::InvalidProxy(_))));
2632 }
2633
2634 #[tokio::test]
2635 async fn test_http_client_get() {
2636 let addr = start_test_server().await.unwrap();
2637 let url = format!("http://{addr}/get");
2638
2639 let client = HttpClient::builder().build().unwrap();
2640 let response = client.get(url, None, None, None, None).await.unwrap();
2641
2642 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2643 assert_eq!(response.headers, HashMap::new());
2644 assert_eq!(response.body.as_ref(), b"hello-world!");
2645 }
2646
2647 #[tokio::test]
2648 async fn test_http_client_post() {
2649 let addr = start_test_server().await.unwrap();
2650 let url = format!("http://{addr}/post");
2651
2652 let client = HttpClient::builder().build().unwrap();
2653 let response = client
2654 .post(url, None, None, Some(b"post-body-73".to_vec()), None, None)
2655 .await
2656 .unwrap();
2657
2658 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2659 assert_eq!(response.headers, HashMap::new());
2660 assert_eq!(response.body.as_ref(), b"post-body-73");
2661 }
2662
2663 #[tokio::test]
2664 async fn test_http_client_patch() {
2665 let addr = start_test_server().await.unwrap();
2666 let url = format!("http://{addr}/patch");
2667
2668 let client = HttpClient::builder().build().unwrap();
2669 let response = client
2670 .patch(url, None, None, Some(b"patch-body-91".to_vec()), None, None)
2671 .await
2672 .unwrap();
2673
2674 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2675 assert_eq!(response.headers, HashMap::new());
2676 assert_eq!(response.body.as_ref(), b"patch-body-91");
2677 }
2678
2679 #[tokio::test]
2680 async fn test_http_client_delete() {
2681 let addr = start_test_server().await.unwrap();
2682 let url = format!("http://{addr}/delete");
2683
2684 let client = HttpClient::builder().build().unwrap();
2685 let response = client.delete(url, None, None, None, None).await.unwrap();
2686
2687 assert_eq!(response.status.as_u16(), StatusCode::OK.as_u16());
2688 assert_eq!(response.headers, HashMap::new());
2689 assert_eq!(response.body.as_ref(), b"");
2690 }
2691}
2692
2693#[cfg(test)]
2694mod rate_limit_tests {
2695 use std::{num::NonZeroU32, sync::Arc, time::Duration};
2696
2697 #[cfg(all(feature = "simulation", madsim))]
2698 use madsim::task as test_task;
2699 #[cfg(not(all(feature = "simulation", madsim)))]
2700 use tokio::task as test_task;
2701 use ustr::Ustr;
2702
2703 use super::HttpClient;
2704 use crate::ratelimiter::{RateLimiter, quota::Quota};
2705
2706 #[tokio::test]
2707 async fn test_http_client_awaits_multiple_rate_limiters() {
2708 let quota = Quota::per_minute(NonZeroU32::MIN);
2709 let request_key = Ustr::from("scope:request");
2710 let order_key = Ustr::from("scope:order");
2711
2712 let request_limiter = Arc::new(RateLimiter::new_with_quota(
2713 None,
2714 vec![(request_key, quota)],
2715 ));
2716 let order_limiter = Arc::new(RateLimiter::new_with_quota(None, vec![(order_key, quota)]));
2717 let client = HttpClient::builder()
2718 .rate_limiters(vec![
2719 Arc::clone(&request_limiter),
2720 Arc::clone(&order_limiter),
2721 ])
2722 .build()
2723 .unwrap();
2724
2725 client
2726 .await_rate_limits(Some(&[request_key, order_key]))
2727 .await;
2728
2729 assert!(request_limiter.check_key(&request_key).is_err());
2730 assert!(order_limiter.check_key(&order_key).is_err());
2731 }
2732
2733 #[cfg_attr(
2734 not(all(feature = "simulation", madsim)),
2735 tokio::test(start_paused = true)
2736 )]
2737 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2738 async fn test_http_client_reserves_multiple_rate_limits_together() {
2739 let global_key = Ustr::from("scope:global");
2740 let order_key = Ustr::from("scope:order");
2741
2742 let global_limiter = Arc::new(RateLimiter::new_with_quota(
2743 None,
2744 vec![(
2745 global_key,
2746 Quota::with_period(Duration::from_secs(1)).unwrap(),
2747 )],
2748 ));
2749
2750 let order_limiter = Arc::new(RateLimiter::new_with_quota(
2751 None,
2752 vec![(
2753 order_key,
2754 Quota::with_period(Duration::from_secs(10)).unwrap(),
2755 )],
2756 ));
2757 order_limiter.check_key(&order_key).unwrap();
2758
2759 let client = HttpClient::builder()
2760 .rate_limiters(vec![
2761 Arc::clone(&global_limiter),
2762 Arc::clone(&order_limiter),
2763 ])
2764 .build()
2765 .unwrap();
2766
2767 let request = test_task::spawn(async move {
2768 client
2769 .await_rate_limits(Some(&[global_key, order_key]))
2770 .await;
2771 });
2772
2773 test_task::yield_now().await;
2774
2775 global_limiter.check_key(&global_key).unwrap();
2776 assert!(!request.is_finished());
2777
2778 advance_test_clock(Duration::from_millis(9_999)).await;
2779 global_limiter.until_key_ready(&global_key).await;
2780 advance_test_clock(Duration::from_millis(1)).await;
2781 test_task::yield_now().await;
2782 assert!(!request.is_finished());
2783
2784 advance_test_clock(Duration::from_millis(998)).await;
2785 test_task::yield_now().await;
2786 assert!(!request.is_finished());
2787
2788 advance_test_clock(Duration::from_millis(1)).await;
2789 request.await.unwrap();
2790
2791 assert!(global_limiter.check_key(&global_key).is_err());
2792 assert!(order_limiter.check_key(&order_key).is_err());
2793 }
2794
2795 #[cfg(all(feature = "simulation", madsim))]
2796 async fn advance_test_clock(duration: Duration) {
2797 madsim::time::advance(duration);
2798 test_task::yield_now().await;
2799 }
2800
2801 #[cfg(not(all(feature = "simulation", madsim)))]
2802 async fn advance_test_clock(duration: Duration) {
2803 tokio::time::advance(duration).await;
2804 }
2805}
2806
2807#[cfg(test)]
2808pub(super) mod content_encoding_tests {
2809 use std::{collections::HashMap, io::Write, time::Duration};
2810
2811 use bytes::Bytes;
2812 use flate2::{Compression, write::GzEncoder};
2813 use http_body_util::Full;
2814 use rstest::rstest;
2815
2816 use super::{HttpClient, HttpClientError, REQUEST_TIMEOUT_MESSAGE};
2817 use crate::dst::time;
2818
2819 #[rstest]
2820 #[case::large_body(false, false)]
2821 #[case::large_body_deadline(true, false)]
2822 #[case::empty_members(false, true)]
2823 #[case::empty_members_deadline(true, true)]
2824 #[cfg_attr(
2825 not(all(feature = "simulation", madsim)),
2826 tokio::test(start_paused = true)
2827 )]
2828 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2829 async fn gzip_decoding_yields_and_obeys_deadline(
2830 #[case] expires: bool,
2831 #[case] empty_members: bool,
2832 ) {
2833 let (payload, compressed) = if empty_members {
2834 (Vec::new(), gzip(&[]).repeat(4_000))
2835 } else {
2836 let payload = vec![b'x'; 2 * 1024 * 1024];
2837 let compressed = gzip(&payload);
2838 (payload, compressed)
2839 };
2840
2841 let body = Bytes::from(compressed);
2842 let body = Full::new(body);
2843 let response = http::Response::builder()
2844 .header("Content-Encoding", "gzip")
2845 .body(body)
2846 .unwrap();
2847 let client = HttpClient::builder()
2848 .use_system_proxy(false)
2849 .build()
2850 .unwrap()
2851 .client;
2852 let deadline = expires.then(|| time::Instant::now() + Duration::from_secs(1));
2853 let mut request = Box::pin(client.consume_response(response, deadline, false));
2854
2855 assert!(futures_util::poll!(request.as_mut()).is_pending());
2856
2857 if expires {
2858 #[cfg(not(all(feature = "simulation", madsim)))]
2859 tokio::time::advance(Duration::from_secs(2)).await;
2860 #[cfg(all(feature = "simulation", madsim))]
2861 madsim::time::advance(Duration::from_secs(2));
2862 }
2863
2864 let result = request.await;
2865 if expires {
2866 let HttpClientError::TimeoutError(message) = result.unwrap_err() else {
2867 panic!("expected decoding timeout");
2868 };
2869
2870 assert_eq!(message, REQUEST_TIMEOUT_MESSAGE);
2871 } else {
2872 let response = result.unwrap();
2873 assert_eq!(response.status.as_u16(), 200);
2874 assert_eq!(response.headers, HashMap::new());
2875 assert_eq!(response.body.as_ref(), payload);
2876 }
2877 }
2878
2879 #[rstest]
2880 #[case::stored_blocks(false)]
2881 #[case::header_fields(true)]
2882 #[cfg_attr(not(all(feature = "simulation", madsim)), tokio::test)]
2883 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
2884 async fn gzip_decoding_resumes_at_input_boundaries(#[case] metadata: bool) {
2885 let payload = vec![b'x'; 128 * 1024];
2886
2887 let mut encoder = if metadata {
2888 flate2::GzBuilder::new()
2889 .filename(vec![b'f'; 32 * 1024])
2890 .comment(vec![b'c'; 32 * 1024])
2891 .write(Vec::new(), Compression::default())
2892 } else {
2893 GzEncoder::new(Vec::new(), Compression::none())
2894 };
2895
2896 encoder.write_all(&payload).unwrap();
2897 let compressed = encoder.finish().unwrap();
2898 let response = http::Response::builder()
2899 .header("Content-Encoding", "gzip")
2900 .body(Full::new(Bytes::from(compressed)))
2901 .unwrap();
2902 let client = HttpClient::builder()
2903 .use_system_proxy(false)
2904 .build()
2905 .unwrap()
2906 .client;
2907
2908 let response = client
2909 .consume_response(response, None, false)
2910 .await
2911 .unwrap();
2912
2913 assert_eq!(response.status.as_u16(), 200);
2914 assert_eq!(response.headers, HashMap::new());
2915 assert_eq!(response.body.as_ref(), payload);
2916 }
2917
2918 pub(in crate::http) fn gzip(data: &[u8]) -> Vec<u8> {
2919 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
2920 encoder.write_all(data).unwrap();
2921 encoder.finish().unwrap()
2922 }
2923}