1use std::{
57 collections::VecDeque,
58 fmt::{Debug, Display},
59 future::Future,
60 pin::pin,
61 sync::{
62 Arc, OnceLock,
63 atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering},
64 },
65 time::Duration,
66};
67
68use futures_util::{SinkExt, StreamExt};
69use http::HeaderName;
70use nautilus_core::string::secret::REDACTED;
71use nautilus_cryptography::providers::install_cryptographic_provider;
72use parking_lot::RwLock;
73#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
74use rustls::ClientConfig;
75#[cfg(feature = "transport-sockudo")]
76use sockudo_ws::{
77 Config as SockudoConfig, Http1, Role, Stream as SockudoStream,
78 WebSocketStream as SockudoWebSocketStream,
79};
80#[cfg(feature = "transport-sockudo")]
81use tokio::io::{AsyncRead, AsyncWrite};
82#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
83use tokio_rustls::TlsConnector;
84use tokio_tungstenite::tungstenite::{
85 client::IntoClientRequest, handshake::client::Request, http::HeaderValue, stream::Mode,
86};
87use tokio_util::sync::CancellationToken;
88use ustr::Ustr;
89
90#[cfg(not(feature = "turmoil"))]
91use super::proxy::{ProxyKind, WsTarget, tunnel_via_proxy};
92use super::{
93 auth::{AuthState, AuthTracker},
94 config::{InitialConnectRetryPolicy, TransportBackend, WebSocketConfig},
95 consts::{
96 CONNECTION_STATE_CHECK_INTERVAL_MS, GRACEFUL_SHUTDOWN_DELAY_MS,
97 GRACEFUL_SHUTDOWN_TIMEOUT_SECS,
98 },
99 frame::FrameStream,
100 types::{
101 CloseHandler, EpochMessageHandler, EpochPingHandler, MessageHandler, MessageReader,
102 MessageWriter, PingHandler, WriterCommand,
103 },
104};
105#[cfg(feature = "transport-sockudo")]
106use crate::net::TcpStream;
107#[cfg(feature = "transport-sockudo")]
108use crate::transport::sockudo::{
109 PrefixedIo, SockudoTransport, client_handshake_with_headers, validate_extra_headers,
110};
111use crate::{
112 RECONNECTED, SocketState, SocketStateSink,
113 backoff::{
114 ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
115 },
116 dst,
117 error::{SendError, is_connection_drop_io_error},
118 logging::{log_task_aborted, log_task_started, log_task_stopped},
119 mode::{
120 ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
121 ReconnectRequestOutcome,
122 },
123 ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota},
124 retry::{RetryConfig, RetryError, RetryManager},
125 transport::{
126 BoxedWsTransport, CloseFrame, Message, TransportError, error::retryable_status,
127 tungstenite::TungsteniteTransport,
128 },
129 writer::{DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
130};
131
132const WRITE_TIMEOUT_SECS: u64 = 5;
133const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
134const INITIAL_CONNECT_OPERATION: &str = "WebSocket initial connection";
135
136const MAX_CONTROL_FRAME_PAYLOAD_BYTES: usize = 125;
138
139pub struct WebSocketClientInner {
164 config: WebSocketConfig,
165 reconnect_headers: ReconnectHeaders,
166 handler: Option<IncomingHandler>,
167 ping_handler: Option<IncomingPingHandler>,
168 close_handler: Option<CloseHandler>,
169 read_task: Option<dst::task::JoinHandle<()>>,
170 read_fence: Option<ReadSessionFence>,
171 write_task: dst::task::JoinHandle<()>,
172 writer_tx: WriterSender<WriterCommand>,
173 heartbeat_task: Option<dst::task::JoinHandle<()>>,
174 connection_mode: Arc<AtomicU8>,
175 connection_epoch: Arc<AtomicU64>,
176 state_notify: Arc<tokio::sync::Notify>,
177 controller_notify: Arc<tokio::sync::Notify>,
178 reconnect_published: Arc<AtomicBool>,
179 connect_timeout: Duration,
180 heartbeat_timeout: Option<Duration>,
181 backoff: ExponentialBackoff,
182 reconnect_throttle: ReconnectThrottle,
183 reconnect_max_attempts: Option<u32>,
184 reconnection_attempt_count: u32,
185 auth_tracker: Arc<OnceLock<AuthTracker>>,
186 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
187 state_sink: Option<SocketStateSink>,
188 connection_rate_limit: Option<ConnectionRateLimit>,
189}
190
191#[derive(Clone, Debug)]
192struct ConnectionRateLimit {
193 limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
194 keys: Arc<[Ustr]>,
195}
196
197#[derive(Default)]
198struct InitialConnectOptions {
199 retry_policy: Option<InitialConnectRetryPolicy>,
200 cancellation_token: Option<CancellationToken>,
201}
202
203impl WebSocketClientInner {
204 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
212 #[expect(
213 clippy::unused_async,
214 clippy::unused_async_trait_impl,
215 reason = "async signature for consistency with connect-based constructors"
216 )]
217 pub async fn new_with_writer(
218 config: WebSocketConfig,
219 writer: MessageWriter,
220 ) -> Result<Self, TransportError> {
221 Self::new_with_writer_and_state_sink(config, writer, None)
222 }
223
224 fn new_with_writer_and_state_sink(
225 mut config: WebSocketConfig,
226 writer: MessageWriter,
227 state_sink: Option<SocketStateSink>,
228 ) -> Result<Self, TransportError> {
229 install_cryptographic_provider();
230
231 crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
232 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
233 })?;
234
235 if config.heartbeat_interval_secs == Some(0) {
236 return Err(TransportError::Io(std::io::Error::new(
237 std::io::ErrorKind::InvalidInput,
238 "Heartbeat interval cannot be zero",
239 )));
240 }
241
242 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
243 let connection_epoch = Arc::new(AtomicU64::new(0));
244 let state_notify = Arc::new(tokio::sync::Notify::new());
245 let controller_notify = Arc::new(tokio::sync::Notify::new());
246 let reconnect_published = Arc::new(AtomicBool::new(true));
247 let outcome =
248 ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
249 debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
250
251 let read_task = None;
253 let read_fence = None;
254
255 let backoff = ExponentialBackoff::new(
257 Duration::from_secs(2),
258 Duration::from_secs(30),
259 1.5,
260 100,
261 true,
262 )
263 .map_err(|e| {
264 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
265 })?;
266
267 let auth_tracker = Arc::new(OnceLock::new());
268 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
269
270 let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
271 config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
272 );
273 let write_task = Self::spawn_write_task(
274 Arc::clone(&connection_mode),
275 Arc::clone(&controller_notify),
276 Arc::clone(&reconnect_published),
277 writer,
278 writer_rx,
279 Arc::clone(&connection_epoch),
280 Arc::clone(&auth_tracker),
281 Arc::clone(&reconnect_buffer_waits_for_auth),
282 state_sink.clone(),
283 );
284
285 let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_interval| {
286 Self::spawn_heartbeat_task(
287 Arc::clone(&connection_mode),
288 heartbeat_interval,
289 config.heartbeat_payload.clone(),
290 writer_tx.clone(),
291 )
292 });
293
294 let reconnect_max_attempts = None; let connect_timeout = Duration::from_secs(10);
296
297 let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
298
299 Ok(Self {
300 config,
301 reconnect_headers,
302 handler: None, ping_handler: None,
304 close_handler: None,
305 writer_tx,
306 connection_mode,
307 connection_epoch,
308 state_notify,
309 controller_notify,
310 reconnect_published,
311 connect_timeout,
312 heartbeat_timeout: None,
313 heartbeat_task,
314 read_task,
315 read_fence,
316 write_task,
317 backoff,
318 reconnect_throttle: ReconnectThrottle::default(),
319 reconnect_max_attempts,
320 reconnection_attempt_count: 0,
321 auth_tracker,
322 reconnect_buffer_waits_for_auth,
323 state_sink,
324 connection_rate_limit: None,
325 })
326 }
327
328 pub async fn connect_url(
337 config: WebSocketConfig,
338 message_handler: Option<MessageHandler>,
339 ping_handler: Option<PingHandler>,
340 ) -> Result<Self, TransportError> {
341 Self::connect_url_with_handler(
342 config,
343 message_handler.map(IncomingHandler::Message),
344 ping_handler.map(IncomingPingHandler::Ping),
345 None,
346 None,
347 None,
348 InitialConnectOptions::default(),
349 )
350 .await
351 }
352
353 async fn connect_url_with_handler(
354 config: WebSocketConfig,
355 handler: Option<IncomingHandler>,
356 ping_handler: Option<IncomingPingHandler>,
357 close_handler: Option<CloseHandler>,
358 state_sink: Option<SocketStateSink>,
359 connection_rate_limit: Option<ConnectionRateLimit>,
360 initial_connect_options: InitialConnectOptions,
361 ) -> Result<Self, TransportError> {
362 install_cryptographic_provider();
363
364 let is_stream_mode = handler.is_none();
365
366 if is_stream_mode {
370 crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
371 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
372 })?;
373
374 if config.heartbeat_interval_secs == Some(0) {
375 return Err(TransportError::Io(std::io::Error::new(
376 std::io::ErrorKind::InvalidInput,
377 "Heartbeat interval cannot be zero",
378 )));
379 }
380
381 validate_honored_inbound_limits(&config)?;
382 } else {
383 config.validate().map_err(|e| {
384 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
385 })?;
386 }
387
388 let heartbeat_timeout = config.resolved_heartbeat_timeout().map(Duration::from_secs);
389 let frame_activity = (!is_stream_mode && heartbeat_timeout.is_some())
390 .then(|| Arc::new(AtomicBool::new(false)));
391 let reconnect_max_attempts = config.reconnect_max_attempts;
392
393 let connect_timeout = if is_stream_mode {
395 Duration::from_secs(10)
396 } else {
397 Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000))
398 };
399
400 let backoff = ExponentialBackoff::new(
401 Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
402 Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
403 config.reconnect_backoff_factor.unwrap_or(1.5),
404 config.reconnect_jitter_ms.unwrap_or(100),
405 true, )
407 .map_err(|e| {
408 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
409 })?;
410
411 let cancellation_token = initial_connect_options
412 .cancellation_token
413 .unwrap_or_default();
414 let retry_policy = initial_connect_options.retry_policy;
415 let max_attempts = retry_policy
416 .as_ref()
417 .map_or(1, |policy| policy.max_attempts.get());
418 let retry_manager = RetryManager::new(initial_connect_retry_config(retry_policy.as_ref())?);
419 let attempt = AtomicU32::new(0);
420 let operation = || {
421 attempt.fetch_add(1, Ordering::Relaxed);
422
423 async {
424 await_initial_connect_attempt(&cancellation_token, async {
425 if let Some(rate_limit) = &connection_rate_limit {
426 rate_limit
427 .limiter
428 .await_keys_ready(Some(&rate_limit.keys))
429 .await;
430 }
431
432 dst::time::timeout(
434 connect_timeout,
435 Box::pin(Self::connect_with_server_activity(
436 &config.url,
437 config.headers.clone(),
438 config.backend,
439 config.proxy_url.as_deref(),
440 config.max_message_size_bytes,
441 config.max_frame_size_bytes,
442 frame_activity.clone(),
443 )),
444 )
445 .await
446 .unwrap_or_else(|_| {
447 Err(TransportError::Io(std::io::Error::new(
448 std::io::ErrorKind::TimedOut,
449 format!(
450 "connection timed out after {}s",
451 connect_timeout.as_secs_f64()
452 ),
453 )))
454 })
455 })
456 .await
457 .map_err(InitialConnectError)
458 }
459 };
460
461 let classify = |error: &InitialConnectError| {
462 let error = &error.0;
463 let retryable = is_retryable_initial_connect_error(error);
464 let attempt = attempt.load(Ordering::Relaxed);
465 if retryable && attempt < max_attempts && !cancellation_token.is_cancelled() {
466 log::warn!(
467 "WebSocket connection attempt {attempt}/{max_attempts} to {REDACTED} failed: {}",
468 error.redacted_message()
469 );
470 }
471
472 retryable
473 };
474
475 let transport = retry_manager
476 .invocation(INITIAL_CONNECT_OPERATION, operation, classify, |e| {
477 InitialConnectError(initial_connect_retry_error(e))
478 })
479 .cancellation_token(&cancellation_token)
480 .execute()
481 .await
482 .map_err(|e| e.0)?;
483
484 let attempt = attempt.load(Ordering::Relaxed);
485 if attempt > 1 {
486 log::info!("WebSocket connection established after {attempt} attempts");
487 }
488
489 let (writer, reader) = transport;
490 let mut config = config;
491 let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
492
493 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
494 let connection_epoch = Arc::new(AtomicU64::new(0));
495 let state_notify = Arc::new(tokio::sync::Notify::new());
496 let controller_notify = Arc::new(tokio::sync::Notify::new());
497 let reconnect_published = Arc::new(AtomicBool::new(true));
498 let outcome =
499 ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
500 debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
501
502 let (read_task, read_fence) = if is_stream_mode {
503 (None, None)
504 } else {
505 let read_fence = ReadSessionFence::new();
506 let read_task = Self::spawn_message_handler_task(
507 Arc::clone(&connection_mode),
508 Arc::clone(&state_notify),
509 read_fence.clone(),
510 reader,
511 0,
512 handler.as_ref(),
513 ping_handler.as_ref(),
514 close_handler.as_ref(),
515 state_sink.as_ref(),
516 config.idle_timeout_ms,
517 heartbeat_timeout,
518 frame_activity,
519 );
520 (Some(read_task), Some(read_fence))
521 };
522
523 let auth_tracker = Arc::new(OnceLock::new());
524 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
525
526 let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
527 config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
528 );
529 let write_task = Self::spawn_write_task(
530 Arc::clone(&connection_mode),
531 Arc::clone(&controller_notify),
532 Arc::clone(&reconnect_published),
533 writer,
534 writer_rx,
535 Arc::clone(&connection_epoch),
536 Arc::clone(&auth_tracker),
537 Arc::clone(&reconnect_buffer_waits_for_auth),
538 state_sink.clone(),
539 );
540
541 let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_secs| {
543 Self::spawn_heartbeat_task(
544 Arc::clone(&connection_mode),
545 heartbeat_secs,
546 config.heartbeat_payload.clone(),
547 writer_tx.clone(),
548 )
549 });
550
551 Ok(Self {
552 config,
553 reconnect_headers,
554 handler,
555 ping_handler,
556 close_handler,
557 read_task,
558 read_fence,
559 write_task,
560 writer_tx,
561 heartbeat_task,
562 connection_mode,
563 connection_epoch,
564 state_notify,
565 controller_notify,
566 reconnect_published,
567 connect_timeout,
568 heartbeat_timeout,
569 backoff,
570 reconnect_throttle: ReconnectThrottle::default(),
571 reconnect_max_attempts,
572 reconnection_attempt_count: 0,
573 auth_tracker,
574 reconnect_buffer_waits_for_auth,
575 state_sink,
576 connection_rate_limit,
577 })
578 }
579
580 #[inline]
602 pub async fn connect_with_server(
603 url: &str,
604 headers: Vec<(String, String)>,
605 backend: TransportBackend,
606 proxy_url: Option<&str>,
607 max_message_size_bytes: Option<usize>,
608 max_frame_size_bytes: Option<usize>,
609 ) -> Result<(MessageWriter, MessageReader), TransportError> {
610 Self::connect_with_server_activity(
611 url,
612 headers,
613 backend,
614 proxy_url,
615 max_message_size_bytes,
616 max_frame_size_bytes,
617 None,
618 )
619 .await
620 }
621
622 async fn connect_with_server_activity(
623 url: &str,
624 headers: Vec<(String, String)>,
625 backend: TransportBackend,
626 proxy_url: Option<&str>,
627 max_message_size_bytes: Option<usize>,
628 max_frame_size_bytes: Option<usize>,
629 frame_activity: Option<Arc<AtomicBool>>,
630 ) -> Result<(MessageWriter, MessageReader), TransportError> {
631 #[cfg(all(feature = "simulation", madsim))]
632 if backend == TransportBackend::Sockudo {
633 return Err(TransportError::Other(
634 "Sockudo timers are unsupported under simulation; select Tungstenite".into(),
635 ));
636 }
637
638 #[cfg(all(feature = "simulation", madsim))]
639 if proxy_url.is_some() || !url.starts_with("ws://") {
640 return Err(TransportError::InvalidUrl(
641 "WebSocket simulation requires plaintext ws:// endpoints without a proxy".into(),
642 ));
643 }
644
645 let limits = InboundConfig {
646 max_message_size_bytes,
647 max_frame_size_bytes,
648 frame_activity,
649 };
650
651 match backend {
652 TransportBackend::Tungstenite => match proxy_url {
653 Some(proxy) => {
654 Box::pin(Self::connect_tungstenite_via_proxy(
655 url, headers, proxy, limits,
656 ))
657 .await
658 }
659 None => Self::connect_tungstenite(url, headers, limits).await,
660 },
661 TransportBackend::Sockudo => {
662 #[cfg(feature = "transport-sockudo")]
663 {
664 match proxy_url {
665 Some(proxy) => {
666 Box::pin(Self::connect_sockudo_via_proxy(url, headers, proxy, limits))
667 .await
668 }
669 None => Self::connect_sockudo(url, headers, limits).await,
670 }
671 }
672
673 #[cfg(not(feature = "transport-sockudo"))]
674 {
675 Err(TransportError::Other(
676 "sockudo backend selected but the transport-sockudo \
677 Cargo feature is not enabled"
678 .to_string(),
679 ))
680 }
681 }
682 }
683 }
684
685 async fn connect_tungstenite(
686 url: &str,
687 headers: Vec<(String, String)>,
688 limits: InboundConfig,
689 ) -> Result<(MessageWriter, MessageReader), TransportError> {
690 let request = tungstenite_request(url, headers)?;
691 let mode = tokio_tungstenite::tungstenite::client::uri_mode(request.uri())?;
692 let host = request
693 .uri()
694 .host()
695 .ok_or_else(|| TransportError::InvalidUrl("missing hostname".into()))?;
696 let port = request
697 .uri()
698 .port_u16()
699 .unwrap_or(if matches!(mode, Mode::Tls) { 443 } else { 80 });
700 let address = format!("{host}:{port}");
701 let socket = dst::net::TcpStream::connect(address.as_str()).await?;
702 #[cfg(not(all(feature = "simulation", madsim)))]
703 crate::net::apply_socket_options(&socket);
704 let socket = crate::tls::tcp_tls(&request, mode, socket, None).await?;
705 let socket = FrameStream::new(socket, limits.frame_activity.clone());
706 let (stream, _) = tokio_tungstenite::client_async_with_config(
707 request,
708 socket,
709 tungstenite_protocol_config(&limits),
710 )
711 .await?;
712 let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
713 Ok(transport.split())
714 }
715
716 #[inline]
724 #[cfg(not(feature = "turmoil"))]
725 async fn connect_tungstenite_via_proxy(
726 url: &str,
727 headers: Vec<(String, String)>,
728 proxy_url: &str,
729 limits: InboundConfig,
730 ) -> Result<(MessageWriter, MessageReader), TransportError> {
731 let proxy = match ProxyKind::parse(proxy_url)? {
732 ProxyKind::Http(target) => target,
733 ProxyKind::Unsupported { scheme } => {
734 log::warn!(
735 "WebSocket proxy_url scheme '{scheme}' is not yet supported; \
736 connecting without a WebSocket proxy"
737 );
738 return Self::connect_tungstenite(url, headers, limits).await;
739 }
740 };
741
742 let request = tungstenite_request(url, headers)?;
743
744 let target = WsTarget::parse(url)?;
745 let stream = tunnel_via_proxy(&target, &proxy).await?;
746
747 let transport: BoxedWsTransport =
751 Box::pin(proxied_ws_handshake(request, stream, limits)).await?;
752
753 Ok(transport.split())
754 }
755
756 #[inline]
759 #[cfg(feature = "turmoil")]
760 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
761 #[expect(
762 clippy::unused_async,
763 clippy::unused_async_trait_impl,
764 reason = "signature mirrors the production variant; both are awaited in the dispatcher"
765 )]
766 async fn connect_tungstenite_via_proxy(
767 _url: &str,
768 _headers: Vec<(String, String)>,
769 _proxy_url: &str,
770 _limits: InboundConfig,
771 ) -> Result<(MessageWriter, MessageReader), TransportError> {
772 Err(TransportError::Other(
773 "proxy_url is not supported under the turmoil simulator".to_string(),
774 ))
775 }
776
777 #[inline]
786 #[cfg(feature = "transport-sockudo")]
787 async fn connect_sockudo(
788 url: &str,
789 headers: Vec<(String, String)>,
790 limits: InboundConfig,
791 ) -> Result<(MessageWriter, MessageReader), TransportError> {
792 let target = SockudoTarget::parse(url)?;
793 validate_extra_headers(&headers).map_err(TransportError::from)?;
794
795 #[cfg(feature = "turmoil")]
796 if target.is_tls {
797 return Err(TransportError::Tls(
798 "wss:// is not supported under the turmoil simulator; use ws://".to_string(),
799 ));
800 }
801
802 let tcp_stream = TcpStream::connect((target.host.as_str(), target.port))
803 .await
804 .map_err(TransportError::Io)?;
805
806 crate::net::apply_socket_options(&tcp_stream);
807
808 #[cfg(not(feature = "turmoil"))]
809 if target.is_tls {
810 let mut root_store = rustls::RootCertStore::empty();
811 root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
812 let config = ClientConfig::builder()
813 .with_root_certificates(root_store)
814 .with_no_client_auth();
815 let connector = TlsConnector::from(std::sync::Arc::new(config));
816 let domain = rustls::pki_types::ServerName::try_from(target.host.clone())
817 .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
818 let tls_stream = connector
819 .connect(domain, tcp_stream)
820 .await
821 .map_err(TransportError::Io)?;
822 return Self::finish_sockudo_handshake(tls_stream, &target, &headers, limits).await;
823 }
824
825 Self::finish_sockudo_handshake(tcp_stream, &target, &headers, limits).await
826 }
827
828 #[inline]
834 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
835 async fn connect_sockudo_via_proxy(
836 url: &str,
837 headers: Vec<(String, String)>,
838 proxy_url: &str,
839 limits: InboundConfig,
840 ) -> Result<(MessageWriter, MessageReader), TransportError> {
841 let proxy = match ProxyKind::parse(proxy_url)? {
842 ProxyKind::Http(target) => target,
843 ProxyKind::Unsupported { scheme } => {
844 log::warn!(
845 "WebSocket proxy_url scheme '{scheme}' is not yet supported; \
846 connecting without a WebSocket proxy"
847 );
848 return Self::connect_sockudo(url, headers, limits).await;
849 }
850 };
851
852 let target = SockudoTarget::parse(url)?;
853 validate_extra_headers(&headers).map_err(TransportError::from)?;
854
855 let ws_target = WsTarget::parse(url)?;
859 let stream = tunnel_via_proxy(&ws_target, &proxy).await?;
860
861 Self::finish_sockudo_handshake(stream, &target, &headers, limits).await
862 }
863
864 #[inline]
867 #[cfg(all(feature = "transport-sockudo", feature = "turmoil"))]
868 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
869 #[expect(
870 clippy::unused_async,
871 clippy::unused_async_trait_impl,
872 reason = "signature mirrors the production variant; both are awaited in the dispatcher"
873 )]
874 async fn connect_sockudo_via_proxy(
875 _url: &str,
876 _headers: Vec<(String, String)>,
877 _proxy_url: &str,
878 _limits: InboundConfig,
879 ) -> Result<(MessageWriter, MessageReader), TransportError> {
880 Err(TransportError::Other(
881 "proxy_url is not supported under the turmoil simulator".to_string(),
882 ))
883 }
884
885 #[cfg(feature = "transport-sockudo")]
886 async fn finish_sockudo_handshake<S>(
887 stream: S,
888 target: &SockudoTarget,
889 headers: &[(String, String)],
890 limits: InboundConfig,
891 ) -> Result<(MessageWriter, MessageReader), TransportError>
892 where
893 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
894 {
895 let mut stream = FrameStream::new(stream, limits.frame_activity.clone());
899 let handshake =
900 client_handshake_with_headers(&mut stream, &target.host_header, &target.path, headers)
901 .await?;
902
903 let stream = match handshake.leftover {
906 Some(prefix) => SockudoStream::<Http1>::new(PrefixedIo::new(stream, prefix)),
907 None => SockudoStream::<Http1>::new(stream),
908 };
909
910 let ws = SockudoWebSocketStream::from_raw(
911 stream,
912 Role::Client,
913 sockudo_protocol_config(&limits),
914 );
915 let transport: BoxedWsTransport = Box::pin(SockudoTransport::new(ws));
916 Ok(transport.split())
917 }
918}
919
920#[derive(Clone, Debug, Default)]
921struct InboundConfig {
922 max_message_size_bytes: Option<usize>,
923 max_frame_size_bytes: Option<usize>,
924 frame_activity: Option<Arc<AtomicBool>>,
925}
926
927fn validate_honored_inbound_limits(config: &WebSocketConfig) -> Result<(), TransportError> {
928 if config.max_message_size_bytes == Some(0) {
929 return Err(TransportError::Io(std::io::Error::new(
930 std::io::ErrorKind::InvalidInput,
931 "max_message_size_bytes must be positive",
932 )));
933 }
934
935 if config.max_frame_size_bytes == Some(0) {
936 return Err(TransportError::Io(std::io::Error::new(
937 std::io::ErrorKind::InvalidInput,
938 "max_frame_size_bytes must be positive",
939 )));
940 }
941
942 Ok(())
943}
944
945fn tungstenite_protocol_config(
946 limits: &InboundConfig,
947) -> Option<tokio_tungstenite::tungstenite::protocol::WebSocketConfig> {
948 if limits.max_message_size_bytes.is_none() && limits.max_frame_size_bytes.is_none() {
949 return None;
950 }
951
952 let mut config = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
953 if let Some(size) = limits.max_message_size_bytes {
954 config.max_message_size = Some(size);
955 }
956
957 if let Some(size) = limits.max_frame_size_bytes {
958 config.max_frame_size = Some(size);
959 }
960
961 Some(config)
962}
963
964#[cfg(feature = "transport-sockudo")]
965fn sockudo_protocol_config(limits: &InboundConfig) -> SockudoConfig {
966 let mut config = SockudoConfig::builder()
967 .auto_ping(false)
968 .idle_timeout(0)
969 .build();
970
971 if let Some(size) = limits.max_message_size_bytes {
972 config.max_message_size = size;
973 }
974
975 if let Some(size) = limits.max_frame_size_bytes {
976 config.max_frame_size = size;
977 }
978
979 config
980}
981
982fn tungstenite_request(
983 url: &str,
984 headers: Vec<(String, String)>,
985) -> Result<Request, TransportError> {
986 let mut request = url.into_client_request().map_err(TransportError::from)?;
987
988 for (key, value) in headers {
989 let value = HeaderValue::from_str(&value)
990 .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
991 let name: HeaderName = key
992 .parse()
993 .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
994 request.headers_mut().insert(name, value);
995 }
996
997 Ok(request)
998}
999
1000fn is_connection_drop_transport_error(err: &TransportError) -> bool {
1001 err.is_closed() || matches!(err, TransportError::Io(e) if is_connection_drop_io_error(e))
1002}
1003
1004fn is_retryable_initial_connect_error(err: &TransportError) -> bool {
1005 match err {
1006 TransportError::ConnectionClosed
1007 | TransportError::ConnectionReset
1008 | TransportError::ClosedByPeer(_) => true,
1009 TransportError::Io(error) => !matches!(
1010 error.kind(),
1011 std::io::ErrorKind::InvalidInput
1012 | std::io::ErrorKind::InvalidData
1013 | std::io::ErrorKind::Unsupported
1014 | std::io::ErrorKind::PermissionDenied
1015 ),
1016 TransportError::UpgradeRejected(status) | TransportError::ProxyConnectRejected(status) => {
1017 retryable_status(*status)
1018 }
1019 TransportError::InvalidUrl(_)
1020 | TransportError::Handshake(_)
1021 | TransportError::Tls(_)
1022 | TransportError::Protocol(_)
1023 | TransportError::MessageTooLarge
1024 | TransportError::FrameTooLarge
1025 | TransportError::InvalidUtf8
1026 | TransportError::Other(_) => false,
1027 }
1028}
1029
1030fn initial_connect_retry_config(
1031 policy: Option<&InitialConnectRetryPolicy>,
1032) -> Result<RetryConfig, TransportError> {
1033 if let Some(policy) = policy {
1034 ExponentialBackoff::new(
1035 policy.delay_initial,
1036 policy.delay_max,
1037 policy.backoff_factor,
1038 policy.jitter_ms,
1039 false,
1040 )
1041 .map_err(|e| {
1042 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
1043 })?;
1044
1045 Ok(RetryConfig {
1046 max_retries: policy.max_attempts.get() - 1,
1047 initial_delay_ms: duration_to_millis("delay_initial", policy.delay_initial)?,
1048 max_delay_ms: duration_to_millis("delay_max", policy.delay_max)?,
1049 backoff_factor: policy.backoff_factor,
1050 jitter_ms: policy.jitter_ms,
1051 operation_timeout_ms: None,
1052 immediate_first: false,
1053 max_elapsed_ms: None,
1054 })
1055 } else {
1056 Ok(RetryConfig {
1057 max_retries: 0,
1058 initial_delay_ms: 1,
1059 max_delay_ms: 1,
1060 backoff_factor: 1.0,
1061 jitter_ms: 0,
1062 operation_timeout_ms: None,
1063 immediate_first: false,
1064 max_elapsed_ms: None,
1065 })
1066 }
1067}
1068
1069fn duration_to_millis(field: &str, duration: Duration) -> Result<u64, TransportError> {
1070 let nanoseconds = duration.as_nanos();
1071 if nanoseconds > u128::from(u64::MAX) {
1072 return Err(TransportError::Io(std::io::Error::new(
1073 std::io::ErrorKind::InvalidInput,
1074 format!("{field} exceeds the maximum backoff duration"),
1075 )));
1076 }
1077
1078 let milliseconds = nanoseconds
1079 .div_ceil(1_000_000)
1080 .min(u128::from(u64::MAX / 1_000_000));
1081 u64::try_from(milliseconds).map_err(|_| {
1082 TransportError::Io(std::io::Error::new(
1083 std::io::ErrorKind::InvalidInput,
1084 format!("{field} exceeds the maximum backoff duration"),
1085 ))
1086 })
1087}
1088
1089async fn await_initial_connect_attempt<F, T>(
1090 cancellation_token: &CancellationToken,
1091 attempt: F,
1092) -> Result<T, TransportError>
1093where
1094 F: Future<Output = Result<T, TransportError>>,
1095{
1096 tokio::select! {
1097 biased;
1098 () = cancellation_token.cancelled() => Err(initial_connect_cancelled()),
1099 result = attempt => result,
1100 }
1101}
1102
1103struct InitialConnectError(TransportError);
1104
1105impl std::error::Error for InitialConnectError {}
1106
1107impl Display for InitialConnectError {
1108 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1109 f.write_str(&self.0.redacted_message())
1110 }
1111}
1112
1113impl Debug for InitialConnectError {
1114 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1115 Display::fmt(self, f)
1116 }
1117}
1118
1119fn initial_connect_retry_error(error: RetryError) -> TransportError {
1120 let kind = match error {
1121 RetryError::Canceled => return initial_connect_cancelled(),
1122 RetryError::InvalidConfiguration { .. } => std::io::ErrorKind::InvalidInput,
1123 RetryError::OperationTimeout { .. } | RetryError::ElapsedBudgetExceeded { .. } => {
1124 std::io::ErrorKind::TimedOut
1125 }
1126 };
1127
1128 TransportError::Io(std::io::Error::new(kind, error))
1129}
1130
1131fn initial_connect_cancelled() -> TransportError {
1132 TransportError::Io(std::io::Error::new(
1133 std::io::ErrorKind::Interrupted,
1134 "initial WebSocket connection cancelled",
1135 ))
1136}
1137
1138fn read_termination_log_level(connection_state: &AtomicU8) -> log::Level {
1140 let mode = ConnectionMode::from_atomic(connection_state);
1141 if mode.is_disconnect() || mode.is_closed() {
1142 log::Level::Debug
1143 } else {
1144 log::Level::Warn
1145 }
1146}
1147
1148#[cfg(test)]
1149mod connection_error_tests {
1150 use std::io;
1151
1152 use rstest::rstest;
1153
1154 use super::*;
1155 use crate::transport::CloseFrame;
1156
1157 #[rstest]
1158 #[case(TransportError::ConnectionClosed, true)]
1159 #[case(TransportError::ConnectionReset, true)]
1160 #[case(TransportError::ClosedByPeer(Some(CloseFrame::new(1000, "bye"))), true)]
1161 #[case(TransportError::ClosedByPeer(None), true)]
1162 #[case(TransportError::Io(io::Error::from(io::ErrorKind::BrokenPipe)), true)]
1163 #[case(
1164 TransportError::Io(io::Error::from(io::ErrorKind::ConnectionReset)),
1165 true
1166 )]
1167 #[case(TransportError::Io(io::Error::from(io::ErrorKind::TimedOut)), true)]
1168 #[case(
1169 TransportError::Io(io::Error::from(io::ErrorKind::UnexpectedEof)),
1170 true
1171 )]
1172 #[case(
1173 TransportError::Io(io::Error::from(io::ErrorKind::InvalidInput)),
1174 false
1175 )]
1176 #[case(TransportError::InvalidUrl("http://example.com".into()), false)]
1177 #[case(TransportError::Handshake("bad".into()), false)]
1178 #[case(TransportError::Protocol("bad opcode".into()), false)]
1179 #[case(TransportError::Tls("bad certificate".into()), false)]
1180 #[case(TransportError::MessageTooLarge, false)]
1181 #[case(TransportError::FrameTooLarge, false)]
1182 #[case(TransportError::InvalidUtf8, false)]
1183 #[case(TransportError::Other("backend protocol mismatch".into()), false)]
1184 fn connection_drop_transport_error_classification(
1185 #[case] err: TransportError,
1186 #[case] expected: bool,
1187 ) {
1188 assert_eq!(is_connection_drop_transport_error(&err), expected);
1189 }
1190
1191 #[rstest]
1192 #[case(Duration::ZERO, 0)]
1193 #[case(Duration::from_nanos(1), 1)]
1194 #[case(Duration::from_micros(500), 1)]
1195 #[case(Duration::from_nanos(1_000_001), 2)]
1196 #[case(Duration::from_millis(500), 500)]
1197 #[case(Duration::from_secs(5), 5_000)]
1198 fn duration_to_millis_rounds_up(#[case] duration: Duration, #[case] expected: u64) {
1199 assert_eq!(duration_to_millis("delay", duration).unwrap(), expected);
1200 }
1201
1202 #[rstest]
1203 fn duration_to_millis_caps_at_backoff_maximum() {
1204 assert_eq!(
1205 duration_to_millis("delay", Duration::from_nanos(u64::MAX)).unwrap(),
1206 u64::MAX / 1_000_000
1207 );
1208 }
1209
1210 #[rstest]
1211 fn duration_to_millis_rejects_excessive_duration() {
1212 let error = duration_to_millis("delay", Duration::from_secs(u64::MAX)).unwrap_err();
1213
1214 let TransportError::Io(error) = error else {
1215 panic!("expected an I/O error, was: {error:?}");
1216 };
1217
1218 assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1219 assert_eq!(
1220 error.to_string(),
1221 "delay exceeds the maximum backoff duration"
1222 );
1223 }
1224
1225 #[rstest]
1226 fn initial_connect_retry_config_preserves_duration_validation() {
1227 let policy = InitialConnectRetryPolicy {
1228 max_attempts: std::num::NonZeroU32::new(2).unwrap(),
1229 delay_initial: Duration::from_micros(900),
1230 delay_max: Duration::from_micros(500),
1231 backoff_factor: 2.0,
1232 jitter_ms: 0,
1233 };
1234
1235 let error = initial_connect_retry_config(Some(&policy)).unwrap_err();
1236
1237 let TransportError::Io(error) = error else {
1238 panic!("expected an I/O error, was: {error:?}");
1239 };
1240
1241 assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1242 assert!(
1243 error
1244 .to_string()
1245 .contains("delay_max must be >= delay_initial")
1246 );
1247 }
1248
1249 #[rstest]
1250 #[tokio::test]
1251 async fn initial_connect_attempt_prefers_cancellation_over_ready_result() {
1252 let token = CancellationToken::new();
1253 token.cancel();
1254
1255 let error = await_initial_connect_attempt(&token, std::future::ready(Ok(())))
1256 .await
1257 .expect_err("cancellation should take priority");
1258
1259 assert!(
1260 matches!(error, TransportError::Io(ref error) if error.kind() == io::ErrorKind::Interrupted)
1261 );
1262 }
1263
1264 #[rstest]
1265 fn unset_inbound_limits_preserve_backend_defaults() {
1266 let limits = InboundConfig::default();
1267 assert!(tungstenite_protocol_config(&limits).is_none());
1268
1269 #[cfg(feature = "transport-sockudo")]
1270 {
1271 let config = sockudo_protocol_config(&limits);
1272 let default = SockudoConfig::default();
1273 assert_eq!(config.max_message_size, default.max_message_size);
1274 assert_eq!(config.max_frame_size, default.max_frame_size);
1275 assert!(!config.auto_ping);
1276 assert_eq!(config.idle_timeout, 0);
1277 }
1278 }
1279
1280 #[rstest]
1281 fn configured_inbound_limit_overrides_only_that_cap() {
1282 let limits = InboundConfig {
1283 max_message_size_bytes: Some(32),
1284 max_frame_size_bytes: None,
1285 frame_activity: None,
1286 };
1287
1288 let config = tungstenite_protocol_config(&limits).expect("override should build a config");
1289 let default = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
1290
1291 assert_eq!(config.max_message_size, Some(32));
1292 assert_eq!(config.max_frame_size, default.max_frame_size);
1293
1294 #[cfg(feature = "transport-sockudo")]
1295 {
1296 let config = sockudo_protocol_config(&limits);
1297 let default = SockudoConfig::default();
1298 assert_eq!(config.max_message_size, 32);
1299 assert_eq!(config.max_frame_size, default.max_frame_size);
1300 assert!(!config.auto_ping);
1301 assert_eq!(config.idle_timeout, 0);
1302 }
1303 }
1304}
1305
1306#[cfg(not(feature = "turmoil"))]
1311async fn proxied_ws_handshake<S>(
1312 request: tokio_tungstenite::tungstenite::handshake::client::Request,
1313 stream: S,
1314 limits: InboundConfig,
1315) -> Result<BoxedWsTransport, TransportError>
1316where
1317 S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
1318{
1319 let stream = FrameStream::new(stream, limits.frame_activity.clone());
1320 let (ws, _resp) = tokio_tungstenite::client_async_with_config(
1321 request,
1322 stream,
1323 tungstenite_protocol_config(&limits),
1324 )
1325 .await
1326 .map_err(TransportError::from)?;
1327 Ok(Box::pin(TungsteniteTransport::new(ws)))
1328}
1329
1330#[cfg(feature = "transport-sockudo")]
1337#[derive(Debug, PartialEq, Eq)]
1338struct SockudoTarget {
1339 host: String,
1340 host_header: String,
1341 port: u16,
1342 path: String,
1343 is_tls: bool,
1344}
1345
1346#[cfg(feature = "transport-sockudo")]
1347impl SockudoTarget {
1348 fn parse(url: &str) -> Result<Self, TransportError> {
1349 let parsed = url::Url::parse(url)
1350 .map_err(|e| TransportError::InvalidUrl(format!("invalid WebSocket URL: {e}")))?;
1351
1352 let scheme = parsed.scheme();
1353
1354 let is_tls = match scheme {
1355 "ws" => false,
1356 "wss" => true,
1357 other => {
1358 return Err(TransportError::InvalidUrl(format!(
1359 "expected ws:// or wss:// scheme, was {other}"
1360 )));
1361 }
1362 };
1363
1364 let raw_host = parsed
1365 .host_str()
1366 .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
1367
1368 let is_bracketed = raw_host.starts_with('[') && raw_host.ends_with(']');
1373 let host = if is_bracketed {
1374 raw_host[1..raw_host.len() - 1].to_string()
1375 } else {
1376 raw_host.to_string()
1377 };
1378
1379 let explicit_port = parsed.port();
1380 let port = explicit_port.unwrap_or(if is_tls { 443 } else { 80 });
1381
1382 let host_header = match explicit_port {
1383 Some(p) => format!("{raw_host}:{p}"),
1384 None => raw_host.to_string(),
1385 };
1386
1387 let path = if parsed.path().is_empty() {
1388 "/".to_string()
1389 } else {
1390 let mut p = parsed.path().to_string();
1391 if let Some(query) = parsed.query() {
1392 p.push('?');
1393 p.push_str(query);
1394 }
1395
1396 p
1397 };
1398
1399 Ok(Self {
1400 host,
1401 host_header,
1402 port,
1403 path,
1404 is_tls,
1405 })
1406 }
1407}
1408
1409impl WebSocketClientInner {
1410 pub async fn reconnect(&mut self) -> Result<(), TransportError> {
1431 Box::pin(self.reconnect_with_outcome()).await.map(|_| ())
1432 }
1433
1434 async fn wait_for_reconnect_publication(&self) -> bool {
1435 let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1436
1437 loop {
1438 let mut notified = pin!(self.controller_notify.notified());
1439 notified.as_mut().enable();
1440
1441 if !ConnectionMode::from_atomic(&self.connection_mode).is_reconnect() {
1442 return false;
1443 }
1444
1445 if self.reconnect_published.load(Ordering::SeqCst) {
1446 return true;
1447 }
1448
1449 tokio::select! {
1450 biased;
1451 () = notified => {}
1452 () = dst::time::sleep(fallback_interval) => {}
1453 }
1454 }
1455 }
1456
1457 async fn reconnect_with_outcome(&mut self) -> Result<ReconnectOutcome, TransportError> {
1458 log::info!("Reconnecting");
1459
1460 if self.handler.is_none() {
1461 log::warn!(
1462 "Auto-reconnect disabled for stream-based WebSocket client; \
1463 stream users must manually reconnect by creating a new connection"
1464 );
1465 self.connection_mode
1467 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1468 fail_registered_auth(
1469 self.auth_tracker.as_ref(),
1470 "WebSocket stream mode cannot reconnect",
1471 );
1472 self.state_notify.notify_waiters();
1476 return Ok(ReconnectOutcome::Aborted);
1477 }
1478
1479 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1480 log::debug!("Reconnect aborted due to disconnect state");
1481 return Ok(ReconnectOutcome::Aborted);
1482 }
1483
1484 if let Some(rate_limit) = &self.connection_rate_limit {
1485 rate_limit
1486 .limiter
1487 .await_keys_ready(Some(&rate_limit.keys))
1488 .await;
1489 }
1490
1491 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1492 log::debug!("Reconnect aborted during connection rate-limit wait");
1493 return Ok(ReconnectOutcome::Aborted);
1494 }
1495
1496 let frame_activity = self
1497 .heartbeat_timeout
1498 .map(|_| Arc::new(AtomicBool::new(false)));
1499
1500 let (new_writer, reader) = dst::time::timeout(
1502 self.connect_timeout,
1503 Box::pin(Self::connect_with_server_activity(
1504 &self.config.url,
1505 self.reconnect_headers.snapshot(),
1506 self.config.backend,
1507 self.config.proxy_url.as_deref(),
1508 self.config.max_message_size_bytes,
1509 self.config.max_frame_size_bytes,
1510 frame_activity.clone(),
1511 )),
1512 )
1513 .await
1514 .map_err(|_| {
1515 TransportError::Io(std::io::Error::new(
1516 std::io::ErrorKind::TimedOut,
1517 format!(
1518 "reconnection timed out after {}s",
1519 self.connect_timeout.as_secs_f64()
1520 ),
1521 ))
1522 })??;
1523
1524 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1525 log::debug!("Reconnect aborted mid-flight (after connect)");
1526 return Ok(ReconnectOutcome::Aborted);
1527 }
1528
1529 let (tx, rx) = tokio::sync::oneshot::channel();
1532
1533 if let Err(e) = self
1534 .writer_tx
1535 .send_update(WriterCommand::Update(new_writer, tx))
1536 {
1537 log::error!("{e}");
1538 return Err(TransportError::Io(std::io::Error::new(
1539 std::io::ErrorKind::BrokenPipe,
1540 format!("Failed to send update command: {e}"),
1541 )));
1542 }
1543
1544 let connection_epoch = match rx.await {
1546 Ok(connection_epoch) => {
1547 log::debug!("Writer confirmed socket update: epoch={connection_epoch}");
1548 connection_epoch
1549 }
1550 Err(e) => {
1551 log::error!("Writer dropped update channel: {e}");
1552 return Err(TransportError::Io(std::io::Error::new(
1553 std::io::ErrorKind::BrokenPipe,
1554 "Writer task dropped response channel",
1555 )));
1556 }
1557 };
1558
1559 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1561
1562 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1563 log::debug!("Reconnect aborted mid-flight (after delay)");
1564 return Ok(ReconnectOutcome::Aborted);
1565 }
1566
1567 if let Some(read_fence) = self.read_fence.take() {
1568 read_fence.invalidate();
1569 }
1570
1571 if let Some(ref read_task) = self.read_task.take()
1572 && !read_task.is_finished()
1573 {
1574 read_task.abort();
1575 log_task_aborted("read");
1576 }
1577
1578 if !self.wait_for_reconnect_publication().await {
1579 log::debug!("Reconnect aborted before state publication completed");
1580 return Ok(ReconnectOutcome::Aborted);
1581 }
1582
1583 if ConnectionMode::complete_reconnect_with_sink(
1586 &self.connection_mode,
1587 self.state_sink.as_ref(),
1588 ) == ReconnectOutcome::Aborted
1589 {
1590 log::debug!("Reconnect aborted (state changed during reconnect)");
1591 return Ok(ReconnectOutcome::Aborted);
1592 }
1593
1594 let read_fence = ReadSessionFence::new();
1595 self.read_task = Some(Self::spawn_message_handler_task(
1596 Arc::clone(&self.connection_mode),
1597 Arc::clone(&self.state_notify),
1598 read_fence.clone(),
1599 reader,
1600 connection_epoch,
1601 self.handler.as_ref(),
1602 self.ping_handler.as_ref(),
1603 self.close_handler.as_ref(),
1604 self.state_sink.as_ref(),
1605 self.config.idle_timeout_ms,
1606 self.heartbeat_timeout,
1607 frame_activity,
1608 ));
1609 self.read_fence = Some(read_fence);
1610
1611 log::info!("Reconnect succeeded");
1612 Ok(ReconnectOutcome::Reconnected)
1613 }
1614
1615 #[inline]
1621 #[must_use]
1622 pub fn is_alive(&self) -> bool {
1623 match &self.read_task {
1624 Some(read_task) => !read_task.is_finished() && !self.write_task.is_finished(),
1625 None => !self.write_task.is_finished(),
1626 }
1627 }
1628
1629 #[expect(
1630 clippy::too_many_arguments,
1631 reason = "both handler modes share the same reader lifecycle"
1632 )]
1633 fn spawn_message_handler_task(
1634 connection_state: Arc<AtomicU8>,
1635 state_notify: Arc<tokio::sync::Notify>,
1636 read_fence: ReadSessionFence,
1637 mut reader: MessageReader,
1638 connection_epoch: u64,
1639 handler: Option<&IncomingHandler>,
1640 ping_handler: Option<&IncomingPingHandler>,
1641 close_handler: Option<&CloseHandler>,
1642 state_sink: Option<&SocketStateSink>,
1643 idle_timeout_ms: Option<u64>,
1644 heartbeat_timeout: Option<Duration>,
1645 frame_activity: Option<Arc<AtomicBool>>,
1646 ) -> dst::task::JoinHandle<()> {
1647 log::debug!("Started message handler task 'read'");
1648
1649 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1650 let idle_timeout = idle_timeout_ms.map(Duration::from_millis);
1651
1652 let handler = handler.cloned();
1653 let ping_handler = ping_handler.cloned();
1654 let close_handler = close_handler.cloned();
1655 let state_sink = state_sink.cloned();
1656
1657 dst::task::spawn(async move {
1658 let mut last_data_time = dst::time::Instant::now();
1659 let mut last_frame_time = dst::time::Instant::now();
1660
1661 loop {
1662 if !ConnectionMode::from_atomic(&connection_state).is_active()
1663 || !read_fence.is_valid()
1664 {
1665 break;
1666 }
1667
1668 let read_result = dst::time::timeout(check_interval, reader.next()).await;
1669
1670 if let Ok(Some(Ok(ref message))) = read_result
1671 && (!ConnectionMode::from_atomic(&connection_state).is_active()
1672 || !read_fence.is_valid())
1673 {
1674 log::debug!(
1675 "Dropping WebSocket message with {} bytes after session ended",
1676 message.as_bytes().len()
1677 );
1678 break;
1679 }
1680
1681 if matches!(&read_result, Ok(Some(Ok(_)))) {
1682 last_frame_time = dst::time::Instant::now();
1683 }
1684
1685 match read_result {
1686 Ok(Some(Ok(Message::Binary(data)))) => {
1687 log::trace!("Received message <binary> {} bytes", data.len());
1688 last_data_time = dst::time::Instant::now();
1689
1690 if !ConnectionMode::from_atomic(&connection_state).is_active()
1691 || !read_fence.is_valid()
1692 {
1693 log::debug!(
1694 "Dropping WebSocket message with {} bytes after session ended",
1695 data.len()
1696 );
1697 break;
1698 }
1699
1700 if let Some(ref handler) = handler {
1701 handler.handle(connection_epoch, Message::Binary(data));
1702 }
1703 }
1704 Ok(Some(Ok(Message::Text(data)))) => {
1705 log::trace!("Received text frame ({} bytes)", data.len());
1706 last_data_time = dst::time::Instant::now();
1707
1708 if !ConnectionMode::from_atomic(&connection_state).is_active()
1709 || !read_fence.is_valid()
1710 {
1711 log::debug!(
1712 "Dropping WebSocket message with {} bytes after session ended",
1713 data.len()
1714 );
1715 break;
1716 }
1717
1718 if let Some(ref handler) = handler {
1719 handler.handle(connection_epoch, Message::Text(data));
1720 }
1721 }
1722 Ok(Some(Ok(Message::Ping(ping_data)))) => {
1723 log::trace!("Received ping frame ({} bytes)", ping_data.len());
1724 if let Some(ref handler) = ping_handler {
1729 if !ConnectionMode::from_atomic(&connection_state).is_active()
1730 || !read_fence.is_valid()
1731 {
1732 log::debug!(
1733 "Dropping WebSocket ping with {} bytes after session ended",
1734 ping_data.len()
1735 );
1736 break;
1737 }
1738
1739 handler.handle(connection_epoch, ping_data.to_vec());
1740 }
1741
1742 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1743 break;
1744 }
1745 }
1746 Ok(Some(Ok(Message::Pong(_)))) => {
1747 log::trace!("Received pong");
1748 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1751 break;
1752 }
1753 }
1754 Ok(Some(Ok(Message::Close(frame)))) => {
1755 Self::dispatch_close_frame(
1756 &connection_state,
1757 &read_fence,
1758 close_handler.as_ref(),
1759 state_sink.as_ref(),
1760 frame,
1761 );
1762 break;
1763 }
1764 Ok(Some(Err(e))) => {
1765 if is_connection_drop_transport_error(&e) {
1766 log::warn!("Received connection error, terminating: {e}");
1767 } else {
1768 log::error!("Received transport error, terminating: {e}");
1769 }
1770
1771 break;
1772 }
1773 Ok(None) => {
1774 log::log!(
1775 read_termination_log_level(&connection_state),
1776 "Connection closed by peer (no close frame), terminating"
1777 );
1778 break;
1779 }
1780 Err(_) => {
1781 if heartbeat_timeout_exceeded(
1782 &mut last_frame_time,
1783 heartbeat_timeout,
1784 frame_activity.as_deref(),
1785 ) {
1786 break;
1787 }
1788
1789 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1790 break;
1791 }
1792 }
1793 }
1794 }
1795
1796 state_notify.notify_one();
1798 })
1799 }
1800
1801 fn dispatch_close_frame(
1802 connection_state: &AtomicU8,
1803 read_fence: &ReadSessionFence,
1804 close_handler: Option<&CloseHandler>,
1805 state_sink: Option<&SocketStateSink>,
1806 frame: Option<CloseFrame>,
1807 ) {
1808 match &frame {
1809 Some(frame) => log::log!(
1810 read_termination_log_level(connection_state),
1811 "Received close frame, terminating: code={}, reason={REDACTED}",
1812 frame.code
1813 ),
1814 None => log::log!(
1815 read_termination_log_level(connection_state),
1816 "Received close frame with no code or reason, terminating"
1817 ),
1818 }
1819
1820 if !ConnectionMode::from_atomic(connection_state).is_active() || !read_fence.is_valid() {
1821 return;
1822 }
1823
1824 if let Some(handler) = close_handler
1825 && !handler(frame)
1826 {
1827 ConnectionMode::close_websocket_on_loss(connection_state, state_sink);
1828 }
1829 }
1830
1831 fn buffer_for_replay(
1839 buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1840 msg: Message,
1841 permit: tokio::sync::OwnedSemaphorePermit,
1842 ) {
1843 if msg.is_control() {
1844 return;
1845 }
1846
1847 log::debug!(
1848 "Buffering message for replay (buffer size: {})",
1849 buffer.len() + 1
1850 );
1851
1852 buffer.push_back((msg, permit));
1853 }
1854
1855 async fn drain_reconnect_buffer(
1860 buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1861 writer: &mut MessageWriter,
1862 connection_state: &AtomicU8,
1863 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1864 reconnect_buffer_waits_for_auth: &AtomicBool,
1865 ) -> bool {
1866 if buffer.is_empty() {
1867 return false;
1868 }
1869
1870 let initial_buffer_len = buffer.len();
1871 log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
1872
1873 while !buffer.is_empty() {
1874 match Self::reconnect_buffer_action(
1875 reconnect_buffer_waits_for_auth,
1876 auth_tracker,
1877 connection_state,
1878 ) {
1879 ReconnectBufferAction::Drain => {}
1880 ReconnectBufferAction::Wait => return false,
1881 ReconnectBufferAction::Discard => {
1882 log::warn!(
1883 "Discarding {} buffered messages after authentication failed",
1884 buffer.len()
1885 );
1886 buffer.clear();
1887 return false;
1888 }
1889 }
1890
1891 let msg_to_send = buffer
1893 .front()
1894 .expect("reconnect buffer should not be empty")
1895 .0
1896 .clone();
1897
1898 if let Err(e) = writer.send(msg_to_send).await {
1899 if is_connection_drop_transport_error(&e) {
1900 log::warn!(
1901 "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1902 buffer.len()
1903 );
1904 } else {
1905 log::error!(
1906 "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1907 buffer.len()
1908 );
1909 }
1910
1911 return true;
1912 }
1913
1914 buffer.pop_front();
1916 }
1917
1918 if buffer.is_empty() {
1919 log::info!("Successfully sent all {initial_buffer_len} buffered messages");
1920 }
1921
1922 false
1923 }
1924
1925 fn can_drain_reconnect_buffer(
1926 reconnect_buffer_waits_for_auth: &AtomicBool,
1927 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1928 ) -> ReconnectBufferAction {
1929 if !reconnect_buffer_waits_for_auth.load(Ordering::Acquire) {
1930 return ReconnectBufferAction::Drain;
1931 }
1932
1933 match auth_tracker.get().map(AuthTracker::auth_state) {
1934 Some(AuthState::Authenticated) => ReconnectBufferAction::Drain,
1935 Some(AuthState::Failed) => ReconnectBufferAction::Discard,
1936 Some(AuthState::Unauthenticated) | None => ReconnectBufferAction::Wait,
1937 }
1938 }
1939
1940 fn reconnect_buffer_action(
1941 reconnect_buffer_waits_for_auth: &AtomicBool,
1942 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1943 connection_state: &AtomicU8,
1944 ) -> ReconnectBufferAction {
1945 let action =
1946 Self::can_drain_reconnect_buffer(reconnect_buffer_waits_for_auth, auth_tracker);
1947
1948 if action == ReconnectBufferAction::Drain
1950 && !ConnectionMode::from_atomic(connection_state).is_active()
1951 {
1952 ReconnectBufferAction::Wait
1953 } else {
1954 action
1955 }
1956 }
1957
1958 #[expect(
1959 clippy::too_many_arguments,
1960 reason = "writer task owns the transport and shared lifecycle coordination state"
1961 )]
1962 fn spawn_write_task(
1963 connection_state: Arc<AtomicU8>,
1964 controller_notify: Arc<tokio::sync::Notify>,
1965 reconnect_published: Arc<AtomicBool>,
1966 writer: MessageWriter,
1967 mut writer_rx: WriterReceiver<WriterCommand>,
1968 connection_epoch: Arc<AtomicU64>,
1969 auth_tracker: Arc<OnceLock<AuthTracker>>,
1970 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1971 state_sink: Option<SocketStateSink>,
1972 ) -> dst::task::JoinHandle<()> {
1973 log_task_started("write");
1974
1975 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1977
1978 dst::task::spawn(async move {
1979 let mut active_writer = writer;
1980 let mut reconnect_buffer: VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)> =
1983 VecDeque::new();
1984
1985 loop {
1986 let mode = ConnectionMode::from_atomic(&connection_state);
1987
1988 match mode {
1989 ConnectionMode::Disconnect => {
1990 if !reconnect_buffer.is_empty() {
1992 log::warn!(
1993 "Discarding {} buffered messages due to disconnect",
1994 reconnect_buffer.len()
1995 );
1996 reconnect_buffer.clear();
1997 }
1998
1999 _ = dst::time::timeout(
2002 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2003 active_writer.close(),
2004 )
2005 .await;
2006 break;
2007 }
2008 ConnectionMode::Closed => {
2009 if !reconnect_buffer.is_empty() {
2011 log::warn!(
2012 "Discarding {} buffered messages due to closed connection",
2013 reconnect_buffer.len()
2014 );
2015 reconnect_buffer.clear();
2016 }
2017
2018 break;
2019 }
2020 _ => {}
2021 }
2022
2023 if mode.is_active() && !reconnect_buffer.is_empty() {
2024 match Self::reconnect_buffer_action(
2025 reconnect_buffer_waits_for_auth.as_ref(),
2026 &auth_tracker,
2027 &connection_state,
2028 ) {
2029 ReconnectBufferAction::Drain => {
2030 let drain_result = dst::time::timeout(
2031 Duration::from_secs(WRITE_TIMEOUT_SECS),
2032 Self::drain_reconnect_buffer(
2033 &mut reconnect_buffer,
2034 &mut active_writer,
2035 &connection_state,
2036 &auth_tracker,
2037 reconnect_buffer_waits_for_auth.as_ref(),
2038 ),
2039 )
2040 .await;
2041
2042 let send_error = drain_result.unwrap_or_else(|_| {
2043 log::warn!(
2044 "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
2045 reconnect_buffer.len()
2046 );
2047 true
2048 });
2049
2050 if send_error {
2052 _ = request_websocket_reconnect(
2053 &connection_state,
2054 &reconnect_published,
2055 state_sink.as_ref(),
2056 &auth_tracker,
2057 &controller_notify,
2058 || {},
2059 );
2060 }
2061
2062 continue;
2063 }
2064 ReconnectBufferAction::Discard => {
2065 log::warn!(
2066 "Discarding {} buffered messages after authentication failed",
2067 reconnect_buffer.len()
2068 );
2069 reconnect_buffer.clear();
2070 continue;
2071 }
2072 ReconnectBufferAction::Wait => {}
2073 }
2074 }
2075
2076 match dst::time::timeout(check_interval, writer_rx.recv()).await {
2077 Ok(Some((msg, permit))) => {
2078 let mode = ConnectionMode::from_atomic(&connection_state);
2080
2081 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
2082 break;
2083 }
2084
2085 match msg {
2086 WriterCommand::Update(new_writer, tx) => {
2087 log::debug!("Received new writer");
2088
2089 dst::time::sleep(Duration::from_millis(100)).await;
2091
2092 _ = dst::time::timeout(
2095 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2096 active_writer.close(),
2097 )
2098 .await;
2099
2100 active_writer = new_writer;
2101 let epoch = connection_epoch.fetch_add(1, Ordering::AcqRel) + 1;
2102 log::debug!("Updated writer: epoch={epoch}");
2103
2104 if let Err(e) = tx.send(epoch) {
2105 log::error!(
2106 "Failed to report writer update to controller: {e:?}"
2107 );
2108 }
2109 }
2110 WriterCommand::Send(msg)
2111 if mode.is_reconnect()
2112 || (!reconnect_buffer.is_empty() && !msg.is_control()) =>
2113 {
2114 Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2115 }
2116 WriterCommand::Heartbeat(_)
2117 | WriterCommand::SendPongOnConnection { .. }
2118 if mode.is_reconnect() => {}
2119 WriterCommand::SendOnConnection { response_tx, .. }
2120 if mode.is_reconnect() =>
2121 {
2122 _ = response_tx.send(Err(SendError::ConnectionChanged));
2123 }
2124 WriterCommand::SendPongOnConnection {
2125 data,
2126 connection_epoch: expected_epoch,
2127 } => {
2128 let epoch = connection_epoch.load(Ordering::Acquire);
2129 if epoch != expected_epoch {
2130 continue;
2131 }
2132
2133 let send_result = dst::time::timeout(
2134 Duration::from_secs(WRITE_TIMEOUT_SECS),
2135 active_writer.send(Message::Pong(data.into())),
2136 )
2137 .await;
2138
2139 let send_failed = match send_result {
2140 Ok(Ok(())) => false,
2141 Ok(Err(e)) => {
2142 if is_connection_drop_transport_error(&e) {
2143 log::warn!("Failed to send pong: {e}");
2144 } else {
2145 log::error!("Failed to send pong: {e}");
2146 }
2147
2148 true
2149 }
2150 Err(_) => {
2151 log::warn!(
2152 "Timed out sending pong after {WRITE_TIMEOUT_SECS}s"
2153 );
2154 true
2155 }
2156 };
2157
2158 if send_failed
2159 && request_websocket_reconnect(
2160 &connection_state,
2161 &reconnect_published,
2162 state_sink.as_ref(),
2163 &auth_tracker,
2164 &controller_notify,
2165 || {},
2166 ) == ReconnectRequestOutcome::Accepted
2167 {
2168 log::warn!("Writer triggering reconnect");
2169 }
2170 }
2171 WriterCommand::SendOnConnection {
2172 message,
2173 connection_epoch: expected_epoch,
2174 response_tx,
2175 } => {
2176 let epoch = connection_epoch.load(Ordering::Acquire);
2177 if epoch != expected_epoch {
2178 _ = response_tx.send(Err(SendError::ConnectionChanged));
2179 continue;
2180 }
2181
2182 let send_result = dst::time::timeout(
2183 Duration::from_secs(WRITE_TIMEOUT_SECS),
2184 active_writer.send(message),
2185 )
2186 .await;
2187
2188 let result = match send_result {
2192 Ok(Ok(())) => Ok(()),
2193 Ok(Err(e)) => {
2194 if is_connection_drop_transport_error(&e) {
2195 log::warn!("Failed to send message: {e}");
2196 } else {
2197 log::error!("Failed to send message: {e}");
2198 }
2199
2200 Err(SendError::BrokenPipe(e.to_string()))
2201 }
2202 Err(_) => {
2203 log::warn!(
2204 "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
2205 );
2206
2207 Err(SendError::WriteTimeout)
2208 }
2209 };
2210
2211 let send_failed = result.is_err();
2212 _ = response_tx.send(result);
2213
2214 if send_failed
2215 && request_websocket_reconnect(
2216 &connection_state,
2217 &reconnect_published,
2218 state_sink.as_ref(),
2219 &auth_tracker,
2220 &controller_notify,
2221 || {},
2222 ) == ReconnectRequestOutcome::Accepted
2223 {
2224 log::warn!("Writer triggering reconnect");
2225 }
2226 }
2227 WriterCommand::Send(msg) => {
2228 let send_failed =
2229 Self::write_outbound(&mut active_writer, msg.clone()).await;
2230 if send_failed {
2231 Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2232
2233 if request_websocket_reconnect(
2235 &connection_state,
2236 &reconnect_published,
2237 state_sink.as_ref(),
2238 &auth_tracker,
2239 &controller_notify,
2240 || {},
2241 ) == ReconnectRequestOutcome::Accepted
2242 {
2243 log::warn!("Writer triggering reconnect");
2244 }
2245 }
2246 }
2247 WriterCommand::Heartbeat(msg) => {
2248 let send_failed =
2249 Self::write_outbound(&mut active_writer, msg).await;
2250
2251 if send_failed
2252 && request_websocket_reconnect(
2253 &connection_state,
2254 &reconnect_published,
2255 state_sink.as_ref(),
2256 &auth_tracker,
2257 &controller_notify,
2258 || {},
2259 ) == ReconnectRequestOutcome::Accepted
2260 {
2261 log::warn!("Writer triggering reconnect");
2262 }
2263 }
2264 }
2265 }
2266 Ok(None) => {
2267 log::debug!("Writer channel closed, terminating writer task");
2269 break;
2270 }
2271 Err(_) => {
2272 }
2274 }
2275 }
2276
2277 _ = dst::time::timeout(
2280 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2281 active_writer.close(),
2282 )
2283 .await;
2284
2285 log_task_stopped("write");
2286 })
2287 }
2288
2289 async fn write_outbound(writer: &mut MessageWriter, msg: Message) -> bool {
2290 let send_result =
2291 dst::time::timeout(Duration::from_secs(WRITE_TIMEOUT_SECS), writer.send(msg)).await;
2292
2293 match send_result {
2294 Ok(Ok(())) => false,
2295 Ok(Err(e)) => {
2296 if is_connection_drop_transport_error(&e) {
2297 log::warn!("Failed to send message: {e}");
2298 } else {
2299 log::error!("Failed to send message: {e}");
2300 }
2301
2302 true
2303 }
2304 Err(_) => {
2305 log::warn!("Timed out sending message after {WRITE_TIMEOUT_SECS}s");
2306 true
2307 }
2308 }
2309 }
2310
2311 fn spawn_heartbeat_task(
2312 connection_state: Arc<AtomicU8>,
2313 heartbeat_secs: u64,
2314 message: Option<String>,
2315 writer_tx: WriterSender<WriterCommand>,
2316 ) -> dst::task::JoinHandle<()> {
2317 log_task_started("heartbeat");
2318
2319 dst::task::spawn(async move {
2320 let interval = Duration::from_secs(heartbeat_secs);
2321 loop {
2322 dst::time::sleep(interval).await;
2323
2324 match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
2325 ConnectionMode::Active => {
2326 let msg = match &message {
2327 Some(text) => {
2328 WriterCommand::Heartbeat(Message::Text(text.clone().into()))
2329 }
2330 None => WriterCommand::Heartbeat(Message::Ping(vec![].into())),
2331 };
2332
2333 match writer_tx.send_control(msg) {
2334 Ok(()) => log::trace!("Sent heartbeat to writer task"),
2335 Err(e) => {
2336 log::warn!("Failed to send heartbeat to writer task: {e}");
2337 }
2338 }
2339 }
2340 ConnectionMode::Reconnect => {}
2341 ConnectionMode::Disconnect | ConnectionMode::Closed => break,
2342 }
2343 }
2344
2345 log_task_stopped("heartbeat");
2346 })
2347 }
2348}
2349
2350fn heartbeat_timeout_exceeded(
2351 last_frame_time: &mut dst::time::Instant,
2352 timeout: Option<Duration>,
2353 frame_activity: Option<&AtomicBool>,
2354) -> bool {
2355 if frame_activity.is_some_and(|activity| activity.swap(false, Ordering::Relaxed)) {
2356 *last_frame_time = dst::time::Instant::now();
2357 }
2358
2359 if let Some(timeout) = timeout {
2360 let elapsed = last_frame_time.elapsed();
2361 if elapsed >= timeout {
2362 log::warn!(
2363 "Heartbeat timeout: no frame received for {:.1}s",
2364 elapsed.as_secs_f64()
2365 );
2366 return true;
2367 }
2368 }
2369
2370 false
2371}
2372
2373fn idle_timeout_exceeded(
2374 last_data_time: dst::time::Instant,
2375 idle_timeout: Option<Duration>,
2376) -> bool {
2377 if let Some(timeout) = idle_timeout {
2378 let idle_duration = last_data_time.elapsed();
2379 if idle_duration >= timeout {
2380 log::warn!(
2381 "Read idle timeout: no data received for {:.1}s",
2382 idle_duration.as_secs_f64()
2383 );
2384 return true;
2385 }
2386 }
2387
2388 false
2389}
2390
2391impl Drop for WebSocketClientInner {
2392 fn drop(&mut self) {
2393 if let Some(read_fence) = self.read_fence.take() {
2394 read_fence.invalidate();
2395 }
2396
2397 if let Some(ref read_task) = self.read_task.take()
2398 && !read_task.is_finished()
2399 {
2400 read_task.abort();
2401 log_task_aborted("read");
2402 }
2403
2404 if !self.write_task.is_finished() {
2405 self.write_task.abort();
2406 log_task_aborted("write");
2407 }
2408
2409 if let Some(ref handle) = self.heartbeat_task.take()
2410 && !handle.is_finished()
2411 {
2412 handle.abort();
2413 log_task_aborted("heartbeat");
2414 }
2415 }
2416}
2417
2418#[expect(
2419 clippy::missing_fields_in_debug,
2420 reason = "handler closures and internal task handles are intentionally omitted"
2421)]
2422impl Debug for WebSocketClientInner {
2423 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2424 f.debug_struct(stringify!(WebSocketClientInner))
2425 .field("config", &self.config)
2426 .field(
2427 "connection_mode",
2428 &ConnectionMode::from_atomic(&self.connection_mode),
2429 )
2430 .field("connect_timeout", &self.connect_timeout)
2431 .field("is_stream_mode", &self.handler.is_none())
2432 .finish()
2433 }
2434}
2435
2436#[derive(Clone)]
2437enum IncomingHandler {
2438 Message(MessageHandler),
2439 Epoch(EpochMessageHandler),
2440}
2441
2442impl IncomingHandler {
2443 fn handle(&self, connection_epoch: u64, message: Message) {
2444 match self {
2445 Self::Message(handler) => handler(message),
2446 Self::Epoch(handler) => handler(connection_epoch, message),
2447 }
2448 }
2449}
2450
2451#[derive(Clone)]
2452enum IncomingPingHandler {
2453 Ping(PingHandler),
2454 Epoch(EpochPingHandler),
2455}
2456
2457impl IncomingPingHandler {
2458 fn handle(&self, connection_epoch: u64, data: Vec<u8>) {
2459 match self {
2460 Self::Ping(handler) => handler(data),
2461 Self::Epoch(handler) => handler(connection_epoch, data),
2462 }
2463 }
2464}
2465
2466#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2467enum ReconnectBufferAction {
2468 Drain,
2469 Wait,
2470 Discard,
2471}
2472
2473pub struct WebSocketClient {
2480 pub(crate) controller_task: dst::task::JoinHandle<()>,
2481 pub(crate) connection_mode: Arc<AtomicU8>,
2482 pub(crate) connection_epoch: Arc<AtomicU64>,
2483 pub(crate) state_notify: Arc<tokio::sync::Notify>,
2484 pub(crate) connect_timeout: Duration,
2485 pub(crate) rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
2486 pub(crate) writer_tx: WriterSender<WriterCommand>,
2487 auth_tracker: Arc<OnceLock<AuthTracker>>,
2488 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
2489 reconnect_headers: ReconnectHeaders,
2490 state_sink: Option<SocketStateSink>,
2491 controller_lifecycle: Arc<ControllerLifecycle>,
2492 controller_notify: Arc<tokio::sync::Notify>,
2493 reconnect_published: Arc<AtomicBool>,
2494 reconnect_supported: bool,
2495}
2496
2497#[derive(Clone)]
2501pub struct ReconnectHeaders {
2502 inner: Arc<RwLock<Vec<(String, String)>>>,
2503}
2504
2505impl ReconnectHeaders {
2506 fn new(headers: Vec<(String, String)>) -> Self {
2507 Self {
2508 inner: Arc::new(RwLock::new(headers)),
2509 }
2510 }
2511
2512 pub fn update(&self, name: &str, value: &str) -> Result<(), TransportError> {
2518 let name = HeaderName::from_bytes(name.as_bytes()).map_err(|e| {
2519 TransportError::Io(std::io::Error::new(
2520 std::io::ErrorKind::InvalidInput,
2521 format!("Invalid WebSocket reconnect header name: {e}"),
2522 ))
2523 })?;
2524
2525 HeaderValue::from_str(value).map_err(|e| {
2526 TransportError::Io(std::io::Error::new(
2527 std::io::ErrorKind::InvalidInput,
2528 format!("Invalid WebSocket reconnect header value: {e}"),
2529 ))
2530 })?;
2531
2532 let name = name.as_str();
2533 let mut headers = self.inner.write();
2534 headers.retain(|(existing, _)| !existing.eq_ignore_ascii_case(name));
2535 headers.push((name.to_string(), value.to_string()));
2536 Ok(())
2537 }
2538
2539 fn snapshot(&self) -> Vec<(String, String)> {
2540 self.inner.read().clone()
2541 }
2542}
2543
2544impl Debug for ReconnectHeaders {
2545 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2546 f.debug_struct(stringify!(ReconnectHeaders))
2547 .finish_non_exhaustive()
2548 }
2549}
2550
2551#[derive(Clone)]
2553pub struct WebSocketReconnectHandle {
2554 connection_mode: Arc<AtomicU8>,
2555 auth_tracker: Arc<OnceLock<AuthTracker>>,
2556 state_sink: Option<SocketStateSink>,
2557 controller_lifecycle: Arc<ControllerLifecycle>,
2558 controller_notify: Arc<tokio::sync::Notify>,
2559 reconnect_published: Arc<AtomicBool>,
2560 supported: bool,
2561}
2562
2563impl Debug for WebSocketReconnectHandle {
2564 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2565 f.debug_struct(stringify!(WebSocketReconnectHandle))
2566 .field(
2567 "connection_mode",
2568 &ConnectionMode::from_atomic(&self.connection_mode),
2569 )
2570 .field("supported", &self.supported)
2571 .finish_non_exhaustive()
2572 }
2573}
2574
2575impl WebSocketReconnectHandle {
2576 #[must_use]
2584 pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
2585 if !self.supported {
2586 return ReconnectRequestOutcome::Unsupported;
2587 }
2588
2589 let Some(request) = self.controller_lifecycle.enter_request() else {
2590 return ReconnectRequestOutcome::Closed;
2591 };
2592
2593 let mut request = Some(request);
2594 request_websocket_reconnect(
2595 &self.connection_mode,
2596 &self.reconnect_published,
2597 self.state_sink.as_ref(),
2598 &self.auth_tracker,
2599 &self.controller_notify,
2600 || drop(request.take()),
2601 )
2602 }
2603}
2604
2605struct ReconnectPublication<'a> {
2606 published: &'a AtomicBool,
2607 controller_notify: &'a tokio::sync::Notify,
2608}
2609
2610impl Drop for ReconnectPublication<'_> {
2611 fn drop(&mut self) {
2612 self.published.store(true, Ordering::SeqCst);
2613 self.controller_notify.notify_one();
2614 }
2615}
2616
2617fn request_websocket_reconnect<F>(
2618 connection_mode: &AtomicU8,
2619 reconnect_published: &AtomicBool,
2620 state_sink: Option<&SocketStateSink>,
2621 auth_tracker: &OnceLock<AuthTracker>,
2622 controller_notify: &tokio::sync::Notify,
2623 on_handoff: F,
2624) -> ReconnectRequestOutcome
2625where
2626 F: FnOnce(),
2627{
2628 if reconnect_published
2629 .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst)
2630 .is_err()
2631 {
2632 return match ConnectionMode::from_atomic(connection_mode) {
2633 ConnectionMode::Active | ConnectionMode::Reconnect => {
2634 ReconnectRequestOutcome::AlreadyReconnecting
2635 }
2636 ConnectionMode::Disconnect => ReconnectRequestOutcome::Disconnected,
2637 ConnectionMode::Closed => ReconnectRequestOutcome::Closed,
2638 };
2639 }
2640
2641 let outcome = ConnectionMode::request_reconnect_outcome(connection_mode);
2642 if outcome != ReconnectRequestOutcome::Accepted {
2643 reconnect_published.store(true, Ordering::SeqCst);
2644 return outcome;
2645 }
2646
2647 let _publication = ReconnectPublication {
2648 published: reconnect_published,
2649 controller_notify,
2650 };
2651
2652 if let Some(tracker) = auth_tracker.get() {
2653 tracker.invalidate();
2654 }
2655
2656 controller_notify.notify_one();
2657 on_handoff();
2658
2659 if let Some(sink) = state_sink {
2660 sink.publish_websocket(SocketState::Disconnected);
2661 }
2662
2663 ReconnectRequestOutcome::Accepted
2664}
2665
2666#[cfg(test)]
2667mod reconnect_request_tests {
2668 use std::sync::{Arc, OnceLock, atomic::AtomicU8};
2669
2670 use rstest::rstest;
2671
2672 use super::*;
2673
2674 fn handle(
2675 mode: ConnectionMode,
2676 supported: bool,
2677 ) -> (
2678 WebSocketReconnectHandle,
2679 AuthTracker,
2680 Arc<tokio::sync::Notify>,
2681 ) {
2682 let tracker = AuthTracker::new();
2683 let _receiver = tracker.begin();
2684 tracker.succeed();
2685 let auth_tracker = Arc::new(OnceLock::new());
2686 auth_tracker
2687 .set(tracker.clone())
2688 .expect("auth tracker should be unset");
2689 let notify = Arc::new(tokio::sync::Notify::new());
2690
2691 let handle = WebSocketReconnectHandle {
2692 connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
2693 auth_tracker,
2694 state_sink: None,
2695 controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2696 controller_notify: Arc::clone(¬ify),
2697 reconnect_published: Arc::new(AtomicBool::new(true)),
2698 supported,
2699 };
2700
2701 (handle, tracker, notify)
2702 }
2703
2704 #[rstest]
2705 #[tokio::test]
2706 async fn accepted_request_invalidates_auth_and_wakes_controller_once() {
2707 let (handle, tracker, notify) = handle(ConnectionMode::Active, true);
2708
2709 assert_eq!(
2710 handle.request_reconnect(),
2711 ReconnectRequestOutcome::Accepted
2712 );
2713 assert!(!tracker.is_authenticated());
2714 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2715 .await
2716 .expect("accepted request should notify controller");
2717
2718 let _receiver = tracker.begin();
2719 tracker.succeed();
2720 assert_eq!(
2721 handle.request_reconnect(),
2722 ReconnectRequestOutcome::AlreadyReconnecting
2723 );
2724 assert!(tracker.is_authenticated());
2725 assert!(
2726 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2727 .await
2728 .is_err(),
2729 "duplicate request should not notify controller",
2730 );
2731
2732 handle
2733 .connection_mode
2734 .store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
2735 assert_eq!(
2736 handle.request_reconnect(),
2737 ReconnectRequestOutcome::Accepted
2738 );
2739 assert!(!tracker.is_authenticated());
2740 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2741 .await
2742 .expect("a later reconnect cycle should notify the controller");
2743 }
2744
2745 #[rstest]
2746 #[case(
2747 ConnectionMode::Disconnect,
2748 true,
2749 ReconnectRequestOutcome::Disconnected
2750 )]
2751 #[case(
2752 ConnectionMode::Reconnect,
2753 true,
2754 ReconnectRequestOutcome::AlreadyReconnecting
2755 )]
2756 #[case(ConnectionMode::Closed, true, ReconnectRequestOutcome::Closed)]
2757 #[case(ConnectionMode::Active, false, ReconnectRequestOutcome::Unsupported)]
2758 #[tokio::test]
2759 async fn rejected_request_preserves_auth_and_does_not_wake_controller(
2760 #[case] mode: ConnectionMode,
2761 #[case] supported: bool,
2762 #[case] expected: ReconnectRequestOutcome,
2763 ) {
2764 let (handle, tracker, notify) = handle(mode, supported);
2765
2766 assert_eq!(handle.request_reconnect(), expected);
2767 assert!(tracker.is_authenticated());
2768 assert!(
2769 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2770 .await
2771 .is_err(),
2772 "rejected request should not notify controller",
2773 );
2774 }
2775
2776 #[rstest]
2777 #[tokio::test]
2778 async fn reconnect_loss_callback_rejects_nested_request() {
2779 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2780 let controller_notify = Arc::new(tokio::sync::Notify::new());
2781 let handle_slot = Arc::new(OnceLock::<WebSocketReconnectHandle>::new());
2782 let handle_slot_callback = Arc::clone(&handle_slot);
2783 let nested_outcomes = Arc::new(parking_lot::Mutex::new(Vec::new()));
2784 let nested_outcomes_callback = Arc::clone(&nested_outcomes);
2785 let states = Arc::new(parking_lot::Mutex::new(Vec::new()));
2786 let states_callback = Arc::clone(&states);
2787 let sink = SocketStateSink::new(move |state| {
2788 states_callback.lock().push(state);
2789 nested_outcomes_callback
2790 .lock()
2791 .push(handle_slot_callback.get().unwrap().request_reconnect());
2792 });
2793
2794 let handle = WebSocketReconnectHandle {
2795 connection_mode,
2796 auth_tracker: Arc::new(OnceLock::new()),
2797 state_sink: Some(sink),
2798 controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2799 controller_notify,
2800 reconnect_published: Arc::new(AtomicBool::new(true)),
2801 supported: true,
2802 };
2803
2804 handle_slot.set(handle.clone()).unwrap();
2805 let (result_tx, result_rx) = tokio::sync::oneshot::channel();
2806 std::thread::spawn(move || {
2807 _ = result_tx.send(handle.request_reconnect());
2808 });
2809
2810 assert_eq!(
2811 tokio::time::timeout(Duration::from_secs(1), result_rx)
2812 .await
2813 .expect("reentrant reconnect callback deadlocked")
2814 .unwrap(),
2815 ReconnectRequestOutcome::Accepted
2816 );
2817 assert_eq!(
2818 *nested_outcomes.lock(),
2819 vec![ReconnectRequestOutcome::AlreadyReconnecting]
2820 );
2821 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2822 }
2823
2824 #[rstest]
2825 fn closed_stream_handle_remains_unsupported() {
2826 let (handle, tracker, _notify) = handle(ConnectionMode::Closed, false);
2827 handle.controller_lifecycle.close_and_abort();
2828
2829 assert_eq!(
2830 handle.request_reconnect(),
2831 ReconnectRequestOutcome::Unsupported
2832 );
2833 assert!(tracker.is_authenticated());
2834 }
2835}
2836
2837impl Debug for WebSocketClient {
2838 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2839 f.debug_struct(stringify!(WebSocketClient)).finish()
2840 }
2841}
2842
2843#[bon::bon]
2844impl WebSocketClient {
2845 #[builder(
2864 builder_type = WebSocketClientStreamBuilder,
2865 finish_fn = connect
2866 )]
2867 pub async fn stream_builder(
2868 config: WebSocketConfig,
2869 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2870 default_quota: Option<Quota>,
2871 state_sink: Option<SocketStateSink>,
2872 ) -> Result<(MessageReader, Self), TransportError> {
2873 install_cryptographic_provider();
2874 validate_honored_inbound_limits(&config)?;
2875
2876 let connect_timeout = Duration::from_secs(10);
2879
2880 let (writer, reader) = dst::time::timeout(
2881 connect_timeout,
2882 Box::pin(WebSocketClientInner::connect_with_server(
2883 &config.url,
2884 config.headers.clone(),
2885 config.backend,
2886 config.proxy_url.as_deref(),
2887 config.max_message_size_bytes,
2888 config.max_frame_size_bytes,
2889 )),
2890 )
2891 .await
2892 .map_err(|_| {
2893 TransportError::Io(std::io::Error::new(
2894 std::io::ErrorKind::TimedOut,
2895 format!(
2896 "connection timed out after {}s",
2897 connect_timeout.as_secs_f64()
2898 ),
2899 ))
2900 })??;
2901
2902 let inner =
2904 WebSocketClientInner::new_with_writer_and_state_sink(config, writer, state_sink)?;
2905
2906 let connection_mode = Arc::clone(&inner.connection_mode);
2907 let connection_epoch = Arc::clone(&inner.connection_epoch);
2908 let state_notify = Arc::clone(&inner.state_notify);
2909 let controller_notify = Arc::clone(&inner.controller_notify);
2910 let reconnect_published = Arc::clone(&inner.reconnect_published);
2911 let connect_timeout = inner.connect_timeout;
2912 let auth_tracker = Arc::clone(&inner.auth_tracker);
2913 let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
2914 let reconnect_headers = inner.reconnect_headers.clone();
2915 let state_sink = inner.state_sink.clone();
2916 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, None)?;
2917 let writer_tx = inner.writer_tx.clone();
2918 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2919
2920 let controller_task = Self::spawn_controller_task(
2921 inner,
2922 Arc::clone(&connection_mode),
2923 Arc::clone(&state_notify),
2924 Arc::clone(&auth_tracker),
2925 Arc::clone(&controller_lifecycle),
2926 Arc::clone(&controller_notify),
2927 Arc::clone(&reconnect_published),
2928 );
2929 let abort_handle = controller_task.abort_handle();
2930 controller_lifecycle.set_abort(move || abort_handle.abort());
2931
2932 Ok((
2933 reader,
2934 Self {
2935 controller_task,
2936 connection_mode,
2937 connection_epoch,
2938 state_notify,
2939 connect_timeout,
2940 rate_limiter,
2941 writer_tx,
2942 auth_tracker,
2943 reconnect_buffer_waits_for_auth,
2944 reconnect_headers,
2945 state_sink,
2946 controller_lifecycle,
2947 controller_notify,
2948 reconnect_published,
2949 reconnect_supported: false,
2950 },
2951 ))
2952 }
2953
2954 #[builder(finish_fn = connect)]
2998 pub async fn builder(
2999 config: WebSocketConfig,
3000 message_handler: MessageHandler,
3001 ping_handler: Option<PingHandler>,
3002 close_handler: Option<CloseHandler>,
3003 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3004 default_quota: Option<Quota>,
3005 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3006 state_sink: Option<SocketStateSink>,
3007 connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3008 #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3009 initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3010 cancellation_token: Option<CancellationToken>,
3011 ) -> Result<Self, TransportError> {
3012 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3013 let connection_rate_limit =
3014 Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3015 Self::connect_with_handler_scoped(
3016 config,
3017 IncomingHandler::Message(message_handler),
3018 ping_handler.map(IncomingPingHandler::Ping),
3019 close_handler,
3020 rate_limiter,
3021 state_sink,
3022 connection_rate_limit,
3023 InitialConnectOptions {
3024 retry_policy: initial_connect_retry_policy,
3025 cancellation_token,
3026 },
3027 )
3028 .await
3029 }
3030
3031 #[builder(
3058 builder_type = WebSocketClientEpochBuilder,
3059 finish_fn = connect
3060 )]
3061 pub async fn epoch_builder(
3062 config: WebSocketConfig,
3063 epoch_handler: EpochMessageHandler,
3064 ping_handler: Option<PingHandler>,
3065 epoch_ping_handler: Option<EpochPingHandler>,
3066 close_handler: Option<CloseHandler>,
3067 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3068 default_quota: Option<Quota>,
3069 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3070 state_sink: Option<SocketStateSink>,
3071 connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3072 #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3073 initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3074 cancellation_token: Option<CancellationToken>,
3075 ) -> Result<Self, TransportError> {
3076 let ping_handler = match (ping_handler, epoch_ping_handler) {
3077 (Some(_), Some(_)) => {
3078 return Err(TransportError::Io(std::io::Error::new(
3079 std::io::ErrorKind::InvalidInput,
3080 "Cannot configure both ping_handler and epoch_ping_handler",
3081 )));
3082 }
3083 (Some(handler), None) => Some(IncomingPingHandler::Ping(handler)),
3084 (None, Some(handler)) => Some(IncomingPingHandler::Epoch(handler)),
3085 (None, None) => None,
3086 };
3087
3088 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3089 let connection_rate_limit =
3090 Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3091 Self::connect_with_handler_scoped(
3092 config,
3093 IncomingHandler::Epoch(epoch_handler),
3094 ping_handler,
3095 close_handler,
3096 rate_limiter,
3097 state_sink,
3098 connection_rate_limit,
3099 InitialConnectOptions {
3100 retry_policy: initial_connect_retry_policy,
3101 cancellation_token,
3102 },
3103 )
3104 .await
3105 }
3106
3107 fn resolve_connection_rate_limit(
3108 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3109 keys: Arc<[Ustr]>,
3110 ) -> Result<Option<ConnectionRateLimit>, TransportError> {
3111 if rate_limiter.is_none() && !keys.is_empty() {
3112 return Err(TransportError::Io(std::io::Error::new(
3113 std::io::ErrorKind::InvalidInput,
3114 "Connection rate keys require a connection rate limiter",
3115 )));
3116 }
3117
3118 if rate_limiter.is_some() && keys.is_empty() {
3119 return Err(TransportError::Io(std::io::Error::new(
3120 std::io::ErrorKind::InvalidInput,
3121 "Connection rate limiter requires at least one connection rate key",
3122 )));
3123 }
3124
3125 Ok(rate_limiter.map(|limiter| ConnectionRateLimit { limiter, keys }))
3126 }
3127
3128 fn resolve_rate_limiter(
3129 default_quota: Option<Quota>,
3130 keyed_quotas: Vec<(String, Quota)>,
3131 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3132 ) -> Result<Arc<RateLimiter<Ustr, MonotonicClock>>, TransportError> {
3133 if let Some(rate_limiter) = rate_limiter {
3134 if default_quota.is_some() || !keyed_quotas.is_empty() {
3135 return Err(TransportError::Io(std::io::Error::new(
3136 std::io::ErrorKind::InvalidInput,
3137 "Cannot combine a shared rate limiter with quota configuration",
3138 )));
3139 }
3140
3141 return Ok(rate_limiter);
3142 }
3143
3144 let keyed_quotas = keyed_quotas
3145 .into_iter()
3146 .map(|(key, quota)| (Ustr::from(&key), quota))
3147 .collect();
3148 Ok(Arc::new(RateLimiter::new_with_quota(
3149 default_quota,
3150 keyed_quotas,
3151 )))
3152 }
3153
3154 #[expect(
3155 clippy::too_many_arguments,
3156 reason = "handler delivery and connection controls are independent options"
3157 )]
3158 async fn connect_with_handler_scoped(
3159 config: WebSocketConfig,
3160 handler: IncomingHandler,
3161 ping_handler: Option<IncomingPingHandler>,
3162 close_handler: Option<CloseHandler>,
3163 rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
3164 state_sink: Option<SocketStateSink>,
3165 connection_rate_limit: Option<ConnectionRateLimit>,
3166 initial_connect_options: InitialConnectOptions,
3167 ) -> Result<Self, TransportError> {
3168 log::debug!("Connecting");
3169 let inner = WebSocketClientInner::connect_url_with_handler(
3170 config,
3171 Some(handler),
3172 ping_handler,
3173 close_handler,
3174 state_sink,
3175 connection_rate_limit,
3176 initial_connect_options,
3177 )
3178 .await?;
3179 let connection_mode = Arc::clone(&inner.connection_mode);
3180 let connection_epoch = Arc::clone(&inner.connection_epoch);
3181 let state_notify = Arc::clone(&inner.state_notify);
3182 let controller_notify = Arc::clone(&inner.controller_notify);
3183 let reconnect_published = Arc::clone(&inner.reconnect_published);
3184 let writer_tx = inner.writer_tx.clone();
3185 let connect_timeout = inner.connect_timeout;
3186 let auth_tracker = Arc::clone(&inner.auth_tracker);
3187 let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
3188 let reconnect_headers = inner.reconnect_headers.clone();
3189 let state_sink = inner.state_sink.clone();
3190 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
3191
3192 let controller_task = Self::spawn_controller_task(
3193 inner,
3194 Arc::clone(&connection_mode),
3195 Arc::clone(&state_notify),
3196 Arc::clone(&auth_tracker),
3197 Arc::clone(&controller_lifecycle),
3198 Arc::clone(&controller_notify),
3199 Arc::clone(&reconnect_published),
3200 );
3201 let abort_handle = controller_task.abort_handle();
3202 controller_lifecycle.set_abort(move || abort_handle.abort());
3203
3204 Ok(Self {
3205 controller_task,
3206 connection_mode,
3207 connection_epoch,
3208 state_notify,
3209 connect_timeout,
3210 rate_limiter,
3211 writer_tx,
3212 auth_tracker,
3213 reconnect_buffer_waits_for_auth,
3214 reconnect_headers,
3215 state_sink,
3216 controller_lifecycle,
3217 controller_notify,
3218 reconnect_published,
3219 reconnect_supported: true,
3220 })
3221 }
3222
3223 #[must_use]
3225 pub fn reconnect_headers(&self) -> ReconnectHeaders {
3226 self.reconnect_headers.clone()
3227 }
3228
3229 #[must_use]
3231 pub fn reconnect_handle(&self) -> WebSocketReconnectHandle {
3232 WebSocketReconnectHandle {
3233 connection_mode: Arc::clone(&self.connection_mode),
3234 auth_tracker: Arc::clone(&self.auth_tracker),
3235 state_sink: self.state_sink.clone(),
3236 controller_lifecycle: Arc::clone(&self.controller_lifecycle),
3237 controller_notify: Arc::clone(&self.controller_notify),
3238 reconnect_published: Arc::clone(&self.reconnect_published),
3239 supported: self.reconnect_supported,
3240 }
3241 }
3242
3243 #[must_use]
3248 pub fn request_reconnect(&self) -> bool {
3249 self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
3250 }
3251
3252 #[must_use]
3254 pub fn connection_mode(&self) -> ConnectionMode {
3255 ConnectionMode::from_atomic(&self.connection_mode)
3256 }
3257
3258 #[must_use]
3263 pub fn connection_epoch(&self) -> u64 {
3264 self.connection_epoch.load(Ordering::Acquire)
3265 }
3266
3267 #[must_use]
3272 pub fn connection_mode_atomic(&self) -> Arc<AtomicU8> {
3273 Arc::clone(&self.connection_mode)
3274 }
3275
3276 #[must_use]
3281 pub fn connection_epoch_atomic(&self) -> Arc<AtomicU64> {
3282 Arc::clone(&self.connection_epoch)
3283 }
3284
3285 #[inline]
3290 #[must_use]
3291 pub fn is_active(&self) -> bool {
3292 self.connection_mode().is_active()
3293 }
3294
3295 #[must_use]
3297 pub fn is_disconnected(&self) -> bool {
3298 self.controller_task.is_finished()
3299 }
3300
3301 #[inline]
3306 #[must_use]
3307 pub fn is_reconnecting(&self) -> bool {
3308 self.connection_mode().is_reconnect()
3309 }
3310
3311 pub fn set_auth_tracker(&self, tracker: AuthTracker, reconnect_buffer_waits_for_auth: bool) {
3322 let _ = self.auth_tracker.set(tracker);
3323 self.reconnect_buffer_waits_for_auth
3324 .store(reconnect_buffer_waits_for_auth, Ordering::Release);
3325 }
3326
3327 #[inline]
3331 #[must_use]
3332 pub fn is_disconnecting(&self) -> bool {
3333 self.connection_mode().is_disconnect()
3334 }
3335
3336 #[inline]
3342 #[must_use]
3343 pub fn is_closed(&self) -> bool {
3344 self.connection_mode().is_closed()
3345 }
3346
3347 #[inline]
3351 fn check_not_terminal(&self) -> Result<(), SendError> {
3352 match self.connection_mode() {
3353 ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
3354 _ => Ok(()),
3355 }
3356 }
3357
3358 async fn await_rate_limit_or_closed(&self, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3360 const CHECK_INTERVAL_MS: u64 = 100;
3361
3362 tokio::select! {
3363 biased;
3364 () = self.rate_limiter.await_keys_ready(keys) => Ok(()),
3365 () = async {
3366 loop {
3367 let mut notified = pin!(self.state_notify.notified());
3369 notified.as_mut().enable();
3370
3371 if matches!(self.connection_mode(), ConnectionMode::Disconnect | ConnectionMode::Closed) {
3372 break;
3373 }
3374 tokio::select! {
3375 biased;
3376 () = notified => {}
3377 () = dst::time::sleep(Duration::from_millis(CHECK_INTERVAL_MS)) => {}
3378 }
3379 }
3380 } => Err(SendError::Closed),
3381 }
3382 }
3383
3384 async fn wait_for_active(&self) -> Result<(), SendError> {
3390 const FALLBACK_INTERVAL_MS: u64 = 100;
3391
3392 let mode = self.connection_mode();
3393 if mode.is_active() {
3394 return Ok(());
3395 }
3396
3397 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3398 return Err(SendError::Closed);
3399 }
3400
3401 log::debug!("Waiting for client to become ACTIVE before sending...");
3402
3403 let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
3404
3405 dst::time::timeout(self.connect_timeout, async {
3406 loop {
3407 let mut notified = pin!(self.state_notify.notified());
3409 notified.as_mut().enable();
3410
3411 let mode = self.connection_mode();
3412 if mode.is_active() {
3413 return Ok(());
3414 }
3415
3416 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3417 return Err(());
3418 }
3419
3420 tokio::select! {
3421 biased;
3422 () = notified => {}
3423 () = dst::time::sleep(fallback_interval) => {}
3424 }
3425 }
3426 })
3427 .await
3428 .map_err(|_| SendError::Timeout)?
3429 .map_err(|()| SendError::Closed)
3430 }
3431
3432 pub fn notify_closed(&self) {
3445 let mode = self.connection_mode();
3446 if mode.is_disconnect() || mode.is_closed() {
3447 return;
3448 }
3449
3450 log::debug!("Stream reader signalled EOF, transitioning to CLOSED");
3451
3452 if ConnectionMode::close_websocket_on_loss(&self.connection_mode, self.state_sink.as_ref())
3453 {
3454 fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3455 self.state_notify.notify_waiters();
3456 }
3457 }
3458
3459 pub async fn disconnect(&self) {
3463 log::debug!("Disconnecting");
3464
3465 if ConnectionMode::request_disconnect(&self.connection_mode)
3467 && let Some(tracker) = self.auth_tracker.get()
3468 {
3469 tracker.fail("WebSocket client disconnected");
3470 }
3471
3472 self.state_notify.notify_waiters();
3473
3474 if dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
3475 while !self.is_disconnected() {
3476 dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
3477 }
3478
3479 if !self.controller_task.is_finished() {
3480 self.controller_task.abort();
3481 log_task_aborted("controller");
3482 }
3483 })
3484 .await
3485 == Ok(())
3486 {
3487 log::debug!("Controller task finished");
3488 } else {
3489 log::warn!("Timeout waiting for controller task to finish");
3490
3491 if !self.controller_task.is_finished() {
3492 self.controller_task.abort();
3493 log_task_aborted("controller");
3494 }
3495
3496 self.connection_mode
3497 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3498 }
3499 }
3500
3501 #[allow(unused_variables)]
3511 pub async fn send_text(&self, data: String, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3512 self.check_not_terminal()?;
3513
3514 self.await_rate_limit_or_closed(keys).await?;
3515 self.wait_for_active().await?;
3516
3517 log::trace!("Sending text frame ({} bytes)", data.len());
3518
3519 let msg = Message::Text(data.into());
3520 self.writer_tx.send(WriterCommand::Send(msg))
3521 }
3522
3523 pub async fn send_text_on_connection(
3540 &self,
3541 data: String,
3542 keys: Option<&[Ustr]>,
3543 connection_epoch: u64,
3544 ) -> Result<(), SendError> {
3545 self.check_not_terminal()?;
3546 self.await_rate_limit_or_closed(keys).await?;
3547 self.wait_for_active().await?;
3548
3549 log::trace!(
3550 "Sending text frame once: epoch={connection_epoch} ({} bytes)",
3551 data.len()
3552 );
3553
3554 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3555 self.writer_tx
3556 .send_control(WriterCommand::SendOnConnection {
3557 message: Message::Text(data.into()),
3558 connection_epoch,
3559 response_tx,
3560 })?;
3561
3562 response_rx
3563 .await
3564 .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3565 }
3566
3567 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3579 #[expect(
3580 clippy::unused_async,
3581 clippy::unused_async_trait_impl,
3582 reason = "skipping instead of waiting removes the only await; the signature is public API shared with the other send methods"
3583 )]
3584 pub async fn send_pong(&self, data: Vec<u8>) -> Result<(), SendError> {
3585 validate_pong_payload(&data)?;
3586
3587 if !self.connection_mode().is_active() {
3588 log::debug!("Skipping pong: connection not active");
3589 return Ok(());
3590 }
3591
3592 log::trace!("Sending pong frame ({} bytes)", data.len());
3593
3594 self.writer_tx
3595 .send_control(WriterCommand::Send(Message::Pong(data.into())))
3596 }
3597
3598 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3608 #[expect(
3609 clippy::unused_async,
3610 clippy::unused_async_trait_impl,
3611 reason = "the public send API is async even though this method only enqueues"
3612 )]
3613 pub async fn send_pong_on_connection(
3614 &self,
3615 data: Vec<u8>,
3616 connection_epoch: u64,
3617 ) -> Result<(), SendError> {
3618 validate_pong_payload(&data)?;
3619
3620 if !self.connection_mode().is_active() {
3621 log::debug!("Skipping pong: connection not active");
3622 return Ok(());
3623 }
3624
3625 log::trace!(
3626 "Sending pong frame once: epoch={connection_epoch} ({} bytes)",
3627 data.len()
3628 );
3629
3630 self.writer_tx
3631 .send_control(WriterCommand::SendPongOnConnection {
3632 data,
3633 connection_epoch,
3634 })
3635 }
3636
3637 #[allow(unused_variables)]
3647 pub async fn send_bytes(&self, data: Vec<u8>, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3648 self.check_not_terminal()?;
3649
3650 self.await_rate_limit_or_closed(keys).await?;
3651 self.wait_for_active().await?;
3652
3653 log::trace!("Sending binary frame ({} bytes)", data.len());
3654
3655 let msg = Message::Binary(data.into());
3656 self.writer_tx.send(WriterCommand::Send(msg))
3657 }
3658
3659 pub async fn send_bytes_on_connection(
3676 &self,
3677 data: Vec<u8>,
3678 keys: Option<&[Ustr]>,
3679 connection_epoch: u64,
3680 ) -> Result<(), SendError> {
3681 self.check_not_terminal()?;
3682 self.await_rate_limit_or_closed(keys).await?;
3683 self.wait_for_active().await?;
3684
3685 log::trace!(
3686 "Sending binary frame once: epoch={connection_epoch} ({} bytes)",
3687 data.len()
3688 );
3689
3690 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3691 self.writer_tx
3692 .send_control(WriterCommand::SendOnConnection {
3693 message: Message::Binary(data.into()),
3694 connection_epoch,
3695 response_tx,
3696 })?;
3697
3698 response_rx
3699 .await
3700 .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3701 }
3702
3703 pub async fn send_close_message(&self) -> Result<(), SendError> {
3709 self.wait_for_active().await?;
3710
3711 let msg = Message::Close(None);
3712 self.writer_tx.send_control(WriterCommand::Send(msg))
3713 }
3714
3715 fn spawn_controller_task(
3716 mut inner: WebSocketClientInner,
3717 connection_mode: Arc<AtomicU8>,
3718 state_notify: Arc<tokio::sync::Notify>,
3719 auth_tracker: Arc<OnceLock<AuthTracker>>,
3720 controller_lifecycle: Arc<ControllerLifecycle>,
3721 controller_notify: Arc<tokio::sync::Notify>,
3722 reconnect_published: Arc<AtomicBool>,
3723 ) -> dst::task::JoinHandle<()> {
3724 dst::task::spawn(async move {
3725 let _activity = controller_lifecycle.activity();
3726 log_task_started("controller");
3727
3728 let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
3729 let mut reconnected_at = None;
3730
3731 loop {
3732 tokio::select! {
3733 biased;
3734 () = controller_notify.notified() => {}
3735 () = state_notify.notified() => {}
3736 () = dst::time::sleep(fallback_interval) => {}
3737 }
3738
3739 let mut mode = ConnectionMode::from_atomic(&connection_mode);
3740
3741 if mode.is_disconnect() {
3742 log::debug!("Disconnecting");
3743
3744 let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
3745 if dst::time::timeout(timeout, async {
3746 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
3748
3749 if let Some(read_fence) = inner.read_fence.take() {
3750 read_fence.invalidate();
3751 }
3752
3753 if let Some(task) = &inner.read_task
3754 && !task.is_finished()
3755 {
3756 task.abort();
3757 log_task_aborted("read");
3758 }
3759
3760 if let Some(task) = &inner.heartbeat_task
3761 && !task.is_finished()
3762 {
3763 task.abort();
3764 log_task_aborted("heartbeat");
3765 }
3766 })
3767 .await
3768 .is_err()
3769 {
3770 log::warn!("Shutdown timed out after {}s", timeout.as_secs());
3771 }
3772
3773 log::debug!("Closed");
3774 break; }
3776
3777 if mode.is_closed() {
3778 log::debug!("Connection closed");
3779 fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3780 state_notify.notify_waiters();
3781 break;
3782 }
3783
3784 if mode.is_active() && !inner.is_alive() {
3785 let target = if inner.handler.is_none() {
3786 ConnectionMode::Closed
3787 } else {
3788 ConnectionMode::Reconnect
3789 };
3790
3791 let transitioned = if target.is_closed() {
3792 ConnectionMode::close_websocket_on_loss(
3793 &connection_mode,
3794 inner.state_sink.as_ref(),
3795 )
3796 } else {
3797 request_websocket_reconnect(
3798 &connection_mode,
3799 &reconnect_published,
3800 inner.state_sink.as_ref(),
3801 &auth_tracker,
3802 &controller_notify,
3803 || {},
3804 ) == ReconnectRequestOutcome::Accepted
3805 };
3806
3807 if transitioned {
3808 if target.is_closed() {
3809 fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3810 }
3811
3812 log::info!("Detected dead connection, transitioning to {target:?}");
3813 }
3814
3815 mode = ConnectionMode::from_atomic(&connection_mode);
3816 }
3817
3818 if mode.is_reconnect() {
3819 if let Some(tracker) = auth_tracker.get() {
3820 tracker.invalidate();
3821 }
3822
3823 let reconnect_uptime = reconnected_at
3824 .take()
3825 .map(|started: dst::time::Instant| started.elapsed());
3826 let previous_reconnect_stable = reconnect_uptime
3827 .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
3828
3829 if previous_reconnect_stable {
3830 inner.backoff.reset();
3831 inner.reconnection_attempt_count = 0;
3832 log::debug!(
3833 "WebSocket remained active for at least {}s, resetting reconnect cycle",
3834 RECONNECT_STABILITY_THRESHOLD.as_secs()
3835 );
3836 }
3837
3838 if let Some(max_attempts) = inner.reconnect_max_attempts
3839 && inner.reconnection_attempt_count >= max_attempts
3840 {
3841 log::error!(
3842 "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
3843 );
3844 connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3845 fail_registered_auth(
3846 auth_tracker.as_ref(),
3847 "WebSocket reconnect attempts exhausted",
3848 );
3849 state_notify.notify_waiters();
3850 break;
3851 }
3852
3853 let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
3854 {
3855 inner.backoff.next_duration()
3856 } else {
3857 Duration::ZERO
3858 };
3859
3860 let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
3861 if !duration.is_zero() {
3862 log::warn!("Backing off for {}s...", duration.as_secs_f64());
3863
3864 if !wait_reconnect_delay(
3865 duration,
3866 connection_mode.as_ref(),
3867 state_notify.as_ref(),
3868 )
3869 .await
3870 {
3871 log::debug!("Backoff interrupted by terminal state");
3872 continue;
3873 }
3874 }
3875
3876 inner.reconnection_attempt_count += 1;
3877 inner.reconnect_throttle.record_attempt();
3878 log::debug!(
3879 "Reconnection attempt {} of {}",
3880 inner.reconnection_attempt_count,
3881 inner
3882 .reconnect_max_attempts
3883 .map_or_else(|| "unlimited".to_string(), |m| m.to_string())
3884 );
3885
3886 let reconnect_result = tokio::select! {
3888 biased;
3889 result = inner.reconnect_with_outcome() => Some(result),
3890 () = async {
3891 loop {
3892 let mut notified = pin!(state_notify.notified());
3894 notified.as_mut().enable();
3895
3896 if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
3897 break;
3898 }
3899 notified.await;
3900 }
3901 } => None,
3902 };
3903
3904 match reconnect_result {
3905 None => {
3906 log::debug!("Reconnect interrupted by disconnect");
3907 }
3908 Some(Ok(ReconnectOutcome::Reconnected)) => {
3909 reconnected_at = Some(dst::time::Instant::now());
3910
3911 state_notify.notify_waiters();
3912
3913 if ConnectionMode::from_atomic(&connection_mode).is_active() {
3917 if let Some(ref handler) = inner.handler {
3918 let connection_epoch =
3919 inner.connection_epoch.load(Ordering::Acquire);
3920 let reconnected_msg =
3921 Message::Text(RECONNECTED.to_string().into());
3922 handler.handle(connection_epoch, reconnected_msg);
3923 match handler {
3924 IncomingHandler::Message(_) => {
3925 log::debug!("Sent reconnected message to handler");
3926 }
3927 IncomingHandler::Epoch(_) => {
3928 log::debug!(
3929 "Sent reconnected message to epoch handler: \
3930 epoch={connection_epoch}",
3931 );
3932 }
3933 }
3934 }
3935
3936 log::debug!("Reconnected successfully");
3937 } else {
3938 log::debug!("Skipping reconnect handlers due to disconnect state");
3939 }
3940 }
3941 Some(Ok(ReconnectOutcome::Aborted)) => {
3942 log::debug!("Reconnect aborted");
3943 }
3944 Some(Err(e)) => {
3945 let duration = inner.backoff.next_duration();
3946 log::warn!(
3947 "Reconnect attempt {} failed: {}",
3948 inner.reconnection_attempt_count,
3949 e.redacted_message()
3950 );
3951
3952 if !duration.is_zero() {
3953 log::warn!("Backing off for {}s...", duration.as_secs_f64());
3954
3955 if !wait_reconnect_delay(
3956 duration,
3957 connection_mode.as_ref(),
3958 state_notify.as_ref(),
3959 )
3960 .await
3961 {
3962 log::debug!("Backoff interrupted by terminal state");
3963 }
3964 }
3965 }
3966 }
3967 }
3968 }
3969
3970 inner
3971 .connection_mode
3972 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3973
3974 log_task_stopped("controller");
3975 })
3976 }
3977}
3978
3979fn fail_registered_auth(auth_tracker: &OnceLock<AuthTracker>, reason: &str) {
3980 if let Some(tracker) = auth_tracker.get() {
3981 tracker.fail(reason);
3982 }
3983}
3984
3985fn validate_pong_payload(data: &[u8]) -> Result<(), SendError> {
3986 if data.len() > MAX_CONTROL_FRAME_PAYLOAD_BYTES {
3987 return Err(SendError::InvalidInput(format!(
3988 "pong payload exceeds {MAX_CONTROL_FRAME_PAYLOAD_BYTES} bytes"
3989 )));
3990 }
3991
3992 Ok(())
3993}
3994
3995impl Drop for WebSocketClient {
3996 fn drop(&mut self) {
3997 self.connection_mode
3998 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3999 fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
4000 self.state_notify.notify_waiters();
4001 self.controller_notify.notify_waiters();
4002 self.controller_lifecycle.close_and_abort();
4003 }
4004}
4005
4006#[cfg(test)]
4007#[cfg(not(feature = "turmoil"))]
4008#[cfg(not(all(feature = "simulation", madsim)))] #[cfg(target_os = "linux")] mod tests {
4011 use std::{
4012 collections::HashMap,
4013 num::NonZeroU32,
4014 sync::{
4015 Arc,
4016 atomic::{AtomicU8, AtomicUsize, Ordering},
4017 },
4018 time::Duration,
4019 };
4020
4021 use axum::{Router, routing::post};
4022 use futures_util::{SinkExt, StreamExt};
4023 use log::Level;
4024 use nautilus_common::testing::wait_until_async;
4025 use nautilus_core::string::secret::REDACTED;
4026 use parking_lot::Mutex;
4027 use rstest::rstest;
4028 use tokio::{
4029 io::{AsyncReadExt, AsyncWriteExt},
4030 net::TcpListener,
4031 sync::{mpsc, oneshot},
4032 task::{self, JoinHandle},
4033 };
4034 use tokio_tungstenite::{
4035 accept_async, accept_hdr_async,
4036 tungstenite::{
4037 Message as WsMessage,
4038 handshake::server::{self, Callback},
4039 http::HeaderValue,
4040 protocol::frame::{CloseFrame as WsCloseFrame, coding::CloseCode},
4041 },
4042 };
4043
4044 use crate::{
4045 RECONNECTED, SocketState, SocketStateSink,
4046 error::SendError,
4047 http::{HttpClient, Method},
4048 logging::tests::capture_logs_for,
4049 mode::{ConnectionMode, ReadSessionFence},
4050 ratelimiter::quota::Quota,
4051 transport::{CloseFrame, Message, TransportError},
4052 websocket::{
4053 AuthTracker, CloseHandler, InitialConnectRetryPolicy, ReconnectHeaders,
4054 TransportBackend, WebSocketClient, WebSocketClientInner, WebSocketConfig,
4055 auth::AuthState,
4056 types::{WriterCommand, channel_epoch_message_handler, channel_message_handler},
4057 },
4058 };
4059
4060 const SECRET_MARKER: &str = "OUTBOUND_SECRET_MARKER";
4061 const PING_TRIGGER: &str = "send-test-ping";
4062 const HOSTILE_CLOSE_TRIGGER: &str = "close-hostile";
4063 const CODED_CLOSE_TRIGGER: &str = "close-coded";
4064 const CODED_CLOSE_CODE: u16 = 4003;
4065 const CODED_CLOSE_REASON: &str = "draining";
4066 const HOSTILE_CLOSE_REASON: &str = "injected\r\n\u{1b}[31mforged\u{1b}[0m";
4067 const NETWORK_LOG_TARGETS: &[&str] = &[
4068 "nautilus_network::http::client",
4069 "nautilus_network::websocket::client",
4070 ];
4071
4072 struct TestServer {
4073 task: JoinHandle<()>,
4074 port: u16,
4075 }
4076
4077 #[derive(Debug, Clone)]
4078 struct TestCallback {
4079 key: String,
4080 value: HeaderValue,
4081 }
4082
4083 impl Callback for TestCallback {
4084 #[expect(clippy::panic_in_result_fn)]
4085 fn on_request(
4086 self,
4087 request: &server::Request,
4088 response: server::Response,
4089 ) -> Result<server::Response, server::ErrorResponse> {
4090 let _ = response;
4091 let value = request.headers().get(&self.key);
4092 assert!(value.is_some());
4093
4094 if let Some(value) = request.headers().get(&self.key) {
4095 assert_eq!(value, self.value);
4096 }
4097
4098 Ok(response)
4099 }
4100 }
4101
4102 impl TestServer {
4103 async fn setup() -> Self {
4104 let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4105 let port = TcpListener::local_addr(&server).unwrap().port();
4106
4107 let header_key = "test".to_string();
4108 let header_value = "test".to_string();
4109
4110 let test_call_back = TestCallback {
4111 key: header_key,
4112 value: HeaderValue::from_str(&header_value).unwrap(),
4113 };
4114
4115 let task = task::spawn(async move {
4116 loop {
4118 let (conn, _) = server.accept().await.unwrap();
4119 let mut websocket = accept_hdr_async(conn, test_call_back.clone())
4120 .await
4121 .unwrap();
4122
4123 task::spawn(async move {
4124 while let Some(Ok(msg)) = websocket.next().await {
4125 match msg {
4126 WsMessage::Text(txt) if txt == "close-now" => {
4127 log::debug!("Forcibly closing from server side");
4128 let _ = websocket.close(None).await;
4130 break;
4131 }
4132 WsMessage::Text(txt) if txt == HOSTILE_CLOSE_TRIGGER => {
4133 let _ = websocket
4137 .send(WsMessage::Close(Some(WsCloseFrame {
4138 code: CloseCode::Error,
4139 reason: HOSTILE_CLOSE_REASON.into(),
4140 })))
4141 .await;
4142
4143 break;
4144 }
4145 WsMessage::Text(txt) if txt == CODED_CLOSE_TRIGGER => {
4146 let _ = websocket
4147 .send(WsMessage::Close(Some(WsCloseFrame {
4148 code: CloseCode::from(CODED_CLOSE_CODE),
4149 reason: CODED_CLOSE_REASON.into(),
4150 })))
4151 .await;
4152
4153 break;
4154 }
4155 WsMessage::Text(txt) if txt == PING_TRIGGER => {
4156 let ping = format!("{SECRET_MARKER}:ping");
4157 if websocket.send(WsMessage::Ping(ping.into())).await.is_err() {
4158 break;
4159 }
4160 }
4161 WsMessage::Text(_) | WsMessage::Binary(_) => {
4163 if websocket.send(msg).await.is_err() {
4164 break;
4165 }
4166 }
4167 WsMessage::Close(_frame) => {
4169 let _ = websocket.close(None).await;
4170 break;
4171 }
4172 _ => {}
4174 }
4175 }
4176 });
4177 }
4178 });
4179
4180 Self { task, port }
4181 }
4182 }
4183
4184 impl Drop for TestServer {
4185 fn drop(&mut self) {
4186 self.task.abort();
4187 }
4188 }
4189
4190 async fn setup_test_client(port: u16) -> WebSocketClient {
4191 let config = WebSocketConfig {
4192 url: format!("ws://127.0.0.1:{port}"),
4193 headers: vec![("test".into(), "test".into())],
4194 heartbeat_interval_secs: None,
4195 heartbeat_payload: None,
4196 connect_timeout_ms: None,
4197 reconnect_delay_initial_ms: None,
4198 reconnect_backoff_factor: None,
4199 reconnect_delay_max_ms: None,
4200 reconnect_jitter_ms: None,
4201 reconnect_max_attempts: None,
4202 heartbeat_timeout_secs: None,
4203 idle_timeout_ms: None,
4204 writer_capacity: None,
4205 backend: TransportBackend::Tungstenite,
4206 proxy_url: None,
4207 max_message_size_bytes: None,
4208 max_frame_size_bytes: None,
4209 };
4210
4211 WebSocketClient::builder()
4212 .config(config)
4213 .message_handler(Arc::new(|_| {}))
4214 .connect()
4215 .await
4216 .expect("Failed to connect")
4217 }
4218
4219 #[rstest]
4220 #[case::handler(false)]
4221 #[case::stream(true)]
4222 #[tokio::test]
4223 async fn test_writer_capacity_from_config(#[case] stream_mode: bool) {
4224 let server = TestServer::setup().await;
4225 let config = WebSocketConfig::builder()
4226 .url(format!("ws://127.0.0.1:{}", server.port))
4227 .headers(vec![("test".to_string(), "test".to_string())])
4228 .backend(TransportBackend::Tungstenite)
4229 .writer_capacity(2)
4230 .build()
4231 .unwrap();
4232
4233 let (client, _reader) = if stream_mode {
4234 let (reader, client) = WebSocketClient::stream_builder()
4235 .config(config)
4236 .connect()
4237 .await
4238 .unwrap();
4239 (client, Some(reader))
4240 } else {
4241 let client = WebSocketClient::builder()
4242 .config(config)
4243 .message_handler(Arc::new(|_| {}))
4244 .connect()
4245 .await
4246 .unwrap();
4247 (client, None)
4248 };
4249
4250 client.send_text("first".to_string(), None).await.unwrap();
4252 client.send_bytes(vec![2], None).await.unwrap();
4253 let overflow = client.send_text("overflow".to_string(), None).await;
4254
4255 assert_eq!(client.connection_mode(), ConnectionMode::Active);
4256 assert!(matches!(overflow, Err(SendError::BufferFull)));
4257 client.disconnect().await;
4258 }
4259
4260 #[rstest]
4261 #[tokio::test]
4262 async fn test_send_reports_writer_buffer_full() {
4263 let server = TestServer::setup().await;
4264 let mut client = setup_test_client(server.port).await;
4265 let (writer_tx, mut writer_rx) = crate::writer::channel(1);
4266 client.writer_tx = writer_tx;
4267
4268 client.send_text("first".to_string(), None).await.unwrap();
4269 let result = client.send_bytes(vec![2], None).await;
4270 assert!(matches!(result, Err(SendError::BufferFull)));
4271 let (command, permit) = writer_rx.recv().await.unwrap();
4272 assert!(
4273 matches!(command, WriterCommand::Send(message) if message == Message::text("first"))
4274 );
4275 assert!(matches!(
4276 client.send_text("overflow".to_string(), None).await,
4277 Err(SendError::BufferFull)
4278 ));
4279 drop(permit);
4280 client.send_bytes(vec![3], None).await.unwrap();
4281 let (command, _) = writer_rx.recv().await.unwrap();
4282 assert!(
4283 matches!(command, WriterCommand::Send(message) if message == Message::Binary(vec![3].into()))
4284 );
4285 client.disconnect().await;
4286 }
4287
4288 async fn setup_reconnecting_client(port: u16) -> WebSocketClient {
4289 let config = WebSocketConfig {
4290 url: format!("ws://127.0.0.1:{port}"),
4291 headers: vec![],
4292 heartbeat_interval_secs: None,
4293 heartbeat_payload: None,
4294 connect_timeout_ms: Some(5_000),
4295 reconnect_delay_initial_ms: Some(1),
4296 reconnect_backoff_factor: Some(1.0),
4297 reconnect_delay_max_ms: Some(1),
4298 reconnect_jitter_ms: Some(0),
4299 reconnect_max_attempts: None,
4300 heartbeat_timeout_secs: None,
4301 idle_timeout_ms: None,
4302 writer_capacity: None,
4303 backend: TransportBackend::Tungstenite,
4304 proxy_url: None,
4305 max_message_size_bytes: None,
4306 max_frame_size_bytes: None,
4307 };
4308
4309 WebSocketClient::builder()
4310 .config(config)
4311 .message_handler(Arc::new(|_| {}))
4312 .connect()
4313 .await
4314 .expect("client should connect")
4315 }
4316
4317 async fn wait_for_mode(client: &WebSocketClient, expected: ConnectionMode) {
4318 crate::dst::time::timeout(Duration::from_secs(5), async {
4319 loop {
4320 if ConnectionMode::from_atomic(&client.connection_mode) == expected {
4321 break;
4322 }
4323
4324 crate::dst::time::sleep(Duration::from_millis(1)).await;
4325 }
4326 })
4327 .await
4328 .expect("client should reach expected connection mode");
4329 }
4330
4331 async fn setup_http_test_server() -> (JoinHandle<()>, u16) {
4332 let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4333 let port = server.local_addr().unwrap().port();
4334
4335 let app = Router::new().route(
4336 "/logging",
4337 post(|| async {
4338 (
4339 [("x-secret-response", SECRET_MARKER)],
4340 format!("{SECRET_MARKER}:response"),
4341 )
4342 }),
4343 );
4344
4345 let task = task::spawn(async move {
4346 axum::serve(server, app).await.unwrap();
4347 });
4348
4349 (task, port)
4350 }
4351
4352 #[rstest]
4353 #[tokio::test]
4354 async fn test_network_logs_omit_payload_bodies() {
4355 let server = TestServer::setup().await;
4356 let client = setup_test_client(server.port).await;
4357 let capture = capture_logs_for(NETWORK_LOG_TARGETS).await;
4358 let binary = format!("{SECRET_MARKER}:binary").into_bytes();
4359 let binary_marker = format!("{binary:?}");
4360
4361 client
4362 .send_text(format!("{SECRET_MARKER}:café"), None)
4363 .await
4364 .unwrap();
4365 client
4366 .send_text_on_connection(
4367 format!("{SECRET_MARKER}:owned-é"),
4368 None,
4369 client.connection_epoch(),
4370 )
4371 .await
4372 .unwrap();
4373 client.send_bytes(binary.clone(), None).await.unwrap();
4374 client
4375 .send_bytes_on_connection(binary, None, client.connection_epoch())
4376 .await
4377 .unwrap();
4378 client
4379 .send_text(PING_TRIGGER.to_string(), None)
4380 .await
4381 .unwrap();
4382
4383 tokio::time::timeout(Duration::from_secs(2), async {
4384 loop {
4385 if capture.messages().iter().any(|(level, message)| {
4386 matches!(level, Level::Trace | Level::Warn)
4387 && message == "Received ping frame (27 bytes)"
4388 }) {
4389 break;
4390 }
4391
4392 tokio::task::yield_now().await;
4393 }
4394 })
4395 .await
4396 .expect("timed out waiting for inbound WebSocket metadata log");
4397
4398 let (http_task, http_port) = setup_http_test_server().await;
4399 let invalid_headers =
4400 HashMap::from([("x-secret-default".to_string(), format!("{SECRET_MARKER}\n"))]);
4401 let invalid_header_error = HttpClient::builder()
4402 .headers(invalid_headers)
4403 .build()
4404 .unwrap_err();
4405 let http_client = HttpClient::builder().build().unwrap();
4406 let params = HashMap::from([("secret".to_string(), vec![SECRET_MARKER.to_string()])]);
4407 let headers = HashMap::from([
4408 (
4409 "X-Secret-Request".to_string(),
4410 format!("{SECRET_MARKER}:first"),
4411 ),
4412 (
4413 "x-secret-request".to_string(),
4414 format!("{SECRET_MARKER}:second"),
4415 ),
4416 ]);
4417 let http_body = format!("{SECRET_MARKER}:http-body").into_bytes();
4418 http_client
4419 .request(
4420 Method::POST,
4421 format!("http://127.0.0.1:{http_port}/logging"),
4422 Some(¶ms),
4423 Some(headers),
4424 Some(http_body),
4425 None,
4426 None,
4427 )
4428 .await
4429 .unwrap();
4430
4431 let rejecting = TcpListener::bind("127.0.0.1:0").await.unwrap();
4434 let rejecting_port = rejecting.local_addr().unwrap().port();
4435
4436 let rejecting_server = task::spawn(async move {
4437 loop {
4438 let (mut stream, _) = rejecting.accept().await.unwrap();
4439 let mut request = [0; 1024];
4440 let _ = stream.read(&mut request).await.unwrap();
4441 stream
4442 .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
4443 .await
4444 .unwrap();
4445 }
4446 });
4447
4448 let retry_config = WebSocketConfig {
4449 url: format!("ws://127.0.0.1:{rejecting_port}/?token={SECRET_MARKER}"),
4450 headers: vec![],
4451 heartbeat_interval_secs: None,
4452 heartbeat_payload: None,
4453 connect_timeout_ms: Some(5_000),
4454 reconnect_delay_initial_ms: None,
4455 reconnect_backoff_factor: None,
4456 reconnect_delay_max_ms: None,
4457 reconnect_jitter_ms: None,
4458 reconnect_max_attempts: None,
4459 heartbeat_timeout_secs: None,
4460 idle_timeout_ms: None,
4461 writer_capacity: None,
4462 backend: TransportBackend::Tungstenite,
4463 proxy_url: None,
4464 max_message_size_bytes: None,
4465 max_frame_size_bytes: None,
4466 };
4467
4468 let retry_error = WebSocketClient::builder()
4469 .config(retry_config)
4470 .message_handler(Arc::new(|_| {}))
4471 .initial_connect_retry_policy(InitialConnectRetryPolicy {
4472 max_attempts: NonZeroU32::new(2).unwrap(),
4473 delay_initial: Duration::from_millis(1),
4474 delay_max: Duration::from_millis(1),
4475 backoff_factor: 1.0,
4476 jitter_ms: 0,
4477 })
4478 .connect()
4479 .await
4480 .expect_err("server always rejects the upgrade");
4481
4482 rejecting_server.abort();
4483
4484 assert!(
4485 matches!(retry_error, TransportError::UpgradeRejected(503)),
4486 "expected a 503 upgrade rejection, was: {retry_error:?}"
4487 );
4488
4489 let messages: Vec<_> = capture
4490 .messages()
4491 .into_iter()
4492 .filter(|(level, message)| {
4493 matches!(level, Level::Trace | Level::Warn)
4494 && (message.starts_with("Sending ")
4495 || message.starts_with("Received ")
4496 || message.starts_with("Replaced ")
4497 || message.starts_with("WebSocket connection attempt "))
4498 })
4499 .map(|(_, message)| message)
4500 .collect();
4501
4502 let invalid_header_message = invalid_header_error.to_string();
4503
4504 let retry_warning = messages
4507 .iter()
4508 .find(|message| message.starts_with("WebSocket connection attempt 1/2 "))
4509 .expect("initial-connect retry warning was not captured");
4510 assert!(
4511 retry_warning.contains(REDACTED),
4512 "retry warning omitted the redaction placeholder: {retry_warning}"
4513 );
4514
4515 assert!(
4516 messages.iter().all(|message| {
4517 !message.contains(SECRET_MARKER) && !message.contains(&binary_marker)
4518 }),
4519 "network logs exposed the secret marker: {messages:?}"
4520 );
4521 assert!(
4522 !invalid_header_message.contains(SECRET_MARKER),
4523 "invalid header error exposed the secret marker: {invalid_header_message}"
4524 );
4525 assert!(
4526 invalid_header_message.contains("x-secret-default"),
4527 "invalid header error omitted safe header metadata: {invalid_header_message}"
4528 );
4529 assert!(
4530 messages
4531 .iter()
4532 .any(|message| message == "Sending text frame (28 bytes)"),
4533 "text send metadata missing or inaccurate: {messages:?}"
4534 );
4535 assert!(
4536 messages
4537 .iter()
4538 .any(|message| { message == "Sending text frame once: epoch=0 (31 bytes)" }),
4539 "ownership-bound text metadata missing or inaccurate: {messages:?}"
4540 );
4541 assert!(
4542 messages
4543 .iter()
4544 .any(|message| message == "Sending binary frame (29 bytes)"),
4545 "binary send metadata missing or inaccurate: {messages:?}"
4546 );
4547 assert!(
4548 messages
4549 .iter()
4550 .any(|message| message == "Sending binary frame once: epoch=0 (29 bytes)"),
4551 "ownership-bound binary metadata missing or inaccurate: {messages:?}"
4552 );
4553 assert!(
4554 messages
4555 .iter()
4556 .any(|message| message == "Received text frame (28 bytes)"),
4557 "text receive metadata missing or inaccurate: {messages:?}"
4558 );
4559 assert!(
4560 messages
4561 .iter()
4562 .any(|message| message == "Received text frame (31 bytes)"),
4563 "ownership-bound text receive metadata missing or inaccurate: {messages:?}"
4564 );
4565 assert!(
4566 messages
4567 .iter()
4568 .any(|message| message == "Received message <binary> 29 bytes"),
4569 "binary receive metadata missing or inaccurate: {messages:?}"
4570 );
4571 assert!(
4572 messages
4573 .iter()
4574 .any(|message| message == "Received ping frame (27 bytes)"),
4575 "ping receive metadata missing or inaccurate: {messages:?}"
4576 );
4577 assert!(
4579 messages.iter().any(|message| {
4580 message
4581 == "Sending HTTP request: method=POST extra_headers=3 query_bytes=29 \
4582 body_bytes=32"
4583 }),
4584 "HTTP request metadata missing or inaccurate: {messages:?}"
4585 );
4586 assert!(
4587 messages
4588 .iter()
4589 .any(|message| message == "Replaced duplicate request header 'x-secret-request'"),
4590 "duplicate header metadata missing: {messages:?}"
4591 );
4592 assert!(
4593 messages.iter().any(|message| {
4594 message.starts_with("Received HTTP response: status=200 OK headers=")
4595 && message.ends_with(" body_bytes=31")
4596 }),
4597 "HTTP response metadata missing or inaccurate: {messages:?}"
4598 );
4599
4600 client.disconnect().await;
4601 http_task.abort();
4602 }
4603
4604 #[rstest]
4608 #[tokio::test]
4609 async fn test_send_pong_skips_gated_reconnect() {
4610 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4611 let port = listener.local_addr().unwrap().port();
4612 let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
4613 let (handshake_gate_tx, handshake_gate_rx) = oneshot::channel();
4614 let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
4615
4616 let server_task = task::spawn(async move {
4617 let (first, _) = listener.accept().await.unwrap();
4618 let first_websocket = accept_async(first).await.unwrap();
4619 drop(first_websocket);
4620
4621 let (second, _) = listener.accept().await.unwrap();
4622 second_accepted_tx.send(()).unwrap();
4623 handshake_gate_rx.await.unwrap();
4624 let mut replacement = accept_async(second).await.unwrap();
4625
4626 while let Some(message) = replacement.next().await {
4627 match message.unwrap() {
4628 WsMessage::Pong(data) => observed_tx.send(data.to_vec()).unwrap(),
4629 WsMessage::Close(_) => {
4630 let _ = replacement.close(None).await;
4631 break;
4632 }
4633 _ => {}
4634 }
4635 }
4636 });
4637
4638 let client = setup_reconnecting_client(port).await;
4639
4640 crate::dst::time::timeout(Duration::from_secs(5), second_accepted_rx)
4641 .await
4642 .expect("replacement connection should be accepted")
4643 .unwrap();
4644 wait_for_mode(&client, ConnectionMode::Reconnect).await;
4645
4646 let rejected = client.send_pong(vec![2; 126]).await;
4647 assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4648
4649 crate::dst::time::timeout(
4652 Duration::from_secs(1),
4653 client.send_pong(b"stale-pong".to_vec()),
4654 )
4655 .await
4656 .expect("pong raised during reconnect should not wait for the replacement")
4657 .expect("skipped pong should report success");
4658
4659 handshake_gate_tx.send(()).unwrap();
4660 wait_for_mode(&client, ConnectionMode::Active).await;
4661
4662 let fresh_payload = b"fresh-pong".to_vec();
4663 client.send_pong(fresh_payload.clone()).await.unwrap();
4664 assert_eq!(
4665 crate::dst::time::timeout(Duration::from_secs(5), observed_rx.recv())
4666 .await
4667 .expect("replacement connection should receive the fresh pong"),
4668 Some(fresh_payload)
4669 );
4670
4671 client.disconnect().await;
4672 server_task.await.unwrap();
4673 assert_eq!(
4674 observed_rx.try_recv(),
4675 Err(mpsc::error::TryRecvError::Disconnected),
4676 "replacement connection should receive exactly one pong"
4677 );
4678 }
4679
4680 #[rstest]
4683 #[tokio::test]
4684 async fn test_pong_validation_precedes_inactive_skip() {
4685 let server = TestServer::setup().await;
4686 let client = setup_test_client(server.port).await;
4687 client
4688 .connection_mode
4689 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4690
4691 let result = client.send_pong(vec![2; 126]).await;
4692 let epoch_result = client.send_pong_on_connection(vec![2; 126], 0).await;
4693
4694 assert!(matches!(result, Err(SendError::InvalidInput(_))));
4695 assert!(matches!(epoch_result, Err(SendError::InvalidInput(_))));
4696 }
4697
4698 #[rstest]
4701 #[tokio::test]
4702 async fn test_pong_payload_limit_preserves_connection() {
4703 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4704 let port = listener.local_addr().unwrap().port();
4705 let (pong_tx, mut pong_rx) = mpsc::unbounded_channel();
4706
4707 let server_task = task::spawn(async move {
4708 let (stream, _) = listener.accept().await.unwrap();
4709 let mut websocket = accept_async(stream).await.unwrap();
4710
4711 while let Some(message) = websocket.next().await {
4712 match message.unwrap() {
4713 WsMessage::Pong(data) => pong_tx.send(data.to_vec()).unwrap(),
4714 WsMessage::Close(_) => {
4715 let _ = websocket.close(None).await;
4716 break;
4717 }
4718 _ => {}
4719 }
4720 }
4721 });
4722
4723 let client = setup_reconnecting_client(port).await;
4724 let accepted = vec![1; 125];
4725 let follow_up = vec![3; 125];
4726
4727 client.send_pong(accepted.clone()).await.unwrap();
4728 assert_eq!(
4729 crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4730 .await
4731 .unwrap(),
4732 Some(accepted)
4733 );
4734
4735 let rejected = client.send_pong(vec![2; 126]).await;
4736 assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4737
4738 client.send_pong(follow_up.clone()).await.unwrap();
4739 assert_eq!(
4740 crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4741 .await
4742 .unwrap(),
4743 Some(follow_up)
4744 );
4745
4746 client.disconnect().await;
4747 server_task.await.unwrap();
4748 }
4749
4750 #[tokio::test]
4751 async fn test_websocket_basic() {
4752 let server = TestServer::setup().await;
4753 let client = setup_test_client(server.port).await;
4754
4755 assert!(!client.is_disconnected());
4756
4757 client.disconnect().await;
4758 assert!(client.is_disconnected());
4759 }
4760
4761 #[rstest]
4762 #[tokio::test]
4763 async fn test_drop_sets_shared_connection_mode_closed() {
4764 let server = TestServer::setup().await;
4765 let client = setup_test_client(server.port).await;
4766 let connection_mode = client.connection_mode_atomic();
4767
4768 drop(client);
4769
4770 assert_eq!(
4771 ConnectionMode::from_atomic(&connection_mode),
4772 ConnectionMode::Closed
4773 );
4774 }
4775
4776 #[rstest]
4777 #[tokio::test]
4778 async fn test_notify_closed_closes_reconnecting_client() {
4779 let server = TestServer::setup().await;
4780 let client = setup_test_client(server.port).await;
4781 client
4782 .connection_mode
4783 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4784
4785 client.notify_closed();
4786
4787 assert!(client.is_closed());
4788 }
4789
4790 #[tokio::test]
4791 async fn test_websocket_heartbeat() {
4792 let server = TestServer::setup().await;
4793 let client = setup_test_client(server.port).await;
4794
4795 tokio::time::sleep(std::time::Duration::from_secs(3)).await;
4797
4798 client.disconnect().await;
4800 assert!(client.is_disconnected());
4801 }
4802
4803 #[rstest]
4804 #[tokio::test]
4805 async fn test_websocket_reconnect_exhausted() {
4806 let config = WebSocketConfig {
4807 url: "ws://127.0.0.1:9997".into(), headers: vec![],
4809 heartbeat_interval_secs: None,
4810 heartbeat_payload: None,
4811 connect_timeout_ms: None,
4812 reconnect_delay_initial_ms: None,
4813 reconnect_backoff_factor: None,
4814 reconnect_delay_max_ms: None,
4815 reconnect_jitter_ms: None,
4816 reconnect_max_attempts: None,
4817 heartbeat_timeout_secs: None,
4818 idle_timeout_ms: None,
4819 writer_capacity: None,
4820 backend: TransportBackend::Tungstenite,
4821 proxy_url: None,
4822 max_message_size_bytes: None,
4823 max_frame_size_bytes: None,
4824 };
4825
4826 let states = Arc::new(Mutex::new(Vec::new()));
4827 let states_callback = Arc::clone(&states);
4828 let sink = SocketStateSink::new(move |state| {
4829 states_callback.lock().push(state);
4830 });
4831
4832 let res = WebSocketClient::builder()
4833 .config(config)
4834 .message_handler(Arc::new(|_| {}))
4835 .state_sink(sink)
4836 .connect()
4837 .await;
4838 assert!(res.is_err(), "Should fail quickly with no server");
4839 assert_eq!(*states.lock(), Vec::new());
4840 }
4841
4842 #[tokio::test]
4843 async fn test_websocket_forced_close_reconnect() {
4844 let server = TestServer::setup().await;
4845 let client = setup_test_client(server.port).await;
4846
4847 client.send_text("Hello".into(), None).await.unwrap();
4849
4850 client.send_text("close-now".into(), None).await.unwrap();
4852
4853 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
4855
4856 assert!(!client.is_disconnected());
4858
4859 client.disconnect().await;
4861 assert!(client.is_disconnected());
4862 }
4863
4864 #[rstest]
4865 #[case::plain(false)]
4866 #[case::epoch(true)]
4867 #[tokio::test]
4868 async fn test_peer_close_preserves_forwarded_messages(#[case] epoch_aware: bool) {
4869 let timeout = Duration::from_secs(5);
4870 let subscription = "subscribe-retained";
4871 let server = TestServer::setup().await;
4872 let config = WebSocketConfig::builder()
4873 .url(format!("ws://127.0.0.1:{}", server.port))
4874 .headers(vec![("test".to_string(), "test".to_string())])
4875 .backend(TransportBackend::Tungstenite)
4876 .reconnect_delay_initial_ms(50)
4877 .reconnect_jitter_ms(0)
4878 .build()
4879 .unwrap();
4880 let (handler, mut raw_rx) = channel_epoch_message_handler();
4881
4882 let ws_client = if epoch_aware {
4883 WebSocketClient::epoch_builder()
4884 .config(config)
4885 .epoch_handler(handler)
4886 .connect()
4887 .await
4888 } else {
4889 WebSocketClient::builder()
4890 .config(config)
4891 .message_handler(Arc::new(move |message| handler(0, message)))
4892 .connect()
4893 .await
4894 }
4895 .unwrap();
4896
4897 let ws_client = Arc::new(ws_client);
4898 let (bytes_tx, mut bytes_rx) = tokio::sync::mpsc::unbounded_channel();
4899 let forwarding_client = Arc::clone(&ws_client);
4900
4901 let forwarding_task = tokio::spawn(async move {
4902 while let Some((epoch, message)) = raw_rx.recv().await {
4903 let data = match message {
4904 WsMessage::Text(text) => text.as_bytes().to_vec(),
4905 WsMessage::Binary(data) => data.to_vec(),
4906 WsMessage::Close(_) => break,
4907 _ => continue,
4908 };
4909
4910 if data == RECONNECTED.as_bytes() {
4911 forwarding_client
4912 .send_text(subscription.into(), None)
4913 .await
4914 .unwrap();
4915 }
4916
4917 if bytes_tx.send((epoch, data)).is_err() {
4918 break;
4919 }
4920 }
4921 });
4922
4923 ws_client
4924 .send_text(subscription.into(), None)
4925 .await
4926 .unwrap();
4927 let initial = tokio::time::timeout(timeout, bytes_rx.recv())
4928 .await
4929 .expect("initial subscription response timed out")
4930 .expect("forwarding task ended before peer close");
4931
4932 ws_client.send_text("close-now".into(), None).await.unwrap();
4933 let reconnected = tokio::time::timeout(timeout, bytes_rx.recv())
4934 .await
4935 .expect("reconnect notification timed out")
4936 .expect("peer close terminated the forwarding task");
4937 let replayed = tokio::time::timeout(timeout, bytes_rx.recv())
4938 .await
4939 .expect("subscription replay response timed out")
4940 .expect("forwarding task ended before subscription replay");
4941 let payload = vec![3, 7, 11];
4942 ws_client.send_bytes(payload.clone(), None).await.unwrap();
4943 let live = tokio::time::timeout(timeout, bytes_rx.recv())
4944 .await
4945 .expect("replacement connection data timed out")
4946 .expect("forwarding task ended before replacement data");
4947
4948 let expected_epoch = u64::from(epoch_aware);
4949 assert_eq!(initial, (0, subscription.as_bytes().to_vec()));
4950 assert_eq!(
4951 reconnected,
4952 (expected_epoch, RECONNECTED.as_bytes().to_vec())
4953 );
4954 assert_eq!(replayed, (expected_epoch, subscription.as_bytes().to_vec()));
4955 assert_eq!(live, (expected_epoch, payload));
4956 assert_eq!(ws_client.connection_epoch(), 1);
4957 assert!(!forwarding_task.is_finished());
4958
4959 ws_client.disconnect().await;
4960 forwarding_task.abort();
4961 }
4962
4963 #[rstest]
4964 #[case(true)]
4965 #[case(false)]
4966 #[tokio::test]
4967 async fn test_peer_close_frame_is_forwarded_to_close_handler(#[case] reconnect: bool) {
4968 let server = TestServer::setup().await;
4969 let (handler, mut handler_rx) = channel_message_handler();
4970 let (state_tx, mut state_rx) = tokio::sync::mpsc::unbounded_channel();
4971
4972 let client = WebSocketClient::builder()
4973 .config(WebSocketConfig {
4974 url: format!("ws://127.0.0.1:{}", server.port),
4975 headers: vec![("test".into(), "test".into())],
4976 heartbeat_interval_secs: None,
4977 heartbeat_payload: None,
4978 connect_timeout_ms: Some(1_000),
4979 reconnect_delay_initial_ms: Some(1_000),
4980 reconnect_backoff_factor: None,
4981 reconnect_delay_max_ms: None,
4982 reconnect_jitter_ms: Some(0),
4983 reconnect_max_attempts: None,
4984 heartbeat_timeout_secs: None,
4985 idle_timeout_ms: None,
4986 writer_capacity: None,
4987 backend: TransportBackend::Tungstenite,
4988 proxy_url: None,
4989 max_message_size_bytes: None,
4990 max_frame_size_bytes: None,
4991 })
4992 .message_handler(Arc::new(|_| {}))
4993 .state_sink(SocketStateSink::new(move |state| {
4994 state_tx.send(state).expect("state channel closed");
4995 }))
4996 .close_handler(Arc::new(move |frame| {
4997 handler(Message::Close(frame));
4998 reconnect
4999 }))
5000 .connect()
5001 .await
5002 .expect("Failed to connect");
5003
5004 let tracker = AuthTracker::new();
5005 let pending_auth = tracker.begin();
5006 client.set_auth_tracker(tracker.clone(), false);
5007
5008 client
5009 .send_text(CODED_CLOSE_TRIGGER.to_string(), None)
5010 .await
5011 .unwrap();
5012
5013 let message = tokio::time::timeout(Duration::from_secs(10), handler_rx.recv())
5014 .await
5015 .expect("close frame was not forwarded")
5016 .expect("handler channel closed");
5017
5018 let WsMessage::Close(Some(frame)) = message else {
5019 panic!("expected a close frame, was {message:?}");
5020 };
5021
5022 let connected = state_rx.recv().await.expect("initial socket state");
5023 let disconnected = tokio::time::timeout(Duration::from_secs(5), state_rx.recv())
5024 .await
5025 .expect("peer close must publish socket loss")
5026 .expect("state channel closed");
5027
5028 assert_eq!(u16::from(frame.code), CODED_CLOSE_CODE);
5029 assert_eq!(frame.reason.as_str(), CODED_CLOSE_REASON);
5030 assert_eq!(connected, SocketState::Connected);
5031 assert_eq!(disconnected, SocketState::Disconnected);
5032
5033 if !reconnect {
5034 let auth_result = tokio::time::timeout(Duration::from_secs(1), pending_auth)
5035 .await
5036 .expect("terminal close should resolve pending authentication")
5037 .expect("authentication sender should report its terminal result");
5038
5039 assert_eq!(auth_result, Err("WebSocket client closed".to_string()));
5040 assert_eq!(tracker.auth_state(), AuthState::Failed);
5041 }
5042
5043 client.disconnect().await;
5044 }
5045
5046 #[rstest]
5047 #[case(true, ConnectionMode::Active)]
5048 #[case(false, ConnectionMode::Closed)]
5049 fn test_peer_close_callback_controls_reconnect(
5050 #[case] reconnect: bool,
5051 #[case] expected_mode: ConnectionMode,
5052 ) {
5053 let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5054 let fence = ReadSessionFence::new();
5055 let received = Arc::new(Mutex::new(Vec::new()));
5056 let states = Arc::new(Mutex::new(Vec::new()));
5057 let sink_states = Arc::clone(&states);
5058 let sink = SocketStateSink::new(move |state| sink_states.lock().push(state));
5059 let callback_received = Arc::clone(&received);
5060 let callback: CloseHandler = Arc::new(move |frame| {
5061 callback_received.lock().push(frame);
5062 reconnect
5063 });
5064
5065 let frame = CloseFrame::new(CODED_CLOSE_CODE, CODED_CLOSE_REASON);
5066
5067 WebSocketClientInner::dispatch_close_frame(
5068 &mode,
5069 &fence,
5070 Some(&callback),
5071 Some(&sink),
5072 Some(frame.clone()),
5073 );
5074
5075 assert_eq!(*received.lock(), vec![Some(frame)]);
5076 assert_eq!(ConnectionMode::from_atomic(&mode), expected_mode);
5077 assert_eq!(
5078 *states.lock(),
5079 if reconnect {
5080 Vec::new()
5081 } else {
5082 vec![SocketState::Disconnected]
5083 },
5084 );
5085 }
5086
5087 #[rstest]
5088 fn test_peer_close_callback_ignores_retired_reader() {
5089 let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5090 let fence = ReadSessionFence::new();
5091 fence.invalidate();
5092 let calls = Arc::new(AtomicUsize::new(0));
5093 let callback_calls = Arc::clone(&calls);
5094 let callback: CloseHandler = Arc::new(move |_| {
5095 callback_calls.fetch_add(1, Ordering::SeqCst);
5096 false
5097 });
5098
5099 WebSocketClientInner::dispatch_close_frame(&mode, &fence, Some(&callback), None, None);
5100
5101 assert_eq!(calls.load(Ordering::SeqCst), 0);
5102 assert_eq!(ConnectionMode::from_atomic(&mode), ConnectionMode::Active);
5103 }
5104
5105 #[rstest]
5107 #[tokio::test]
5108 async fn test_close_frame_reason_is_redacted_in_logs() {
5109 let server = TestServer::setup().await;
5110 let client = setup_test_client(server.port).await;
5111 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5112
5113 client
5114 .send_text(HOSTILE_CLOSE_TRIGGER.to_string(), None)
5115 .await
5116 .unwrap();
5117
5118 tokio::time::timeout(Duration::from_secs(5), async {
5119 loop {
5120 if capture.messages().iter().any(|(_, message)| {
5121 message.starts_with("Received close frame, terminating: code=")
5122 }) {
5123 break;
5124 }
5125
5126 tokio::task::yield_now().await;
5127 }
5128 })
5129 .await
5130 .expect("timed out waiting for the close frame log");
5131
5132 let close_frame_logs: Vec<String> = capture
5133 .messages()
5134 .iter()
5135 .map(|(_, message)| message.clone())
5136 .filter(|message| message.starts_with("Received close frame, terminating: code="))
5137 .collect();
5138
5139 let expected = "Received close frame, terminating: code=1011, reason=<redacted>";
5140 assert!(
5141 close_frame_logs.iter().any(|message| message == expected),
5142 "close frame reason was not redacted: {close_frame_logs:?}"
5143 );
5144 assert!(
5145 close_frame_logs.iter().all(|message| {
5146 !message.contains('\r') && !message.contains('\n') && !message.contains('\u{1b}')
5147 }),
5148 "close frame log forged line breaks or escape sequences: {close_frame_logs:?}"
5149 );
5150
5151 assert!(
5152 close_frame_logs
5153 .iter()
5154 .all(|message| !message.contains("injected") && !message.contains("forged"))
5155 );
5156
5157 client.disconnect().await;
5158 }
5159
5160 #[rstest]
5161 #[tokio::test]
5162 async fn test_state_sink_reports_initial_loss_and_recovery() {
5163 let server = TestServer::setup().await;
5164
5165 let config = WebSocketConfig {
5166 url: format!("ws://127.0.0.1:{}", server.port),
5167 headers: vec![("test".into(), "test".into())],
5168 heartbeat_interval_secs: None,
5169 heartbeat_payload: None,
5170 connect_timeout_ms: Some(1_000),
5171 reconnect_delay_initial_ms: Some(1),
5172 reconnect_backoff_factor: Some(1.0),
5173 reconnect_delay_max_ms: Some(1),
5174 reconnect_jitter_ms: Some(0),
5175 reconnect_max_attempts: Some(3),
5176 heartbeat_timeout_secs: None,
5177 idle_timeout_ms: None,
5178 writer_capacity: None,
5179 backend: TransportBackend::Tungstenite,
5180 proxy_url: None,
5181 max_message_size_bytes: None,
5182 max_frame_size_bytes: None,
5183 };
5184
5185 let states = Arc::new(Mutex::new(Vec::new()));
5186 let states_callback = Arc::clone(&states);
5187 let sink = SocketStateSink::new(move |state| {
5188 states_callback.lock().push(state);
5189 });
5190
5191 let client = WebSocketClient::builder()
5192 .config(config)
5193 .message_handler(Arc::new(|_| {}))
5194 .state_sink(sink)
5195 .connect()
5196 .await
5197 .unwrap();
5198
5199 assert_eq!(*states.lock(), vec![SocketState::Connected]);
5200
5201 client.send_text("close-now".into(), None).await.unwrap();
5202 wait_until_async(
5203 || {
5204 let states = Arc::clone(&states);
5205 async move { states.lock().len() == 3 }
5206 },
5207 Duration::from_secs(5),
5208 )
5209 .await;
5210
5211 assert_eq!(
5212 *states.lock(),
5213 vec![
5214 SocketState::Connected,
5215 SocketState::Disconnected,
5216 SocketState::Connected,
5217 ]
5218 );
5219
5220 client.disconnect().await;
5221 assert_eq!(states.lock().len(), 3);
5222 }
5223
5224 #[rstest]
5225 #[tokio::test]
5226 async fn test_drop_suppresses_socket_state_event() {
5227 let server = TestServer::setup().await;
5228
5229 let config = WebSocketConfig {
5230 url: format!("ws://127.0.0.1:{}", server.port),
5231 headers: vec![("test".into(), "test".into())],
5232 heartbeat_interval_secs: None,
5233 heartbeat_payload: None,
5234 connect_timeout_ms: Some(1_000),
5235 reconnect_delay_initial_ms: Some(1),
5236 reconnect_backoff_factor: Some(1.0),
5237 reconnect_delay_max_ms: Some(1),
5238 reconnect_jitter_ms: Some(0),
5239 reconnect_max_attempts: Some(3),
5240 heartbeat_timeout_secs: None,
5241 idle_timeout_ms: None,
5242 writer_capacity: None,
5243 backend: TransportBackend::Tungstenite,
5244 proxy_url: None,
5245 max_message_size_bytes: None,
5246 max_frame_size_bytes: None,
5247 };
5248
5249 let states = Arc::new(Mutex::new(Vec::new()));
5250 let states_callback = Arc::clone(&states);
5251 let sink = SocketStateSink::new(move |state| {
5252 states_callback.lock().push(state);
5253 });
5254
5255 let client = WebSocketClient::builder()
5256 .config(config)
5257 .message_handler(Arc::new(|_| {}))
5258 .state_sink(sink)
5259 .connect()
5260 .await
5261 .unwrap();
5262
5263 drop(client);
5264 crate::dst::time::sleep(Duration::from_millis(25)).await;
5265
5266 assert_eq!(*states.lock(), vec![SocketState::Connected]);
5267 }
5268
5269 #[rstest]
5270 #[tokio::test]
5271 async fn test_stream_state_sink_reports_reader_loss() {
5272 let server = TestServer::setup().await;
5273
5274 let config = WebSocketConfig {
5275 url: format!("ws://127.0.0.1:{}", server.port),
5276 headers: vec![("test".into(), "test".into())],
5277 heartbeat_interval_secs: None,
5278 heartbeat_payload: None,
5279 connect_timeout_ms: None,
5280 reconnect_delay_initial_ms: None,
5281 reconnect_backoff_factor: None,
5282 reconnect_delay_max_ms: None,
5283 reconnect_jitter_ms: None,
5284 reconnect_max_attempts: None,
5285 heartbeat_timeout_secs: None,
5286 idle_timeout_ms: None,
5287 writer_capacity: None,
5288 backend: TransportBackend::Tungstenite,
5289 proxy_url: None,
5290 max_message_size_bytes: None,
5291 max_frame_size_bytes: None,
5292 };
5293
5294 let states = Arc::new(Mutex::new(Vec::new()));
5295 let states_callback = Arc::clone(&states);
5296 let sink = SocketStateSink::new(move |state| {
5297 states_callback.lock().push(state);
5298 });
5299
5300 let (_reader, client) = WebSocketClient::stream_builder()
5301 .config(config)
5302 .state_sink(sink)
5303 .connect()
5304 .await
5305 .unwrap();
5306
5307 client.notify_closed();
5308
5309 assert!(client.is_closed());
5310 assert_eq!(
5311 *states.lock(),
5312 vec![SocketState::Connected, SocketState::Disconnected]
5313 );
5314 }
5315
5316 #[rstest]
5317 #[tokio::test]
5318 async fn test_state_sink_emits_no_retry_or_exhaustion_events() {
5319 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5320 let port = listener.local_addr().unwrap().port();
5321 let server_task = task::spawn(async move {
5322 let (connection, _) = listener.accept().await.unwrap();
5323 let mut websocket = accept_async(connection).await.unwrap();
5324 while let Some(Ok(message)) = websocket.next().await {
5325 if matches!(&message, WsMessage::Text(text) if text.as_str() == "close-now") {
5326 websocket.close(None).await.unwrap();
5327 break;
5328 }
5329 }
5330 });
5331
5332 let config = WebSocketConfig {
5333 url: format!("ws://127.0.0.1:{port}"),
5334 headers: vec![],
5335 heartbeat_interval_secs: None,
5336 heartbeat_payload: None,
5337 connect_timeout_ms: Some(100),
5338 reconnect_delay_initial_ms: Some(1),
5339 reconnect_backoff_factor: Some(1.0),
5340 reconnect_delay_max_ms: Some(1),
5341 reconnect_jitter_ms: Some(0),
5342 reconnect_max_attempts: Some(2),
5343 heartbeat_timeout_secs: None,
5344 idle_timeout_ms: None,
5345 writer_capacity: None,
5346 backend: TransportBackend::Tungstenite,
5347 proxy_url: None,
5348 max_message_size_bytes: None,
5349 max_frame_size_bytes: None,
5350 };
5351
5352 let states = Arc::new(Mutex::new(Vec::new()));
5353 let states_callback = Arc::clone(&states);
5354 let sink = SocketStateSink::new(move |state| {
5355 states_callback.lock().push(state);
5356 });
5357
5358 let client = WebSocketClient::builder()
5359 .config(config)
5360 .message_handler(Arc::new(|_| {}))
5361 .state_sink(sink)
5362 .connect()
5363 .await
5364 .unwrap();
5365
5366 client.send_text("close-now".into(), None).await.unwrap();
5367 wait_until_async(
5368 || async { client.is_disconnected() },
5369 Duration::from_secs(5),
5370 )
5371 .await;
5372 assert_eq!(
5373 *states.lock(),
5374 vec![SocketState::Connected, SocketState::Disconnected]
5375 );
5376
5377 server_task.await.unwrap();
5378 }
5379
5380 #[rstest]
5381 #[case(
5382 "invalid header",
5383 "value",
5384 "Invalid WebSocket reconnect header name: invalid HTTP header name"
5385 )]
5386 #[case(
5387 "x-test",
5388 "invalid\nvalue",
5389 "Invalid WebSocket reconnect header value: failed to parse header value"
5390 )]
5391 fn test_reconnect_headers_reject_invalid_update(
5392 #[case] name: &str,
5393 #[case] value: &str,
5394 #[case] expected_message: &str,
5395 ) {
5396 let initial = vec![("x-existing".to_string(), "initial".to_string())];
5397 let headers = ReconnectHeaders::new(initial.clone());
5398
5399 let error = headers.update(name, value).unwrap_err();
5400
5401 let TransportError::Io(error) = error else {
5402 panic!("expected I/O error, was {error:?}");
5403 };
5404
5405 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
5406 assert_eq!(error.to_string(), expected_message);
5407 assert_eq!(headers.snapshot(), initial);
5408 }
5409
5410 #[tokio::test]
5411 #[allow(clippy::result_large_err)]
5412 async fn test_reconnect_uses_updated_headers_without_interrupting_active_connection() {
5413 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5414 let port = listener.local_addr().unwrap().port();
5415 let (header_tx, mut header_rx) = tokio::sync::mpsc::unbounded_channel();
5416
5417 let server_task = task::spawn(async move {
5418 loop {
5419 let (conn, _) = listener.accept().await.unwrap();
5420 let header_tx = header_tx.clone();
5421
5422 let mut websocket = accept_hdr_async(
5423 conn,
5424 move |request: &server::Request, response: server::Response| {
5425 let values = request
5426 .headers()
5427 .get_all("authorization")
5428 .iter()
5429 .map(|value| value.to_str().unwrap().to_string())
5430 .collect::<Vec<_>>();
5431 header_tx.send(values).unwrap();
5432 Ok(response)
5433 },
5434 )
5435 .await
5436 .unwrap();
5437
5438 task::spawn(async move {
5439 while let Some(Ok(msg)) = websocket.next().await {
5440 if matches!(&msg, WsMessage::Text(text) if text.as_str() == "close-now") {
5441 let _ = websocket.close(None).await;
5442 break;
5443 }
5444 }
5445 });
5446 }
5447 });
5448
5449 let config = WebSocketConfig {
5450 url: format!("ws://127.0.0.1:{port}"),
5451 headers: vec![("Authorization".into(), "Bearer initial".into())],
5452 heartbeat_interval_secs: None,
5453 heartbeat_payload: None,
5454 connect_timeout_ms: Some(1_000),
5455 reconnect_delay_initial_ms: Some(50),
5456 reconnect_delay_max_ms: Some(50),
5457 reconnect_backoff_factor: Some(1.0),
5458 reconnect_jitter_ms: Some(0),
5459 reconnect_max_attempts: None,
5460 heartbeat_timeout_secs: None,
5461 idle_timeout_ms: None,
5462 writer_capacity: None,
5463 backend: TransportBackend::Tungstenite,
5464 proxy_url: None,
5465 max_message_size_bytes: None,
5466 max_frame_size_bytes: None,
5467 };
5468
5469 let client = WebSocketClient::builder()
5470 .config(config)
5471 .message_handler(Arc::new(|_| {}))
5472 .connect()
5473 .await
5474 .unwrap();
5475
5476 let initial = header_rx.recv().await.unwrap();
5477 let reconnect_headers = client.reconnect_headers();
5478 reconnect_headers
5479 .update("authorization", "Bearer refreshed")
5480 .unwrap();
5481
5482 tokio::time::sleep(Duration::from_millis(100)).await;
5483 assert_eq!(initial, vec!["Bearer initial"]);
5484 assert!(client.is_active());
5485 assert!(header_rx.try_recv().is_err());
5486 assert!(!format!("{reconnect_headers:?}").contains("refreshed"));
5487
5488 client.send_text("close-now".into(), None).await.unwrap();
5489 let refreshed = tokio::time::timeout(Duration::from_secs(3), header_rx.recv())
5490 .await
5491 .unwrap()
5492 .unwrap();
5493
5494 assert_eq!(refreshed, vec!["Bearer refreshed"]);
5495
5496 client.disconnect().await;
5497 server_task.abort();
5498 }
5499
5500 #[tokio::test]
5501 async fn test_rate_limiter() {
5502 let server = TestServer::setup().await;
5503 let quota = Quota::per_second(NonZeroU32::new(2).unwrap()).unwrap();
5504
5505 let config = WebSocketConfig {
5506 url: format!("ws://127.0.0.1:{}", server.port),
5507 headers: vec![("test".into(), "test".into())],
5508 heartbeat_interval_secs: None,
5509 heartbeat_payload: None,
5510 connect_timeout_ms: None,
5511 reconnect_delay_initial_ms: None,
5512 reconnect_backoff_factor: None,
5513 reconnect_delay_max_ms: None,
5514 reconnect_jitter_ms: None,
5515 reconnect_max_attempts: None,
5516 heartbeat_timeout_secs: None,
5517 idle_timeout_ms: None,
5518 writer_capacity: None,
5519 backend: TransportBackend::Tungstenite,
5520 proxy_url: None,
5521 max_message_size_bytes: None,
5522 max_frame_size_bytes: None,
5523 };
5524
5525 let client = WebSocketClient::builder()
5526 .config(config)
5527 .message_handler(Arc::new(|_| {}))
5528 .keyed_quotas(vec![("default".into(), quota)])
5529 .connect()
5530 .await
5531 .unwrap();
5532
5533 let keys: [ustr::Ustr; 1] = [ustr::Ustr::from("default")];
5536 let start = std::time::Instant::now();
5537 client
5538 .send_text("test1".into(), Some(keys.as_slice()))
5539 .await
5540 .unwrap();
5541 client
5542 .send_text("test2".into(), Some(keys.as_slice()))
5543 .await
5544 .unwrap();
5545 let after_burst = start.elapsed();
5546 client
5547 .send_text("test3".into(), Some(keys.as_slice()))
5548 .await
5549 .unwrap();
5550 let after_third = start.elapsed();
5551
5552 assert!(
5553 after_burst < std::time::Duration::from_millis(300),
5554 "Burst sends should not be rate limited, took {after_burst:?}"
5555 );
5556 assert!(
5557 after_third >= std::time::Duration::from_millis(400),
5558 "Third send should wait for quota replenishment, took {after_third:?}"
5559 );
5560
5561 client.disconnect().await;
5563 assert!(client.is_disconnected());
5564 }
5565
5566 #[tokio::test]
5567 async fn test_concurrent_writers() {
5568 let server = TestServer::setup().await;
5569 let client = Arc::new(setup_test_client(server.port).await);
5570
5571 let mut handles = vec![];
5572
5573 for i in 0..10 {
5574 let client = Arc::clone(&client);
5575 handles.push(task::spawn(async move {
5576 client.send_text(format!("test{i}"), None).await.unwrap();
5577 }));
5578 }
5579
5580 for handle in handles {
5581 handle.await.unwrap();
5582 }
5583
5584 client.disconnect().await;
5586 assert!(client.is_disconnected());
5587 }
5588}
5589
5590#[cfg(test)]
5591#[cfg(not(feature = "turmoil"))]
5592#[cfg(not(all(feature = "simulation", madsim)))] mod rust_tests {
5594 use std::{
5595 pin::Pin,
5596 sync::{
5597 Arc, OnceLock,
5598 atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
5599 },
5600 task::{Context, Poll},
5601 };
5602
5603 use futures_util::{SinkExt, StreamExt};
5604 #[cfg(target_os = "linux")]
5605 use log::Level;
5606 use nautilus_common::testing::wait_until_async;
5607 use parking_lot::{Condvar, Mutex};
5608 use rstest::rstest;
5609 #[cfg(feature = "transport-sockudo")]
5610 use sockudo_ws::handshake as sockudo_handshake;
5611 #[cfg(feature = "transport-sockudo")]
5612 use tokio::io::AsyncRead;
5613 use tokio::{
5614 io::{AsyncReadExt, AsyncWriteExt},
5615 net::TcpListener,
5616 sync::oneshot,
5617 task::{self, JoinHandle},
5618 time::{Duration, sleep},
5619 };
5620 use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
5621 #[cfg(feature = "transport-sockudo")]
5622 use tokio_tungstenite::{
5623 accept_hdr_async,
5624 tungstenite::{
5625 handshake::server::{self, Callback},
5626 http::HeaderValue,
5627 },
5628 };
5629
5630 use super::*;
5631 #[cfg(target_os = "linux")]
5632 use crate::logging::tests::capture_logs_for;
5633 use crate::{
5634 SocketState,
5635 websocket::types::{channel_epoch_message_handler, channel_message_handler},
5636 };
5637
5638 const TEST_TIMEOUT: Duration = Duration::from_secs(10);
5639
5640 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5643 const RECONNECT_LOG_TARGETS: &[&str] = &[
5644 "nautilus_network::transport::sockudo",
5645 "nautilus_network::websocket::client",
5646 ];
5647
5648 #[cfg(target_os = "linux")]
5649 #[rstest]
5650 #[case::tungstenite(TransportBackend::Tungstenite)]
5651 #[cfg_attr(
5652 feature = "transport-sockudo",
5653 case::sockudo(TransportBackend::Sockudo)
5654 )]
5655 #[tokio::test]
5656 async fn test_initial_connect_redacts_error_text(#[case] backend: TransportBackend) {
5657 let capture = capture_logs_for(&["nautilus_network::retry"]).await;
5658 let socket = tokio::net::TcpSocket::new_v4().unwrap();
5659 socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
5660 let mut config = reconnect_test_config(socket.local_addr().unwrap().port());
5661 config.backend = backend;
5662 let error = WebSocketClient::builder()
5663 .config(config)
5664 .message_handler(Arc::new(|_| {}))
5665 .connect()
5666 .await
5667 .unwrap_err();
5668 let messages = capture.messages();
5669 let raw_message = error.to_string();
5670 let redacted_message = error.redacted_message();
5671
5672 assert!(
5673 matches!(error, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::ConnectionRefused)
5674 );
5675 assert_ne!(raw_message, redacted_message);
5676 assert_eq!(redacted_message, "I/O error: ConnectionRefused");
5677 assert!(messages.contains(&(
5678 Level::Trace,
5679 format!(
5680 "Operation '{INITIAL_CONNECT_OPERATION}' retries exhausted after 1 attempts: {redacted_message}"
5681 ),
5682 )), "{messages:?}");
5683 assert!(
5684 messages
5685 .iter()
5686 .all(|(_, message)| !message.contains(&raw_message)),
5687 "{messages:?}"
5688 );
5689 }
5690
5691 #[cfg(target_os = "linux")]
5692 #[rstest]
5693 #[case::tungstenite(TransportBackend::Tungstenite)]
5694 #[cfg_attr(
5695 feature = "transport-sockudo",
5696 case::sockudo(TransportBackend::Sockudo)
5697 )]
5698 #[tokio::test]
5699 async fn test_reconnect_redacts_error_text(#[case] backend: TransportBackend) {
5700 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5701 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5702 let port = listener.local_addr().unwrap().port();
5703 let (close_tx, close_rx) = oneshot::channel();
5704
5705 let server = tokio::spawn(async move {
5706 let (stream, _) = listener.accept().await.unwrap();
5707 let ws_client = accept_async(stream).await.unwrap();
5708 close_rx.await.unwrap();
5709 drop(ws_client);
5710 let (_stream, _) = listener.accept().await.unwrap();
5711 std::future::pending::<()>().await;
5712 });
5713
5714 let mut config = reconnect_test_config(port);
5715 config.backend = backend;
5716 config.reconnect_max_attempts = Some(1);
5717 config.reconnect_delay_initial_ms = Some(1);
5718 config.reconnect_delay_max_ms = Some(1);
5719 config.reconnect_jitter_ms = Some(0);
5720 let client = WebSocketClient::builder()
5721 .config(config)
5722 .message_handler(Arc::new(|_| {}))
5723 .connect()
5724 .await
5725 .unwrap();
5726 close_tx.send(()).unwrap();
5727
5728 wait_until_async(
5729 || async {
5730 capture.messages().iter().any(|(level, message)| {
5731 *level == Level::Warn && message.starts_with("Reconnect attempt 1 failed:")
5732 })
5733 },
5734 TEST_TIMEOUT,
5735 )
5736 .await;
5737
5738 client.disconnect().await;
5739 server.abort();
5740 await_task_termination(server, "redaction peer").await;
5741 let messages = capture.messages();
5742 let warnings: Vec<_> = messages
5743 .iter()
5744 .filter(|(_, message)| message.starts_with("Reconnect attempt "))
5745 .cloned()
5746 .collect();
5747
5748 assert_eq!(
5749 warnings,
5750 vec![(
5751 Level::Warn,
5752 "Reconnect attempt 1 failed: I/O error: TimedOut".into()
5753 )],
5754 );
5755 }
5756
5757 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5758 #[tokio::test]
5759 async fn test_sockudo_handshake_trace_redacts_authentication() {
5760 let capture = capture_logs_for(&[
5761 "tungstenite::handshake::client",
5762 "nautilus_network::transport::sockudo",
5763 "nautilus_network::retry",
5764 "nautilus_network::websocket::client",
5765 ])
5766 .await;
5767 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5768 let addr = listener.local_addr().unwrap();
5769 let peer = tokio::spawn(async move {
5770 let (mut stream, _) = listener.accept().await.unwrap();
5771 let request = read_http_request(&mut stream).await;
5772 stream
5773 .write_all(
5774 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
5775 )
5776 .await
5777 .unwrap();
5778 stream.flush().await.unwrap();
5779 request
5780 });
5781
5782 let config = WebSocketConfig::builder()
5783 .url(format!(
5784 "ws://{addr}/TRACE_PATH_SECRET?token=TRACE_QUERY_SECRET"
5785 ))
5786 .headers(vec![
5787 ("KALSHI-ACCESS-KEY".into(), "TRACE_KEY_SECRET".into()),
5788 (
5789 "KALSHI-ACCESS-SIGNATURE".into(),
5790 "TRACE_SIGNATURE_SECRET".into(),
5791 ),
5792 ])
5793 .backend(TransportBackend::Sockudo)
5794 .build()
5795 .unwrap();
5796 let error = WebSocketClient::builder()
5797 .config(config)
5798 .message_handler(Arc::new(|_| {}))
5799 .connect()
5800 .await
5801 .unwrap_err();
5802 let request = String::from_utf8(peer.await.unwrap()).unwrap();
5803 let messages = capture.messages();
5804
5805 assert!(matches!(error, TransportError::UpgradeRejected(503)));
5806
5807 for secret in [
5808 "TRACE_KEY_SECRET",
5809 "TRACE_SIGNATURE_SECRET",
5810 "TRACE_PATH_SECRET",
5811 "TRACE_QUERY_SECRET",
5812 ] {
5813 assert!(request.contains(secret));
5814 assert!(
5815 messages
5816 .iter()
5817 .all(|(_, message)| !message.contains(secret)),
5818 "{messages:?}"
5819 );
5820 }
5821
5822 assert!(
5823 messages
5824 .iter()
5825 .any(|(level, message)| *level == log::Level::Trace
5826 && message.contains("retries exhausted after 1 attempts")),
5827 "{messages:?}"
5828 );
5829 }
5830
5831 struct CondvarReleaseGuard<'a> {
5832 release: &'a (Mutex<bool>, Condvar),
5833 }
5834
5835 impl<'a> CondvarReleaseGuard<'a> {
5836 fn new(release: &'a (Mutex<bool>, Condvar)) -> Self {
5837 Self { release }
5838 }
5839
5840 fn release(&self) {
5841 let (lock, condvar) = self.release;
5842 let mut released = lock.lock();
5843 *released = true;
5844 condvar.notify_all();
5845 }
5846 }
5847
5848 impl Drop for CondvarReleaseGuard<'_> {
5849 fn drop(&mut self) {
5850 self.release();
5851 }
5852 }
5853
5854 async fn recv_rendezvous<T: Send + 'static>(
5855 receiver: std::sync::mpsc::Receiver<T>,
5856 name: &'static str,
5857 ) -> T {
5858 let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
5859
5860 match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
5861 Ok(Ok(Ok(value))) => value,
5862 Ok(Ok(Err(e))) => {
5863 panic!("{name} did not arrive within the test timeout: {e}")
5864 }
5865 Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
5866 Err(e) => panic!("{name} receive task did not finish: {e}"),
5867 }
5868 }
5869
5870 async fn await_task_termination(task: tokio::task::JoinHandle<()>, name: &'static str) {
5871 match tokio::time::timeout(TEST_TIMEOUT, task).await {
5872 Ok(Ok(())) => {}
5873 Ok(Err(e)) if e.is_cancelled() => {}
5874 Ok(Err(e)) => panic!("{name} failed: {e}"),
5875 Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
5876 }
5877 }
5878
5879 fn reconnect_test_config(port: u16) -> WebSocketConfig {
5880 WebSocketConfig {
5881 url: format!("ws://127.0.0.1:{port}"),
5882 headers: vec![],
5883 heartbeat_interval_secs: None,
5884 heartbeat_payload: None,
5885 connect_timeout_ms: Some(1_000),
5886 reconnect_delay_initial_ms: None,
5887 reconnect_delay_max_ms: None,
5888 reconnect_backoff_factor: None,
5889 reconnect_jitter_ms: None,
5890 reconnect_max_attempts: None,
5891 heartbeat_timeout_secs: None,
5892 idle_timeout_ms: None,
5893 writer_capacity: None,
5894 backend: TransportBackend::Tungstenite,
5895 proxy_url: None,
5896 max_message_size_bytes: None,
5897 max_frame_size_bytes: None,
5898 }
5899 }
5900
5901 fn initial_connect_retry_policy(
5902 max_attempts: u32,
5903 initial_delay_ms: u64,
5904 ) -> InitialConnectRetryPolicy {
5905 InitialConnectRetryPolicy {
5906 max_attempts: std::num::NonZeroU32::new(max_attempts).unwrap(),
5907 delay_initial: Duration::from_millis(initial_delay_ms),
5908 delay_max: Duration::from_millis(initial_delay_ms),
5909 backoff_factor: 2.0,
5910 jitter_ms: 0,
5911 }
5912 }
5913
5914 #[rstest]
5915 #[case(TransportError::UpgradeRejected(408), true)]
5916 #[case(TransportError::UpgradeRejected(425), true)]
5917 #[case(TransportError::UpgradeRejected(429), true)]
5918 #[case(TransportError::UpgradeRejected(500), true)]
5919 #[case(TransportError::UpgradeRejected(599), true)]
5920 #[case(TransportError::UpgradeRejected(404), false)]
5921 #[case(TransportError::UpgradeRejected(600), false)]
5922 #[case(TransportError::ProxyConnectRejected(429), true)]
5923 #[case(TransportError::ProxyConnectRejected(407), false)]
5924 #[case(TransportError::ConnectionClosed, true)]
5925 #[case(
5926 TransportError::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
5927 true
5928 )]
5929 #[case(
5930 TransportError::Io(std::io::Error::from(std::io::ErrorKind::Interrupted)),
5931 true
5932 )]
5933 #[case(
5934 TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidInput)),
5935 false
5936 )]
5937 #[case(
5938 TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidData)),
5939 false
5940 )]
5941 #[case(
5942 TransportError::Io(std::io::Error::from(std::io::ErrorKind::Unsupported)),
5943 false
5944 )]
5945 #[case(
5946 TransportError::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
5947 false
5948 )]
5949 #[case(TransportError::Handshake("malformed".to_string()), false)]
5950 fn initial_connect_error_classification(#[case] error: TransportError, #[case] expected: bool) {
5951 assert_eq!(is_retryable_initial_connect_error(&error), expected);
5952 }
5953
5954 async fn rejected_upgrade_attempts(
5955 status: u16,
5956 backend: TransportBackend,
5957 retry_policy: Option<InitialConnectRetryPolicy>,
5958 ) -> u64 {
5959 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5960 let port = listener.local_addr().unwrap().port();
5961 let accepted = Arc::new(AtomicU64::new(0));
5962 let accepted_server = Arc::clone(&accepted);
5963
5964 let server = tokio::spawn(async move {
5965 loop {
5966 let (mut stream, _) = listener.accept().await.unwrap();
5967 accepted_server.fetch_add(1, Ordering::SeqCst);
5968 let mut request = Vec::new();
5969
5970 loop {
5971 let mut chunk = [0; 1024];
5972 let n = stream.read(&mut chunk).await.unwrap();
5973 if n == 0 {
5974 break;
5975 }
5976
5977 request.extend_from_slice(&chunk[..n]);
5978 if request.windows(4).any(|window| window == b"\r\n\r\n") {
5979 break;
5980 }
5981 }
5982
5983 stream
5984 .write_all(format!("HTTP/1.1 {status}\r\n\r\n").as_bytes())
5985 .await
5986 .unwrap();
5987 }
5988 });
5989
5990 let (handler, _rx) = channel_message_handler();
5991 let mut config = reconnect_test_config(port);
5992 config.backend = backend;
5993 WebSocketClient::builder()
5994 .config(config)
5995 .message_handler(handler)
5996 .maybe_initial_connect_retry_policy(retry_policy)
5997 .connect()
5998 .await
5999 .expect_err("server always rejects the upgrade");
6000
6001 let attempts = accepted.load(Ordering::SeqCst);
6002 server.abort();
6003 attempts
6004 }
6005
6006 async fn closed_proxy_attempts() -> u64 {
6007 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6008 let proxy_addr = listener.local_addr().unwrap();
6009 let accepted = Arc::new(AtomicU64::new(0));
6010 let accepted_server = Arc::clone(&accepted);
6011
6012 let server = tokio::spawn(async move {
6013 loop {
6014 let (mut stream, _) = listener.accept().await.unwrap();
6015 accepted_server.fetch_add(1, Ordering::SeqCst);
6016 let mut request = [0; 1024];
6017 let _ = stream.read(&mut request).await.unwrap();
6018 }
6019 });
6020
6021 let (handler, _rx) = channel_message_handler();
6022 let mut config = reconnect_test_config(9);
6023 config.proxy_url = Some(format!("http://{proxy_addr}"));
6024 WebSocketClient::builder()
6025 .config(config)
6026 .message_handler(handler)
6027 .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6028 .connect()
6029 .await
6030 .expect_err("proxy always closes before its CONNECT response");
6031
6032 let attempts = accepted.load(Ordering::SeqCst);
6033 server.abort();
6034 attempts
6035 }
6036
6037 #[rstest]
6038 #[tokio::test]
6039 async fn initial_connect_cancellation_interrupts_in_flight_handshake() {
6040 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6041 let port = listener.local_addr().unwrap().port();
6042 let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
6043
6044 let server = tokio::spawn(async move {
6045 let (_stream, _) = listener.accept().await.unwrap();
6046 let _ = accepted_tx.send(());
6047 std::future::pending::<()>().await;
6048 });
6049
6050 let token = CancellationToken::new();
6051 let (handler, _rx) = channel_message_handler();
6052 let mut config = reconnect_test_config(port);
6053 config.connect_timeout_ms = Some(10_000);
6054 let connect = WebSocketClient::builder()
6055 .config(config)
6056 .message_handler(handler)
6057 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6058 .cancellation_token(token.clone())
6059 .connect();
6060 tokio::pin!(connect);
6061
6062 tokio::select! {
6063 biased;
6064 result = &mut connect => panic!("connect completed before cancellation: {result:?}"),
6065 result = accepted_rx => result.unwrap(),
6066 }
6067 token.cancel();
6068
6069 let err = tokio::time::timeout(Duration::from_millis(250), connect)
6070 .await
6071 .expect("cancellation should interrupt the handshake")
6072 .expect_err("cancelled connect should fail");
6073 assert!(
6074 matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::Interrupted)
6075 );
6076 server.abort();
6077 }
6078
6079 #[rstest]
6080 #[tokio::test]
6081 async fn timed_out_initial_connect_attempt_is_retried() {
6082 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6083 let port = listener.local_addr().unwrap().port();
6084 let accepted = Arc::new(AtomicU64::new(0));
6085 let accepted_server = Arc::clone(&accepted);
6086
6087 let server = tokio::spawn(async move {
6090 let mut held = Vec::new();
6091
6092 loop {
6093 let (stream, _) = listener.accept().await.unwrap();
6094 accepted_server.fetch_add(1, Ordering::SeqCst);
6095 held.push(stream);
6096 }
6097 });
6098
6099 let (handler, _rx) = channel_message_handler();
6100 let mut config = reconnect_test_config(port);
6101 config.connect_timeout_ms = Some(50);
6102 let err = WebSocketClient::builder()
6103 .config(config)
6104 .message_handler(handler)
6105 .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6106 .connect()
6107 .await
6108 .expect_err("every attempt times out");
6109
6110 assert!(
6111 matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::TimedOut)
6112 );
6113
6114 let counted = tokio::time::timeout(Duration::from_secs(1), async {
6119 while accepted.load(Ordering::SeqCst) < 3 {
6120 tokio::time::sleep(Duration::from_millis(5)).await;
6121 }
6122 })
6123 .await;
6124
6125 assert!(
6126 counted.is_ok(),
6127 "expected 3 connection attempts, observed {}",
6128 accepted.load(Ordering::SeqCst)
6129 );
6130 assert_eq!(accepted.load(Ordering::SeqCst), 3);
6133 server.abort();
6134 }
6135
6136 #[rstest]
6137 #[case::too_many_requests(429)]
6138 #[case::server_error(503)]
6139 #[tokio::test]
6140 async fn transient_upgrade_rejection_is_retried(#[case] status: u16) {
6141 assert_eq!(
6142 rejected_upgrade_attempts(
6143 status,
6144 TransportBackend::Tungstenite,
6145 Some(initial_connect_retry_policy(3, 1)),
6146 )
6147 .await,
6148 3
6149 );
6150 }
6151
6152 #[rstest]
6153 #[tokio::test]
6154 async fn initial_connect_without_retry_policy_makes_one_attempt() {
6155 assert_eq!(
6156 rejected_upgrade_attempts(503, TransportBackend::Tungstenite, None).await,
6157 1
6158 );
6159 }
6160
6161 #[cfg(feature = "transport-sockudo")]
6162 #[rstest]
6163 #[tokio::test]
6164 async fn sockudo_too_many_requests_upgrade_rejection_is_retried() {
6165 assert_eq!(
6166 rejected_upgrade_attempts(
6167 429,
6168 TransportBackend::Sockudo,
6169 Some(initial_connect_retry_policy(3, 1)),
6170 )
6171 .await,
6172 3
6173 );
6174 }
6175
6176 #[rstest]
6177 #[tokio::test]
6178 async fn permanent_upgrade_rejection_is_not_retried() {
6179 assert_eq!(
6180 rejected_upgrade_attempts(
6181 404,
6182 TransportBackend::Tungstenite,
6183 Some(initial_connect_retry_policy(3, 1)),
6184 )
6185 .await,
6186 1
6187 );
6188 }
6189
6190 #[rstest]
6191 #[tokio::test]
6192 async fn proxy_close_before_connect_response_is_retried() {
6193 assert_eq!(closed_proxy_attempts().await, 3);
6194 }
6195
6196 #[rstest]
6197 #[tokio::test]
6198 async fn initial_connect_cancellation_interrupts_retry_backoff() {
6199 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6200 let port = listener.local_addr().unwrap().port();
6201 let accepted = Arc::new(AtomicU64::new(0));
6202 let accepted_server = Arc::clone(&accepted);
6203 let (rejected_tx, rejected_rx) = oneshot::channel();
6204 let server = tokio::spawn(async move {
6205 let mut rejected_tx = Some(rejected_tx);
6206
6207 loop {
6208 let (mut stream, _) = listener.accept().await.unwrap();
6209 accepted_server.fetch_add(1, Ordering::SeqCst);
6210 let mut request = [0; 1024];
6211 let _ = stream.read(&mut request).await.unwrap();
6212 stream
6213 .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
6214 .await
6215 .unwrap();
6216
6217 if let Some(rejected_tx) = rejected_tx.take() {
6218 rejected_tx.send(()).unwrap();
6219 }
6220 }
6221 });
6222
6223 let token = CancellationToken::new();
6224 let (handler, _rx) = channel_message_handler();
6225 let connect = WebSocketClient::builder()
6226 .config(reconnect_test_config(port))
6227 .message_handler(handler)
6228 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6229 .cancellation_token(token.clone())
6230 .connect();
6231 tokio::pin!(connect);
6232
6233 tokio::select! {
6234 biased;
6235 result = &mut connect => panic!("connect completed before backoff: {result:?}"),
6236 result = rejected_rx => result.unwrap(),
6237 }
6238 assert!(futures_util::poll!(&mut connect).is_pending());
6239 token.cancel();
6240
6241 let error = tokio::time::timeout(Duration::from_millis(250), connect)
6242 .await
6243 .expect("cancellation should interrupt retry backoff")
6244 .expect_err("cancelled connect should fail");
6245
6246 assert!(
6247 matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6248 );
6249 assert_eq!(accepted.load(Ordering::SeqCst), 1);
6250 server.abort();
6251 }
6252
6253 #[rstest]
6254 #[tokio::test]
6255 async fn permanent_initial_connect_error_does_not_wait_for_retry_ladder() {
6256 let (handler, _rx) = channel_message_handler();
6257 let mut config = reconnect_test_config(1);
6258 config.url = "not a websocket URL".to_string();
6259 let connect = WebSocketClient::builder()
6260 .config(config)
6261 .message_handler(handler)
6262 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6263 .connect();
6264
6265 let err = tokio::time::timeout(Duration::from_millis(250), connect)
6266 .await
6267 .expect("permanent error should not enter retry backoff")
6268 .expect_err("invalid URL should fail");
6269 assert!(matches!(
6270 err,
6271 TransportError::InvalidUrl(_) | TransportError::Handshake(_)
6272 ));
6273 }
6274
6275 #[rstest]
6276 #[tokio::test(start_paused = true)]
6277 async fn connection_rate_limit_gates_initial_connect_and_reconnect() {
6278 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6279 let port = listener.local_addr().unwrap().port();
6280 let server = tokio::spawn(async move {
6281 let (stream, _) = listener.accept().await.unwrap();
6282 let _first = accept_async(stream).await.unwrap();
6283 let (stream, _) = listener.accept().await.unwrap();
6284 let _second = accept_async(stream).await.unwrap();
6285 std::future::pending::<()>().await;
6286 });
6287
6288 let key = Ustr::from("connection-attempt");
6289
6290 let limiter = Arc::new(RateLimiter::new_with_quota(
6291 None,
6292 vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6293 ));
6294 limiter.await_keys_ready(Some(&[key])).await;
6295
6296 let rate_limit = ConnectionRateLimit {
6297 limiter,
6298 keys: Arc::from([key]),
6299 };
6300
6301 let (handler, _rx) = channel_message_handler();
6302 let mut config = reconnect_test_config(port);
6303 config.connect_timeout_ms = Some(10_000);
6304 let initial = WebSocketClientInner::connect_url_with_handler(
6305 config,
6306 Some(IncomingHandler::Message(handler)),
6307 None,
6308 None,
6309 None,
6310 Some(rate_limit),
6311 InitialConnectOptions::default(),
6312 );
6313 tokio::pin!(initial);
6314 assert!(futures_util::poll!(&mut initial).is_pending());
6315 tokio::time::advance(Duration::from_secs(1)).await;
6316 tokio::task::yield_now().await;
6317
6318 tokio::time::resume();
6321 let mut inner = initial.await.unwrap();
6322 tokio::time::pause();
6323
6324 inner
6325 .connection_mode
6326 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6327 let reconnect = inner.reconnect_with_outcome();
6328 tokio::pin!(reconnect);
6329 assert!(futures_util::poll!(&mut reconnect).is_pending());
6330 tokio::time::advance(Duration::from_secs(1)).await;
6331 tokio::task::yield_now().await;
6332 tokio::time::resume();
6333 assert_eq!(reconnect.await.unwrap(), ReconnectOutcome::Reconnected);
6334
6335 server.abort();
6336 }
6337
6338 #[rstest]
6339 #[tokio::test(start_paused = true)]
6340 async fn initial_connect_cancellation_interrupts_connection_rate_limit_wait() {
6341 let key = Ustr::from("connection-attempt");
6342
6343 let limiter = Arc::new(RateLimiter::new_with_quota(
6344 None,
6345 vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6346 ));
6347 limiter.await_keys_ready(Some(&[key])).await;
6348
6349 let rate_limit = ConnectionRateLimit {
6350 limiter,
6351 keys: Arc::from([key]),
6352 };
6353
6354 let token = CancellationToken::new();
6355 let (handler, _rx) = channel_message_handler();
6356
6357 let connect = WebSocketClientInner::connect_url_with_handler(
6358 reconnect_test_config(1),
6359 Some(IncomingHandler::Message(handler)),
6360 None,
6361 None,
6362 None,
6363 Some(rate_limit),
6364 InitialConnectOptions {
6365 retry_policy: None,
6366 cancellation_token: Some(token.clone()),
6367 },
6368 );
6369
6370 tokio::pin!(connect);
6371 assert!(futures_util::poll!(&mut connect).is_pending());
6372
6373 token.cancel();
6374 let error = connect.await.expect_err("cancelled connect should fail");
6375
6376 assert!(
6377 matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6378 );
6379 }
6380
6381 #[rstest]
6382 #[tokio::test]
6383 async fn test_reconnect_outcome_is_aborted_before_connect() {
6384 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6385 let port = listener.local_addr().unwrap().port();
6386 let server = tokio::spawn(async move {
6387 let (stream, _) = listener.accept().await.unwrap();
6388 let _websocket = accept_async(stream).await.unwrap();
6389 std::future::pending::<()>().await;
6390 });
6391
6392 let (handler, _rx) = channel_message_handler();
6393 let mut inner =
6394 WebSocketClientInner::connect_url(reconnect_test_config(port), Some(handler), None)
6395 .await
6396 .unwrap();
6397 inner
6398 .connection_mode
6399 .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
6400
6401 let outcome = inner.reconnect_with_outcome().await.unwrap();
6402
6403 assert_eq!(outcome, ReconnectOutcome::Aborted);
6404 server.abort();
6405 }
6406
6407 #[rstest]
6408 #[tokio::test]
6409 async fn test_stream_reconnect_outcome_is_aborted_and_notifies_closed() {
6410 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6411 let port = listener.local_addr().unwrap().port();
6412 let server = tokio::spawn(async move {
6413 let (stream, _) = listener.accept().await.unwrap();
6414 let _websocket = accept_async(stream).await.unwrap();
6415 std::future::pending::<()>().await;
6416 });
6417
6418 let mut inner = WebSocketClientInner::connect_url(reconnect_test_config(port), None, None)
6419 .await
6420 .unwrap();
6421 let state_notify = Arc::clone(&inner.state_notify);
6422 let mut notified = std::pin::pin!(state_notify.notified());
6423 notified.as_mut().enable();
6424
6425 let outcome = inner.reconnect_with_outcome().await.unwrap();
6426
6427 assert_eq!(outcome, ReconnectOutcome::Aborted);
6428 assert_eq!(
6429 ConnectionMode::from_atomic(&inner.connection_mode),
6430 ConnectionMode::Closed
6431 );
6432 tokio::time::timeout(TEST_TIMEOUT, notified)
6433 .await
6434 .expect("stream close notification was not published");
6435 server.abort();
6436 }
6437
6438 #[rstest]
6439 #[tokio::test]
6440 async fn test_reconnect_outcome_is_reconnected_with_handler() {
6441 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6442 let port = listener.local_addr().unwrap().port();
6443 let server = tokio::spawn(async move {
6444 let (stream, _) = listener.accept().await.unwrap();
6445 let _first = accept_async(stream).await.unwrap();
6446 let (stream, _) = listener.accept().await.unwrap();
6447 let mut second = accept_async(stream).await.unwrap();
6448 second
6449 .send(WsMessage::Text("replacement".into()))
6450 .await
6451 .unwrap();
6452 std::future::pending::<()>().await;
6453 });
6454
6455 let (epoch_handler, mut epoch_rx) = channel_epoch_message_handler();
6456 let mut inner = WebSocketClientInner::connect_url_with_handler(
6457 reconnect_test_config(port),
6458 Some(IncomingHandler::Epoch(epoch_handler)),
6459 None,
6460 None,
6461 None,
6462 None,
6463 InitialConnectOptions::default(),
6464 )
6465 .await
6466 .unwrap();
6467 inner
6468 .connection_mode
6469 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6470
6471 let outcome = inner.reconnect_with_outcome().await.unwrap();
6472
6473 assert_eq!(outcome, ReconnectOutcome::Reconnected);
6474 assert_eq!(
6475 ConnectionMode::from_atomic(&inner.connection_mode),
6476 ConnectionMode::Active
6477 );
6478 let (epoch, message) = tokio::time::timeout(TEST_TIMEOUT, epoch_rx.recv())
6479 .await
6480 .expect("replacement epoch message was not delivered")
6481 .expect("epoch handler channel closed");
6482 assert_eq!(epoch, 1);
6483 assert_eq!(message, WsMessage::Text("replacement".into()));
6484 server.abort();
6485 }
6486
6487 #[rstest]
6488 #[tokio::test]
6489 async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
6490 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6491 let port = listener.local_addr().unwrap().port();
6492 let server = tokio::spawn(async move {
6493 let (stream, _) = listener.accept().await.unwrap();
6494 let _websocket = accept_async(stream).await.unwrap();
6495 std::future::pending::<()>().await;
6496 });
6497
6498 let mut config = reconnect_test_config(port);
6499 config.heartbeat_interval_secs = Some(60);
6500 let (handler, _handler_rx) = channel_message_handler();
6501 let inner = WebSocketClientInner::connect_url(config, Some(handler), None)
6502 .await
6503 .unwrap();
6504 let read_fence = inner
6505 .read_fence
6506 .clone()
6507 .expect("read fence should exist in handler mode");
6508 let read_abort = inner
6509 .read_task
6510 .as_ref()
6511 .expect("read task should be spawned in handler mode")
6512 .abort_handle();
6513 let write_abort = inner.write_task.abort_handle();
6514 let heartbeat_abort = inner
6515 .heartbeat_task
6516 .as_ref()
6517 .expect("heartbeat task should be spawned for a configured heartbeat")
6518 .abort_handle();
6519
6520 assert!(read_fence.is_valid(), "read fence should start valid");
6521 assert!(
6522 !read_abort.is_finished(),
6523 "read task should be running before drop"
6524 );
6525 assert!(
6526 !write_abort.is_finished(),
6527 "write task should be running before drop"
6528 );
6529 assert!(
6530 !heartbeat_abort.is_finished(),
6531 "heartbeat task should be running before drop"
6532 );
6533
6534 drop(inner);
6535 wait_until_async(
6536 || async {
6537 read_abort.is_finished()
6538 && write_abort.is_finished()
6539 && heartbeat_abort.is_finished()
6540 },
6541 TEST_TIMEOUT,
6542 )
6543 .await;
6544
6545 assert!(!read_fence.is_valid(), "read fence was not invalidated");
6546 assert!(read_abort.is_finished(), "read task was not aborted");
6547 assert!(write_abort.is_finished(), "write task was not aborted");
6548 assert!(
6549 heartbeat_abort.is_finished(),
6550 "heartbeat task was not aborted"
6551 );
6552 server.abort();
6553 }
6554
6555 struct RecordingServer {
6556 task: JoinHandle<()>,
6557 port: u16,
6558 messages: Arc<tokio::sync::Mutex<Vec<String>>>,
6559 binary_frames: Arc<tokio::sync::Mutex<Vec<Vec<u8>>>>,
6560 control_frames: Arc<tokio::sync::Mutex<Vec<String>>>,
6561 connections: Arc<AtomicUsize>,
6562 }
6563
6564 #[cfg(feature = "transport-sockudo")]
6565 async fn read_http_request<S>(stream: &mut S) -> Vec<u8>
6566 where
6567 S: AsyncRead + Unpin,
6568 {
6569 let mut buf = Vec::new();
6570 let mut chunk = [0u8; 256];
6571 loop {
6572 let n = stream.read(&mut chunk).await.unwrap();
6573 assert!(n > 0, "HTTP request closed before headers completed");
6574 buf.extend_from_slice(&chunk[..n]);
6575 if buf.windows(4).any(|window| window == b"\r\n\r\n") {
6576 return buf;
6577 }
6578 }
6579 }
6580
6581 #[cfg(feature = "transport-sockudo")]
6582 fn extract_header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
6583 request.lines().find_map(|line| {
6584 let (header_name, header_value) = line.split_once(':')?;
6585 if header_name.eq_ignore_ascii_case(name) {
6586 Some(header_value.trim())
6587 } else {
6588 None
6589 }
6590 })
6591 }
6592
6593 #[cfg(feature = "transport-sockudo")]
6594 #[derive(Debug, Clone)]
6595 struct HeaderAssertCallback {
6596 key: String,
6597 value: HeaderValue,
6598 }
6599
6600 #[cfg(feature = "transport-sockudo")]
6601 impl Callback for HeaderAssertCallback {
6602 #[expect(
6603 clippy::panic_in_result_fn,
6604 reason = "assertion failures should fail the test"
6605 )]
6606 fn on_request(
6607 self,
6608 request: &server::Request,
6609 response: server::Response,
6610 ) -> Result<server::Response, server::ErrorResponse> {
6611 assert_eq!(request.headers().get(&self.key), Some(&self.value));
6612 Ok(response)
6613 }
6614 }
6615
6616 impl RecordingServer {
6617 async fn setup() -> Self {
6618 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6619 let port = listener.local_addr().unwrap().port();
6620 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6621 let messages_clone = Arc::clone(&messages);
6622 let binary_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6623 let binary_frames_clone = Arc::clone(&binary_frames);
6624 let control_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6625 let control_frames_clone = Arc::clone(&control_frames);
6626 let connections = Arc::new(AtomicUsize::new(0));
6627 let connections_clone = Arc::clone(&connections);
6628
6629 let task = task::spawn(async move {
6630 loop {
6631 let (stream, _) = listener.accept().await.unwrap();
6632 let mut websocket = accept_async(stream).await.unwrap();
6633 connections_clone.fetch_add(1, Ordering::SeqCst);
6634 let messages = Arc::clone(&messages_clone);
6635 let binary_frames = Arc::clone(&binary_frames_clone);
6636 let control_frames = Arc::clone(&control_frames_clone);
6637
6638 task::spawn(async move {
6639 while let Some(Ok(msg)) = websocket.next().await {
6640 match msg {
6641 WsMessage::Text(text) => {
6642 messages.lock().await.push(text.to_string());
6643 }
6644 WsMessage::Binary(data) => {
6645 binary_frames.lock().await.push(data.to_vec());
6646 }
6647 WsMessage::Pong(data) => {
6648 control_frames
6649 .lock()
6650 .await
6651 .push(format!("pong:{}", String::from_utf8_lossy(&data)));
6652 }
6653 WsMessage::Close(_) => {
6654 let _ = websocket.close(None).await;
6655 break;
6656 }
6657 _ => {}
6658 }
6659 }
6660 });
6661 }
6662 });
6663
6664 Self {
6665 task,
6666 port,
6667 messages,
6668 binary_frames,
6669 control_frames,
6670 connections,
6671 }
6672 }
6673
6674 async fn messages(&self) -> Vec<String> {
6675 self.messages.lock().await.clone()
6676 }
6677
6678 async fn binary_frames(&self) -> Vec<Vec<u8>> {
6679 self.binary_frames.lock().await.clone()
6680 }
6681
6682 async fn control_frames(&self) -> Vec<String> {
6683 self.control_frames.lock().await.clone()
6684 }
6685
6686 async fn wait_for_connections(&self, expected: usize) {
6687 wait_until_async(
6688 || async { self.connections.load(Ordering::SeqCst) == expected },
6689 TEST_TIMEOUT,
6690 )
6691 .await;
6692 }
6693 }
6694
6695 impl Drop for RecordingServer {
6696 fn drop(&mut self) {
6697 self.task.abort();
6698 }
6699 }
6700
6701 #[rstest]
6702 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6703 async fn test_manual_reconnect_waits_for_slow_loss_callback_and_new_auth() {
6704 let server = RecordingServer::setup().await;
6705 let tracker = AuthTracker::new();
6706 let _initial_auth = tracker.begin();
6707 tracker.succeed();
6708 let states = Arc::new(Mutex::new(Vec::new()));
6709 let states_callback = Arc::clone(&states);
6710 let auth_at_loss = Arc::new(Mutex::new(Vec::new()));
6711 let auth_at_loss_callback = Arc::clone(&auth_at_loss);
6712 let tracker_callback = tracker.clone();
6713 let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6714 let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6715 let callback_release_clone = Arc::clone(&callback_release);
6716 let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6717
6718 let sink = SocketStateSink::new(move |state| {
6719 states_callback.lock().push(state);
6720 if state == SocketState::Disconnected {
6721 auth_at_loss_callback
6722 .lock()
6723 .push(tracker_callback.auth_state());
6724 callback_entered_tx.send(()).unwrap();
6725 let (lock, condvar) = callback_release_clone.as_ref();
6726 let mut released = lock.lock();
6727 while !*released {
6728 condvar.wait(&mut released);
6729 }
6730 }
6731 });
6732
6733 let (handler, mut handler_rx) = channel_message_handler();
6734 let client = WebSocketClient::builder()
6735 .config(reconnect_test_config(server.port))
6736 .message_handler(handler)
6737 .state_sink(sink)
6738 .connect()
6739 .await
6740 .unwrap();
6741 client.set_auth_tracker(tracker.clone(), true);
6742 server.wait_for_connections(1).await;
6743
6744 let handle = client.reconnect_handle();
6745 let (request_tx, request_rx) = std::sync::mpsc::channel();
6746 let request_thread = std::thread::spawn(move || {
6747 request_tx.send(handle.request_reconnect()).unwrap();
6748 });
6749
6750 recv_rendezvous(callback_entered_rx, "slow reconnect callback entry").await;
6751 client
6752 .writer_tx
6753 .send(WriterCommand::Send(Message::text("buffered")))
6754 .unwrap();
6755 server.wait_for_connections(2).await;
6756 tokio::time::sleep(Duration::from_millis(250)).await;
6757
6758 assert_eq!(client.connection_mode(), ConnectionMode::Reconnect);
6759 assert!(!client.reconnect_published.load(Ordering::SeqCst));
6760 assert_eq!(tracker.auth_state(), AuthState::Unauthenticated);
6761 assert_eq!(*auth_at_loss.lock(), vec![AuthState::Unauthenticated]);
6762 assert_eq!(
6763 *states.lock(),
6764 vec![SocketState::Connected, SocketState::Disconnected]
6765 );
6766 assert!(server.messages().await.is_empty());
6767
6768 callback_release_guard.release();
6769 assert_eq!(
6770 recv_rendezvous(request_rx, "manual reconnect result").await,
6771 ReconnectRequestOutcome::Accepted
6772 );
6773 request_thread.join().unwrap();
6774 wait_until_async(|| async { client.is_active() }, TEST_TIMEOUT).await;
6775 wait_until_async(
6776 || {
6777 let states = Arc::clone(&states);
6778 async move { states.lock().len() == 3 }
6779 },
6780 TEST_TIMEOUT,
6781 )
6782 .await;
6783
6784 let notification = tokio::time::timeout(TEST_TIMEOUT, handler_rx.recv())
6785 .await
6786 .expect("reconnect notification was not delivered")
6787 .expect("handler channel closed");
6788 assert_eq!(notification, WsMessage::Text(RECONNECTED.into()));
6789 assert!(handler_rx.try_recv().is_err());
6790 tokio::time::sleep(Duration::from_millis(200)).await;
6791 assert!(server.messages().await.is_empty());
6792
6793 let _replacement_auth = tracker.begin();
6796 client
6797 .send_bytes_on_connection(b"auth".to_vec(), None, client.connection_epoch())
6798 .await
6799 .unwrap();
6800 wait_until_async(
6801 || {
6802 let binary_frames = Arc::clone(&server.binary_frames);
6803 async move { binary_frames.lock().await.len() == 1 }
6804 },
6805 TEST_TIMEOUT,
6806 )
6807 .await;
6808 assert_eq!(server.binary_frames().await, vec![b"auth".to_vec()]);
6809 assert!(server.messages().await.is_empty());
6810
6811 tracker.succeed();
6812 wait_until_async(
6813 || {
6814 let messages = Arc::clone(&server.messages);
6815 async move { messages.lock().await.len() == 1 }
6816 },
6817 TEST_TIMEOUT,
6818 )
6819 .await;
6820
6821 client.send_text("live".into(), None).await.unwrap();
6822 wait_until_async(
6823 || {
6824 let messages = Arc::clone(&server.messages);
6825 async move { messages.lock().await.len() == 2 }
6826 },
6827 TEST_TIMEOUT,
6828 )
6829 .await;
6830
6831 assert_eq!(server.messages().await, vec!["buffered", "live"]);
6832 assert_eq!(server.connections.load(Ordering::SeqCst), 2);
6833 assert_eq!(
6834 *states.lock(),
6835 vec![
6836 SocketState::Connected,
6837 SocketState::Disconnected,
6838 SocketState::Connected,
6839 ]
6840 );
6841
6842 client.disconnect().await;
6843 }
6844
6845 #[rstest]
6846 #[tokio::test]
6847 async fn test_reconnect_handle_is_closed_after_client_drop() {
6848 let server = RecordingServer::setup().await;
6849 let (handler, _handler_rx) = channel_message_handler();
6850 let client = WebSocketClient::builder()
6851 .config(reconnect_test_config(server.port))
6852 .message_handler(handler)
6853 .connect()
6854 .await
6855 .unwrap();
6856 let tracker = AuthTracker::new();
6857 let pending_auth = tracker.begin();
6858 client.set_auth_tracker(tracker.clone(), true);
6859 let controller_abort = client.controller_task.abort_handle();
6860 let handle = client.reconnect_handle();
6861
6862 drop(client);
6863 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6864
6865 assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
6866 assert_eq!(tracker.auth_state(), AuthState::Failed);
6867 assert_eq!(
6868 tokio::time::timeout(TEST_TIMEOUT, pending_auth)
6869 .await
6870 .expect("client drop should resolve pending authentication")
6871 .expect("authentication sender should report its terminal result"),
6872 Err("WebSocket client closed".to_string())
6873 );
6874 }
6875
6876 #[rstest]
6877 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6878 async fn test_concurrent_drop_closes_accepted_reconnect() {
6879 let server = RecordingServer::setup().await;
6880 let tracker = AuthTracker::new();
6881 let _initial_auth = tracker.begin();
6882 tracker.succeed();
6883 let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6884 let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6885 let callback_release_clone = Arc::clone(&callback_release);
6886 let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6887 let states = Arc::new(Mutex::new(Vec::new()));
6888 let states_callback = Arc::clone(&states);
6889 let sink = SocketStateSink::new(move |state| {
6890 states_callback.lock().push(state);
6891 if state == SocketState::Disconnected {
6892 callback_entered_tx.send(()).unwrap();
6893 let (lock, condvar) = callback_release_clone.as_ref();
6894 let mut released = lock.lock();
6895 while !*released {
6896 condvar.wait(&mut released);
6897 }
6898 }
6899 });
6900
6901 let (handler, _handler_rx) = channel_message_handler();
6902 let client = WebSocketClient::builder()
6903 .config(reconnect_test_config(server.port))
6904 .message_handler(handler)
6905 .state_sink(sink)
6906 .connect()
6907 .await
6908 .unwrap();
6909 client.set_auth_tracker(tracker.clone(), true);
6910 let controller_abort = client.controller_task.abort_handle();
6911 let connection_mode = Arc::clone(&client.connection_mode);
6912 let handle = client.reconnect_handle();
6913 let surviving_handle = handle.clone();
6914 let (request_tx, request_rx) = std::sync::mpsc::channel();
6915 let request_thread = std::thread::spawn(move || {
6916 request_tx.send(handle.request_reconnect()).unwrap();
6917 });
6918
6919 recv_rendezvous(callback_entered_rx, "concurrent drop callback entry").await;
6920 drop(client);
6921 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6922 assert_eq!(
6923 ConnectionMode::from_atomic(&connection_mode),
6924 ConnectionMode::Closed
6925 );
6926 assert_eq!(tracker.auth_state(), AuthState::Failed);
6927
6928 callback_release_guard.release();
6929 assert_eq!(
6930 recv_rendezvous(request_rx, "concurrent drop reconnect result").await,
6931 ReconnectRequestOutcome::Accepted
6932 );
6933 request_thread.join().unwrap();
6934 assert_eq!(
6935 surviving_handle.request_reconnect(),
6936 ReconnectRequestOutcome::Closed
6937 );
6938 assert_eq!(
6939 *states.lock(),
6940 vec![SocketState::Connected, SocketState::Disconnected]
6941 );
6942 }
6943
6944 #[rstest]
6945 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6946 async fn test_reconnect_callback_can_drop_client() {
6947 let server = RecordingServer::setup().await;
6948 let client_slot = Arc::new(Mutex::new(None::<WebSocketClient>));
6949 let client_slot_callback = Arc::clone(&client_slot);
6950 let states = Arc::new(Mutex::new(Vec::new()));
6951 let states_callback = Arc::clone(&states);
6952 let sink = SocketStateSink::new(move |state| {
6953 states_callback.lock().push(state);
6954 if state == SocketState::Disconnected {
6955 drop(client_slot_callback.lock().take());
6956 }
6957 });
6958
6959 let (handler, _handler_rx) = channel_message_handler();
6960 let client = WebSocketClient::builder()
6961 .config(reconnect_test_config(server.port))
6962 .message_handler(handler)
6963 .state_sink(sink)
6964 .connect()
6965 .await
6966 .unwrap();
6967 let tracker = AuthTracker::new();
6968 let _initial_auth = tracker.begin();
6969 tracker.succeed();
6970 client.set_auth_tracker(tracker.clone(), true);
6971 let controller_abort = client.controller_task.abort_handle();
6972 let connection_mode = Arc::clone(&client.connection_mode);
6973 let handle = client.reconnect_handle();
6974 let surviving_handle = handle.clone();
6975 *client_slot.lock() = Some(client);
6976 let (result_tx, result_rx) = std::sync::mpsc::channel();
6977 std::thread::spawn(move || {
6978 result_tx.send(handle.request_reconnect()).unwrap();
6979 });
6980
6981 assert_eq!(
6982 recv_rendezvous(result_rx, "callback drop reconnect result").await,
6983 ReconnectRequestOutcome::Accepted
6984 );
6985 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6986
6987 assert!(client_slot.lock().is_none());
6988 assert_eq!(
6989 ConnectionMode::from_atomic(&connection_mode),
6990 ConnectionMode::Closed
6991 );
6992 assert_eq!(tracker.auth_state(), AuthState::Failed);
6993 assert_eq!(
6994 surviving_handle.request_reconnect(),
6995 ReconnectRequestOutcome::Closed
6996 );
6997 assert_eq!(
6998 *states.lock(),
6999 vec![SocketState::Connected, SocketState::Disconnected]
7000 );
7001 }
7002
7003 #[rstest]
7004 #[tokio::test]
7005 async fn test_reconnect_then_disconnect() {
7006 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7008 let port = listener.local_addr().unwrap().port();
7009
7010 let server = task::spawn(async move {
7012 let (stream, _) = listener.accept().await.unwrap();
7013 let ws = accept_async(stream).await.unwrap();
7014 drop(ws);
7015 sleep(Duration::from_secs(1)).await;
7017 });
7018
7019 let (handler, _rx) = channel_message_handler();
7021
7022 let config = WebSocketConfig {
7024 url: format!("ws://127.0.0.1:{port}"),
7025 headers: vec![],
7026 heartbeat_interval_secs: None,
7027 heartbeat_payload: None,
7028 connect_timeout_ms: Some(1_000),
7029 reconnect_delay_initial_ms: Some(50),
7030 reconnect_delay_max_ms: Some(100),
7031 reconnect_backoff_factor: Some(1.0),
7032 reconnect_jitter_ms: Some(0),
7033 reconnect_max_attempts: None,
7034 heartbeat_timeout_secs: None,
7035 idle_timeout_ms: None,
7036 writer_capacity: None,
7037 backend: TransportBackend::Tungstenite,
7038 proxy_url: None,
7039 max_message_size_bytes: None,
7040 max_frame_size_bytes: None,
7041 };
7042
7043 let client = WebSocketClient::builder()
7045 .config(config)
7046 .message_handler(handler)
7047 .connect()
7048 .await
7049 .unwrap();
7050
7051 sleep(Duration::from_millis(100)).await;
7053 client.disconnect().await;
7055 assert!(client.is_disconnected());
7056 server.abort();
7057 }
7058
7059 #[rstest]
7060 #[tokio::test]
7061 async fn test_reconnect_state_flips_when_reader_stops() {
7062 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7064 let port = listener.local_addr().unwrap().port();
7065 let server = task::spawn(async move {
7066 if let Ok((stream, _)) = listener.accept().await
7067 && let Ok(ws) = accept_async(stream).await
7068 {
7069 drop(ws);
7070 }
7071
7072 sleep(Duration::from_millis(50)).await;
7073 });
7074
7075 let (handler, _rx) = channel_message_handler();
7076
7077 let config = WebSocketConfig {
7078 url: format!("ws://127.0.0.1:{port}"),
7079 headers: vec![],
7080 heartbeat_interval_secs: None,
7081 heartbeat_payload: None,
7082 connect_timeout_ms: Some(1_000),
7083 reconnect_delay_initial_ms: Some(50),
7084 reconnect_delay_max_ms: Some(100),
7085 reconnect_backoff_factor: Some(1.0),
7086 reconnect_jitter_ms: Some(0),
7087 reconnect_max_attempts: None,
7088 heartbeat_timeout_secs: None,
7089 idle_timeout_ms: None,
7090 writer_capacity: None,
7091 backend: TransportBackend::Tungstenite,
7092 proxy_url: None,
7093 max_message_size_bytes: None,
7094 max_frame_size_bytes: None,
7095 };
7096
7097 let client = WebSocketClient::builder()
7098 .config(config)
7099 .message_handler(handler)
7100 .connect()
7101 .await
7102 .unwrap();
7103
7104 tokio::time::timeout(Duration::from_secs(2), async {
7105 loop {
7106 if client.is_reconnecting() {
7107 break;
7108 }
7109
7110 tokio::time::sleep(Duration::from_millis(10)).await;
7111 }
7112 })
7113 .await
7114 .expect("client did not enter RECONNECT state");
7115
7116 client.disconnect().await;
7117 server.abort();
7118 }
7119
7120 #[rstest]
7121 #[tokio::test]
7122 async fn test_stream_mode_disables_auto_reconnect() {
7123 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7126 let port = listener.local_addr().unwrap().port();
7127 let server = task::spawn(async move {
7128 if let Ok((stream, _)) = listener.accept().await
7129 && let Ok(_ws) = accept_async(stream).await
7130 {
7131 sleep(Duration::from_millis(100)).await;
7133 }
7134 });
7135
7136 let config = WebSocketConfig {
7137 url: format!("ws://127.0.0.1:{port}"),
7138 headers: vec![],
7139 heartbeat_interval_secs: None,
7140 heartbeat_payload: None,
7141 connect_timeout_ms: Some(1_000),
7142 reconnect_delay_initial_ms: Some(50),
7143 reconnect_delay_max_ms: Some(100),
7144 reconnect_backoff_factor: Some(1.0),
7145 reconnect_jitter_ms: Some(0),
7146 reconnect_max_attempts: None,
7147 heartbeat_timeout_secs: None,
7148 idle_timeout_ms: None,
7149 writer_capacity: None,
7150 backend: TransportBackend::Tungstenite,
7151 proxy_url: None,
7152 max_message_size_bytes: None,
7153 max_frame_size_bytes: None,
7154 };
7155
7156 let (_reader, _client) = WebSocketClient::stream_builder()
7157 .config(config)
7158 .connect()
7159 .await
7160 .unwrap();
7161
7162 server.abort();
7170 }
7171
7172 #[rstest]
7173 #[tokio::test]
7174 async fn test_message_handler_mode_allows_auto_reconnect() {
7175 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7177 let port = listener.local_addr().unwrap().port();
7178
7179 let server = task::spawn(async move {
7180 if let Ok((stream, _)) = listener.accept().await
7182 && let Ok(ws) = accept_async(stream).await
7183 {
7184 drop(ws);
7185 }
7186
7187 sleep(Duration::from_millis(50)).await;
7188 });
7189
7190 let (handler, _rx) = channel_message_handler();
7191
7192 let config = WebSocketConfig {
7193 url: format!("ws://127.0.0.1:{port}"),
7194 headers: vec![],
7195 heartbeat_interval_secs: None,
7196 heartbeat_payload: None,
7197 connect_timeout_ms: Some(1_000),
7198 reconnect_delay_initial_ms: Some(50),
7199 reconnect_delay_max_ms: Some(100),
7200 reconnect_backoff_factor: Some(1.0),
7201 reconnect_jitter_ms: Some(0),
7202 reconnect_max_attempts: None,
7203 heartbeat_timeout_secs: None,
7204 idle_timeout_ms: None,
7205 writer_capacity: None,
7206 backend: TransportBackend::Tungstenite,
7207 proxy_url: None,
7208 max_message_size_bytes: None,
7209 max_frame_size_bytes: None,
7210 };
7211
7212 let client = WebSocketClient::builder()
7213 .config(config)
7214 .message_handler(handler)
7215 .connect()
7216 .await
7217 .unwrap();
7218
7219 tokio::time::timeout(Duration::from_secs(2), async {
7221 loop {
7222 if client.is_reconnecting() || client.is_closed() {
7223 break;
7224 }
7225
7226 tokio::time::sleep(Duration::from_millis(10)).await;
7227 }
7228 })
7229 .await
7230 .expect("client should attempt reconnection or close");
7231
7232 assert!(
7235 client.is_reconnecting() || client.is_closed(),
7236 "Client with message handler should attempt reconnection"
7237 );
7238
7239 client.disconnect().await;
7240 server.abort();
7241 }
7242
7243 #[rstest]
7244 #[tokio::test]
7245 async fn test_handler_mode_reconnect_with_new_connection() {
7246 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7248 let port = listener.local_addr().unwrap().port();
7249
7250 let server = task::spawn(async move {
7251 if let Ok((stream, _)) = listener.accept().await
7253 && let Ok(ws) = accept_async(stream).await
7254 {
7255 drop(ws);
7256 }
7257
7258 sleep(Duration::from_millis(100)).await;
7260
7261 if let Ok((stream, _)) = listener.accept().await
7263 && let Ok(mut ws) = accept_async(stream).await
7264 {
7265 use futures_util::SinkExt;
7266 let _ = ws
7267 .send(WsMessage::Text("reconnected".to_string().into()))
7268 .await;
7269 sleep(Duration::from_secs(1)).await;
7270 }
7271 });
7272
7273 let (handler, mut rx) = channel_message_handler();
7274
7275 let config = WebSocketConfig {
7276 url: format!("ws://127.0.0.1:{port}"),
7277 headers: vec![],
7278 heartbeat_interval_secs: None,
7279 heartbeat_payload: None,
7280 connect_timeout_ms: Some(2_000),
7281 reconnect_delay_initial_ms: Some(50),
7282 reconnect_delay_max_ms: Some(200),
7283 reconnect_backoff_factor: Some(1.5),
7284 reconnect_jitter_ms: Some(10),
7285 reconnect_max_attempts: None,
7286 heartbeat_timeout_secs: None,
7287 idle_timeout_ms: None,
7288 writer_capacity: None,
7289 backend: TransportBackend::Tungstenite,
7290 proxy_url: None,
7291 max_message_size_bytes: None,
7292 max_frame_size_bytes: None,
7293 };
7294
7295 let client = WebSocketClient::builder()
7296 .config(config)
7297 .message_handler(handler)
7298 .connect()
7299 .await
7300 .unwrap();
7301
7302 let result = tokio::time::timeout(Duration::from_secs(5), async {
7304 loop {
7305 if let Ok(msg) = rx.try_recv()
7306 && matches!(msg, WsMessage::Text(ref text) if AsRef::<str>::as_ref(text) == "reconnected")
7307 {
7308 return true;
7309 }
7310
7311 tokio::time::sleep(Duration::from_millis(10)).await;
7312 }
7313 })
7314 .await;
7315
7316 assert!(
7317 result.is_ok(),
7318 "Should receive message after reconnection within timeout"
7319 );
7320
7321 client.disconnect().await;
7322 server.abort();
7323 }
7324
7325 #[rstest]
7326 #[tokio::test]
7327 async fn test_stream_mode_no_auto_reconnect() {
7328 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7331 let port = listener.local_addr().unwrap().port();
7332
7333 let server = task::spawn(async move {
7334 if let Ok((stream, _)) = listener.accept().await
7336 && let Ok(mut ws) = accept_async(stream).await
7337 {
7338 use futures_util::SinkExt;
7339 let _ = ws.send(WsMessage::Text("hello".to_string().into())).await;
7340 sleep(Duration::from_millis(50)).await;
7341 }
7343 });
7344
7345 let config = WebSocketConfig {
7346 url: format!("ws://127.0.0.1:{port}"),
7347 headers: vec![],
7348 heartbeat_interval_secs: None,
7349 heartbeat_payload: None,
7350 connect_timeout_ms: Some(1_000),
7351 reconnect_delay_initial_ms: Some(50),
7352 reconnect_delay_max_ms: Some(100),
7353 reconnect_backoff_factor: Some(1.0),
7354 reconnect_jitter_ms: Some(0),
7355 reconnect_max_attempts: None,
7356 heartbeat_timeout_secs: None,
7357 idle_timeout_ms: None,
7358 writer_capacity: None,
7359 backend: TransportBackend::Tungstenite,
7360 proxy_url: None,
7361 max_message_size_bytes: None,
7362 max_frame_size_bytes: None,
7363 };
7364
7365 let (mut reader, client) = WebSocketClient::stream_builder()
7366 .config(config)
7367 .connect()
7368 .await
7369 .unwrap();
7370
7371 assert!(client.is_active(), "Client should start as active");
7373
7374 let msg = reader.next().await;
7376 assert!(
7377 matches!(&msg, Some(Ok(Message::Text(bytes))) if bytes.as_ref() == b"hello"),
7378 "Should receive initial message"
7379 );
7380
7381 while let Some(msg) = reader.next().await {
7383 if msg.is_err() || matches!(msg, Ok(Message::Close(_))) {
7384 break;
7385 }
7386 }
7387
7388 sleep(Duration::from_millis(200)).await;
7391 assert!(
7392 client.is_active(),
7393 "Stream mode client stays ACTIVE before notify_closed()"
7394 );
7395
7396 client.notify_closed();
7398
7399 assert!(
7400 client.is_closed(),
7401 "Stream mode client should be CLOSED after notify_closed()"
7402 );
7403 assert!(
7404 !client.is_reconnecting(),
7405 "Stream mode client should never attempt reconnection"
7406 );
7407
7408 client.disconnect().await;
7409 server.abort();
7410 }
7411
7412 #[rstest]
7413 #[tokio::test]
7414 async fn test_send_timeout_uses_configured_connect_timeout() {
7415 use nautilus_common::testing::wait_until_async;
7418
7419 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7420 let port = listener.local_addr().unwrap().port();
7421
7422 let server = task::spawn(async move {
7423 if let Ok((stream, _)) = listener.accept().await
7425 && let Ok(ws) = accept_async(stream).await
7426 {
7427 drop(ws);
7428 }
7429
7430 sleep(Duration::from_mins(1)).await;
7432 });
7433
7434 let (handler, _rx) = channel_message_handler();
7435
7436 let config = WebSocketConfig {
7438 url: format!("ws://127.0.0.1:{port}"),
7439 headers: vec![],
7440 heartbeat_interval_secs: None,
7441 heartbeat_payload: None,
7442 connect_timeout_ms: Some(2_000), reconnect_delay_initial_ms: Some(50),
7444 reconnect_delay_max_ms: Some(100),
7445 reconnect_backoff_factor: Some(1.0),
7446 reconnect_jitter_ms: Some(0),
7447 reconnect_max_attempts: None,
7448 heartbeat_timeout_secs: None,
7449 idle_timeout_ms: None,
7450 writer_capacity: None,
7451 backend: TransportBackend::Tungstenite,
7452 proxy_url: None,
7453 max_message_size_bytes: None,
7454 max_frame_size_bytes: None,
7455 };
7456
7457 let client = WebSocketClient::builder()
7458 .config(config)
7459 .message_handler(handler)
7460 .connect()
7461 .await
7462 .unwrap();
7463
7464 wait_until_async(
7466 || async { client.is_reconnecting() },
7467 Duration::from_secs(3),
7468 )
7469 .await;
7470
7471 let start = std::time::Instant::now();
7473 let send_result = client.send_text("test".to_string(), None).await;
7474 let elapsed = start.elapsed();
7475
7476 assert!(
7477 send_result.is_err(),
7478 "Send should fail when client stuck in RECONNECT"
7479 );
7480 assert!(
7481 matches!(send_result, Err(crate::error::SendError::Timeout)),
7482 "Send should return Timeout error, was: {send_result:?}"
7483 );
7484 assert!(
7487 elapsed >= Duration::from_millis(1800),
7488 "Send should timeout after at least 2s (configured timeout), took {elapsed:?}"
7489 );
7490
7491 client.disconnect().await;
7492 server.abort();
7493 }
7494
7495 #[rstest]
7496 #[tokio::test]
7497 async fn test_send_waits_during_reconnection() {
7498 use nautilus_common::testing::wait_until_async;
7500
7501 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7502 let port = listener.local_addr().unwrap().port();
7503
7504 let server = task::spawn(async move {
7505 if let Ok((stream, _)) = listener.accept().await
7507 && let Ok(ws) = accept_async(stream).await
7508 {
7509 drop(ws);
7510 }
7511
7512 sleep(Duration::from_millis(500)).await;
7514
7515 if let Ok((stream, _)) = listener.accept().await
7517 && let Ok(mut ws) = accept_async(stream).await
7518 {
7519 while let Some(Ok(msg)) = ws.next().await {
7521 if ws.send(msg).await.is_err() {
7522 break;
7523 }
7524 }
7525 }
7526 });
7527
7528 let (handler, _rx) = channel_message_handler();
7529
7530 let config = WebSocketConfig {
7531 url: format!("ws://127.0.0.1:{port}"),
7532 headers: vec![],
7533 heartbeat_interval_secs: None,
7534 heartbeat_payload: None,
7535 connect_timeout_ms: Some(5_000), reconnect_delay_initial_ms: Some(100),
7537 reconnect_delay_max_ms: Some(200),
7538 reconnect_backoff_factor: Some(1.0),
7539 reconnect_jitter_ms: Some(0),
7540 reconnect_max_attempts: None,
7541 heartbeat_timeout_secs: None,
7542 idle_timeout_ms: None,
7543 writer_capacity: None,
7544 backend: TransportBackend::Tungstenite,
7545 proxy_url: None,
7546 max_message_size_bytes: None,
7547 max_frame_size_bytes: None,
7548 };
7549
7550 let client = WebSocketClient::builder()
7551 .config(config)
7552 .message_handler(handler)
7553 .connect()
7554 .await
7555 .unwrap();
7556
7557 wait_until_async(
7559 || async { client.is_reconnecting() },
7560 Duration::from_secs(2),
7561 )
7562 .await;
7563
7564 let send_result = tokio::time::timeout(
7566 Duration::from_secs(3),
7567 client.send_text("test_message".to_string(), None),
7568 )
7569 .await;
7570
7571 assert!(
7572 send_result.is_ok() && send_result.unwrap().is_ok(),
7573 "Send should succeed after waiting for reconnection"
7574 );
7575
7576 client.disconnect().await;
7577 server.abort();
7578 }
7579
7580 #[rstest]
7581 #[tokio::test]
7582 async fn test_rate_limiter_before_active_wait() {
7583 use std::{num::NonZeroU32, sync::Arc};
7588
7589 use nautilus_common::testing::wait_until_async;
7590
7591 use crate::ratelimiter::quota::Quota;
7592
7593 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7594 let port = listener.local_addr().unwrap().port();
7595
7596 let server = task::spawn(async move {
7597 if let Ok((stream, _)) = listener.accept().await
7599 && let Ok(mut ws) = accept_async(stream).await
7600 {
7601 if let Some(Ok(_)) = ws.next().await {
7603 drop(ws);
7604 }
7605 }
7606
7607 sleep(Duration::from_millis(500)).await;
7609
7610 if let Ok((stream, _)) = listener.accept().await
7612 && let Ok(mut ws) = accept_async(stream).await
7613 {
7614 while let Some(Ok(msg)) = ws.next().await {
7615 if ws.send(msg).await.is_err() {
7616 break;
7617 }
7618 }
7619 }
7620 });
7621
7622 let (handler, _rx) = channel_message_handler();
7623
7624 let config = WebSocketConfig {
7625 url: format!("ws://127.0.0.1:{port}"),
7626 headers: vec![],
7627 heartbeat_interval_secs: None,
7628 heartbeat_payload: None,
7629 connect_timeout_ms: Some(5_000),
7630 reconnect_delay_initial_ms: Some(50),
7631 reconnect_delay_max_ms: Some(100),
7632 reconnect_backoff_factor: Some(1.0),
7633 reconnect_jitter_ms: Some(0),
7634 reconnect_max_attempts: None,
7635 heartbeat_timeout_secs: None,
7636 idle_timeout_ms: None,
7637 writer_capacity: None,
7638 backend: TransportBackend::Tungstenite,
7639 proxy_url: None,
7640 max_message_size_bytes: None,
7641 max_frame_size_bytes: None,
7642 };
7643
7644 let quota = Quota::per_second(NonZeroU32::new(1).unwrap())
7646 .unwrap()
7647 .allow_burst(NonZeroU32::new(1).unwrap());
7648
7649 let client = Arc::new(
7650 WebSocketClient::builder()
7651 .config(config)
7652 .message_handler(handler)
7653 .keyed_quotas(vec![("test_key".to_string(), quota)])
7654 .connect()
7655 .await
7656 .unwrap(),
7657 );
7658
7659 let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7661 client
7662 .send_text("msg1".to_string(), Some(test_key.as_slice()))
7663 .await
7664 .unwrap();
7665
7666 wait_until_async(
7668 || async { client.is_reconnecting() },
7669 Duration::from_secs(2),
7670 )
7671 .await;
7672
7673 let start = std::time::Instant::now();
7675 let send_result = client
7676 .send_text("msg2".to_string(), Some(test_key.as_slice()))
7677 .await;
7678 let elapsed = start.elapsed();
7679
7680 assert!(
7682 send_result.is_ok(),
7683 "Send should succeed after rate limit + reconnection, was: {send_result:?}"
7684 );
7685 assert!(
7689 elapsed >= Duration::from_millis(850),
7690 "Should wait for rate limit (~1s), waited {elapsed:?}"
7691 );
7692
7693 client.disconnect().await;
7694 server.abort();
7695 }
7696
7697 #[rstest]
7698 #[tokio::test]
7699 async fn test_disconnect_during_reconnect_exits_cleanly() {
7700 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7703 let port = listener.local_addr().unwrap().port();
7704
7705 let server = task::spawn(async move {
7706 if let Ok((stream, _)) = listener.accept().await
7708 && let Ok(ws) = accept_async(stream).await
7709 {
7710 drop(ws);
7711 }
7712
7713 sleep(Duration::from_mins(1)).await;
7715 });
7716
7717 let (handler, _rx) = channel_message_handler();
7718
7719 let config = WebSocketConfig {
7720 url: format!("ws://127.0.0.1:{port}"),
7721 headers: vec![],
7722 heartbeat_interval_secs: None,
7723 heartbeat_payload: None,
7724 connect_timeout_ms: Some(2_000), reconnect_delay_initial_ms: Some(100),
7726 reconnect_delay_max_ms: Some(200),
7727 reconnect_backoff_factor: Some(1.0),
7728 reconnect_jitter_ms: Some(0),
7729 reconnect_max_attempts: None,
7730 heartbeat_timeout_secs: None,
7731 idle_timeout_ms: None,
7732 writer_capacity: None,
7733 backend: TransportBackend::Tungstenite,
7734 proxy_url: None,
7735 max_message_size_bytes: None,
7736 max_frame_size_bytes: None,
7737 };
7738
7739 let client = WebSocketClient::builder()
7740 .config(config)
7741 .message_handler(handler)
7742 .connect()
7743 .await
7744 .unwrap();
7745
7746 tokio::time::timeout(Duration::from_secs(2), async {
7748 while !client.is_reconnecting() {
7749 sleep(Duration::from_millis(10)).await;
7750 }
7751 })
7752 .await
7753 .expect("Client should enter RECONNECT state");
7754
7755 client.disconnect().await;
7757
7758 assert!(
7760 client.is_disconnected(),
7761 "Client should be cleanly disconnected"
7762 );
7763
7764 server.abort();
7765 }
7766
7767 #[rstest]
7768 #[tokio::test]
7769 async fn test_send_fails_fast_when_closed_before_rate_limit() {
7770 use std::{num::NonZeroU32, sync::Arc};
7773
7774 use nautilus_common::testing::wait_until_async;
7775
7776 use crate::ratelimiter::quota::Quota;
7777
7778 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7779 let port = listener.local_addr().unwrap().port();
7780
7781 let server = task::spawn(async move {
7782 if let Ok((stream, _)) = listener.accept().await
7784 && let Ok(ws) = accept_async(stream).await
7785 {
7786 drop(ws);
7787 }
7788
7789 sleep(Duration::from_mins(1)).await;
7790 });
7791
7792 let (handler, _rx) = channel_message_handler();
7793
7794 let config = WebSocketConfig {
7795 url: format!("ws://127.0.0.1:{port}"),
7796 headers: vec![],
7797 heartbeat_interval_secs: None,
7798 heartbeat_payload: None,
7799 connect_timeout_ms: Some(5_000),
7800 reconnect_delay_initial_ms: Some(50),
7801 reconnect_delay_max_ms: Some(100),
7802 reconnect_backoff_factor: Some(1.0),
7803 reconnect_jitter_ms: Some(0),
7804 reconnect_max_attempts: None,
7805 heartbeat_timeout_secs: None,
7806 idle_timeout_ms: None,
7807 writer_capacity: None,
7808 backend: TransportBackend::Tungstenite,
7809 proxy_url: None,
7810 max_message_size_bytes: None,
7811 max_frame_size_bytes: None,
7812 };
7813
7814 let quota = Quota::with_period(Duration::from_secs(10))
7817 .unwrap()
7818 .allow_burst(NonZeroU32::new(1).unwrap());
7819
7820 let client = Arc::new(
7821 WebSocketClient::builder()
7822 .config(config)
7823 .message_handler(handler)
7824 .keyed_quotas(vec![("test_key".to_string(), quota)])
7825 .connect()
7826 .await
7827 .unwrap(),
7828 );
7829
7830 wait_until_async(
7832 || async { client.is_reconnecting() || client.is_closed() },
7833 Duration::from_secs(2),
7834 )
7835 .await;
7836
7837 client.disconnect().await;
7839 assert!(
7840 !client.is_active(),
7841 "Client should not be active after disconnect"
7842 );
7843
7844 let start = std::time::Instant::now();
7846 let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7847 let result = client
7848 .send_text("test".to_string(), Some(test_key.as_slice()))
7849 .await;
7850 let elapsed = start.elapsed();
7851
7852 assert!(result.is_err(), "Send should fail when client is closed");
7854 assert!(
7855 matches!(result, Err(crate::error::SendError::Closed)),
7856 "Send should return Closed error, was: {result:?}"
7857 );
7858
7859 assert!(
7861 elapsed < Duration::from_millis(100),
7862 "Send should fail fast without rate limiting, took {elapsed:?}"
7863 );
7864
7865 server.abort();
7866 }
7867
7868 #[rstest]
7869 #[tokio::test]
7870 async fn test_builder_rejects_shared_rate_limiter_with_quotas() {
7871 let (handler, _rx) = channel_message_handler();
7872 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7873 let result = WebSocketClient::builder()
7874 .config(reconnect_test_config(1))
7875 .message_handler(handler)
7876 .default_quota(Quota::with_period(Duration::from_secs(1)).unwrap())
7877 .rate_limiter(rate_limiter)
7878 .connect()
7879 .await;
7880
7881 assert_eq!(
7882 result.unwrap_err().to_string(),
7883 "I/O error: Cannot combine a shared rate limiter with quota configuration"
7884 );
7885 }
7886
7887 #[rstest]
7888 #[tokio::test]
7889 async fn test_builder_rejects_connection_keys_without_limiter() {
7890 let (handler, _rx) = channel_message_handler();
7891 let result = WebSocketClient::builder()
7892 .config(reconnect_test_config(1))
7893 .message_handler(handler)
7894 .connection_rate_keys(Arc::from([Ustr::from("connect")]))
7895 .connect()
7896 .await;
7897
7898 assert_eq!(
7899 result.unwrap_err().to_string(),
7900 "I/O error: Connection rate keys require a connection rate limiter"
7901 );
7902 }
7903
7904 #[rstest]
7905 #[tokio::test]
7906 async fn test_builder_rejects_connection_limiter_without_keys() {
7907 let (handler, _rx) = channel_message_handler();
7908 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7909 let result = WebSocketClient::builder()
7910 .config(reconnect_test_config(1))
7911 .message_handler(handler)
7912 .connection_rate_limiter(rate_limiter)
7913 .connect()
7914 .await;
7915
7916 assert_eq!(
7917 result.unwrap_err().to_string(),
7918 "I/O error: Connection rate limiter requires at least one connection rate key"
7919 );
7920 }
7921
7922 #[rstest]
7923 #[tokio::test]
7924 async fn test_epoch_builder_rejects_both_ping_handler_types() {
7925 let (handler, _rx) = channel_epoch_message_handler();
7926 let ping_handler: PingHandler = Arc::new(|_| {});
7927 let epoch_ping_handler: EpochPingHandler = Arc::new(|_, _| {});
7928 let result = WebSocketClient::epoch_builder()
7929 .config(reconnect_test_config(1))
7930 .epoch_handler(handler)
7931 .ping_handler(ping_handler)
7932 .epoch_ping_handler(epoch_ping_handler)
7933 .connect()
7934 .await;
7935
7936 assert_eq!(
7937 result.unwrap_err().to_string(),
7938 "I/O error: Cannot configure both ping_handler and epoch_ping_handler"
7939 );
7940 }
7941
7942 #[rstest]
7943 #[tokio::test]
7944 async fn test_connect_url_rejects_invalid_reconnect_timing_before_connect() {
7945 let (handler, _rx) = channel_message_handler();
7946
7947 let config = WebSocketConfig {
7948 url: "ws://127.0.0.1:1".to_string(),
7949 headers: vec![],
7950 heartbeat_interval_secs: None,
7951 heartbeat_payload: None,
7952 connect_timeout_ms: Some(0),
7953 reconnect_delay_initial_ms: Some(100),
7954 reconnect_delay_max_ms: Some(500),
7955 reconnect_backoff_factor: Some(1.5),
7956 reconnect_jitter_ms: Some(0),
7957 reconnect_max_attempts: None,
7958 heartbeat_timeout_secs: None,
7959 idle_timeout_ms: None,
7960 writer_capacity: None,
7961 backend: TransportBackend::Tungstenite,
7962 proxy_url: None,
7963 max_message_size_bytes: None,
7964 max_frame_size_bytes: None,
7965 };
7966
7967 let err = WebSocketClientInner::connect_url(config, Some(handler), None)
7968 .await
7969 .expect_err("invalid reconnect timing should be rejected");
7970
7971 match err {
7972 TransportError::Io(error) => {
7973 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
7974 assert!(
7975 error.to_string().contains("connect_timeout_ms"),
7976 "error should mention zero reconnect timeout, was: {error}"
7977 );
7978 }
7979 other => panic!("expected InvalidInput IO error, was: {other:?}"),
7980 }
7981 }
7982
7983 #[rstest]
7984 #[tokio::test]
7985 async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
7986 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7987 let port = listener.local_addr().unwrap().port();
7988 let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
7989 let accepted_clone = Arc::clone(&accepted);
7990
7991 let server = task::spawn(async move {
7992 let (stream, _) = listener.accept().await.unwrap();
7993 accepted_clone.store(true, Ordering::SeqCst);
7994 accept_async(stream).await.unwrap();
7995 });
7996
7997 let (handler, _rx) = channel_message_handler();
7998
7999 let config = WebSocketConfig {
8000 url: format!("ws://127.0.0.1:{port}"),
8001 headers: vec![],
8002 heartbeat_interval_secs: None,
8003 heartbeat_payload: None,
8004 connect_timeout_ms: Some(1_000),
8005 reconnect_delay_initial_ms: Some(50),
8006 reconnect_delay_max_ms: Some(100),
8007 reconnect_backoff_factor: Some(100.1),
8008 reconnect_jitter_ms: Some(0),
8009 reconnect_max_attempts: None,
8010 heartbeat_timeout_secs: None,
8011 idle_timeout_ms: None,
8012 writer_capacity: None,
8013 backend: TransportBackend::Tungstenite,
8014 proxy_url: None,
8015 max_message_size_bytes: None,
8016 max_frame_size_bytes: None,
8017 };
8018
8019 let error = WebSocketClientInner::connect_url(config, Some(handler), None)
8020 .await
8021 .expect_err("invalid reconnect backoff should be rejected");
8022
8023 match error {
8024 TransportError::Io(error) => {
8025 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
8026 assert!(
8027 error.to_string().contains("factor"),
8028 "error should mention the invalid factor, was: {error}"
8029 );
8030 }
8031 other => panic!("expected InvalidInput IO error, was: {other:?}"),
8032 }
8033
8034 assert!(
8035 !accepted.load(Ordering::SeqCst),
8036 "invalid reconnect backoff must be rejected before connecting"
8037 );
8038 server.abort();
8039 }
8040
8041 #[rstest]
8042 #[tokio::test]
8043 async fn test_client_without_handler_sets_stream_mode() {
8044 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8048 let port = listener.local_addr().unwrap().port();
8049
8050 let server = task::spawn(async move {
8051 if let Ok((stream, _)) = listener.accept().await
8053 && let Ok(ws) = accept_async(stream).await
8054 {
8055 drop(ws); }
8057 });
8058
8059 let config = WebSocketConfig {
8060 url: format!("ws://127.0.0.1:{port}"),
8061 headers: vec![],
8062 heartbeat_interval_secs: None,
8063 heartbeat_payload: None,
8064 connect_timeout_ms: Some(1_000),
8065 reconnect_delay_initial_ms: Some(100),
8066 reconnect_delay_max_ms: Some(500),
8067 reconnect_backoff_factor: Some(1.5),
8068 reconnect_jitter_ms: Some(0),
8069 reconnect_max_attempts: None,
8070 heartbeat_timeout_secs: None,
8071 idle_timeout_ms: None,
8072 writer_capacity: None,
8073 backend: TransportBackend::Tungstenite,
8074 proxy_url: None,
8075 max_message_size_bytes: None,
8076 max_frame_size_bytes: None,
8077 };
8078
8079 let inner = WebSocketClientInner::connect_url(config, None, None)
8081 .await
8082 .unwrap();
8083
8084 assert!(
8086 inner.handler.is_none(),
8087 "Client without handler should not retain an internal handler"
8088 );
8089
8090 server.abort();
8094 }
8095
8096 #[rstest]
8097 #[tokio::test]
8098 async fn test_idle_timeout_triggers_reconnect() {
8099 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8100 let port = listener.local_addr().unwrap().port();
8101
8102 let server = task::spawn(async move {
8104 let (stream, _) = listener.accept().await.unwrap();
8105 let _ws = accept_async(stream).await.unwrap();
8106 sleep(Duration::from_secs(5)).await;
8108 });
8109
8110 let (handler, _rx) = channel_message_handler();
8111
8112 let config = WebSocketConfig {
8113 url: format!("ws://127.0.0.1:{port}"),
8114 headers: vec![],
8115 heartbeat_interval_secs: None,
8116 heartbeat_payload: None,
8117 connect_timeout_ms: Some(2_000),
8118 reconnect_delay_initial_ms: Some(50),
8119 reconnect_delay_max_ms: Some(100),
8120 reconnect_backoff_factor: Some(1.0),
8121 reconnect_jitter_ms: Some(0),
8122 reconnect_max_attempts: Some(1),
8123 heartbeat_timeout_secs: None,
8124 idle_timeout_ms: Some(500),
8125 writer_capacity: None,
8126 backend: TransportBackend::Tungstenite,
8127 proxy_url: None,
8128 max_message_size_bytes: None,
8129 max_frame_size_bytes: None,
8130 };
8131
8132 let client = WebSocketClient::builder()
8133 .config(config)
8134 .message_handler(handler)
8135 .connect()
8136 .await
8137 .unwrap();
8138
8139 assert!(client.is_active());
8140
8141 wait_until_async(
8143 || async { client.is_reconnecting() || client.is_disconnected() },
8144 Duration::from_secs(3),
8145 )
8146 .await;
8147
8148 assert!(
8149 !client.is_active(),
8150 "Client should not be active after idle timeout"
8151 );
8152
8153 client.disconnect().await;
8154 server.abort();
8155 }
8156
8157 #[rstest]
8158 #[tokio::test]
8159 async fn test_idle_timeout_resets_on_data() {
8160 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8161 let port = listener.local_addr().unwrap().port();
8162
8163 let server = task::spawn(async move {
8165 let (stream, _) = listener.accept().await.unwrap();
8166 let mut ws = accept_async(stream).await.unwrap();
8167
8168 for _ in 0..10 {
8169 sleep(Duration::from_millis(200)).await;
8170
8171 if ws.send(WsMessage::Text("ping".into())).await.is_err() {
8172 break;
8173 }
8174 }
8175 });
8176
8177 let (handler, _rx) = channel_message_handler();
8178
8179 let config = WebSocketConfig {
8180 url: format!("ws://127.0.0.1:{port}"),
8181 headers: vec![],
8182 heartbeat_interval_secs: None,
8183 heartbeat_payload: None,
8184 connect_timeout_ms: Some(2_000),
8185 reconnect_delay_initial_ms: Some(50),
8186 reconnect_delay_max_ms: Some(100),
8187 reconnect_backoff_factor: Some(1.0),
8188 reconnect_jitter_ms: Some(0),
8189 reconnect_max_attempts: Some(1),
8190 heartbeat_timeout_secs: None,
8191 idle_timeout_ms: Some(1_000),
8192 writer_capacity: None,
8193 backend: TransportBackend::Tungstenite,
8194 proxy_url: None,
8195 max_message_size_bytes: None,
8196 max_frame_size_bytes: None,
8197 };
8198
8199 let client = WebSocketClient::builder()
8200 .config(config)
8201 .message_handler(handler)
8202 .connect()
8203 .await
8204 .unwrap();
8205
8206 assert!(client.is_active());
8207
8208 sleep(Duration::from_millis(1_500)).await;
8210
8211 assert!(
8212 client.is_active(),
8213 "Client should remain active when data is flowing"
8214 );
8215
8216 client.disconnect().await;
8217 server.abort();
8218 }
8219
8220 #[rstest]
8221 #[tokio::test]
8222 async fn test_idle_timeout_fires_when_only_pings_received() {
8223 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8229 let port = listener.local_addr().unwrap().port();
8230 let server = task::spawn(async move {
8231 let (stream, _) = listener.accept().await.unwrap();
8232 let mut ws = accept_async(stream).await.unwrap();
8233
8234 for _ in 0..60 {
8235 sleep(Duration::from_millis(100)).await;
8236
8237 if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
8238 break;
8239 }
8240 }
8241 });
8242
8243 let (handler, _rx) = channel_message_handler();
8244
8245 let config = WebSocketConfig {
8246 url: format!("ws://127.0.0.1:{port}"),
8247 headers: vec![],
8248 heartbeat_interval_secs: None,
8249 heartbeat_payload: None,
8250 connect_timeout_ms: Some(2_000),
8251 reconnect_delay_initial_ms: Some(50),
8252 reconnect_delay_max_ms: Some(100),
8253 reconnect_backoff_factor: Some(1.0),
8254 reconnect_jitter_ms: Some(0),
8255 reconnect_max_attempts: Some(1),
8256 heartbeat_timeout_secs: None,
8257 idle_timeout_ms: Some(500),
8258 writer_capacity: None,
8259 backend: TransportBackend::Tungstenite,
8260 proxy_url: None,
8261 max_message_size_bytes: None,
8262 max_frame_size_bytes: None,
8263 };
8264
8265 let client = WebSocketClient::builder()
8266 .config(config)
8267 .message_handler(handler)
8268 .connect()
8269 .await
8270 .unwrap();
8271
8272 assert!(client.is_active());
8273
8274 wait_until_async(
8278 || async { client.is_reconnecting() || client.is_disconnected() },
8279 Duration::from_millis(1_500),
8280 )
8281 .await;
8282
8283 assert!(
8284 !client.is_active(),
8285 "Client should not be active after idle timeout when only pings/pongs flow"
8286 );
8287
8288 client.disconnect().await;
8289 server.abort();
8290 }
8291
8292 #[rstest]
8293 #[tokio::test]
8294 async fn test_idle_timeout_fires_when_only_pongs_received() {
8295 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8300 let port = listener.local_addr().unwrap().port();
8301 let server = task::spawn(async move {
8302 let (stream, _) = listener.accept().await.unwrap();
8303 let mut ws = accept_async(stream).await.unwrap();
8304
8305 let deadline = tokio::time::Instant::now() + Duration::from_secs(6);
8309 while tokio::time::Instant::now() < deadline {
8310 if let Ok(Some(Err(_)) | None) =
8311 tokio::time::timeout(Duration::from_millis(100), ws.next()).await
8312 {
8313 break;
8314 }
8315 }
8316 });
8317
8318 let (handler, _rx) = channel_message_handler();
8319
8320 let config = WebSocketConfig {
8321 url: format!("ws://127.0.0.1:{port}"),
8322 headers: vec![],
8323 heartbeat_interval_secs: Some(1),
8324 heartbeat_payload: None,
8325 connect_timeout_ms: Some(2_000),
8326 reconnect_delay_initial_ms: Some(50),
8327 reconnect_delay_max_ms: Some(100),
8328 reconnect_backoff_factor: Some(1.0),
8329 reconnect_jitter_ms: Some(0),
8330 reconnect_max_attempts: Some(1),
8331 heartbeat_timeout_secs: None,
8332 idle_timeout_ms: Some(1_500),
8333 writer_capacity: None,
8334 backend: TransportBackend::Tungstenite,
8335 proxy_url: None,
8336 max_message_size_bytes: None,
8337 max_frame_size_bytes: None,
8338 };
8339
8340 let client = WebSocketClient::builder()
8341 .config(config)
8342 .message_handler(handler)
8343 .connect()
8344 .await
8345 .unwrap();
8346
8347 assert!(client.is_active());
8348
8349 wait_until_async(
8353 || async { client.is_reconnecting() || client.is_disconnected() },
8354 Duration::from_millis(2_500),
8355 )
8356 .await;
8357
8358 assert!(
8359 !client.is_active(),
8360 "Client should not be active after idle timeout when only pongs flow"
8361 );
8362
8363 client.disconnect().await;
8364 server.abort();
8365 }
8366
8367 #[rstest]
8368 #[tokio::test]
8369 async fn test_disconnect_during_backoff_exits_promptly() {
8370 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8374 let port = listener.local_addr().unwrap().port();
8375
8376 let server = task::spawn(async move {
8377 if let Ok((stream, _)) = listener.accept().await {
8379 let _ = accept_async(stream).await;
8380 }
8381
8382 sleep(Duration::from_mins(1)).await;
8384 });
8385
8386 let (handler, _rx) = channel_message_handler();
8387
8388 let config = WebSocketConfig {
8389 url: format!("ws://127.0.0.1:{port}"),
8390 headers: vec![],
8391 heartbeat_interval_secs: None,
8392 heartbeat_payload: None,
8393 connect_timeout_ms: Some(1_000),
8394 reconnect_delay_initial_ms: Some(10_000), reconnect_delay_max_ms: Some(10_000),
8396 reconnect_backoff_factor: Some(1.0),
8397 reconnect_jitter_ms: Some(0),
8398 reconnect_max_attempts: None,
8399 heartbeat_timeout_secs: None,
8400 idle_timeout_ms: None,
8401 writer_capacity: None,
8402 backend: TransportBackend::Tungstenite,
8403 proxy_url: None,
8404 max_message_size_bytes: None,
8405 max_frame_size_bytes: None,
8406 };
8407
8408 let client = WebSocketClient::builder()
8409 .config(config)
8410 .message_handler(handler)
8411 .connect()
8412 .await
8413 .unwrap();
8414
8415 wait_until_async(
8417 || async { client.is_reconnecting() },
8418 Duration::from_secs(3),
8419 )
8420 .await;
8421
8422 sleep(Duration::from_millis(1_500)).await;
8424
8425 let start = std::time::Instant::now();
8427 client.disconnect().await;
8428 let elapsed = start.elapsed();
8429
8430 assert!(client.is_disconnected(), "Client should be disconnected");
8431 assert!(
8433 elapsed < Duration::from_secs(2),
8434 "Disconnect should interrupt backoff sleep, took {elapsed:?}"
8435 );
8436
8437 server.abort();
8438 }
8439
8440 #[rstest]
8441 #[tokio::test]
8442 async fn test_rate_limit_cancelled_on_disconnect() {
8443 use std::{num::NonZeroU32, sync::Arc};
8446
8447 use crate::ratelimiter::quota::Quota;
8448
8449 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8450 let port = listener.local_addr().unwrap().port();
8451 let server = task::spawn(async move {
8452 if let Ok((stream, _)) = listener.accept().await {
8453 let mut ws = accept_async(stream).await.unwrap();
8454 while let Some(Ok(msg)) = ws.next().await {
8456 if ws.send(msg).await.is_err() {
8457 break;
8458 }
8459 }
8460 }
8461 });
8462
8463 let (handler, _rx) = channel_message_handler();
8464
8465 let config = WebSocketConfig {
8466 url: format!("ws://127.0.0.1:{port}"),
8467 headers: vec![],
8468 heartbeat_interval_secs: None,
8469 heartbeat_payload: None,
8470 connect_timeout_ms: Some(5_000),
8471 reconnect_delay_initial_ms: Some(100),
8472 reconnect_delay_max_ms: Some(500),
8473 reconnect_backoff_factor: Some(1.5),
8474 reconnect_jitter_ms: Some(0),
8475 reconnect_max_attempts: None,
8476 heartbeat_timeout_secs: None,
8477 idle_timeout_ms: None,
8478 writer_capacity: None,
8479 backend: TransportBackend::Tungstenite,
8480 proxy_url: None,
8481 max_message_size_bytes: None,
8482 max_frame_size_bytes: None,
8483 };
8484
8485 let quota = Quota::with_period(Duration::from_mins(1))
8487 .unwrap()
8488 .allow_burst(NonZeroU32::new(1).unwrap());
8489
8490 let client = Arc::new(
8491 WebSocketClient::builder()
8492 .config(config)
8493 .message_handler(handler)
8494 .keyed_quotas(vec![("rate_key".to_string(), quota)])
8495 .connect()
8496 .await
8497 .unwrap(),
8498 );
8499
8500 let test_key: [Ustr; 1] = [Ustr::from("rate_key")];
8501
8502 client
8504 .send_text("exhaust".to_string(), Some(test_key.as_slice()))
8505 .await
8506 .unwrap();
8507
8508 let client_clone = Arc::clone(&client);
8510 let send_handle = task::spawn(async move {
8511 client_clone
8512 .send_text("blocked".to_string(), Some(&[Ustr::from("rate_key")]))
8513 .await
8514 });
8515
8516 sleep(Duration::from_millis(200)).await;
8518
8519 let start = std::time::Instant::now();
8521 client.disconnect().await;
8522 let elapsed_disconnect = start.elapsed();
8523
8524 let result = tokio::time::timeout(Duration::from_secs(2), send_handle)
8526 .await
8527 .expect("Send task should complete quickly")
8528 .expect("Send task should not panic");
8529
8530 assert!(
8531 matches!(result, Err(crate::error::SendError::Closed)),
8532 "Blocked send should return Closed, was: {result:?}"
8533 );
8534
8535 assert!(
8537 elapsed_disconnect < Duration::from_secs(3),
8538 "Disconnect should not wait for rate limiter, took {elapsed_disconnect:?}"
8539 );
8540
8541 server.abort();
8542 }
8543
8544 #[rstest]
8545 #[tokio::test]
8546 async fn test_stream_mode_transitions_to_closed_on_dead_write_task() {
8547 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8551 let port = listener.local_addr().unwrap().port();
8552 let server = task::spawn(async move {
8553 if let Ok((stream, _)) = listener.accept().await
8554 && let Ok(ws) = accept_async(stream).await
8555 {
8556 drop(ws);
8558 }
8559 });
8560
8561 let config = WebSocketConfig {
8562 url: format!("ws://127.0.0.1:{port}"),
8563 headers: vec![],
8564 heartbeat_interval_secs: None,
8565 heartbeat_payload: None,
8566 connect_timeout_ms: Some(1_000),
8567 reconnect_delay_initial_ms: Some(50),
8568 reconnect_delay_max_ms: Some(100),
8569 reconnect_backoff_factor: Some(1.0),
8570 reconnect_jitter_ms: Some(0),
8571 reconnect_max_attempts: None,
8572 heartbeat_timeout_secs: None,
8573 idle_timeout_ms: None,
8574 writer_capacity: None,
8575 backend: TransportBackend::Tungstenite,
8576 proxy_url: None,
8577 max_message_size_bytes: None,
8578 max_frame_size_bytes: None,
8579 };
8580
8581 let (_reader, client) = WebSocketClient::stream_builder()
8582 .config(config)
8583 .connect()
8584 .await
8585 .unwrap();
8586
8587 assert!(client.is_active(), "Client should start active");
8588
8589 sleep(Duration::from_millis(100)).await;
8591
8592 for _ in 0..20 {
8594 let _ = client.send_text("ping".to_string(), None).await;
8595 sleep(Duration::from_millis(50)).await;
8596
8597 if !client.is_active() {
8598 break;
8599 }
8600 }
8601
8602 wait_until_async(|| async { !client.is_active() }, Duration::from_secs(5)).await;
8604
8605 assert!(
8607 client.is_closed() || client.is_disconnected(),
8608 "Stream mode should transition to CLOSED, not RECONNECT. \
8609 is_reconnecting={}, is_closed={}, is_disconnected={}",
8610 client.is_reconnecting(),
8611 client.is_closed(),
8612 client.is_disconnected(),
8613 );
8614 assert!(
8615 !client.is_reconnecting(),
8616 "Stream mode should never attempt reconnection"
8617 );
8618
8619 server.abort();
8620 }
8621
8622 #[derive(Default)]
8623 struct BlockingFailState {
8624 send_entered: AtomicBool,
8625 send_entered_notify: tokio::sync::Notify,
8626 released: AtomicBool,
8627 fail: AtomicBool,
8628 waker: parking_lot::Mutex<Option<std::task::Waker>>,
8629 }
8630
8631 impl BlockingFailState {
8632 fn trigger_failure(&self) {
8633 self.fail.store(true, Ordering::SeqCst);
8634 self.release_send();
8635 }
8636
8637 fn release_send(&self) {
8638 self.released.store(true, Ordering::SeqCst);
8639
8640 if let Some(waker) = self.waker.lock().take() {
8641 waker.wake();
8642 }
8643 }
8644 }
8645
8646 struct BlockingFailTransport {
8648 state: Arc<BlockingFailState>,
8649 }
8650
8651 impl futures_util::Stream for BlockingFailTransport {
8652 type Item = Result<Message, TransportError>;
8653
8654 fn poll_next(
8655 self: std::pin::Pin<&mut Self>,
8656 _cx: &mut std::task::Context<'_>,
8657 ) -> std::task::Poll<Option<Self::Item>> {
8658 std::task::Poll::Pending
8659 }
8660 }
8661
8662 impl futures_util::Sink<Message> for BlockingFailTransport {
8663 type Error = TransportError;
8664
8665 fn poll_ready(
8666 self: std::pin::Pin<&mut Self>,
8667 _cx: &mut std::task::Context<'_>,
8668 ) -> std::task::Poll<Result<(), Self::Error>> {
8669 std::task::Poll::Ready(Ok(()))
8670 }
8671
8672 fn start_send(self: std::pin::Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8673 Ok(())
8674 }
8675
8676 fn poll_flush(
8677 self: std::pin::Pin<&mut Self>,
8678 cx: &mut std::task::Context<'_>,
8679 ) -> std::task::Poll<Result<(), Self::Error>> {
8680 *self.state.waker.lock() = Some(cx.waker().clone());
8683 self.state.send_entered.store(true, Ordering::SeqCst);
8684 self.state.send_entered_notify.notify_one();
8685
8686 if !self.state.released.load(Ordering::SeqCst) {
8687 std::task::Poll::Pending
8688 } else if self.state.fail.load(Ordering::SeqCst) {
8689 std::task::Poll::Ready(Err(TransportError::ConnectionReset))
8690 } else {
8691 std::task::Poll::Ready(Ok(()))
8692 }
8693 }
8694
8695 fn poll_close(
8696 self: std::pin::Pin<&mut Self>,
8697 _cx: &mut std::task::Context<'_>,
8698 ) -> std::task::Poll<Result<(), Self::Error>> {
8699 std::task::Poll::Ready(Ok(()))
8700 }
8701 }
8702
8703 struct BlockingMessageState {
8704 polled_tx: Mutex<Option<std::sync::mpsc::Sender<()>>>,
8705 release: (Mutex<bool>, parking_lot::Condvar),
8706 message: Mutex<Option<Message>>,
8707 }
8708
8709 struct BlockingMessageTransport {
8710 state: Arc<BlockingMessageState>,
8711 }
8712
8713 impl futures_util::Stream for BlockingMessageTransport {
8714 type Item = Result<Message, TransportError>;
8715
8716 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8717 if let Some(polled_tx) = self.state.polled_tx.lock().take() {
8718 polled_tx.send(()).unwrap();
8719 }
8720
8721 let (lock, condvar) = &self.state.release;
8722 let mut released = lock.lock();
8723 while !*released {
8724 condvar.wait(&mut released);
8725 }
8726
8727 Poll::Ready(self.state.message.lock().take().map(Ok))
8728 }
8729 }
8730
8731 impl futures_util::Sink<Message> for BlockingMessageTransport {
8732 type Error = TransportError;
8733
8734 fn poll_ready(
8735 self: Pin<&mut Self>,
8736 _cx: &mut Context<'_>,
8737 ) -> Poll<Result<(), Self::Error>> {
8738 Poll::Ready(Ok(()))
8739 }
8740
8741 fn start_send(self: Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8742 Ok(())
8743 }
8744
8745 fn poll_flush(
8746 self: Pin<&mut Self>,
8747 _cx: &mut Context<'_>,
8748 ) -> Poll<Result<(), Self::Error>> {
8749 Poll::Ready(Ok(()))
8750 }
8751
8752 fn poll_close(
8753 self: Pin<&mut Self>,
8754 _cx: &mut Context<'_>,
8755 ) -> Poll<Result<(), Self::Error>> {
8756 Poll::Ready(Ok(()))
8757 }
8758 }
8759
8760 struct RecordingState {
8761 messages: Arc<Mutex<Vec<Message>>>,
8762 recorded_notify: tokio::sync::Notify,
8763 }
8764
8765 struct RecordingTransport {
8766 state: Arc<RecordingState>,
8767 }
8768
8769 impl futures_util::Stream for RecordingTransport {
8770 type Item = Result<Message, TransportError>;
8771
8772 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8773 Poll::Pending
8774 }
8775 }
8776
8777 impl futures_util::Sink<Message> for RecordingTransport {
8778 type Error = TransportError;
8779
8780 fn poll_ready(
8781 self: Pin<&mut Self>,
8782 _cx: &mut Context<'_>,
8783 ) -> Poll<Result<(), Self::Error>> {
8784 Poll::Ready(Ok(()))
8785 }
8786
8787 fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> {
8788 self.state.messages.lock().push(item);
8789 self.state.recorded_notify.notify_one();
8790 Ok(())
8791 }
8792
8793 fn poll_flush(
8794 self: Pin<&mut Self>,
8795 _cx: &mut Context<'_>,
8796 ) -> Poll<Result<(), Self::Error>> {
8797 Poll::Ready(Ok(()))
8798 }
8799
8800 fn poll_close(
8801 self: Pin<&mut Self>,
8802 _cx: &mut Context<'_>,
8803 ) -> Poll<Result<(), Self::Error>> {
8804 Poll::Ready(Ok(()))
8805 }
8806 }
8807
8808 #[rstest]
8809 #[tokio::test(start_paused = true)]
8810 async fn test_pong_is_bound_to_connection_epoch() {
8811 let initial_state = Arc::new(RecordingState {
8812 messages: Arc::new(Mutex::new(Vec::new())),
8813 recorded_notify: tokio::sync::Notify::new(),
8814 });
8815
8816 let initial_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8817 state: initial_state,
8818 });
8819
8820 let (writer, _reader) = initial_transport.split();
8821 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8822 let state_notify = Arc::new(tokio::sync::Notify::new());
8823 let auth_tracker = Arc::new(OnceLock::new());
8824 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
8825 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
8826 let write_task = WebSocketClientInner::spawn_write_task(
8827 Arc::clone(&connection_state),
8828 Arc::clone(&state_notify),
8829 Arc::new(AtomicBool::new(true)),
8830 writer,
8831 writer_rx,
8832 Arc::new(AtomicU64::new(0)),
8833 auth_tracker,
8834 reconnect_buffer_waits_for_auth,
8835 None,
8836 );
8837
8838 let recorded = Arc::new(Mutex::new(Vec::new()));
8839
8840 let replacement_state = Arc::new(RecordingState {
8841 messages: Arc::clone(&recorded),
8842 recorded_notify: tokio::sync::Notify::new(),
8843 });
8844
8845 let replacement_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8846 state: replacement_state,
8847 });
8848
8849 let (replacement_writer, _reader) = replacement_transport.split();
8850 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
8851 writer_tx
8852 .send(WriterCommand::Update(replacement_writer, update_tx))
8853 .unwrap();
8854 writer_tx
8855 .send_control(WriterCommand::SendPongOnConnection {
8856 data: b"stale-pong".to_vec(),
8857 connection_epoch: 0,
8858 })
8859 .unwrap();
8860
8861 tokio::time::advance(Duration::from_millis(100)).await;
8862 assert_eq!(update_rx.await.unwrap(), 1);
8863
8864 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8865 writer_tx
8866 .send_control(WriterCommand::SendOnConnection {
8867 message: Message::text("sentinel-1"),
8868 connection_epoch: 1,
8869 response_tx: sentinel_tx,
8870 })
8871 .unwrap();
8872
8873 sentinel_rx.await.unwrap().unwrap();
8874 assert_eq!(recorded.lock().as_slice(), &[Message::text("sentinel-1")]);
8875
8876 writer_tx
8877 .send_control(WriterCommand::SendPongOnConnection {
8878 data: b"fresh-pong".to_vec(),
8879 connection_epoch: 1,
8880 })
8881 .unwrap();
8882
8883 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8884 writer_tx
8885 .send_control(WriterCommand::SendOnConnection {
8886 message: Message::text("sentinel-2"),
8887 connection_epoch: 1,
8888 response_tx: sentinel_tx,
8889 })
8890 .unwrap();
8891
8892 sentinel_rx.await.unwrap().unwrap();
8893 assert_eq!(
8894 recorded.lock().as_slice(),
8895 &[
8896 Message::text("sentinel-1"),
8897 Message::Pong(b"fresh-pong".to_vec().into()),
8898 Message::text("sentinel-2"),
8899 ]
8900 );
8901
8902 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
8903 state_notify.notify_waiters();
8904 drop(writer_tx);
8905 write_task.await.unwrap();
8906 }
8907
8908 #[rstest]
8909 #[case(Message::text("stale"))]
8910 #[case(Message::Binary(vec![1, 2, 3].into()))]
8911 #[case(Message::Ping(vec![1, 2, 3].into()))]
8912 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8913 async fn test_message_handler_drops_old_session_message(#[case] message: Message) {
8914 let (polled_tx, polled_rx) = std::sync::mpsc::channel();
8915
8916 let state = Arc::new(BlockingMessageState {
8917 polled_tx: Mutex::new(Some(polled_tx)),
8918 release: (Mutex::new(false), Condvar::new()),
8919 message: Mutex::new(Some(message)),
8920 });
8921
8922 let release_guard = CondvarReleaseGuard::new(&state.release);
8923
8924 let transport: BoxedWsTransport = Box::pin(BlockingMessageTransport {
8925 state: Arc::clone(&state),
8926 });
8927
8928 let (_writer, reader) = transport.split();
8929 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8930 let state_notify = Arc::new(tokio::sync::Notify::new());
8931 let read_fence = ReadSessionFence::new();
8932 let message_count = Arc::new(AtomicUsize::new(0));
8933 let ping_count = Arc::new(AtomicUsize::new(0));
8934 let message_count_clone = Arc::clone(&message_count);
8935 let ping_count_clone = Arc::clone(&ping_count);
8936 let message_handler: MessageHandler =
8937 Arc::new(move |_| _ = message_count_clone.fetch_add(1, Ordering::SeqCst));
8938 let message_handler = IncomingHandler::Message(message_handler);
8939 let ping_handler: PingHandler =
8940 Arc::new(move |_| _ = ping_count_clone.fetch_add(1, Ordering::SeqCst));
8941 let ping_handler = IncomingPingHandler::Ping(ping_handler);
8942
8943 let read_task = WebSocketClientInner::spawn_message_handler_task(
8944 Arc::clone(&connection_state),
8945 state_notify,
8946 read_fence.clone(),
8947 reader,
8948 0,
8949 Some(&message_handler),
8950 Some(&ping_handler),
8951 None,
8952 None,
8953 None,
8954 None,
8955 None,
8956 );
8957
8958 recv_rendezvous(polled_rx, "WebSocket reader poll entry").await;
8959 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8960 read_fence.invalidate();
8961 read_task.abort();
8962 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
8963 release_guard.release();
8964 await_task_termination(read_task, "old WebSocket read task").await;
8965
8966 assert_eq!(message_count.load(Ordering::SeqCst), 0);
8967 assert_eq!(ping_count.load(Ordering::SeqCst), 0);
8968 }
8969
8970 #[rstest]
8971 #[tokio::test]
8972 async fn test_reconnect_buffer_drain_stops_after_reconnect_request() {
8973 let state = Arc::new(BlockingFailState::default());
8974
8975 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
8976 state: Arc::clone(&state),
8977 });
8978
8979 let (mut writer, _reader) = transport.split();
8980 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8981 let task_connection_state = Arc::clone(&connection_state);
8982
8983 let drain_task = tokio::spawn(async move {
8984 let auth_tracker = Arc::new(OnceLock::new());
8985 let reconnect_buffer_waits_for_auth = AtomicBool::new(false);
8986 let (sender, mut receiver) = crate::writer::channel(2);
8987 sender.send(Message::text("admitted")).unwrap();
8988 sender.send(Message::text("held-for-reconnect")).unwrap();
8989 let mut buffer = VecDeque::from([
8990 receiver.recv().await.unwrap(),
8991 receiver.recv().await.unwrap(),
8992 ]);
8993 let send_error = WebSocketClientInner::drain_reconnect_buffer(
8994 &mut buffer,
8995 &mut writer,
8996 &task_connection_state,
8997 &auth_tracker,
8998 &reconnect_buffer_waits_for_auth,
8999 )
9000 .await;
9001 (buffer, send_error)
9002 });
9003
9004 state.send_entered_notify.notified().await;
9005 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
9006 state.release_send();
9007
9008 let (buffer, send_error) = tokio::time::timeout(TEST_TIMEOUT, drain_task)
9009 .await
9010 .expect("buffer drain should stop after reconnect acceptance")
9011 .unwrap();
9012 assert!(!send_error);
9013 assert_eq!(
9014 buffer
9015 .into_iter()
9016 .map(|(message, _)| message)
9017 .collect::<Vec<_>>(),
9018 vec![Message::text("held-for-reconnect")]
9019 );
9020 }
9021
9022 #[rstest]
9023 #[case(0)]
9024 #[case(usize::MAX)]
9025 #[tokio::test]
9026 async fn test_stream_writer_rejects_invalid_capacity(#[case] capacity: usize) {
9027 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9028 state: Arc::new(BlockingFailState::default()),
9029 });
9030
9031 let (writer, _reader) = transport.split();
9032 let mut config = WebSocketConfig::builder()
9033 .url("ws://localhost".to_string())
9034 .build()
9035 .unwrap();
9036 config.writer_capacity = Some(capacity);
9037
9038 let result = WebSocketClientInner::new_with_writer(config, writer).await;
9039
9040 assert!(
9041 matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9042 );
9043 }
9044
9045 #[rstest]
9046 #[case(0)]
9047 #[case(usize::MAX)]
9048 #[tokio::test]
9049 async fn test_handlerless_connect_rejects_invalid_capacity(#[case] capacity: usize) {
9050 let mut config = WebSocketConfig::builder()
9051 .url("ws://127.0.0.1:1".to_string())
9052 .build()
9053 .unwrap();
9054 config.writer_capacity = Some(capacity);
9055
9056 let result = WebSocketClientInner::connect_url(config, None, None).await;
9057
9058 assert!(
9059 matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9060 );
9061 }
9062
9063 #[rstest]
9064 #[tokio::test(start_paused = true)]
9065 async fn test_stalled_websocket_send_reconnects_and_replays() {
9066 let state = Arc::new(BlockingFailState::default());
9067
9068 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9069 state: Arc::clone(&state),
9070 });
9071
9072 let (writer, _reader) = transport.split();
9073 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9074 let state_notify = Arc::new(tokio::sync::Notify::new());
9075 let auth_tracker = Arc::new(OnceLock::new());
9076 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9077 let (writer_tx, writer_rx) = crate::writer::channel(2);
9078 let states = Arc::new(Mutex::new(Vec::new()));
9079 let states_callback = Arc::clone(&states);
9080 let sink = SocketStateSink::new(move |state| {
9081 states_callback.lock().push(state);
9082 });
9083
9084 let write_task = WebSocketClientInner::spawn_write_task(
9085 Arc::clone(&connection_state),
9086 Arc::clone(&state_notify),
9087 Arc::new(AtomicBool::new(true)),
9088 writer,
9089 writer_rx,
9090 Arc::new(AtomicU64::new(0)),
9091 Arc::clone(&auth_tracker),
9092 reconnect_buffer_waits_for_auth,
9093 Some(sink),
9094 );
9095
9096 writer_tx
9097 .send(WriterCommand::Send(Message::text("complete-message")))
9098 .unwrap();
9099 state.send_entered_notify.notified().await;
9100 writer_tx
9101 .send(WriterCommand::Send(Message::text("second-message")))
9102 .unwrap();
9103 assert_eq!(
9104 ConnectionMode::from_atomic(&connection_state),
9105 ConnectionMode::Active
9106 );
9107
9108 for _ in 0..100 {
9109 assert!(matches!(
9110 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9111 Err(SendError::BufferFull)
9112 ));
9113 }
9114
9115 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9116 tokio::task::yield_now().await;
9117 assert_eq!(
9118 ConnectionMode::from_atomic(&connection_state),
9119 ConnectionMode::Reconnect
9120 );
9121 assert!(matches!(
9122 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9123 Err(SendError::BufferFull)
9124 ));
9125
9126 let recorded = Arc::new(Mutex::new(Vec::new()));
9127
9128 let recording_state = Arc::new(RecordingState {
9129 messages: Arc::clone(&recorded),
9130 recorded_notify: tokio::sync::Notify::new(),
9131 });
9132
9133 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9134 state: Arc::clone(&recording_state),
9135 });
9136
9137 let (new_writer, _reader) = transport.split();
9138 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9139 writer_tx
9140 .send_update(WriterCommand::Update(new_writer, update_tx))
9141 .unwrap();
9142
9143 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9144 assert_eq!(
9145 tokio::time::timeout(Duration::from_secs(1), update_rx)
9146 .await
9147 .expect("writer update should not remain queued behind a stalled send")
9148 .unwrap(),
9149 1,
9150 "the replacement sink should install as connection epoch 1"
9151 );
9152 assert_eq!(
9153 ConnectionMode::from_atomic(&connection_state),
9154 ConnectionMode::Reconnect
9155 );
9156
9157 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9158 state_notify.notify_waiters();
9159 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9160 recording_state.recorded_notify.notified().await;
9161 assert_eq!(
9162 recorded.lock().as_slice(),
9163 &[
9164 Message::text("complete-message"),
9165 Message::text("second-message")
9166 ]
9167 );
9168
9169 writer_tx
9170 .send(WriterCommand::Send(Message::text("second-message")))
9171 .unwrap();
9172
9173 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9174 state_notify.notify_waiters();
9175 drop(writer_tx);
9176 write_task.await.unwrap();
9177
9178 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
9179 }
9180
9181 #[rstest]
9182 #[case(Message::Ping(vec![1, 2, 3].into()))]
9183 #[case(Message::Pong(vec![4, 5, 6].into()))]
9184 #[case(Message::Close(None))]
9185 #[tokio::test(start_paused = true)]
9186 async fn test_stalled_control_frame_is_not_replayed(#[case] control: Message) {
9187 let state = Arc::new(BlockingFailState::default());
9190
9191 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9192 state: Arc::clone(&state),
9193 });
9194
9195 let (writer, _reader) = transport.split();
9196 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9197 let state_notify = Arc::new(tokio::sync::Notify::new());
9198 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9199 let write_task = WebSocketClientInner::spawn_write_task(
9200 Arc::clone(&connection_state),
9201 Arc::clone(&state_notify),
9202 Arc::new(AtomicBool::new(true)),
9203 writer,
9204 writer_rx,
9205 Arc::new(AtomicU64::new(0)),
9206 Arc::new(OnceLock::new()),
9207 Arc::new(AtomicBool::new(false)),
9208 None,
9209 );
9210
9211 writer_tx.send(WriterCommand::Send(control)).unwrap();
9212 state.send_entered_notify.notified().await;
9213
9214 let recorded = Arc::new(Mutex::new(Vec::new()));
9215
9216 let recording_state = Arc::new(RecordingState {
9217 messages: Arc::clone(&recorded),
9218 recorded_notify: tokio::sync::Notify::new(),
9219 });
9220
9221 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9222 state: recording_state,
9223 });
9224
9225 let (new_writer, _reader) = transport.split();
9226 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9227 writer_tx
9228 .send(WriterCommand::Update(new_writer, update_tx))
9229 .unwrap();
9230
9231 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9232 assert_eq!(
9233 tokio::time::timeout(Duration::from_secs(1), update_rx)
9234 .await
9235 .expect("writer update should not remain queued behind a stalled send")
9236 .unwrap(),
9237 1,
9238 "the replacement sink should install as connection epoch 1"
9239 );
9240 assert_eq!(
9241 ConnectionMode::from_atomic(&connection_state),
9242 ConnectionMode::Reconnect,
9243 "a failed control-frame write should still trigger a reconnect"
9244 );
9245
9246 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9247 state_notify.notify_waiters();
9248
9249 for _ in 0..5 {
9251 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9252 }
9253
9254 let replayed = recorded.lock().clone();
9255 assert!(
9256 replayed.is_empty(),
9257 "a failed control frame must not reach the replacement connection, was {replayed:?}"
9258 );
9259
9260 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9261 state_notify.notify_waiters();
9262 drop(writer_tx);
9263 write_task.await.unwrap();
9264 }
9265
9266 #[rstest]
9267 #[case(Message::Ping(vec![1, 2, 3].into()))]
9268 #[case(Message::Pong(vec![4, 5, 6].into()))]
9269 #[case(Message::Close(None))]
9270 #[tokio::test(start_paused = true)]
9271 async fn test_control_frame_enqueued_during_reconnect_is_not_replayed(
9272 #[case] control: Message,
9273 ) {
9274 let recorded = Arc::new(Mutex::new(Vec::new()));
9277
9278 let recording_state = Arc::new(RecordingState {
9279 messages: Arc::clone(&recorded),
9280 recorded_notify: tokio::sync::Notify::new(),
9281 });
9282
9283 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9284 state: recording_state,
9285 });
9286
9287 let (writer, _reader) = transport.split();
9288 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9289 let state_notify = Arc::new(tokio::sync::Notify::new());
9290 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9291 let write_task = WebSocketClientInner::spawn_write_task(
9292 Arc::clone(&connection_state),
9293 Arc::clone(&state_notify),
9294 Arc::new(AtomicBool::new(true)),
9295 writer,
9296 writer_rx,
9297 Arc::new(AtomicU64::new(0)),
9298 Arc::new(OnceLock::new()),
9299 Arc::new(AtomicBool::new(false)),
9300 None,
9301 );
9302
9303 writer_tx.send(WriterCommand::Send(control)).unwrap();
9304 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9305
9306 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9307 state_notify.notify_waiters();
9308
9309 for _ in 0..5 {
9311 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9312 }
9313
9314 let replayed = recorded.lock().clone();
9315 assert!(
9316 replayed.is_empty(),
9317 "a control frame enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9318 );
9319
9320 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9321 state_notify.notify_waiters();
9322 drop(writer_tx);
9323 write_task.await.unwrap();
9324 }
9325
9326 #[rstest]
9327 #[tokio::test(start_paused = true)]
9328 async fn test_stalled_text_heartbeat_is_not_replayed() {
9329 let state = Arc::new(BlockingFailState::default());
9330
9331 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9332 state: Arc::clone(&state),
9333 });
9334
9335 let (writer, _reader) = transport.split();
9336 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9337 let state_notify = Arc::new(tokio::sync::Notify::new());
9338 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9339 let write_task = WebSocketClientInner::spawn_write_task(
9340 Arc::clone(&connection_state),
9341 Arc::clone(&state_notify),
9342 Arc::new(AtomicBool::new(true)),
9343 writer,
9344 writer_rx,
9345 Arc::new(AtomicU64::new(0)),
9346 Arc::new(OnceLock::new()),
9347 Arc::new(AtomicBool::new(false)),
9348 None,
9349 );
9350
9351 writer_tx
9352 .send(WriterCommand::Heartbeat(Message::text(
9353 "{\"op\":\"heartbeat\"}",
9354 )))
9355 .unwrap();
9356 state.send_entered_notify.notified().await;
9357
9358 let recorded = Arc::new(Mutex::new(Vec::new()));
9359
9360 let recording_state = Arc::new(RecordingState {
9361 messages: Arc::clone(&recorded),
9362 recorded_notify: tokio::sync::Notify::new(),
9363 });
9364
9365 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9366 state: recording_state,
9367 });
9368
9369 let (new_writer, _reader) = transport.split();
9370 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9371 writer_tx
9372 .send(WriterCommand::Update(new_writer, update_tx))
9373 .unwrap();
9374
9375 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9376 assert_eq!(
9377 tokio::time::timeout(Duration::from_secs(1), update_rx)
9378 .await
9379 .expect("writer update should not remain queued behind a stalled send")
9380 .unwrap(),
9381 1,
9382 "the replacement sink should install as connection epoch 1"
9383 );
9384 assert_eq!(
9385 ConnectionMode::from_atomic(&connection_state),
9386 ConnectionMode::Reconnect,
9387 "a failed heartbeat write should still trigger a reconnect"
9388 );
9389
9390 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9391 state_notify.notify_waiters();
9392
9393 for _ in 0..5 {
9394 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9395 }
9396
9397 let replayed = recorded.lock().clone();
9398 assert!(
9399 replayed.is_empty(),
9400 "a failed text heartbeat must not reach the replacement connection, was {replayed:?}"
9401 );
9402
9403 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9404 state_notify.notify_waiters();
9405 drop(writer_tx);
9406 write_task.await.unwrap();
9407 }
9408
9409 #[rstest]
9410 #[tokio::test(start_paused = true)]
9411 async fn test_text_heartbeat_enqueued_during_reconnect_is_not_replayed() {
9412 let recorded = Arc::new(Mutex::new(Vec::new()));
9413
9414 let recording_state = Arc::new(RecordingState {
9415 messages: Arc::clone(&recorded),
9416 recorded_notify: tokio::sync::Notify::new(),
9417 });
9418
9419 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9420 state: recording_state,
9421 });
9422
9423 let (writer, _reader) = transport.split();
9424 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9425 let state_notify = Arc::new(tokio::sync::Notify::new());
9426 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9427 let write_task = WebSocketClientInner::spawn_write_task(
9428 Arc::clone(&connection_state),
9429 Arc::clone(&state_notify),
9430 Arc::new(AtomicBool::new(true)),
9431 writer,
9432 writer_rx,
9433 Arc::new(AtomicU64::new(0)),
9434 Arc::new(OnceLock::new()),
9435 Arc::new(AtomicBool::new(false)),
9436 None,
9437 );
9438
9439 writer_tx
9440 .send(WriterCommand::Heartbeat(Message::text(
9441 "{\"op\":\"heartbeat\"}",
9442 )))
9443 .unwrap();
9444 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9445
9446 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9447 state_notify.notify_waiters();
9448
9449 for _ in 0..5 {
9450 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9451 }
9452
9453 let replayed = recorded.lock().clone();
9454 assert!(
9455 replayed.is_empty(),
9456 "a text heartbeat enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9457 );
9458
9459 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9460 state_notify.notify_waiters();
9461 drop(writer_tx);
9462 write_task.await.unwrap();
9463 }
9464
9465 #[rstest]
9466 #[case::text(
9467 Some("{\"op\":\"heartbeat\"}"),
9468 Message::text("{\"op\":\"heartbeat\"}")
9469 )]
9470 #[case::ping(None, Message::Ping(vec![].into()))]
9471 #[tokio::test(start_paused = true)]
9472 async fn test_heartbeat_task_enqueues_writer_heartbeat_command(
9473 #[case] payload: Option<&str>,
9474 #[case] expected: Message,
9475 ) {
9476 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9477 let (writer_tx, mut writer_rx) =
9478 crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9479 let task = WebSocketClientInner::spawn_heartbeat_task(
9480 Arc::clone(&connection_state),
9481 1,
9482 payload.map(ToString::to_string),
9483 writer_tx,
9484 );
9485
9486 tokio::time::advance(Duration::from_secs(1)).await;
9487 let cmd = writer_rx
9488 .recv()
9489 .await
9490 .expect("heartbeat task should enqueue");
9491
9492 match cmd.0 {
9493 WriterCommand::Heartbeat(msg) => assert_eq!(msg, expected),
9494 other => panic!("expected Heartbeat, was {other:?}"),
9495 }
9496
9497 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9498 tokio::time::advance(Duration::from_secs(1)).await;
9499 task.await.unwrap();
9500 }
9501
9502 #[cfg(target_os = "linux")]
9503 #[rstest]
9504 #[tokio::test]
9505 async fn test_heartbeat_task_warns_when_writer_channel_closed() {
9506 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
9507 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9508 let (writer_tx, mut writer_rx) =
9509 crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9510 let task = WebSocketClientInner::spawn_heartbeat_task(
9511 Arc::clone(&connection_state),
9512 1,
9513 None,
9514 writer_tx,
9515 );
9516
9517 let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
9518 .await
9519 .expect("timed out waiting for the first heartbeat")
9520 .expect("heartbeat channel closed before the first heartbeat");
9521 drop(writer_rx);
9522 tokio::time::sleep(Duration::from_millis(1_200)).await;
9523
9524 let messages = capture.messages();
9525 assert!(
9526 messages.iter().any(|(level, message)| {
9527 *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
9528 }),
9529 "expected WARN when the writer channel is closed, was {messages:?}"
9530 );
9531 assert!(
9532 !messages.iter().any(|(level, _)| *level == Level::Error),
9533 "a closed writer channel must not log ERROR, was {messages:?}"
9534 );
9535
9536 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9537 tokio::time::timeout(Duration::from_secs(2), task)
9538 .await
9539 .expect("heartbeat task should stop after close")
9540 .unwrap();
9541 }
9542
9543 #[rstest]
9544 #[tokio::test(start_paused = true)]
9545 async fn test_stalled_ownership_bound_send_times_out_without_replay() {
9546 let state = Arc::new(BlockingFailState::default());
9547
9548 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9549 state: Arc::clone(&state),
9550 });
9551
9552 let (writer, _reader) = transport.split();
9553 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9554 let state_notify = Arc::new(tokio::sync::Notify::new());
9555 let auth_tracker = Arc::new(OnceLock::new());
9556 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9557 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9558 let write_task = WebSocketClientInner::spawn_write_task(
9559 Arc::clone(&connection_state),
9560 Arc::clone(&state_notify),
9561 Arc::new(AtomicBool::new(true)),
9562 writer,
9563 writer_rx,
9564 Arc::new(AtomicU64::new(0)),
9565 Arc::clone(&auth_tracker),
9566 reconnect_buffer_waits_for_auth,
9567 None,
9568 );
9569
9570 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
9571 writer_tx
9572 .send_control(WriterCommand::SendOnConnection {
9573 message: Message::text("ownership-bound"),
9574 connection_epoch: 0,
9575 response_tx,
9576 })
9577 .unwrap();
9578
9579 state.send_entered_notify.notified().await;
9580
9581 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9582 let outcome = tokio::time::timeout(Duration::from_secs(1), response_rx)
9583 .await
9584 .expect("a stalled ownership-bound send must not wedge the writer task")
9585 .unwrap();
9586 assert!(
9587 matches!(outcome, Err(SendError::WriteTimeout)),
9588 "expected the write deadline to be reported, was {outcome:?}"
9589 );
9590 assert_eq!(
9591 ConnectionMode::from_atomic(&connection_state),
9592 ConnectionMode::Reconnect
9593 );
9594
9595 let recorded = Arc::new(Mutex::new(Vec::new()));
9598
9599 let recording_state = Arc::new(RecordingState {
9600 messages: Arc::clone(&recorded),
9601 recorded_notify: tokio::sync::Notify::new(),
9602 });
9603
9604 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9605 state: Arc::clone(&recording_state),
9606 });
9607
9608 let (new_writer, _reader) = transport.split();
9609 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9610 writer_tx
9611 .send(WriterCommand::Update(new_writer, update_tx))
9612 .unwrap();
9613
9614 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9615 assert_eq!(
9616 tokio::time::timeout(Duration::from_secs(1), update_rx)
9617 .await
9618 .expect("writer update should not remain queued behind a stalled send")
9619 .unwrap(),
9620 1
9621 );
9622
9623 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9624 state_notify.notify_waiters();
9625 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9626
9627 for name in ["sentinel-1", "sentinel-2"] {
9633 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
9634 writer_tx
9635 .send_control(WriterCommand::SendOnConnection {
9636 message: Message::text(name),
9637 connection_epoch: 1,
9638 response_tx: sentinel_tx,
9639 })
9640 .unwrap();
9641
9642 recording_state.recorded_notify.notified().await;
9643 sentinel_rx
9644 .await
9645 .unwrap()
9646 .expect("the sentinel should send on the replacement connection");
9647 }
9648
9649 assert_eq!(
9650 recorded.lock().as_slice(),
9651 &[Message::text("sentinel-1"), Message::text("sentinel-2")],
9652 "an ownership-bound message must never be replayed after its deadline expires"
9653 );
9654
9655 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9656 state_notify.notify_waiters();
9657 drop(writer_tx);
9658 write_task.await.unwrap();
9659 }
9660
9661 #[rstest]
9662 #[tokio::test(start_paused = true)]
9663 async fn test_stalled_websocket_replay_reconnects_and_retries_buffer() {
9664 let initial_messages = Arc::new(Mutex::new(Vec::new()));
9665
9666 let initial_recording_state = Arc::new(RecordingState {
9667 messages: initial_messages,
9668 recorded_notify: tokio::sync::Notify::new(),
9669 });
9670
9671 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9672 state: initial_recording_state,
9673 });
9674
9675 let (writer, _reader) = transport.split();
9676 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9677 let state_notify = Arc::new(tokio::sync::Notify::new());
9678 let auth_tracker = Arc::new(OnceLock::new());
9679 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9680 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9681 let write_task = WebSocketClientInner::spawn_write_task(
9682 Arc::clone(&connection_state),
9683 Arc::clone(&state_notify),
9684 Arc::new(AtomicBool::new(true)),
9685 writer,
9686 writer_rx,
9687 Arc::new(AtomicU64::new(0)),
9688 Arc::clone(&auth_tracker),
9689 reconnect_buffer_waits_for_auth,
9690 None,
9691 );
9692
9693 writer_tx
9694 .send(WriterCommand::Send(Message::text("buffered-message")))
9695 .unwrap();
9696
9697 let blocking_state = Arc::new(BlockingFailState::default());
9698
9699 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9700 state: Arc::clone(&blocking_state),
9701 });
9702
9703 let (blocking_writer, _reader) = transport.split();
9704 let (blocking_tx, blocking_rx) = tokio::sync::oneshot::channel();
9705 writer_tx
9706 .send(WriterCommand::Update(blocking_writer, blocking_tx))
9707 .unwrap();
9708 assert_eq!(blocking_rx.await.unwrap(), 1);
9709
9710 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9711 state_notify.notify_waiters();
9712 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9713 blocking_state.send_entered_notify.notified().await;
9714
9715 let recorded = Arc::new(Mutex::new(Vec::new()));
9716
9717 let recording_state = Arc::new(RecordingState {
9718 messages: Arc::clone(&recorded),
9719 recorded_notify: tokio::sync::Notify::new(),
9720 });
9721
9722 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9723 state: Arc::clone(&recording_state),
9724 });
9725
9726 let (new_writer, _reader) = transport.split();
9727 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9728 writer_tx
9729 .send(WriterCommand::Update(new_writer, update_tx))
9730 .unwrap();
9731
9732 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9733 assert_eq!(
9734 tokio::time::timeout(Duration::from_secs(1), update_rx)
9735 .await
9736 .expect("writer update should not remain queued behind stalled replay")
9737 .unwrap(),
9738 2,
9739 "the second replacement sink should install as connection epoch 2"
9740 );
9741 assert_eq!(
9742 ConnectionMode::from_atomic(&connection_state),
9743 ConnectionMode::Reconnect
9744 );
9745
9746 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9747 state_notify.notify_waiters();
9748 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9749 recording_state.recorded_notify.notified().await;
9750 assert_eq!(
9751 recorded.lock().as_slice(),
9752 &[Message::text("buffered-message")]
9753 );
9754
9755 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9756 state_notify.notify_waiters();
9757 drop(writer_tx);
9758 write_task.await.unwrap();
9759 }
9760
9761 #[rstest]
9762 #[tokio::test]
9763 async fn test_new_with_writer_rejects_zero_heartbeat() {
9764 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9767 state: Arc::new(BlockingFailState::default()),
9768 });
9769
9770 let (writer, _reader) = transport.split();
9771
9772 let config = WebSocketConfig {
9773 url: "ws://127.0.0.1:1".to_string(),
9774 headers: vec![],
9775 heartbeat_interval_secs: Some(0),
9776 heartbeat_payload: None,
9777 connect_timeout_ms: None,
9778 reconnect_delay_initial_ms: None,
9779 reconnect_delay_max_ms: None,
9780 reconnect_backoff_factor: None,
9781 reconnect_jitter_ms: None,
9782 reconnect_max_attempts: None,
9783 heartbeat_timeout_secs: None,
9784 idle_timeout_ms: None,
9785 writer_capacity: None,
9786 backend: TransportBackend::Tungstenite,
9787 proxy_url: None,
9788 max_message_size_bytes: None,
9789 max_frame_size_bytes: None,
9790 };
9791
9792 let err = WebSocketClientInner::new_with_writer(config, writer)
9793 .await
9794 .expect_err("zero heartbeat should be rejected in stream mode");
9795 assert!(
9796 err.to_string()
9797 .contains("Heartbeat interval cannot be zero"),
9798 "error should mention zero heartbeat, was: {err}"
9799 );
9800 }
9801
9802 #[rstest]
9803 #[tokio::test]
9804 async fn test_connect_times_out_on_silent_server() {
9805 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9808 let port = listener.local_addr().unwrap().port();
9809
9810 let server = task::spawn(async move {
9811 if let Ok((_stream, _)) = listener.accept().await {
9813 sleep(Duration::from_secs(30)).await;
9814 }
9815 });
9816
9817 let (handler, _rx) = channel_message_handler();
9818
9819 let config = WebSocketConfig {
9820 url: format!("ws://127.0.0.1:{port}"),
9821 headers: vec![],
9822 heartbeat_interval_secs: None,
9823 heartbeat_payload: None,
9824 connect_timeout_ms: Some(500),
9825 reconnect_delay_initial_ms: Some(50),
9826 reconnect_delay_max_ms: Some(100),
9827 reconnect_backoff_factor: Some(1.0),
9828 reconnect_jitter_ms: Some(0),
9829 reconnect_max_attempts: None,
9830 heartbeat_timeout_secs: None,
9831 idle_timeout_ms: None,
9832 writer_capacity: None,
9833 backend: TransportBackend::Tungstenite,
9834 proxy_url: None,
9835 max_message_size_bytes: None,
9836 max_frame_size_bytes: None,
9837 };
9838
9839 let result = tokio::time::timeout(
9840 Duration::from_secs(5),
9841 WebSocketClient::builder()
9842 .config(config)
9843 .message_handler(handler)
9844 .connect(),
9845 )
9846 .await
9847 .expect("connect should not hang on a silent server");
9848
9849 assert!(result.is_err(), "connect should fail with a timeout error");
9850 let err_msg = result.unwrap_err().to_string();
9851 assert!(
9852 err_msg.contains("timed out"),
9853 "error should mention the timeout, was: {err_msg}"
9854 );
9855
9856 server.abort();
9857 }
9858
9859 #[rstest]
9860 #[tokio::test]
9861 async fn test_reconnect_succeeds_with_timeout_shorter_than_swap_ceremony() {
9862 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9868 let port = listener.local_addr().unwrap().port();
9869 let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel();
9870
9871 let server = task::spawn(async move {
9872 if let Ok((stream, _)) = listener.accept().await
9875 && let Ok(ws) = accept_async(stream).await
9876 {
9877 let _ = release_first_rx.await;
9878 drop(ws);
9879 }
9880
9881 loop {
9884 let Ok((stream, _)) = listener.accept().await else {
9885 break;
9886 };
9887
9888 if let Ok(mut ws) = accept_async(stream).await {
9889 loop {
9890 if ws
9891 .send(WsMessage::Text("reconnected-msg".to_string().into()))
9892 .await
9893 .is_err()
9894 {
9895 break;
9896 }
9897
9898 sleep(Duration::from_millis(50)).await;
9899 }
9900 }
9901 }
9902 });
9903
9904 let (handler, mut rx) = channel_message_handler();
9905
9906 let config = WebSocketConfig {
9907 url: format!("ws://127.0.0.1:{port}"),
9908 headers: vec![],
9909 heartbeat_interval_secs: None,
9910 heartbeat_payload: None,
9911 connect_timeout_ms: Some(150), reconnect_delay_initial_ms: Some(25),
9913 reconnect_delay_max_ms: Some(50),
9914 reconnect_backoff_factor: Some(1.0),
9915 reconnect_jitter_ms: Some(0),
9916 reconnect_max_attempts: None,
9917 heartbeat_timeout_secs: None,
9918 idle_timeout_ms: None,
9919 writer_capacity: None,
9920 backend: TransportBackend::Tungstenite,
9921 proxy_url: None,
9922 max_message_size_bytes: None,
9923 max_frame_size_bytes: None,
9924 };
9925
9926 let client = WebSocketClient::builder()
9927 .config(config)
9928 .message_handler(handler)
9929 .connect()
9930 .await
9931 .expect("Initial WebSocket connection should succeed before triggering reconnect");
9932 wait_until_async(|| async { client.is_active() }, Duration::from_secs(2)).await;
9933 release_first_tx
9934 .send(())
9935 .expect("server should still be holding the first connection");
9936
9937 let received = tokio::time::timeout(Duration::from_secs(5), async {
9938 loop {
9939 match rx.recv().await {
9940 Some(WsMessage::Text(text)) if text.as_str() == "reconnected-msg" => {
9941 return true;
9942 }
9943 Some(_) => {}
9944 None => return false,
9945 }
9946 }
9947 })
9948 .await
9949 .expect("Timed out waiting for a message from the replacement WebSocket connection");
9950
9951 assert!(
9952 received,
9953 "Replacement WebSocket message channel closed before delivering the reconnect message"
9954 );
9955 assert!(
9956 client.is_active(),
9957 "WebSocket client should be active after delivering the reconnect message"
9958 );
9959
9960 client.disconnect().await;
9961 server.abort();
9962 }
9963
9964 #[rstest]
9965 #[tokio::test]
9966 async fn test_idle_timeout_fires_under_ping_flood() {
9967 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9971 let port = listener.local_addr().unwrap().port();
9972 let server = task::spawn(async move {
9973 let (stream, _) = listener.accept().await.unwrap();
9974 let mut ws = accept_async(stream).await.unwrap();
9975
9976 for _ in 0..600 {
9977 sleep(Duration::from_millis(5)).await;
9978
9979 if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
9980 break;
9981 }
9982 }
9983 });
9984
9985 let (handler, _rx) = channel_message_handler();
9986
9987 let config = WebSocketConfig {
9988 url: format!("ws://127.0.0.1:{port}"),
9989 headers: vec![],
9990 heartbeat_interval_secs: None,
9991 heartbeat_payload: None,
9992 connect_timeout_ms: Some(2_000),
9993 reconnect_delay_initial_ms: Some(50),
9994 reconnect_delay_max_ms: Some(100),
9995 reconnect_backoff_factor: Some(1.0),
9996 reconnect_jitter_ms: Some(0),
9997 reconnect_max_attempts: Some(1),
9998 heartbeat_timeout_secs: None,
9999 idle_timeout_ms: Some(500),
10000 writer_capacity: None,
10001 backend: TransportBackend::Tungstenite,
10002 proxy_url: None,
10003 max_message_size_bytes: None,
10004 max_frame_size_bytes: None,
10005 };
10006
10007 let client = WebSocketClient::builder()
10008 .config(config)
10009 .message_handler(handler)
10010 .connect()
10011 .await
10012 .unwrap();
10013
10014 assert!(client.is_active());
10015
10016 wait_until_async(
10017 || async { client.is_reconnecting() || client.is_disconnected() },
10018 Duration::from_millis(1_500),
10019 )
10020 .await;
10021
10022 assert!(
10023 !client.is_active(),
10024 "Client should not be active after idle timeout under a ping flood"
10025 );
10026
10027 client.disconnect().await;
10028 server.abort();
10029 }
10030
10031 #[rstest]
10032 #[tokio::test]
10033 async fn test_send_failure_does_not_overwrite_disconnect() {
10034 let state = Arc::new(BlockingFailState::default());
10035
10036 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10037 state: Arc::clone(&state),
10038 });
10039
10040 let (writer, _reader) = transport.split();
10041
10042 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10043 let state_notify = Arc::new(tokio::sync::Notify::new());
10044 let auth_tracker = Arc::new(OnceLock::new());
10045 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
10046 let connection_epoch = Arc::new(AtomicU64::new(0));
10047
10048 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10049 let write_task = WebSocketClientInner::spawn_write_task(
10050 Arc::clone(&connection_state),
10051 Arc::clone(&state_notify),
10052 Arc::new(AtomicBool::new(true)),
10053 writer,
10054 writer_rx,
10055 connection_epoch,
10056 Arc::clone(&auth_tracker),
10057 Arc::clone(&reconnect_buffer_waits_for_auth),
10058 None,
10059 );
10060
10061 writer_tx
10062 .send(WriterCommand::Send(Message::text("doomed")))
10063 .unwrap();
10064
10065 wait_until_async(
10067 || {
10068 let state = Arc::clone(&state);
10069 async move { state.send_entered.load(Ordering::SeqCst) }
10070 },
10071 Duration::from_secs(2),
10072 )
10073 .await;
10074
10075 connection_state.store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
10078 state.trigger_failure();
10079
10080 tokio::time::timeout(Duration::from_secs(2), write_task)
10081 .await
10082 .expect("write task should exit after disconnect")
10083 .unwrap();
10084
10085 assert_eq!(
10086 ConnectionMode::from_atomic(&connection_state),
10087 ConnectionMode::Disconnect,
10088 "Send failure must not resurrect a disconnecting client into RECONNECT"
10089 );
10090 }
10091
10092 #[tokio::test]
10093 async fn send_on_connection_write_failure_reports_broken_pipe_and_reconnects() {
10094 let state = Arc::new(BlockingFailState::default());
10095
10096 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10097 state: Arc::clone(&state),
10098 });
10099
10100 let (writer, _reader) = transport.split();
10101 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10102 let state_notify = Arc::new(tokio::sync::Notify::new());
10103 let connection_epoch = Arc::new(AtomicU64::new(0));
10104 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10105 let write_task = WebSocketClientInner::spawn_write_task(
10106 Arc::clone(&connection_state),
10107 Arc::clone(&state_notify),
10108 Arc::new(AtomicBool::new(true)),
10109 writer,
10110 writer_rx,
10111 Arc::clone(&connection_epoch),
10112 Arc::new(OnceLock::new()),
10113 Arc::new(AtomicBool::new(false)),
10114 None,
10115 );
10116
10117 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10118 writer_tx
10119 .send_control(WriterCommand::SendOnConnection {
10120 message: Message::text("doomed"),
10121 connection_epoch: 0,
10122 response_tx,
10123 })
10124 .unwrap();
10125
10126 wait_until_async(
10127 || {
10128 let state = Arc::clone(&state);
10129 async move { state.send_entered.load(Ordering::SeqCst) }
10130 },
10131 Duration::from_secs(2),
10132 )
10133 .await;
10134
10135 state.trigger_failure();
10136
10137 match response_rx.await.unwrap().unwrap_err() {
10138 SendError::BrokenPipe(message) => assert_eq!(message, "connection reset"),
10139 other => panic!("expected broken-pipe send error, was {other:?}"),
10140 }
10141
10142 wait_until_async(
10143 || async {
10144 ConnectionMode::from_atomic(&connection_state) == ConnectionMode::Reconnect
10145 },
10146 Duration::from_secs(2),
10147 )
10148 .await;
10149
10150 assert_eq!(
10151 ConnectionMode::from_atomic(&connection_state),
10152 ConnectionMode::Reconnect,
10153 );
10154 assert_eq!(connection_epoch.load(Ordering::Acquire), 0);
10155
10156 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10157 state_notify.notify_waiters();
10158 drop(writer_tx);
10159 write_task.await.unwrap();
10160 }
10161
10162 #[rstest]
10163 #[tokio::test]
10164 async fn test_send_bytes_on_connection_rejects_stale_epoch_without_replay() {
10165 let server = RecordingServer::setup().await;
10166 let (handler, _handler_rx) = channel_message_handler();
10167 let client = WebSocketClient::builder()
10168 .config(reconnect_test_config(server.port))
10169 .message_handler(handler)
10170 .connect()
10171 .await
10172 .unwrap();
10173 let stale_epoch = client.connection_epoch();
10174
10175 client
10176 .send_bytes_on_connection(vec![0x00, 0x7f, 0x80, 0xff], None, stale_epoch)
10177 .await
10178 .unwrap();
10179 assert!(client.request_reconnect());
10180 wait_until_async(
10181 || async { client.is_active() && client.connection_epoch() == stale_epoch + 1 },
10182 TEST_TIMEOUT,
10183 )
10184 .await;
10185
10186 let stale_result = client
10187 .send_bytes_on_connection(vec![0xde, 0xad], None, stale_epoch)
10188 .await;
10189 assert!(matches!(stale_result, Err(SendError::ConnectionChanged)));
10190
10191 client
10192 .send_bytes_on_connection(
10193 vec![0xfe, 0xed, 0xfa, 0xce],
10194 None,
10195 client.connection_epoch(),
10196 )
10197 .await
10198 .unwrap();
10199 wait_until_async(
10200 || {
10201 let binary_frames = Arc::clone(&server.binary_frames);
10202 async move { binary_frames.lock().await.len() == 2 }
10203 },
10204 TEST_TIMEOUT,
10205 )
10206 .await;
10207
10208 assert_eq!(
10209 server.binary_frames().await,
10210 vec![vec![0x00, 0x7f, 0x80, 0xff], vec![0xfe, 0xed, 0xfa, 0xce]],
10211 );
10212 assert!(server.messages().await.is_empty());
10213
10214 client.disconnect().await;
10215 }
10216
10217 #[tokio::test]
10218 async fn send_on_connection_rejects_stale_epoch_without_replay() {
10219 let server = RecordingServer::setup().await;
10220 let url = format!("ws://127.0.0.1:{}", server.port);
10221 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10222 &url,
10223 vec![],
10224 TransportBackend::Tungstenite,
10225 None,
10226 None,
10227 None,
10228 )
10229 .await
10230 .unwrap();
10231 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10232 let state_notify = Arc::new(tokio::sync::Notify::new());
10233 let auth_tracker = Arc::new(OnceLock::new());
10234 let connection_epoch = Arc::new(AtomicU64::new(0));
10235 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10236 let write_task = WebSocketClientInner::spawn_write_task(
10237 Arc::clone(&connection_state),
10238 Arc::clone(&state_notify),
10239 Arc::new(AtomicBool::new(true)),
10240 writer,
10241 writer_rx,
10242 Arc::clone(&connection_epoch),
10243 auth_tracker,
10244 Arc::new(AtomicBool::new(false)),
10245 None,
10246 );
10247
10248 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10249 writer_tx
10250 .send_control(WriterCommand::SendOnConnection {
10251 message: Message::text("epoch-0"),
10252 connection_epoch: 0,
10253 response_tx,
10254 })
10255 .unwrap();
10256
10257 response_rx.await.unwrap().unwrap();
10258
10259 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10260 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10261 writer_tx
10262 .send_control(WriterCommand::SendOnConnection {
10263 message: Message::text("during-reconnect"),
10264 connection_epoch: 0,
10265 response_tx,
10266 })
10267 .unwrap();
10268
10269 assert!(matches!(
10270 response_rx.await.unwrap(),
10271 Err(SendError::ConnectionChanged),
10272 ));
10273
10274 let (replacement, _reader) = WebSocketClientInner::connect_with_server(
10275 &url,
10276 vec![],
10277 TransportBackend::Tungstenite,
10278 None,
10279 None,
10280 None,
10281 )
10282 .await
10283 .unwrap();
10284 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10285 writer_tx
10286 .send(WriterCommand::Update(replacement, update_tx))
10287 .unwrap();
10288 assert_eq!(update_rx.await.unwrap(), 1);
10289 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10290
10291 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10292 writer_tx
10293 .send_control(WriterCommand::SendOnConnection {
10294 message: Message::text("stale"),
10295 connection_epoch: 0,
10296 response_tx,
10297 })
10298 .unwrap();
10299
10300 assert!(matches!(
10301 response_rx.await.unwrap(),
10302 Err(SendError::ConnectionChanged),
10303 ));
10304
10305 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10306 writer_tx
10307 .send_control(WriterCommand::SendOnConnection {
10308 message: Message::text("epoch-1"),
10309 connection_epoch: 1,
10310 response_tx,
10311 })
10312 .unwrap();
10313
10314 response_rx.await.unwrap().unwrap();
10315
10316 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10317 let (second_replacement, _reader) = WebSocketClientInner::connect_with_server(
10318 &url,
10319 vec![],
10320 TransportBackend::Tungstenite,
10321 None,
10322 None,
10323 None,
10324 )
10325 .await
10326 .unwrap();
10327 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10328 writer_tx
10329 .send(WriterCommand::Update(second_replacement, update_tx))
10330 .unwrap();
10331 assert_eq!(update_rx.await.unwrap(), 2);
10332 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10333
10334 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10335 writer_tx
10336 .send_control(WriterCommand::SendOnConnection {
10337 message: Message::text("stale-after-second-reconnect"),
10338 connection_epoch: 1,
10339 response_tx,
10340 })
10341 .unwrap();
10342
10343 assert!(matches!(
10344 response_rx.await.unwrap(),
10345 Err(SendError::ConnectionChanged),
10346 ));
10347
10348 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10349 writer_tx
10350 .send_control(WriterCommand::SendOnConnection {
10351 message: Message::text("epoch-2"),
10352 connection_epoch: 2,
10353 response_tx,
10354 })
10355 .unwrap();
10356
10357 response_rx.await.unwrap().unwrap();
10358
10359 wait_until_async(
10360 || {
10361 let messages = Arc::clone(&server.messages);
10362 async move { messages.lock().await.len() == 3 }
10363 },
10364 Duration::from_secs(2),
10365 )
10366 .await;
10367
10368 assert_eq!(connection_epoch.load(Ordering::Acquire), 2);
10369 assert_eq!(
10370 server.messages().await,
10371 vec![
10372 "epoch-0".to_string(),
10373 "epoch-1".to_string(),
10374 "epoch-2".to_string(),
10375 ],
10376 );
10377
10378 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10379 state_notify.notify_waiters();
10380 drop(writer_tx);
10381 write_task.abort();
10382 }
10383
10384 #[rstest]
10385 fn test_reconnect_buffer_action_requires_active_mode() {
10386 let connection_state = AtomicU8::new(ConnectionMode::Active.as_u8());
10387 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
10388 let auth_tracker = Arc::new(OnceLock::new());
10389 let tracker = AuthTracker::new();
10390 tracker.succeed();
10391 auth_tracker.set(tracker).unwrap();
10392
10393 assert_eq!(
10394 WebSocketClientInner::reconnect_buffer_action(
10395 &reconnect_buffer_waits_for_auth,
10396 &auth_tracker,
10397 &connection_state,
10398 ),
10399 ReconnectBufferAction::Drain,
10400 );
10401
10402 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10403 assert_eq!(
10404 WebSocketClientInner::reconnect_buffer_action(
10405 &reconnect_buffer_waits_for_auth,
10406 &auth_tracker,
10407 &connection_state,
10408 ),
10409 ReconnectBufferAction::Wait,
10410 );
10411 }
10412
10413 #[tokio::test]
10414 async fn test_write_task_waits_for_auth_before_replaying_buffer() {
10415 use nautilus_common::testing::wait_until_async;
10416
10417 let server = RecordingServer::setup().await;
10418 let url = format!("ws://127.0.0.1:{}", server.port);
10419 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10420 &url,
10421 vec![],
10422 TransportBackend::Tungstenite,
10423 None,
10424 None,
10425 None,
10426 )
10427 .await
10428 .unwrap();
10429
10430 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10431 let state_notify = Arc::new(tokio::sync::Notify::new());
10432 let auth_tracker = Arc::new(OnceLock::new());
10433 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10434 let connection_epoch = Arc::new(AtomicU64::new(0));
10435 let tracker = AuthTracker::new();
10436 auth_tracker.set(tracker.clone()).unwrap();
10437
10438 let (writer_tx, writer_rx) = crate::writer::channel(1);
10439 let write_task = WebSocketClientInner::spawn_write_task(
10440 Arc::clone(&connection_state),
10441 Arc::clone(&state_notify),
10442 Arc::new(AtomicBool::new(true)),
10443 writer,
10444 writer_rx,
10445 Arc::clone(&connection_epoch),
10446 Arc::clone(&auth_tracker),
10447 Arc::clone(&reconnect_buffer_waits_for_auth),
10448 None,
10449 );
10450
10451 writer_tx
10452 .send(WriterCommand::Send(Message::Text("stale".into())))
10453 .unwrap();
10454
10455 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10456 &url,
10457 vec![],
10458 TransportBackend::Tungstenite,
10459 None,
10460 None,
10461 None,
10462 )
10463 .await
10464 .unwrap();
10465 let (tx, rx) = tokio::sync::oneshot::channel();
10466 writer_tx
10467 .send_update(WriterCommand::Update(new_writer, tx))
10468 .unwrap();
10469 assert_eq!(rx.await.unwrap(), 1);
10470
10471 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10472
10473 tokio::time::sleep(Duration::from_millis(300)).await;
10474 assert!(
10475 server.messages().await.is_empty(),
10476 "buffered messages should wait for re-authentication"
10477 );
10478
10479 assert!(matches!(
10480 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
10481 Err(SendError::BufferFull)
10482 ));
10483 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10484 writer_tx
10485 .send_control(WriterCommand::SendOnConnection {
10486 message: Message::text("authenticate"),
10487 connection_epoch: 1,
10488 response_tx,
10489 })
10490 .unwrap();
10491
10492 response_rx.await.unwrap().unwrap();
10493 tracker.succeed();
10494
10495 wait_until_async(
10496 || {
10497 let messages = Arc::clone(&server.messages);
10498 async move { messages.lock().await.len() == 2 }
10499 },
10500 Duration::from_secs(3),
10501 )
10502 .await;
10503
10504 assert_eq!(
10505 server.messages().await,
10506 vec!["authenticate".to_string(), "stale".to_string()]
10507 );
10508
10509 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10510 state_notify.notify_waiters();
10511 drop(writer_tx);
10512 write_task.abort();
10513 }
10514
10515 #[rstest]
10516 #[tokio::test]
10517 async fn test_write_task_buffers_send_behind_pending_replay() {
10518 use nautilus_common::testing::wait_until_async;
10519
10520 let server = RecordingServer::setup().await;
10521 let url = format!("ws://127.0.0.1:{}", server.port);
10522 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10523 &url,
10524 vec![],
10525 TransportBackend::Tungstenite,
10526 None,
10527 None,
10528 None,
10529 )
10530 .await
10531 .unwrap();
10532
10533 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10534 let state_notify = Arc::new(tokio::sync::Notify::new());
10535 let auth_tracker = Arc::new(OnceLock::new());
10536 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10537 let connection_epoch = Arc::new(AtomicU64::new(0));
10538 let tracker = AuthTracker::new();
10539 auth_tracker.set(tracker.clone()).unwrap();
10540
10541 let (writer_tx, writer_rx) = crate::writer::channel(2);
10544 let write_task = WebSocketClientInner::spawn_write_task(
10545 Arc::clone(&connection_state),
10546 Arc::clone(&state_notify),
10547 Arc::new(AtomicBool::new(true)),
10548 writer,
10549 writer_rx,
10550 Arc::clone(&connection_epoch),
10551 Arc::clone(&auth_tracker),
10552 Arc::clone(&reconnect_buffer_waits_for_auth),
10553 None,
10554 );
10555
10556 writer_tx
10557 .send(WriterCommand::Send(Message::Text("stale".into())))
10558 .unwrap();
10559
10560 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10561 &url,
10562 vec![],
10563 TransportBackend::Tungstenite,
10564 None,
10565 None,
10566 None,
10567 )
10568 .await
10569 .unwrap();
10570 let (tx, rx) = tokio::sync::oneshot::channel();
10571 writer_tx
10572 .send_update(WriterCommand::Update(new_writer, tx))
10573 .unwrap();
10574 assert_eq!(rx.await.unwrap(), 1);
10575
10576 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10578 tokio::time::sleep(Duration::from_millis(300)).await;
10579 assert!(
10580 server.messages().await.is_empty(),
10581 "buffered message should wait for re-authentication"
10582 );
10583
10584 writer_tx
10587 .send(WriterCommand::Send(Message::Text("fresh".into())))
10588 .unwrap();
10589 tokio::time::sleep(Duration::from_millis(300)).await;
10590 assert!(
10591 server.messages().await.is_empty(),
10592 "a send arriving before the gate opens must not reach the connection, was {:?}",
10593 server.messages().await
10594 );
10595
10596 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10597 writer_tx
10598 .send_control(WriterCommand::SendOnConnection {
10599 message: Message::text("authenticate"),
10600 connection_epoch: 1,
10601 response_tx,
10602 })
10603 .unwrap();
10604
10605 response_rx.await.unwrap().unwrap();
10606 tracker.succeed();
10607
10608 wait_until_async(
10609 || {
10610 let messages = Arc::clone(&server.messages);
10611 async move { messages.lock().await.len() == 3 }
10612 },
10613 Duration::from_secs(3),
10614 )
10615 .await;
10616
10617 assert_eq!(
10618 server.messages().await,
10619 vec![
10620 "authenticate".to_string(),
10621 "stale".to_string(),
10622 "fresh".to_string(),
10623 ],
10624 "replay must preserve FIFO order behind the authentication handshake"
10625 );
10626
10627 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10628 state_notify.notify_waiters();
10629 drop(writer_tx);
10630 write_task.abort();
10631 }
10632
10633 #[rstest]
10634 #[tokio::test]
10635 async fn test_write_task_control_send_reaches_connection_behind_pending_replay() {
10636 use nautilus_common::testing::wait_until_async;
10637
10638 let server = RecordingServer::setup().await;
10639 let url = format!("ws://127.0.0.1:{}", server.port);
10640 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10641 &url,
10642 vec![],
10643 TransportBackend::Tungstenite,
10644 None,
10645 None,
10646 None,
10647 )
10648 .await
10649 .unwrap();
10650
10651 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10652 let state_notify = Arc::new(tokio::sync::Notify::new());
10653 let auth_tracker = Arc::new(OnceLock::new());
10654 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10655 let connection_epoch = Arc::new(AtomicU64::new(0));
10656 let tracker = AuthTracker::new();
10657 auth_tracker.set(tracker.clone()).unwrap();
10658
10659 let (writer_tx, writer_rx) = crate::writer::channel(2);
10660 let write_task = WebSocketClientInner::spawn_write_task(
10661 Arc::clone(&connection_state),
10662 Arc::clone(&state_notify),
10663 Arc::new(AtomicBool::new(true)),
10664 writer,
10665 writer_rx,
10666 Arc::clone(&connection_epoch),
10667 Arc::clone(&auth_tracker),
10668 Arc::clone(&reconnect_buffer_waits_for_auth),
10669 None,
10670 );
10671
10672 writer_tx
10673 .send(WriterCommand::Send(Message::Text("stale".into())))
10674 .unwrap();
10675
10676 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10677 &url,
10678 vec![],
10679 TransportBackend::Tungstenite,
10680 None,
10681 None,
10682 None,
10683 )
10684 .await
10685 .unwrap();
10686 let (tx, rx) = tokio::sync::oneshot::channel();
10687 writer_tx
10688 .send_update(WriterCommand::Update(new_writer, tx))
10689 .unwrap();
10690 assert_eq!(rx.await.unwrap(), 1);
10691
10692 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10694 tokio::time::sleep(Duration::from_millis(300)).await;
10695 assert!(
10696 server.messages().await.is_empty(),
10697 "buffered message should wait for re-authentication"
10698 );
10699
10700 writer_tx
10704 .send_control(WriterCommand::Send(Message::Pong(
10705 b"keepalive".to_vec().into(),
10706 )))
10707 .unwrap();
10708
10709 wait_until_async(
10710 || {
10711 let control_frames = Arc::clone(&server.control_frames);
10712 async move { !control_frames.lock().await.is_empty() }
10713 },
10714 Duration::from_secs(3),
10715 )
10716 .await;
10717
10718 assert_eq!(
10719 server.control_frames().await,
10720 vec!["pong:keepalive".to_string()],
10721 "a control frame sent during the wait must reach the new connection without replay"
10722 );
10723 assert!(
10724 server.messages().await.is_empty(),
10725 "ordinary sends must still wait for re-authentication, was {:?}",
10726 server.messages().await
10727 );
10728
10729 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10730 writer_tx
10731 .send_control(WriterCommand::SendOnConnection {
10732 message: Message::text("authenticate"),
10733 connection_epoch: 1,
10734 response_tx,
10735 })
10736 .unwrap();
10737
10738 response_rx.await.unwrap().unwrap();
10739 tracker.succeed();
10740
10741 wait_until_async(
10742 || {
10743 let messages = Arc::clone(&server.messages);
10744 async move { messages.lock().await.len() == 2 }
10745 },
10746 Duration::from_secs(3),
10747 )
10748 .await;
10749
10750 assert_eq!(
10751 server.messages().await,
10752 vec!["authenticate".to_string(), "stale".to_string()],
10753 "the control frame must not disturb ordinary replay ordering"
10754 );
10755
10756 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10757 state_notify.notify_waiters();
10758 drop(writer_tx);
10759 write_task.abort();
10760 }
10761
10762 #[tokio::test]
10763 async fn test_write_task_discards_buffer_after_auth_failure() {
10764 let server = RecordingServer::setup().await;
10765 let url = format!("ws://127.0.0.1:{}", server.port);
10766 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10767 &url,
10768 vec![],
10769 TransportBackend::Tungstenite,
10770 None,
10771 None,
10772 None,
10773 )
10774 .await
10775 .unwrap();
10776
10777 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10778 let state_notify = Arc::new(tokio::sync::Notify::new());
10779 let auth_tracker = Arc::new(OnceLock::new());
10780 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10781 let connection_epoch = Arc::new(AtomicU64::new(0));
10782 let tracker = AuthTracker::new();
10783 auth_tracker.set(tracker.clone()).unwrap();
10784
10785 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10786 let write_task = WebSocketClientInner::spawn_write_task(
10787 Arc::clone(&connection_state),
10788 Arc::clone(&state_notify),
10789 Arc::new(AtomicBool::new(true)),
10790 writer,
10791 writer_rx,
10792 Arc::clone(&connection_epoch),
10793 Arc::clone(&auth_tracker),
10794 Arc::clone(&reconnect_buffer_waits_for_auth),
10795 None,
10796 );
10797
10798 writer_tx
10799 .send(WriterCommand::Send(Message::Text("stale".into())))
10800 .unwrap();
10801
10802 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10803 &url,
10804 vec![],
10805 TransportBackend::Tungstenite,
10806 None,
10807 None,
10808 None,
10809 )
10810 .await
10811 .unwrap();
10812 let (tx, rx) = tokio::sync::oneshot::channel();
10813 writer_tx
10814 .send(WriterCommand::Update(new_writer, tx))
10815 .unwrap();
10816 assert_eq!(rx.await.unwrap(), 1);
10817
10818 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10819 tracker.fail("rejected");
10820 tracker.invalidate();
10821 assert_eq!(tracker.auth_state(), AuthState::Failed);
10822 tokio::time::sleep(Duration::from_millis(300)).await;
10823 assert!(
10824 server.messages().await.is_empty(),
10825 "buffered messages should be discarded after authentication failure"
10826 );
10827
10828 let _auth_receiver = tracker.begin();
10829 tracker.succeed();
10830 tokio::time::sleep(Duration::from_millis(300)).await;
10831 assert!(
10832 server.messages().await.is_empty(),
10833 "discarded buffered messages should not replay on a later auth success"
10834 );
10835
10836 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10837 state_notify.notify_waiters();
10838 drop(writer_tx);
10839 write_task.abort();
10840 }
10841
10842 #[rstest]
10843 #[tokio::test]
10844 async fn test_zero_idle_timeout_rejected() {
10845 let (handler, _rx) = channel_message_handler();
10846
10847 let config = WebSocketConfig {
10848 url: "ws://127.0.0.1:9999".to_string(),
10849 headers: vec![],
10850 heartbeat_interval_secs: None,
10851 heartbeat_payload: None,
10852 connect_timeout_ms: None,
10853 reconnect_delay_initial_ms: None,
10854 reconnect_delay_max_ms: None,
10855 reconnect_backoff_factor: None,
10856 reconnect_jitter_ms: None,
10857 reconnect_max_attempts: None,
10858 heartbeat_timeout_secs: None,
10859 idle_timeout_ms: Some(0),
10860 writer_capacity: None,
10861 backend: TransportBackend::Tungstenite,
10862 proxy_url: None,
10863 max_message_size_bytes: None,
10864 max_frame_size_bytes: None,
10865 };
10866
10867 let result = WebSocketClient::builder()
10868 .config(config)
10869 .message_handler(handler)
10870 .connect()
10871 .await;
10872
10873 assert!(result.is_err(), "Zero idle timeout should be rejected");
10874 let err_msg = result.unwrap_err().to_string();
10875 assert!(
10876 err_msg.contains("idle_timeout_ms"),
10877 "Error should name the offending field, was: {err_msg}"
10878 );
10879 }
10880
10881 #[rstest]
10882 #[tokio::test]
10883 async fn test_zero_heartbeat_timeout_rejected() {
10884 let (handler, _rx) = channel_message_handler();
10885
10886 let config = WebSocketConfig {
10887 url: "ws://127.0.0.1:9999".to_string(),
10888 headers: vec![],
10889 heartbeat_interval_secs: Some(30),
10890 heartbeat_payload: None,
10891 connect_timeout_ms: None,
10892 reconnect_delay_initial_ms: None,
10893 reconnect_delay_max_ms: None,
10894 reconnect_backoff_factor: None,
10895 reconnect_jitter_ms: None,
10896 reconnect_max_attempts: None,
10897 heartbeat_timeout_secs: Some(0),
10898 idle_timeout_ms: None,
10899 writer_capacity: None,
10900 backend: TransportBackend::Tungstenite,
10901 proxy_url: None,
10902 max_message_size_bytes: None,
10903 max_frame_size_bytes: None,
10904 };
10905
10906 let result = WebSocketClient::builder()
10907 .config(config)
10908 .message_handler(handler)
10909 .connect()
10910 .await;
10911
10912 assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
10913 let err_msg = result.unwrap_err().to_string();
10914 assert!(
10915 err_msg.contains("heartbeat_timeout_secs"),
10916 "Error should name the offending field, was: {err_msg}"
10917 );
10918 }
10919
10920 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
10921 #[rstest]
10922 #[tokio::test]
10923 async fn test_sockudo_backend_rejects_reserved_headers_before_connect() {
10924 let (handler, _rx) = channel_message_handler();
10925
10926 let config = WebSocketConfig {
10927 url: "ws://127.0.0.1:1".to_string(),
10928 headers: vec![("Host".to_string(), "example.com".to_string())],
10929 heartbeat_interval_secs: None,
10930 heartbeat_payload: None,
10931 connect_timeout_ms: None,
10932 reconnect_delay_initial_ms: None,
10933 reconnect_delay_max_ms: None,
10934 reconnect_backoff_factor: None,
10935 reconnect_jitter_ms: None,
10936 reconnect_max_attempts: None,
10937 heartbeat_timeout_secs: None,
10938 idle_timeout_ms: None,
10939 writer_capacity: None,
10940 backend: TransportBackend::Sockudo,
10941 proxy_url: None,
10942 max_message_size_bytes: None,
10943 max_frame_size_bytes: None,
10944 };
10945
10946 let err = WebSocketClient::builder()
10947 .config(config)
10948 .message_handler(handler)
10949 .connect()
10950 .await
10951 .expect_err("reserved header should fail before TCP connect");
10952
10953 assert!(
10954 err.to_string()
10955 .contains("reserved upgrade header not allowed in extra_headers"),
10956 "expected reserved-header failure, was: {err}"
10957 );
10958 }
10959
10960 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
10962 #[rstest]
10963 #[tokio::test]
10964 async fn test_sockudo_reconnect_recovers_from_retryable_upgrade_rejection() {
10965 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
10966 let port = listener.local_addr().unwrap().port();
10967
10968 let server = task::spawn(async move {
10969 let (stream, _) = listener.accept().await.unwrap();
10971 drop(accept_async(stream).await.unwrap());
10972
10973 let (mut stream, _) = listener.accept().await.unwrap();
10975 let _request = read_http_request(&mut stream).await;
10976 stream
10977 .write_all(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
10978 .await
10979 .unwrap();
10980 stream.flush().await.unwrap();
10981 sleep(Duration::from_millis(50)).await;
10982 drop(stream);
10983
10984 let (stream, _) = listener.accept().await.unwrap();
10986 let mut ws = accept_async(stream).await.unwrap();
10987 ws.send(WsMessage::Text("recovered".to_string().into()))
10988 .await
10989 .unwrap();
10990 sleep(Duration::from_secs(1)).await;
10991 });
10992
10993 let (handler, mut rx) = channel_message_handler();
10994
10995 let config = WebSocketConfig {
10996 url: format!(
10997 "ws://user-secret:password-secret@127.0.0.1:{port}/path-secret?token=query-secret"
10998 ),
10999 headers: vec![],
11000 heartbeat_interval_secs: None,
11001 heartbeat_payload: None,
11002 connect_timeout_ms: Some(2_000),
11003 reconnect_delay_initial_ms: Some(50),
11004 reconnect_delay_max_ms: Some(100),
11005 reconnect_backoff_factor: Some(1.0),
11006 reconnect_jitter_ms: Some(0),
11007 reconnect_max_attempts: None,
11008 heartbeat_timeout_secs: None,
11009 idle_timeout_ms: None,
11010 writer_capacity: None,
11011 backend: TransportBackend::Sockudo,
11012 proxy_url: None,
11013 max_message_size_bytes: None,
11014 max_frame_size_bytes: None,
11015 };
11016
11017 let capture = capture_logs_for(RECONNECT_LOG_TARGETS).await;
11021
11022 let client = WebSocketClient::builder()
11023 .config(config)
11024 .message_handler(handler)
11025 .connect()
11026 .await
11027 .expect("sockudo connect should succeed");
11028
11029 let recovered = tokio::time::timeout(Duration::from_secs(10), async {
11030 loop {
11031 if let Ok(WsMessage::Text(text)) = rx.try_recv()
11032 && text.as_str() == "recovered"
11033 {
11034 return;
11035 }
11036
11037 tokio::time::sleep(Duration::from_millis(10)).await;
11038 }
11039 })
11040 .await;
11041
11042 let messages = capture.messages();
11043 client.disconnect().await;
11044 server.abort();
11045
11046 recovered.expect("client should recover after the retryable upgrade rejection");
11047
11048 let errors: Vec<_> = messages
11049 .iter()
11050 .filter(|(level, _)| *level == Level::Error)
11051 .collect();
11052 assert!(
11053 errors.is_empty(),
11054 "a retryable rejection must not log at ERROR, was: {errors:?}"
11055 );
11056
11057 assert!(
11059 messages.iter().any(|(level, message)| *level == Level::Warn
11060 && message.contains(&format!(
11061 "Sockudo handshake rejected by 127.0.0.1:{port} with retryable status 502"
11062 ))),
11063 "expected a warning naming the retryable rejection, was: {messages:?}"
11064 );
11065
11066 for secret in [
11067 "user-secret",
11068 "password-secret",
11069 "path-secret",
11070 "query-secret",
11071 ] {
11072 assert!(
11073 messages
11074 .iter()
11075 .all(|(_, message)| !message.contains(secret)),
11076 "logs must not contain {secret}, was: {messages:?}"
11077 );
11078 }
11079 }
11080
11081 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11082 #[rstest]
11083 #[tokio::test]
11084 async fn test_sockudo_backend_replays_leftover_without_custom_headers() {
11085 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11086 let port = listener.local_addr().unwrap().port();
11087 let server = task::spawn(async move {
11088 if let Ok((mut stream, _)) = listener.accept().await {
11089 let request = read_http_request(&mut stream).await;
11090 let request = String::from_utf8(request).unwrap();
11091 let sec_websocket_key = extract_header(&request, "Sec-WebSocket-Key").unwrap();
11092 let accept = sockudo_handshake::generate_accept_key(sec_websocket_key);
11093 let mut response = format!(
11094 concat!(
11095 "HTTP/1.1 101 Switching Protocols\r\n",
11096 "Upgrade: websocket\r\n",
11097 "Connection: Upgrade\r\n",
11098 "Sec-WebSocket-Accept: {}\r\n",
11099 "\r\n",
11100 ),
11101 accept
11102 )
11103 .into_bytes();
11104 response.extend_from_slice(b"\x81\x05hello");
11105 stream.write_all(&response).await.unwrap();
11106 }
11107 });
11108
11109 let (handler, mut rx) = channel_message_handler();
11110
11111 let config = WebSocketConfig {
11112 url: format!("ws://127.0.0.1:{port}/ws"),
11113 headers: vec![],
11114 heartbeat_interval_secs: None,
11115 heartbeat_payload: None,
11116 connect_timeout_ms: Some(2_000),
11117 reconnect_delay_initial_ms: Some(50),
11118 reconnect_delay_max_ms: Some(100),
11119 reconnect_backoff_factor: Some(1.0),
11120 reconnect_jitter_ms: Some(0),
11121 reconnect_max_attempts: None,
11122 heartbeat_timeout_secs: None,
11123 idle_timeout_ms: None,
11124 writer_capacity: None,
11125 backend: TransportBackend::Sockudo,
11126 proxy_url: None,
11127 max_message_size_bytes: None,
11128 max_frame_size_bytes: None,
11129 };
11130
11131 let client = WebSocketClient::builder()
11132 .config(config)
11133 .message_handler(handler)
11134 .connect()
11135 .await
11136 .expect("sockudo connect without custom headers");
11137
11138 let received = tokio::time::timeout(Duration::from_secs(3), async {
11139 loop {
11140 if let Ok(msg) = rx.try_recv() {
11141 return msg;
11142 }
11143
11144 tokio::time::sleep(Duration::from_millis(10)).await;
11145 }
11146 })
11147 .await
11148 .expect("did not receive leftover frame before timeout");
11149
11150 match received {
11151 WsMessage::Text(t) => assert_eq!(t.as_str(), "hello"),
11152 other => panic!("expected text, was {other:?}"),
11153 }
11154
11155 client.disconnect().await;
11156 tokio::time::timeout(Duration::from_secs(3), server)
11157 .await
11158 .expect("server did not close before timeout")
11159 .unwrap();
11160 }
11161
11162 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11163 #[rstest]
11164 #[tokio::test]
11165 async fn test_sockudo_backend_sends_custom_headers() {
11166 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11167 let port = listener.local_addr().unwrap().port();
11168 let server = task::spawn(async move {
11169 if let Ok((stream, _)) = listener.accept().await {
11170 let callback = HeaderAssertCallback {
11171 key: "X-Test".to_string(),
11172 value: HeaderValue::from_static("value"),
11173 };
11174
11175 if let Ok(mut ws) = accept_hdr_async(stream, callback).await {
11176 while let Some(Ok(msg)) = ws.next().await {
11177 if msg.is_text() || msg.is_binary() {
11178 if ws.send(msg).await.is_err() {
11179 break;
11180 }
11181
11182 continue;
11183 }
11184
11185 if msg.is_close() {
11186 let _ = ws.close(None).await;
11187 break;
11188 }
11189 }
11190 }
11191 }
11192 });
11193
11194 let (handler, mut rx) = channel_message_handler();
11195
11196 let config = WebSocketConfig {
11197 url: format!("ws://127.0.0.1:{port}"),
11198 headers: vec![("X-Test".to_string(), "value".to_string())],
11199 heartbeat_interval_secs: None,
11200 heartbeat_payload: None,
11201 connect_timeout_ms: Some(2_000),
11202 reconnect_delay_initial_ms: Some(50),
11203 reconnect_delay_max_ms: Some(100),
11204 reconnect_backoff_factor: Some(1.0),
11205 reconnect_jitter_ms: Some(0),
11206 reconnect_max_attempts: None,
11207 heartbeat_timeout_secs: None,
11208 idle_timeout_ms: None,
11209 writer_capacity: None,
11210 backend: TransportBackend::Sockudo,
11211 proxy_url: None,
11212 max_message_size_bytes: None,
11213 max_frame_size_bytes: None,
11214 };
11215
11216 let client = WebSocketClient::builder()
11217 .config(config)
11218 .message_handler(handler)
11219 .connect()
11220 .await
11221 .expect("sockudo connect with custom headers");
11222
11223 client.send_text("ping".to_string(), None).await.unwrap();
11224
11225 let received = tokio::time::timeout(Duration::from_secs(3), async {
11226 loop {
11227 if let Ok(msg) = rx.try_recv() {
11228 return msg;
11229 }
11230
11231 tokio::time::sleep(Duration::from_millis(10)).await;
11232 }
11233 })
11234 .await
11235 .expect("did not receive echo before timeout");
11236
11237 match received {
11238 WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11239 other => panic!("expected text, was {other:?}"),
11240 }
11241
11242 client.disconnect().await;
11243 tokio::time::timeout(Duration::from_secs(3), server)
11244 .await
11245 .expect("server did not close before timeout")
11246 .unwrap();
11247 }
11248
11249 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11250 #[rstest]
11251 #[tokio::test]
11252 async fn test_sockudo_backend_round_trip_text() {
11253 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11255 let port = listener.local_addr().unwrap().port();
11256 let server = task::spawn(async move {
11257 if let Ok((stream, _)) = listener.accept().await
11258 && let Ok(mut ws) = accept_async(stream).await
11259 {
11260 while let Some(Ok(msg)) = ws.next().await {
11261 match msg {
11262 WsMessage::Text(_) | WsMessage::Binary(_) => {
11263 if ws.send(msg).await.is_err() {
11264 break;
11265 }
11266 }
11267 WsMessage::Close(_) => {
11268 let _ = ws.close(None).await;
11269 break;
11270 }
11271 _ => {}
11272 }
11273 }
11274 }
11275 });
11276
11277 let (handler, mut rx) = channel_message_handler();
11278
11279 let config = WebSocketConfig {
11280 url: format!("ws://127.0.0.1:{port}"),
11281 headers: vec![],
11282 heartbeat_interval_secs: None,
11283 heartbeat_payload: None,
11284 connect_timeout_ms: Some(2_000),
11285 reconnect_delay_initial_ms: Some(50),
11286 reconnect_delay_max_ms: Some(100),
11287 reconnect_backoff_factor: Some(1.0),
11288 reconnect_jitter_ms: Some(0),
11289 reconnect_max_attempts: None,
11290 heartbeat_timeout_secs: None,
11291 idle_timeout_ms: None,
11292 writer_capacity: None,
11293 backend: TransportBackend::Sockudo,
11294 proxy_url: None,
11295 max_message_size_bytes: None,
11296 max_frame_size_bytes: None,
11297 };
11298
11299 let client = WebSocketClient::builder()
11300 .config(config)
11301 .message_handler(handler)
11302 .connect()
11303 .await
11304 .expect("sockudo connect");
11305
11306 client.send_text("ping".to_string(), None).await.unwrap();
11307
11308 let received = tokio::time::timeout(Duration::from_secs(3), async {
11309 loop {
11310 if let Ok(msg) = rx.try_recv() {
11311 return msg;
11312 }
11313
11314 tokio::time::sleep(Duration::from_millis(10)).await;
11315 }
11316 })
11317 .await
11318 .expect("did not receive echo before timeout");
11319
11320 match received {
11321 WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11322 other => panic!("expected text, was {other:?}"),
11323 }
11324
11325 client.disconnect().await;
11326 server.abort();
11327 }
11328
11329 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11330 #[rstest]
11331 #[case::ws_default_port("ws://example.com/ws", "example.com", "example.com", 80, "/ws", false)]
11332 #[case::wss_default_port(
11333 "wss://example.com/ws",
11334 "example.com",
11335 "example.com",
11336 443,
11337 "/ws",
11338 true
11339 )]
11340 #[case::ws_explicit_default(
11343 "ws://example.com:80/ws",
11344 "example.com",
11345 "example.com",
11346 80,
11347 "/ws",
11348 false
11349 )]
11350 #[case::ws_non_default(
11351 "ws://example.com:8443/feed",
11352 "example.com",
11353 "example.com:8443",
11354 8443,
11355 "/feed",
11356 false
11357 )]
11358 #[case::wss_non_default(
11359 "wss://example.com:9443/feed",
11360 "example.com",
11361 "example.com:9443",
11362 9443,
11363 "/feed",
11364 true
11365 )]
11366 #[case::root_path(
11367 "ws://example.com:9000/",
11368 "example.com",
11369 "example.com:9000",
11370 9000,
11371 "/",
11372 false
11373 )]
11374 #[case::query_string(
11375 "ws://example.com/feed?token=abc&channel=trades",
11376 "example.com",
11377 "example.com",
11378 80,
11379 "/feed?token=abc&channel=trades",
11380 false
11381 )]
11382 #[case::ipv6_default("ws://[::1]/feed", "::1", "[::1]", 80, "/feed", false)]
11384 #[case::ipv6_explicit_port("ws://[::1]:9000/feed", "::1", "[::1]:9000", 9000, "/feed", false)]
11385 #[case::ipv6_wss(
11386 "wss://[2001:db8::1]:8443/",
11387 "2001:db8::1",
11388 "[2001:db8::1]:8443",
11389 8443,
11390 "/",
11391 true
11392 )]
11393 fn sockudo_target_parses_url(
11394 #[case] url: &str,
11395 #[case] host: &str,
11396 #[case] host_header: &str,
11397 #[case] port: u16,
11398 #[case] path: &str,
11399 #[case] is_tls: bool,
11400 ) {
11401 let target = super::SockudoTarget::parse(url).expect("parse should succeed");
11402 assert_eq!(target.host, host);
11403 assert_eq!(target.host_header, host_header);
11404 assert_eq!(target.port, port);
11405 assert_eq!(target.path, path);
11406 assert_eq!(target.is_tls, is_tls);
11407 }
11408
11409 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11410 #[rstest]
11411 fn sockudo_target_rejects_unsupported_scheme() {
11412 let err = super::SockudoTarget::parse("http://example.com/feed").expect_err("not a ws URL");
11413 let msg = err.to_string();
11414 assert!(
11415 msg.contains("expected ws:// or wss://"),
11416 "unexpected error: {msg}"
11417 );
11418 }
11419
11420 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11421 #[rstest]
11422 fn sockudo_target_rejects_malformed_url() {
11423 const SECRET: &str = "malformed-websocket-url-secret";
11424 let url = format!("not a url {SECRET}");
11425 let err = super::SockudoTarget::parse(&url).expect_err("malformed URL");
11426 let message = err.to_string();
11427
11428 assert!(
11429 matches!(err, super::TransportError::InvalidUrl(_)),
11430 "expected InvalidUrl, was: {err:?}"
11431 );
11432 assert!(!message.contains(SECRET));
11433 assert!(!message.contains(&url));
11434 }
11435}
11436
11437#[cfg(test)]
11438mod property_tests {
11439 use std::{
11440 collections::{HashSet, VecDeque},
11441 sync::{Arc, OnceLock, atomic::AtomicBool},
11442 };
11443
11444 use proptest::prelude::*;
11445 use rstest::rstest;
11446
11447 use super::{super::auth::AuthResultReceiver, *};
11448
11449 const AUTH_FAILED: &str = "model auth failed";
11450
11451 #[derive(Debug, Clone)]
11452 enum ReconnectBufferTraceOp {
11453 BeginAuth,
11454 AuthSucceeds,
11455 AuthFails,
11456 AuthInvalidates,
11457 ReconnectStarts,
11458 ReconnectCompletes,
11459 BufferedMessage(u8),
11460 }
11461
11462 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
11463 enum ModelConnectionMode {
11464 Active,
11465 Reconnect,
11466 }
11467
11468 #[derive(Debug, Clone, Copy)]
11469 enum ExpectedReconnectBufferAction {
11470 Drain,
11471 Wait,
11472 Discard,
11473 }
11474
11475 #[derive(Debug)]
11476 struct ReconnectBufferModel {
11477 mode: ModelConnectionMode,
11478 auth_state: AuthState,
11479 buffer: VecDeque<String>,
11480 released: Vec<String>,
11481 discarded: Vec<String>,
11482 live_sent: Vec<String>,
11483 handler_controls: Vec<&'static str>,
11484 next_message_index: usize,
11485 }
11486
11487 impl ReconnectBufferModel {
11488 fn new() -> Self {
11489 Self {
11490 mode: ModelConnectionMode::Active,
11491 auth_state: AuthState::Unauthenticated,
11492 buffer: VecDeque::new(),
11493 released: Vec::new(),
11494 discarded: Vec::new(),
11495 live_sent: Vec::new(),
11496 handler_controls: Vec::new(),
11497 next_message_index: 0,
11498 }
11499 }
11500
11501 fn next_payload(&mut self, raw: u8) -> String {
11502 let payload = format!("message-{}-{raw}", self.next_message_index);
11503 self.next_message_index += 1;
11504 payload
11505 }
11506
11507 fn expected_action(&self, waits_for_auth: bool) -> ExpectedReconnectBufferAction {
11508 if !waits_for_auth {
11509 return ExpectedReconnectBufferAction::Drain;
11510 }
11511
11512 match self.auth_state {
11513 AuthState::Authenticated => ExpectedReconnectBufferAction::Drain,
11514 AuthState::Failed => ExpectedReconnectBufferAction::Discard,
11515 AuthState::Unauthenticated => ExpectedReconnectBufferAction::Wait,
11516 }
11517 }
11518 }
11519
11520 fn reconnect_buffer_trace_op_strategy() -> impl Strategy<Value = ReconnectBufferTraceOp> {
11521 prop_oneof![
11522 Just(ReconnectBufferTraceOp::BeginAuth),
11523 Just(ReconnectBufferTraceOp::AuthSucceeds),
11524 Just(ReconnectBufferTraceOp::AuthFails),
11525 Just(ReconnectBufferTraceOp::AuthInvalidates),
11526 Just(ReconnectBufferTraceOp::ReconnectStarts),
11527 Just(ReconnectBufferTraceOp::ReconnectCompletes),
11528 any::<u8>().prop_map(ReconnectBufferTraceOp::BufferedMessage),
11529 ]
11530 }
11531
11532 fn reconnect_buffer_actions_match(
11533 actual: ReconnectBufferAction,
11534 expected: ExpectedReconnectBufferAction,
11535 ) -> bool {
11536 matches!(
11537 (actual, expected),
11538 (
11539 ReconnectBufferAction::Drain,
11540 ExpectedReconnectBufferAction::Drain
11541 ) | (
11542 ReconnectBufferAction::Wait,
11543 ExpectedReconnectBufferAction::Wait
11544 ) | (
11545 ReconnectBufferAction::Discard,
11546 ExpectedReconnectBufferAction::Discard
11547 )
11548 )
11549 }
11550
11551 fn apply_ready_reconnect_buffer_action(
11552 model: &mut ReconnectBufferModel,
11553 reconnect_buffer_waits_for_auth: &AtomicBool,
11554 auth_tracker: &Arc<OnceLock<AuthTracker>>,
11555 waits_for_auth: bool,
11556 step: usize,
11557 op: &ReconnectBufferTraceOp,
11558 ) -> Result<(), TestCaseError> {
11559 if model.mode != ModelConnectionMode::Active || model.buffer.is_empty() {
11560 return Ok(());
11561 }
11562
11563 let expected = model.expected_action(waits_for_auth);
11564 let actual = WebSocketClientInner::can_drain_reconnect_buffer(
11565 reconnect_buffer_waits_for_auth,
11566 auth_tracker,
11567 );
11568
11569 prop_assert!(
11570 reconnect_buffer_actions_match(actual, expected),
11571 "reconnect buffer action mismatch at step {}, op {:?}, waits_for_auth={}, auth_state={:?}",
11572 step,
11573 op,
11574 waits_for_auth,
11575 model.auth_state
11576 );
11577
11578 match expected {
11579 ExpectedReconnectBufferAction::Drain => {
11580 model.released.extend(model.buffer.drain(..));
11581 }
11582 ExpectedReconnectBufferAction::Wait => {}
11583 ExpectedReconnectBufferAction::Discard => {
11584 model.discarded.extend(model.buffer.drain(..));
11585 }
11586 }
11587
11588 Ok(())
11589 }
11590
11591 fn assert_reconnected_control_stays_separate(
11592 model: &ReconnectBufferModel,
11593 step: usize,
11594 ) -> Result<(), TestCaseError> {
11595 prop_assert!(
11596 model
11597 .handler_controls
11598 .iter()
11599 .all(|message| *message == RECONNECTED),
11600 "handler control stream contained a non-RECONNECTED message at step {}",
11601 step
11602 );
11603 prop_assert!(
11604 !model.buffer.iter().any(|message| message == RECONNECTED),
11605 "RECONNECTED control message entered reconnect buffer at step {}",
11606 step
11607 );
11608 prop_assert!(
11609 !model.released.iter().any(|message| message == RECONNECTED),
11610 "RECONNECTED control message entered replayed messages at step {}",
11611 step
11612 );
11613 prop_assert!(
11614 !model.discarded.iter().any(|message| message == RECONNECTED),
11615 "RECONNECTED control message entered discarded messages at step {}",
11616 step
11617 );
11618 prop_assert!(
11619 !model.live_sent.iter().any(|message| message == RECONNECTED),
11620 "RECONNECTED control message entered application sends at step {}",
11621 step
11622 );
11623
11624 Ok(())
11625 }
11626
11627 fn assert_messages_accounted_once(
11628 model: &ReconnectBufferModel,
11629 step: usize,
11630 ) -> Result<(), TestCaseError> {
11631 let mut seen = HashSet::new();
11632
11633 for message in model
11634 .released
11635 .iter()
11636 .chain(model.discarded.iter())
11637 .chain(model.buffer.iter())
11638 .chain(model.live_sent.iter())
11639 {
11640 prop_assert!(
11641 seen.insert(message.as_str()),
11642 "message {} appeared more than once at step {}",
11643 message,
11644 step
11645 );
11646 }
11647
11648 Ok(())
11649 }
11650
11651 fn apply_reconnect_buffer_trace_op(
11652 model: &mut ReconnectBufferModel,
11653 tracker: &AuthTracker,
11654 auth_receivers: &mut Vec<AuthResultReceiver>,
11655 op: &ReconnectBufferTraceOp,
11656 ) -> Result<(), TestCaseError> {
11657 match op {
11658 ReconnectBufferTraceOp::BeginAuth => {
11659 auth_receivers.push(tracker.begin());
11660 model.auth_state = AuthState::Unauthenticated;
11661 }
11662 ReconnectBufferTraceOp::AuthSucceeds => {
11663 tracker.succeed();
11664 model.auth_state = AuthState::Authenticated;
11665 }
11666 ReconnectBufferTraceOp::AuthFails => {
11667 tracker.fail(AUTH_FAILED);
11668 model.auth_state = AuthState::Failed;
11669 }
11670 ReconnectBufferTraceOp::AuthInvalidates => {
11671 tracker.invalidate();
11672
11673 if model.auth_state == AuthState::Authenticated {
11674 model.auth_state = AuthState::Unauthenticated;
11675 }
11676 }
11677 ReconnectBufferTraceOp::ReconnectStarts => {
11678 tracker.invalidate();
11679
11680 if model.auth_state == AuthState::Authenticated {
11681 model.auth_state = AuthState::Unauthenticated;
11682 }
11683
11684 model.mode = ModelConnectionMode::Reconnect;
11685 }
11686 ReconnectBufferTraceOp::ReconnectCompletes => {
11687 model.mode = ModelConnectionMode::Active;
11688 model.handler_controls.push(RECONNECTED);
11689 }
11690 ReconnectBufferTraceOp::BufferedMessage(raw) => {
11691 let payload = model.next_payload(*raw);
11692 prop_assert_ne!(payload.as_str(), RECONNECTED);
11693
11694 if model.mode == ModelConnectionMode::Reconnect {
11695 model.buffer.push_back(payload);
11696 } else {
11697 model.live_sent.push(payload);
11698 }
11699 }
11700 }
11701
11702 Ok(())
11703 }
11704
11705 proptest! {
11706 #![proptest_config(ProptestConfig::with_cases(256))]
11707
11708 #[rstest]
11711 fn test_reconnect_buffer_trace_matches_auth_gate_model(
11712 waits_for_auth in any::<bool>(),
11713 ops in proptest::collection::vec(reconnect_buffer_trace_op_strategy(), 1..100)
11714 ) {
11715 let auth_tracker = Arc::new(OnceLock::new());
11716 let reconnect_buffer_waits_for_auth = AtomicBool::new(waits_for_auth);
11717 let tracker = AuthTracker::new();
11718 auth_tracker.set(tracker.clone()).unwrap();
11719 let mut auth_receivers = Vec::new();
11720 let mut model = ReconnectBufferModel::new();
11721
11722 for (step, op) in ops.iter().enumerate() {
11723 apply_reconnect_buffer_trace_op(
11724 &mut model,
11725 &tracker,
11726 &mut auth_receivers,
11727 op,
11728 )?;
11729
11730 prop_assert_eq!(
11731 tracker.auth_state(),
11732 model.auth_state,
11733 "auth state mismatch at step {}, op {:?}",
11734 step,
11735 op
11736 );
11737
11738 apply_ready_reconnect_buffer_action(
11739 &mut model,
11740 &reconnect_buffer_waits_for_auth,
11741 &auth_tracker,
11742 waits_for_auth,
11743 step,
11744 op,
11745 )?;
11746 assert_reconnected_control_stays_separate(&model, step)?;
11747 prop_assert_eq!(
11748 model.handler_controls.len(),
11749 ops[..=step]
11750 .iter()
11751 .filter(|op| matches!(op, ReconnectBufferTraceOp::ReconnectCompletes))
11752 .count(),
11753 "handler control count mismatch at step {}",
11754 step
11755 );
11756 assert_messages_accounted_once(&model, step)?;
11757 }
11758 }
11759
11760 #[rstest]
11763 fn test_reconnect_buffer_releases_after_auth_success_once(
11764 payloads in proptest::collection::vec(any::<u8>(), 1..32),
11765 extra_success_ticks in 0usize..16
11766 ) {
11767 let auth_tracker = Arc::new(OnceLock::new());
11768 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11769 let tracker = AuthTracker::new();
11770 auth_tracker.set(tracker.clone()).unwrap();
11771 let mut auth_receivers = Vec::new();
11772 let mut model = ReconnectBufferModel::new();
11773
11774 apply_reconnect_buffer_trace_op(
11775 &mut model,
11776 &tracker,
11777 &mut auth_receivers,
11778 &ReconnectBufferTraceOp::ReconnectStarts,
11779 )?;
11780 apply_reconnect_buffer_trace_op(
11781 &mut model,
11782 &tracker,
11783 &mut auth_receivers,
11784 &ReconnectBufferTraceOp::BeginAuth,
11785 )?;
11786
11787 for payload in payloads {
11788 apply_reconnect_buffer_trace_op(
11789 &mut model,
11790 &tracker,
11791 &mut auth_receivers,
11792 &ReconnectBufferTraceOp::BufferedMessage(payload),
11793 )?;
11794 }
11795
11796 let buffered_len = model.buffer.len();
11797 apply_reconnect_buffer_trace_op(
11798 &mut model,
11799 &tracker,
11800 &mut auth_receivers,
11801 &ReconnectBufferTraceOp::ReconnectCompletes,
11802 )?;
11803 apply_ready_reconnect_buffer_action(
11804 &mut model,
11805 &reconnect_buffer_waits_for_auth,
11806 &auth_tracker,
11807 true,
11808 0,
11809 &ReconnectBufferTraceOp::ReconnectCompletes,
11810 )?;
11811
11812 prop_assert_eq!(model.released.len(), 0);
11813 prop_assert_eq!(model.buffer.len(), buffered_len);
11814
11815 apply_reconnect_buffer_trace_op(
11816 &mut model,
11817 &tracker,
11818 &mut auth_receivers,
11819 &ReconnectBufferTraceOp::AuthSucceeds,
11820 )?;
11821 apply_ready_reconnect_buffer_action(
11822 &mut model,
11823 &reconnect_buffer_waits_for_auth,
11824 &auth_tracker,
11825 true,
11826 1,
11827 &ReconnectBufferTraceOp::AuthSucceeds,
11828 )?;
11829
11830 prop_assert_eq!(model.released.len(), buffered_len);
11831 prop_assert!(model.buffer.is_empty());
11832 assert_messages_accounted_once(&model, 1)?;
11833
11834 for tick in 0..extra_success_ticks {
11835 apply_reconnect_buffer_trace_op(
11836 &mut model,
11837 &tracker,
11838 &mut auth_receivers,
11839 &ReconnectBufferTraceOp::AuthSucceeds,
11840 )?;
11841 apply_ready_reconnect_buffer_action(
11842 &mut model,
11843 &reconnect_buffer_waits_for_auth,
11844 &auth_tracker,
11845 true,
11846 tick + 2,
11847 &ReconnectBufferTraceOp::AuthSucceeds,
11848 )?;
11849 prop_assert_eq!(
11850 model.released.len(),
11851 buffered_len,
11852 "buffered messages replayed more than once at tick {}",
11853 tick
11854 );
11855 }
11856 }
11857
11858 #[rstest]
11861 fn test_reconnect_buffer_discards_after_auth_failure(
11862 before_failure_payloads in proptest::collection::vec(any::<u8>(), 0..16),
11863 after_failure_payloads in proptest::collection::vec(any::<u8>(), 1..16),
11864 later_success_ticks in 0usize..16
11865 ) {
11866 let auth_tracker = Arc::new(OnceLock::new());
11867 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11868 let tracker = AuthTracker::new();
11869 auth_tracker.set(tracker.clone()).unwrap();
11870 let mut auth_receivers = Vec::new();
11871 let mut model = ReconnectBufferModel::new();
11872
11873 apply_reconnect_buffer_trace_op(
11874 &mut model,
11875 &tracker,
11876 &mut auth_receivers,
11877 &ReconnectBufferTraceOp::ReconnectStarts,
11878 )?;
11879 apply_reconnect_buffer_trace_op(
11880 &mut model,
11881 &tracker,
11882 &mut auth_receivers,
11883 &ReconnectBufferTraceOp::BeginAuth,
11884 )?;
11885
11886 for payload in before_failure_payloads {
11887 apply_reconnect_buffer_trace_op(
11888 &mut model,
11889 &tracker,
11890 &mut auth_receivers,
11891 &ReconnectBufferTraceOp::BufferedMessage(payload),
11892 )?;
11893 }
11894
11895 apply_reconnect_buffer_trace_op(
11896 &mut model,
11897 &tracker,
11898 &mut auth_receivers,
11899 &ReconnectBufferTraceOp::AuthFails,
11900 )?;
11901
11902 for payload in after_failure_payloads {
11903 apply_reconnect_buffer_trace_op(
11904 &mut model,
11905 &tracker,
11906 &mut auth_receivers,
11907 &ReconnectBufferTraceOp::BufferedMessage(payload),
11908 )?;
11909 }
11910
11911 let buffered_len = model.buffer.len();
11912 apply_reconnect_buffer_trace_op(
11913 &mut model,
11914 &tracker,
11915 &mut auth_receivers,
11916 &ReconnectBufferTraceOp::ReconnectCompletes,
11917 )?;
11918 apply_ready_reconnect_buffer_action(
11919 &mut model,
11920 &reconnect_buffer_waits_for_auth,
11921 &auth_tracker,
11922 true,
11923 0,
11924 &ReconnectBufferTraceOp::ReconnectCompletes,
11925 )?;
11926
11927 prop_assert_eq!(model.discarded.len(), buffered_len);
11928 prop_assert!(model.released.is_empty());
11929 prop_assert!(model.buffer.is_empty());
11930 assert_messages_accounted_once(&model, 0)?;
11931
11932 for tick in 0..later_success_ticks {
11933 apply_reconnect_buffer_trace_op(
11934 &mut model,
11935 &tracker,
11936 &mut auth_receivers,
11937 &ReconnectBufferTraceOp::BeginAuth,
11938 )?;
11939 apply_reconnect_buffer_trace_op(
11940 &mut model,
11941 &tracker,
11942 &mut auth_receivers,
11943 &ReconnectBufferTraceOp::AuthSucceeds,
11944 )?;
11945 apply_ready_reconnect_buffer_action(
11946 &mut model,
11947 &reconnect_buffer_waits_for_auth,
11948 &auth_tracker,
11949 true,
11950 tick + 1,
11951 &ReconnectBufferTraceOp::AuthSucceeds,
11952 )?;
11953 prop_assert!(
11954 model.released.is_empty(),
11955 "discarded messages replayed after later auth success at tick {}",
11956 tick
11957 );
11958 }
11959 }
11960 }
11961}
11962
11963#[cfg(test)]
11964#[cfg(feature = "turmoil")]
11965mod turmoil_tests {
11966 use std::{sync::Arc, time::Duration};
11967
11968 use futures_util::{SinkExt, StreamExt};
11969 use nautilus_common::testing::wait_until_async;
11970 use rstest::rstest;
11971 use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
11972 use turmoil::{Builder, net};
11973
11974 use super::*;
11975 use crate::websocket::types::channel_message_handler;
11976
11977 const AUTH_BUFFER_WAIT_SEED: u64 = 0xA17B_0001;
11978 const AUTH_BUFFER_DISCARD_SEED: u64 = 0xA17B_0002;
11979
11980 fn seeded_turmoil_builder(seed: u64) -> Builder {
11981 let mut builder = Builder::new();
11982 builder.rng_seed(seed);
11983 builder
11984 }
11985
11986 #[rstest]
11987 fn test_turmoil_reconnect_buffer_waits_for_auth() {
11988 let mut sim = seeded_turmoil_builder(AUTH_BUFFER_WAIT_SEED).build();
11989 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
11990 let server_messages = Arc::clone(&messages);
11991 sim.host("server", move || {
11992 let messages = Arc::clone(&server_messages);
11993 auth_buffer_server(messages)
11994 });
11995
11996 sim.client("client", async move {
11997 let tracker = AuthTracker::new();
11998 let (handler, _rx) = channel_message_handler();
11999 let client = WebSocketClient::builder()
12000 .config(turmoil_websocket_config())
12001 .message_handler(handler)
12002 .connect()
12003 .await
12004 .expect("Should connect");
12005
12006 client.set_auth_tracker(tracker.clone(), true);
12007 assert!(client.is_active(), "Client should start active");
12008
12009 wait_until_async(
12010 || async { client.is_reconnecting() },
12011 Duration::from_secs(3),
12012 )
12013 .await;
12014
12015 client
12016 .writer_tx
12017 .send(WriterCommand::Send(Message::Text("stale".into())))
12018 .unwrap();
12019
12020 wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12021
12022 let _auth_receiver = tracker.begin();
12023
12024 tokio::time::sleep(Duration::from_millis(300)).await;
12025 assert!(
12026 messages.lock().await.is_empty(),
12027 "buffered messages should wait for auth after reconnect"
12028 );
12029
12030 tracker.succeed();
12031
12032 wait_until_async(
12033 || {
12034 let messages = Arc::clone(&messages);
12035 async move { messages.lock().await.as_slice() == ["stale"] }
12036 },
12037 Duration::from_secs(3),
12038 )
12039 .await;
12040
12041 assert_eq!(messages.lock().await.as_slice(), ["stale"]);
12042
12043 client.disconnect().await;
12044 assert!(client.is_disconnected());
12045
12046 Ok(())
12047 });
12048
12049 sim.run().unwrap();
12050 }
12051
12052 #[rstest]
12053 fn test_turmoil_reconnect_buffer_discards_after_auth_failure() {
12054 let mut sim = seeded_turmoil_builder(AUTH_BUFFER_DISCARD_SEED).build();
12055 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
12056 let server_messages = Arc::clone(&messages);
12057 sim.host("server", move || {
12058 let messages = Arc::clone(&server_messages);
12059 auth_buffer_server(messages)
12060 });
12061
12062 sim.client("client", async move {
12063 let tracker = AuthTracker::new();
12064 let (handler, _rx) = channel_message_handler();
12065 let client = WebSocketClient::builder()
12066 .config(turmoil_websocket_config())
12067 .message_handler(handler)
12068 .connect()
12069 .await
12070 .expect("Should connect");
12071
12072 client.set_auth_tracker(tracker.clone(), true);
12073 assert!(client.is_active(), "Client should start active");
12074
12075 wait_until_async(
12076 || async { client.is_reconnecting() },
12077 Duration::from_secs(3),
12078 )
12079 .await;
12080
12081 client
12082 .writer_tx
12083 .send(WriterCommand::Send(Message::Text("stale".into())))
12084 .unwrap();
12085
12086 wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12087
12088 let _auth_receiver = tracker.begin();
12089 tracker.fail("rejected");
12090
12091 tokio::time::sleep(Duration::from_millis(300)).await;
12092 assert!(
12093 messages.lock().await.is_empty(),
12094 "buffered messages should be discarded after auth failure"
12095 );
12096
12097 let _retry_auth_receiver = tracker.begin();
12098 tracker.succeed();
12099
12100 tokio::time::sleep(Duration::from_millis(300)).await;
12101 assert!(
12102 messages.lock().await.is_empty(),
12103 "discarded messages should not replay on a later auth success"
12104 );
12105
12106 client.disconnect().await;
12107 assert!(client.is_disconnected());
12108
12109 Ok(())
12110 });
12111
12112 sim.run().unwrap();
12113 }
12114
12115 fn turmoil_websocket_config() -> WebSocketConfig {
12116 WebSocketConfig {
12117 url: "ws://server:8080".to_string(),
12118 headers: vec![],
12119 heartbeat_interval_secs: None,
12120 heartbeat_payload: None,
12121 connect_timeout_ms: Some(5_000),
12122 reconnect_delay_initial_ms: Some(50),
12123 reconnect_delay_max_ms: Some(200),
12124 reconnect_backoff_factor: Some(1.0),
12125 reconnect_jitter_ms: Some(0),
12126 reconnect_max_attempts: None,
12127 heartbeat_timeout_secs: None,
12128 idle_timeout_ms: None,
12129 writer_capacity: None,
12130 backend: TransportBackend::Tungstenite,
12131 proxy_url: None,
12132 max_message_size_bytes: None,
12133 max_frame_size_bytes: None,
12134 }
12135 }
12136
12137 async fn auth_buffer_server(
12138 messages: Arc<tokio::sync::Mutex<Vec<String>>>,
12139 ) -> Result<(), Box<dyn std::error::Error>> {
12140 let listener = net::TcpListener::bind("0.0.0.0:8080").await?;
12141
12142 let (stream, _) = listener.accept().await?;
12143 let mut websocket = accept_async(stream).await?;
12144 let _ = websocket.send(WsMessage::Text("first".into())).await;
12145 drop(websocket);
12146
12147 tokio::time::sleep(Duration::from_millis(200)).await;
12148
12149 let (stream, _) = listener.accept().await?;
12150 let mut websocket = accept_async(stream).await?;
12151
12152 while let Some(msg) = websocket.next().await {
12153 match msg {
12154 Ok(WsMessage::Text(text)) => {
12155 messages.lock().await.push(text.to_string());
12156 }
12157 Ok(WsMessage::Close(_)) => {
12158 let _ = websocket.close(None).await;
12159 break;
12160 }
12161 Ok(_) => {}
12162 Err(_) => break,
12163 }
12164 }
12165
12166 Ok(())
12167 }
12168}