1use std::{
57 collections::VecDeque,
58 fmt::{Debug, Display},
59 future::Future,
60 pin::pin,
61 sync::{
62 Arc, OnceLock,
63 atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering},
64 },
65 time::Duration,
66};
67
68use futures_util::{SinkExt, StreamExt};
69use http::HeaderName;
70use nautilus_core::string::secret::REDACTED;
71use nautilus_cryptography::providers::install_cryptographic_provider;
72use parking_lot::RwLock;
73#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
74use rustls::ClientConfig;
75#[cfg(feature = "transport-sockudo")]
76use sockudo_ws::{
77 Config as SockudoConfig, Http1, Role, Stream as SockudoStream,
78 WebSocketStream as SockudoWebSocketStream,
79};
80#[cfg(feature = "transport-sockudo")]
81use tokio::io::{AsyncRead, AsyncWrite};
82#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
83use tokio_rustls::TlsConnector;
84use tokio_tungstenite::tungstenite::{
85 client::IntoClientRequest, handshake::client::Request, http::HeaderValue, stream::Mode,
86};
87use tokio_util::sync::CancellationToken;
88use ustr::Ustr;
89
90#[cfg(not(feature = "turmoil"))]
91use super::proxy::{ProxyTarget, WsTarget, tunnel_via_proxy};
92use super::{
93 auth::{AuthState, AuthTracker},
94 config::{InitialConnectRetryPolicy, TransportBackend, WebSocketConfig},
95 consts::{
96 CONNECTION_STATE_CHECK_INTERVAL_MS, GRACEFUL_SHUTDOWN_DELAY_MS,
97 GRACEFUL_SHUTDOWN_TIMEOUT_SECS,
98 },
99 frame::FrameStream,
100 types::{
101 CloseHandler, EpochMessageHandler, EpochPingHandler, MessageHandler, MessageReader,
102 MessageWriter, PingHandler, WriterCommand,
103 },
104};
105#[cfg(feature = "transport-sockudo")]
106use crate::net::TcpStream;
107#[cfg(feature = "transport-sockudo")]
108use crate::transport::sockudo::{
109 PrefixedIo, SockudoTransport, client_handshake_with_headers, validate_extra_headers,
110};
111use crate::{
112 RECONNECTED, SocketState, SocketStateSink,
113 backoff::{
114 ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
115 },
116 dst,
117 error::{SendError, is_connection_drop_io_error},
118 logging::{log_task_aborted, log_task_started, log_task_stopped},
119 mode::{
120 ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
121 ReconnectRequestOutcome,
122 },
123 ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota},
124 retry::{RetryConfig, RetryError, RetryManager},
125 transport::{
126 BoxedWsTransport, CloseFrame, Message, TransportError, error::retryable_status,
127 tungstenite::TungsteniteTransport,
128 },
129 writer::{DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
130};
131
132const WRITE_TIMEOUT_SECS: u64 = 5;
133const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
134const INITIAL_CONNECT_OPERATION: &str = "WebSocket initial connection";
135
136const MAX_CONTROL_FRAME_PAYLOAD_BYTES: usize = 125;
138
139pub struct WebSocketClientInner {
164 config: WebSocketConfig,
165 reconnect_headers: ReconnectHeaders,
166 handler: Option<IncomingHandler>,
167 ping_handler: Option<IncomingPingHandler>,
168 close_handler: Option<CloseHandler>,
169 read_task: Option<dst::task::JoinHandle<()>>,
170 read_fence: Option<ReadSessionFence>,
171 write_task: dst::task::JoinHandle<()>,
172 writer_tx: WriterSender<WriterCommand>,
173 heartbeat_task: Option<dst::task::JoinHandle<()>>,
174 connection_mode: Arc<AtomicU8>,
175 connection_epoch: Arc<AtomicU64>,
176 state_notify: Arc<tokio::sync::Notify>,
177 controller_notify: Arc<tokio::sync::Notify>,
178 reconnect_published: Arc<AtomicBool>,
179 connect_timeout: Duration,
180 heartbeat_timeout: Option<Duration>,
181 backoff: ExponentialBackoff,
182 reconnect_throttle: ReconnectThrottle,
183 reconnect_max_attempts: Option<u32>,
184 reconnection_attempt_count: u32,
185 auth_tracker: Arc<OnceLock<AuthTracker>>,
186 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
187 state_sink: Option<SocketStateSink>,
188 connection_rate_limit: Option<ConnectionRateLimit>,
189}
190
191#[derive(Clone, Debug)]
192struct ConnectionRateLimit {
193 limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
194 keys: Arc<[Ustr]>,
195}
196
197#[derive(Default)]
198struct InitialConnectOptions {
199 retry_policy: Option<InitialConnectRetryPolicy>,
200 cancellation_token: Option<CancellationToken>,
201}
202
203impl WebSocketClientInner {
204 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
212 #[expect(
213 clippy::unused_async,
214 clippy::unused_async_trait_impl,
215 reason = "async signature for consistency with connect-based constructors"
216 )]
217 pub async fn new_with_writer(
218 config: WebSocketConfig,
219 writer: MessageWriter,
220 ) -> Result<Self, TransportError> {
221 Self::new_with_writer_and_state_sink(config, writer, None)
222 }
223
224 fn new_with_writer_and_state_sink(
225 mut config: WebSocketConfig,
226 writer: MessageWriter,
227 state_sink: Option<SocketStateSink>,
228 ) -> Result<Self, TransportError> {
229 install_cryptographic_provider();
230
231 crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
232 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
233 })?;
234
235 if config.heartbeat_interval_secs == Some(0) {
236 return Err(TransportError::Io(std::io::Error::new(
237 std::io::ErrorKind::InvalidInput,
238 "Heartbeat interval cannot be zero",
239 )));
240 }
241
242 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
243 let connection_epoch = Arc::new(AtomicU64::new(0));
244 let state_notify = Arc::new(tokio::sync::Notify::new());
245 let controller_notify = Arc::new(tokio::sync::Notify::new());
246 let reconnect_published = Arc::new(AtomicBool::new(true));
247 let outcome =
248 ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
249 debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
250
251 let read_task = None;
253 let read_fence = None;
254
255 let backoff = ExponentialBackoff::new(
257 Duration::from_secs(2),
258 Duration::from_secs(30),
259 1.5,
260 100,
261 true,
262 )
263 .map_err(|e| {
264 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
265 })?;
266
267 let auth_tracker = Arc::new(OnceLock::new());
268 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
269
270 let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
271 config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
272 );
273 let write_task = Self::spawn_write_task(
274 Arc::clone(&connection_mode),
275 Arc::clone(&controller_notify),
276 Arc::clone(&reconnect_published),
277 writer,
278 writer_rx,
279 Arc::clone(&connection_epoch),
280 Arc::clone(&auth_tracker),
281 Arc::clone(&reconnect_buffer_waits_for_auth),
282 state_sink.clone(),
283 );
284
285 let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_interval| {
286 Self::spawn_heartbeat_task(
287 Arc::clone(&connection_mode),
288 heartbeat_interval,
289 config.heartbeat_payload.clone(),
290 writer_tx.clone(),
291 )
292 });
293
294 let reconnect_max_attempts = None; let connect_timeout = Duration::from_secs(10);
296
297 let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
298
299 Ok(Self {
300 config,
301 reconnect_headers,
302 handler: None, ping_handler: None,
304 close_handler: None,
305 writer_tx,
306 connection_mode,
307 connection_epoch,
308 state_notify,
309 controller_notify,
310 reconnect_published,
311 connect_timeout,
312 heartbeat_timeout: None,
313 heartbeat_task,
314 read_task,
315 read_fence,
316 write_task,
317 backoff,
318 reconnect_throttle: ReconnectThrottle::default(),
319 reconnect_max_attempts,
320 reconnection_attempt_count: 0,
321 auth_tracker,
322 reconnect_buffer_waits_for_auth,
323 state_sink,
324 connection_rate_limit: None,
325 })
326 }
327
328 pub async fn connect_url(
337 config: WebSocketConfig,
338 message_handler: Option<MessageHandler>,
339 ping_handler: Option<PingHandler>,
340 ) -> Result<Self, TransportError> {
341 Self::connect_url_with_handler(
342 config,
343 message_handler.map(IncomingHandler::Message),
344 ping_handler.map(IncomingPingHandler::Ping),
345 None,
346 None,
347 None,
348 InitialConnectOptions::default(),
349 )
350 .await
351 }
352
353 async fn connect_url_with_handler(
354 config: WebSocketConfig,
355 handler: Option<IncomingHandler>,
356 ping_handler: Option<IncomingPingHandler>,
357 close_handler: Option<CloseHandler>,
358 state_sink: Option<SocketStateSink>,
359 connection_rate_limit: Option<ConnectionRateLimit>,
360 initial_connect_options: InitialConnectOptions,
361 ) -> Result<Self, TransportError> {
362 install_cryptographic_provider();
363
364 let is_stream_mode = handler.is_none();
365
366 if is_stream_mode {
370 crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
371 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
372 })?;
373
374 if config.heartbeat_interval_secs == Some(0) {
375 return Err(TransportError::Io(std::io::Error::new(
376 std::io::ErrorKind::InvalidInput,
377 "Heartbeat interval cannot be zero",
378 )));
379 }
380
381 validate_honored_inbound_limits(&config)?;
382 } else {
383 config.validate().map_err(|e| {
384 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
385 })?;
386 }
387
388 let heartbeat_timeout = config.resolved_heartbeat_timeout().map(Duration::from_secs);
389 let frame_activity = (!is_stream_mode && heartbeat_timeout.is_some())
390 .then(|| Arc::new(AtomicBool::new(false)));
391 let reconnect_max_attempts = config.reconnect_max_attempts;
392
393 let connect_timeout = if is_stream_mode {
395 Duration::from_secs(10)
396 } else {
397 Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000))
398 };
399
400 let backoff = ExponentialBackoff::new(
401 Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
402 Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
403 config.reconnect_backoff_factor.unwrap_or(1.5),
404 config.reconnect_jitter_ms.unwrap_or(100),
405 true, )
407 .map_err(|e| {
408 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
409 })?;
410
411 let cancellation_token = initial_connect_options
412 .cancellation_token
413 .unwrap_or_default();
414 let retry_policy = initial_connect_options.retry_policy;
415 let max_attempts = retry_policy
416 .as_ref()
417 .map_or(1, |policy| policy.max_attempts.get());
418 let retry_manager = RetryManager::new(initial_connect_retry_config(retry_policy.as_ref())?);
419 let attempt = AtomicU32::new(0);
420 let operation = || {
421 attempt.fetch_add(1, Ordering::Relaxed);
422
423 async {
424 await_initial_connect_attempt(&cancellation_token, async {
425 if let Some(rate_limit) = &connection_rate_limit {
426 rate_limit
427 .limiter
428 .await_keys_ready(Some(&rate_limit.keys))
429 .await;
430 }
431
432 dst::time::timeout(
434 connect_timeout,
435 Box::pin(Self::connect_with_server_activity(
436 &config.url,
437 config.headers.clone(),
438 config.backend,
439 config.proxy_url.as_deref(),
440 config.max_message_size_bytes,
441 config.max_frame_size_bytes,
442 frame_activity.clone(),
443 )),
444 )
445 .await
446 .unwrap_or_else(|_| {
447 Err(TransportError::Io(std::io::Error::new(
448 std::io::ErrorKind::TimedOut,
449 format!(
450 "connection timed out after {}s",
451 connect_timeout.as_secs_f64()
452 ),
453 )))
454 })
455 })
456 .await
457 .map_err(InitialConnectError)
458 }
459 };
460
461 let classify = |error: &InitialConnectError| {
462 let error = &error.0;
463 let retryable = is_retryable_initial_connect_error(error);
464 let attempt = attempt.load(Ordering::Relaxed);
465 if retryable && attempt < max_attempts && !cancellation_token.is_cancelled() {
466 log::warn!(
467 "WebSocket connection attempt {attempt}/{max_attempts} to {REDACTED} failed: {}",
468 error.redacted_message()
469 );
470 }
471
472 retryable
473 };
474
475 let transport = retry_manager
476 .invocation(INITIAL_CONNECT_OPERATION, operation, classify, |e| {
477 InitialConnectError(initial_connect_retry_error(e))
478 })
479 .cancellation_token(&cancellation_token)
480 .execute()
481 .await
482 .map_err(|e| e.0)?;
483
484 let attempt = attempt.load(Ordering::Relaxed);
485 if attempt > 1 {
486 log::info!("WebSocket connection established after {attempt} attempts");
487 }
488
489 let (writer, reader) = transport;
490 let mut config = config;
491 let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
492
493 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
494 let connection_epoch = Arc::new(AtomicU64::new(0));
495 let state_notify = Arc::new(tokio::sync::Notify::new());
496 let controller_notify = Arc::new(tokio::sync::Notify::new());
497 let reconnect_published = Arc::new(AtomicBool::new(true));
498 let outcome =
499 ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
500 debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
501
502 let (read_task, read_fence) = if is_stream_mode {
503 (None, None)
504 } else {
505 let read_fence = ReadSessionFence::new();
506 let read_task = Self::spawn_message_handler_task(
507 Arc::clone(&connection_mode),
508 Arc::clone(&state_notify),
509 read_fence.clone(),
510 reader,
511 0,
512 handler.as_ref(),
513 ping_handler.as_ref(),
514 close_handler.as_ref(),
515 state_sink.as_ref(),
516 config.idle_timeout_ms,
517 heartbeat_timeout,
518 frame_activity,
519 );
520 (Some(read_task), Some(read_fence))
521 };
522
523 let auth_tracker = Arc::new(OnceLock::new());
524 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
525
526 let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
527 config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
528 );
529 let write_task = Self::spawn_write_task(
530 Arc::clone(&connection_mode),
531 Arc::clone(&controller_notify),
532 Arc::clone(&reconnect_published),
533 writer,
534 writer_rx,
535 Arc::clone(&connection_epoch),
536 Arc::clone(&auth_tracker),
537 Arc::clone(&reconnect_buffer_waits_for_auth),
538 state_sink.clone(),
539 );
540
541 let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_secs| {
543 Self::spawn_heartbeat_task(
544 Arc::clone(&connection_mode),
545 heartbeat_secs,
546 config.heartbeat_payload.clone(),
547 writer_tx.clone(),
548 )
549 });
550
551 Ok(Self {
552 config,
553 reconnect_headers,
554 handler,
555 ping_handler,
556 close_handler,
557 read_task,
558 read_fence,
559 write_task,
560 writer_tx,
561 heartbeat_task,
562 connection_mode,
563 connection_epoch,
564 state_notify,
565 controller_notify,
566 reconnect_published,
567 connect_timeout,
568 heartbeat_timeout,
569 backoff,
570 reconnect_throttle: ReconnectThrottle::default(),
571 reconnect_max_attempts,
572 reconnection_attempt_count: 0,
573 auth_tracker,
574 reconnect_buffer_waits_for_auth,
575 state_sink,
576 connection_rate_limit,
577 })
578 }
579
580 #[inline]
602 pub async fn connect_with_server(
603 url: &str,
604 headers: Vec<(String, String)>,
605 backend: TransportBackend,
606 proxy_url: Option<&str>,
607 max_message_size_bytes: Option<usize>,
608 max_frame_size_bytes: Option<usize>,
609 ) -> Result<(MessageWriter, MessageReader), TransportError> {
610 Self::connect_with_server_activity(
611 url,
612 headers,
613 backend,
614 proxy_url,
615 max_message_size_bytes,
616 max_frame_size_bytes,
617 None,
618 )
619 .await
620 }
621
622 async fn connect_with_server_activity(
623 url: &str,
624 headers: Vec<(String, String)>,
625 backend: TransportBackend,
626 proxy_url: Option<&str>,
627 max_message_size_bytes: Option<usize>,
628 max_frame_size_bytes: Option<usize>,
629 frame_activity: Option<Arc<AtomicBool>>,
630 ) -> Result<(MessageWriter, MessageReader), TransportError> {
631 #[cfg(all(feature = "simulation", madsim))]
632 if backend == TransportBackend::Sockudo {
633 return Err(TransportError::Other(
634 "Sockudo timers are unsupported under simulation; select Tungstenite".into(),
635 ));
636 }
637
638 #[cfg(all(feature = "simulation", madsim))]
639 if proxy_url.is_some() || !url.starts_with("ws://") {
640 return Err(TransportError::InvalidUrl(
641 "WebSocket simulation requires plaintext ws:// endpoints without a proxy".into(),
642 ));
643 }
644
645 let limits = InboundConfig {
646 max_message_size_bytes,
647 max_frame_size_bytes,
648 frame_activity,
649 };
650
651 match backend {
652 TransportBackend::Tungstenite => match proxy_url {
653 Some(proxy) => {
654 Box::pin(Self::connect_tungstenite_via_proxy(
655 url, headers, proxy, limits,
656 ))
657 .await
658 }
659 None => Self::connect_tungstenite(url, headers, limits).await,
660 },
661 TransportBackend::Sockudo => {
662 #[cfg(feature = "transport-sockudo")]
663 {
664 match proxy_url {
665 Some(proxy) => {
666 Box::pin(Self::connect_sockudo_via_proxy(url, headers, proxy, limits))
667 .await
668 }
669 None => Self::connect_sockudo(url, headers, limits).await,
670 }
671 }
672
673 #[cfg(not(feature = "transport-sockudo"))]
674 {
675 Err(TransportError::Other(
676 "sockudo backend selected but the transport-sockudo \
677 Cargo feature is not enabled"
678 .to_string(),
679 ))
680 }
681 }
682 }
683 }
684
685 async fn connect_tungstenite(
686 url: &str,
687 headers: Vec<(String, String)>,
688 limits: InboundConfig,
689 ) -> Result<(MessageWriter, MessageReader), TransportError> {
690 let request = tungstenite_request(url, headers)?;
691 let mode = tokio_tungstenite::tungstenite::client::uri_mode(request.uri())?;
692 let host = request
693 .uri()
694 .host()
695 .ok_or_else(|| TransportError::InvalidUrl("missing hostname".into()))?;
696 let port = request
697 .uri()
698 .port_u16()
699 .unwrap_or(if matches!(mode, Mode::Tls) { 443 } else { 80 });
700 let address = format!("{host}:{port}");
701 let socket = dst::net::TcpStream::connect(address.as_str()).await?;
702 #[cfg(not(all(feature = "simulation", madsim)))]
703 crate::net::apply_socket_options(&socket);
704 let socket = crate::tls::tcp_tls(&request, mode, socket, None).await?;
705 let socket = FrameStream::new(socket, limits.frame_activity.clone());
706 let (stream, _) = tokio_tungstenite::client_async_with_config(
707 request,
708 socket,
709 tungstenite_protocol_config(&limits),
710 )
711 .await?;
712 let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
713 Ok(transport.split())
714 }
715
716 #[inline]
723 #[cfg(not(feature = "turmoil"))]
724 async fn connect_tungstenite_via_proxy(
725 url: &str,
726 headers: Vec<(String, String)>,
727 proxy_url: &str,
728 limits: InboundConfig,
729 ) -> Result<(MessageWriter, MessageReader), TransportError> {
730 let proxy = ProxyTarget::parse(proxy_url)?;
731
732 let request = tungstenite_request(url, headers)?;
733
734 let target = WsTarget::parse(url)?;
735 let stream = tunnel_via_proxy(&target, &proxy).await?;
736
737 let transport: BoxedWsTransport =
741 Box::pin(proxied_ws_handshake(request, stream, limits)).await?;
742
743 Ok(transport.split())
744 }
745
746 #[inline]
749 #[cfg(feature = "turmoil")]
750 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
751 #[expect(
752 clippy::unused_async,
753 clippy::unused_async_trait_impl,
754 reason = "signature mirrors the production variant; both are awaited in the dispatcher"
755 )]
756 async fn connect_tungstenite_via_proxy(
757 _url: &str,
758 _headers: Vec<(String, String)>,
759 _proxy_url: &str,
760 _limits: InboundConfig,
761 ) -> Result<(MessageWriter, MessageReader), TransportError> {
762 Err(TransportError::Other(
763 "proxy_url is not supported under the turmoil simulator".to_string(),
764 ))
765 }
766
767 #[inline]
776 #[cfg(feature = "transport-sockudo")]
777 async fn connect_sockudo(
778 url: &str,
779 headers: Vec<(String, String)>,
780 limits: InboundConfig,
781 ) -> Result<(MessageWriter, MessageReader), TransportError> {
782 let target = SockudoTarget::parse(url)?;
783 validate_extra_headers(&headers).map_err(TransportError::from)?;
784
785 #[cfg(feature = "turmoil")]
786 if target.is_tls {
787 return Err(TransportError::Tls(
788 "wss:// is not supported under the turmoil simulator; use ws://".to_string(),
789 ));
790 }
791
792 let tcp_stream = TcpStream::connect((target.host.as_str(), target.port))
793 .await
794 .map_err(TransportError::Io)?;
795
796 crate::net::apply_socket_options(&tcp_stream);
797
798 #[cfg(not(feature = "turmoil"))]
799 if target.is_tls {
800 let mut root_store = rustls::RootCertStore::empty();
801 root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
802 let config = ClientConfig::builder()
803 .with_root_certificates(root_store)
804 .with_no_client_auth();
805 let connector = TlsConnector::from(std::sync::Arc::new(config));
806 let domain = rustls::pki_types::ServerName::try_from(target.host.clone())
807 .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
808 let tls_stream = connector
809 .connect(domain, tcp_stream)
810 .await
811 .map_err(TransportError::Io)?;
812 return Self::finish_sockudo_handshake(tls_stream, &target, &headers, limits).await;
813 }
814
815 Self::finish_sockudo_handshake(tcp_stream, &target, &headers, limits).await
816 }
817
818 #[inline]
824 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
825 async fn connect_sockudo_via_proxy(
826 url: &str,
827 headers: Vec<(String, String)>,
828 proxy_url: &str,
829 limits: InboundConfig,
830 ) -> Result<(MessageWriter, MessageReader), TransportError> {
831 let proxy = ProxyTarget::parse(proxy_url)?;
832
833 let target = SockudoTarget::parse(url)?;
834 validate_extra_headers(&headers).map_err(TransportError::from)?;
835
836 let ws_target = WsTarget::parse(url)?;
840 let stream = tunnel_via_proxy(&ws_target, &proxy).await?;
841
842 Self::finish_sockudo_handshake(stream, &target, &headers, limits).await
843 }
844
845 #[inline]
848 #[cfg(all(feature = "transport-sockudo", feature = "turmoil"))]
849 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
850 #[expect(
851 clippy::unused_async,
852 clippy::unused_async_trait_impl,
853 reason = "signature mirrors the production variant; both are awaited in the dispatcher"
854 )]
855 async fn connect_sockudo_via_proxy(
856 _url: &str,
857 _headers: Vec<(String, String)>,
858 _proxy_url: &str,
859 _limits: InboundConfig,
860 ) -> Result<(MessageWriter, MessageReader), TransportError> {
861 Err(TransportError::Other(
862 "proxy_url is not supported under the turmoil simulator".to_string(),
863 ))
864 }
865
866 #[cfg(feature = "transport-sockudo")]
867 async fn finish_sockudo_handshake<S>(
868 stream: S,
869 target: &SockudoTarget,
870 headers: &[(String, String)],
871 limits: InboundConfig,
872 ) -> Result<(MessageWriter, MessageReader), TransportError>
873 where
874 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
875 {
876 let mut stream = FrameStream::new(stream, limits.frame_activity.clone());
880 let handshake =
881 client_handshake_with_headers(&mut stream, &target.host_header, &target.path, headers)
882 .await?;
883
884 let stream = match handshake.leftover {
887 Some(prefix) => SockudoStream::<Http1>::new(PrefixedIo::new(stream, prefix)),
888 None => SockudoStream::<Http1>::new(stream),
889 };
890
891 let ws = SockudoWebSocketStream::from_raw(
892 stream,
893 Role::Client,
894 sockudo_protocol_config(&limits),
895 );
896 let transport: BoxedWsTransport = Box::pin(SockudoTransport::new(ws));
897 Ok(transport.split())
898 }
899}
900
901#[derive(Clone, Debug, Default)]
902struct InboundConfig {
903 max_message_size_bytes: Option<usize>,
904 max_frame_size_bytes: Option<usize>,
905 frame_activity: Option<Arc<AtomicBool>>,
906}
907
908fn validate_honored_inbound_limits(config: &WebSocketConfig) -> Result<(), TransportError> {
909 if config.max_message_size_bytes == Some(0) {
910 return Err(TransportError::Io(std::io::Error::new(
911 std::io::ErrorKind::InvalidInput,
912 "max_message_size_bytes must be positive",
913 )));
914 }
915
916 if config.max_frame_size_bytes == Some(0) {
917 return Err(TransportError::Io(std::io::Error::new(
918 std::io::ErrorKind::InvalidInput,
919 "max_frame_size_bytes must be positive",
920 )));
921 }
922
923 Ok(())
924}
925
926fn tungstenite_protocol_config(
927 limits: &InboundConfig,
928) -> Option<tokio_tungstenite::tungstenite::protocol::WebSocketConfig> {
929 if limits.max_message_size_bytes.is_none() && limits.max_frame_size_bytes.is_none() {
930 return None;
931 }
932
933 let mut config = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
934 if let Some(size) = limits.max_message_size_bytes {
935 config.max_message_size = Some(size);
936 }
937
938 if let Some(size) = limits.max_frame_size_bytes {
939 config.max_frame_size = Some(size);
940 }
941
942 Some(config)
943}
944
945#[cfg(feature = "transport-sockudo")]
946fn sockudo_protocol_config(limits: &InboundConfig) -> SockudoConfig {
947 let mut config = SockudoConfig::builder()
948 .auto_ping(false)
949 .idle_timeout(0)
950 .build();
951
952 if let Some(size) = limits.max_message_size_bytes {
953 config.max_message_size = size;
954 }
955
956 if let Some(size) = limits.max_frame_size_bytes {
957 config.max_frame_size = size;
958 }
959
960 config
961}
962
963fn tungstenite_request(
964 url: &str,
965 headers: Vec<(String, String)>,
966) -> Result<Request, TransportError> {
967 let mut request = url.into_client_request().map_err(TransportError::from)?;
968
969 for (key, value) in headers {
970 let value = HeaderValue::from_str(&value)
971 .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
972 let name: HeaderName = key
973 .parse()
974 .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
975 request.headers_mut().insert(name, value);
976 }
977
978 Ok(request)
979}
980
981fn is_connection_drop_transport_error(err: &TransportError) -> bool {
982 err.is_closed() || matches!(err, TransportError::Io(e) if is_connection_drop_io_error(e))
983}
984
985fn is_retryable_initial_connect_error(err: &TransportError) -> bool {
986 match err {
987 TransportError::ConnectionClosed
988 | TransportError::ConnectionReset
989 | TransportError::ClosedByPeer(_) => true,
990 TransportError::Io(error) => !matches!(
991 error.kind(),
992 std::io::ErrorKind::InvalidInput
993 | std::io::ErrorKind::InvalidData
994 | std::io::ErrorKind::Unsupported
995 | std::io::ErrorKind::PermissionDenied
996 ),
997 TransportError::UpgradeRejected(status) | TransportError::ProxyConnectRejected(status) => {
998 retryable_status(*status)
999 }
1000 TransportError::InvalidUrl(_)
1001 | TransportError::Handshake(_)
1002 | TransportError::Tls(_)
1003 | TransportError::Protocol(_)
1004 | TransportError::MessageTooLarge
1005 | TransportError::FrameTooLarge
1006 | TransportError::InvalidUtf8
1007 | TransportError::Other(_) => false,
1008 }
1009}
1010
1011fn initial_connect_retry_config(
1012 policy: Option<&InitialConnectRetryPolicy>,
1013) -> Result<RetryConfig, TransportError> {
1014 if let Some(policy) = policy {
1015 ExponentialBackoff::new(
1016 policy.delay_initial,
1017 policy.delay_max,
1018 policy.backoff_factor,
1019 policy.jitter_ms,
1020 false,
1021 )
1022 .map_err(|e| {
1023 TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
1024 })?;
1025
1026 Ok(RetryConfig {
1027 max_retries: policy.max_attempts.get() - 1,
1028 initial_delay_ms: duration_to_millis("delay_initial", policy.delay_initial)?,
1029 max_delay_ms: duration_to_millis("delay_max", policy.delay_max)?,
1030 backoff_factor: policy.backoff_factor,
1031 jitter_ms: policy.jitter_ms,
1032 operation_timeout_ms: None,
1033 immediate_first: false,
1034 max_elapsed_ms: None,
1035 })
1036 } else {
1037 Ok(RetryConfig {
1038 max_retries: 0,
1039 initial_delay_ms: 1,
1040 max_delay_ms: 1,
1041 backoff_factor: 1.0,
1042 jitter_ms: 0,
1043 operation_timeout_ms: None,
1044 immediate_first: false,
1045 max_elapsed_ms: None,
1046 })
1047 }
1048}
1049
1050fn duration_to_millis(field: &str, duration: Duration) -> Result<u64, TransportError> {
1051 let nanoseconds = duration.as_nanos();
1052 if nanoseconds > u128::from(u64::MAX) {
1053 return Err(TransportError::Io(std::io::Error::new(
1054 std::io::ErrorKind::InvalidInput,
1055 format!("{field} exceeds the maximum backoff duration"),
1056 )));
1057 }
1058
1059 let milliseconds = nanoseconds
1060 .div_ceil(1_000_000)
1061 .min(u128::from(u64::MAX / 1_000_000));
1062 u64::try_from(milliseconds).map_err(|_| {
1063 TransportError::Io(std::io::Error::new(
1064 std::io::ErrorKind::InvalidInput,
1065 format!("{field} exceeds the maximum backoff duration"),
1066 ))
1067 })
1068}
1069
1070async fn await_initial_connect_attempt<F, T>(
1071 cancellation_token: &CancellationToken,
1072 attempt: F,
1073) -> Result<T, TransportError>
1074where
1075 F: Future<Output = Result<T, TransportError>>,
1076{
1077 tokio::select! {
1078 biased;
1079 () = cancellation_token.cancelled() => Err(initial_connect_cancelled()),
1080 result = attempt => result,
1081 }
1082}
1083
1084struct InitialConnectError(TransportError);
1085
1086impl std::error::Error for InitialConnectError {}
1087
1088impl Display for InitialConnectError {
1089 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1090 f.write_str(&self.0.redacted_message())
1091 }
1092}
1093
1094impl Debug for InitialConnectError {
1095 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1096 Display::fmt(self, f)
1097 }
1098}
1099
1100fn initial_connect_retry_error(error: RetryError) -> TransportError {
1101 let kind = match error {
1102 RetryError::Canceled => return initial_connect_cancelled(),
1103 RetryError::InvalidConfiguration { .. } => std::io::ErrorKind::InvalidInput,
1104 RetryError::OperationTimeout { .. } | RetryError::ElapsedBudgetExceeded { .. } => {
1105 std::io::ErrorKind::TimedOut
1106 }
1107 };
1108
1109 TransportError::Io(std::io::Error::new(kind, error))
1110}
1111
1112fn initial_connect_cancelled() -> TransportError {
1113 TransportError::Io(std::io::Error::new(
1114 std::io::ErrorKind::Interrupted,
1115 "initial WebSocket connection cancelled",
1116 ))
1117}
1118
1119fn read_termination_log_level(connection_state: &AtomicU8) -> log::Level {
1121 let mode = ConnectionMode::from_atomic(connection_state);
1122 if mode.is_disconnect() || mode.is_closed() {
1123 log::Level::Debug
1124 } else {
1125 log::Level::Warn
1126 }
1127}
1128
1129#[cfg(test)]
1130mod connection_error_tests {
1131 use std::io;
1132
1133 use rstest::rstest;
1134
1135 use super::*;
1136 use crate::transport::CloseFrame;
1137
1138 #[rstest]
1139 #[case(TransportError::ConnectionClosed, true)]
1140 #[case(TransportError::ConnectionReset, true)]
1141 #[case(TransportError::ClosedByPeer(Some(CloseFrame::new(1000, "bye"))), true)]
1142 #[case(TransportError::ClosedByPeer(None), true)]
1143 #[case(TransportError::Io(io::Error::from(io::ErrorKind::BrokenPipe)), true)]
1144 #[case(
1145 TransportError::Io(io::Error::from(io::ErrorKind::ConnectionReset)),
1146 true
1147 )]
1148 #[case(TransportError::Io(io::Error::from(io::ErrorKind::TimedOut)), true)]
1149 #[case(
1150 TransportError::Io(io::Error::from(io::ErrorKind::UnexpectedEof)),
1151 true
1152 )]
1153 #[case(
1154 TransportError::Io(io::Error::from(io::ErrorKind::InvalidInput)),
1155 false
1156 )]
1157 #[case(TransportError::InvalidUrl("http://example.com".into()), false)]
1158 #[case(TransportError::Handshake("bad".into()), false)]
1159 #[case(TransportError::Protocol("bad opcode".into()), false)]
1160 #[case(TransportError::Tls("bad certificate".into()), false)]
1161 #[case(TransportError::MessageTooLarge, false)]
1162 #[case(TransportError::FrameTooLarge, false)]
1163 #[case(TransportError::InvalidUtf8, false)]
1164 #[case(TransportError::Other("backend protocol mismatch".into()), false)]
1165 fn connection_drop_transport_error_classification(
1166 #[case] err: TransportError,
1167 #[case] expected: bool,
1168 ) {
1169 assert_eq!(is_connection_drop_transport_error(&err), expected);
1170 }
1171
1172 #[rstest]
1173 #[case(Duration::ZERO, 0)]
1174 #[case(Duration::from_nanos(1), 1)]
1175 #[case(Duration::from_micros(500), 1)]
1176 #[case(Duration::from_nanos(1_000_001), 2)]
1177 #[case(Duration::from_millis(500), 500)]
1178 #[case(Duration::from_secs(5), 5_000)]
1179 fn duration_to_millis_rounds_up(#[case] duration: Duration, #[case] expected: u64) {
1180 assert_eq!(duration_to_millis("delay", duration).unwrap(), expected);
1181 }
1182
1183 #[rstest]
1184 fn duration_to_millis_caps_at_backoff_maximum() {
1185 assert_eq!(
1186 duration_to_millis("delay", Duration::from_nanos(u64::MAX)).unwrap(),
1187 u64::MAX / 1_000_000
1188 );
1189 }
1190
1191 #[rstest]
1192 fn duration_to_millis_rejects_excessive_duration() {
1193 let error = duration_to_millis("delay", Duration::from_secs(u64::MAX)).unwrap_err();
1194
1195 let TransportError::Io(error) = error else {
1196 panic!("expected an I/O error, was: {error:?}");
1197 };
1198
1199 assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1200 assert_eq!(
1201 error.to_string(),
1202 "delay exceeds the maximum backoff duration"
1203 );
1204 }
1205
1206 #[rstest]
1207 fn initial_connect_retry_config_preserves_duration_validation() {
1208 let policy = InitialConnectRetryPolicy {
1209 max_attempts: std::num::NonZeroU32::new(2).unwrap(),
1210 delay_initial: Duration::from_micros(900),
1211 delay_max: Duration::from_micros(500),
1212 backoff_factor: 2.0,
1213 jitter_ms: 0,
1214 };
1215
1216 let error = initial_connect_retry_config(Some(&policy)).unwrap_err();
1217
1218 let TransportError::Io(error) = error else {
1219 panic!("expected an I/O error, was: {error:?}");
1220 };
1221
1222 assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1223 assert!(
1224 error
1225 .to_string()
1226 .contains("delay_max must be >= delay_initial")
1227 );
1228 }
1229
1230 #[rstest]
1231 #[tokio::test]
1232 async fn initial_connect_attempt_prefers_cancellation_over_ready_result() {
1233 let token = CancellationToken::new();
1234 token.cancel();
1235
1236 let error = await_initial_connect_attempt(&token, std::future::ready(Ok(())))
1237 .await
1238 .expect_err("cancellation should take priority");
1239
1240 assert!(
1241 matches!(error, TransportError::Io(ref error) if error.kind() == io::ErrorKind::Interrupted)
1242 );
1243 }
1244
1245 #[rstest]
1246 fn unset_inbound_limits_preserve_backend_defaults() {
1247 let limits = InboundConfig::default();
1248 assert!(tungstenite_protocol_config(&limits).is_none());
1249
1250 #[cfg(feature = "transport-sockudo")]
1251 {
1252 let config = sockudo_protocol_config(&limits);
1253 let default = SockudoConfig::default();
1254 assert_eq!(config.max_message_size, default.max_message_size);
1255 assert_eq!(config.max_frame_size, default.max_frame_size);
1256 assert!(!config.auto_ping);
1257 assert_eq!(config.idle_timeout, 0);
1258 }
1259 }
1260
1261 #[rstest]
1262 fn configured_inbound_limit_overrides_only_that_cap() {
1263 let limits = InboundConfig {
1264 max_message_size_bytes: Some(32),
1265 max_frame_size_bytes: None,
1266 frame_activity: None,
1267 };
1268
1269 let config = tungstenite_protocol_config(&limits).expect("override should build a config");
1270 let default = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
1271
1272 assert_eq!(config.max_message_size, Some(32));
1273 assert_eq!(config.max_frame_size, default.max_frame_size);
1274
1275 #[cfg(feature = "transport-sockudo")]
1276 {
1277 let config = sockudo_protocol_config(&limits);
1278 let default = SockudoConfig::default();
1279 assert_eq!(config.max_message_size, 32);
1280 assert_eq!(config.max_frame_size, default.max_frame_size);
1281 assert!(!config.auto_ping);
1282 assert_eq!(config.idle_timeout, 0);
1283 }
1284 }
1285}
1286
1287#[cfg(not(feature = "turmoil"))]
1292async fn proxied_ws_handshake<S>(
1293 request: tokio_tungstenite::tungstenite::handshake::client::Request,
1294 stream: S,
1295 limits: InboundConfig,
1296) -> Result<BoxedWsTransport, TransportError>
1297where
1298 S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
1299{
1300 let stream = FrameStream::new(stream, limits.frame_activity.clone());
1301 let (ws, _resp) = tokio_tungstenite::client_async_with_config(
1302 request,
1303 stream,
1304 tungstenite_protocol_config(&limits),
1305 )
1306 .await
1307 .map_err(TransportError::from)?;
1308 Ok(Box::pin(TungsteniteTransport::new(ws)))
1309}
1310
1311#[cfg(feature = "transport-sockudo")]
1318#[derive(Debug, PartialEq, Eq)]
1319struct SockudoTarget {
1320 host: String,
1321 host_header: String,
1322 port: u16,
1323 path: String,
1324 is_tls: bool,
1325}
1326
1327#[cfg(feature = "transport-sockudo")]
1328impl SockudoTarget {
1329 fn parse(url: &str) -> Result<Self, TransportError> {
1330 let parsed = url::Url::parse(url)
1331 .map_err(|e| TransportError::InvalidUrl(format!("invalid WebSocket URL: {e}")))?;
1332
1333 let scheme = parsed.scheme();
1334
1335 let is_tls = match scheme {
1336 "ws" => false,
1337 "wss" => true,
1338 other => {
1339 return Err(TransportError::InvalidUrl(format!(
1340 "expected ws:// or wss:// scheme, was {other}"
1341 )));
1342 }
1343 };
1344
1345 let raw_host = parsed
1346 .host_str()
1347 .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
1348
1349 let is_bracketed = raw_host.starts_with('[') && raw_host.ends_with(']');
1354 let host = if is_bracketed {
1355 raw_host[1..raw_host.len() - 1].to_string()
1356 } else {
1357 raw_host.to_string()
1358 };
1359
1360 let explicit_port = parsed.port();
1361 let port = explicit_port.unwrap_or(if is_tls { 443 } else { 80 });
1362
1363 let host_header = match explicit_port {
1364 Some(p) => format!("{raw_host}:{p}"),
1365 None => raw_host.to_string(),
1366 };
1367
1368 let path = if parsed.path().is_empty() {
1369 "/".to_string()
1370 } else {
1371 let mut p = parsed.path().to_string();
1372 if let Some(query) = parsed.query() {
1373 p.push('?');
1374 p.push_str(query);
1375 }
1376
1377 p
1378 };
1379
1380 Ok(Self {
1381 host,
1382 host_header,
1383 port,
1384 path,
1385 is_tls,
1386 })
1387 }
1388}
1389
1390impl WebSocketClientInner {
1391 pub async fn reconnect(&mut self) -> Result<(), TransportError> {
1412 Box::pin(self.reconnect_with_outcome()).await.map(|_| ())
1413 }
1414
1415 async fn wait_for_reconnect_publication(&self) -> bool {
1416 let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1417
1418 loop {
1419 let mut notified = pin!(self.controller_notify.notified());
1420 notified.as_mut().enable();
1421
1422 if !ConnectionMode::from_atomic(&self.connection_mode).is_reconnect() {
1423 return false;
1424 }
1425
1426 if self.reconnect_published.load(Ordering::SeqCst) {
1427 return true;
1428 }
1429
1430 tokio::select! {
1431 biased;
1432 () = notified => {}
1433 () = dst::time::sleep(fallback_interval) => {}
1434 }
1435 }
1436 }
1437
1438 async fn reconnect_with_outcome(&mut self) -> Result<ReconnectOutcome, TransportError> {
1439 log::info!("Reconnecting");
1440
1441 if self.handler.is_none() {
1442 log::warn!(
1443 "Auto-reconnect disabled for stream-based WebSocket client; \
1444 stream users must manually reconnect by creating a new connection"
1445 );
1446 self.connection_mode
1448 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1449 fail_registered_auth(
1450 self.auth_tracker.as_ref(),
1451 "WebSocket stream mode cannot reconnect",
1452 );
1453 self.state_notify.notify_waiters();
1457 return Ok(ReconnectOutcome::Aborted);
1458 }
1459
1460 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1461 log::debug!("Reconnect aborted due to disconnect state");
1462 return Ok(ReconnectOutcome::Aborted);
1463 }
1464
1465 if let Some(rate_limit) = &self.connection_rate_limit {
1466 rate_limit
1467 .limiter
1468 .await_keys_ready(Some(&rate_limit.keys))
1469 .await;
1470 }
1471
1472 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1473 log::debug!("Reconnect aborted during connection rate-limit wait");
1474 return Ok(ReconnectOutcome::Aborted);
1475 }
1476
1477 let frame_activity = self
1478 .heartbeat_timeout
1479 .map(|_| Arc::new(AtomicBool::new(false)));
1480
1481 let (new_writer, reader) = dst::time::timeout(
1483 self.connect_timeout,
1484 Box::pin(Self::connect_with_server_activity(
1485 &self.config.url,
1486 self.reconnect_headers.snapshot(),
1487 self.config.backend,
1488 self.config.proxy_url.as_deref(),
1489 self.config.max_message_size_bytes,
1490 self.config.max_frame_size_bytes,
1491 frame_activity.clone(),
1492 )),
1493 )
1494 .await
1495 .map_err(|_| {
1496 TransportError::Io(std::io::Error::new(
1497 std::io::ErrorKind::TimedOut,
1498 format!(
1499 "reconnection timed out after {}s",
1500 self.connect_timeout.as_secs_f64()
1501 ),
1502 ))
1503 })??;
1504
1505 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1506 log::debug!("Reconnect aborted mid-flight (after connect)");
1507 return Ok(ReconnectOutcome::Aborted);
1508 }
1509
1510 let (tx, rx) = tokio::sync::oneshot::channel();
1513
1514 if let Err(e) = self
1515 .writer_tx
1516 .send_update(WriterCommand::Update(new_writer, tx))
1517 {
1518 log::error!("{e}");
1519 return Err(TransportError::Io(std::io::Error::new(
1520 std::io::ErrorKind::BrokenPipe,
1521 format!("Failed to send update command: {e}"),
1522 )));
1523 }
1524
1525 let connection_epoch = match rx.await {
1527 Ok(connection_epoch) => {
1528 log::debug!("Writer confirmed socket update: epoch={connection_epoch}");
1529 connection_epoch
1530 }
1531 Err(e) => {
1532 log::error!("Writer dropped update channel: {e}");
1533 return Err(TransportError::Io(std::io::Error::new(
1534 std::io::ErrorKind::BrokenPipe,
1535 "Writer task dropped response channel",
1536 )));
1537 }
1538 };
1539
1540 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1542
1543 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1544 log::debug!("Reconnect aborted mid-flight (after delay)");
1545 return Ok(ReconnectOutcome::Aborted);
1546 }
1547
1548 if let Some(read_fence) = self.read_fence.take() {
1549 read_fence.invalidate();
1550 }
1551
1552 if let Some(ref read_task) = self.read_task.take()
1553 && !read_task.is_finished()
1554 {
1555 read_task.abort();
1556 log_task_aborted("read");
1557 }
1558
1559 if !self.wait_for_reconnect_publication().await {
1560 log::debug!("Reconnect aborted before state publication completed");
1561 return Ok(ReconnectOutcome::Aborted);
1562 }
1563
1564 if ConnectionMode::complete_reconnect_with_sink(
1567 &self.connection_mode,
1568 self.state_sink.as_ref(),
1569 ) == ReconnectOutcome::Aborted
1570 {
1571 log::debug!("Reconnect aborted (state changed during reconnect)");
1572 return Ok(ReconnectOutcome::Aborted);
1573 }
1574
1575 let read_fence = ReadSessionFence::new();
1576 self.read_task = Some(Self::spawn_message_handler_task(
1577 Arc::clone(&self.connection_mode),
1578 Arc::clone(&self.state_notify),
1579 read_fence.clone(),
1580 reader,
1581 connection_epoch,
1582 self.handler.as_ref(),
1583 self.ping_handler.as_ref(),
1584 self.close_handler.as_ref(),
1585 self.state_sink.as_ref(),
1586 self.config.idle_timeout_ms,
1587 self.heartbeat_timeout,
1588 frame_activity,
1589 ));
1590 self.read_fence = Some(read_fence);
1591
1592 log::info!("Reconnect succeeded");
1593 Ok(ReconnectOutcome::Reconnected)
1594 }
1595
1596 #[inline]
1602 #[must_use]
1603 pub fn is_alive(&self) -> bool {
1604 match &self.read_task {
1605 Some(read_task) => !read_task.is_finished() && !self.write_task.is_finished(),
1606 None => !self.write_task.is_finished(),
1607 }
1608 }
1609
1610 #[expect(
1611 clippy::too_many_arguments,
1612 reason = "both handler modes share the same reader lifecycle"
1613 )]
1614 fn spawn_message_handler_task(
1615 connection_state: Arc<AtomicU8>,
1616 state_notify: Arc<tokio::sync::Notify>,
1617 read_fence: ReadSessionFence,
1618 mut reader: MessageReader,
1619 connection_epoch: u64,
1620 handler: Option<&IncomingHandler>,
1621 ping_handler: Option<&IncomingPingHandler>,
1622 close_handler: Option<&CloseHandler>,
1623 state_sink: Option<&SocketStateSink>,
1624 idle_timeout_ms: Option<u64>,
1625 heartbeat_timeout: Option<Duration>,
1626 frame_activity: Option<Arc<AtomicBool>>,
1627 ) -> dst::task::JoinHandle<()> {
1628 log::debug!("Started message handler task 'read'");
1629
1630 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1631 let idle_timeout = idle_timeout_ms.map(Duration::from_millis);
1632
1633 let handler = handler.cloned();
1634 let ping_handler = ping_handler.cloned();
1635 let close_handler = close_handler.cloned();
1636 let state_sink = state_sink.cloned();
1637
1638 dst::task::spawn(async move {
1639 let mut last_data_time = dst::time::Instant::now();
1640 let mut last_frame_time = dst::time::Instant::now();
1641
1642 loop {
1643 if !ConnectionMode::from_atomic(&connection_state).is_active()
1644 || !read_fence.is_valid()
1645 {
1646 break;
1647 }
1648
1649 let read_result = dst::time::timeout(check_interval, reader.next()).await;
1650
1651 if let Ok(Some(Ok(ref message))) = read_result
1652 && (!ConnectionMode::from_atomic(&connection_state).is_active()
1653 || !read_fence.is_valid())
1654 {
1655 log::debug!(
1656 "Dropping WebSocket message with {} bytes after session ended",
1657 message.as_bytes().len()
1658 );
1659 break;
1660 }
1661
1662 if matches!(&read_result, Ok(Some(Ok(_)))) {
1663 last_frame_time = dst::time::Instant::now();
1664 }
1665
1666 match read_result {
1667 Ok(Some(Ok(Message::Binary(data)))) => {
1668 log::trace!("Received message <binary> {} bytes", data.len());
1669 last_data_time = dst::time::Instant::now();
1670
1671 if !ConnectionMode::from_atomic(&connection_state).is_active()
1672 || !read_fence.is_valid()
1673 {
1674 log::debug!(
1675 "Dropping WebSocket message with {} bytes after session ended",
1676 data.len()
1677 );
1678 break;
1679 }
1680
1681 if let Some(ref handler) = handler {
1682 handler.handle(connection_epoch, Message::Binary(data));
1683 }
1684 }
1685 Ok(Some(Ok(Message::Text(data)))) => {
1686 log::trace!("Received text frame ({} bytes)", data.len());
1687 last_data_time = dst::time::Instant::now();
1688
1689 if !ConnectionMode::from_atomic(&connection_state).is_active()
1690 || !read_fence.is_valid()
1691 {
1692 log::debug!(
1693 "Dropping WebSocket message with {} bytes after session ended",
1694 data.len()
1695 );
1696 break;
1697 }
1698
1699 if let Some(ref handler) = handler {
1700 handler.handle(connection_epoch, Message::Text(data));
1701 }
1702 }
1703 Ok(Some(Ok(Message::Ping(ping_data)))) => {
1704 log::trace!("Received ping frame ({} bytes)", ping_data.len());
1705 if let Some(ref handler) = ping_handler {
1710 if !ConnectionMode::from_atomic(&connection_state).is_active()
1711 || !read_fence.is_valid()
1712 {
1713 log::debug!(
1714 "Dropping WebSocket ping with {} bytes after session ended",
1715 ping_data.len()
1716 );
1717 break;
1718 }
1719
1720 handler.handle(connection_epoch, ping_data.to_vec());
1721 }
1722
1723 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1724 break;
1725 }
1726 }
1727 Ok(Some(Ok(Message::Pong(_)))) => {
1728 log::trace!("Received pong");
1729 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1732 break;
1733 }
1734 }
1735 Ok(Some(Ok(Message::Close(frame)))) => {
1736 Self::dispatch_close_frame(
1737 &connection_state,
1738 &read_fence,
1739 close_handler.as_ref(),
1740 state_sink.as_ref(),
1741 frame,
1742 );
1743 break;
1744 }
1745 Ok(Some(Err(e))) => {
1746 if is_connection_drop_transport_error(&e) {
1747 log::warn!("Received connection error, terminating: {e}");
1748 } else {
1749 log::error!("Received transport error, terminating: {e}");
1750 }
1751
1752 break;
1753 }
1754 Ok(None) => {
1755 log::log!(
1756 read_termination_log_level(&connection_state),
1757 "Connection closed by peer (no close frame), terminating"
1758 );
1759 break;
1760 }
1761 Err(_) => {
1762 if heartbeat_timeout_exceeded(
1763 &mut last_frame_time,
1764 heartbeat_timeout,
1765 frame_activity.as_deref(),
1766 ) {
1767 break;
1768 }
1769
1770 if idle_timeout_exceeded(last_data_time, idle_timeout) {
1771 break;
1772 }
1773 }
1774 }
1775 }
1776
1777 state_notify.notify_one();
1779 })
1780 }
1781
1782 fn dispatch_close_frame(
1783 connection_state: &AtomicU8,
1784 read_fence: &ReadSessionFence,
1785 close_handler: Option<&CloseHandler>,
1786 state_sink: Option<&SocketStateSink>,
1787 frame: Option<CloseFrame>,
1788 ) {
1789 match &frame {
1790 Some(frame) => log::log!(
1791 read_termination_log_level(connection_state),
1792 "Received close frame, terminating: code={}, reason={REDACTED}",
1793 frame.code
1794 ),
1795 None => log::log!(
1796 read_termination_log_level(connection_state),
1797 "Received close frame with no code or reason, terminating"
1798 ),
1799 }
1800
1801 if !ConnectionMode::from_atomic(connection_state).is_active() || !read_fence.is_valid() {
1802 return;
1803 }
1804
1805 if let Some(handler) = close_handler
1806 && !handler(frame)
1807 {
1808 ConnectionMode::close_websocket_on_loss(connection_state, state_sink);
1809 }
1810 }
1811
1812 fn buffer_for_replay(
1820 buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1821 msg: Message,
1822 permit: tokio::sync::OwnedSemaphorePermit,
1823 ) {
1824 if msg.is_control() {
1825 return;
1826 }
1827
1828 log::debug!(
1829 "Buffering message for replay (buffer size: {})",
1830 buffer.len() + 1
1831 );
1832
1833 buffer.push_back((msg, permit));
1834 }
1835
1836 async fn drain_reconnect_buffer(
1841 buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1842 writer: &mut MessageWriter,
1843 connection_state: &AtomicU8,
1844 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1845 reconnect_buffer_waits_for_auth: &AtomicBool,
1846 ) -> bool {
1847 if buffer.is_empty() {
1848 return false;
1849 }
1850
1851 let initial_buffer_len = buffer.len();
1852 log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
1853
1854 while !buffer.is_empty() {
1855 match Self::reconnect_buffer_action(
1856 reconnect_buffer_waits_for_auth,
1857 auth_tracker,
1858 connection_state,
1859 ) {
1860 ReconnectBufferAction::Drain => {}
1861 ReconnectBufferAction::Wait => return false,
1862 ReconnectBufferAction::Discard => {
1863 log::warn!(
1864 "Discarding {} buffered messages after authentication failed",
1865 buffer.len()
1866 );
1867 buffer.clear();
1868 return false;
1869 }
1870 }
1871
1872 let msg_to_send = buffer
1874 .front()
1875 .expect("reconnect buffer should not be empty")
1876 .0
1877 .clone();
1878
1879 if let Err(e) = writer.send(msg_to_send).await {
1880 if is_connection_drop_transport_error(&e) {
1881 log::warn!(
1882 "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1883 buffer.len()
1884 );
1885 } else {
1886 log::error!(
1887 "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1888 buffer.len()
1889 );
1890 }
1891
1892 return true;
1893 }
1894
1895 buffer.pop_front();
1897 }
1898
1899 if buffer.is_empty() {
1900 log::info!("Successfully sent all {initial_buffer_len} buffered messages");
1901 }
1902
1903 false
1904 }
1905
1906 fn can_drain_reconnect_buffer(
1907 reconnect_buffer_waits_for_auth: &AtomicBool,
1908 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1909 ) -> ReconnectBufferAction {
1910 if !reconnect_buffer_waits_for_auth.load(Ordering::Acquire) {
1911 return ReconnectBufferAction::Drain;
1912 }
1913
1914 match auth_tracker.get().map(AuthTracker::auth_state) {
1915 Some(AuthState::Authenticated) => ReconnectBufferAction::Drain,
1916 Some(AuthState::Failed) => ReconnectBufferAction::Discard,
1917 Some(AuthState::Unauthenticated) | None => ReconnectBufferAction::Wait,
1918 }
1919 }
1920
1921 fn reconnect_buffer_action(
1922 reconnect_buffer_waits_for_auth: &AtomicBool,
1923 auth_tracker: &Arc<OnceLock<AuthTracker>>,
1924 connection_state: &AtomicU8,
1925 ) -> ReconnectBufferAction {
1926 let action =
1927 Self::can_drain_reconnect_buffer(reconnect_buffer_waits_for_auth, auth_tracker);
1928
1929 if action == ReconnectBufferAction::Drain
1931 && !ConnectionMode::from_atomic(connection_state).is_active()
1932 {
1933 ReconnectBufferAction::Wait
1934 } else {
1935 action
1936 }
1937 }
1938
1939 #[expect(
1940 clippy::too_many_arguments,
1941 reason = "writer task owns the transport and shared lifecycle coordination state"
1942 )]
1943 fn spawn_write_task(
1944 connection_state: Arc<AtomicU8>,
1945 controller_notify: Arc<tokio::sync::Notify>,
1946 reconnect_published: Arc<AtomicBool>,
1947 writer: MessageWriter,
1948 mut writer_rx: WriterReceiver<WriterCommand>,
1949 connection_epoch: Arc<AtomicU64>,
1950 auth_tracker: Arc<OnceLock<AuthTracker>>,
1951 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1952 state_sink: Option<SocketStateSink>,
1953 ) -> dst::task::JoinHandle<()> {
1954 log_task_started("write");
1955
1956 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1958
1959 dst::task::spawn(async move {
1960 let mut active_writer = writer;
1961 let mut reconnect_buffer: VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)> =
1964 VecDeque::new();
1965
1966 loop {
1967 let mode = ConnectionMode::from_atomic(&connection_state);
1968
1969 match mode {
1970 ConnectionMode::Disconnect => {
1971 if !reconnect_buffer.is_empty() {
1973 log::warn!(
1974 "Discarding {} buffered messages due to disconnect",
1975 reconnect_buffer.len()
1976 );
1977 reconnect_buffer.clear();
1978 }
1979
1980 _ = dst::time::timeout(
1983 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
1984 active_writer.close(),
1985 )
1986 .await;
1987 break;
1988 }
1989 ConnectionMode::Closed => {
1990 if !reconnect_buffer.is_empty() {
1992 log::warn!(
1993 "Discarding {} buffered messages due to closed connection",
1994 reconnect_buffer.len()
1995 );
1996 reconnect_buffer.clear();
1997 }
1998
1999 break;
2000 }
2001 _ => {}
2002 }
2003
2004 if mode.is_active() && !reconnect_buffer.is_empty() {
2005 match Self::reconnect_buffer_action(
2006 reconnect_buffer_waits_for_auth.as_ref(),
2007 &auth_tracker,
2008 &connection_state,
2009 ) {
2010 ReconnectBufferAction::Drain => {
2011 let drain_result = dst::time::timeout(
2012 Duration::from_secs(WRITE_TIMEOUT_SECS),
2013 Self::drain_reconnect_buffer(
2014 &mut reconnect_buffer,
2015 &mut active_writer,
2016 &connection_state,
2017 &auth_tracker,
2018 reconnect_buffer_waits_for_auth.as_ref(),
2019 ),
2020 )
2021 .await;
2022
2023 let send_error = drain_result.unwrap_or_else(|_| {
2024 log::warn!(
2025 "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
2026 reconnect_buffer.len()
2027 );
2028 true
2029 });
2030
2031 if send_error {
2033 _ = request_websocket_reconnect(
2034 &connection_state,
2035 &reconnect_published,
2036 state_sink.as_ref(),
2037 &auth_tracker,
2038 &controller_notify,
2039 || {},
2040 );
2041 }
2042
2043 continue;
2044 }
2045 ReconnectBufferAction::Discard => {
2046 log::warn!(
2047 "Discarding {} buffered messages after authentication failed",
2048 reconnect_buffer.len()
2049 );
2050 reconnect_buffer.clear();
2051 continue;
2052 }
2053 ReconnectBufferAction::Wait => {}
2054 }
2055 }
2056
2057 match dst::time::timeout(check_interval, writer_rx.recv()).await {
2058 Ok(Some((msg, permit))) => {
2059 let mode = ConnectionMode::from_atomic(&connection_state);
2061
2062 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
2063 break;
2064 }
2065
2066 match msg {
2067 WriterCommand::Update(new_writer, tx) => {
2068 log::debug!("Received new writer");
2069
2070 dst::time::sleep(Duration::from_millis(100)).await;
2072
2073 _ = dst::time::timeout(
2076 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2077 active_writer.close(),
2078 )
2079 .await;
2080
2081 active_writer = new_writer;
2082 let epoch = connection_epoch.fetch_add(1, Ordering::AcqRel) + 1;
2083 log::debug!("Updated writer: epoch={epoch}");
2084
2085 if let Err(e) = tx.send(epoch) {
2086 log::error!(
2087 "Failed to report writer update to controller: {e:?}"
2088 );
2089 }
2090 }
2091 WriterCommand::Send(msg)
2092 if mode.is_reconnect()
2093 || (!reconnect_buffer.is_empty() && !msg.is_control()) =>
2094 {
2095 Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2096 }
2097 WriterCommand::Heartbeat(_)
2098 | WriterCommand::SendPongOnConnection { .. }
2099 if mode.is_reconnect() => {}
2100 WriterCommand::SendOnConnection { response_tx, .. }
2101 if mode.is_reconnect() =>
2102 {
2103 _ = response_tx.send(Err(SendError::ConnectionChanged));
2104 }
2105 WriterCommand::SendPongOnConnection {
2106 data,
2107 connection_epoch: expected_epoch,
2108 } => {
2109 let epoch = connection_epoch.load(Ordering::Acquire);
2110 if epoch != expected_epoch {
2111 continue;
2112 }
2113
2114 let send_result = dst::time::timeout(
2115 Duration::from_secs(WRITE_TIMEOUT_SECS),
2116 active_writer.send(Message::Pong(data.into())),
2117 )
2118 .await;
2119
2120 let send_failed = match send_result {
2121 Ok(Ok(())) => false,
2122 Ok(Err(e)) => {
2123 if is_connection_drop_transport_error(&e) {
2124 log::warn!("Failed to send pong: {e}");
2125 } else {
2126 log::error!("Failed to send pong: {e}");
2127 }
2128
2129 true
2130 }
2131 Err(_) => {
2132 log::warn!(
2133 "Timed out sending pong after {WRITE_TIMEOUT_SECS}s"
2134 );
2135 true
2136 }
2137 };
2138
2139 if send_failed
2140 && request_websocket_reconnect(
2141 &connection_state,
2142 &reconnect_published,
2143 state_sink.as_ref(),
2144 &auth_tracker,
2145 &controller_notify,
2146 || {},
2147 ) == ReconnectRequestOutcome::Accepted
2148 {
2149 log::warn!("Writer triggering reconnect");
2150 }
2151 }
2152 WriterCommand::SendOnConnection {
2153 message,
2154 connection_epoch: expected_epoch,
2155 response_tx,
2156 } => {
2157 let epoch = connection_epoch.load(Ordering::Acquire);
2158 if epoch != expected_epoch {
2159 _ = response_tx.send(Err(SendError::ConnectionChanged));
2160 continue;
2161 }
2162
2163 let send_result = dst::time::timeout(
2164 Duration::from_secs(WRITE_TIMEOUT_SECS),
2165 active_writer.send(message),
2166 )
2167 .await;
2168
2169 let result = match send_result {
2173 Ok(Ok(())) => Ok(()),
2174 Ok(Err(e)) => {
2175 if is_connection_drop_transport_error(&e) {
2176 log::warn!("Failed to send message: {e}");
2177 } else {
2178 log::error!("Failed to send message: {e}");
2179 }
2180
2181 Err(SendError::BrokenPipe(e.to_string()))
2182 }
2183 Err(_) => {
2184 log::warn!(
2185 "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
2186 );
2187
2188 Err(SendError::WriteTimeout)
2189 }
2190 };
2191
2192 let send_failed = result.is_err();
2193 _ = response_tx.send(result);
2194
2195 if send_failed
2196 && request_websocket_reconnect(
2197 &connection_state,
2198 &reconnect_published,
2199 state_sink.as_ref(),
2200 &auth_tracker,
2201 &controller_notify,
2202 || {},
2203 ) == ReconnectRequestOutcome::Accepted
2204 {
2205 log::warn!("Writer triggering reconnect");
2206 }
2207 }
2208 WriterCommand::Send(msg) => {
2209 let send_failed =
2210 Self::write_outbound(&mut active_writer, msg.clone()).await;
2211 if send_failed {
2212 Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2213
2214 if request_websocket_reconnect(
2216 &connection_state,
2217 &reconnect_published,
2218 state_sink.as_ref(),
2219 &auth_tracker,
2220 &controller_notify,
2221 || {},
2222 ) == ReconnectRequestOutcome::Accepted
2223 {
2224 log::warn!("Writer triggering reconnect");
2225 }
2226 }
2227 }
2228 WriterCommand::Heartbeat(msg) => {
2229 let send_failed =
2230 Self::write_outbound(&mut active_writer, msg).await;
2231
2232 if send_failed
2233 && request_websocket_reconnect(
2234 &connection_state,
2235 &reconnect_published,
2236 state_sink.as_ref(),
2237 &auth_tracker,
2238 &controller_notify,
2239 || {},
2240 ) == ReconnectRequestOutcome::Accepted
2241 {
2242 log::warn!("Writer triggering reconnect");
2243 }
2244 }
2245 }
2246 }
2247 Ok(None) => {
2248 log::debug!("Writer channel closed, terminating writer task");
2250 break;
2251 }
2252 Err(_) => {
2253 }
2255 }
2256 }
2257
2258 _ = dst::time::timeout(
2261 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2262 active_writer.close(),
2263 )
2264 .await;
2265
2266 log_task_stopped("write");
2267 })
2268 }
2269
2270 async fn write_outbound(writer: &mut MessageWriter, msg: Message) -> bool {
2271 let send_result =
2272 dst::time::timeout(Duration::from_secs(WRITE_TIMEOUT_SECS), writer.send(msg)).await;
2273
2274 match send_result {
2275 Ok(Ok(())) => false,
2276 Ok(Err(e)) => {
2277 if is_connection_drop_transport_error(&e) {
2278 log::warn!("Failed to send message: {e}");
2279 } else {
2280 log::error!("Failed to send message: {e}");
2281 }
2282
2283 true
2284 }
2285 Err(_) => {
2286 log::warn!("Timed out sending message after {WRITE_TIMEOUT_SECS}s");
2287 true
2288 }
2289 }
2290 }
2291
2292 fn spawn_heartbeat_task(
2293 connection_state: Arc<AtomicU8>,
2294 heartbeat_secs: u64,
2295 message: Option<String>,
2296 writer_tx: WriterSender<WriterCommand>,
2297 ) -> dst::task::JoinHandle<()> {
2298 log_task_started("heartbeat");
2299
2300 dst::task::spawn(async move {
2301 let interval = Duration::from_secs(heartbeat_secs);
2302 loop {
2303 dst::time::sleep(interval).await;
2304
2305 match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
2306 ConnectionMode::Active => {
2307 let msg = match &message {
2308 Some(text) => {
2309 WriterCommand::Heartbeat(Message::Text(text.clone().into()))
2310 }
2311 None => WriterCommand::Heartbeat(Message::Ping(vec![].into())),
2312 };
2313
2314 match writer_tx.send_control(msg) {
2315 Ok(()) => log::trace!("Sent heartbeat to writer task"),
2316 Err(e) => {
2317 log::warn!("Failed to send heartbeat to writer task: {e}");
2318 }
2319 }
2320 }
2321 ConnectionMode::Reconnect => {}
2322 ConnectionMode::Disconnect | ConnectionMode::Closed => break,
2323 }
2324 }
2325
2326 log_task_stopped("heartbeat");
2327 })
2328 }
2329}
2330
2331fn heartbeat_timeout_exceeded(
2332 last_frame_time: &mut dst::time::Instant,
2333 timeout: Option<Duration>,
2334 frame_activity: Option<&AtomicBool>,
2335) -> bool {
2336 if frame_activity.is_some_and(|activity| activity.swap(false, Ordering::Relaxed)) {
2337 *last_frame_time = dst::time::Instant::now();
2338 }
2339
2340 if let Some(timeout) = timeout {
2341 let elapsed = last_frame_time.elapsed();
2342 if elapsed >= timeout {
2343 log::warn!(
2344 "Heartbeat timeout: no frame received for {:.1}s",
2345 elapsed.as_secs_f64()
2346 );
2347 return true;
2348 }
2349 }
2350
2351 false
2352}
2353
2354fn idle_timeout_exceeded(
2355 last_data_time: dst::time::Instant,
2356 idle_timeout: Option<Duration>,
2357) -> bool {
2358 if let Some(timeout) = idle_timeout {
2359 let idle_duration = last_data_time.elapsed();
2360 if idle_duration >= timeout {
2361 log::warn!(
2362 "Read idle timeout: no data received for {:.1}s",
2363 idle_duration.as_secs_f64()
2364 );
2365 return true;
2366 }
2367 }
2368
2369 false
2370}
2371
2372impl Drop for WebSocketClientInner {
2373 fn drop(&mut self) {
2374 if let Some(read_fence) = self.read_fence.take() {
2375 read_fence.invalidate();
2376 }
2377
2378 if let Some(ref read_task) = self.read_task.take()
2379 && !read_task.is_finished()
2380 {
2381 read_task.abort();
2382 log_task_aborted("read");
2383 }
2384
2385 if !self.write_task.is_finished() {
2386 self.write_task.abort();
2387 log_task_aborted("write");
2388 }
2389
2390 if let Some(ref handle) = self.heartbeat_task.take()
2391 && !handle.is_finished()
2392 {
2393 handle.abort();
2394 log_task_aborted("heartbeat");
2395 }
2396 }
2397}
2398
2399#[expect(
2400 clippy::missing_fields_in_debug,
2401 reason = "handler closures and internal task handles are intentionally omitted"
2402)]
2403impl Debug for WebSocketClientInner {
2404 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2405 f.debug_struct(stringify!(WebSocketClientInner))
2406 .field("config", &self.config)
2407 .field(
2408 "connection_mode",
2409 &ConnectionMode::from_atomic(&self.connection_mode),
2410 )
2411 .field("connect_timeout", &self.connect_timeout)
2412 .field("is_stream_mode", &self.handler.is_none())
2413 .finish()
2414 }
2415}
2416
2417#[derive(Clone)]
2418enum IncomingHandler {
2419 Message(MessageHandler),
2420 Epoch(EpochMessageHandler),
2421}
2422
2423impl IncomingHandler {
2424 fn handle(&self, connection_epoch: u64, message: Message) {
2425 match self {
2426 Self::Message(handler) => handler(message),
2427 Self::Epoch(handler) => handler(connection_epoch, message),
2428 }
2429 }
2430}
2431
2432#[derive(Clone)]
2433enum IncomingPingHandler {
2434 Ping(PingHandler),
2435 Epoch(EpochPingHandler),
2436}
2437
2438impl IncomingPingHandler {
2439 fn handle(&self, connection_epoch: u64, data: Vec<u8>) {
2440 match self {
2441 Self::Ping(handler) => handler(data),
2442 Self::Epoch(handler) => handler(connection_epoch, data),
2443 }
2444 }
2445}
2446
2447#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2448enum ReconnectBufferAction {
2449 Drain,
2450 Wait,
2451 Discard,
2452}
2453
2454pub struct WebSocketClient {
2461 pub(crate) controller_task: dst::task::JoinHandle<()>,
2462 pub(crate) connection_mode: Arc<AtomicU8>,
2463 pub(crate) connection_epoch: Arc<AtomicU64>,
2464 pub(crate) state_notify: Arc<tokio::sync::Notify>,
2465 pub(crate) connect_timeout: Duration,
2466 pub(crate) rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
2467 pub(crate) writer_tx: WriterSender<WriterCommand>,
2468 auth_tracker: Arc<OnceLock<AuthTracker>>,
2469 reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
2470 reconnect_headers: ReconnectHeaders,
2471 state_sink: Option<SocketStateSink>,
2472 controller_lifecycle: Arc<ControllerLifecycle>,
2473 controller_notify: Arc<tokio::sync::Notify>,
2474 reconnect_published: Arc<AtomicBool>,
2475 reconnect_supported: bool,
2476}
2477
2478#[derive(Clone)]
2482pub struct ReconnectHeaders {
2483 inner: Arc<RwLock<Vec<(String, String)>>>,
2484}
2485
2486impl ReconnectHeaders {
2487 fn new(headers: Vec<(String, String)>) -> Self {
2488 Self {
2489 inner: Arc::new(RwLock::new(headers)),
2490 }
2491 }
2492
2493 pub fn update(&self, name: &str, value: &str) -> Result<(), TransportError> {
2499 let name = HeaderName::from_bytes(name.as_bytes()).map_err(|e| {
2500 TransportError::Io(std::io::Error::new(
2501 std::io::ErrorKind::InvalidInput,
2502 format!("Invalid WebSocket reconnect header name: {e}"),
2503 ))
2504 })?;
2505
2506 HeaderValue::from_str(value).map_err(|e| {
2507 TransportError::Io(std::io::Error::new(
2508 std::io::ErrorKind::InvalidInput,
2509 format!("Invalid WebSocket reconnect header value: {e}"),
2510 ))
2511 })?;
2512
2513 let name = name.as_str();
2514 let mut headers = self.inner.write();
2515 headers.retain(|(existing, _)| !existing.eq_ignore_ascii_case(name));
2516 headers.push((name.to_string(), value.to_string()));
2517 Ok(())
2518 }
2519
2520 fn snapshot(&self) -> Vec<(String, String)> {
2521 self.inner.read().clone()
2522 }
2523}
2524
2525impl Debug for ReconnectHeaders {
2526 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2527 f.debug_struct(stringify!(ReconnectHeaders))
2528 .finish_non_exhaustive()
2529 }
2530}
2531
2532#[derive(Clone)]
2534pub struct WebSocketReconnectHandle {
2535 connection_mode: Arc<AtomicU8>,
2536 auth_tracker: Arc<OnceLock<AuthTracker>>,
2537 state_sink: Option<SocketStateSink>,
2538 controller_lifecycle: Arc<ControllerLifecycle>,
2539 controller_notify: Arc<tokio::sync::Notify>,
2540 reconnect_published: Arc<AtomicBool>,
2541 supported: bool,
2542}
2543
2544impl Debug for WebSocketReconnectHandle {
2545 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2546 f.debug_struct(stringify!(WebSocketReconnectHandle))
2547 .field(
2548 "connection_mode",
2549 &ConnectionMode::from_atomic(&self.connection_mode),
2550 )
2551 .field("supported", &self.supported)
2552 .finish_non_exhaustive()
2553 }
2554}
2555
2556impl WebSocketReconnectHandle {
2557 #[must_use]
2565 pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
2566 if !self.supported {
2567 return ReconnectRequestOutcome::Unsupported;
2568 }
2569
2570 let Some(request) = self.controller_lifecycle.enter_request() else {
2571 return ReconnectRequestOutcome::Closed;
2572 };
2573
2574 let mut request = Some(request);
2575 request_websocket_reconnect(
2576 &self.connection_mode,
2577 &self.reconnect_published,
2578 self.state_sink.as_ref(),
2579 &self.auth_tracker,
2580 &self.controller_notify,
2581 || drop(request.take()),
2582 )
2583 }
2584}
2585
2586struct ReconnectPublication<'a> {
2587 published: &'a AtomicBool,
2588 controller_notify: &'a tokio::sync::Notify,
2589}
2590
2591impl Drop for ReconnectPublication<'_> {
2592 fn drop(&mut self) {
2593 self.published.store(true, Ordering::SeqCst);
2594 self.controller_notify.notify_one();
2595 }
2596}
2597
2598fn request_websocket_reconnect<F>(
2599 connection_mode: &AtomicU8,
2600 reconnect_published: &AtomicBool,
2601 state_sink: Option<&SocketStateSink>,
2602 auth_tracker: &OnceLock<AuthTracker>,
2603 controller_notify: &tokio::sync::Notify,
2604 on_handoff: F,
2605) -> ReconnectRequestOutcome
2606where
2607 F: FnOnce(),
2608{
2609 if reconnect_published
2610 .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst)
2611 .is_err()
2612 {
2613 return match ConnectionMode::from_atomic(connection_mode) {
2614 ConnectionMode::Active | ConnectionMode::Reconnect => {
2615 ReconnectRequestOutcome::AlreadyReconnecting
2616 }
2617 ConnectionMode::Disconnect => ReconnectRequestOutcome::Disconnected,
2618 ConnectionMode::Closed => ReconnectRequestOutcome::Closed,
2619 };
2620 }
2621
2622 let outcome = ConnectionMode::request_reconnect_outcome(connection_mode);
2623 if outcome != ReconnectRequestOutcome::Accepted {
2624 reconnect_published.store(true, Ordering::SeqCst);
2625 return outcome;
2626 }
2627
2628 let _publication = ReconnectPublication {
2629 published: reconnect_published,
2630 controller_notify,
2631 };
2632
2633 if let Some(tracker) = auth_tracker.get() {
2634 tracker.invalidate();
2635 }
2636
2637 controller_notify.notify_one();
2638 on_handoff();
2639
2640 if let Some(sink) = state_sink {
2641 sink.publish_websocket(SocketState::Disconnected);
2642 }
2643
2644 ReconnectRequestOutcome::Accepted
2645}
2646
2647#[cfg(test)]
2648mod reconnect_request_tests {
2649 use std::sync::{Arc, OnceLock, atomic::AtomicU8};
2650
2651 use rstest::rstest;
2652
2653 use super::*;
2654
2655 fn handle(
2656 mode: ConnectionMode,
2657 supported: bool,
2658 ) -> (
2659 WebSocketReconnectHandle,
2660 AuthTracker,
2661 Arc<tokio::sync::Notify>,
2662 ) {
2663 let tracker = AuthTracker::new();
2664 let _receiver = tracker.begin();
2665 tracker.succeed();
2666 let auth_tracker = Arc::new(OnceLock::new());
2667 auth_tracker
2668 .set(tracker.clone())
2669 .expect("auth tracker should be unset");
2670 let notify = Arc::new(tokio::sync::Notify::new());
2671
2672 let handle = WebSocketReconnectHandle {
2673 connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
2674 auth_tracker,
2675 state_sink: None,
2676 controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2677 controller_notify: Arc::clone(¬ify),
2678 reconnect_published: Arc::new(AtomicBool::new(true)),
2679 supported,
2680 };
2681
2682 (handle, tracker, notify)
2683 }
2684
2685 #[rstest]
2686 #[tokio::test]
2687 async fn accepted_request_invalidates_auth_and_wakes_controller_once() {
2688 let (handle, tracker, notify) = handle(ConnectionMode::Active, true);
2689
2690 assert_eq!(
2691 handle.request_reconnect(),
2692 ReconnectRequestOutcome::Accepted
2693 );
2694 assert!(!tracker.is_authenticated());
2695 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2696 .await
2697 .expect("accepted request should notify controller");
2698
2699 let _receiver = tracker.begin();
2700 tracker.succeed();
2701 assert_eq!(
2702 handle.request_reconnect(),
2703 ReconnectRequestOutcome::AlreadyReconnecting
2704 );
2705 assert!(tracker.is_authenticated());
2706 assert!(
2707 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2708 .await
2709 .is_err(),
2710 "duplicate request should not notify controller",
2711 );
2712
2713 handle
2714 .connection_mode
2715 .store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
2716 assert_eq!(
2717 handle.request_reconnect(),
2718 ReconnectRequestOutcome::Accepted
2719 );
2720 assert!(!tracker.is_authenticated());
2721 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2722 .await
2723 .expect("a later reconnect cycle should notify the controller");
2724 }
2725
2726 #[rstest]
2727 #[case(
2728 ConnectionMode::Disconnect,
2729 true,
2730 ReconnectRequestOutcome::Disconnected
2731 )]
2732 #[case(
2733 ConnectionMode::Reconnect,
2734 true,
2735 ReconnectRequestOutcome::AlreadyReconnecting
2736 )]
2737 #[case(ConnectionMode::Closed, true, ReconnectRequestOutcome::Closed)]
2738 #[case(ConnectionMode::Active, false, ReconnectRequestOutcome::Unsupported)]
2739 #[tokio::test]
2740 async fn rejected_request_preserves_auth_and_does_not_wake_controller(
2741 #[case] mode: ConnectionMode,
2742 #[case] supported: bool,
2743 #[case] expected: ReconnectRequestOutcome,
2744 ) {
2745 let (handle, tracker, notify) = handle(mode, supported);
2746
2747 assert_eq!(handle.request_reconnect(), expected);
2748 assert!(tracker.is_authenticated());
2749 assert!(
2750 tokio::time::timeout(Duration::from_millis(10), notify.notified())
2751 .await
2752 .is_err(),
2753 "rejected request should not notify controller",
2754 );
2755 }
2756
2757 #[rstest]
2758 #[tokio::test]
2759 async fn reconnect_loss_callback_rejects_nested_request() {
2760 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2761 let controller_notify = Arc::new(tokio::sync::Notify::new());
2762 let handle_slot = Arc::new(OnceLock::<WebSocketReconnectHandle>::new());
2763 let handle_slot_callback = Arc::clone(&handle_slot);
2764 let nested_outcomes = Arc::new(parking_lot::Mutex::new(Vec::new()));
2765 let nested_outcomes_callback = Arc::clone(&nested_outcomes);
2766 let states = Arc::new(parking_lot::Mutex::new(Vec::new()));
2767 let states_callback = Arc::clone(&states);
2768 let sink = SocketStateSink::new(move |state| {
2769 states_callback.lock().push(state);
2770 nested_outcomes_callback
2771 .lock()
2772 .push(handle_slot_callback.get().unwrap().request_reconnect());
2773 });
2774
2775 let handle = WebSocketReconnectHandle {
2776 connection_mode,
2777 auth_tracker: Arc::new(OnceLock::new()),
2778 state_sink: Some(sink),
2779 controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2780 controller_notify,
2781 reconnect_published: Arc::new(AtomicBool::new(true)),
2782 supported: true,
2783 };
2784
2785 handle_slot.set(handle.clone()).unwrap();
2786 let (result_tx, result_rx) = tokio::sync::oneshot::channel();
2787 std::thread::spawn(move || {
2788 _ = result_tx.send(handle.request_reconnect());
2789 });
2790
2791 assert_eq!(
2792 tokio::time::timeout(Duration::from_secs(1), result_rx)
2793 .await
2794 .expect("reentrant reconnect callback deadlocked")
2795 .unwrap(),
2796 ReconnectRequestOutcome::Accepted
2797 );
2798 assert_eq!(
2799 *nested_outcomes.lock(),
2800 vec![ReconnectRequestOutcome::AlreadyReconnecting]
2801 );
2802 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2803 }
2804
2805 #[rstest]
2806 fn closed_stream_handle_remains_unsupported() {
2807 let (handle, tracker, _notify) = handle(ConnectionMode::Closed, false);
2808 handle.controller_lifecycle.close_and_abort();
2809
2810 assert_eq!(
2811 handle.request_reconnect(),
2812 ReconnectRequestOutcome::Unsupported
2813 );
2814 assert!(tracker.is_authenticated());
2815 }
2816}
2817
2818impl Debug for WebSocketClient {
2819 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2820 f.debug_struct(stringify!(WebSocketClient)).finish()
2821 }
2822}
2823
2824#[bon::bon]
2825impl WebSocketClient {
2826 #[builder(
2845 builder_type = WebSocketClientStreamBuilder,
2846 finish_fn = connect
2847 )]
2848 pub async fn stream_builder(
2849 config: WebSocketConfig,
2850 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2851 default_quota: Option<Quota>,
2852 state_sink: Option<SocketStateSink>,
2853 ) -> Result<(MessageReader, Self), TransportError> {
2854 install_cryptographic_provider();
2855 validate_honored_inbound_limits(&config)?;
2856
2857 let connect_timeout = Duration::from_secs(10);
2860
2861 let (writer, reader) = dst::time::timeout(
2862 connect_timeout,
2863 Box::pin(WebSocketClientInner::connect_with_server(
2864 &config.url,
2865 config.headers.clone(),
2866 config.backend,
2867 config.proxy_url.as_deref(),
2868 config.max_message_size_bytes,
2869 config.max_frame_size_bytes,
2870 )),
2871 )
2872 .await
2873 .map_err(|_| {
2874 TransportError::Io(std::io::Error::new(
2875 std::io::ErrorKind::TimedOut,
2876 format!(
2877 "connection timed out after {}s",
2878 connect_timeout.as_secs_f64()
2879 ),
2880 ))
2881 })??;
2882
2883 let inner =
2885 WebSocketClientInner::new_with_writer_and_state_sink(config, writer, state_sink)?;
2886
2887 let connection_mode = Arc::clone(&inner.connection_mode);
2888 let connection_epoch = Arc::clone(&inner.connection_epoch);
2889 let state_notify = Arc::clone(&inner.state_notify);
2890 let controller_notify = Arc::clone(&inner.controller_notify);
2891 let reconnect_published = Arc::clone(&inner.reconnect_published);
2892 let connect_timeout = inner.connect_timeout;
2893 let auth_tracker = Arc::clone(&inner.auth_tracker);
2894 let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
2895 let reconnect_headers = inner.reconnect_headers.clone();
2896 let state_sink = inner.state_sink.clone();
2897 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, None)?;
2898 let writer_tx = inner.writer_tx.clone();
2899 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2900
2901 let controller_task = Self::spawn_controller_task(
2902 inner,
2903 Arc::clone(&connection_mode),
2904 Arc::clone(&state_notify),
2905 Arc::clone(&auth_tracker),
2906 Arc::clone(&controller_lifecycle),
2907 Arc::clone(&controller_notify),
2908 Arc::clone(&reconnect_published),
2909 );
2910 let abort_handle = controller_task.abort_handle();
2911 controller_lifecycle.set_abort(move || abort_handle.abort());
2912
2913 Ok((
2914 reader,
2915 Self {
2916 controller_task,
2917 connection_mode,
2918 connection_epoch,
2919 state_notify,
2920 connect_timeout,
2921 rate_limiter,
2922 writer_tx,
2923 auth_tracker,
2924 reconnect_buffer_waits_for_auth,
2925 reconnect_headers,
2926 state_sink,
2927 controller_lifecycle,
2928 controller_notify,
2929 reconnect_published,
2930 reconnect_supported: false,
2931 },
2932 ))
2933 }
2934
2935 #[builder(finish_fn = connect)]
2979 pub async fn builder(
2980 config: WebSocketConfig,
2981 message_handler: MessageHandler,
2982 ping_handler: Option<PingHandler>,
2983 close_handler: Option<CloseHandler>,
2984 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2985 default_quota: Option<Quota>,
2986 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
2987 state_sink: Option<SocketStateSink>,
2988 connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
2989 #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
2990 initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
2991 cancellation_token: Option<CancellationToken>,
2992 ) -> Result<Self, TransportError> {
2993 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
2994 let connection_rate_limit =
2995 Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
2996 Self::connect_with_handler_scoped(
2997 config,
2998 IncomingHandler::Message(message_handler),
2999 ping_handler.map(IncomingPingHandler::Ping),
3000 close_handler,
3001 rate_limiter,
3002 state_sink,
3003 connection_rate_limit,
3004 InitialConnectOptions {
3005 retry_policy: initial_connect_retry_policy,
3006 cancellation_token,
3007 },
3008 )
3009 .await
3010 }
3011
3012 #[builder(
3039 builder_type = WebSocketClientEpochBuilder,
3040 finish_fn = connect
3041 )]
3042 pub async fn epoch_builder(
3043 config: WebSocketConfig,
3044 epoch_handler: EpochMessageHandler,
3045 ping_handler: Option<PingHandler>,
3046 epoch_ping_handler: Option<EpochPingHandler>,
3047 close_handler: Option<CloseHandler>,
3048 #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3049 default_quota: Option<Quota>,
3050 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3051 state_sink: Option<SocketStateSink>,
3052 connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3053 #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3054 initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3055 cancellation_token: Option<CancellationToken>,
3056 ) -> Result<Self, TransportError> {
3057 let ping_handler = match (ping_handler, epoch_ping_handler) {
3058 (Some(_), Some(_)) => {
3059 return Err(TransportError::Io(std::io::Error::new(
3060 std::io::ErrorKind::InvalidInput,
3061 "Cannot configure both ping_handler and epoch_ping_handler",
3062 )));
3063 }
3064 (Some(handler), None) => Some(IncomingPingHandler::Ping(handler)),
3065 (None, Some(handler)) => Some(IncomingPingHandler::Epoch(handler)),
3066 (None, None) => None,
3067 };
3068
3069 let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3070 let connection_rate_limit =
3071 Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3072 Self::connect_with_handler_scoped(
3073 config,
3074 IncomingHandler::Epoch(epoch_handler),
3075 ping_handler,
3076 close_handler,
3077 rate_limiter,
3078 state_sink,
3079 connection_rate_limit,
3080 InitialConnectOptions {
3081 retry_policy: initial_connect_retry_policy,
3082 cancellation_token,
3083 },
3084 )
3085 .await
3086 }
3087
3088 fn resolve_connection_rate_limit(
3089 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3090 keys: Arc<[Ustr]>,
3091 ) -> Result<Option<ConnectionRateLimit>, TransportError> {
3092 if rate_limiter.is_none() && !keys.is_empty() {
3093 return Err(TransportError::Io(std::io::Error::new(
3094 std::io::ErrorKind::InvalidInput,
3095 "Connection rate keys require a connection rate limiter",
3096 )));
3097 }
3098
3099 if rate_limiter.is_some() && keys.is_empty() {
3100 return Err(TransportError::Io(std::io::Error::new(
3101 std::io::ErrorKind::InvalidInput,
3102 "Connection rate limiter requires at least one connection rate key",
3103 )));
3104 }
3105
3106 Ok(rate_limiter.map(|limiter| ConnectionRateLimit { limiter, keys }))
3107 }
3108
3109 fn resolve_rate_limiter(
3110 default_quota: Option<Quota>,
3111 keyed_quotas: Vec<(String, Quota)>,
3112 rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3113 ) -> Result<Arc<RateLimiter<Ustr, MonotonicClock>>, TransportError> {
3114 if let Some(rate_limiter) = rate_limiter {
3115 if default_quota.is_some() || !keyed_quotas.is_empty() {
3116 return Err(TransportError::Io(std::io::Error::new(
3117 std::io::ErrorKind::InvalidInput,
3118 "Cannot combine a shared rate limiter with quota configuration",
3119 )));
3120 }
3121
3122 return Ok(rate_limiter);
3123 }
3124
3125 let keyed_quotas = keyed_quotas
3126 .into_iter()
3127 .map(|(key, quota)| (Ustr::from(&key), quota))
3128 .collect();
3129 Ok(Arc::new(RateLimiter::new_with_quota(
3130 default_quota,
3131 keyed_quotas,
3132 )))
3133 }
3134
3135 #[expect(
3136 clippy::too_many_arguments,
3137 reason = "handler delivery and connection controls are independent options"
3138 )]
3139 async fn connect_with_handler_scoped(
3140 config: WebSocketConfig,
3141 handler: IncomingHandler,
3142 ping_handler: Option<IncomingPingHandler>,
3143 close_handler: Option<CloseHandler>,
3144 rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
3145 state_sink: Option<SocketStateSink>,
3146 connection_rate_limit: Option<ConnectionRateLimit>,
3147 initial_connect_options: InitialConnectOptions,
3148 ) -> Result<Self, TransportError> {
3149 log::debug!("Connecting");
3150 let inner = WebSocketClientInner::connect_url_with_handler(
3151 config,
3152 Some(handler),
3153 ping_handler,
3154 close_handler,
3155 state_sink,
3156 connection_rate_limit,
3157 initial_connect_options,
3158 )
3159 .await?;
3160 let connection_mode = Arc::clone(&inner.connection_mode);
3161 let connection_epoch = Arc::clone(&inner.connection_epoch);
3162 let state_notify = Arc::clone(&inner.state_notify);
3163 let controller_notify = Arc::clone(&inner.controller_notify);
3164 let reconnect_published = Arc::clone(&inner.reconnect_published);
3165 let writer_tx = inner.writer_tx.clone();
3166 let connect_timeout = inner.connect_timeout;
3167 let auth_tracker = Arc::clone(&inner.auth_tracker);
3168 let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
3169 let reconnect_headers = inner.reconnect_headers.clone();
3170 let state_sink = inner.state_sink.clone();
3171 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
3172
3173 let controller_task = Self::spawn_controller_task(
3174 inner,
3175 Arc::clone(&connection_mode),
3176 Arc::clone(&state_notify),
3177 Arc::clone(&auth_tracker),
3178 Arc::clone(&controller_lifecycle),
3179 Arc::clone(&controller_notify),
3180 Arc::clone(&reconnect_published),
3181 );
3182 let abort_handle = controller_task.abort_handle();
3183 controller_lifecycle.set_abort(move || abort_handle.abort());
3184
3185 Ok(Self {
3186 controller_task,
3187 connection_mode,
3188 connection_epoch,
3189 state_notify,
3190 connect_timeout,
3191 rate_limiter,
3192 writer_tx,
3193 auth_tracker,
3194 reconnect_buffer_waits_for_auth,
3195 reconnect_headers,
3196 state_sink,
3197 controller_lifecycle,
3198 controller_notify,
3199 reconnect_published,
3200 reconnect_supported: true,
3201 })
3202 }
3203
3204 #[must_use]
3206 pub fn reconnect_headers(&self) -> ReconnectHeaders {
3207 self.reconnect_headers.clone()
3208 }
3209
3210 #[must_use]
3212 pub fn reconnect_handle(&self) -> WebSocketReconnectHandle {
3213 WebSocketReconnectHandle {
3214 connection_mode: Arc::clone(&self.connection_mode),
3215 auth_tracker: Arc::clone(&self.auth_tracker),
3216 state_sink: self.state_sink.clone(),
3217 controller_lifecycle: Arc::clone(&self.controller_lifecycle),
3218 controller_notify: Arc::clone(&self.controller_notify),
3219 reconnect_published: Arc::clone(&self.reconnect_published),
3220 supported: self.reconnect_supported,
3221 }
3222 }
3223
3224 #[must_use]
3229 pub fn request_reconnect(&self) -> bool {
3230 self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
3231 }
3232
3233 #[must_use]
3235 pub fn connection_mode(&self) -> ConnectionMode {
3236 ConnectionMode::from_atomic(&self.connection_mode)
3237 }
3238
3239 #[must_use]
3244 pub fn connection_epoch(&self) -> u64 {
3245 self.connection_epoch.load(Ordering::Acquire)
3246 }
3247
3248 #[must_use]
3253 pub fn connection_mode_atomic(&self) -> Arc<AtomicU8> {
3254 Arc::clone(&self.connection_mode)
3255 }
3256
3257 #[must_use]
3262 pub fn connection_epoch_atomic(&self) -> Arc<AtomicU64> {
3263 Arc::clone(&self.connection_epoch)
3264 }
3265
3266 #[inline]
3271 #[must_use]
3272 pub fn is_active(&self) -> bool {
3273 self.connection_mode().is_active()
3274 }
3275
3276 #[must_use]
3278 pub fn is_disconnected(&self) -> bool {
3279 self.controller_task.is_finished()
3280 }
3281
3282 #[inline]
3287 #[must_use]
3288 pub fn is_reconnecting(&self) -> bool {
3289 self.connection_mode().is_reconnect()
3290 }
3291
3292 pub fn set_auth_tracker(&self, tracker: AuthTracker, reconnect_buffer_waits_for_auth: bool) {
3303 let _ = self.auth_tracker.set(tracker);
3304 self.reconnect_buffer_waits_for_auth
3305 .store(reconnect_buffer_waits_for_auth, Ordering::Release);
3306 }
3307
3308 #[inline]
3312 #[must_use]
3313 pub fn is_disconnecting(&self) -> bool {
3314 self.connection_mode().is_disconnect()
3315 }
3316
3317 #[inline]
3323 #[must_use]
3324 pub fn is_closed(&self) -> bool {
3325 self.connection_mode().is_closed()
3326 }
3327
3328 #[inline]
3332 fn check_not_terminal(&self) -> Result<(), SendError> {
3333 match self.connection_mode() {
3334 ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
3335 _ => Ok(()),
3336 }
3337 }
3338
3339 async fn await_rate_limit_or_closed(&self, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3341 const CHECK_INTERVAL_MS: u64 = 100;
3342
3343 tokio::select! {
3344 biased;
3345 () = self.rate_limiter.await_keys_ready(keys) => Ok(()),
3346 () = async {
3347 loop {
3348 let mut notified = pin!(self.state_notify.notified());
3350 notified.as_mut().enable();
3351
3352 if matches!(self.connection_mode(), ConnectionMode::Disconnect | ConnectionMode::Closed) {
3353 break;
3354 }
3355 tokio::select! {
3356 biased;
3357 () = notified => {}
3358 () = dst::time::sleep(Duration::from_millis(CHECK_INTERVAL_MS)) => {}
3359 }
3360 }
3361 } => Err(SendError::Closed),
3362 }
3363 }
3364
3365 async fn wait_for_active(&self) -> Result<(), SendError> {
3371 const FALLBACK_INTERVAL_MS: u64 = 100;
3372
3373 let mode = self.connection_mode();
3374 if mode.is_active() {
3375 return Ok(());
3376 }
3377
3378 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3379 return Err(SendError::Closed);
3380 }
3381
3382 log::debug!("Waiting for client to become ACTIVE before sending...");
3383
3384 let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
3385
3386 dst::time::timeout(self.connect_timeout, async {
3387 loop {
3388 let mut notified = pin!(self.state_notify.notified());
3390 notified.as_mut().enable();
3391
3392 let mode = self.connection_mode();
3393 if mode.is_active() {
3394 return Ok(());
3395 }
3396
3397 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3398 return Err(());
3399 }
3400
3401 tokio::select! {
3402 biased;
3403 () = notified => {}
3404 () = dst::time::sleep(fallback_interval) => {}
3405 }
3406 }
3407 })
3408 .await
3409 .map_err(|_| SendError::Timeout)?
3410 .map_err(|()| SendError::Closed)
3411 }
3412
3413 pub fn notify_closed(&self) {
3426 let mode = self.connection_mode();
3427 if mode.is_disconnect() || mode.is_closed() {
3428 return;
3429 }
3430
3431 log::debug!("Stream reader signalled EOF, transitioning to CLOSED");
3432
3433 if ConnectionMode::close_websocket_on_loss(&self.connection_mode, self.state_sink.as_ref())
3434 {
3435 fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3436 self.state_notify.notify_waiters();
3437 }
3438 }
3439
3440 pub async fn disconnect(&self) {
3444 log::debug!("Disconnecting");
3445
3446 if ConnectionMode::request_disconnect(&self.connection_mode)
3448 && let Some(tracker) = self.auth_tracker.get()
3449 {
3450 tracker.fail("WebSocket client disconnected");
3451 }
3452
3453 self.state_notify.notify_waiters();
3454
3455 if dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
3456 while !self.is_disconnected() {
3457 dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
3458 }
3459
3460 if !self.controller_task.is_finished() {
3461 self.controller_task.abort();
3462 log_task_aborted("controller");
3463 }
3464 })
3465 .await
3466 == Ok(())
3467 {
3468 log::debug!("Controller task finished");
3469 } else {
3470 log::warn!("Timeout waiting for controller task to finish");
3471
3472 if !self.controller_task.is_finished() {
3473 self.controller_task.abort();
3474 log_task_aborted("controller");
3475 }
3476
3477 self.connection_mode
3478 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3479 }
3480 }
3481
3482 #[allow(unused_variables)]
3492 pub async fn send_text(&self, data: String, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3493 self.check_not_terminal()?;
3494
3495 self.await_rate_limit_or_closed(keys).await?;
3496 self.wait_for_active().await?;
3497
3498 log::trace!("Sending text frame ({} bytes)", data.len());
3499
3500 let msg = Message::Text(data.into());
3501 self.writer_tx.send(WriterCommand::Send(msg))
3502 }
3503
3504 pub async fn send_text_on_connection(
3521 &self,
3522 data: String,
3523 keys: Option<&[Ustr]>,
3524 connection_epoch: u64,
3525 ) -> Result<(), SendError> {
3526 self.check_not_terminal()?;
3527 self.await_rate_limit_or_closed(keys).await?;
3528 self.wait_for_active().await?;
3529
3530 log::trace!(
3531 "Sending text frame once: epoch={connection_epoch} ({} bytes)",
3532 data.len()
3533 );
3534
3535 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3536 self.writer_tx
3537 .send_control(WriterCommand::SendOnConnection {
3538 message: Message::Text(data.into()),
3539 connection_epoch,
3540 response_tx,
3541 })?;
3542
3543 response_rx
3544 .await
3545 .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3546 }
3547
3548 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3560 #[expect(
3561 clippy::unused_async,
3562 clippy::unused_async_trait_impl,
3563 reason = "skipping instead of waiting removes the only await; the signature is public API shared with the other send methods"
3564 )]
3565 pub async fn send_pong(&self, data: Vec<u8>) -> Result<(), SendError> {
3566 validate_pong_payload(&data)?;
3567
3568 if !self.connection_mode().is_active() {
3569 log::debug!("Skipping pong: connection not active");
3570 return Ok(());
3571 }
3572
3573 log::trace!("Sending pong frame ({} bytes)", data.len());
3574
3575 self.writer_tx
3576 .send_control(WriterCommand::Send(Message::Pong(data.into())))
3577 }
3578
3579 #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3589 #[expect(
3590 clippy::unused_async,
3591 clippy::unused_async_trait_impl,
3592 reason = "the public send API is async even though this method only enqueues"
3593 )]
3594 pub async fn send_pong_on_connection(
3595 &self,
3596 data: Vec<u8>,
3597 connection_epoch: u64,
3598 ) -> Result<(), SendError> {
3599 validate_pong_payload(&data)?;
3600
3601 if !self.connection_mode().is_active() {
3602 log::debug!("Skipping pong: connection not active");
3603 return Ok(());
3604 }
3605
3606 log::trace!(
3607 "Sending pong frame once: epoch={connection_epoch} ({} bytes)",
3608 data.len()
3609 );
3610
3611 self.writer_tx
3612 .send_control(WriterCommand::SendPongOnConnection {
3613 data,
3614 connection_epoch,
3615 })
3616 }
3617
3618 #[allow(unused_variables)]
3628 pub async fn send_bytes(&self, data: Vec<u8>, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3629 self.check_not_terminal()?;
3630
3631 self.await_rate_limit_or_closed(keys).await?;
3632 self.wait_for_active().await?;
3633
3634 log::trace!("Sending binary frame ({} bytes)", data.len());
3635
3636 let msg = Message::Binary(data.into());
3637 self.writer_tx.send(WriterCommand::Send(msg))
3638 }
3639
3640 pub async fn send_bytes_on_connection(
3657 &self,
3658 data: Vec<u8>,
3659 keys: Option<&[Ustr]>,
3660 connection_epoch: u64,
3661 ) -> Result<(), SendError> {
3662 self.check_not_terminal()?;
3663 self.await_rate_limit_or_closed(keys).await?;
3664 self.wait_for_active().await?;
3665
3666 log::trace!(
3667 "Sending binary frame once: epoch={connection_epoch} ({} bytes)",
3668 data.len()
3669 );
3670
3671 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3672 self.writer_tx
3673 .send_control(WriterCommand::SendOnConnection {
3674 message: Message::Binary(data.into()),
3675 connection_epoch,
3676 response_tx,
3677 })?;
3678
3679 response_rx
3680 .await
3681 .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3682 }
3683
3684 pub async fn send_close_message(&self) -> Result<(), SendError> {
3690 self.wait_for_active().await?;
3691
3692 let msg = Message::Close(None);
3693 self.writer_tx.send_control(WriterCommand::Send(msg))
3694 }
3695
3696 fn spawn_controller_task(
3697 mut inner: WebSocketClientInner,
3698 connection_mode: Arc<AtomicU8>,
3699 state_notify: Arc<tokio::sync::Notify>,
3700 auth_tracker: Arc<OnceLock<AuthTracker>>,
3701 controller_lifecycle: Arc<ControllerLifecycle>,
3702 controller_notify: Arc<tokio::sync::Notify>,
3703 reconnect_published: Arc<AtomicBool>,
3704 ) -> dst::task::JoinHandle<()> {
3705 dst::task::spawn(async move {
3706 let _activity = controller_lifecycle.activity();
3707 log_task_started("controller");
3708
3709 let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
3710 let mut reconnected_at = None;
3711
3712 loop {
3713 tokio::select! {
3714 biased;
3715 () = controller_notify.notified() => {}
3716 () = state_notify.notified() => {}
3717 () = dst::time::sleep(fallback_interval) => {}
3718 }
3719
3720 let mut mode = ConnectionMode::from_atomic(&connection_mode);
3721
3722 if mode.is_disconnect() {
3723 log::debug!("Disconnecting");
3724
3725 let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
3726 if dst::time::timeout(timeout, async {
3727 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
3729
3730 if let Some(read_fence) = inner.read_fence.take() {
3731 read_fence.invalidate();
3732 }
3733
3734 if let Some(task) = &inner.read_task
3735 && !task.is_finished()
3736 {
3737 task.abort();
3738 log_task_aborted("read");
3739 }
3740
3741 if let Some(task) = &inner.heartbeat_task
3742 && !task.is_finished()
3743 {
3744 task.abort();
3745 log_task_aborted("heartbeat");
3746 }
3747 })
3748 .await
3749 .is_err()
3750 {
3751 log::warn!("Shutdown timed out after {}s", timeout.as_secs());
3752 }
3753
3754 log::debug!("Closed");
3755 break; }
3757
3758 if mode.is_closed() {
3759 log::debug!("Connection closed");
3760 fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3761 state_notify.notify_waiters();
3762 break;
3763 }
3764
3765 if mode.is_active() && !inner.is_alive() {
3766 let target = if inner.handler.is_none() {
3767 ConnectionMode::Closed
3768 } else {
3769 ConnectionMode::Reconnect
3770 };
3771
3772 let transitioned = if target.is_closed() {
3773 ConnectionMode::close_websocket_on_loss(
3774 &connection_mode,
3775 inner.state_sink.as_ref(),
3776 )
3777 } else {
3778 request_websocket_reconnect(
3779 &connection_mode,
3780 &reconnect_published,
3781 inner.state_sink.as_ref(),
3782 &auth_tracker,
3783 &controller_notify,
3784 || {},
3785 ) == ReconnectRequestOutcome::Accepted
3786 };
3787
3788 if transitioned {
3789 if target.is_closed() {
3790 fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3791 }
3792
3793 log::info!("Detected dead connection, transitioning to {target:?}");
3794 }
3795
3796 mode = ConnectionMode::from_atomic(&connection_mode);
3797 }
3798
3799 if mode.is_reconnect() {
3800 if let Some(tracker) = auth_tracker.get() {
3801 tracker.invalidate();
3802 }
3803
3804 let reconnect_uptime = reconnected_at
3805 .take()
3806 .map(|started: dst::time::Instant| started.elapsed());
3807 let previous_reconnect_stable = reconnect_uptime
3808 .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
3809
3810 if previous_reconnect_stable {
3811 inner.backoff.reset();
3812 inner.reconnection_attempt_count = 0;
3813 log::debug!(
3814 "WebSocket remained active for at least {}s, resetting reconnect cycle",
3815 RECONNECT_STABILITY_THRESHOLD.as_secs()
3816 );
3817 }
3818
3819 if let Some(max_attempts) = inner.reconnect_max_attempts
3820 && inner.reconnection_attempt_count >= max_attempts
3821 {
3822 log::error!(
3823 "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
3824 );
3825 connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3826 fail_registered_auth(
3827 auth_tracker.as_ref(),
3828 "WebSocket reconnect attempts exhausted",
3829 );
3830 state_notify.notify_waiters();
3831 break;
3832 }
3833
3834 let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
3835 {
3836 inner.backoff.next_duration()
3837 } else {
3838 Duration::ZERO
3839 };
3840
3841 let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
3842 if !duration.is_zero() {
3843 log::warn!("Backing off for {}s...", duration.as_secs_f64());
3844
3845 if !wait_reconnect_delay(
3846 duration,
3847 connection_mode.as_ref(),
3848 state_notify.as_ref(),
3849 )
3850 .await
3851 {
3852 log::debug!("Backoff interrupted by terminal state");
3853 continue;
3854 }
3855 }
3856
3857 inner.reconnection_attempt_count += 1;
3858 inner.reconnect_throttle.record_attempt();
3859 log::debug!(
3860 "Reconnection attempt {} of {}",
3861 inner.reconnection_attempt_count,
3862 inner
3863 .reconnect_max_attempts
3864 .map_or_else(|| "unlimited".to_string(), |m| m.to_string())
3865 );
3866
3867 let reconnect_result = tokio::select! {
3869 biased;
3870 result = inner.reconnect_with_outcome() => Some(result),
3871 () = async {
3872 loop {
3873 let mut notified = pin!(state_notify.notified());
3875 notified.as_mut().enable();
3876
3877 if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
3878 break;
3879 }
3880 notified.await;
3881 }
3882 } => None,
3883 };
3884
3885 match reconnect_result {
3886 None => {
3887 log::debug!("Reconnect interrupted by disconnect");
3888 }
3889 Some(Ok(ReconnectOutcome::Reconnected)) => {
3890 reconnected_at = Some(dst::time::Instant::now());
3891
3892 state_notify.notify_waiters();
3893
3894 if ConnectionMode::from_atomic(&connection_mode).is_active() {
3898 if let Some(ref handler) = inner.handler {
3899 let connection_epoch =
3900 inner.connection_epoch.load(Ordering::Acquire);
3901 let reconnected_msg =
3902 Message::Text(RECONNECTED.to_string().into());
3903 handler.handle(connection_epoch, reconnected_msg);
3904 match handler {
3905 IncomingHandler::Message(_) => {
3906 log::debug!("Sent reconnected message to handler");
3907 }
3908 IncomingHandler::Epoch(_) => {
3909 log::debug!(
3910 "Sent reconnected message to epoch handler: \
3911 epoch={connection_epoch}",
3912 );
3913 }
3914 }
3915 }
3916
3917 log::debug!("Reconnected successfully");
3918 } else {
3919 log::debug!("Skipping reconnect handlers due to disconnect state");
3920 }
3921 }
3922 Some(Ok(ReconnectOutcome::Aborted)) => {
3923 log::debug!("Reconnect aborted");
3924 }
3925 Some(Err(e)) => {
3926 let duration = inner.backoff.next_duration();
3927 log::warn!(
3928 "Reconnect attempt {} failed: {}",
3929 inner.reconnection_attempt_count,
3930 e.redacted_message()
3931 );
3932
3933 if !duration.is_zero() {
3934 log::warn!("Backing off for {}s...", duration.as_secs_f64());
3935
3936 if !wait_reconnect_delay(
3937 duration,
3938 connection_mode.as_ref(),
3939 state_notify.as_ref(),
3940 )
3941 .await
3942 {
3943 log::debug!("Backoff interrupted by terminal state");
3944 }
3945 }
3946 }
3947 }
3948 }
3949 }
3950
3951 inner
3952 .connection_mode
3953 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3954
3955 log_task_stopped("controller");
3956 })
3957 }
3958}
3959
3960fn fail_registered_auth(auth_tracker: &OnceLock<AuthTracker>, reason: &str) {
3961 if let Some(tracker) = auth_tracker.get() {
3962 tracker.fail(reason);
3963 }
3964}
3965
3966fn validate_pong_payload(data: &[u8]) -> Result<(), SendError> {
3967 if data.len() > MAX_CONTROL_FRAME_PAYLOAD_BYTES {
3968 return Err(SendError::InvalidInput(format!(
3969 "pong payload exceeds {MAX_CONTROL_FRAME_PAYLOAD_BYTES} bytes"
3970 )));
3971 }
3972
3973 Ok(())
3974}
3975
3976impl Drop for WebSocketClient {
3977 fn drop(&mut self) {
3978 self.connection_mode
3979 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3980 fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3981 self.state_notify.notify_waiters();
3982 self.controller_notify.notify_waiters();
3983 self.controller_lifecycle.close_and_abort();
3984 }
3985}
3986
3987#[cfg(test)]
3988#[cfg(not(feature = "turmoil"))]
3989#[cfg(not(all(feature = "simulation", madsim)))] #[cfg(target_os = "linux")] mod tests {
3992 use std::{
3993 collections::HashMap,
3994 num::NonZeroU32,
3995 sync::{
3996 Arc,
3997 atomic::{AtomicU8, AtomicUsize, Ordering},
3998 },
3999 time::Duration,
4000 };
4001
4002 use axum::{Router, routing::post};
4003 use futures_util::{SinkExt, StreamExt};
4004 use log::Level;
4005 use nautilus_common::testing::wait_until_async;
4006 use nautilus_core::string::secret::REDACTED;
4007 use parking_lot::Mutex;
4008 use rstest::rstest;
4009 use tokio::{
4010 io::{AsyncReadExt, AsyncWriteExt},
4011 net::TcpListener,
4012 sync::{mpsc, oneshot},
4013 task::{self, JoinHandle},
4014 };
4015 use tokio_tungstenite::{
4016 accept_async, accept_hdr_async,
4017 tungstenite::{
4018 Message as WsMessage,
4019 handshake::server::{self, Callback},
4020 http::HeaderValue,
4021 protocol::frame::{CloseFrame as WsCloseFrame, coding::CloseCode},
4022 },
4023 };
4024
4025 use crate::{
4026 RECONNECTED, SocketState, SocketStateSink,
4027 error::SendError,
4028 http::{HttpClient, Method},
4029 logging::tests::capture_logs_for,
4030 mode::{ConnectionMode, ReadSessionFence},
4031 ratelimiter::quota::Quota,
4032 transport::{CloseFrame, Message, TransportError},
4033 websocket::{
4034 AuthTracker, CloseHandler, InitialConnectRetryPolicy, ReconnectHeaders,
4035 TransportBackend, WebSocketClient, WebSocketClientInner, WebSocketConfig,
4036 auth::AuthState,
4037 types::{WriterCommand, channel_epoch_message_handler, channel_message_handler},
4038 },
4039 };
4040
4041 const SECRET_MARKER: &str = "OUTBOUND_SECRET_MARKER";
4042 const PING_TRIGGER: &str = "send-test-ping";
4043 const HOSTILE_CLOSE_TRIGGER: &str = "close-hostile";
4044 const CODED_CLOSE_TRIGGER: &str = "close-coded";
4045 const CODED_CLOSE_CODE: u16 = 4003;
4046 const CODED_CLOSE_REASON: &str = "draining";
4047 const HOSTILE_CLOSE_REASON: &str = "injected\r\n\u{1b}[31mforged\u{1b}[0m";
4048 const NETWORK_LOG_TARGETS: &[&str] = &[
4049 "nautilus_network::http::client",
4050 "nautilus_network::websocket::client",
4051 ];
4052
4053 struct TestServer {
4054 task: JoinHandle<()>,
4055 port: u16,
4056 }
4057
4058 #[derive(Debug, Clone)]
4059 struct TestCallback {
4060 key: String,
4061 value: HeaderValue,
4062 }
4063
4064 impl Callback for TestCallback {
4065 #[expect(clippy::panic_in_result_fn)]
4066 fn on_request(
4067 self,
4068 request: &server::Request,
4069 response: server::Response,
4070 ) -> Result<server::Response, server::ErrorResponse> {
4071 let _ = response;
4072 let value = request.headers().get(&self.key);
4073 assert!(value.is_some());
4074
4075 if let Some(value) = request.headers().get(&self.key) {
4076 assert_eq!(value, self.value);
4077 }
4078
4079 Ok(response)
4080 }
4081 }
4082
4083 impl TestServer {
4084 async fn setup() -> Self {
4085 let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4086 let port = TcpListener::local_addr(&server).unwrap().port();
4087
4088 let header_key = "test".to_string();
4089 let header_value = "test".to_string();
4090
4091 let test_call_back = TestCallback {
4092 key: header_key,
4093 value: HeaderValue::from_str(&header_value).unwrap(),
4094 };
4095
4096 let task = task::spawn(async move {
4097 loop {
4099 let (conn, _) = server.accept().await.unwrap();
4100 let mut websocket = accept_hdr_async(conn, test_call_back.clone())
4101 .await
4102 .unwrap();
4103
4104 task::spawn(async move {
4105 while let Some(Ok(msg)) = websocket.next().await {
4106 match msg {
4107 WsMessage::Text(txt) if txt == "close-now" => {
4108 log::debug!("Forcibly closing from server side");
4109 let _ = websocket.close(None).await;
4111 break;
4112 }
4113 WsMessage::Text(txt) if txt == HOSTILE_CLOSE_TRIGGER => {
4114 let _ = websocket
4118 .send(WsMessage::Close(Some(WsCloseFrame {
4119 code: CloseCode::Error,
4120 reason: HOSTILE_CLOSE_REASON.into(),
4121 })))
4122 .await;
4123
4124 break;
4125 }
4126 WsMessage::Text(txt) if txt == CODED_CLOSE_TRIGGER => {
4127 let _ = websocket
4128 .send(WsMessage::Close(Some(WsCloseFrame {
4129 code: CloseCode::from(CODED_CLOSE_CODE),
4130 reason: CODED_CLOSE_REASON.into(),
4131 })))
4132 .await;
4133
4134 break;
4135 }
4136 WsMessage::Text(txt) if txt == PING_TRIGGER => {
4137 let ping = format!("{SECRET_MARKER}:ping");
4138 if websocket.send(WsMessage::Ping(ping.into())).await.is_err() {
4139 break;
4140 }
4141 }
4142 WsMessage::Text(_) | WsMessage::Binary(_) => {
4144 if websocket.send(msg).await.is_err() {
4145 break;
4146 }
4147 }
4148 WsMessage::Close(_frame) => {
4150 let _ = websocket.close(None).await;
4151 break;
4152 }
4153 _ => {}
4155 }
4156 }
4157 });
4158 }
4159 });
4160
4161 Self { task, port }
4162 }
4163 }
4164
4165 impl Drop for TestServer {
4166 fn drop(&mut self) {
4167 self.task.abort();
4168 }
4169 }
4170
4171 async fn setup_test_client(port: u16) -> WebSocketClient {
4172 let config = WebSocketConfig {
4173 url: format!("ws://127.0.0.1:{port}"),
4174 headers: vec![("test".into(), "test".into())],
4175 heartbeat_interval_secs: None,
4176 heartbeat_payload: None,
4177 connect_timeout_ms: None,
4178 reconnect_delay_initial_ms: None,
4179 reconnect_backoff_factor: None,
4180 reconnect_delay_max_ms: None,
4181 reconnect_jitter_ms: None,
4182 reconnect_max_attempts: None,
4183 heartbeat_timeout_secs: None,
4184 idle_timeout_ms: None,
4185 writer_capacity: None,
4186 backend: TransportBackend::Tungstenite,
4187 proxy_url: None,
4188 max_message_size_bytes: None,
4189 max_frame_size_bytes: None,
4190 };
4191
4192 WebSocketClient::builder()
4193 .config(config)
4194 .message_handler(Arc::new(|_| {}))
4195 .connect()
4196 .await
4197 .expect("Failed to connect")
4198 }
4199
4200 #[rstest]
4201 #[case::handler(false)]
4202 #[case::stream(true)]
4203 #[tokio::test]
4204 async fn test_writer_capacity_from_config(#[case] stream_mode: bool) {
4205 let server = TestServer::setup().await;
4206 let config = WebSocketConfig::builder()
4207 .url(format!("ws://127.0.0.1:{}", server.port))
4208 .headers(vec![("test".to_string(), "test".to_string())])
4209 .backend(TransportBackend::Tungstenite)
4210 .writer_capacity(2)
4211 .build()
4212 .unwrap();
4213
4214 let (client, _reader) = if stream_mode {
4215 let (reader, client) = WebSocketClient::stream_builder()
4216 .config(config)
4217 .connect()
4218 .await
4219 .unwrap();
4220 (client, Some(reader))
4221 } else {
4222 let client = WebSocketClient::builder()
4223 .config(config)
4224 .message_handler(Arc::new(|_| {}))
4225 .connect()
4226 .await
4227 .unwrap();
4228 (client, None)
4229 };
4230
4231 client.send_text("first".to_string(), None).await.unwrap();
4233 client.send_bytes(vec![2], None).await.unwrap();
4234 let overflow = client.send_text("overflow".to_string(), None).await;
4235
4236 assert_eq!(client.connection_mode(), ConnectionMode::Active);
4237 assert!(matches!(overflow, Err(SendError::BufferFull)));
4238 client.disconnect().await;
4239 }
4240
4241 #[rstest]
4242 #[tokio::test]
4243 async fn test_send_reports_writer_buffer_full() {
4244 let server = TestServer::setup().await;
4245 let mut client = setup_test_client(server.port).await;
4246 let (writer_tx, mut writer_rx) = crate::writer::channel(1);
4247 client.writer_tx = writer_tx;
4248
4249 client.send_text("first".to_string(), None).await.unwrap();
4250 let result = client.send_bytes(vec![2], None).await;
4251 assert!(matches!(result, Err(SendError::BufferFull)));
4252 let (command, permit) = writer_rx.recv().await.unwrap();
4253 assert!(
4254 matches!(command, WriterCommand::Send(message) if message == Message::text("first"))
4255 );
4256 assert!(matches!(
4257 client.send_text("overflow".to_string(), None).await,
4258 Err(SendError::BufferFull)
4259 ));
4260 drop(permit);
4261 client.send_bytes(vec![3], None).await.unwrap();
4262 let (command, _) = writer_rx.recv().await.unwrap();
4263 assert!(
4264 matches!(command, WriterCommand::Send(message) if message == Message::Binary(vec![3].into()))
4265 );
4266 client.disconnect().await;
4267 }
4268
4269 async fn setup_reconnecting_client(port: u16) -> WebSocketClient {
4270 let config = WebSocketConfig {
4271 url: format!("ws://127.0.0.1:{port}"),
4272 headers: vec![],
4273 heartbeat_interval_secs: None,
4274 heartbeat_payload: None,
4275 connect_timeout_ms: Some(5_000),
4276 reconnect_delay_initial_ms: Some(1),
4277 reconnect_backoff_factor: Some(1.0),
4278 reconnect_delay_max_ms: Some(1),
4279 reconnect_jitter_ms: Some(0),
4280 reconnect_max_attempts: None,
4281 heartbeat_timeout_secs: None,
4282 idle_timeout_ms: None,
4283 writer_capacity: None,
4284 backend: TransportBackend::Tungstenite,
4285 proxy_url: None,
4286 max_message_size_bytes: None,
4287 max_frame_size_bytes: None,
4288 };
4289
4290 WebSocketClient::builder()
4291 .config(config)
4292 .message_handler(Arc::new(|_| {}))
4293 .connect()
4294 .await
4295 .expect("client should connect")
4296 }
4297
4298 async fn wait_for_mode(client: &WebSocketClient, expected: ConnectionMode) {
4299 crate::dst::time::timeout(Duration::from_secs(5), async {
4300 loop {
4301 if ConnectionMode::from_atomic(&client.connection_mode) == expected {
4302 break;
4303 }
4304
4305 crate::dst::time::sleep(Duration::from_millis(1)).await;
4306 }
4307 })
4308 .await
4309 .expect("client should reach expected connection mode");
4310 }
4311
4312 async fn setup_http_test_server() -> (JoinHandle<()>, u16) {
4313 let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4314 let port = server.local_addr().unwrap().port();
4315
4316 let app = Router::new().route(
4317 "/logging",
4318 post(|| async {
4319 (
4320 [("x-secret-response", SECRET_MARKER)],
4321 format!("{SECRET_MARKER}:response"),
4322 )
4323 }),
4324 );
4325
4326 let task = task::spawn(async move {
4327 axum::serve(server, app).await.unwrap();
4328 });
4329
4330 (task, port)
4331 }
4332
4333 #[rstest]
4334 #[tokio::test]
4335 async fn test_network_logs_omit_payload_bodies() {
4336 let server = TestServer::setup().await;
4337 let client = setup_test_client(server.port).await;
4338 let capture = capture_logs_for(NETWORK_LOG_TARGETS).await;
4339 let binary = format!("{SECRET_MARKER}:binary").into_bytes();
4340 let binary_marker = format!("{binary:?}");
4341
4342 client
4343 .send_text(format!("{SECRET_MARKER}:café"), None)
4344 .await
4345 .unwrap();
4346 client
4347 .send_text_on_connection(
4348 format!("{SECRET_MARKER}:owned-é"),
4349 None,
4350 client.connection_epoch(),
4351 )
4352 .await
4353 .unwrap();
4354 client.send_bytes(binary.clone(), None).await.unwrap();
4355 client
4356 .send_bytes_on_connection(binary, None, client.connection_epoch())
4357 .await
4358 .unwrap();
4359 client
4360 .send_text(PING_TRIGGER.to_string(), None)
4361 .await
4362 .unwrap();
4363
4364 tokio::time::timeout(Duration::from_secs(2), async {
4365 loop {
4366 if capture.messages().iter().any(|(level, message)| {
4367 matches!(level, Level::Trace | Level::Warn)
4368 && message == "Received ping frame (27 bytes)"
4369 }) {
4370 break;
4371 }
4372
4373 tokio::task::yield_now().await;
4374 }
4375 })
4376 .await
4377 .expect("timed out waiting for inbound WebSocket metadata log");
4378
4379 let (http_task, http_port) = setup_http_test_server().await;
4380 let invalid_headers =
4381 HashMap::from([("x-secret-default".to_string(), format!("{SECRET_MARKER}\n"))]);
4382 let invalid_header_error = HttpClient::builder()
4383 .headers(invalid_headers)
4384 .build()
4385 .unwrap_err();
4386 let http_client = HttpClient::builder().build().unwrap();
4387 let params = HashMap::from([("secret".to_string(), vec![SECRET_MARKER.to_string()])]);
4388 let headers = HashMap::from([
4389 (
4390 "X-Secret-Request".to_string(),
4391 format!("{SECRET_MARKER}:first"),
4392 ),
4393 (
4394 "x-secret-request".to_string(),
4395 format!("{SECRET_MARKER}:second"),
4396 ),
4397 ]);
4398 let http_body = format!("{SECRET_MARKER}:http-body").into_bytes();
4399 http_client
4400 .request(
4401 Method::POST,
4402 format!("http://127.0.0.1:{http_port}/logging"),
4403 Some(¶ms),
4404 Some(headers),
4405 Some(http_body),
4406 None,
4407 None,
4408 )
4409 .await
4410 .unwrap();
4411
4412 let rejecting = TcpListener::bind("127.0.0.1:0").await.unwrap();
4415 let rejecting_port = rejecting.local_addr().unwrap().port();
4416
4417 let rejecting_server = task::spawn(async move {
4418 loop {
4419 let (mut stream, _) = rejecting.accept().await.unwrap();
4420 let mut request = [0; 1024];
4421 let _ = stream.read(&mut request).await.unwrap();
4422 stream
4423 .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
4424 .await
4425 .unwrap();
4426 }
4427 });
4428
4429 let retry_config = WebSocketConfig {
4430 url: format!("ws://127.0.0.1:{rejecting_port}/?token={SECRET_MARKER}"),
4431 headers: vec![],
4432 heartbeat_interval_secs: None,
4433 heartbeat_payload: None,
4434 connect_timeout_ms: Some(5_000),
4435 reconnect_delay_initial_ms: None,
4436 reconnect_backoff_factor: None,
4437 reconnect_delay_max_ms: None,
4438 reconnect_jitter_ms: None,
4439 reconnect_max_attempts: None,
4440 heartbeat_timeout_secs: None,
4441 idle_timeout_ms: None,
4442 writer_capacity: None,
4443 backend: TransportBackend::Tungstenite,
4444 proxy_url: None,
4445 max_message_size_bytes: None,
4446 max_frame_size_bytes: None,
4447 };
4448
4449 let retry_error = WebSocketClient::builder()
4450 .config(retry_config)
4451 .message_handler(Arc::new(|_| {}))
4452 .initial_connect_retry_policy(InitialConnectRetryPolicy {
4453 max_attempts: NonZeroU32::new(2).unwrap(),
4454 delay_initial: Duration::from_millis(1),
4455 delay_max: Duration::from_millis(1),
4456 backoff_factor: 1.0,
4457 jitter_ms: 0,
4458 })
4459 .connect()
4460 .await
4461 .expect_err("server always rejects the upgrade");
4462
4463 rejecting_server.abort();
4464
4465 assert!(
4466 matches!(retry_error, TransportError::UpgradeRejected(503)),
4467 "expected a 503 upgrade rejection, was: {retry_error:?}"
4468 );
4469
4470 let messages: Vec<_> = capture
4471 .messages()
4472 .into_iter()
4473 .filter(|(level, message)| {
4474 matches!(level, Level::Trace | Level::Warn)
4475 && (message.starts_with("Sending ")
4476 || message.starts_with("Received ")
4477 || message.starts_with("Replaced ")
4478 || message.starts_with("WebSocket connection attempt "))
4479 })
4480 .map(|(_, message)| message)
4481 .collect();
4482
4483 let invalid_header_message = invalid_header_error.to_string();
4484
4485 let retry_warning = messages
4488 .iter()
4489 .find(|message| message.starts_with("WebSocket connection attempt 1/2 "))
4490 .expect("initial-connect retry warning was not captured");
4491 assert!(
4492 retry_warning.contains(REDACTED),
4493 "retry warning omitted the redaction placeholder: {retry_warning}"
4494 );
4495
4496 assert!(
4497 messages.iter().all(|message| {
4498 !message.contains(SECRET_MARKER) && !message.contains(&binary_marker)
4499 }),
4500 "network logs exposed the secret marker: {messages:?}"
4501 );
4502 assert!(
4503 !invalid_header_message.contains(SECRET_MARKER),
4504 "invalid header error exposed the secret marker: {invalid_header_message}"
4505 );
4506 assert!(
4507 invalid_header_message.contains("x-secret-default"),
4508 "invalid header error omitted safe header metadata: {invalid_header_message}"
4509 );
4510 assert!(
4511 messages
4512 .iter()
4513 .any(|message| message == "Sending text frame (28 bytes)"),
4514 "text send metadata missing or inaccurate: {messages:?}"
4515 );
4516 assert!(
4517 messages
4518 .iter()
4519 .any(|message| { message == "Sending text frame once: epoch=0 (31 bytes)" }),
4520 "ownership-bound text metadata missing or inaccurate: {messages:?}"
4521 );
4522 assert!(
4523 messages
4524 .iter()
4525 .any(|message| message == "Sending binary frame (29 bytes)"),
4526 "binary send metadata missing or inaccurate: {messages:?}"
4527 );
4528 assert!(
4529 messages
4530 .iter()
4531 .any(|message| message == "Sending binary frame once: epoch=0 (29 bytes)"),
4532 "ownership-bound binary metadata missing or inaccurate: {messages:?}"
4533 );
4534 assert!(
4535 messages
4536 .iter()
4537 .any(|message| message == "Received text frame (28 bytes)"),
4538 "text receive metadata missing or inaccurate: {messages:?}"
4539 );
4540 assert!(
4541 messages
4542 .iter()
4543 .any(|message| message == "Received text frame (31 bytes)"),
4544 "ownership-bound text receive metadata missing or inaccurate: {messages:?}"
4545 );
4546 assert!(
4547 messages
4548 .iter()
4549 .any(|message| message == "Received message <binary> 29 bytes"),
4550 "binary receive metadata missing or inaccurate: {messages:?}"
4551 );
4552 assert!(
4553 messages
4554 .iter()
4555 .any(|message| message == "Received ping frame (27 bytes)"),
4556 "ping receive metadata missing or inaccurate: {messages:?}"
4557 );
4558 assert!(
4560 messages.iter().any(|message| {
4561 message
4562 == "Sending HTTP request: method=POST extra_headers=3 query_bytes=29 \
4563 body_bytes=32"
4564 }),
4565 "HTTP request metadata missing or inaccurate: {messages:?}"
4566 );
4567 assert!(
4568 messages
4569 .iter()
4570 .any(|message| message == "Replaced duplicate request header 'x-secret-request'"),
4571 "duplicate header metadata missing: {messages:?}"
4572 );
4573 assert!(
4574 messages.iter().any(|message| {
4575 message.starts_with("Received HTTP response: status=200 OK headers=")
4576 && message.ends_with(" body_bytes=31")
4577 }),
4578 "HTTP response metadata missing or inaccurate: {messages:?}"
4579 );
4580
4581 client.disconnect().await;
4582 http_task.abort();
4583 }
4584
4585 #[rstest]
4589 #[tokio::test]
4590 async fn test_send_pong_skips_gated_reconnect() {
4591 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4592 let port = listener.local_addr().unwrap().port();
4593 let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
4594 let (handshake_gate_tx, handshake_gate_rx) = oneshot::channel();
4595 let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
4596
4597 let server_task = task::spawn(async move {
4598 let (first, _) = listener.accept().await.unwrap();
4599 let first_websocket = accept_async(first).await.unwrap();
4600 drop(first_websocket);
4601
4602 let (second, _) = listener.accept().await.unwrap();
4603 second_accepted_tx.send(()).unwrap();
4604 handshake_gate_rx.await.unwrap();
4605 let mut replacement = accept_async(second).await.unwrap();
4606
4607 while let Some(message) = replacement.next().await {
4608 match message.unwrap() {
4609 WsMessage::Pong(data) => observed_tx.send(data.to_vec()).unwrap(),
4610 WsMessage::Close(_) => {
4611 let _ = replacement.close(None).await;
4612 break;
4613 }
4614 _ => {}
4615 }
4616 }
4617 });
4618
4619 let client = setup_reconnecting_client(port).await;
4620
4621 crate::dst::time::timeout(Duration::from_secs(5), second_accepted_rx)
4622 .await
4623 .expect("replacement connection should be accepted")
4624 .unwrap();
4625 wait_for_mode(&client, ConnectionMode::Reconnect).await;
4626
4627 let rejected = client.send_pong(vec![2; 126]).await;
4628 assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4629
4630 crate::dst::time::timeout(
4633 Duration::from_secs(1),
4634 client.send_pong(b"stale-pong".to_vec()),
4635 )
4636 .await
4637 .expect("pong raised during reconnect should not wait for the replacement")
4638 .expect("skipped pong should report success");
4639
4640 handshake_gate_tx.send(()).unwrap();
4641 wait_for_mode(&client, ConnectionMode::Active).await;
4642
4643 let fresh_payload = b"fresh-pong".to_vec();
4644 client.send_pong(fresh_payload.clone()).await.unwrap();
4645 assert_eq!(
4646 crate::dst::time::timeout(Duration::from_secs(5), observed_rx.recv())
4647 .await
4648 .expect("replacement connection should receive the fresh pong"),
4649 Some(fresh_payload)
4650 );
4651
4652 client.disconnect().await;
4653 server_task.await.unwrap();
4654 assert_eq!(
4655 observed_rx.try_recv(),
4656 Err(mpsc::error::TryRecvError::Disconnected),
4657 "replacement connection should receive exactly one pong"
4658 );
4659 }
4660
4661 #[rstest]
4664 #[tokio::test]
4665 async fn test_pong_validation_precedes_inactive_skip() {
4666 let server = TestServer::setup().await;
4667 let client = setup_test_client(server.port).await;
4668 client
4669 .connection_mode
4670 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4671
4672 let result = client.send_pong(vec![2; 126]).await;
4673 let epoch_result = client.send_pong_on_connection(vec![2; 126], 0).await;
4674
4675 assert!(matches!(result, Err(SendError::InvalidInput(_))));
4676 assert!(matches!(epoch_result, Err(SendError::InvalidInput(_))));
4677 }
4678
4679 #[rstest]
4682 #[tokio::test]
4683 async fn test_pong_payload_limit_preserves_connection() {
4684 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4685 let port = listener.local_addr().unwrap().port();
4686 let (pong_tx, mut pong_rx) = mpsc::unbounded_channel();
4687
4688 let server_task = task::spawn(async move {
4689 let (stream, _) = listener.accept().await.unwrap();
4690 let mut websocket = accept_async(stream).await.unwrap();
4691
4692 while let Some(message) = websocket.next().await {
4693 match message.unwrap() {
4694 WsMessage::Pong(data) => pong_tx.send(data.to_vec()).unwrap(),
4695 WsMessage::Close(_) => {
4696 let _ = websocket.close(None).await;
4697 break;
4698 }
4699 _ => {}
4700 }
4701 }
4702 });
4703
4704 let client = setup_reconnecting_client(port).await;
4705 let accepted = vec![1; 125];
4706 let follow_up = vec![3; 125];
4707
4708 client.send_pong(accepted.clone()).await.unwrap();
4709 assert_eq!(
4710 crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4711 .await
4712 .unwrap(),
4713 Some(accepted)
4714 );
4715
4716 let rejected = client.send_pong(vec![2; 126]).await;
4717 assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4718
4719 client.send_pong(follow_up.clone()).await.unwrap();
4720 assert_eq!(
4721 crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4722 .await
4723 .unwrap(),
4724 Some(follow_up)
4725 );
4726
4727 client.disconnect().await;
4728 server_task.await.unwrap();
4729 }
4730
4731 #[tokio::test]
4732 async fn test_websocket_basic() {
4733 let server = TestServer::setup().await;
4734 let client = setup_test_client(server.port).await;
4735
4736 assert!(!client.is_disconnected());
4737
4738 client.disconnect().await;
4739 assert!(client.is_disconnected());
4740 }
4741
4742 #[rstest]
4743 #[tokio::test]
4744 async fn test_drop_sets_shared_connection_mode_closed() {
4745 let server = TestServer::setup().await;
4746 let client = setup_test_client(server.port).await;
4747 let connection_mode = client.connection_mode_atomic();
4748
4749 drop(client);
4750
4751 assert_eq!(
4752 ConnectionMode::from_atomic(&connection_mode),
4753 ConnectionMode::Closed
4754 );
4755 }
4756
4757 #[rstest]
4758 #[tokio::test]
4759 async fn test_notify_closed_closes_reconnecting_client() {
4760 let server = TestServer::setup().await;
4761 let client = setup_test_client(server.port).await;
4762 client
4763 .connection_mode
4764 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4765
4766 client.notify_closed();
4767
4768 assert!(client.is_closed());
4769 }
4770
4771 #[tokio::test]
4772 async fn test_websocket_heartbeat() {
4773 let server = TestServer::setup().await;
4774 let client = setup_test_client(server.port).await;
4775
4776 tokio::time::sleep(std::time::Duration::from_secs(3)).await;
4778
4779 client.disconnect().await;
4781 assert!(client.is_disconnected());
4782 }
4783
4784 #[rstest]
4785 #[tokio::test]
4786 async fn test_websocket_reconnect_exhausted() {
4787 let config = WebSocketConfig {
4788 url: "ws://127.0.0.1:9997".into(), headers: vec![],
4790 heartbeat_interval_secs: None,
4791 heartbeat_payload: None,
4792 connect_timeout_ms: None,
4793 reconnect_delay_initial_ms: None,
4794 reconnect_backoff_factor: None,
4795 reconnect_delay_max_ms: None,
4796 reconnect_jitter_ms: None,
4797 reconnect_max_attempts: None,
4798 heartbeat_timeout_secs: None,
4799 idle_timeout_ms: None,
4800 writer_capacity: None,
4801 backend: TransportBackend::Tungstenite,
4802 proxy_url: None,
4803 max_message_size_bytes: None,
4804 max_frame_size_bytes: None,
4805 };
4806
4807 let states = Arc::new(Mutex::new(Vec::new()));
4808 let states_callback = Arc::clone(&states);
4809 let sink = SocketStateSink::new(move |state| {
4810 states_callback.lock().push(state);
4811 });
4812
4813 let res = WebSocketClient::builder()
4814 .config(config)
4815 .message_handler(Arc::new(|_| {}))
4816 .state_sink(sink)
4817 .connect()
4818 .await;
4819 assert!(res.is_err(), "Should fail quickly with no server");
4820 assert_eq!(*states.lock(), Vec::new());
4821 }
4822
4823 #[tokio::test]
4824 async fn test_websocket_forced_close_reconnect() {
4825 let server = TestServer::setup().await;
4826 let client = setup_test_client(server.port).await;
4827
4828 client.send_text("Hello".into(), None).await.unwrap();
4830
4831 client.send_text("close-now".into(), None).await.unwrap();
4833
4834 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
4836
4837 assert!(!client.is_disconnected());
4839
4840 client.disconnect().await;
4842 assert!(client.is_disconnected());
4843 }
4844
4845 #[rstest]
4846 #[case::plain(false)]
4847 #[case::epoch(true)]
4848 #[tokio::test]
4849 async fn test_peer_close_preserves_forwarded_messages(#[case] epoch_aware: bool) {
4850 let timeout = Duration::from_secs(5);
4851 let subscription = "subscribe-retained";
4852 let server = TestServer::setup().await;
4853 let config = WebSocketConfig::builder()
4854 .url(format!("ws://127.0.0.1:{}", server.port))
4855 .headers(vec![("test".to_string(), "test".to_string())])
4856 .backend(TransportBackend::Tungstenite)
4857 .reconnect_delay_initial_ms(50)
4858 .reconnect_jitter_ms(0)
4859 .build()
4860 .unwrap();
4861 let (handler, mut raw_rx) = channel_epoch_message_handler();
4862
4863 let ws_client = if epoch_aware {
4864 WebSocketClient::epoch_builder()
4865 .config(config)
4866 .epoch_handler(handler)
4867 .connect()
4868 .await
4869 } else {
4870 WebSocketClient::builder()
4871 .config(config)
4872 .message_handler(Arc::new(move |message| handler(0, message)))
4873 .connect()
4874 .await
4875 }
4876 .unwrap();
4877
4878 let ws_client = Arc::new(ws_client);
4879 let (bytes_tx, mut bytes_rx) = tokio::sync::mpsc::unbounded_channel();
4880 let forwarding_client = Arc::clone(&ws_client);
4881
4882 let forwarding_task = tokio::spawn(async move {
4883 while let Some((epoch, message)) = raw_rx.recv().await {
4884 let data = match message {
4885 WsMessage::Text(text) => text.as_bytes().to_vec(),
4886 WsMessage::Binary(data) => data.to_vec(),
4887 WsMessage::Close(_) => break,
4888 _ => continue,
4889 };
4890
4891 if data == RECONNECTED.as_bytes() {
4892 forwarding_client
4893 .send_text(subscription.into(), None)
4894 .await
4895 .unwrap();
4896 }
4897
4898 if bytes_tx.send((epoch, data)).is_err() {
4899 break;
4900 }
4901 }
4902 });
4903
4904 ws_client
4905 .send_text(subscription.into(), None)
4906 .await
4907 .unwrap();
4908 let initial = tokio::time::timeout(timeout, bytes_rx.recv())
4909 .await
4910 .expect("initial subscription response timed out")
4911 .expect("forwarding task ended before peer close");
4912
4913 ws_client.send_text("close-now".into(), None).await.unwrap();
4914 let reconnected = tokio::time::timeout(timeout, bytes_rx.recv())
4915 .await
4916 .expect("reconnect notification timed out")
4917 .expect("peer close terminated the forwarding task");
4918 let replayed = tokio::time::timeout(timeout, bytes_rx.recv())
4919 .await
4920 .expect("subscription replay response timed out")
4921 .expect("forwarding task ended before subscription replay");
4922 let payload = vec![3, 7, 11];
4923 ws_client.send_bytes(payload.clone(), None).await.unwrap();
4924 let live = tokio::time::timeout(timeout, bytes_rx.recv())
4925 .await
4926 .expect("replacement connection data timed out")
4927 .expect("forwarding task ended before replacement data");
4928
4929 let expected_epoch = u64::from(epoch_aware);
4930 assert_eq!(initial, (0, subscription.as_bytes().to_vec()));
4931 assert_eq!(
4932 reconnected,
4933 (expected_epoch, RECONNECTED.as_bytes().to_vec())
4934 );
4935 assert_eq!(replayed, (expected_epoch, subscription.as_bytes().to_vec()));
4936 assert_eq!(live, (expected_epoch, payload));
4937 assert_eq!(ws_client.connection_epoch(), 1);
4938 assert!(!forwarding_task.is_finished());
4939
4940 ws_client.disconnect().await;
4941 forwarding_task.abort();
4942 }
4943
4944 #[rstest]
4945 #[case(true)]
4946 #[case(false)]
4947 #[tokio::test]
4948 async fn test_peer_close_frame_is_forwarded_to_close_handler(#[case] reconnect: bool) {
4949 let server = TestServer::setup().await;
4950 let (handler, mut handler_rx) = channel_message_handler();
4951 let (state_tx, mut state_rx) = tokio::sync::mpsc::unbounded_channel();
4952
4953 let client = WebSocketClient::builder()
4954 .config(WebSocketConfig {
4955 url: format!("ws://127.0.0.1:{}", server.port),
4956 headers: vec![("test".into(), "test".into())],
4957 heartbeat_interval_secs: None,
4958 heartbeat_payload: None,
4959 connect_timeout_ms: Some(1_000),
4960 reconnect_delay_initial_ms: Some(1_000),
4961 reconnect_backoff_factor: None,
4962 reconnect_delay_max_ms: None,
4963 reconnect_jitter_ms: Some(0),
4964 reconnect_max_attempts: None,
4965 heartbeat_timeout_secs: None,
4966 idle_timeout_ms: None,
4967 writer_capacity: None,
4968 backend: TransportBackend::Tungstenite,
4969 proxy_url: None,
4970 max_message_size_bytes: None,
4971 max_frame_size_bytes: None,
4972 })
4973 .message_handler(Arc::new(|_| {}))
4974 .state_sink(SocketStateSink::new(move |state| {
4975 state_tx.send(state).expect("state channel closed");
4976 }))
4977 .close_handler(Arc::new(move |frame| {
4978 handler(Message::Close(frame));
4979 reconnect
4980 }))
4981 .connect()
4982 .await
4983 .expect("Failed to connect");
4984
4985 let tracker = AuthTracker::new();
4986 let pending_auth = tracker.begin();
4987 client.set_auth_tracker(tracker.clone(), false);
4988
4989 client
4990 .send_text(CODED_CLOSE_TRIGGER.to_string(), None)
4991 .await
4992 .unwrap();
4993
4994 let message = tokio::time::timeout(Duration::from_secs(10), handler_rx.recv())
4995 .await
4996 .expect("close frame was not forwarded")
4997 .expect("handler channel closed");
4998
4999 let WsMessage::Close(Some(frame)) = message else {
5000 panic!("expected a close frame, was {message:?}");
5001 };
5002
5003 let connected = state_rx.recv().await.expect("initial socket state");
5004 let disconnected = tokio::time::timeout(Duration::from_secs(5), state_rx.recv())
5005 .await
5006 .expect("peer close must publish socket loss")
5007 .expect("state channel closed");
5008
5009 assert_eq!(u16::from(frame.code), CODED_CLOSE_CODE);
5010 assert_eq!(frame.reason.as_str(), CODED_CLOSE_REASON);
5011 assert_eq!(connected, SocketState::Connected);
5012 assert_eq!(disconnected, SocketState::Disconnected);
5013
5014 if !reconnect {
5015 let auth_result = tokio::time::timeout(Duration::from_secs(1), pending_auth)
5016 .await
5017 .expect("terminal close should resolve pending authentication")
5018 .expect("authentication sender should report its terminal result");
5019
5020 assert_eq!(auth_result, Err("WebSocket client closed".to_string()));
5021 assert_eq!(tracker.auth_state(), AuthState::Failed);
5022 }
5023
5024 client.disconnect().await;
5025 }
5026
5027 #[rstest]
5028 #[case(true, ConnectionMode::Active)]
5029 #[case(false, ConnectionMode::Closed)]
5030 fn test_peer_close_callback_controls_reconnect(
5031 #[case] reconnect: bool,
5032 #[case] expected_mode: ConnectionMode,
5033 ) {
5034 let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5035 let fence = ReadSessionFence::new();
5036 let received = Arc::new(Mutex::new(Vec::new()));
5037 let states = Arc::new(Mutex::new(Vec::new()));
5038 let sink_states = Arc::clone(&states);
5039 let sink = SocketStateSink::new(move |state| sink_states.lock().push(state));
5040 let callback_received = Arc::clone(&received);
5041 let callback: CloseHandler = Arc::new(move |frame| {
5042 callback_received.lock().push(frame);
5043 reconnect
5044 });
5045
5046 let frame = CloseFrame::new(CODED_CLOSE_CODE, CODED_CLOSE_REASON);
5047
5048 WebSocketClientInner::dispatch_close_frame(
5049 &mode,
5050 &fence,
5051 Some(&callback),
5052 Some(&sink),
5053 Some(frame.clone()),
5054 );
5055
5056 assert_eq!(*received.lock(), vec![Some(frame)]);
5057 assert_eq!(ConnectionMode::from_atomic(&mode), expected_mode);
5058 assert_eq!(
5059 *states.lock(),
5060 if reconnect {
5061 Vec::new()
5062 } else {
5063 vec![SocketState::Disconnected]
5064 },
5065 );
5066 }
5067
5068 #[rstest]
5069 fn test_peer_close_callback_ignores_retired_reader() {
5070 let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5071 let fence = ReadSessionFence::new();
5072 fence.invalidate();
5073 let calls = Arc::new(AtomicUsize::new(0));
5074 let callback_calls = Arc::clone(&calls);
5075 let callback: CloseHandler = Arc::new(move |_| {
5076 callback_calls.fetch_add(1, Ordering::SeqCst);
5077 false
5078 });
5079
5080 WebSocketClientInner::dispatch_close_frame(&mode, &fence, Some(&callback), None, None);
5081
5082 assert_eq!(calls.load(Ordering::SeqCst), 0);
5083 assert_eq!(ConnectionMode::from_atomic(&mode), ConnectionMode::Active);
5084 }
5085
5086 #[rstest]
5088 #[tokio::test]
5089 async fn test_close_frame_reason_is_redacted_in_logs() {
5090 let server = TestServer::setup().await;
5091 let client = setup_test_client(server.port).await;
5092 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5093
5094 client
5095 .send_text(HOSTILE_CLOSE_TRIGGER.to_string(), None)
5096 .await
5097 .unwrap();
5098
5099 tokio::time::timeout(Duration::from_secs(5), async {
5100 loop {
5101 if capture.messages().iter().any(|(_, message)| {
5102 message.starts_with("Received close frame, terminating: code=")
5103 }) {
5104 break;
5105 }
5106
5107 tokio::task::yield_now().await;
5108 }
5109 })
5110 .await
5111 .expect("timed out waiting for the close frame log");
5112
5113 let close_frame_logs: Vec<String> = capture
5114 .messages()
5115 .iter()
5116 .map(|(_, message)| message.clone())
5117 .filter(|message| message.starts_with("Received close frame, terminating: code="))
5118 .collect();
5119
5120 let expected = "Received close frame, terminating: code=1011, reason=<redacted>";
5121 assert!(
5122 close_frame_logs.iter().any(|message| message == expected),
5123 "close frame reason was not redacted: {close_frame_logs:?}"
5124 );
5125 assert!(
5126 close_frame_logs.iter().all(|message| {
5127 !message.contains('\r') && !message.contains('\n') && !message.contains('\u{1b}')
5128 }),
5129 "close frame log forged line breaks or escape sequences: {close_frame_logs:?}"
5130 );
5131
5132 assert!(
5133 close_frame_logs
5134 .iter()
5135 .all(|message| !message.contains("injected") && !message.contains("forged"))
5136 );
5137
5138 client.disconnect().await;
5139 }
5140
5141 #[rstest]
5142 #[tokio::test]
5143 async fn test_state_sink_reports_initial_loss_and_recovery() {
5144 let server = TestServer::setup().await;
5145
5146 let config = WebSocketConfig {
5147 url: format!("ws://127.0.0.1:{}", server.port),
5148 headers: vec![("test".into(), "test".into())],
5149 heartbeat_interval_secs: None,
5150 heartbeat_payload: None,
5151 connect_timeout_ms: Some(1_000),
5152 reconnect_delay_initial_ms: Some(1),
5153 reconnect_backoff_factor: Some(1.0),
5154 reconnect_delay_max_ms: Some(1),
5155 reconnect_jitter_ms: Some(0),
5156 reconnect_max_attempts: Some(3),
5157 heartbeat_timeout_secs: None,
5158 idle_timeout_ms: None,
5159 writer_capacity: None,
5160 backend: TransportBackend::Tungstenite,
5161 proxy_url: None,
5162 max_message_size_bytes: None,
5163 max_frame_size_bytes: None,
5164 };
5165
5166 let states = Arc::new(Mutex::new(Vec::new()));
5167 let states_callback = Arc::clone(&states);
5168 let sink = SocketStateSink::new(move |state| {
5169 states_callback.lock().push(state);
5170 });
5171
5172 let client = WebSocketClient::builder()
5173 .config(config)
5174 .message_handler(Arc::new(|_| {}))
5175 .state_sink(sink)
5176 .connect()
5177 .await
5178 .unwrap();
5179
5180 assert_eq!(*states.lock(), vec![SocketState::Connected]);
5181
5182 client.send_text("close-now".into(), None).await.unwrap();
5183 wait_until_async(
5184 || {
5185 let states = Arc::clone(&states);
5186 async move { states.lock().len() == 3 }
5187 },
5188 Duration::from_secs(5),
5189 )
5190 .await;
5191
5192 assert_eq!(
5193 *states.lock(),
5194 vec![
5195 SocketState::Connected,
5196 SocketState::Disconnected,
5197 SocketState::Connected,
5198 ]
5199 );
5200
5201 client.disconnect().await;
5202 assert_eq!(states.lock().len(), 3);
5203 }
5204
5205 #[rstest]
5206 #[tokio::test]
5207 async fn test_drop_suppresses_socket_state_event() {
5208 let server = TestServer::setup().await;
5209
5210 let config = WebSocketConfig {
5211 url: format!("ws://127.0.0.1:{}", server.port),
5212 headers: vec![("test".into(), "test".into())],
5213 heartbeat_interval_secs: None,
5214 heartbeat_payload: None,
5215 connect_timeout_ms: Some(1_000),
5216 reconnect_delay_initial_ms: Some(1),
5217 reconnect_backoff_factor: Some(1.0),
5218 reconnect_delay_max_ms: Some(1),
5219 reconnect_jitter_ms: Some(0),
5220 reconnect_max_attempts: Some(3),
5221 heartbeat_timeout_secs: None,
5222 idle_timeout_ms: None,
5223 writer_capacity: None,
5224 backend: TransportBackend::Tungstenite,
5225 proxy_url: None,
5226 max_message_size_bytes: None,
5227 max_frame_size_bytes: None,
5228 };
5229
5230 let states = Arc::new(Mutex::new(Vec::new()));
5231 let states_callback = Arc::clone(&states);
5232 let sink = SocketStateSink::new(move |state| {
5233 states_callback.lock().push(state);
5234 });
5235
5236 let client = WebSocketClient::builder()
5237 .config(config)
5238 .message_handler(Arc::new(|_| {}))
5239 .state_sink(sink)
5240 .connect()
5241 .await
5242 .unwrap();
5243
5244 drop(client);
5245 crate::dst::time::sleep(Duration::from_millis(25)).await;
5246
5247 assert_eq!(*states.lock(), vec![SocketState::Connected]);
5248 }
5249
5250 #[rstest]
5251 #[tokio::test]
5252 async fn test_stream_state_sink_reports_reader_loss() {
5253 let server = TestServer::setup().await;
5254
5255 let config = WebSocketConfig {
5256 url: format!("ws://127.0.0.1:{}", server.port),
5257 headers: vec![("test".into(), "test".into())],
5258 heartbeat_interval_secs: None,
5259 heartbeat_payload: None,
5260 connect_timeout_ms: None,
5261 reconnect_delay_initial_ms: None,
5262 reconnect_backoff_factor: None,
5263 reconnect_delay_max_ms: None,
5264 reconnect_jitter_ms: None,
5265 reconnect_max_attempts: None,
5266 heartbeat_timeout_secs: None,
5267 idle_timeout_ms: None,
5268 writer_capacity: None,
5269 backend: TransportBackend::Tungstenite,
5270 proxy_url: None,
5271 max_message_size_bytes: None,
5272 max_frame_size_bytes: None,
5273 };
5274
5275 let states = Arc::new(Mutex::new(Vec::new()));
5276 let states_callback = Arc::clone(&states);
5277 let sink = SocketStateSink::new(move |state| {
5278 states_callback.lock().push(state);
5279 });
5280
5281 let (_reader, client) = WebSocketClient::stream_builder()
5282 .config(config)
5283 .state_sink(sink)
5284 .connect()
5285 .await
5286 .unwrap();
5287
5288 client.notify_closed();
5289
5290 assert!(client.is_closed());
5291 assert_eq!(
5292 *states.lock(),
5293 vec![SocketState::Connected, SocketState::Disconnected]
5294 );
5295 }
5296
5297 #[rstest]
5298 #[tokio::test]
5299 async fn test_state_sink_emits_no_retry_or_exhaustion_events() {
5300 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5301 let port = listener.local_addr().unwrap().port();
5302 let server_task = task::spawn(async move {
5303 let (connection, _) = listener.accept().await.unwrap();
5304 let mut websocket = accept_async(connection).await.unwrap();
5305 while let Some(Ok(message)) = websocket.next().await {
5306 if matches!(&message, WsMessage::Text(text) if text.as_str() == "close-now") {
5307 websocket.close(None).await.unwrap();
5308 break;
5309 }
5310 }
5311 });
5312
5313 let config = WebSocketConfig {
5314 url: format!("ws://127.0.0.1:{port}"),
5315 headers: vec![],
5316 heartbeat_interval_secs: None,
5317 heartbeat_payload: None,
5318 connect_timeout_ms: Some(100),
5319 reconnect_delay_initial_ms: Some(1),
5320 reconnect_backoff_factor: Some(1.0),
5321 reconnect_delay_max_ms: Some(1),
5322 reconnect_jitter_ms: Some(0),
5323 reconnect_max_attempts: Some(2),
5324 heartbeat_timeout_secs: None,
5325 idle_timeout_ms: None,
5326 writer_capacity: None,
5327 backend: TransportBackend::Tungstenite,
5328 proxy_url: None,
5329 max_message_size_bytes: None,
5330 max_frame_size_bytes: None,
5331 };
5332
5333 let states = Arc::new(Mutex::new(Vec::new()));
5334 let states_callback = Arc::clone(&states);
5335 let sink = SocketStateSink::new(move |state| {
5336 states_callback.lock().push(state);
5337 });
5338
5339 let client = WebSocketClient::builder()
5340 .config(config)
5341 .message_handler(Arc::new(|_| {}))
5342 .state_sink(sink)
5343 .connect()
5344 .await
5345 .unwrap();
5346
5347 client.send_text("close-now".into(), None).await.unwrap();
5348 wait_until_async(
5349 || async { client.is_disconnected() },
5350 Duration::from_secs(5),
5351 )
5352 .await;
5353 assert_eq!(
5354 *states.lock(),
5355 vec![SocketState::Connected, SocketState::Disconnected]
5356 );
5357
5358 server_task.await.unwrap();
5359 }
5360
5361 #[rstest]
5362 #[case(
5363 "invalid header",
5364 "value",
5365 "Invalid WebSocket reconnect header name: invalid HTTP header name"
5366 )]
5367 #[case(
5368 "x-test",
5369 "invalid\nvalue",
5370 "Invalid WebSocket reconnect header value: failed to parse header value"
5371 )]
5372 fn test_reconnect_headers_reject_invalid_update(
5373 #[case] name: &str,
5374 #[case] value: &str,
5375 #[case] expected_message: &str,
5376 ) {
5377 let initial = vec![("x-existing".to_string(), "initial".to_string())];
5378 let headers = ReconnectHeaders::new(initial.clone());
5379
5380 let error = headers.update(name, value).unwrap_err();
5381
5382 let TransportError::Io(error) = error else {
5383 panic!("expected I/O error, was {error:?}");
5384 };
5385
5386 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
5387 assert_eq!(error.to_string(), expected_message);
5388 assert_eq!(headers.snapshot(), initial);
5389 }
5390
5391 #[tokio::test]
5392 #[allow(clippy::result_large_err)]
5393 async fn test_reconnect_uses_updated_headers_without_interrupting_active_connection() {
5394 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5395 let port = listener.local_addr().unwrap().port();
5396 let (header_tx, mut header_rx) = tokio::sync::mpsc::unbounded_channel();
5397
5398 let server_task = task::spawn(async move {
5399 loop {
5400 let (conn, _) = listener.accept().await.unwrap();
5401 let header_tx = header_tx.clone();
5402
5403 let mut websocket = accept_hdr_async(
5404 conn,
5405 move |request: &server::Request, response: server::Response| {
5406 let values = request
5407 .headers()
5408 .get_all("authorization")
5409 .iter()
5410 .map(|value| value.to_str().unwrap().to_string())
5411 .collect::<Vec<_>>();
5412 header_tx.send(values).unwrap();
5413 Ok(response)
5414 },
5415 )
5416 .await
5417 .unwrap();
5418
5419 task::spawn(async move {
5420 while let Some(Ok(msg)) = websocket.next().await {
5421 if matches!(&msg, WsMessage::Text(text) if text.as_str() == "close-now") {
5422 let _ = websocket.close(None).await;
5423 break;
5424 }
5425 }
5426 });
5427 }
5428 });
5429
5430 let config = WebSocketConfig {
5431 url: format!("ws://127.0.0.1:{port}"),
5432 headers: vec![("Authorization".into(), "Bearer initial".into())],
5433 heartbeat_interval_secs: None,
5434 heartbeat_payload: None,
5435 connect_timeout_ms: Some(1_000),
5436 reconnect_delay_initial_ms: Some(50),
5437 reconnect_delay_max_ms: Some(50),
5438 reconnect_backoff_factor: Some(1.0),
5439 reconnect_jitter_ms: Some(0),
5440 reconnect_max_attempts: None,
5441 heartbeat_timeout_secs: None,
5442 idle_timeout_ms: None,
5443 writer_capacity: None,
5444 backend: TransportBackend::Tungstenite,
5445 proxy_url: None,
5446 max_message_size_bytes: None,
5447 max_frame_size_bytes: None,
5448 };
5449
5450 let client = WebSocketClient::builder()
5451 .config(config)
5452 .message_handler(Arc::new(|_| {}))
5453 .connect()
5454 .await
5455 .unwrap();
5456
5457 let initial = header_rx.recv().await.unwrap();
5458 let reconnect_headers = client.reconnect_headers();
5459 reconnect_headers
5460 .update("authorization", "Bearer refreshed")
5461 .unwrap();
5462
5463 tokio::time::sleep(Duration::from_millis(100)).await;
5464 assert_eq!(initial, vec!["Bearer initial"]);
5465 assert!(client.is_active());
5466 assert!(header_rx.try_recv().is_err());
5467 assert!(!format!("{reconnect_headers:?}").contains("refreshed"));
5468
5469 client.send_text("close-now".into(), None).await.unwrap();
5470 let refreshed = tokio::time::timeout(Duration::from_secs(3), header_rx.recv())
5471 .await
5472 .unwrap()
5473 .unwrap();
5474
5475 assert_eq!(refreshed, vec!["Bearer refreshed"]);
5476
5477 client.disconnect().await;
5478 server_task.abort();
5479 }
5480
5481 #[tokio::test]
5482 async fn test_rate_limiter() {
5483 let server = TestServer::setup().await;
5484 let quota = Quota::per_second(NonZeroU32::new(2).unwrap()).unwrap();
5485
5486 let config = WebSocketConfig {
5487 url: format!("ws://127.0.0.1:{}", server.port),
5488 headers: vec![("test".into(), "test".into())],
5489 heartbeat_interval_secs: None,
5490 heartbeat_payload: None,
5491 connect_timeout_ms: None,
5492 reconnect_delay_initial_ms: None,
5493 reconnect_backoff_factor: None,
5494 reconnect_delay_max_ms: None,
5495 reconnect_jitter_ms: None,
5496 reconnect_max_attempts: None,
5497 heartbeat_timeout_secs: None,
5498 idle_timeout_ms: None,
5499 writer_capacity: None,
5500 backend: TransportBackend::Tungstenite,
5501 proxy_url: None,
5502 max_message_size_bytes: None,
5503 max_frame_size_bytes: None,
5504 };
5505
5506 let client = WebSocketClient::builder()
5507 .config(config)
5508 .message_handler(Arc::new(|_| {}))
5509 .keyed_quotas(vec![("default".into(), quota)])
5510 .connect()
5511 .await
5512 .unwrap();
5513
5514 let keys: [ustr::Ustr; 1] = [ustr::Ustr::from("default")];
5517 let start = std::time::Instant::now();
5518 client
5519 .send_text("test1".into(), Some(keys.as_slice()))
5520 .await
5521 .unwrap();
5522 client
5523 .send_text("test2".into(), Some(keys.as_slice()))
5524 .await
5525 .unwrap();
5526 let after_burst = start.elapsed();
5527 client
5528 .send_text("test3".into(), Some(keys.as_slice()))
5529 .await
5530 .unwrap();
5531 let after_third = start.elapsed();
5532
5533 assert!(
5534 after_burst < std::time::Duration::from_millis(300),
5535 "Burst sends should not be rate limited, took {after_burst:?}"
5536 );
5537 assert!(
5538 after_third >= std::time::Duration::from_millis(400),
5539 "Third send should wait for quota replenishment, took {after_third:?}"
5540 );
5541
5542 client.disconnect().await;
5544 assert!(client.is_disconnected());
5545 }
5546
5547 #[tokio::test]
5548 async fn test_concurrent_writers() {
5549 let server = TestServer::setup().await;
5550 let client = Arc::new(setup_test_client(server.port).await);
5551
5552 let mut handles = vec![];
5553
5554 for i in 0..10 {
5555 let client = Arc::clone(&client);
5556 handles.push(task::spawn(async move {
5557 client.send_text(format!("test{i}"), None).await.unwrap();
5558 }));
5559 }
5560
5561 for handle in handles {
5562 handle.await.unwrap();
5563 }
5564
5565 client.disconnect().await;
5567 assert!(client.is_disconnected());
5568 }
5569}
5570
5571#[cfg(test)]
5572#[cfg(not(feature = "turmoil"))]
5573#[cfg(not(all(feature = "simulation", madsim)))] mod rust_tests {
5575 use std::{
5576 pin::Pin,
5577 sync::{
5578 Arc, OnceLock,
5579 atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
5580 },
5581 task::{Context, Poll},
5582 };
5583
5584 use futures_util::{SinkExt, StreamExt};
5585 #[cfg(target_os = "linux")]
5586 use log::Level;
5587 use nautilus_common::testing::wait_until_async;
5588 use parking_lot::{Condvar, Mutex};
5589 use rstest::rstest;
5590 #[cfg(feature = "transport-sockudo")]
5591 use sockudo_ws::handshake as sockudo_handshake;
5592 #[cfg(feature = "transport-sockudo")]
5593 use tokio::io::AsyncRead;
5594 use tokio::{
5595 io::{AsyncReadExt, AsyncWriteExt},
5596 net::TcpListener,
5597 sync::oneshot,
5598 task::{self, JoinHandle},
5599 time::{Duration, sleep},
5600 };
5601 use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
5602 #[cfg(feature = "transport-sockudo")]
5603 use tokio_tungstenite::{
5604 accept_hdr_async,
5605 tungstenite::{
5606 handshake::server::{self, Callback},
5607 http::HeaderValue,
5608 },
5609 };
5610
5611 use super::*;
5612 #[cfg(target_os = "linux")]
5613 use crate::logging::tests::capture_logs_for;
5614 use crate::{
5615 SocketState,
5616 websocket::types::{channel_epoch_message_handler, channel_message_handler},
5617 };
5618
5619 const TEST_TIMEOUT: Duration = Duration::from_secs(10);
5620
5621 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5624 const RECONNECT_LOG_TARGETS: &[&str] = &[
5625 "nautilus_network::transport::sockudo",
5626 "nautilus_network::websocket::client",
5627 ];
5628
5629 #[cfg(target_os = "linux")]
5630 #[rstest]
5631 #[case::tungstenite(TransportBackend::Tungstenite)]
5632 #[cfg_attr(
5633 feature = "transport-sockudo",
5634 case::sockudo(TransportBackend::Sockudo)
5635 )]
5636 #[tokio::test]
5637 async fn test_initial_connect_redacts_error_text(#[case] backend: TransportBackend) {
5638 let capture = capture_logs_for(&["nautilus_network::retry"]).await;
5639 let socket = tokio::net::TcpSocket::new_v4().unwrap();
5640 socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
5641 let mut config = reconnect_test_config(socket.local_addr().unwrap().port());
5642 config.backend = backend;
5643 let error = WebSocketClient::builder()
5644 .config(config)
5645 .message_handler(Arc::new(|_| {}))
5646 .connect()
5647 .await
5648 .unwrap_err();
5649 let messages = capture.messages();
5650 let raw_message = error.to_string();
5651 let redacted_message = error.redacted_message();
5652
5653 assert!(
5654 matches!(error, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::ConnectionRefused)
5655 );
5656 assert_ne!(raw_message, redacted_message);
5657 assert_eq!(redacted_message, "I/O error: ConnectionRefused");
5658 assert!(messages.contains(&(
5659 Level::Trace,
5660 format!(
5661 "Operation '{INITIAL_CONNECT_OPERATION}' retries exhausted after 1 attempts: {redacted_message}"
5662 ),
5663 )), "{messages:?}");
5664 assert!(
5665 messages
5666 .iter()
5667 .all(|(_, message)| !message.contains(&raw_message)),
5668 "{messages:?}"
5669 );
5670 }
5671
5672 #[cfg(target_os = "linux")]
5673 #[rstest]
5674 #[case::tungstenite(TransportBackend::Tungstenite)]
5675 #[cfg_attr(
5676 feature = "transport-sockudo",
5677 case::sockudo(TransportBackend::Sockudo)
5678 )]
5679 #[tokio::test]
5680 async fn test_reconnect_redacts_error_text(#[case] backend: TransportBackend) {
5681 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5682 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5683 let port = listener.local_addr().unwrap().port();
5684 let (close_tx, close_rx) = oneshot::channel();
5685
5686 let server = tokio::spawn(async move {
5687 let (stream, _) = listener.accept().await.unwrap();
5688 let ws_client = accept_async(stream).await.unwrap();
5689 close_rx.await.unwrap();
5690 drop(ws_client);
5691 let (_stream, _) = listener.accept().await.unwrap();
5692 std::future::pending::<()>().await;
5693 });
5694
5695 let mut config = reconnect_test_config(port);
5696 config.backend = backend;
5697 config.reconnect_max_attempts = Some(1);
5698 config.reconnect_delay_initial_ms = Some(1);
5699 config.reconnect_delay_max_ms = Some(1);
5700 config.reconnect_jitter_ms = Some(0);
5701 let client = WebSocketClient::builder()
5702 .config(config)
5703 .message_handler(Arc::new(|_| {}))
5704 .connect()
5705 .await
5706 .unwrap();
5707 close_tx.send(()).unwrap();
5708
5709 wait_until_async(
5710 || async {
5711 capture.messages().iter().any(|(level, message)| {
5712 *level == Level::Warn && message.starts_with("Reconnect attempt 1 failed:")
5713 })
5714 },
5715 TEST_TIMEOUT,
5716 )
5717 .await;
5718
5719 client.disconnect().await;
5720 server.abort();
5721 await_task_termination(server, "redaction peer").await;
5722 let messages = capture.messages();
5723 let warnings: Vec<_> = messages
5724 .iter()
5725 .filter(|(_, message)| message.starts_with("Reconnect attempt "))
5726 .cloned()
5727 .collect();
5728
5729 assert_eq!(
5730 warnings,
5731 vec![(
5732 Level::Warn,
5733 "Reconnect attempt 1 failed: I/O error: TimedOut".into()
5734 )],
5735 );
5736 }
5737
5738 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5739 #[tokio::test]
5740 async fn test_sockudo_handshake_trace_redacts_authentication() {
5741 let capture = capture_logs_for(&[
5742 "tungstenite::handshake::client",
5743 "nautilus_network::transport::sockudo",
5744 "nautilus_network::retry",
5745 "nautilus_network::websocket::client",
5746 ])
5747 .await;
5748 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5749 let addr = listener.local_addr().unwrap();
5750 let peer = tokio::spawn(async move {
5751 let (mut stream, _) = listener.accept().await.unwrap();
5752 let request = read_http_request(&mut stream).await;
5753 stream
5754 .write_all(
5755 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
5756 )
5757 .await
5758 .unwrap();
5759 stream.flush().await.unwrap();
5760 request
5761 });
5762
5763 let config = WebSocketConfig::builder()
5764 .url(format!(
5765 "ws://{addr}/TRACE_PATH_SECRET?token=TRACE_QUERY_SECRET"
5766 ))
5767 .headers(vec![
5768 ("KALSHI-ACCESS-KEY".into(), "TRACE_KEY_SECRET".into()),
5769 (
5770 "KALSHI-ACCESS-SIGNATURE".into(),
5771 "TRACE_SIGNATURE_SECRET".into(),
5772 ),
5773 ])
5774 .backend(TransportBackend::Sockudo)
5775 .build()
5776 .unwrap();
5777 let error = WebSocketClient::builder()
5778 .config(config)
5779 .message_handler(Arc::new(|_| {}))
5780 .connect()
5781 .await
5782 .unwrap_err();
5783 let request = String::from_utf8(peer.await.unwrap()).unwrap();
5784 let messages = capture.messages();
5785
5786 assert!(matches!(error, TransportError::UpgradeRejected(503)));
5787
5788 for secret in [
5789 "TRACE_KEY_SECRET",
5790 "TRACE_SIGNATURE_SECRET",
5791 "TRACE_PATH_SECRET",
5792 "TRACE_QUERY_SECRET",
5793 ] {
5794 assert!(request.contains(secret));
5795 assert!(
5796 messages
5797 .iter()
5798 .all(|(_, message)| !message.contains(secret)),
5799 "{messages:?}"
5800 );
5801 }
5802
5803 assert!(
5804 messages
5805 .iter()
5806 .any(|(level, message)| *level == log::Level::Trace
5807 && message.contains("retries exhausted after 1 attempts")),
5808 "{messages:?}"
5809 );
5810 }
5811
5812 struct CondvarReleaseGuard<'a> {
5813 release: &'a (Mutex<bool>, Condvar),
5814 }
5815
5816 impl<'a> CondvarReleaseGuard<'a> {
5817 fn new(release: &'a (Mutex<bool>, Condvar)) -> Self {
5818 Self { release }
5819 }
5820
5821 fn release(&self) {
5822 let (lock, condvar) = self.release;
5823 let mut released = lock.lock();
5824 *released = true;
5825 condvar.notify_all();
5826 }
5827 }
5828
5829 impl Drop for CondvarReleaseGuard<'_> {
5830 fn drop(&mut self) {
5831 self.release();
5832 }
5833 }
5834
5835 async fn recv_rendezvous<T: Send + 'static>(
5836 receiver: std::sync::mpsc::Receiver<T>,
5837 name: &'static str,
5838 ) -> T {
5839 let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
5840
5841 match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
5842 Ok(Ok(Ok(value))) => value,
5843 Ok(Ok(Err(e))) => {
5844 panic!("{name} did not arrive within the test timeout: {e}")
5845 }
5846 Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
5847 Err(e) => panic!("{name} receive task did not finish: {e}"),
5848 }
5849 }
5850
5851 async fn await_task_termination(task: tokio::task::JoinHandle<()>, name: &'static str) {
5852 match tokio::time::timeout(TEST_TIMEOUT, task).await {
5853 Ok(Ok(())) => {}
5854 Ok(Err(e)) if e.is_cancelled() => {}
5855 Ok(Err(e)) => panic!("{name} failed: {e}"),
5856 Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
5857 }
5858 }
5859
5860 fn reconnect_test_config(port: u16) -> WebSocketConfig {
5861 WebSocketConfig {
5862 url: format!("ws://127.0.0.1:{port}"),
5863 headers: vec![],
5864 heartbeat_interval_secs: None,
5865 heartbeat_payload: None,
5866 connect_timeout_ms: Some(1_000),
5867 reconnect_delay_initial_ms: None,
5868 reconnect_delay_max_ms: None,
5869 reconnect_backoff_factor: None,
5870 reconnect_jitter_ms: None,
5871 reconnect_max_attempts: None,
5872 heartbeat_timeout_secs: None,
5873 idle_timeout_ms: None,
5874 writer_capacity: None,
5875 backend: TransportBackend::Tungstenite,
5876 proxy_url: None,
5877 max_message_size_bytes: None,
5878 max_frame_size_bytes: None,
5879 }
5880 }
5881
5882 fn initial_connect_retry_policy(
5883 max_attempts: u32,
5884 initial_delay_ms: u64,
5885 ) -> InitialConnectRetryPolicy {
5886 InitialConnectRetryPolicy {
5887 max_attempts: std::num::NonZeroU32::new(max_attempts).unwrap(),
5888 delay_initial: Duration::from_millis(initial_delay_ms),
5889 delay_max: Duration::from_millis(initial_delay_ms),
5890 backoff_factor: 2.0,
5891 jitter_ms: 0,
5892 }
5893 }
5894
5895 #[rstest]
5896 #[case(TransportError::UpgradeRejected(408), true)]
5897 #[case(TransportError::UpgradeRejected(425), true)]
5898 #[case(TransportError::UpgradeRejected(429), true)]
5899 #[case(TransportError::UpgradeRejected(500), true)]
5900 #[case(TransportError::UpgradeRejected(599), true)]
5901 #[case(TransportError::UpgradeRejected(404), false)]
5902 #[case(TransportError::UpgradeRejected(600), false)]
5903 #[case(TransportError::ProxyConnectRejected(429), true)]
5904 #[case(TransportError::ProxyConnectRejected(407), false)]
5905 #[case(TransportError::ConnectionClosed, true)]
5906 #[case(
5907 TransportError::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
5908 true
5909 )]
5910 #[case(
5911 TransportError::Io(std::io::Error::from(std::io::ErrorKind::Interrupted)),
5912 true
5913 )]
5914 #[case(
5915 TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidInput)),
5916 false
5917 )]
5918 #[case(
5919 TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidData)),
5920 false
5921 )]
5922 #[case(
5923 TransportError::Io(std::io::Error::from(std::io::ErrorKind::Unsupported)),
5924 false
5925 )]
5926 #[case(
5927 TransportError::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
5928 false
5929 )]
5930 #[case(TransportError::Handshake("malformed".to_string()), false)]
5931 fn initial_connect_error_classification(#[case] error: TransportError, #[case] expected: bool) {
5932 assert_eq!(is_retryable_initial_connect_error(&error), expected);
5933 }
5934
5935 async fn rejected_upgrade_attempts(
5936 status: u16,
5937 backend: TransportBackend,
5938 retry_policy: Option<InitialConnectRetryPolicy>,
5939 ) -> u64 {
5940 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5941 let port = listener.local_addr().unwrap().port();
5942 let accepted = Arc::new(AtomicU64::new(0));
5943 let accepted_server = Arc::clone(&accepted);
5944
5945 let server = tokio::spawn(async move {
5946 loop {
5947 let (mut stream, _) = listener.accept().await.unwrap();
5948 accepted_server.fetch_add(1, Ordering::SeqCst);
5949 let mut request = Vec::new();
5950
5951 loop {
5952 let mut chunk = [0; 1024];
5953 let n = stream.read(&mut chunk).await.unwrap();
5954 if n == 0 {
5955 break;
5956 }
5957
5958 request.extend_from_slice(&chunk[..n]);
5959 if request.windows(4).any(|window| window == b"\r\n\r\n") {
5960 break;
5961 }
5962 }
5963
5964 stream
5965 .write_all(format!("HTTP/1.1 {status}\r\n\r\n").as_bytes())
5966 .await
5967 .unwrap();
5968 }
5969 });
5970
5971 let (handler, _rx) = channel_message_handler();
5972 let mut config = reconnect_test_config(port);
5973 config.backend = backend;
5974 WebSocketClient::builder()
5975 .config(config)
5976 .message_handler(handler)
5977 .maybe_initial_connect_retry_policy(retry_policy)
5978 .connect()
5979 .await
5980 .expect_err("server always rejects the upgrade");
5981
5982 let attempts = accepted.load(Ordering::SeqCst);
5983 server.abort();
5984 attempts
5985 }
5986
5987 async fn closed_proxy_attempts() -> u64 {
5988 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5989 let proxy_addr = listener.local_addr().unwrap();
5990 let accepted = Arc::new(AtomicU64::new(0));
5991 let accepted_server = Arc::clone(&accepted);
5992
5993 let server = tokio::spawn(async move {
5994 loop {
5995 let (mut stream, _) = listener.accept().await.unwrap();
5996 accepted_server.fetch_add(1, Ordering::SeqCst);
5997 let mut request = [0; 1024];
5998 let _ = stream.read(&mut request).await.unwrap();
5999 }
6000 });
6001
6002 let (handler, _rx) = channel_message_handler();
6003 let mut config = reconnect_test_config(9);
6004 config.proxy_url = Some(format!("http://{proxy_addr}"));
6005 WebSocketClient::builder()
6006 .config(config)
6007 .message_handler(handler)
6008 .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6009 .connect()
6010 .await
6011 .expect_err("proxy always closes before its CONNECT response");
6012
6013 let attempts = accepted.load(Ordering::SeqCst);
6014 server.abort();
6015 attempts
6016 }
6017
6018 #[rstest]
6019 #[tokio::test]
6020 async fn initial_connect_cancellation_interrupts_in_flight_handshake() {
6021 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6022 let port = listener.local_addr().unwrap().port();
6023 let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
6024
6025 let server = tokio::spawn(async move {
6026 let (_stream, _) = listener.accept().await.unwrap();
6027 let _ = accepted_tx.send(());
6028 std::future::pending::<()>().await;
6029 });
6030
6031 let token = CancellationToken::new();
6032 let (handler, _rx) = channel_message_handler();
6033 let mut config = reconnect_test_config(port);
6034 config.connect_timeout_ms = Some(10_000);
6035 let connect = WebSocketClient::builder()
6036 .config(config)
6037 .message_handler(handler)
6038 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6039 .cancellation_token(token.clone())
6040 .connect();
6041 tokio::pin!(connect);
6042
6043 tokio::select! {
6044 biased;
6045 result = &mut connect => panic!("connect completed before cancellation: {result:?}"),
6046 result = accepted_rx => result.unwrap(),
6047 }
6048 token.cancel();
6049
6050 let err = tokio::time::timeout(Duration::from_millis(250), connect)
6051 .await
6052 .expect("cancellation should interrupt the handshake")
6053 .expect_err("cancelled connect should fail");
6054 assert!(
6055 matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::Interrupted)
6056 );
6057 server.abort();
6058 }
6059
6060 #[rstest]
6061 #[tokio::test]
6062 async fn timed_out_initial_connect_attempt_is_retried() {
6063 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6064 let port = listener.local_addr().unwrap().port();
6065 let accepted = Arc::new(AtomicU64::new(0));
6066 let accepted_server = Arc::clone(&accepted);
6067
6068 let server = tokio::spawn(async move {
6071 let mut held = Vec::new();
6072
6073 loop {
6074 let (stream, _) = listener.accept().await.unwrap();
6075 accepted_server.fetch_add(1, Ordering::SeqCst);
6076 held.push(stream);
6077 }
6078 });
6079
6080 let (handler, _rx) = channel_message_handler();
6081 let mut config = reconnect_test_config(port);
6082 config.connect_timeout_ms = Some(50);
6083 let err = WebSocketClient::builder()
6084 .config(config)
6085 .message_handler(handler)
6086 .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6087 .connect()
6088 .await
6089 .expect_err("every attempt times out");
6090
6091 assert!(
6092 matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::TimedOut)
6093 );
6094
6095 let counted = tokio::time::timeout(Duration::from_secs(1), async {
6100 while accepted.load(Ordering::SeqCst) < 3 {
6101 tokio::time::sleep(Duration::from_millis(5)).await;
6102 }
6103 })
6104 .await;
6105
6106 assert!(
6107 counted.is_ok(),
6108 "expected 3 connection attempts, observed {}",
6109 accepted.load(Ordering::SeqCst)
6110 );
6111 assert_eq!(accepted.load(Ordering::SeqCst), 3);
6114 server.abort();
6115 }
6116
6117 #[rstest]
6118 #[case::too_many_requests(429)]
6119 #[case::server_error(503)]
6120 #[tokio::test]
6121 async fn transient_upgrade_rejection_is_retried(#[case] status: u16) {
6122 assert_eq!(
6123 rejected_upgrade_attempts(
6124 status,
6125 TransportBackend::Tungstenite,
6126 Some(initial_connect_retry_policy(3, 1)),
6127 )
6128 .await,
6129 3
6130 );
6131 }
6132
6133 #[rstest]
6134 #[tokio::test]
6135 async fn initial_connect_without_retry_policy_makes_one_attempt() {
6136 assert_eq!(
6137 rejected_upgrade_attempts(503, TransportBackend::Tungstenite, None).await,
6138 1
6139 );
6140 }
6141
6142 #[cfg(feature = "transport-sockudo")]
6143 #[rstest]
6144 #[tokio::test]
6145 async fn sockudo_too_many_requests_upgrade_rejection_is_retried() {
6146 assert_eq!(
6147 rejected_upgrade_attempts(
6148 429,
6149 TransportBackend::Sockudo,
6150 Some(initial_connect_retry_policy(3, 1)),
6151 )
6152 .await,
6153 3
6154 );
6155 }
6156
6157 #[rstest]
6158 #[tokio::test]
6159 async fn permanent_upgrade_rejection_is_not_retried() {
6160 assert_eq!(
6161 rejected_upgrade_attempts(
6162 404,
6163 TransportBackend::Tungstenite,
6164 Some(initial_connect_retry_policy(3, 1)),
6165 )
6166 .await,
6167 1
6168 );
6169 }
6170
6171 #[rstest]
6172 #[tokio::test]
6173 async fn proxy_close_before_connect_response_is_retried() {
6174 assert_eq!(closed_proxy_attempts().await, 3);
6175 }
6176
6177 #[rstest]
6178 #[tokio::test]
6179 async fn initial_connect_cancellation_interrupts_retry_backoff() {
6180 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6181 let port = listener.local_addr().unwrap().port();
6182 let accepted = Arc::new(AtomicU64::new(0));
6183 let accepted_server = Arc::clone(&accepted);
6184 let (rejected_tx, rejected_rx) = oneshot::channel();
6185 let server = tokio::spawn(async move {
6186 let mut rejected_tx = Some(rejected_tx);
6187
6188 loop {
6189 let (mut stream, _) = listener.accept().await.unwrap();
6190 accepted_server.fetch_add(1, Ordering::SeqCst);
6191 let mut request = [0; 1024];
6192 let _ = stream.read(&mut request).await.unwrap();
6193 stream
6194 .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
6195 .await
6196 .unwrap();
6197
6198 if let Some(rejected_tx) = rejected_tx.take() {
6199 rejected_tx.send(()).unwrap();
6200 }
6201 }
6202 });
6203
6204 let token = CancellationToken::new();
6205 let (handler, _rx) = channel_message_handler();
6206 let connect = WebSocketClient::builder()
6207 .config(reconnect_test_config(port))
6208 .message_handler(handler)
6209 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6210 .cancellation_token(token.clone())
6211 .connect();
6212 tokio::pin!(connect);
6213
6214 tokio::select! {
6215 biased;
6216 result = &mut connect => panic!("connect completed before backoff: {result:?}"),
6217 result = rejected_rx => result.unwrap(),
6218 }
6219 assert!(futures_util::poll!(&mut connect).is_pending());
6220 token.cancel();
6221
6222 let error = tokio::time::timeout(Duration::from_millis(250), connect)
6223 .await
6224 .expect("cancellation should interrupt retry backoff")
6225 .expect_err("cancelled connect should fail");
6226
6227 assert!(
6228 matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6229 );
6230 assert_eq!(accepted.load(Ordering::SeqCst), 1);
6231 server.abort();
6232 }
6233
6234 #[rstest]
6235 #[tokio::test]
6236 async fn permanent_initial_connect_error_does_not_wait_for_retry_ladder() {
6237 let (handler, _rx) = channel_message_handler();
6238 let mut config = reconnect_test_config(1);
6239 config.url = "not a websocket URL".to_string();
6240 let connect = WebSocketClient::builder()
6241 .config(config)
6242 .message_handler(handler)
6243 .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6244 .connect();
6245
6246 let err = tokio::time::timeout(Duration::from_millis(250), connect)
6247 .await
6248 .expect("permanent error should not enter retry backoff")
6249 .expect_err("invalid URL should fail");
6250 assert!(matches!(
6251 err,
6252 TransportError::InvalidUrl(_) | TransportError::Handshake(_)
6253 ));
6254 }
6255
6256 #[rstest]
6257 #[tokio::test(start_paused = true)]
6258 async fn connection_rate_limit_gates_initial_connect_and_reconnect() {
6259 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6260 let port = listener.local_addr().unwrap().port();
6261 let server = tokio::spawn(async move {
6262 let (stream, _) = listener.accept().await.unwrap();
6263 let _first = accept_async(stream).await.unwrap();
6264 let (stream, _) = listener.accept().await.unwrap();
6265 let _second = accept_async(stream).await.unwrap();
6266 std::future::pending::<()>().await;
6267 });
6268
6269 let key = Ustr::from("connection-attempt");
6270
6271 let limiter = Arc::new(RateLimiter::new_with_quota(
6272 None,
6273 vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6274 ));
6275 limiter.await_keys_ready(Some(&[key])).await;
6276
6277 let rate_limit = ConnectionRateLimit {
6278 limiter,
6279 keys: Arc::from([key]),
6280 };
6281
6282 let (handler, _rx) = channel_message_handler();
6283 let mut config = reconnect_test_config(port);
6284 config.connect_timeout_ms = Some(10_000);
6285 let initial = WebSocketClientInner::connect_url_with_handler(
6286 config,
6287 Some(IncomingHandler::Message(handler)),
6288 None,
6289 None,
6290 None,
6291 Some(rate_limit),
6292 InitialConnectOptions::default(),
6293 );
6294 tokio::pin!(initial);
6295 assert!(futures_util::poll!(&mut initial).is_pending());
6296 tokio::time::advance(Duration::from_secs(1)).await;
6297 tokio::task::yield_now().await;
6298
6299 tokio::time::resume();
6302 let mut inner = initial.await.unwrap();
6303 tokio::time::pause();
6304
6305 inner
6306 .connection_mode
6307 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6308 let reconnect = inner.reconnect_with_outcome();
6309 tokio::pin!(reconnect);
6310 assert!(futures_util::poll!(&mut reconnect).is_pending());
6311 tokio::time::advance(Duration::from_secs(1)).await;
6312 tokio::task::yield_now().await;
6313 tokio::time::resume();
6314 assert_eq!(reconnect.await.unwrap(), ReconnectOutcome::Reconnected);
6315
6316 server.abort();
6317 }
6318
6319 #[rstest]
6320 #[tokio::test(start_paused = true)]
6321 async fn initial_connect_cancellation_interrupts_connection_rate_limit_wait() {
6322 let key = Ustr::from("connection-attempt");
6323
6324 let limiter = Arc::new(RateLimiter::new_with_quota(
6325 None,
6326 vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6327 ));
6328 limiter.await_keys_ready(Some(&[key])).await;
6329
6330 let rate_limit = ConnectionRateLimit {
6331 limiter,
6332 keys: Arc::from([key]),
6333 };
6334
6335 let token = CancellationToken::new();
6336 let (handler, _rx) = channel_message_handler();
6337
6338 let connect = WebSocketClientInner::connect_url_with_handler(
6339 reconnect_test_config(1),
6340 Some(IncomingHandler::Message(handler)),
6341 None,
6342 None,
6343 None,
6344 Some(rate_limit),
6345 InitialConnectOptions {
6346 retry_policy: None,
6347 cancellation_token: Some(token.clone()),
6348 },
6349 );
6350
6351 tokio::pin!(connect);
6352 assert!(futures_util::poll!(&mut connect).is_pending());
6353
6354 token.cancel();
6355 let error = connect.await.expect_err("cancelled connect should fail");
6356
6357 assert!(
6358 matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6359 );
6360 }
6361
6362 #[rstest]
6363 #[tokio::test]
6364 async fn test_reconnect_outcome_is_aborted_before_connect() {
6365 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6366 let port = listener.local_addr().unwrap().port();
6367 let server = tokio::spawn(async move {
6368 let (stream, _) = listener.accept().await.unwrap();
6369 let _websocket = accept_async(stream).await.unwrap();
6370 std::future::pending::<()>().await;
6371 });
6372
6373 let (handler, _rx) = channel_message_handler();
6374 let mut inner =
6375 WebSocketClientInner::connect_url(reconnect_test_config(port), Some(handler), None)
6376 .await
6377 .unwrap();
6378 inner
6379 .connection_mode
6380 .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
6381
6382 let outcome = inner.reconnect_with_outcome().await.unwrap();
6383
6384 assert_eq!(outcome, ReconnectOutcome::Aborted);
6385 server.abort();
6386 }
6387
6388 #[rstest]
6389 #[tokio::test]
6390 async fn test_stream_reconnect_outcome_is_aborted_and_notifies_closed() {
6391 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6392 let port = listener.local_addr().unwrap().port();
6393 let server = tokio::spawn(async move {
6394 let (stream, _) = listener.accept().await.unwrap();
6395 let _websocket = accept_async(stream).await.unwrap();
6396 std::future::pending::<()>().await;
6397 });
6398
6399 let mut inner = WebSocketClientInner::connect_url(reconnect_test_config(port), None, None)
6400 .await
6401 .unwrap();
6402 let state_notify = Arc::clone(&inner.state_notify);
6403 let mut notified = std::pin::pin!(state_notify.notified());
6404 notified.as_mut().enable();
6405
6406 let outcome = inner.reconnect_with_outcome().await.unwrap();
6407
6408 assert_eq!(outcome, ReconnectOutcome::Aborted);
6409 assert_eq!(
6410 ConnectionMode::from_atomic(&inner.connection_mode),
6411 ConnectionMode::Closed
6412 );
6413 tokio::time::timeout(TEST_TIMEOUT, notified)
6414 .await
6415 .expect("stream close notification was not published");
6416 server.abort();
6417 }
6418
6419 #[rstest]
6420 #[tokio::test]
6421 async fn test_reconnect_outcome_is_reconnected_with_handler() {
6422 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6423 let port = listener.local_addr().unwrap().port();
6424 let server = tokio::spawn(async move {
6425 let (stream, _) = listener.accept().await.unwrap();
6426 let _first = accept_async(stream).await.unwrap();
6427 let (stream, _) = listener.accept().await.unwrap();
6428 let mut second = accept_async(stream).await.unwrap();
6429 second
6430 .send(WsMessage::Text("replacement".into()))
6431 .await
6432 .unwrap();
6433 std::future::pending::<()>().await;
6434 });
6435
6436 let (epoch_handler, mut epoch_rx) = channel_epoch_message_handler();
6437 let mut inner = WebSocketClientInner::connect_url_with_handler(
6438 reconnect_test_config(port),
6439 Some(IncomingHandler::Epoch(epoch_handler)),
6440 None,
6441 None,
6442 None,
6443 None,
6444 InitialConnectOptions::default(),
6445 )
6446 .await
6447 .unwrap();
6448 inner
6449 .connection_mode
6450 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6451
6452 let outcome = inner.reconnect_with_outcome().await.unwrap();
6453
6454 assert_eq!(outcome, ReconnectOutcome::Reconnected);
6455 assert_eq!(
6456 ConnectionMode::from_atomic(&inner.connection_mode),
6457 ConnectionMode::Active
6458 );
6459 let (epoch, message) = tokio::time::timeout(TEST_TIMEOUT, epoch_rx.recv())
6460 .await
6461 .expect("replacement epoch message was not delivered")
6462 .expect("epoch handler channel closed");
6463 assert_eq!(epoch, 1);
6464 assert_eq!(message, WsMessage::Text("replacement".into()));
6465 server.abort();
6466 }
6467
6468 #[rstest]
6469 #[tokio::test]
6470 async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
6471 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6472 let port = listener.local_addr().unwrap().port();
6473 let server = tokio::spawn(async move {
6474 let (stream, _) = listener.accept().await.unwrap();
6475 let _websocket = accept_async(stream).await.unwrap();
6476 std::future::pending::<()>().await;
6477 });
6478
6479 let mut config = reconnect_test_config(port);
6480 config.heartbeat_interval_secs = Some(60);
6481 let (handler, _handler_rx) = channel_message_handler();
6482 let inner = WebSocketClientInner::connect_url(config, Some(handler), None)
6483 .await
6484 .unwrap();
6485 let read_fence = inner
6486 .read_fence
6487 .clone()
6488 .expect("read fence should exist in handler mode");
6489 let read_abort = inner
6490 .read_task
6491 .as_ref()
6492 .expect("read task should be spawned in handler mode")
6493 .abort_handle();
6494 let write_abort = inner.write_task.abort_handle();
6495 let heartbeat_abort = inner
6496 .heartbeat_task
6497 .as_ref()
6498 .expect("heartbeat task should be spawned for a configured heartbeat")
6499 .abort_handle();
6500
6501 assert!(read_fence.is_valid(), "read fence should start valid");
6502 assert!(
6503 !read_abort.is_finished(),
6504 "read task should be running before drop"
6505 );
6506 assert!(
6507 !write_abort.is_finished(),
6508 "write task should be running before drop"
6509 );
6510 assert!(
6511 !heartbeat_abort.is_finished(),
6512 "heartbeat task should be running before drop"
6513 );
6514
6515 drop(inner);
6516 wait_until_async(
6517 || async {
6518 read_abort.is_finished()
6519 && write_abort.is_finished()
6520 && heartbeat_abort.is_finished()
6521 },
6522 TEST_TIMEOUT,
6523 )
6524 .await;
6525
6526 assert!(!read_fence.is_valid(), "read fence was not invalidated");
6527 assert!(read_abort.is_finished(), "read task was not aborted");
6528 assert!(write_abort.is_finished(), "write task was not aborted");
6529 assert!(
6530 heartbeat_abort.is_finished(),
6531 "heartbeat task was not aborted"
6532 );
6533 server.abort();
6534 }
6535
6536 struct RecordingServer {
6537 task: JoinHandle<()>,
6538 port: u16,
6539 messages: Arc<tokio::sync::Mutex<Vec<String>>>,
6540 binary_frames: Arc<tokio::sync::Mutex<Vec<Vec<u8>>>>,
6541 control_frames: Arc<tokio::sync::Mutex<Vec<String>>>,
6542 connections: Arc<AtomicUsize>,
6543 }
6544
6545 #[cfg(feature = "transport-sockudo")]
6546 async fn read_http_request<S>(stream: &mut S) -> Vec<u8>
6547 where
6548 S: AsyncRead + Unpin,
6549 {
6550 let mut buf = Vec::new();
6551 let mut chunk = [0u8; 256];
6552 loop {
6553 let n = stream.read(&mut chunk).await.unwrap();
6554 assert!(n > 0, "HTTP request closed before headers completed");
6555 buf.extend_from_slice(&chunk[..n]);
6556 if buf.windows(4).any(|window| window == b"\r\n\r\n") {
6557 return buf;
6558 }
6559 }
6560 }
6561
6562 #[cfg(feature = "transport-sockudo")]
6563 fn extract_header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
6564 request.lines().find_map(|line| {
6565 let (header_name, header_value) = line.split_once(':')?;
6566 if header_name.eq_ignore_ascii_case(name) {
6567 Some(header_value.trim())
6568 } else {
6569 None
6570 }
6571 })
6572 }
6573
6574 #[cfg(feature = "transport-sockudo")]
6575 #[derive(Debug, Clone)]
6576 struct HeaderAssertCallback {
6577 key: String,
6578 value: HeaderValue,
6579 }
6580
6581 #[cfg(feature = "transport-sockudo")]
6582 impl Callback for HeaderAssertCallback {
6583 #[expect(
6584 clippy::panic_in_result_fn,
6585 reason = "assertion failures should fail the test"
6586 )]
6587 fn on_request(
6588 self,
6589 request: &server::Request,
6590 response: server::Response,
6591 ) -> Result<server::Response, server::ErrorResponse> {
6592 assert_eq!(request.headers().get(&self.key), Some(&self.value));
6593 Ok(response)
6594 }
6595 }
6596
6597 impl RecordingServer {
6598 async fn setup() -> Self {
6599 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6600 let port = listener.local_addr().unwrap().port();
6601 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6602 let messages_clone = Arc::clone(&messages);
6603 let binary_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6604 let binary_frames_clone = Arc::clone(&binary_frames);
6605 let control_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6606 let control_frames_clone = Arc::clone(&control_frames);
6607 let connections = Arc::new(AtomicUsize::new(0));
6608 let connections_clone = Arc::clone(&connections);
6609
6610 let task = task::spawn(async move {
6611 loop {
6612 let (stream, _) = listener.accept().await.unwrap();
6613 let mut websocket = accept_async(stream).await.unwrap();
6614 connections_clone.fetch_add(1, Ordering::SeqCst);
6615 let messages = Arc::clone(&messages_clone);
6616 let binary_frames = Arc::clone(&binary_frames_clone);
6617 let control_frames = Arc::clone(&control_frames_clone);
6618
6619 task::spawn(async move {
6620 while let Some(Ok(msg)) = websocket.next().await {
6621 match msg {
6622 WsMessage::Text(text) => {
6623 messages.lock().await.push(text.to_string());
6624 }
6625 WsMessage::Binary(data) => {
6626 binary_frames.lock().await.push(data.to_vec());
6627 }
6628 WsMessage::Pong(data) => {
6629 control_frames
6630 .lock()
6631 .await
6632 .push(format!("pong:{}", String::from_utf8_lossy(&data)));
6633 }
6634 WsMessage::Close(_) => {
6635 let _ = websocket.close(None).await;
6636 break;
6637 }
6638 _ => {}
6639 }
6640 }
6641 });
6642 }
6643 });
6644
6645 Self {
6646 task,
6647 port,
6648 messages,
6649 binary_frames,
6650 control_frames,
6651 connections,
6652 }
6653 }
6654
6655 async fn messages(&self) -> Vec<String> {
6656 self.messages.lock().await.clone()
6657 }
6658
6659 async fn binary_frames(&self) -> Vec<Vec<u8>> {
6660 self.binary_frames.lock().await.clone()
6661 }
6662
6663 async fn control_frames(&self) -> Vec<String> {
6664 self.control_frames.lock().await.clone()
6665 }
6666
6667 async fn wait_for_connections(&self, expected: usize) {
6668 wait_until_async(
6669 || async { self.connections.load(Ordering::SeqCst) == expected },
6670 TEST_TIMEOUT,
6671 )
6672 .await;
6673 }
6674 }
6675
6676 impl Drop for RecordingServer {
6677 fn drop(&mut self) {
6678 self.task.abort();
6679 }
6680 }
6681
6682 #[rstest]
6683 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6684 async fn test_manual_reconnect_waits_for_slow_loss_callback_and_new_auth() {
6685 let server = RecordingServer::setup().await;
6686 let tracker = AuthTracker::new();
6687 let _initial_auth = tracker.begin();
6688 tracker.succeed();
6689 let states = Arc::new(Mutex::new(Vec::new()));
6690 let states_callback = Arc::clone(&states);
6691 let auth_at_loss = Arc::new(Mutex::new(Vec::new()));
6692 let auth_at_loss_callback = Arc::clone(&auth_at_loss);
6693 let tracker_callback = tracker.clone();
6694 let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6695 let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6696 let callback_release_clone = Arc::clone(&callback_release);
6697 let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6698
6699 let sink = SocketStateSink::new(move |state| {
6700 states_callback.lock().push(state);
6701 if state == SocketState::Disconnected {
6702 auth_at_loss_callback
6703 .lock()
6704 .push(tracker_callback.auth_state());
6705 callback_entered_tx.send(()).unwrap();
6706 let (lock, condvar) = callback_release_clone.as_ref();
6707 let mut released = lock.lock();
6708 while !*released {
6709 condvar.wait(&mut released);
6710 }
6711 }
6712 });
6713
6714 let (handler, mut handler_rx) = channel_message_handler();
6715 let client = WebSocketClient::builder()
6716 .config(reconnect_test_config(server.port))
6717 .message_handler(handler)
6718 .state_sink(sink)
6719 .connect()
6720 .await
6721 .unwrap();
6722 client.set_auth_tracker(tracker.clone(), true);
6723 server.wait_for_connections(1).await;
6724
6725 let handle = client.reconnect_handle();
6726 let (request_tx, request_rx) = std::sync::mpsc::channel();
6727 let request_thread = std::thread::spawn(move || {
6728 request_tx.send(handle.request_reconnect()).unwrap();
6729 });
6730
6731 recv_rendezvous(callback_entered_rx, "slow reconnect callback entry").await;
6732 client
6733 .writer_tx
6734 .send(WriterCommand::Send(Message::text("buffered")))
6735 .unwrap();
6736 server.wait_for_connections(2).await;
6737 tokio::time::sleep(Duration::from_millis(250)).await;
6738
6739 assert_eq!(client.connection_mode(), ConnectionMode::Reconnect);
6740 assert!(!client.reconnect_published.load(Ordering::SeqCst));
6741 assert_eq!(tracker.auth_state(), AuthState::Unauthenticated);
6742 assert_eq!(*auth_at_loss.lock(), vec![AuthState::Unauthenticated]);
6743 assert_eq!(
6744 *states.lock(),
6745 vec![SocketState::Connected, SocketState::Disconnected]
6746 );
6747 assert!(server.messages().await.is_empty());
6748
6749 callback_release_guard.release();
6750 assert_eq!(
6751 recv_rendezvous(request_rx, "manual reconnect result").await,
6752 ReconnectRequestOutcome::Accepted
6753 );
6754 request_thread.join().unwrap();
6755 wait_until_async(|| async { client.is_active() }, TEST_TIMEOUT).await;
6756 wait_until_async(
6757 || {
6758 let states = Arc::clone(&states);
6759 async move { states.lock().len() == 3 }
6760 },
6761 TEST_TIMEOUT,
6762 )
6763 .await;
6764
6765 let notification = tokio::time::timeout(TEST_TIMEOUT, handler_rx.recv())
6766 .await
6767 .expect("reconnect notification was not delivered")
6768 .expect("handler channel closed");
6769 assert_eq!(notification, WsMessage::Text(RECONNECTED.into()));
6770 assert!(handler_rx.try_recv().is_err());
6771 tokio::time::sleep(Duration::from_millis(200)).await;
6772 assert!(server.messages().await.is_empty());
6773
6774 let _replacement_auth = tracker.begin();
6777 client
6778 .send_bytes_on_connection(b"auth".to_vec(), None, client.connection_epoch())
6779 .await
6780 .unwrap();
6781 wait_until_async(
6782 || {
6783 let binary_frames = Arc::clone(&server.binary_frames);
6784 async move { binary_frames.lock().await.len() == 1 }
6785 },
6786 TEST_TIMEOUT,
6787 )
6788 .await;
6789 assert_eq!(server.binary_frames().await, vec![b"auth".to_vec()]);
6790 assert!(server.messages().await.is_empty());
6791
6792 tracker.succeed();
6793 wait_until_async(
6794 || {
6795 let messages = Arc::clone(&server.messages);
6796 async move { messages.lock().await.len() == 1 }
6797 },
6798 TEST_TIMEOUT,
6799 )
6800 .await;
6801
6802 client.send_text("live".into(), None).await.unwrap();
6803 wait_until_async(
6804 || {
6805 let messages = Arc::clone(&server.messages);
6806 async move { messages.lock().await.len() == 2 }
6807 },
6808 TEST_TIMEOUT,
6809 )
6810 .await;
6811
6812 assert_eq!(server.messages().await, vec!["buffered", "live"]);
6813 assert_eq!(server.connections.load(Ordering::SeqCst), 2);
6814 assert_eq!(
6815 *states.lock(),
6816 vec![
6817 SocketState::Connected,
6818 SocketState::Disconnected,
6819 SocketState::Connected,
6820 ]
6821 );
6822
6823 client.disconnect().await;
6824 }
6825
6826 #[rstest]
6827 #[tokio::test]
6828 async fn test_reconnect_handle_is_closed_after_client_drop() {
6829 let server = RecordingServer::setup().await;
6830 let (handler, _handler_rx) = channel_message_handler();
6831 let client = WebSocketClient::builder()
6832 .config(reconnect_test_config(server.port))
6833 .message_handler(handler)
6834 .connect()
6835 .await
6836 .unwrap();
6837 let tracker = AuthTracker::new();
6838 let pending_auth = tracker.begin();
6839 client.set_auth_tracker(tracker.clone(), true);
6840 let controller_abort = client.controller_task.abort_handle();
6841 let handle = client.reconnect_handle();
6842
6843 drop(client);
6844 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6845
6846 assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
6847 assert_eq!(tracker.auth_state(), AuthState::Failed);
6848 assert_eq!(
6849 tokio::time::timeout(TEST_TIMEOUT, pending_auth)
6850 .await
6851 .expect("client drop should resolve pending authentication")
6852 .expect("authentication sender should report its terminal result"),
6853 Err("WebSocket client closed".to_string())
6854 );
6855 }
6856
6857 #[rstest]
6858 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6859 async fn test_concurrent_drop_closes_accepted_reconnect() {
6860 let server = RecordingServer::setup().await;
6861 let tracker = AuthTracker::new();
6862 let _initial_auth = tracker.begin();
6863 tracker.succeed();
6864 let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6865 let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6866 let callback_release_clone = Arc::clone(&callback_release);
6867 let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6868 let states = Arc::new(Mutex::new(Vec::new()));
6869 let states_callback = Arc::clone(&states);
6870 let sink = SocketStateSink::new(move |state| {
6871 states_callback.lock().push(state);
6872 if state == SocketState::Disconnected {
6873 callback_entered_tx.send(()).unwrap();
6874 let (lock, condvar) = callback_release_clone.as_ref();
6875 let mut released = lock.lock();
6876 while !*released {
6877 condvar.wait(&mut released);
6878 }
6879 }
6880 });
6881
6882 let (handler, _handler_rx) = channel_message_handler();
6883 let client = WebSocketClient::builder()
6884 .config(reconnect_test_config(server.port))
6885 .message_handler(handler)
6886 .state_sink(sink)
6887 .connect()
6888 .await
6889 .unwrap();
6890 client.set_auth_tracker(tracker.clone(), true);
6891 let controller_abort = client.controller_task.abort_handle();
6892 let connection_mode = Arc::clone(&client.connection_mode);
6893 let handle = client.reconnect_handle();
6894 let surviving_handle = handle.clone();
6895 let (request_tx, request_rx) = std::sync::mpsc::channel();
6896 let request_thread = std::thread::spawn(move || {
6897 request_tx.send(handle.request_reconnect()).unwrap();
6898 });
6899
6900 recv_rendezvous(callback_entered_rx, "concurrent drop callback entry").await;
6901 drop(client);
6902 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6903 assert_eq!(
6904 ConnectionMode::from_atomic(&connection_mode),
6905 ConnectionMode::Closed
6906 );
6907 assert_eq!(tracker.auth_state(), AuthState::Failed);
6908
6909 callback_release_guard.release();
6910 assert_eq!(
6911 recv_rendezvous(request_rx, "concurrent drop reconnect result").await,
6912 ReconnectRequestOutcome::Accepted
6913 );
6914 request_thread.join().unwrap();
6915 assert_eq!(
6916 surviving_handle.request_reconnect(),
6917 ReconnectRequestOutcome::Closed
6918 );
6919 assert_eq!(
6920 *states.lock(),
6921 vec![SocketState::Connected, SocketState::Disconnected]
6922 );
6923 }
6924
6925 #[rstest]
6926 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6927 async fn test_reconnect_callback_can_drop_client() {
6928 let server = RecordingServer::setup().await;
6929 let client_slot = Arc::new(Mutex::new(None::<WebSocketClient>));
6930 let client_slot_callback = Arc::clone(&client_slot);
6931 let states = Arc::new(Mutex::new(Vec::new()));
6932 let states_callback = Arc::clone(&states);
6933 let sink = SocketStateSink::new(move |state| {
6934 states_callback.lock().push(state);
6935 if state == SocketState::Disconnected {
6936 drop(client_slot_callback.lock().take());
6937 }
6938 });
6939
6940 let (handler, _handler_rx) = channel_message_handler();
6941 let client = WebSocketClient::builder()
6942 .config(reconnect_test_config(server.port))
6943 .message_handler(handler)
6944 .state_sink(sink)
6945 .connect()
6946 .await
6947 .unwrap();
6948 let tracker = AuthTracker::new();
6949 let _initial_auth = tracker.begin();
6950 tracker.succeed();
6951 client.set_auth_tracker(tracker.clone(), true);
6952 let controller_abort = client.controller_task.abort_handle();
6953 let connection_mode = Arc::clone(&client.connection_mode);
6954 let handle = client.reconnect_handle();
6955 let surviving_handle = handle.clone();
6956 *client_slot.lock() = Some(client);
6957 let (result_tx, result_rx) = std::sync::mpsc::channel();
6958 std::thread::spawn(move || {
6959 result_tx.send(handle.request_reconnect()).unwrap();
6960 });
6961
6962 assert_eq!(
6963 recv_rendezvous(result_rx, "callback drop reconnect result").await,
6964 ReconnectRequestOutcome::Accepted
6965 );
6966 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6967
6968 assert!(client_slot.lock().is_none());
6969 assert_eq!(
6970 ConnectionMode::from_atomic(&connection_mode),
6971 ConnectionMode::Closed
6972 );
6973 assert_eq!(tracker.auth_state(), AuthState::Failed);
6974 assert_eq!(
6975 surviving_handle.request_reconnect(),
6976 ReconnectRequestOutcome::Closed
6977 );
6978 assert_eq!(
6979 *states.lock(),
6980 vec![SocketState::Connected, SocketState::Disconnected]
6981 );
6982 }
6983
6984 #[rstest]
6985 #[tokio::test]
6986 async fn test_reconnect_then_disconnect() {
6987 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6989 let port = listener.local_addr().unwrap().port();
6990
6991 let server = task::spawn(async move {
6993 let (stream, _) = listener.accept().await.unwrap();
6994 let ws = accept_async(stream).await.unwrap();
6995 drop(ws);
6996 sleep(Duration::from_secs(1)).await;
6998 });
6999
7000 let (handler, _rx) = channel_message_handler();
7002
7003 let config = WebSocketConfig {
7005 url: format!("ws://127.0.0.1:{port}"),
7006 headers: vec![],
7007 heartbeat_interval_secs: None,
7008 heartbeat_payload: None,
7009 connect_timeout_ms: Some(1_000),
7010 reconnect_delay_initial_ms: Some(50),
7011 reconnect_delay_max_ms: Some(100),
7012 reconnect_backoff_factor: Some(1.0),
7013 reconnect_jitter_ms: Some(0),
7014 reconnect_max_attempts: None,
7015 heartbeat_timeout_secs: None,
7016 idle_timeout_ms: None,
7017 writer_capacity: None,
7018 backend: TransportBackend::Tungstenite,
7019 proxy_url: None,
7020 max_message_size_bytes: None,
7021 max_frame_size_bytes: None,
7022 };
7023
7024 let client = WebSocketClient::builder()
7026 .config(config)
7027 .message_handler(handler)
7028 .connect()
7029 .await
7030 .unwrap();
7031
7032 sleep(Duration::from_millis(100)).await;
7034 client.disconnect().await;
7036 assert!(client.is_disconnected());
7037 server.abort();
7038 }
7039
7040 #[rstest]
7041 #[tokio::test]
7042 async fn test_reconnect_state_flips_when_reader_stops() {
7043 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7045 let port = listener.local_addr().unwrap().port();
7046 let server = task::spawn(async move {
7047 if let Ok((stream, _)) = listener.accept().await
7048 && let Ok(ws) = accept_async(stream).await
7049 {
7050 drop(ws);
7051 }
7052
7053 sleep(Duration::from_millis(50)).await;
7054 });
7055
7056 let (handler, _rx) = channel_message_handler();
7057
7058 let config = WebSocketConfig {
7059 url: format!("ws://127.0.0.1:{port}"),
7060 headers: vec![],
7061 heartbeat_interval_secs: None,
7062 heartbeat_payload: None,
7063 connect_timeout_ms: Some(1_000),
7064 reconnect_delay_initial_ms: Some(50),
7065 reconnect_delay_max_ms: Some(100),
7066 reconnect_backoff_factor: Some(1.0),
7067 reconnect_jitter_ms: Some(0),
7068 reconnect_max_attempts: None,
7069 heartbeat_timeout_secs: None,
7070 idle_timeout_ms: None,
7071 writer_capacity: None,
7072 backend: TransportBackend::Tungstenite,
7073 proxy_url: None,
7074 max_message_size_bytes: None,
7075 max_frame_size_bytes: None,
7076 };
7077
7078 let client = WebSocketClient::builder()
7079 .config(config)
7080 .message_handler(handler)
7081 .connect()
7082 .await
7083 .unwrap();
7084
7085 tokio::time::timeout(Duration::from_secs(2), async {
7086 loop {
7087 if client.is_reconnecting() {
7088 break;
7089 }
7090
7091 tokio::time::sleep(Duration::from_millis(10)).await;
7092 }
7093 })
7094 .await
7095 .expect("client did not enter RECONNECT state");
7096
7097 client.disconnect().await;
7098 server.abort();
7099 }
7100
7101 #[rstest]
7102 #[tokio::test]
7103 async fn test_stream_mode_disables_auto_reconnect() {
7104 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7107 let port = listener.local_addr().unwrap().port();
7108 let server = task::spawn(async move {
7109 if let Ok((stream, _)) = listener.accept().await
7110 && let Ok(_ws) = accept_async(stream).await
7111 {
7112 sleep(Duration::from_millis(100)).await;
7114 }
7115 });
7116
7117 let config = WebSocketConfig {
7118 url: format!("ws://127.0.0.1:{port}"),
7119 headers: vec![],
7120 heartbeat_interval_secs: None,
7121 heartbeat_payload: None,
7122 connect_timeout_ms: Some(1_000),
7123 reconnect_delay_initial_ms: Some(50),
7124 reconnect_delay_max_ms: Some(100),
7125 reconnect_backoff_factor: Some(1.0),
7126 reconnect_jitter_ms: Some(0),
7127 reconnect_max_attempts: None,
7128 heartbeat_timeout_secs: None,
7129 idle_timeout_ms: None,
7130 writer_capacity: None,
7131 backend: TransportBackend::Tungstenite,
7132 proxy_url: None,
7133 max_message_size_bytes: None,
7134 max_frame_size_bytes: None,
7135 };
7136
7137 let (_reader, _client) = WebSocketClient::stream_builder()
7138 .config(config)
7139 .connect()
7140 .await
7141 .unwrap();
7142
7143 server.abort();
7151 }
7152
7153 #[rstest]
7154 #[tokio::test]
7155 async fn test_message_handler_mode_allows_auto_reconnect() {
7156 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7158 let port = listener.local_addr().unwrap().port();
7159
7160 let server = task::spawn(async move {
7161 if let Ok((stream, _)) = listener.accept().await
7163 && let Ok(ws) = accept_async(stream).await
7164 {
7165 drop(ws);
7166 }
7167
7168 sleep(Duration::from_millis(50)).await;
7169 });
7170
7171 let (handler, _rx) = channel_message_handler();
7172
7173 let config = WebSocketConfig {
7174 url: format!("ws://127.0.0.1:{port}"),
7175 headers: vec![],
7176 heartbeat_interval_secs: None,
7177 heartbeat_payload: None,
7178 connect_timeout_ms: Some(1_000),
7179 reconnect_delay_initial_ms: Some(50),
7180 reconnect_delay_max_ms: Some(100),
7181 reconnect_backoff_factor: Some(1.0),
7182 reconnect_jitter_ms: Some(0),
7183 reconnect_max_attempts: None,
7184 heartbeat_timeout_secs: None,
7185 idle_timeout_ms: None,
7186 writer_capacity: None,
7187 backend: TransportBackend::Tungstenite,
7188 proxy_url: None,
7189 max_message_size_bytes: None,
7190 max_frame_size_bytes: None,
7191 };
7192
7193 let client = WebSocketClient::builder()
7194 .config(config)
7195 .message_handler(handler)
7196 .connect()
7197 .await
7198 .unwrap();
7199
7200 tokio::time::timeout(Duration::from_secs(2), async {
7202 loop {
7203 if client.is_reconnecting() || client.is_closed() {
7204 break;
7205 }
7206
7207 tokio::time::sleep(Duration::from_millis(10)).await;
7208 }
7209 })
7210 .await
7211 .expect("client should attempt reconnection or close");
7212
7213 assert!(
7216 client.is_reconnecting() || client.is_closed(),
7217 "Client with message handler should attempt reconnection"
7218 );
7219
7220 client.disconnect().await;
7221 server.abort();
7222 }
7223
7224 #[rstest]
7225 #[tokio::test]
7226 async fn test_handler_mode_reconnect_with_new_connection() {
7227 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7229 let port = listener.local_addr().unwrap().port();
7230
7231 let server = task::spawn(async move {
7232 if let Ok((stream, _)) = listener.accept().await
7234 && let Ok(ws) = accept_async(stream).await
7235 {
7236 drop(ws);
7237 }
7238
7239 sleep(Duration::from_millis(100)).await;
7241
7242 if let Ok((stream, _)) = listener.accept().await
7244 && let Ok(mut ws) = accept_async(stream).await
7245 {
7246 use futures_util::SinkExt;
7247 let _ = ws
7248 .send(WsMessage::Text("reconnected".to_string().into()))
7249 .await;
7250 sleep(Duration::from_secs(1)).await;
7251 }
7252 });
7253
7254 let (handler, mut rx) = channel_message_handler();
7255
7256 let config = WebSocketConfig {
7257 url: format!("ws://127.0.0.1:{port}"),
7258 headers: vec![],
7259 heartbeat_interval_secs: None,
7260 heartbeat_payload: None,
7261 connect_timeout_ms: Some(2_000),
7262 reconnect_delay_initial_ms: Some(50),
7263 reconnect_delay_max_ms: Some(200),
7264 reconnect_backoff_factor: Some(1.5),
7265 reconnect_jitter_ms: Some(10),
7266 reconnect_max_attempts: None,
7267 heartbeat_timeout_secs: None,
7268 idle_timeout_ms: None,
7269 writer_capacity: None,
7270 backend: TransportBackend::Tungstenite,
7271 proxy_url: None,
7272 max_message_size_bytes: None,
7273 max_frame_size_bytes: None,
7274 };
7275
7276 let client = WebSocketClient::builder()
7277 .config(config)
7278 .message_handler(handler)
7279 .connect()
7280 .await
7281 .unwrap();
7282
7283 let result = tokio::time::timeout(Duration::from_secs(5), async {
7285 loop {
7286 if let Ok(msg) = rx.try_recv()
7287 && matches!(msg, WsMessage::Text(ref text) if AsRef::<str>::as_ref(text) == "reconnected")
7288 {
7289 return true;
7290 }
7291
7292 tokio::time::sleep(Duration::from_millis(10)).await;
7293 }
7294 })
7295 .await;
7296
7297 assert!(
7298 result.is_ok(),
7299 "Should receive message after reconnection within timeout"
7300 );
7301
7302 client.disconnect().await;
7303 server.abort();
7304 }
7305
7306 #[rstest]
7307 #[tokio::test]
7308 async fn test_stream_mode_no_auto_reconnect() {
7309 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7312 let port = listener.local_addr().unwrap().port();
7313
7314 let server = task::spawn(async move {
7315 if let Ok((stream, _)) = listener.accept().await
7317 && let Ok(mut ws) = accept_async(stream).await
7318 {
7319 use futures_util::SinkExt;
7320 let _ = ws.send(WsMessage::Text("hello".to_string().into())).await;
7321 sleep(Duration::from_millis(50)).await;
7322 }
7324 });
7325
7326 let config = WebSocketConfig {
7327 url: format!("ws://127.0.0.1:{port}"),
7328 headers: vec![],
7329 heartbeat_interval_secs: None,
7330 heartbeat_payload: None,
7331 connect_timeout_ms: Some(1_000),
7332 reconnect_delay_initial_ms: Some(50),
7333 reconnect_delay_max_ms: Some(100),
7334 reconnect_backoff_factor: Some(1.0),
7335 reconnect_jitter_ms: Some(0),
7336 reconnect_max_attempts: None,
7337 heartbeat_timeout_secs: None,
7338 idle_timeout_ms: None,
7339 writer_capacity: None,
7340 backend: TransportBackend::Tungstenite,
7341 proxy_url: None,
7342 max_message_size_bytes: None,
7343 max_frame_size_bytes: None,
7344 };
7345
7346 let (mut reader, client) = WebSocketClient::stream_builder()
7347 .config(config)
7348 .connect()
7349 .await
7350 .unwrap();
7351
7352 assert!(client.is_active(), "Client should start as active");
7354
7355 let msg = reader.next().await;
7357 assert!(
7358 matches!(&msg, Some(Ok(Message::Text(bytes))) if bytes.as_ref() == b"hello"),
7359 "Should receive initial message"
7360 );
7361
7362 while let Some(msg) = reader.next().await {
7364 if msg.is_err() || matches!(msg, Ok(Message::Close(_))) {
7365 break;
7366 }
7367 }
7368
7369 sleep(Duration::from_millis(200)).await;
7372 assert!(
7373 client.is_active(),
7374 "Stream mode client stays ACTIVE before notify_closed()"
7375 );
7376
7377 client.notify_closed();
7379
7380 assert!(
7381 client.is_closed(),
7382 "Stream mode client should be CLOSED after notify_closed()"
7383 );
7384 assert!(
7385 !client.is_reconnecting(),
7386 "Stream mode client should never attempt reconnection"
7387 );
7388
7389 client.disconnect().await;
7390 server.abort();
7391 }
7392
7393 #[rstest]
7394 #[tokio::test]
7395 async fn test_send_timeout_uses_configured_connect_timeout() {
7396 use nautilus_common::testing::wait_until_async;
7399
7400 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7401 let port = listener.local_addr().unwrap().port();
7402
7403 let server = task::spawn(async move {
7404 if let Ok((stream, _)) = listener.accept().await
7406 && let Ok(ws) = accept_async(stream).await
7407 {
7408 drop(ws);
7409 }
7410
7411 sleep(Duration::from_mins(1)).await;
7413 });
7414
7415 let (handler, _rx) = channel_message_handler();
7416
7417 let config = WebSocketConfig {
7419 url: format!("ws://127.0.0.1:{port}"),
7420 headers: vec![],
7421 heartbeat_interval_secs: None,
7422 heartbeat_payload: None,
7423 connect_timeout_ms: Some(2_000), reconnect_delay_initial_ms: Some(50),
7425 reconnect_delay_max_ms: Some(100),
7426 reconnect_backoff_factor: Some(1.0),
7427 reconnect_jitter_ms: Some(0),
7428 reconnect_max_attempts: None,
7429 heartbeat_timeout_secs: None,
7430 idle_timeout_ms: None,
7431 writer_capacity: None,
7432 backend: TransportBackend::Tungstenite,
7433 proxy_url: None,
7434 max_message_size_bytes: None,
7435 max_frame_size_bytes: None,
7436 };
7437
7438 let client = WebSocketClient::builder()
7439 .config(config)
7440 .message_handler(handler)
7441 .connect()
7442 .await
7443 .unwrap();
7444
7445 wait_until_async(
7447 || async { client.is_reconnecting() },
7448 Duration::from_secs(3),
7449 )
7450 .await;
7451
7452 let start = std::time::Instant::now();
7454 let send_result = client.send_text("test".to_string(), None).await;
7455 let elapsed = start.elapsed();
7456
7457 assert!(
7458 send_result.is_err(),
7459 "Send should fail when client stuck in RECONNECT"
7460 );
7461 assert!(
7462 matches!(send_result, Err(crate::error::SendError::Timeout)),
7463 "Send should return Timeout error, was: {send_result:?}"
7464 );
7465 assert!(
7468 elapsed >= Duration::from_millis(1800),
7469 "Send should timeout after at least 2s (configured timeout), took {elapsed:?}"
7470 );
7471
7472 client.disconnect().await;
7473 server.abort();
7474 }
7475
7476 #[rstest]
7477 #[tokio::test]
7478 async fn test_send_waits_during_reconnection() {
7479 use nautilus_common::testing::wait_until_async;
7481
7482 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7483 let port = listener.local_addr().unwrap().port();
7484
7485 let server = task::spawn(async move {
7486 if let Ok((stream, _)) = listener.accept().await
7488 && let Ok(ws) = accept_async(stream).await
7489 {
7490 drop(ws);
7491 }
7492
7493 sleep(Duration::from_millis(500)).await;
7495
7496 if let Ok((stream, _)) = listener.accept().await
7498 && let Ok(mut ws) = accept_async(stream).await
7499 {
7500 while let Some(Ok(msg)) = ws.next().await {
7502 if ws.send(msg).await.is_err() {
7503 break;
7504 }
7505 }
7506 }
7507 });
7508
7509 let (handler, _rx) = channel_message_handler();
7510
7511 let config = WebSocketConfig {
7512 url: format!("ws://127.0.0.1:{port}"),
7513 headers: vec![],
7514 heartbeat_interval_secs: None,
7515 heartbeat_payload: None,
7516 connect_timeout_ms: Some(5_000), reconnect_delay_initial_ms: Some(100),
7518 reconnect_delay_max_ms: Some(200),
7519 reconnect_backoff_factor: Some(1.0),
7520 reconnect_jitter_ms: Some(0),
7521 reconnect_max_attempts: None,
7522 heartbeat_timeout_secs: None,
7523 idle_timeout_ms: None,
7524 writer_capacity: None,
7525 backend: TransportBackend::Tungstenite,
7526 proxy_url: None,
7527 max_message_size_bytes: None,
7528 max_frame_size_bytes: None,
7529 };
7530
7531 let client = WebSocketClient::builder()
7532 .config(config)
7533 .message_handler(handler)
7534 .connect()
7535 .await
7536 .unwrap();
7537
7538 wait_until_async(
7540 || async { client.is_reconnecting() },
7541 Duration::from_secs(2),
7542 )
7543 .await;
7544
7545 let send_result = tokio::time::timeout(
7547 Duration::from_secs(3),
7548 client.send_text("test_message".to_string(), None),
7549 )
7550 .await;
7551
7552 assert!(
7553 send_result.is_ok() && send_result.unwrap().is_ok(),
7554 "Send should succeed after waiting for reconnection"
7555 );
7556
7557 client.disconnect().await;
7558 server.abort();
7559 }
7560
7561 #[rstest]
7562 #[tokio::test]
7563 async fn test_rate_limiter_before_active_wait() {
7564 use std::{num::NonZeroU32, sync::Arc};
7569
7570 use nautilus_common::testing::wait_until_async;
7571
7572 use crate::ratelimiter::quota::Quota;
7573
7574 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7575 let port = listener.local_addr().unwrap().port();
7576
7577 let server = task::spawn(async move {
7578 if let Ok((stream, _)) = listener.accept().await
7580 && let Ok(mut ws) = accept_async(stream).await
7581 {
7582 if let Some(Ok(_)) = ws.next().await {
7584 drop(ws);
7585 }
7586 }
7587
7588 sleep(Duration::from_millis(500)).await;
7590
7591 if let Ok((stream, _)) = listener.accept().await
7593 && let Ok(mut ws) = accept_async(stream).await
7594 {
7595 while let Some(Ok(msg)) = ws.next().await {
7596 if ws.send(msg).await.is_err() {
7597 break;
7598 }
7599 }
7600 }
7601 });
7602
7603 let (handler, _rx) = channel_message_handler();
7604
7605 let config = WebSocketConfig {
7606 url: format!("ws://127.0.0.1:{port}"),
7607 headers: vec![],
7608 heartbeat_interval_secs: None,
7609 heartbeat_payload: None,
7610 connect_timeout_ms: Some(5_000),
7611 reconnect_delay_initial_ms: Some(50),
7612 reconnect_delay_max_ms: Some(100),
7613 reconnect_backoff_factor: Some(1.0),
7614 reconnect_jitter_ms: Some(0),
7615 reconnect_max_attempts: None,
7616 heartbeat_timeout_secs: None,
7617 idle_timeout_ms: None,
7618 writer_capacity: None,
7619 backend: TransportBackend::Tungstenite,
7620 proxy_url: None,
7621 max_message_size_bytes: None,
7622 max_frame_size_bytes: None,
7623 };
7624
7625 let quota = Quota::per_second(NonZeroU32::new(1).unwrap())
7627 .unwrap()
7628 .allow_burst(NonZeroU32::new(1).unwrap());
7629
7630 let client = Arc::new(
7631 WebSocketClient::builder()
7632 .config(config)
7633 .message_handler(handler)
7634 .keyed_quotas(vec![("test_key".to_string(), quota)])
7635 .connect()
7636 .await
7637 .unwrap(),
7638 );
7639
7640 let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7642 client
7643 .send_text("msg1".to_string(), Some(test_key.as_slice()))
7644 .await
7645 .unwrap();
7646
7647 wait_until_async(
7649 || async { client.is_reconnecting() },
7650 Duration::from_secs(2),
7651 )
7652 .await;
7653
7654 let start = std::time::Instant::now();
7656 let send_result = client
7657 .send_text("msg2".to_string(), Some(test_key.as_slice()))
7658 .await;
7659 let elapsed = start.elapsed();
7660
7661 assert!(
7663 send_result.is_ok(),
7664 "Send should succeed after rate limit + reconnection, was: {send_result:?}"
7665 );
7666 assert!(
7670 elapsed >= Duration::from_millis(850),
7671 "Should wait for rate limit (~1s), waited {elapsed:?}"
7672 );
7673
7674 client.disconnect().await;
7675 server.abort();
7676 }
7677
7678 #[rstest]
7679 #[tokio::test]
7680 async fn test_disconnect_during_reconnect_exits_cleanly() {
7681 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7684 let port = listener.local_addr().unwrap().port();
7685
7686 let server = task::spawn(async move {
7687 if let Ok((stream, _)) = listener.accept().await
7689 && let Ok(ws) = accept_async(stream).await
7690 {
7691 drop(ws);
7692 }
7693
7694 sleep(Duration::from_mins(1)).await;
7696 });
7697
7698 let (handler, _rx) = channel_message_handler();
7699
7700 let config = WebSocketConfig {
7701 url: format!("ws://127.0.0.1:{port}"),
7702 headers: vec![],
7703 heartbeat_interval_secs: None,
7704 heartbeat_payload: None,
7705 connect_timeout_ms: Some(2_000), reconnect_delay_initial_ms: Some(100),
7707 reconnect_delay_max_ms: Some(200),
7708 reconnect_backoff_factor: Some(1.0),
7709 reconnect_jitter_ms: Some(0),
7710 reconnect_max_attempts: None,
7711 heartbeat_timeout_secs: None,
7712 idle_timeout_ms: None,
7713 writer_capacity: None,
7714 backend: TransportBackend::Tungstenite,
7715 proxy_url: None,
7716 max_message_size_bytes: None,
7717 max_frame_size_bytes: None,
7718 };
7719
7720 let client = WebSocketClient::builder()
7721 .config(config)
7722 .message_handler(handler)
7723 .connect()
7724 .await
7725 .unwrap();
7726
7727 tokio::time::timeout(Duration::from_secs(2), async {
7729 while !client.is_reconnecting() {
7730 sleep(Duration::from_millis(10)).await;
7731 }
7732 })
7733 .await
7734 .expect("Client should enter RECONNECT state");
7735
7736 client.disconnect().await;
7738
7739 assert!(
7741 client.is_disconnected(),
7742 "Client should be cleanly disconnected"
7743 );
7744
7745 server.abort();
7746 }
7747
7748 #[rstest]
7749 #[tokio::test]
7750 async fn test_send_fails_fast_when_closed_before_rate_limit() {
7751 use std::{num::NonZeroU32, sync::Arc};
7754
7755 use nautilus_common::testing::wait_until_async;
7756
7757 use crate::ratelimiter::quota::Quota;
7758
7759 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7760 let port = listener.local_addr().unwrap().port();
7761
7762 let server = task::spawn(async move {
7763 if let Ok((stream, _)) = listener.accept().await
7765 && let Ok(ws) = accept_async(stream).await
7766 {
7767 drop(ws);
7768 }
7769
7770 sleep(Duration::from_mins(1)).await;
7771 });
7772
7773 let (handler, _rx) = channel_message_handler();
7774
7775 let config = WebSocketConfig {
7776 url: format!("ws://127.0.0.1:{port}"),
7777 headers: vec![],
7778 heartbeat_interval_secs: None,
7779 heartbeat_payload: None,
7780 connect_timeout_ms: Some(5_000),
7781 reconnect_delay_initial_ms: Some(50),
7782 reconnect_delay_max_ms: Some(100),
7783 reconnect_backoff_factor: Some(1.0),
7784 reconnect_jitter_ms: Some(0),
7785 reconnect_max_attempts: None,
7786 heartbeat_timeout_secs: None,
7787 idle_timeout_ms: None,
7788 writer_capacity: None,
7789 backend: TransportBackend::Tungstenite,
7790 proxy_url: None,
7791 max_message_size_bytes: None,
7792 max_frame_size_bytes: None,
7793 };
7794
7795 let quota = Quota::with_period(Duration::from_secs(10))
7798 .unwrap()
7799 .allow_burst(NonZeroU32::new(1).unwrap());
7800
7801 let client = Arc::new(
7802 WebSocketClient::builder()
7803 .config(config)
7804 .message_handler(handler)
7805 .keyed_quotas(vec![("test_key".to_string(), quota)])
7806 .connect()
7807 .await
7808 .unwrap(),
7809 );
7810
7811 wait_until_async(
7813 || async { client.is_reconnecting() || client.is_closed() },
7814 Duration::from_secs(2),
7815 )
7816 .await;
7817
7818 client.disconnect().await;
7820 assert!(
7821 !client.is_active(),
7822 "Client should not be active after disconnect"
7823 );
7824
7825 let start = std::time::Instant::now();
7827 let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7828 let result = client
7829 .send_text("test".to_string(), Some(test_key.as_slice()))
7830 .await;
7831 let elapsed = start.elapsed();
7832
7833 assert!(result.is_err(), "Send should fail when client is closed");
7835 assert!(
7836 matches!(result, Err(crate::error::SendError::Closed)),
7837 "Send should return Closed error, was: {result:?}"
7838 );
7839
7840 assert!(
7842 elapsed < Duration::from_millis(100),
7843 "Send should fail fast without rate limiting, took {elapsed:?}"
7844 );
7845
7846 server.abort();
7847 }
7848
7849 #[rstest]
7850 #[tokio::test]
7851 async fn test_builder_rejects_shared_rate_limiter_with_quotas() {
7852 let (handler, _rx) = channel_message_handler();
7853 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7854 let result = WebSocketClient::builder()
7855 .config(reconnect_test_config(1))
7856 .message_handler(handler)
7857 .default_quota(Quota::with_period(Duration::from_secs(1)).unwrap())
7858 .rate_limiter(rate_limiter)
7859 .connect()
7860 .await;
7861
7862 assert_eq!(
7863 result.unwrap_err().to_string(),
7864 "I/O error: Cannot combine a shared rate limiter with quota configuration"
7865 );
7866 }
7867
7868 #[rstest]
7869 #[tokio::test]
7870 async fn test_builder_rejects_connection_keys_without_limiter() {
7871 let (handler, _rx) = channel_message_handler();
7872 let result = WebSocketClient::builder()
7873 .config(reconnect_test_config(1))
7874 .message_handler(handler)
7875 .connection_rate_keys(Arc::from([Ustr::from("connect")]))
7876 .connect()
7877 .await;
7878
7879 assert_eq!(
7880 result.unwrap_err().to_string(),
7881 "I/O error: Connection rate keys require a connection rate limiter"
7882 );
7883 }
7884
7885 #[rstest]
7886 #[tokio::test]
7887 async fn test_builder_rejects_connection_limiter_without_keys() {
7888 let (handler, _rx) = channel_message_handler();
7889 let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7890 let result = WebSocketClient::builder()
7891 .config(reconnect_test_config(1))
7892 .message_handler(handler)
7893 .connection_rate_limiter(rate_limiter)
7894 .connect()
7895 .await;
7896
7897 assert_eq!(
7898 result.unwrap_err().to_string(),
7899 "I/O error: Connection rate limiter requires at least one connection rate key"
7900 );
7901 }
7902
7903 #[rstest]
7904 #[tokio::test]
7905 async fn test_epoch_builder_rejects_both_ping_handler_types() {
7906 let (handler, _rx) = channel_epoch_message_handler();
7907 let ping_handler: PingHandler = Arc::new(|_| {});
7908 let epoch_ping_handler: EpochPingHandler = Arc::new(|_, _| {});
7909 let result = WebSocketClient::epoch_builder()
7910 .config(reconnect_test_config(1))
7911 .epoch_handler(handler)
7912 .ping_handler(ping_handler)
7913 .epoch_ping_handler(epoch_ping_handler)
7914 .connect()
7915 .await;
7916
7917 assert_eq!(
7918 result.unwrap_err().to_string(),
7919 "I/O error: Cannot configure both ping_handler and epoch_ping_handler"
7920 );
7921 }
7922
7923 #[rstest]
7924 #[tokio::test]
7925 async fn test_connect_url_rejects_invalid_reconnect_timing_before_connect() {
7926 let (handler, _rx) = channel_message_handler();
7927
7928 let config = WebSocketConfig {
7929 url: "ws://127.0.0.1:1".to_string(),
7930 headers: vec![],
7931 heartbeat_interval_secs: None,
7932 heartbeat_payload: None,
7933 connect_timeout_ms: Some(0),
7934 reconnect_delay_initial_ms: Some(100),
7935 reconnect_delay_max_ms: Some(500),
7936 reconnect_backoff_factor: Some(1.5),
7937 reconnect_jitter_ms: Some(0),
7938 reconnect_max_attempts: None,
7939 heartbeat_timeout_secs: None,
7940 idle_timeout_ms: None,
7941 writer_capacity: None,
7942 backend: TransportBackend::Tungstenite,
7943 proxy_url: None,
7944 max_message_size_bytes: None,
7945 max_frame_size_bytes: None,
7946 };
7947
7948 let err = WebSocketClientInner::connect_url(config, Some(handler), None)
7949 .await
7950 .expect_err("invalid reconnect timing should be rejected");
7951
7952 match err {
7953 TransportError::Io(error) => {
7954 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
7955 assert!(
7956 error.to_string().contains("connect_timeout_ms"),
7957 "error should mention zero reconnect timeout, was: {error}"
7958 );
7959 }
7960 other => panic!("expected InvalidInput IO error, was: {other:?}"),
7961 }
7962 }
7963
7964 #[rstest]
7965 #[tokio::test]
7966 async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
7967 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7968 let port = listener.local_addr().unwrap().port();
7969 let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
7970 let accepted_clone = Arc::clone(&accepted);
7971
7972 let server = task::spawn(async move {
7973 let (stream, _) = listener.accept().await.unwrap();
7974 accepted_clone.store(true, Ordering::SeqCst);
7975 accept_async(stream).await.unwrap();
7976 });
7977
7978 let (handler, _rx) = channel_message_handler();
7979
7980 let config = WebSocketConfig {
7981 url: format!("ws://127.0.0.1:{port}"),
7982 headers: vec![],
7983 heartbeat_interval_secs: None,
7984 heartbeat_payload: None,
7985 connect_timeout_ms: Some(1_000),
7986 reconnect_delay_initial_ms: Some(50),
7987 reconnect_delay_max_ms: Some(100),
7988 reconnect_backoff_factor: Some(100.1),
7989 reconnect_jitter_ms: Some(0),
7990 reconnect_max_attempts: None,
7991 heartbeat_timeout_secs: None,
7992 idle_timeout_ms: None,
7993 writer_capacity: None,
7994 backend: TransportBackend::Tungstenite,
7995 proxy_url: None,
7996 max_message_size_bytes: None,
7997 max_frame_size_bytes: None,
7998 };
7999
8000 let error = WebSocketClientInner::connect_url(config, Some(handler), None)
8001 .await
8002 .expect_err("invalid reconnect backoff should be rejected");
8003
8004 match error {
8005 TransportError::Io(error) => {
8006 assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
8007 assert!(
8008 error.to_string().contains("factor"),
8009 "error should mention the invalid factor, was: {error}"
8010 );
8011 }
8012 other => panic!("expected InvalidInput IO error, was: {other:?}"),
8013 }
8014
8015 assert!(
8016 !accepted.load(Ordering::SeqCst),
8017 "invalid reconnect backoff must be rejected before connecting"
8018 );
8019 server.abort();
8020 }
8021
8022 #[rstest]
8023 #[tokio::test]
8024 async fn test_client_without_handler_sets_stream_mode() {
8025 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8029 let port = listener.local_addr().unwrap().port();
8030
8031 let server = task::spawn(async move {
8032 if let Ok((stream, _)) = listener.accept().await
8034 && let Ok(ws) = accept_async(stream).await
8035 {
8036 drop(ws); }
8038 });
8039
8040 let config = WebSocketConfig {
8041 url: format!("ws://127.0.0.1:{port}"),
8042 headers: vec![],
8043 heartbeat_interval_secs: None,
8044 heartbeat_payload: None,
8045 connect_timeout_ms: Some(1_000),
8046 reconnect_delay_initial_ms: Some(100),
8047 reconnect_delay_max_ms: Some(500),
8048 reconnect_backoff_factor: Some(1.5),
8049 reconnect_jitter_ms: Some(0),
8050 reconnect_max_attempts: None,
8051 heartbeat_timeout_secs: None,
8052 idle_timeout_ms: None,
8053 writer_capacity: None,
8054 backend: TransportBackend::Tungstenite,
8055 proxy_url: None,
8056 max_message_size_bytes: None,
8057 max_frame_size_bytes: None,
8058 };
8059
8060 let inner = WebSocketClientInner::connect_url(config, None, None)
8062 .await
8063 .unwrap();
8064
8065 assert!(
8067 inner.handler.is_none(),
8068 "Client without handler should not retain an internal handler"
8069 );
8070
8071 server.abort();
8075 }
8076
8077 #[rstest]
8078 #[tokio::test]
8079 async fn test_idle_timeout_triggers_reconnect() {
8080 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8081 let port = listener.local_addr().unwrap().port();
8082
8083 let server = task::spawn(async move {
8085 let (stream, _) = listener.accept().await.unwrap();
8086 let _ws = accept_async(stream).await.unwrap();
8087 sleep(Duration::from_secs(5)).await;
8089 });
8090
8091 let (handler, _rx) = channel_message_handler();
8092
8093 let config = WebSocketConfig {
8094 url: format!("ws://127.0.0.1:{port}"),
8095 headers: vec![],
8096 heartbeat_interval_secs: None,
8097 heartbeat_payload: None,
8098 connect_timeout_ms: Some(2_000),
8099 reconnect_delay_initial_ms: Some(50),
8100 reconnect_delay_max_ms: Some(100),
8101 reconnect_backoff_factor: Some(1.0),
8102 reconnect_jitter_ms: Some(0),
8103 reconnect_max_attempts: Some(1),
8104 heartbeat_timeout_secs: None,
8105 idle_timeout_ms: Some(500),
8106 writer_capacity: None,
8107 backend: TransportBackend::Tungstenite,
8108 proxy_url: None,
8109 max_message_size_bytes: None,
8110 max_frame_size_bytes: None,
8111 };
8112
8113 let client = WebSocketClient::builder()
8114 .config(config)
8115 .message_handler(handler)
8116 .connect()
8117 .await
8118 .unwrap();
8119
8120 assert!(client.is_active());
8121
8122 wait_until_async(
8124 || async { client.is_reconnecting() || client.is_disconnected() },
8125 Duration::from_secs(3),
8126 )
8127 .await;
8128
8129 assert!(
8130 !client.is_active(),
8131 "Client should not be active after idle timeout"
8132 );
8133
8134 client.disconnect().await;
8135 server.abort();
8136 }
8137
8138 #[rstest]
8139 #[tokio::test]
8140 async fn test_idle_timeout_resets_on_data() {
8141 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8142 let port = listener.local_addr().unwrap().port();
8143
8144 let server = task::spawn(async move {
8146 let (stream, _) = listener.accept().await.unwrap();
8147 let mut ws = accept_async(stream).await.unwrap();
8148
8149 for _ in 0..10 {
8150 sleep(Duration::from_millis(200)).await;
8151
8152 if ws.send(WsMessage::Text("ping".into())).await.is_err() {
8153 break;
8154 }
8155 }
8156 });
8157
8158 let (handler, _rx) = channel_message_handler();
8159
8160 let config = WebSocketConfig {
8161 url: format!("ws://127.0.0.1:{port}"),
8162 headers: vec![],
8163 heartbeat_interval_secs: None,
8164 heartbeat_payload: None,
8165 connect_timeout_ms: Some(2_000),
8166 reconnect_delay_initial_ms: Some(50),
8167 reconnect_delay_max_ms: Some(100),
8168 reconnect_backoff_factor: Some(1.0),
8169 reconnect_jitter_ms: Some(0),
8170 reconnect_max_attempts: Some(1),
8171 heartbeat_timeout_secs: None,
8172 idle_timeout_ms: Some(1_000),
8173 writer_capacity: None,
8174 backend: TransportBackend::Tungstenite,
8175 proxy_url: None,
8176 max_message_size_bytes: None,
8177 max_frame_size_bytes: None,
8178 };
8179
8180 let client = WebSocketClient::builder()
8181 .config(config)
8182 .message_handler(handler)
8183 .connect()
8184 .await
8185 .unwrap();
8186
8187 assert!(client.is_active());
8188
8189 sleep(Duration::from_millis(1_500)).await;
8191
8192 assert!(
8193 client.is_active(),
8194 "Client should remain active when data is flowing"
8195 );
8196
8197 client.disconnect().await;
8198 server.abort();
8199 }
8200
8201 #[rstest]
8202 #[tokio::test]
8203 async fn test_idle_timeout_fires_when_only_pings_received() {
8204 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8210 let port = listener.local_addr().unwrap().port();
8211 let server = task::spawn(async move {
8212 let (stream, _) = listener.accept().await.unwrap();
8213 let mut ws = accept_async(stream).await.unwrap();
8214
8215 for _ in 0..60 {
8216 sleep(Duration::from_millis(100)).await;
8217
8218 if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
8219 break;
8220 }
8221 }
8222 });
8223
8224 let (handler, _rx) = channel_message_handler();
8225
8226 let config = WebSocketConfig {
8227 url: format!("ws://127.0.0.1:{port}"),
8228 headers: vec![],
8229 heartbeat_interval_secs: None,
8230 heartbeat_payload: None,
8231 connect_timeout_ms: Some(2_000),
8232 reconnect_delay_initial_ms: Some(50),
8233 reconnect_delay_max_ms: Some(100),
8234 reconnect_backoff_factor: Some(1.0),
8235 reconnect_jitter_ms: Some(0),
8236 reconnect_max_attempts: Some(1),
8237 heartbeat_timeout_secs: None,
8238 idle_timeout_ms: Some(500),
8239 writer_capacity: None,
8240 backend: TransportBackend::Tungstenite,
8241 proxy_url: None,
8242 max_message_size_bytes: None,
8243 max_frame_size_bytes: None,
8244 };
8245
8246 let client = WebSocketClient::builder()
8247 .config(config)
8248 .message_handler(handler)
8249 .connect()
8250 .await
8251 .unwrap();
8252
8253 assert!(client.is_active());
8254
8255 wait_until_async(
8259 || async { client.is_reconnecting() || client.is_disconnected() },
8260 Duration::from_millis(1_500),
8261 )
8262 .await;
8263
8264 assert!(
8265 !client.is_active(),
8266 "Client should not be active after idle timeout when only pings/pongs flow"
8267 );
8268
8269 client.disconnect().await;
8270 server.abort();
8271 }
8272
8273 #[rstest]
8274 #[tokio::test]
8275 async fn test_idle_timeout_fires_when_only_pongs_received() {
8276 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8281 let port = listener.local_addr().unwrap().port();
8282 let server = task::spawn(async move {
8283 let (stream, _) = listener.accept().await.unwrap();
8284 let mut ws = accept_async(stream).await.unwrap();
8285
8286 let deadline = tokio::time::Instant::now() + Duration::from_secs(6);
8290 while tokio::time::Instant::now() < deadline {
8291 if let Ok(Some(Err(_)) | None) =
8292 tokio::time::timeout(Duration::from_millis(100), ws.next()).await
8293 {
8294 break;
8295 }
8296 }
8297 });
8298
8299 let (handler, _rx) = channel_message_handler();
8300
8301 let config = WebSocketConfig {
8302 url: format!("ws://127.0.0.1:{port}"),
8303 headers: vec![],
8304 heartbeat_interval_secs: Some(1),
8305 heartbeat_payload: None,
8306 connect_timeout_ms: Some(2_000),
8307 reconnect_delay_initial_ms: Some(50),
8308 reconnect_delay_max_ms: Some(100),
8309 reconnect_backoff_factor: Some(1.0),
8310 reconnect_jitter_ms: Some(0),
8311 reconnect_max_attempts: Some(1),
8312 heartbeat_timeout_secs: None,
8313 idle_timeout_ms: Some(1_500),
8314 writer_capacity: None,
8315 backend: TransportBackend::Tungstenite,
8316 proxy_url: None,
8317 max_message_size_bytes: None,
8318 max_frame_size_bytes: None,
8319 };
8320
8321 let client = WebSocketClient::builder()
8322 .config(config)
8323 .message_handler(handler)
8324 .connect()
8325 .await
8326 .unwrap();
8327
8328 assert!(client.is_active());
8329
8330 wait_until_async(
8334 || async { client.is_reconnecting() || client.is_disconnected() },
8335 Duration::from_millis(2_500),
8336 )
8337 .await;
8338
8339 assert!(
8340 !client.is_active(),
8341 "Client should not be active after idle timeout when only pongs flow"
8342 );
8343
8344 client.disconnect().await;
8345 server.abort();
8346 }
8347
8348 #[rstest]
8349 #[tokio::test]
8350 async fn test_disconnect_during_backoff_exits_promptly() {
8351 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8355 let port = listener.local_addr().unwrap().port();
8356
8357 let server = task::spawn(async move {
8358 if let Ok((stream, _)) = listener.accept().await {
8360 let _ = accept_async(stream).await;
8361 }
8362
8363 sleep(Duration::from_mins(1)).await;
8365 });
8366
8367 let (handler, _rx) = channel_message_handler();
8368
8369 let config = WebSocketConfig {
8370 url: format!("ws://127.0.0.1:{port}"),
8371 headers: vec![],
8372 heartbeat_interval_secs: None,
8373 heartbeat_payload: None,
8374 connect_timeout_ms: Some(1_000),
8375 reconnect_delay_initial_ms: Some(10_000), reconnect_delay_max_ms: Some(10_000),
8377 reconnect_backoff_factor: Some(1.0),
8378 reconnect_jitter_ms: Some(0),
8379 reconnect_max_attempts: None,
8380 heartbeat_timeout_secs: None,
8381 idle_timeout_ms: None,
8382 writer_capacity: None,
8383 backend: TransportBackend::Tungstenite,
8384 proxy_url: None,
8385 max_message_size_bytes: None,
8386 max_frame_size_bytes: None,
8387 };
8388
8389 let client = WebSocketClient::builder()
8390 .config(config)
8391 .message_handler(handler)
8392 .connect()
8393 .await
8394 .unwrap();
8395
8396 wait_until_async(
8398 || async { client.is_reconnecting() },
8399 Duration::from_secs(3),
8400 )
8401 .await;
8402
8403 sleep(Duration::from_millis(1_500)).await;
8405
8406 let start = std::time::Instant::now();
8408 client.disconnect().await;
8409 let elapsed = start.elapsed();
8410
8411 assert!(client.is_disconnected(), "Client should be disconnected");
8412 assert!(
8414 elapsed < Duration::from_secs(2),
8415 "Disconnect should interrupt backoff sleep, took {elapsed:?}"
8416 );
8417
8418 server.abort();
8419 }
8420
8421 #[rstest]
8422 #[tokio::test]
8423 async fn test_rate_limit_cancelled_on_disconnect() {
8424 use std::{num::NonZeroU32, sync::Arc};
8427
8428 use crate::ratelimiter::quota::Quota;
8429
8430 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8431 let port = listener.local_addr().unwrap().port();
8432 let server = task::spawn(async move {
8433 if let Ok((stream, _)) = listener.accept().await {
8434 let mut ws = accept_async(stream).await.unwrap();
8435 while let Some(Ok(msg)) = ws.next().await {
8437 if ws.send(msg).await.is_err() {
8438 break;
8439 }
8440 }
8441 }
8442 });
8443
8444 let (handler, _rx) = channel_message_handler();
8445
8446 let config = WebSocketConfig {
8447 url: format!("ws://127.0.0.1:{port}"),
8448 headers: vec![],
8449 heartbeat_interval_secs: None,
8450 heartbeat_payload: None,
8451 connect_timeout_ms: Some(5_000),
8452 reconnect_delay_initial_ms: Some(100),
8453 reconnect_delay_max_ms: Some(500),
8454 reconnect_backoff_factor: Some(1.5),
8455 reconnect_jitter_ms: Some(0),
8456 reconnect_max_attempts: None,
8457 heartbeat_timeout_secs: None,
8458 idle_timeout_ms: None,
8459 writer_capacity: None,
8460 backend: TransportBackend::Tungstenite,
8461 proxy_url: None,
8462 max_message_size_bytes: None,
8463 max_frame_size_bytes: None,
8464 };
8465
8466 let quota = Quota::with_period(Duration::from_mins(1))
8468 .unwrap()
8469 .allow_burst(NonZeroU32::new(1).unwrap());
8470
8471 let client = Arc::new(
8472 WebSocketClient::builder()
8473 .config(config)
8474 .message_handler(handler)
8475 .keyed_quotas(vec![("rate_key".to_string(), quota)])
8476 .connect()
8477 .await
8478 .unwrap(),
8479 );
8480
8481 let test_key: [Ustr; 1] = [Ustr::from("rate_key")];
8482
8483 client
8485 .send_text("exhaust".to_string(), Some(test_key.as_slice()))
8486 .await
8487 .unwrap();
8488
8489 let client_clone = Arc::clone(&client);
8491 let send_handle = task::spawn(async move {
8492 client_clone
8493 .send_text("blocked".to_string(), Some(&[Ustr::from("rate_key")]))
8494 .await
8495 });
8496
8497 sleep(Duration::from_millis(200)).await;
8499
8500 let start = std::time::Instant::now();
8502 client.disconnect().await;
8503 let elapsed_disconnect = start.elapsed();
8504
8505 let result = tokio::time::timeout(Duration::from_secs(2), send_handle)
8507 .await
8508 .expect("Send task should complete quickly")
8509 .expect("Send task should not panic");
8510
8511 assert!(
8512 matches!(result, Err(crate::error::SendError::Closed)),
8513 "Blocked send should return Closed, was: {result:?}"
8514 );
8515
8516 assert!(
8518 elapsed_disconnect < Duration::from_secs(3),
8519 "Disconnect should not wait for rate limiter, took {elapsed_disconnect:?}"
8520 );
8521
8522 server.abort();
8523 }
8524
8525 #[rstest]
8526 #[tokio::test]
8527 async fn test_stream_mode_transitions_to_closed_on_dead_write_task() {
8528 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8532 let port = listener.local_addr().unwrap().port();
8533 let server = task::spawn(async move {
8534 if let Ok((stream, _)) = listener.accept().await
8535 && let Ok(ws) = accept_async(stream).await
8536 {
8537 drop(ws);
8539 }
8540 });
8541
8542 let config = WebSocketConfig {
8543 url: format!("ws://127.0.0.1:{port}"),
8544 headers: vec![],
8545 heartbeat_interval_secs: None,
8546 heartbeat_payload: None,
8547 connect_timeout_ms: Some(1_000),
8548 reconnect_delay_initial_ms: Some(50),
8549 reconnect_delay_max_ms: Some(100),
8550 reconnect_backoff_factor: Some(1.0),
8551 reconnect_jitter_ms: Some(0),
8552 reconnect_max_attempts: None,
8553 heartbeat_timeout_secs: None,
8554 idle_timeout_ms: None,
8555 writer_capacity: None,
8556 backend: TransportBackend::Tungstenite,
8557 proxy_url: None,
8558 max_message_size_bytes: None,
8559 max_frame_size_bytes: None,
8560 };
8561
8562 let (_reader, client) = WebSocketClient::stream_builder()
8563 .config(config)
8564 .connect()
8565 .await
8566 .unwrap();
8567
8568 assert!(client.is_active(), "Client should start active");
8569
8570 sleep(Duration::from_millis(100)).await;
8572
8573 for _ in 0..20 {
8575 let _ = client.send_text("ping".to_string(), None).await;
8576 sleep(Duration::from_millis(50)).await;
8577
8578 if !client.is_active() {
8579 break;
8580 }
8581 }
8582
8583 wait_until_async(|| async { !client.is_active() }, Duration::from_secs(5)).await;
8585
8586 assert!(
8588 client.is_closed() || client.is_disconnected(),
8589 "Stream mode should transition to CLOSED, not RECONNECT. \
8590 is_reconnecting={}, is_closed={}, is_disconnected={}",
8591 client.is_reconnecting(),
8592 client.is_closed(),
8593 client.is_disconnected(),
8594 );
8595 assert!(
8596 !client.is_reconnecting(),
8597 "Stream mode should never attempt reconnection"
8598 );
8599
8600 server.abort();
8601 }
8602
8603 #[derive(Default)]
8604 struct BlockingFailState {
8605 send_entered: AtomicBool,
8606 send_entered_notify: tokio::sync::Notify,
8607 released: AtomicBool,
8608 fail: AtomicBool,
8609 waker: parking_lot::Mutex<Option<std::task::Waker>>,
8610 }
8611
8612 impl BlockingFailState {
8613 fn trigger_failure(&self) {
8614 self.fail.store(true, Ordering::SeqCst);
8615 self.release_send();
8616 }
8617
8618 fn release_send(&self) {
8619 self.released.store(true, Ordering::SeqCst);
8620
8621 if let Some(waker) = self.waker.lock().take() {
8622 waker.wake();
8623 }
8624 }
8625 }
8626
8627 struct BlockingFailTransport {
8629 state: Arc<BlockingFailState>,
8630 }
8631
8632 impl futures_util::Stream for BlockingFailTransport {
8633 type Item = Result<Message, TransportError>;
8634
8635 fn poll_next(
8636 self: std::pin::Pin<&mut Self>,
8637 _cx: &mut std::task::Context<'_>,
8638 ) -> std::task::Poll<Option<Self::Item>> {
8639 std::task::Poll::Pending
8640 }
8641 }
8642
8643 impl futures_util::Sink<Message> for BlockingFailTransport {
8644 type Error = TransportError;
8645
8646 fn poll_ready(
8647 self: std::pin::Pin<&mut Self>,
8648 _cx: &mut std::task::Context<'_>,
8649 ) -> std::task::Poll<Result<(), Self::Error>> {
8650 std::task::Poll::Ready(Ok(()))
8651 }
8652
8653 fn start_send(self: std::pin::Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8654 Ok(())
8655 }
8656
8657 fn poll_flush(
8658 self: std::pin::Pin<&mut Self>,
8659 cx: &mut std::task::Context<'_>,
8660 ) -> std::task::Poll<Result<(), Self::Error>> {
8661 *self.state.waker.lock() = Some(cx.waker().clone());
8664 self.state.send_entered.store(true, Ordering::SeqCst);
8665 self.state.send_entered_notify.notify_one();
8666
8667 if !self.state.released.load(Ordering::SeqCst) {
8668 std::task::Poll::Pending
8669 } else if self.state.fail.load(Ordering::SeqCst) {
8670 std::task::Poll::Ready(Err(TransportError::ConnectionReset))
8671 } else {
8672 std::task::Poll::Ready(Ok(()))
8673 }
8674 }
8675
8676 fn poll_close(
8677 self: std::pin::Pin<&mut Self>,
8678 _cx: &mut std::task::Context<'_>,
8679 ) -> std::task::Poll<Result<(), Self::Error>> {
8680 std::task::Poll::Ready(Ok(()))
8681 }
8682 }
8683
8684 struct BlockingMessageState {
8685 polled_tx: Mutex<Option<std::sync::mpsc::Sender<()>>>,
8686 release: (Mutex<bool>, parking_lot::Condvar),
8687 message: Mutex<Option<Message>>,
8688 }
8689
8690 struct BlockingMessageTransport {
8691 state: Arc<BlockingMessageState>,
8692 }
8693
8694 impl futures_util::Stream for BlockingMessageTransport {
8695 type Item = Result<Message, TransportError>;
8696
8697 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8698 if let Some(polled_tx) = self.state.polled_tx.lock().take() {
8699 polled_tx.send(()).unwrap();
8700 }
8701
8702 let (lock, condvar) = &self.state.release;
8703 let mut released = lock.lock();
8704 while !*released {
8705 condvar.wait(&mut released);
8706 }
8707
8708 Poll::Ready(self.state.message.lock().take().map(Ok))
8709 }
8710 }
8711
8712 impl futures_util::Sink<Message> for BlockingMessageTransport {
8713 type Error = TransportError;
8714
8715 fn poll_ready(
8716 self: Pin<&mut Self>,
8717 _cx: &mut Context<'_>,
8718 ) -> Poll<Result<(), Self::Error>> {
8719 Poll::Ready(Ok(()))
8720 }
8721
8722 fn start_send(self: Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8723 Ok(())
8724 }
8725
8726 fn poll_flush(
8727 self: Pin<&mut Self>,
8728 _cx: &mut Context<'_>,
8729 ) -> Poll<Result<(), Self::Error>> {
8730 Poll::Ready(Ok(()))
8731 }
8732
8733 fn poll_close(
8734 self: Pin<&mut Self>,
8735 _cx: &mut Context<'_>,
8736 ) -> Poll<Result<(), Self::Error>> {
8737 Poll::Ready(Ok(()))
8738 }
8739 }
8740
8741 struct RecordingState {
8742 messages: Arc<Mutex<Vec<Message>>>,
8743 recorded_notify: tokio::sync::Notify,
8744 }
8745
8746 struct RecordingTransport {
8747 state: Arc<RecordingState>,
8748 }
8749
8750 impl futures_util::Stream for RecordingTransport {
8751 type Item = Result<Message, TransportError>;
8752
8753 fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8754 Poll::Pending
8755 }
8756 }
8757
8758 impl futures_util::Sink<Message> for RecordingTransport {
8759 type Error = TransportError;
8760
8761 fn poll_ready(
8762 self: Pin<&mut Self>,
8763 _cx: &mut Context<'_>,
8764 ) -> Poll<Result<(), Self::Error>> {
8765 Poll::Ready(Ok(()))
8766 }
8767
8768 fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> {
8769 self.state.messages.lock().push(item);
8770 self.state.recorded_notify.notify_one();
8771 Ok(())
8772 }
8773
8774 fn poll_flush(
8775 self: Pin<&mut Self>,
8776 _cx: &mut Context<'_>,
8777 ) -> Poll<Result<(), Self::Error>> {
8778 Poll::Ready(Ok(()))
8779 }
8780
8781 fn poll_close(
8782 self: Pin<&mut Self>,
8783 _cx: &mut Context<'_>,
8784 ) -> Poll<Result<(), Self::Error>> {
8785 Poll::Ready(Ok(()))
8786 }
8787 }
8788
8789 #[rstest]
8790 #[tokio::test(start_paused = true)]
8791 async fn test_pong_is_bound_to_connection_epoch() {
8792 let initial_state = Arc::new(RecordingState {
8793 messages: Arc::new(Mutex::new(Vec::new())),
8794 recorded_notify: tokio::sync::Notify::new(),
8795 });
8796
8797 let initial_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8798 state: initial_state,
8799 });
8800
8801 let (writer, _reader) = initial_transport.split();
8802 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8803 let state_notify = Arc::new(tokio::sync::Notify::new());
8804 let auth_tracker = Arc::new(OnceLock::new());
8805 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
8806 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
8807 let write_task = WebSocketClientInner::spawn_write_task(
8808 Arc::clone(&connection_state),
8809 Arc::clone(&state_notify),
8810 Arc::new(AtomicBool::new(true)),
8811 writer,
8812 writer_rx,
8813 Arc::new(AtomicU64::new(0)),
8814 auth_tracker,
8815 reconnect_buffer_waits_for_auth,
8816 None,
8817 );
8818
8819 let recorded = Arc::new(Mutex::new(Vec::new()));
8820
8821 let replacement_state = Arc::new(RecordingState {
8822 messages: Arc::clone(&recorded),
8823 recorded_notify: tokio::sync::Notify::new(),
8824 });
8825
8826 let replacement_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8827 state: replacement_state,
8828 });
8829
8830 let (replacement_writer, _reader) = replacement_transport.split();
8831 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
8832 writer_tx
8833 .send(WriterCommand::Update(replacement_writer, update_tx))
8834 .unwrap();
8835 writer_tx
8836 .send_control(WriterCommand::SendPongOnConnection {
8837 data: b"stale-pong".to_vec(),
8838 connection_epoch: 0,
8839 })
8840 .unwrap();
8841
8842 tokio::time::advance(Duration::from_millis(100)).await;
8843 assert_eq!(update_rx.await.unwrap(), 1);
8844
8845 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8846 writer_tx
8847 .send_control(WriterCommand::SendOnConnection {
8848 message: Message::text("sentinel-1"),
8849 connection_epoch: 1,
8850 response_tx: sentinel_tx,
8851 })
8852 .unwrap();
8853
8854 sentinel_rx.await.unwrap().unwrap();
8855 assert_eq!(recorded.lock().as_slice(), &[Message::text("sentinel-1")]);
8856
8857 writer_tx
8858 .send_control(WriterCommand::SendPongOnConnection {
8859 data: b"fresh-pong".to_vec(),
8860 connection_epoch: 1,
8861 })
8862 .unwrap();
8863
8864 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8865 writer_tx
8866 .send_control(WriterCommand::SendOnConnection {
8867 message: Message::text("sentinel-2"),
8868 connection_epoch: 1,
8869 response_tx: sentinel_tx,
8870 })
8871 .unwrap();
8872
8873 sentinel_rx.await.unwrap().unwrap();
8874 assert_eq!(
8875 recorded.lock().as_slice(),
8876 &[
8877 Message::text("sentinel-1"),
8878 Message::Pong(b"fresh-pong".to_vec().into()),
8879 Message::text("sentinel-2"),
8880 ]
8881 );
8882
8883 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
8884 state_notify.notify_waiters();
8885 drop(writer_tx);
8886 write_task.await.unwrap();
8887 }
8888
8889 #[rstest]
8890 #[case(Message::text("stale"))]
8891 #[case(Message::Binary(vec![1, 2, 3].into()))]
8892 #[case(Message::Ping(vec![1, 2, 3].into()))]
8893 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8894 async fn test_message_handler_drops_old_session_message(#[case] message: Message) {
8895 let (polled_tx, polled_rx) = std::sync::mpsc::channel();
8896
8897 let state = Arc::new(BlockingMessageState {
8898 polled_tx: Mutex::new(Some(polled_tx)),
8899 release: (Mutex::new(false), Condvar::new()),
8900 message: Mutex::new(Some(message)),
8901 });
8902
8903 let release_guard = CondvarReleaseGuard::new(&state.release);
8904
8905 let transport: BoxedWsTransport = Box::pin(BlockingMessageTransport {
8906 state: Arc::clone(&state),
8907 });
8908
8909 let (_writer, reader) = transport.split();
8910 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8911 let state_notify = Arc::new(tokio::sync::Notify::new());
8912 let read_fence = ReadSessionFence::new();
8913 let message_count = Arc::new(AtomicUsize::new(0));
8914 let ping_count = Arc::new(AtomicUsize::new(0));
8915 let message_count_clone = Arc::clone(&message_count);
8916 let ping_count_clone = Arc::clone(&ping_count);
8917 let message_handler: MessageHandler =
8918 Arc::new(move |_| _ = message_count_clone.fetch_add(1, Ordering::SeqCst));
8919 let message_handler = IncomingHandler::Message(message_handler);
8920 let ping_handler: PingHandler =
8921 Arc::new(move |_| _ = ping_count_clone.fetch_add(1, Ordering::SeqCst));
8922 let ping_handler = IncomingPingHandler::Ping(ping_handler);
8923
8924 let read_task = WebSocketClientInner::spawn_message_handler_task(
8925 Arc::clone(&connection_state),
8926 state_notify,
8927 read_fence.clone(),
8928 reader,
8929 0,
8930 Some(&message_handler),
8931 Some(&ping_handler),
8932 None,
8933 None,
8934 None,
8935 None,
8936 None,
8937 );
8938
8939 recv_rendezvous(polled_rx, "WebSocket reader poll entry").await;
8940 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8941 read_fence.invalidate();
8942 read_task.abort();
8943 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
8944 release_guard.release();
8945 await_task_termination(read_task, "old WebSocket read task").await;
8946
8947 assert_eq!(message_count.load(Ordering::SeqCst), 0);
8948 assert_eq!(ping_count.load(Ordering::SeqCst), 0);
8949 }
8950
8951 #[rstest]
8952 #[tokio::test]
8953 async fn test_reconnect_buffer_drain_stops_after_reconnect_request() {
8954 let state = Arc::new(BlockingFailState::default());
8955
8956 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
8957 state: Arc::clone(&state),
8958 });
8959
8960 let (mut writer, _reader) = transport.split();
8961 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8962 let task_connection_state = Arc::clone(&connection_state);
8963
8964 let drain_task = tokio::spawn(async move {
8965 let auth_tracker = Arc::new(OnceLock::new());
8966 let reconnect_buffer_waits_for_auth = AtomicBool::new(false);
8967 let (sender, mut receiver) = crate::writer::channel(2);
8968 sender.send(Message::text("admitted")).unwrap();
8969 sender.send(Message::text("held-for-reconnect")).unwrap();
8970 let mut buffer = VecDeque::from([
8971 receiver.recv().await.unwrap(),
8972 receiver.recv().await.unwrap(),
8973 ]);
8974 let send_error = WebSocketClientInner::drain_reconnect_buffer(
8975 &mut buffer,
8976 &mut writer,
8977 &task_connection_state,
8978 &auth_tracker,
8979 &reconnect_buffer_waits_for_auth,
8980 )
8981 .await;
8982 (buffer, send_error)
8983 });
8984
8985 state.send_entered_notify.notified().await;
8986 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8987 state.release_send();
8988
8989 let (buffer, send_error) = tokio::time::timeout(TEST_TIMEOUT, drain_task)
8990 .await
8991 .expect("buffer drain should stop after reconnect acceptance")
8992 .unwrap();
8993 assert!(!send_error);
8994 assert_eq!(
8995 buffer
8996 .into_iter()
8997 .map(|(message, _)| message)
8998 .collect::<Vec<_>>(),
8999 vec![Message::text("held-for-reconnect")]
9000 );
9001 }
9002
9003 #[rstest]
9004 #[case(0)]
9005 #[case(usize::MAX)]
9006 #[tokio::test]
9007 async fn test_stream_writer_rejects_invalid_capacity(#[case] capacity: usize) {
9008 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9009 state: Arc::new(BlockingFailState::default()),
9010 });
9011
9012 let (writer, _reader) = transport.split();
9013 let mut config = WebSocketConfig::builder()
9014 .url("ws://localhost".to_string())
9015 .build()
9016 .unwrap();
9017 config.writer_capacity = Some(capacity);
9018
9019 let result = WebSocketClientInner::new_with_writer(config, writer).await;
9020
9021 assert!(
9022 matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9023 );
9024 }
9025
9026 #[rstest]
9027 #[case(0)]
9028 #[case(usize::MAX)]
9029 #[tokio::test]
9030 async fn test_handlerless_connect_rejects_invalid_capacity(#[case] capacity: usize) {
9031 let mut config = WebSocketConfig::builder()
9032 .url("ws://127.0.0.1:1".to_string())
9033 .build()
9034 .unwrap();
9035 config.writer_capacity = Some(capacity);
9036
9037 let result = WebSocketClientInner::connect_url(config, None, None).await;
9038
9039 assert!(
9040 matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9041 );
9042 }
9043
9044 #[rstest]
9045 #[tokio::test(start_paused = true)]
9046 async fn test_stalled_websocket_send_reconnects_and_replays() {
9047 let state = Arc::new(BlockingFailState::default());
9048
9049 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9050 state: Arc::clone(&state),
9051 });
9052
9053 let (writer, _reader) = transport.split();
9054 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9055 let state_notify = Arc::new(tokio::sync::Notify::new());
9056 let auth_tracker = Arc::new(OnceLock::new());
9057 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9058 let (writer_tx, writer_rx) = crate::writer::channel(2);
9059 let states = Arc::new(Mutex::new(Vec::new()));
9060 let states_callback = Arc::clone(&states);
9061 let sink = SocketStateSink::new(move |state| {
9062 states_callback.lock().push(state);
9063 });
9064
9065 let write_task = WebSocketClientInner::spawn_write_task(
9066 Arc::clone(&connection_state),
9067 Arc::clone(&state_notify),
9068 Arc::new(AtomicBool::new(true)),
9069 writer,
9070 writer_rx,
9071 Arc::new(AtomicU64::new(0)),
9072 Arc::clone(&auth_tracker),
9073 reconnect_buffer_waits_for_auth,
9074 Some(sink),
9075 );
9076
9077 writer_tx
9078 .send(WriterCommand::Send(Message::text("complete-message")))
9079 .unwrap();
9080 state.send_entered_notify.notified().await;
9081 writer_tx
9082 .send(WriterCommand::Send(Message::text("second-message")))
9083 .unwrap();
9084 assert_eq!(
9085 ConnectionMode::from_atomic(&connection_state),
9086 ConnectionMode::Active
9087 );
9088
9089 for _ in 0..100 {
9090 assert!(matches!(
9091 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9092 Err(SendError::BufferFull)
9093 ));
9094 }
9095
9096 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9097 tokio::task::yield_now().await;
9098 assert_eq!(
9099 ConnectionMode::from_atomic(&connection_state),
9100 ConnectionMode::Reconnect
9101 );
9102 assert!(matches!(
9103 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9104 Err(SendError::BufferFull)
9105 ));
9106
9107 let recorded = Arc::new(Mutex::new(Vec::new()));
9108
9109 let recording_state = Arc::new(RecordingState {
9110 messages: Arc::clone(&recorded),
9111 recorded_notify: tokio::sync::Notify::new(),
9112 });
9113
9114 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9115 state: Arc::clone(&recording_state),
9116 });
9117
9118 let (new_writer, _reader) = transport.split();
9119 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9120 writer_tx
9121 .send_update(WriterCommand::Update(new_writer, update_tx))
9122 .unwrap();
9123
9124 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9125 assert_eq!(
9126 tokio::time::timeout(Duration::from_secs(1), update_rx)
9127 .await
9128 .expect("writer update should not remain queued behind a stalled send")
9129 .unwrap(),
9130 1,
9131 "the replacement sink should install as connection epoch 1"
9132 );
9133 assert_eq!(
9134 ConnectionMode::from_atomic(&connection_state),
9135 ConnectionMode::Reconnect
9136 );
9137
9138 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9139 state_notify.notify_waiters();
9140 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9141 recording_state.recorded_notify.notified().await;
9142 assert_eq!(
9143 recorded.lock().as_slice(),
9144 &[
9145 Message::text("complete-message"),
9146 Message::text("second-message")
9147 ]
9148 );
9149
9150 writer_tx
9151 .send(WriterCommand::Send(Message::text("second-message")))
9152 .unwrap();
9153
9154 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9155 state_notify.notify_waiters();
9156 drop(writer_tx);
9157 write_task.await.unwrap();
9158
9159 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
9160 }
9161
9162 #[rstest]
9163 #[case(Message::Ping(vec![1, 2, 3].into()))]
9164 #[case(Message::Pong(vec![4, 5, 6].into()))]
9165 #[case(Message::Close(None))]
9166 #[tokio::test(start_paused = true)]
9167 async fn test_stalled_control_frame_is_not_replayed(#[case] control: Message) {
9168 let state = Arc::new(BlockingFailState::default());
9171
9172 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9173 state: Arc::clone(&state),
9174 });
9175
9176 let (writer, _reader) = transport.split();
9177 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9178 let state_notify = Arc::new(tokio::sync::Notify::new());
9179 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9180 let write_task = WebSocketClientInner::spawn_write_task(
9181 Arc::clone(&connection_state),
9182 Arc::clone(&state_notify),
9183 Arc::new(AtomicBool::new(true)),
9184 writer,
9185 writer_rx,
9186 Arc::new(AtomicU64::new(0)),
9187 Arc::new(OnceLock::new()),
9188 Arc::new(AtomicBool::new(false)),
9189 None,
9190 );
9191
9192 writer_tx.send(WriterCommand::Send(control)).unwrap();
9193 state.send_entered_notify.notified().await;
9194
9195 let recorded = Arc::new(Mutex::new(Vec::new()));
9196
9197 let recording_state = Arc::new(RecordingState {
9198 messages: Arc::clone(&recorded),
9199 recorded_notify: tokio::sync::Notify::new(),
9200 });
9201
9202 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9203 state: recording_state,
9204 });
9205
9206 let (new_writer, _reader) = transport.split();
9207 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9208 writer_tx
9209 .send(WriterCommand::Update(new_writer, update_tx))
9210 .unwrap();
9211
9212 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9213 assert_eq!(
9214 tokio::time::timeout(Duration::from_secs(1), update_rx)
9215 .await
9216 .expect("writer update should not remain queued behind a stalled send")
9217 .unwrap(),
9218 1,
9219 "the replacement sink should install as connection epoch 1"
9220 );
9221 assert_eq!(
9222 ConnectionMode::from_atomic(&connection_state),
9223 ConnectionMode::Reconnect,
9224 "a failed control-frame write should still trigger a reconnect"
9225 );
9226
9227 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9228 state_notify.notify_waiters();
9229
9230 for _ in 0..5 {
9232 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9233 }
9234
9235 let replayed = recorded.lock().clone();
9236 assert!(
9237 replayed.is_empty(),
9238 "a failed control frame must not reach the replacement connection, was {replayed:?}"
9239 );
9240
9241 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9242 state_notify.notify_waiters();
9243 drop(writer_tx);
9244 write_task.await.unwrap();
9245 }
9246
9247 #[rstest]
9248 #[case(Message::Ping(vec![1, 2, 3].into()))]
9249 #[case(Message::Pong(vec![4, 5, 6].into()))]
9250 #[case(Message::Close(None))]
9251 #[tokio::test(start_paused = true)]
9252 async fn test_control_frame_enqueued_during_reconnect_is_not_replayed(
9253 #[case] control: Message,
9254 ) {
9255 let recorded = Arc::new(Mutex::new(Vec::new()));
9258
9259 let recording_state = Arc::new(RecordingState {
9260 messages: Arc::clone(&recorded),
9261 recorded_notify: tokio::sync::Notify::new(),
9262 });
9263
9264 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9265 state: recording_state,
9266 });
9267
9268 let (writer, _reader) = transport.split();
9269 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9270 let state_notify = Arc::new(tokio::sync::Notify::new());
9271 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9272 let write_task = WebSocketClientInner::spawn_write_task(
9273 Arc::clone(&connection_state),
9274 Arc::clone(&state_notify),
9275 Arc::new(AtomicBool::new(true)),
9276 writer,
9277 writer_rx,
9278 Arc::new(AtomicU64::new(0)),
9279 Arc::new(OnceLock::new()),
9280 Arc::new(AtomicBool::new(false)),
9281 None,
9282 );
9283
9284 writer_tx.send(WriterCommand::Send(control)).unwrap();
9285 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9286
9287 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9288 state_notify.notify_waiters();
9289
9290 for _ in 0..5 {
9292 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9293 }
9294
9295 let replayed = recorded.lock().clone();
9296 assert!(
9297 replayed.is_empty(),
9298 "a control frame enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9299 );
9300
9301 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9302 state_notify.notify_waiters();
9303 drop(writer_tx);
9304 write_task.await.unwrap();
9305 }
9306
9307 #[rstest]
9308 #[tokio::test(start_paused = true)]
9309 async fn test_stalled_text_heartbeat_is_not_replayed() {
9310 let state = Arc::new(BlockingFailState::default());
9311
9312 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9313 state: Arc::clone(&state),
9314 });
9315
9316 let (writer, _reader) = transport.split();
9317 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9318 let state_notify = Arc::new(tokio::sync::Notify::new());
9319 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9320 let write_task = WebSocketClientInner::spawn_write_task(
9321 Arc::clone(&connection_state),
9322 Arc::clone(&state_notify),
9323 Arc::new(AtomicBool::new(true)),
9324 writer,
9325 writer_rx,
9326 Arc::new(AtomicU64::new(0)),
9327 Arc::new(OnceLock::new()),
9328 Arc::new(AtomicBool::new(false)),
9329 None,
9330 );
9331
9332 writer_tx
9333 .send(WriterCommand::Heartbeat(Message::text(
9334 "{\"op\":\"heartbeat\"}",
9335 )))
9336 .unwrap();
9337 state.send_entered_notify.notified().await;
9338
9339 let recorded = Arc::new(Mutex::new(Vec::new()));
9340
9341 let recording_state = Arc::new(RecordingState {
9342 messages: Arc::clone(&recorded),
9343 recorded_notify: tokio::sync::Notify::new(),
9344 });
9345
9346 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9347 state: recording_state,
9348 });
9349
9350 let (new_writer, _reader) = transport.split();
9351 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9352 writer_tx
9353 .send(WriterCommand::Update(new_writer, update_tx))
9354 .unwrap();
9355
9356 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9357 assert_eq!(
9358 tokio::time::timeout(Duration::from_secs(1), update_rx)
9359 .await
9360 .expect("writer update should not remain queued behind a stalled send")
9361 .unwrap(),
9362 1,
9363 "the replacement sink should install as connection epoch 1"
9364 );
9365 assert_eq!(
9366 ConnectionMode::from_atomic(&connection_state),
9367 ConnectionMode::Reconnect,
9368 "a failed heartbeat write should still trigger a reconnect"
9369 );
9370
9371 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9372 state_notify.notify_waiters();
9373
9374 for _ in 0..5 {
9375 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9376 }
9377
9378 let replayed = recorded.lock().clone();
9379 assert!(
9380 replayed.is_empty(),
9381 "a failed text heartbeat must not reach the replacement connection, was {replayed:?}"
9382 );
9383
9384 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9385 state_notify.notify_waiters();
9386 drop(writer_tx);
9387 write_task.await.unwrap();
9388 }
9389
9390 #[rstest]
9391 #[tokio::test(start_paused = true)]
9392 async fn test_text_heartbeat_enqueued_during_reconnect_is_not_replayed() {
9393 let recorded = Arc::new(Mutex::new(Vec::new()));
9394
9395 let recording_state = Arc::new(RecordingState {
9396 messages: Arc::clone(&recorded),
9397 recorded_notify: tokio::sync::Notify::new(),
9398 });
9399
9400 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9401 state: recording_state,
9402 });
9403
9404 let (writer, _reader) = transport.split();
9405 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9406 let state_notify = Arc::new(tokio::sync::Notify::new());
9407 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9408 let write_task = WebSocketClientInner::spawn_write_task(
9409 Arc::clone(&connection_state),
9410 Arc::clone(&state_notify),
9411 Arc::new(AtomicBool::new(true)),
9412 writer,
9413 writer_rx,
9414 Arc::new(AtomicU64::new(0)),
9415 Arc::new(OnceLock::new()),
9416 Arc::new(AtomicBool::new(false)),
9417 None,
9418 );
9419
9420 writer_tx
9421 .send(WriterCommand::Heartbeat(Message::text(
9422 "{\"op\":\"heartbeat\"}",
9423 )))
9424 .unwrap();
9425 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9426
9427 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9428 state_notify.notify_waiters();
9429
9430 for _ in 0..5 {
9431 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9432 }
9433
9434 let replayed = recorded.lock().clone();
9435 assert!(
9436 replayed.is_empty(),
9437 "a text heartbeat enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9438 );
9439
9440 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9441 state_notify.notify_waiters();
9442 drop(writer_tx);
9443 write_task.await.unwrap();
9444 }
9445
9446 #[rstest]
9447 #[case::text(
9448 Some("{\"op\":\"heartbeat\"}"),
9449 Message::text("{\"op\":\"heartbeat\"}")
9450 )]
9451 #[case::ping(None, Message::Ping(vec![].into()))]
9452 #[tokio::test(start_paused = true)]
9453 async fn test_heartbeat_task_enqueues_writer_heartbeat_command(
9454 #[case] payload: Option<&str>,
9455 #[case] expected: Message,
9456 ) {
9457 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9458 let (writer_tx, mut writer_rx) =
9459 crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9460 let task = WebSocketClientInner::spawn_heartbeat_task(
9461 Arc::clone(&connection_state),
9462 1,
9463 payload.map(ToString::to_string),
9464 writer_tx,
9465 );
9466
9467 tokio::time::advance(Duration::from_secs(1)).await;
9468 let cmd = writer_rx
9469 .recv()
9470 .await
9471 .expect("heartbeat task should enqueue");
9472
9473 match cmd.0 {
9474 WriterCommand::Heartbeat(msg) => assert_eq!(msg, expected),
9475 other => panic!("expected Heartbeat, was {other:?}"),
9476 }
9477
9478 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9479 tokio::time::advance(Duration::from_secs(1)).await;
9480 task.await.unwrap();
9481 }
9482
9483 #[cfg(target_os = "linux")]
9484 #[rstest]
9485 #[tokio::test]
9486 async fn test_heartbeat_task_warns_when_writer_channel_closed() {
9487 let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
9488 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9489 let (writer_tx, mut writer_rx) =
9490 crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9491 let task = WebSocketClientInner::spawn_heartbeat_task(
9492 Arc::clone(&connection_state),
9493 1,
9494 None,
9495 writer_tx,
9496 );
9497
9498 let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
9499 .await
9500 .expect("timed out waiting for the first heartbeat")
9501 .expect("heartbeat channel closed before the first heartbeat");
9502 drop(writer_rx);
9503 tokio::time::sleep(Duration::from_millis(1_200)).await;
9504
9505 let messages = capture.messages();
9506 assert!(
9507 messages.iter().any(|(level, message)| {
9508 *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
9509 }),
9510 "expected WARN when the writer channel is closed, was {messages:?}"
9511 );
9512 assert!(
9513 !messages.iter().any(|(level, _)| *level == Level::Error),
9514 "a closed writer channel must not log ERROR, was {messages:?}"
9515 );
9516
9517 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9518 tokio::time::timeout(Duration::from_secs(2), task)
9519 .await
9520 .expect("heartbeat task should stop after close")
9521 .unwrap();
9522 }
9523
9524 #[rstest]
9525 #[tokio::test(start_paused = true)]
9526 async fn test_stalled_ownership_bound_send_times_out_without_replay() {
9527 let state = Arc::new(BlockingFailState::default());
9528
9529 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9530 state: Arc::clone(&state),
9531 });
9532
9533 let (writer, _reader) = transport.split();
9534 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9535 let state_notify = Arc::new(tokio::sync::Notify::new());
9536 let auth_tracker = Arc::new(OnceLock::new());
9537 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9538 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9539 let write_task = WebSocketClientInner::spawn_write_task(
9540 Arc::clone(&connection_state),
9541 Arc::clone(&state_notify),
9542 Arc::new(AtomicBool::new(true)),
9543 writer,
9544 writer_rx,
9545 Arc::new(AtomicU64::new(0)),
9546 Arc::clone(&auth_tracker),
9547 reconnect_buffer_waits_for_auth,
9548 None,
9549 );
9550
9551 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
9552 writer_tx
9553 .send_control(WriterCommand::SendOnConnection {
9554 message: Message::text("ownership-bound"),
9555 connection_epoch: 0,
9556 response_tx,
9557 })
9558 .unwrap();
9559
9560 state.send_entered_notify.notified().await;
9561
9562 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9563 let outcome = tokio::time::timeout(Duration::from_secs(1), response_rx)
9564 .await
9565 .expect("a stalled ownership-bound send must not wedge the writer task")
9566 .unwrap();
9567 assert!(
9568 matches!(outcome, Err(SendError::WriteTimeout)),
9569 "expected the write deadline to be reported, was {outcome:?}"
9570 );
9571 assert_eq!(
9572 ConnectionMode::from_atomic(&connection_state),
9573 ConnectionMode::Reconnect
9574 );
9575
9576 let recorded = Arc::new(Mutex::new(Vec::new()));
9579
9580 let recording_state = Arc::new(RecordingState {
9581 messages: Arc::clone(&recorded),
9582 recorded_notify: tokio::sync::Notify::new(),
9583 });
9584
9585 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9586 state: Arc::clone(&recording_state),
9587 });
9588
9589 let (new_writer, _reader) = transport.split();
9590 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9591 writer_tx
9592 .send(WriterCommand::Update(new_writer, update_tx))
9593 .unwrap();
9594
9595 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9596 assert_eq!(
9597 tokio::time::timeout(Duration::from_secs(1), update_rx)
9598 .await
9599 .expect("writer update should not remain queued behind a stalled send")
9600 .unwrap(),
9601 1
9602 );
9603
9604 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9605 state_notify.notify_waiters();
9606 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9607
9608 for name in ["sentinel-1", "sentinel-2"] {
9614 let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
9615 writer_tx
9616 .send_control(WriterCommand::SendOnConnection {
9617 message: Message::text(name),
9618 connection_epoch: 1,
9619 response_tx: sentinel_tx,
9620 })
9621 .unwrap();
9622
9623 recording_state.recorded_notify.notified().await;
9624 sentinel_rx
9625 .await
9626 .unwrap()
9627 .expect("the sentinel should send on the replacement connection");
9628 }
9629
9630 assert_eq!(
9631 recorded.lock().as_slice(),
9632 &[Message::text("sentinel-1"), Message::text("sentinel-2")],
9633 "an ownership-bound message must never be replayed after its deadline expires"
9634 );
9635
9636 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9637 state_notify.notify_waiters();
9638 drop(writer_tx);
9639 write_task.await.unwrap();
9640 }
9641
9642 #[rstest]
9643 #[tokio::test(start_paused = true)]
9644 async fn test_stalled_websocket_replay_reconnects_and_retries_buffer() {
9645 let initial_messages = Arc::new(Mutex::new(Vec::new()));
9646
9647 let initial_recording_state = Arc::new(RecordingState {
9648 messages: initial_messages,
9649 recorded_notify: tokio::sync::Notify::new(),
9650 });
9651
9652 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9653 state: initial_recording_state,
9654 });
9655
9656 let (writer, _reader) = transport.split();
9657 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9658 let state_notify = Arc::new(tokio::sync::Notify::new());
9659 let auth_tracker = Arc::new(OnceLock::new());
9660 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9661 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9662 let write_task = WebSocketClientInner::spawn_write_task(
9663 Arc::clone(&connection_state),
9664 Arc::clone(&state_notify),
9665 Arc::new(AtomicBool::new(true)),
9666 writer,
9667 writer_rx,
9668 Arc::new(AtomicU64::new(0)),
9669 Arc::clone(&auth_tracker),
9670 reconnect_buffer_waits_for_auth,
9671 None,
9672 );
9673
9674 writer_tx
9675 .send(WriterCommand::Send(Message::text("buffered-message")))
9676 .unwrap();
9677
9678 let blocking_state = Arc::new(BlockingFailState::default());
9679
9680 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9681 state: Arc::clone(&blocking_state),
9682 });
9683
9684 let (blocking_writer, _reader) = transport.split();
9685 let (blocking_tx, blocking_rx) = tokio::sync::oneshot::channel();
9686 writer_tx
9687 .send(WriterCommand::Update(blocking_writer, blocking_tx))
9688 .unwrap();
9689 assert_eq!(blocking_rx.await.unwrap(), 1);
9690
9691 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9692 state_notify.notify_waiters();
9693 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9694 blocking_state.send_entered_notify.notified().await;
9695
9696 let recorded = Arc::new(Mutex::new(Vec::new()));
9697
9698 let recording_state = Arc::new(RecordingState {
9699 messages: Arc::clone(&recorded),
9700 recorded_notify: tokio::sync::Notify::new(),
9701 });
9702
9703 let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9704 state: Arc::clone(&recording_state),
9705 });
9706
9707 let (new_writer, _reader) = transport.split();
9708 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9709 writer_tx
9710 .send(WriterCommand::Update(new_writer, update_tx))
9711 .unwrap();
9712
9713 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9714 assert_eq!(
9715 tokio::time::timeout(Duration::from_secs(1), update_rx)
9716 .await
9717 .expect("writer update should not remain queued behind stalled replay")
9718 .unwrap(),
9719 2,
9720 "the second replacement sink should install as connection epoch 2"
9721 );
9722 assert_eq!(
9723 ConnectionMode::from_atomic(&connection_state),
9724 ConnectionMode::Reconnect
9725 );
9726
9727 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9728 state_notify.notify_waiters();
9729 tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9730 recording_state.recorded_notify.notified().await;
9731 assert_eq!(
9732 recorded.lock().as_slice(),
9733 &[Message::text("buffered-message")]
9734 );
9735
9736 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9737 state_notify.notify_waiters();
9738 drop(writer_tx);
9739 write_task.await.unwrap();
9740 }
9741
9742 #[rstest]
9743 #[tokio::test]
9744 async fn test_new_with_writer_rejects_zero_heartbeat() {
9745 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9748 state: Arc::new(BlockingFailState::default()),
9749 });
9750
9751 let (writer, _reader) = transport.split();
9752
9753 let config = WebSocketConfig {
9754 url: "ws://127.0.0.1:1".to_string(),
9755 headers: vec![],
9756 heartbeat_interval_secs: Some(0),
9757 heartbeat_payload: None,
9758 connect_timeout_ms: None,
9759 reconnect_delay_initial_ms: None,
9760 reconnect_delay_max_ms: None,
9761 reconnect_backoff_factor: None,
9762 reconnect_jitter_ms: None,
9763 reconnect_max_attempts: None,
9764 heartbeat_timeout_secs: None,
9765 idle_timeout_ms: None,
9766 writer_capacity: None,
9767 backend: TransportBackend::Tungstenite,
9768 proxy_url: None,
9769 max_message_size_bytes: None,
9770 max_frame_size_bytes: None,
9771 };
9772
9773 let err = WebSocketClientInner::new_with_writer(config, writer)
9774 .await
9775 .expect_err("zero heartbeat should be rejected in stream mode");
9776 assert!(
9777 err.to_string()
9778 .contains("Heartbeat interval cannot be zero"),
9779 "error should mention zero heartbeat, was: {err}"
9780 );
9781 }
9782
9783 #[rstest]
9784 #[tokio::test]
9785 async fn test_connect_times_out_on_silent_server() {
9786 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9789 let port = listener.local_addr().unwrap().port();
9790
9791 let server = task::spawn(async move {
9792 if let Ok((_stream, _)) = listener.accept().await {
9794 sleep(Duration::from_secs(30)).await;
9795 }
9796 });
9797
9798 let (handler, _rx) = channel_message_handler();
9799
9800 let config = WebSocketConfig {
9801 url: format!("ws://127.0.0.1:{port}"),
9802 headers: vec![],
9803 heartbeat_interval_secs: None,
9804 heartbeat_payload: None,
9805 connect_timeout_ms: Some(500),
9806 reconnect_delay_initial_ms: Some(50),
9807 reconnect_delay_max_ms: Some(100),
9808 reconnect_backoff_factor: Some(1.0),
9809 reconnect_jitter_ms: Some(0),
9810 reconnect_max_attempts: None,
9811 heartbeat_timeout_secs: None,
9812 idle_timeout_ms: None,
9813 writer_capacity: None,
9814 backend: TransportBackend::Tungstenite,
9815 proxy_url: None,
9816 max_message_size_bytes: None,
9817 max_frame_size_bytes: None,
9818 };
9819
9820 let result = tokio::time::timeout(
9821 Duration::from_secs(5),
9822 WebSocketClient::builder()
9823 .config(config)
9824 .message_handler(handler)
9825 .connect(),
9826 )
9827 .await
9828 .expect("connect should not hang on a silent server");
9829
9830 assert!(result.is_err(), "connect should fail with a timeout error");
9831 let err_msg = result.unwrap_err().to_string();
9832 assert!(
9833 err_msg.contains("timed out"),
9834 "error should mention the timeout, was: {err_msg}"
9835 );
9836
9837 server.abort();
9838 }
9839
9840 #[rstest]
9841 #[tokio::test]
9842 async fn test_reconnect_succeeds_with_timeout_shorter_than_swap_ceremony() {
9843 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9849 let port = listener.local_addr().unwrap().port();
9850 let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel();
9851
9852 let server = task::spawn(async move {
9853 if let Ok((stream, _)) = listener.accept().await
9856 && let Ok(ws) = accept_async(stream).await
9857 {
9858 let _ = release_first_rx.await;
9859 drop(ws);
9860 }
9861
9862 loop {
9865 let Ok((stream, _)) = listener.accept().await else {
9866 break;
9867 };
9868
9869 if let Ok(mut ws) = accept_async(stream).await {
9870 loop {
9871 if ws
9872 .send(WsMessage::Text("reconnected-msg".to_string().into()))
9873 .await
9874 .is_err()
9875 {
9876 break;
9877 }
9878
9879 sleep(Duration::from_millis(50)).await;
9880 }
9881 }
9882 }
9883 });
9884
9885 let (handler, mut rx) = channel_message_handler();
9886
9887 let config = WebSocketConfig {
9888 url: format!("ws://127.0.0.1:{port}"),
9889 headers: vec![],
9890 heartbeat_interval_secs: None,
9891 heartbeat_payload: None,
9892 connect_timeout_ms: Some(150), reconnect_delay_initial_ms: Some(25),
9894 reconnect_delay_max_ms: Some(50),
9895 reconnect_backoff_factor: Some(1.0),
9896 reconnect_jitter_ms: Some(0),
9897 reconnect_max_attempts: None,
9898 heartbeat_timeout_secs: None,
9899 idle_timeout_ms: None,
9900 writer_capacity: None,
9901 backend: TransportBackend::Tungstenite,
9902 proxy_url: None,
9903 max_message_size_bytes: None,
9904 max_frame_size_bytes: None,
9905 };
9906
9907 let client = WebSocketClient::builder()
9908 .config(config)
9909 .message_handler(handler)
9910 .connect()
9911 .await
9912 .expect("Initial WebSocket connection should succeed before triggering reconnect");
9913 wait_until_async(|| async { client.is_active() }, Duration::from_secs(2)).await;
9914 release_first_tx
9915 .send(())
9916 .expect("server should still be holding the first connection");
9917
9918 let received = tokio::time::timeout(Duration::from_secs(5), async {
9919 loop {
9920 match rx.recv().await {
9921 Some(WsMessage::Text(text)) if text.as_str() == "reconnected-msg" => {
9922 return true;
9923 }
9924 Some(_) => {}
9925 None => return false,
9926 }
9927 }
9928 })
9929 .await
9930 .expect("Timed out waiting for a message from the replacement WebSocket connection");
9931
9932 assert!(
9933 received,
9934 "Replacement WebSocket message channel closed before delivering the reconnect message"
9935 );
9936 assert!(
9937 client.is_active(),
9938 "WebSocket client should be active after delivering the reconnect message"
9939 );
9940
9941 client.disconnect().await;
9942 server.abort();
9943 }
9944
9945 #[rstest]
9946 #[tokio::test]
9947 async fn test_idle_timeout_fires_under_ping_flood() {
9948 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9952 let port = listener.local_addr().unwrap().port();
9953 let server = task::spawn(async move {
9954 let (stream, _) = listener.accept().await.unwrap();
9955 let mut ws = accept_async(stream).await.unwrap();
9956
9957 for _ in 0..600 {
9958 sleep(Duration::from_millis(5)).await;
9959
9960 if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
9961 break;
9962 }
9963 }
9964 });
9965
9966 let (handler, _rx) = channel_message_handler();
9967
9968 let config = WebSocketConfig {
9969 url: format!("ws://127.0.0.1:{port}"),
9970 headers: vec![],
9971 heartbeat_interval_secs: None,
9972 heartbeat_payload: None,
9973 connect_timeout_ms: Some(2_000),
9974 reconnect_delay_initial_ms: Some(50),
9975 reconnect_delay_max_ms: Some(100),
9976 reconnect_backoff_factor: Some(1.0),
9977 reconnect_jitter_ms: Some(0),
9978 reconnect_max_attempts: Some(1),
9979 heartbeat_timeout_secs: None,
9980 idle_timeout_ms: Some(500),
9981 writer_capacity: None,
9982 backend: TransportBackend::Tungstenite,
9983 proxy_url: None,
9984 max_message_size_bytes: None,
9985 max_frame_size_bytes: None,
9986 };
9987
9988 let client = WebSocketClient::builder()
9989 .config(config)
9990 .message_handler(handler)
9991 .connect()
9992 .await
9993 .unwrap();
9994
9995 assert!(client.is_active());
9996
9997 wait_until_async(
9998 || async { client.is_reconnecting() || client.is_disconnected() },
9999 Duration::from_millis(1_500),
10000 )
10001 .await;
10002
10003 assert!(
10004 !client.is_active(),
10005 "Client should not be active after idle timeout under a ping flood"
10006 );
10007
10008 client.disconnect().await;
10009 server.abort();
10010 }
10011
10012 #[rstest]
10013 #[tokio::test]
10014 async fn test_send_failure_does_not_overwrite_disconnect() {
10015 let state = Arc::new(BlockingFailState::default());
10016
10017 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10018 state: Arc::clone(&state),
10019 });
10020
10021 let (writer, _reader) = transport.split();
10022
10023 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10024 let state_notify = Arc::new(tokio::sync::Notify::new());
10025 let auth_tracker = Arc::new(OnceLock::new());
10026 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
10027 let connection_epoch = Arc::new(AtomicU64::new(0));
10028
10029 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10030 let write_task = WebSocketClientInner::spawn_write_task(
10031 Arc::clone(&connection_state),
10032 Arc::clone(&state_notify),
10033 Arc::new(AtomicBool::new(true)),
10034 writer,
10035 writer_rx,
10036 connection_epoch,
10037 Arc::clone(&auth_tracker),
10038 Arc::clone(&reconnect_buffer_waits_for_auth),
10039 None,
10040 );
10041
10042 writer_tx
10043 .send(WriterCommand::Send(Message::text("doomed")))
10044 .unwrap();
10045
10046 wait_until_async(
10048 || {
10049 let state = Arc::clone(&state);
10050 async move { state.send_entered.load(Ordering::SeqCst) }
10051 },
10052 Duration::from_secs(2),
10053 )
10054 .await;
10055
10056 connection_state.store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
10059 state.trigger_failure();
10060
10061 tokio::time::timeout(Duration::from_secs(2), write_task)
10062 .await
10063 .expect("write task should exit after disconnect")
10064 .unwrap();
10065
10066 assert_eq!(
10067 ConnectionMode::from_atomic(&connection_state),
10068 ConnectionMode::Disconnect,
10069 "Send failure must not resurrect a disconnecting client into RECONNECT"
10070 );
10071 }
10072
10073 #[tokio::test]
10074 async fn send_on_connection_write_failure_reports_broken_pipe_and_reconnects() {
10075 let state = Arc::new(BlockingFailState::default());
10076
10077 let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10078 state: Arc::clone(&state),
10079 });
10080
10081 let (writer, _reader) = transport.split();
10082 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10083 let state_notify = Arc::new(tokio::sync::Notify::new());
10084 let connection_epoch = Arc::new(AtomicU64::new(0));
10085 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10086 let write_task = WebSocketClientInner::spawn_write_task(
10087 Arc::clone(&connection_state),
10088 Arc::clone(&state_notify),
10089 Arc::new(AtomicBool::new(true)),
10090 writer,
10091 writer_rx,
10092 Arc::clone(&connection_epoch),
10093 Arc::new(OnceLock::new()),
10094 Arc::new(AtomicBool::new(false)),
10095 None,
10096 );
10097
10098 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10099 writer_tx
10100 .send_control(WriterCommand::SendOnConnection {
10101 message: Message::text("doomed"),
10102 connection_epoch: 0,
10103 response_tx,
10104 })
10105 .unwrap();
10106
10107 wait_until_async(
10108 || {
10109 let state = Arc::clone(&state);
10110 async move { state.send_entered.load(Ordering::SeqCst) }
10111 },
10112 Duration::from_secs(2),
10113 )
10114 .await;
10115
10116 state.trigger_failure();
10117
10118 match response_rx.await.unwrap().unwrap_err() {
10119 SendError::BrokenPipe(message) => assert_eq!(message, "connection reset"),
10120 other => panic!("expected broken-pipe send error, was {other:?}"),
10121 }
10122
10123 wait_until_async(
10124 || async {
10125 ConnectionMode::from_atomic(&connection_state) == ConnectionMode::Reconnect
10126 },
10127 Duration::from_secs(2),
10128 )
10129 .await;
10130
10131 assert_eq!(
10132 ConnectionMode::from_atomic(&connection_state),
10133 ConnectionMode::Reconnect,
10134 );
10135 assert_eq!(connection_epoch.load(Ordering::Acquire), 0);
10136
10137 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10138 state_notify.notify_waiters();
10139 drop(writer_tx);
10140 write_task.await.unwrap();
10141 }
10142
10143 #[rstest]
10144 #[tokio::test]
10145 async fn test_send_bytes_on_connection_rejects_stale_epoch_without_replay() {
10146 let server = RecordingServer::setup().await;
10147 let (handler, _handler_rx) = channel_message_handler();
10148 let client = WebSocketClient::builder()
10149 .config(reconnect_test_config(server.port))
10150 .message_handler(handler)
10151 .connect()
10152 .await
10153 .unwrap();
10154 let stale_epoch = client.connection_epoch();
10155
10156 client
10157 .send_bytes_on_connection(vec![0x00, 0x7f, 0x80, 0xff], None, stale_epoch)
10158 .await
10159 .unwrap();
10160 assert!(client.request_reconnect());
10161 wait_until_async(
10162 || async { client.is_active() && client.connection_epoch() == stale_epoch + 1 },
10163 TEST_TIMEOUT,
10164 )
10165 .await;
10166
10167 let stale_result = client
10168 .send_bytes_on_connection(vec![0xde, 0xad], None, stale_epoch)
10169 .await;
10170 assert!(matches!(stale_result, Err(SendError::ConnectionChanged)));
10171
10172 client
10173 .send_bytes_on_connection(
10174 vec![0xfe, 0xed, 0xfa, 0xce],
10175 None,
10176 client.connection_epoch(),
10177 )
10178 .await
10179 .unwrap();
10180 wait_until_async(
10181 || {
10182 let binary_frames = Arc::clone(&server.binary_frames);
10183 async move { binary_frames.lock().await.len() == 2 }
10184 },
10185 TEST_TIMEOUT,
10186 )
10187 .await;
10188
10189 assert_eq!(
10190 server.binary_frames().await,
10191 vec![vec![0x00, 0x7f, 0x80, 0xff], vec![0xfe, 0xed, 0xfa, 0xce]],
10192 );
10193 assert!(server.messages().await.is_empty());
10194
10195 client.disconnect().await;
10196 }
10197
10198 #[tokio::test]
10199 async fn send_on_connection_rejects_stale_epoch_without_replay() {
10200 let server = RecordingServer::setup().await;
10201 let url = format!("ws://127.0.0.1:{}", server.port);
10202 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10203 &url,
10204 vec![],
10205 TransportBackend::Tungstenite,
10206 None,
10207 None,
10208 None,
10209 )
10210 .await
10211 .unwrap();
10212 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10213 let state_notify = Arc::new(tokio::sync::Notify::new());
10214 let auth_tracker = Arc::new(OnceLock::new());
10215 let connection_epoch = Arc::new(AtomicU64::new(0));
10216 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10217 let write_task = WebSocketClientInner::spawn_write_task(
10218 Arc::clone(&connection_state),
10219 Arc::clone(&state_notify),
10220 Arc::new(AtomicBool::new(true)),
10221 writer,
10222 writer_rx,
10223 Arc::clone(&connection_epoch),
10224 auth_tracker,
10225 Arc::new(AtomicBool::new(false)),
10226 None,
10227 );
10228
10229 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10230 writer_tx
10231 .send_control(WriterCommand::SendOnConnection {
10232 message: Message::text("epoch-0"),
10233 connection_epoch: 0,
10234 response_tx,
10235 })
10236 .unwrap();
10237
10238 response_rx.await.unwrap().unwrap();
10239
10240 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10241 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10242 writer_tx
10243 .send_control(WriterCommand::SendOnConnection {
10244 message: Message::text("during-reconnect"),
10245 connection_epoch: 0,
10246 response_tx,
10247 })
10248 .unwrap();
10249
10250 assert!(matches!(
10251 response_rx.await.unwrap(),
10252 Err(SendError::ConnectionChanged),
10253 ));
10254
10255 let (replacement, _reader) = WebSocketClientInner::connect_with_server(
10256 &url,
10257 vec![],
10258 TransportBackend::Tungstenite,
10259 None,
10260 None,
10261 None,
10262 )
10263 .await
10264 .unwrap();
10265 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10266 writer_tx
10267 .send(WriterCommand::Update(replacement, update_tx))
10268 .unwrap();
10269 assert_eq!(update_rx.await.unwrap(), 1);
10270 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10271
10272 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10273 writer_tx
10274 .send_control(WriterCommand::SendOnConnection {
10275 message: Message::text("stale"),
10276 connection_epoch: 0,
10277 response_tx,
10278 })
10279 .unwrap();
10280
10281 assert!(matches!(
10282 response_rx.await.unwrap(),
10283 Err(SendError::ConnectionChanged),
10284 ));
10285
10286 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10287 writer_tx
10288 .send_control(WriterCommand::SendOnConnection {
10289 message: Message::text("epoch-1"),
10290 connection_epoch: 1,
10291 response_tx,
10292 })
10293 .unwrap();
10294
10295 response_rx.await.unwrap().unwrap();
10296
10297 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10298 let (second_replacement, _reader) = WebSocketClientInner::connect_with_server(
10299 &url,
10300 vec![],
10301 TransportBackend::Tungstenite,
10302 None,
10303 None,
10304 None,
10305 )
10306 .await
10307 .unwrap();
10308 let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10309 writer_tx
10310 .send(WriterCommand::Update(second_replacement, update_tx))
10311 .unwrap();
10312 assert_eq!(update_rx.await.unwrap(), 2);
10313 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10314
10315 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10316 writer_tx
10317 .send_control(WriterCommand::SendOnConnection {
10318 message: Message::text("stale-after-second-reconnect"),
10319 connection_epoch: 1,
10320 response_tx,
10321 })
10322 .unwrap();
10323
10324 assert!(matches!(
10325 response_rx.await.unwrap(),
10326 Err(SendError::ConnectionChanged),
10327 ));
10328
10329 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10330 writer_tx
10331 .send_control(WriterCommand::SendOnConnection {
10332 message: Message::text("epoch-2"),
10333 connection_epoch: 2,
10334 response_tx,
10335 })
10336 .unwrap();
10337
10338 response_rx.await.unwrap().unwrap();
10339
10340 wait_until_async(
10341 || {
10342 let messages = Arc::clone(&server.messages);
10343 async move { messages.lock().await.len() == 3 }
10344 },
10345 Duration::from_secs(2),
10346 )
10347 .await;
10348
10349 assert_eq!(connection_epoch.load(Ordering::Acquire), 2);
10350 assert_eq!(
10351 server.messages().await,
10352 vec![
10353 "epoch-0".to_string(),
10354 "epoch-1".to_string(),
10355 "epoch-2".to_string(),
10356 ],
10357 );
10358
10359 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10360 state_notify.notify_waiters();
10361 drop(writer_tx);
10362 write_task.abort();
10363 }
10364
10365 #[rstest]
10366 fn test_reconnect_buffer_action_requires_active_mode() {
10367 let connection_state = AtomicU8::new(ConnectionMode::Active.as_u8());
10368 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
10369 let auth_tracker = Arc::new(OnceLock::new());
10370 let tracker = AuthTracker::new();
10371 tracker.succeed();
10372 auth_tracker.set(tracker).unwrap();
10373
10374 assert_eq!(
10375 WebSocketClientInner::reconnect_buffer_action(
10376 &reconnect_buffer_waits_for_auth,
10377 &auth_tracker,
10378 &connection_state,
10379 ),
10380 ReconnectBufferAction::Drain,
10381 );
10382
10383 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10384 assert_eq!(
10385 WebSocketClientInner::reconnect_buffer_action(
10386 &reconnect_buffer_waits_for_auth,
10387 &auth_tracker,
10388 &connection_state,
10389 ),
10390 ReconnectBufferAction::Wait,
10391 );
10392 }
10393
10394 #[tokio::test]
10395 async fn test_write_task_waits_for_auth_before_replaying_buffer() {
10396 use nautilus_common::testing::wait_until_async;
10397
10398 let server = RecordingServer::setup().await;
10399 let url = format!("ws://127.0.0.1:{}", server.port);
10400 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10401 &url,
10402 vec![],
10403 TransportBackend::Tungstenite,
10404 None,
10405 None,
10406 None,
10407 )
10408 .await
10409 .unwrap();
10410
10411 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10412 let state_notify = Arc::new(tokio::sync::Notify::new());
10413 let auth_tracker = Arc::new(OnceLock::new());
10414 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10415 let connection_epoch = Arc::new(AtomicU64::new(0));
10416 let tracker = AuthTracker::new();
10417 auth_tracker.set(tracker.clone()).unwrap();
10418
10419 let (writer_tx, writer_rx) = crate::writer::channel(1);
10420 let write_task = WebSocketClientInner::spawn_write_task(
10421 Arc::clone(&connection_state),
10422 Arc::clone(&state_notify),
10423 Arc::new(AtomicBool::new(true)),
10424 writer,
10425 writer_rx,
10426 Arc::clone(&connection_epoch),
10427 Arc::clone(&auth_tracker),
10428 Arc::clone(&reconnect_buffer_waits_for_auth),
10429 None,
10430 );
10431
10432 writer_tx
10433 .send(WriterCommand::Send(Message::Text("stale".into())))
10434 .unwrap();
10435
10436 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10437 &url,
10438 vec![],
10439 TransportBackend::Tungstenite,
10440 None,
10441 None,
10442 None,
10443 )
10444 .await
10445 .unwrap();
10446 let (tx, rx) = tokio::sync::oneshot::channel();
10447 writer_tx
10448 .send_update(WriterCommand::Update(new_writer, tx))
10449 .unwrap();
10450 assert_eq!(rx.await.unwrap(), 1);
10451
10452 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10453
10454 tokio::time::sleep(Duration::from_millis(300)).await;
10455 assert!(
10456 server.messages().await.is_empty(),
10457 "buffered messages should wait for re-authentication"
10458 );
10459
10460 assert!(matches!(
10461 writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
10462 Err(SendError::BufferFull)
10463 ));
10464 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10465 writer_tx
10466 .send_control(WriterCommand::SendOnConnection {
10467 message: Message::text("authenticate"),
10468 connection_epoch: 1,
10469 response_tx,
10470 })
10471 .unwrap();
10472
10473 response_rx.await.unwrap().unwrap();
10474 tracker.succeed();
10475
10476 wait_until_async(
10477 || {
10478 let messages = Arc::clone(&server.messages);
10479 async move { messages.lock().await.len() == 2 }
10480 },
10481 Duration::from_secs(3),
10482 )
10483 .await;
10484
10485 assert_eq!(
10486 server.messages().await,
10487 vec!["authenticate".to_string(), "stale".to_string()]
10488 );
10489
10490 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10491 state_notify.notify_waiters();
10492 drop(writer_tx);
10493 write_task.abort();
10494 }
10495
10496 #[rstest]
10497 #[tokio::test]
10498 async fn test_write_task_buffers_send_behind_pending_replay() {
10499 use nautilus_common::testing::wait_until_async;
10500
10501 let server = RecordingServer::setup().await;
10502 let url = format!("ws://127.0.0.1:{}", server.port);
10503 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10504 &url,
10505 vec![],
10506 TransportBackend::Tungstenite,
10507 None,
10508 None,
10509 None,
10510 )
10511 .await
10512 .unwrap();
10513
10514 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10515 let state_notify = Arc::new(tokio::sync::Notify::new());
10516 let auth_tracker = Arc::new(OnceLock::new());
10517 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10518 let connection_epoch = Arc::new(AtomicU64::new(0));
10519 let tracker = AuthTracker::new();
10520 auth_tracker.set(tracker.clone()).unwrap();
10521
10522 let (writer_tx, writer_rx) = crate::writer::channel(2);
10525 let write_task = WebSocketClientInner::spawn_write_task(
10526 Arc::clone(&connection_state),
10527 Arc::clone(&state_notify),
10528 Arc::new(AtomicBool::new(true)),
10529 writer,
10530 writer_rx,
10531 Arc::clone(&connection_epoch),
10532 Arc::clone(&auth_tracker),
10533 Arc::clone(&reconnect_buffer_waits_for_auth),
10534 None,
10535 );
10536
10537 writer_tx
10538 .send(WriterCommand::Send(Message::Text("stale".into())))
10539 .unwrap();
10540
10541 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10542 &url,
10543 vec![],
10544 TransportBackend::Tungstenite,
10545 None,
10546 None,
10547 None,
10548 )
10549 .await
10550 .unwrap();
10551 let (tx, rx) = tokio::sync::oneshot::channel();
10552 writer_tx
10553 .send_update(WriterCommand::Update(new_writer, tx))
10554 .unwrap();
10555 assert_eq!(rx.await.unwrap(), 1);
10556
10557 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10559 tokio::time::sleep(Duration::from_millis(300)).await;
10560 assert!(
10561 server.messages().await.is_empty(),
10562 "buffered message should wait for re-authentication"
10563 );
10564
10565 writer_tx
10568 .send(WriterCommand::Send(Message::Text("fresh".into())))
10569 .unwrap();
10570 tokio::time::sleep(Duration::from_millis(300)).await;
10571 assert!(
10572 server.messages().await.is_empty(),
10573 "a send arriving before the gate opens must not reach the connection, was {:?}",
10574 server.messages().await
10575 );
10576
10577 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10578 writer_tx
10579 .send_control(WriterCommand::SendOnConnection {
10580 message: Message::text("authenticate"),
10581 connection_epoch: 1,
10582 response_tx,
10583 })
10584 .unwrap();
10585
10586 response_rx.await.unwrap().unwrap();
10587 tracker.succeed();
10588
10589 wait_until_async(
10590 || {
10591 let messages = Arc::clone(&server.messages);
10592 async move { messages.lock().await.len() == 3 }
10593 },
10594 Duration::from_secs(3),
10595 )
10596 .await;
10597
10598 assert_eq!(
10599 server.messages().await,
10600 vec![
10601 "authenticate".to_string(),
10602 "stale".to_string(),
10603 "fresh".to_string(),
10604 ],
10605 "replay must preserve FIFO order behind the authentication handshake"
10606 );
10607
10608 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10609 state_notify.notify_waiters();
10610 drop(writer_tx);
10611 write_task.abort();
10612 }
10613
10614 #[rstest]
10615 #[tokio::test]
10616 async fn test_write_task_control_send_reaches_connection_behind_pending_replay() {
10617 use nautilus_common::testing::wait_until_async;
10618
10619 let server = RecordingServer::setup().await;
10620 let url = format!("ws://127.0.0.1:{}", server.port);
10621 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10622 &url,
10623 vec![],
10624 TransportBackend::Tungstenite,
10625 None,
10626 None,
10627 None,
10628 )
10629 .await
10630 .unwrap();
10631
10632 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10633 let state_notify = Arc::new(tokio::sync::Notify::new());
10634 let auth_tracker = Arc::new(OnceLock::new());
10635 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10636 let connection_epoch = Arc::new(AtomicU64::new(0));
10637 let tracker = AuthTracker::new();
10638 auth_tracker.set(tracker.clone()).unwrap();
10639
10640 let (writer_tx, writer_rx) = crate::writer::channel(2);
10641 let write_task = WebSocketClientInner::spawn_write_task(
10642 Arc::clone(&connection_state),
10643 Arc::clone(&state_notify),
10644 Arc::new(AtomicBool::new(true)),
10645 writer,
10646 writer_rx,
10647 Arc::clone(&connection_epoch),
10648 Arc::clone(&auth_tracker),
10649 Arc::clone(&reconnect_buffer_waits_for_auth),
10650 None,
10651 );
10652
10653 writer_tx
10654 .send(WriterCommand::Send(Message::Text("stale".into())))
10655 .unwrap();
10656
10657 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10658 &url,
10659 vec![],
10660 TransportBackend::Tungstenite,
10661 None,
10662 None,
10663 None,
10664 )
10665 .await
10666 .unwrap();
10667 let (tx, rx) = tokio::sync::oneshot::channel();
10668 writer_tx
10669 .send_update(WriterCommand::Update(new_writer, tx))
10670 .unwrap();
10671 assert_eq!(rx.await.unwrap(), 1);
10672
10673 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10675 tokio::time::sleep(Duration::from_millis(300)).await;
10676 assert!(
10677 server.messages().await.is_empty(),
10678 "buffered message should wait for re-authentication"
10679 );
10680
10681 writer_tx
10685 .send_control(WriterCommand::Send(Message::Pong(
10686 b"keepalive".to_vec().into(),
10687 )))
10688 .unwrap();
10689
10690 wait_until_async(
10691 || {
10692 let control_frames = Arc::clone(&server.control_frames);
10693 async move { !control_frames.lock().await.is_empty() }
10694 },
10695 Duration::from_secs(3),
10696 )
10697 .await;
10698
10699 assert_eq!(
10700 server.control_frames().await,
10701 vec!["pong:keepalive".to_string()],
10702 "a control frame sent during the wait must reach the new connection without replay"
10703 );
10704 assert!(
10705 server.messages().await.is_empty(),
10706 "ordinary sends must still wait for re-authentication, was {:?}",
10707 server.messages().await
10708 );
10709
10710 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10711 writer_tx
10712 .send_control(WriterCommand::SendOnConnection {
10713 message: Message::text("authenticate"),
10714 connection_epoch: 1,
10715 response_tx,
10716 })
10717 .unwrap();
10718
10719 response_rx.await.unwrap().unwrap();
10720 tracker.succeed();
10721
10722 wait_until_async(
10723 || {
10724 let messages = Arc::clone(&server.messages);
10725 async move { messages.lock().await.len() == 2 }
10726 },
10727 Duration::from_secs(3),
10728 )
10729 .await;
10730
10731 assert_eq!(
10732 server.messages().await,
10733 vec!["authenticate".to_string(), "stale".to_string()],
10734 "the control frame must not disturb ordinary replay ordering"
10735 );
10736
10737 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10738 state_notify.notify_waiters();
10739 drop(writer_tx);
10740 write_task.abort();
10741 }
10742
10743 #[tokio::test]
10744 async fn test_write_task_discards_buffer_after_auth_failure() {
10745 let server = RecordingServer::setup().await;
10746 let url = format!("ws://127.0.0.1:{}", server.port);
10747 let (writer, _reader) = WebSocketClientInner::connect_with_server(
10748 &url,
10749 vec![],
10750 TransportBackend::Tungstenite,
10751 None,
10752 None,
10753 None,
10754 )
10755 .await
10756 .unwrap();
10757
10758 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10759 let state_notify = Arc::new(tokio::sync::Notify::new());
10760 let auth_tracker = Arc::new(OnceLock::new());
10761 let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10762 let connection_epoch = Arc::new(AtomicU64::new(0));
10763 let tracker = AuthTracker::new();
10764 auth_tracker.set(tracker.clone()).unwrap();
10765
10766 let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10767 let write_task = WebSocketClientInner::spawn_write_task(
10768 Arc::clone(&connection_state),
10769 Arc::clone(&state_notify),
10770 Arc::new(AtomicBool::new(true)),
10771 writer,
10772 writer_rx,
10773 Arc::clone(&connection_epoch),
10774 Arc::clone(&auth_tracker),
10775 Arc::clone(&reconnect_buffer_waits_for_auth),
10776 None,
10777 );
10778
10779 writer_tx
10780 .send(WriterCommand::Send(Message::Text("stale".into())))
10781 .unwrap();
10782
10783 let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10784 &url,
10785 vec![],
10786 TransportBackend::Tungstenite,
10787 None,
10788 None,
10789 None,
10790 )
10791 .await
10792 .unwrap();
10793 let (tx, rx) = tokio::sync::oneshot::channel();
10794 writer_tx
10795 .send(WriterCommand::Update(new_writer, tx))
10796 .unwrap();
10797 assert_eq!(rx.await.unwrap(), 1);
10798
10799 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10800 tracker.fail("rejected");
10801 tracker.invalidate();
10802 assert_eq!(tracker.auth_state(), AuthState::Failed);
10803 tokio::time::sleep(Duration::from_millis(300)).await;
10804 assert!(
10805 server.messages().await.is_empty(),
10806 "buffered messages should be discarded after authentication failure"
10807 );
10808
10809 let _auth_receiver = tracker.begin();
10810 tracker.succeed();
10811 tokio::time::sleep(Duration::from_millis(300)).await;
10812 assert!(
10813 server.messages().await.is_empty(),
10814 "discarded buffered messages should not replay on a later auth success"
10815 );
10816
10817 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10818 state_notify.notify_waiters();
10819 drop(writer_tx);
10820 write_task.abort();
10821 }
10822
10823 #[rstest]
10824 #[tokio::test]
10825 async fn test_zero_idle_timeout_rejected() {
10826 let (handler, _rx) = channel_message_handler();
10827
10828 let config = WebSocketConfig {
10829 url: "ws://127.0.0.1:9999".to_string(),
10830 headers: vec![],
10831 heartbeat_interval_secs: None,
10832 heartbeat_payload: None,
10833 connect_timeout_ms: None,
10834 reconnect_delay_initial_ms: None,
10835 reconnect_delay_max_ms: None,
10836 reconnect_backoff_factor: None,
10837 reconnect_jitter_ms: None,
10838 reconnect_max_attempts: None,
10839 heartbeat_timeout_secs: None,
10840 idle_timeout_ms: Some(0),
10841 writer_capacity: None,
10842 backend: TransportBackend::Tungstenite,
10843 proxy_url: None,
10844 max_message_size_bytes: None,
10845 max_frame_size_bytes: None,
10846 };
10847
10848 let result = WebSocketClient::builder()
10849 .config(config)
10850 .message_handler(handler)
10851 .connect()
10852 .await;
10853
10854 assert!(result.is_err(), "Zero idle timeout should be rejected");
10855 let err_msg = result.unwrap_err().to_string();
10856 assert!(
10857 err_msg.contains("idle_timeout_ms"),
10858 "Error should name the offending field, was: {err_msg}"
10859 );
10860 }
10861
10862 #[rstest]
10863 #[tokio::test]
10864 async fn test_zero_heartbeat_timeout_rejected() {
10865 let (handler, _rx) = channel_message_handler();
10866
10867 let config = WebSocketConfig {
10868 url: "ws://127.0.0.1:9999".to_string(),
10869 headers: vec![],
10870 heartbeat_interval_secs: Some(30),
10871 heartbeat_payload: None,
10872 connect_timeout_ms: None,
10873 reconnect_delay_initial_ms: None,
10874 reconnect_delay_max_ms: None,
10875 reconnect_backoff_factor: None,
10876 reconnect_jitter_ms: None,
10877 reconnect_max_attempts: None,
10878 heartbeat_timeout_secs: Some(0),
10879 idle_timeout_ms: None,
10880 writer_capacity: None,
10881 backend: TransportBackend::Tungstenite,
10882 proxy_url: None,
10883 max_message_size_bytes: None,
10884 max_frame_size_bytes: None,
10885 };
10886
10887 let result = WebSocketClient::builder()
10888 .config(config)
10889 .message_handler(handler)
10890 .connect()
10891 .await;
10892
10893 assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
10894 let err_msg = result.unwrap_err().to_string();
10895 assert!(
10896 err_msg.contains("heartbeat_timeout_secs"),
10897 "Error should name the offending field, was: {err_msg}"
10898 );
10899 }
10900
10901 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
10902 #[rstest]
10903 #[tokio::test]
10904 async fn test_sockudo_backend_rejects_reserved_headers_before_connect() {
10905 let (handler, _rx) = channel_message_handler();
10906
10907 let config = WebSocketConfig {
10908 url: "ws://127.0.0.1:1".to_string(),
10909 headers: vec![("Host".to_string(), "example.com".to_string())],
10910 heartbeat_interval_secs: None,
10911 heartbeat_payload: None,
10912 connect_timeout_ms: None,
10913 reconnect_delay_initial_ms: None,
10914 reconnect_delay_max_ms: None,
10915 reconnect_backoff_factor: None,
10916 reconnect_jitter_ms: None,
10917 reconnect_max_attempts: None,
10918 heartbeat_timeout_secs: None,
10919 idle_timeout_ms: None,
10920 writer_capacity: None,
10921 backend: TransportBackend::Sockudo,
10922 proxy_url: None,
10923 max_message_size_bytes: None,
10924 max_frame_size_bytes: None,
10925 };
10926
10927 let err = WebSocketClient::builder()
10928 .config(config)
10929 .message_handler(handler)
10930 .connect()
10931 .await
10932 .expect_err("reserved header should fail before TCP connect");
10933
10934 assert!(
10935 err.to_string()
10936 .contains("reserved upgrade header not allowed in extra_headers"),
10937 "expected reserved-header failure, was: {err}"
10938 );
10939 }
10940
10941 #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
10943 #[rstest]
10944 #[tokio::test]
10945 async fn test_sockudo_reconnect_recovers_from_retryable_upgrade_rejection() {
10946 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
10947 let port = listener.local_addr().unwrap().port();
10948
10949 let server = task::spawn(async move {
10950 let (stream, _) = listener.accept().await.unwrap();
10952 drop(accept_async(stream).await.unwrap());
10953
10954 let (mut stream, _) = listener.accept().await.unwrap();
10956 let _request = read_http_request(&mut stream).await;
10957 stream
10958 .write_all(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
10959 .await
10960 .unwrap();
10961 stream.flush().await.unwrap();
10962 sleep(Duration::from_millis(50)).await;
10963 drop(stream);
10964
10965 let (stream, _) = listener.accept().await.unwrap();
10967 let mut ws = accept_async(stream).await.unwrap();
10968 ws.send(WsMessage::Text("recovered".to_string().into()))
10969 .await
10970 .unwrap();
10971 sleep(Duration::from_secs(1)).await;
10972 });
10973
10974 let (handler, mut rx) = channel_message_handler();
10975
10976 let config = WebSocketConfig {
10977 url: format!(
10978 "ws://user-secret:password-secret@127.0.0.1:{port}/path-secret?token=query-secret"
10979 ),
10980 headers: vec![],
10981 heartbeat_interval_secs: None,
10982 heartbeat_payload: None,
10983 connect_timeout_ms: Some(2_000),
10984 reconnect_delay_initial_ms: Some(50),
10985 reconnect_delay_max_ms: Some(100),
10986 reconnect_backoff_factor: Some(1.0),
10987 reconnect_jitter_ms: Some(0),
10988 reconnect_max_attempts: None,
10989 heartbeat_timeout_secs: None,
10990 idle_timeout_ms: None,
10991 writer_capacity: None,
10992 backend: TransportBackend::Sockudo,
10993 proxy_url: None,
10994 max_message_size_bytes: None,
10995 max_frame_size_bytes: None,
10996 };
10997
10998 let capture = capture_logs_for(RECONNECT_LOG_TARGETS).await;
11002
11003 let client = WebSocketClient::builder()
11004 .config(config)
11005 .message_handler(handler)
11006 .connect()
11007 .await
11008 .expect("sockudo connect should succeed");
11009
11010 let recovered = tokio::time::timeout(Duration::from_secs(10), async {
11011 loop {
11012 if let Ok(WsMessage::Text(text)) = rx.try_recv()
11013 && text.as_str() == "recovered"
11014 {
11015 return;
11016 }
11017
11018 tokio::time::sleep(Duration::from_millis(10)).await;
11019 }
11020 })
11021 .await;
11022
11023 let messages = capture.messages();
11024 client.disconnect().await;
11025 server.abort();
11026
11027 recovered.expect("client should recover after the retryable upgrade rejection");
11028
11029 let errors: Vec<_> = messages
11030 .iter()
11031 .filter(|(level, _)| *level == Level::Error)
11032 .collect();
11033 assert!(
11034 errors.is_empty(),
11035 "a retryable rejection must not log at ERROR, was: {errors:?}"
11036 );
11037
11038 assert!(
11040 messages.iter().any(|(level, message)| *level == Level::Warn
11041 && message.contains(&format!(
11042 "Sockudo handshake rejected by 127.0.0.1:{port} with retryable status 502"
11043 ))),
11044 "expected a warning naming the retryable rejection, was: {messages:?}"
11045 );
11046
11047 for secret in [
11048 "user-secret",
11049 "password-secret",
11050 "path-secret",
11051 "query-secret",
11052 ] {
11053 assert!(
11054 messages
11055 .iter()
11056 .all(|(_, message)| !message.contains(secret)),
11057 "logs must not contain {secret}, was: {messages:?}"
11058 );
11059 }
11060 }
11061
11062 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11063 #[rstest]
11064 #[tokio::test]
11065 async fn test_sockudo_backend_replays_leftover_without_custom_headers() {
11066 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11067 let port = listener.local_addr().unwrap().port();
11068 let server = task::spawn(async move {
11069 if let Ok((mut stream, _)) = listener.accept().await {
11070 let request = read_http_request(&mut stream).await;
11071 let request = String::from_utf8(request).unwrap();
11072 let sec_websocket_key = extract_header(&request, "Sec-WebSocket-Key").unwrap();
11073 let accept = sockudo_handshake::generate_accept_key(sec_websocket_key);
11074 let mut response = format!(
11075 concat!(
11076 "HTTP/1.1 101 Switching Protocols\r\n",
11077 "Upgrade: websocket\r\n",
11078 "Connection: Upgrade\r\n",
11079 "Sec-WebSocket-Accept: {}\r\n",
11080 "\r\n",
11081 ),
11082 accept
11083 )
11084 .into_bytes();
11085 response.extend_from_slice(b"\x81\x05hello");
11086 stream.write_all(&response).await.unwrap();
11087 }
11088 });
11089
11090 let (handler, mut rx) = channel_message_handler();
11091
11092 let config = WebSocketConfig {
11093 url: format!("ws://127.0.0.1:{port}/ws"),
11094 headers: vec![],
11095 heartbeat_interval_secs: None,
11096 heartbeat_payload: None,
11097 connect_timeout_ms: Some(2_000),
11098 reconnect_delay_initial_ms: Some(50),
11099 reconnect_delay_max_ms: Some(100),
11100 reconnect_backoff_factor: Some(1.0),
11101 reconnect_jitter_ms: Some(0),
11102 reconnect_max_attempts: None,
11103 heartbeat_timeout_secs: None,
11104 idle_timeout_ms: None,
11105 writer_capacity: None,
11106 backend: TransportBackend::Sockudo,
11107 proxy_url: None,
11108 max_message_size_bytes: None,
11109 max_frame_size_bytes: None,
11110 };
11111
11112 let client = WebSocketClient::builder()
11113 .config(config)
11114 .message_handler(handler)
11115 .connect()
11116 .await
11117 .expect("sockudo connect without custom headers");
11118
11119 let received = tokio::time::timeout(Duration::from_secs(3), async {
11120 loop {
11121 if let Ok(msg) = rx.try_recv() {
11122 return msg;
11123 }
11124
11125 tokio::time::sleep(Duration::from_millis(10)).await;
11126 }
11127 })
11128 .await
11129 .expect("did not receive leftover frame before timeout");
11130
11131 match received {
11132 WsMessage::Text(t) => assert_eq!(t.as_str(), "hello"),
11133 other => panic!("expected text, was {other:?}"),
11134 }
11135
11136 client.disconnect().await;
11137 tokio::time::timeout(Duration::from_secs(3), server)
11138 .await
11139 .expect("server did not close before timeout")
11140 .unwrap();
11141 }
11142
11143 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11144 #[rstest]
11145 #[tokio::test]
11146 async fn test_sockudo_backend_sends_custom_headers() {
11147 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11148 let port = listener.local_addr().unwrap().port();
11149 let server = task::spawn(async move {
11150 if let Ok((stream, _)) = listener.accept().await {
11151 let callback = HeaderAssertCallback {
11152 key: "X-Test".to_string(),
11153 value: HeaderValue::from_static("value"),
11154 };
11155
11156 if let Ok(mut ws) = accept_hdr_async(stream, callback).await {
11157 while let Some(Ok(msg)) = ws.next().await {
11158 if msg.is_text() || msg.is_binary() {
11159 if ws.send(msg).await.is_err() {
11160 break;
11161 }
11162
11163 continue;
11164 }
11165
11166 if msg.is_close() {
11167 let _ = ws.close(None).await;
11168 break;
11169 }
11170 }
11171 }
11172 }
11173 });
11174
11175 let (handler, mut rx) = channel_message_handler();
11176
11177 let config = WebSocketConfig {
11178 url: format!("ws://127.0.0.1:{port}"),
11179 headers: vec![("X-Test".to_string(), "value".to_string())],
11180 heartbeat_interval_secs: None,
11181 heartbeat_payload: None,
11182 connect_timeout_ms: Some(2_000),
11183 reconnect_delay_initial_ms: Some(50),
11184 reconnect_delay_max_ms: Some(100),
11185 reconnect_backoff_factor: Some(1.0),
11186 reconnect_jitter_ms: Some(0),
11187 reconnect_max_attempts: None,
11188 heartbeat_timeout_secs: None,
11189 idle_timeout_ms: None,
11190 writer_capacity: None,
11191 backend: TransportBackend::Sockudo,
11192 proxy_url: None,
11193 max_message_size_bytes: None,
11194 max_frame_size_bytes: None,
11195 };
11196
11197 let client = WebSocketClient::builder()
11198 .config(config)
11199 .message_handler(handler)
11200 .connect()
11201 .await
11202 .expect("sockudo connect with custom headers");
11203
11204 client.send_text("ping".to_string(), None).await.unwrap();
11205
11206 let received = tokio::time::timeout(Duration::from_secs(3), async {
11207 loop {
11208 if let Ok(msg) = rx.try_recv() {
11209 return msg;
11210 }
11211
11212 tokio::time::sleep(Duration::from_millis(10)).await;
11213 }
11214 })
11215 .await
11216 .expect("did not receive echo before timeout");
11217
11218 match received {
11219 WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11220 other => panic!("expected text, was {other:?}"),
11221 }
11222
11223 client.disconnect().await;
11224 tokio::time::timeout(Duration::from_secs(3), server)
11225 .await
11226 .expect("server did not close before timeout")
11227 .unwrap();
11228 }
11229
11230 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11231 #[rstest]
11232 #[tokio::test]
11233 async fn test_sockudo_backend_round_trip_text() {
11234 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11236 let port = listener.local_addr().unwrap().port();
11237 let server = task::spawn(async move {
11238 if let Ok((stream, _)) = listener.accept().await
11239 && let Ok(mut ws) = accept_async(stream).await
11240 {
11241 while let Some(Ok(msg)) = ws.next().await {
11242 match msg {
11243 WsMessage::Text(_) | WsMessage::Binary(_) => {
11244 if ws.send(msg).await.is_err() {
11245 break;
11246 }
11247 }
11248 WsMessage::Close(_) => {
11249 let _ = ws.close(None).await;
11250 break;
11251 }
11252 _ => {}
11253 }
11254 }
11255 }
11256 });
11257
11258 let (handler, mut rx) = channel_message_handler();
11259
11260 let config = WebSocketConfig {
11261 url: format!("ws://127.0.0.1:{port}"),
11262 headers: vec![],
11263 heartbeat_interval_secs: None,
11264 heartbeat_payload: None,
11265 connect_timeout_ms: Some(2_000),
11266 reconnect_delay_initial_ms: Some(50),
11267 reconnect_delay_max_ms: Some(100),
11268 reconnect_backoff_factor: Some(1.0),
11269 reconnect_jitter_ms: Some(0),
11270 reconnect_max_attempts: None,
11271 heartbeat_timeout_secs: None,
11272 idle_timeout_ms: None,
11273 writer_capacity: None,
11274 backend: TransportBackend::Sockudo,
11275 proxy_url: None,
11276 max_message_size_bytes: None,
11277 max_frame_size_bytes: None,
11278 };
11279
11280 let client = WebSocketClient::builder()
11281 .config(config)
11282 .message_handler(handler)
11283 .connect()
11284 .await
11285 .expect("sockudo connect");
11286
11287 client.send_text("ping".to_string(), None).await.unwrap();
11288
11289 let received = tokio::time::timeout(Duration::from_secs(3), async {
11290 loop {
11291 if let Ok(msg) = rx.try_recv() {
11292 return msg;
11293 }
11294
11295 tokio::time::sleep(Duration::from_millis(10)).await;
11296 }
11297 })
11298 .await
11299 .expect("did not receive echo before timeout");
11300
11301 match received {
11302 WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11303 other => panic!("expected text, was {other:?}"),
11304 }
11305
11306 client.disconnect().await;
11307 server.abort();
11308 }
11309
11310 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11311 #[rstest]
11312 #[case::ws_default_port("ws://example.com/ws", "example.com", "example.com", 80, "/ws", false)]
11313 #[case::wss_default_port(
11314 "wss://example.com/ws",
11315 "example.com",
11316 "example.com",
11317 443,
11318 "/ws",
11319 true
11320 )]
11321 #[case::ws_explicit_default(
11324 "ws://example.com:80/ws",
11325 "example.com",
11326 "example.com",
11327 80,
11328 "/ws",
11329 false
11330 )]
11331 #[case::ws_non_default(
11332 "ws://example.com:8443/feed",
11333 "example.com",
11334 "example.com:8443",
11335 8443,
11336 "/feed",
11337 false
11338 )]
11339 #[case::wss_non_default(
11340 "wss://example.com:9443/feed",
11341 "example.com",
11342 "example.com:9443",
11343 9443,
11344 "/feed",
11345 true
11346 )]
11347 #[case::root_path(
11348 "ws://example.com:9000/",
11349 "example.com",
11350 "example.com:9000",
11351 9000,
11352 "/",
11353 false
11354 )]
11355 #[case::query_string(
11356 "ws://example.com/feed?token=abc&channel=trades",
11357 "example.com",
11358 "example.com",
11359 80,
11360 "/feed?token=abc&channel=trades",
11361 false
11362 )]
11363 #[case::ipv6_default("ws://[::1]/feed", "::1", "[::1]", 80, "/feed", false)]
11365 #[case::ipv6_explicit_port("ws://[::1]:9000/feed", "::1", "[::1]:9000", 9000, "/feed", false)]
11366 #[case::ipv6_wss(
11367 "wss://[2001:db8::1]:8443/",
11368 "2001:db8::1",
11369 "[2001:db8::1]:8443",
11370 8443,
11371 "/",
11372 true
11373 )]
11374 fn sockudo_target_parses_url(
11375 #[case] url: &str,
11376 #[case] host: &str,
11377 #[case] host_header: &str,
11378 #[case] port: u16,
11379 #[case] path: &str,
11380 #[case] is_tls: bool,
11381 ) {
11382 let target = super::SockudoTarget::parse(url).expect("parse should succeed");
11383 assert_eq!(target.host, host);
11384 assert_eq!(target.host_header, host_header);
11385 assert_eq!(target.port, port);
11386 assert_eq!(target.path, path);
11387 assert_eq!(target.is_tls, is_tls);
11388 }
11389
11390 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11391 #[rstest]
11392 fn sockudo_target_rejects_unsupported_scheme() {
11393 let err = super::SockudoTarget::parse("http://example.com/feed").expect_err("not a ws URL");
11394 let msg = err.to_string();
11395 assert!(
11396 msg.contains("expected ws:// or wss://"),
11397 "unexpected error: {msg}"
11398 );
11399 }
11400
11401 #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11402 #[rstest]
11403 fn sockudo_target_rejects_malformed_url() {
11404 const SECRET: &str = "malformed-websocket-url-secret";
11405 let url = format!("not a url {SECRET}");
11406 let err = super::SockudoTarget::parse(&url).expect_err("malformed URL");
11407 let message = err.to_string();
11408
11409 assert!(
11410 matches!(err, super::TransportError::InvalidUrl(_)),
11411 "expected InvalidUrl, was: {err:?}"
11412 );
11413 assert!(!message.contains(SECRET));
11414 assert!(!message.contains(&url));
11415 }
11416}
11417
11418#[cfg(test)]
11419mod property_tests {
11420 use std::{
11421 collections::{HashSet, VecDeque},
11422 sync::{Arc, OnceLock, atomic::AtomicBool},
11423 };
11424
11425 use proptest::prelude::*;
11426 use rstest::rstest;
11427
11428 use super::{super::auth::AuthResultReceiver, *};
11429
11430 const AUTH_FAILED: &str = "model auth failed";
11431
11432 #[derive(Debug, Clone)]
11433 enum ReconnectBufferTraceOp {
11434 BeginAuth,
11435 AuthSucceeds,
11436 AuthFails,
11437 AuthInvalidates,
11438 ReconnectStarts,
11439 ReconnectCompletes,
11440 BufferedMessage(u8),
11441 }
11442
11443 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
11444 enum ModelConnectionMode {
11445 Active,
11446 Reconnect,
11447 }
11448
11449 #[derive(Debug, Clone, Copy)]
11450 enum ExpectedReconnectBufferAction {
11451 Drain,
11452 Wait,
11453 Discard,
11454 }
11455
11456 #[derive(Debug)]
11457 struct ReconnectBufferModel {
11458 mode: ModelConnectionMode,
11459 auth_state: AuthState,
11460 buffer: VecDeque<String>,
11461 released: Vec<String>,
11462 discarded: Vec<String>,
11463 live_sent: Vec<String>,
11464 handler_controls: Vec<&'static str>,
11465 next_message_index: usize,
11466 }
11467
11468 impl ReconnectBufferModel {
11469 fn new() -> Self {
11470 Self {
11471 mode: ModelConnectionMode::Active,
11472 auth_state: AuthState::Unauthenticated,
11473 buffer: VecDeque::new(),
11474 released: Vec::new(),
11475 discarded: Vec::new(),
11476 live_sent: Vec::new(),
11477 handler_controls: Vec::new(),
11478 next_message_index: 0,
11479 }
11480 }
11481
11482 fn next_payload(&mut self, raw: u8) -> String {
11483 let payload = format!("message-{}-{raw}", self.next_message_index);
11484 self.next_message_index += 1;
11485 payload
11486 }
11487
11488 fn expected_action(&self, waits_for_auth: bool) -> ExpectedReconnectBufferAction {
11489 if !waits_for_auth {
11490 return ExpectedReconnectBufferAction::Drain;
11491 }
11492
11493 match self.auth_state {
11494 AuthState::Authenticated => ExpectedReconnectBufferAction::Drain,
11495 AuthState::Failed => ExpectedReconnectBufferAction::Discard,
11496 AuthState::Unauthenticated => ExpectedReconnectBufferAction::Wait,
11497 }
11498 }
11499 }
11500
11501 fn reconnect_buffer_trace_op_strategy() -> impl Strategy<Value = ReconnectBufferTraceOp> {
11502 prop_oneof![
11503 Just(ReconnectBufferTraceOp::BeginAuth),
11504 Just(ReconnectBufferTraceOp::AuthSucceeds),
11505 Just(ReconnectBufferTraceOp::AuthFails),
11506 Just(ReconnectBufferTraceOp::AuthInvalidates),
11507 Just(ReconnectBufferTraceOp::ReconnectStarts),
11508 Just(ReconnectBufferTraceOp::ReconnectCompletes),
11509 any::<u8>().prop_map(ReconnectBufferTraceOp::BufferedMessage),
11510 ]
11511 }
11512
11513 fn reconnect_buffer_actions_match(
11514 actual: ReconnectBufferAction,
11515 expected: ExpectedReconnectBufferAction,
11516 ) -> bool {
11517 matches!(
11518 (actual, expected),
11519 (
11520 ReconnectBufferAction::Drain,
11521 ExpectedReconnectBufferAction::Drain
11522 ) | (
11523 ReconnectBufferAction::Wait,
11524 ExpectedReconnectBufferAction::Wait
11525 ) | (
11526 ReconnectBufferAction::Discard,
11527 ExpectedReconnectBufferAction::Discard
11528 )
11529 )
11530 }
11531
11532 fn apply_ready_reconnect_buffer_action(
11533 model: &mut ReconnectBufferModel,
11534 reconnect_buffer_waits_for_auth: &AtomicBool,
11535 auth_tracker: &Arc<OnceLock<AuthTracker>>,
11536 waits_for_auth: bool,
11537 step: usize,
11538 op: &ReconnectBufferTraceOp,
11539 ) -> Result<(), TestCaseError> {
11540 if model.mode != ModelConnectionMode::Active || model.buffer.is_empty() {
11541 return Ok(());
11542 }
11543
11544 let expected = model.expected_action(waits_for_auth);
11545 let actual = WebSocketClientInner::can_drain_reconnect_buffer(
11546 reconnect_buffer_waits_for_auth,
11547 auth_tracker,
11548 );
11549
11550 prop_assert!(
11551 reconnect_buffer_actions_match(actual, expected),
11552 "reconnect buffer action mismatch at step {}, op {:?}, waits_for_auth={}, auth_state={:?}",
11553 step,
11554 op,
11555 waits_for_auth,
11556 model.auth_state
11557 );
11558
11559 match expected {
11560 ExpectedReconnectBufferAction::Drain => {
11561 model.released.extend(model.buffer.drain(..));
11562 }
11563 ExpectedReconnectBufferAction::Wait => {}
11564 ExpectedReconnectBufferAction::Discard => {
11565 model.discarded.extend(model.buffer.drain(..));
11566 }
11567 }
11568
11569 Ok(())
11570 }
11571
11572 fn assert_reconnected_control_stays_separate(
11573 model: &ReconnectBufferModel,
11574 step: usize,
11575 ) -> Result<(), TestCaseError> {
11576 prop_assert!(
11577 model
11578 .handler_controls
11579 .iter()
11580 .all(|message| *message == RECONNECTED),
11581 "handler control stream contained a non-RECONNECTED message at step {}",
11582 step
11583 );
11584 prop_assert!(
11585 !model.buffer.iter().any(|message| message == RECONNECTED),
11586 "RECONNECTED control message entered reconnect buffer at step {}",
11587 step
11588 );
11589 prop_assert!(
11590 !model.released.iter().any(|message| message == RECONNECTED),
11591 "RECONNECTED control message entered replayed messages at step {}",
11592 step
11593 );
11594 prop_assert!(
11595 !model.discarded.iter().any(|message| message == RECONNECTED),
11596 "RECONNECTED control message entered discarded messages at step {}",
11597 step
11598 );
11599 prop_assert!(
11600 !model.live_sent.iter().any(|message| message == RECONNECTED),
11601 "RECONNECTED control message entered application sends at step {}",
11602 step
11603 );
11604
11605 Ok(())
11606 }
11607
11608 fn assert_messages_accounted_once(
11609 model: &ReconnectBufferModel,
11610 step: usize,
11611 ) -> Result<(), TestCaseError> {
11612 let mut seen = HashSet::new();
11613
11614 for message in model
11615 .released
11616 .iter()
11617 .chain(model.discarded.iter())
11618 .chain(model.buffer.iter())
11619 .chain(model.live_sent.iter())
11620 {
11621 prop_assert!(
11622 seen.insert(message.as_str()),
11623 "message {} appeared more than once at step {}",
11624 message,
11625 step
11626 );
11627 }
11628
11629 Ok(())
11630 }
11631
11632 fn apply_reconnect_buffer_trace_op(
11633 model: &mut ReconnectBufferModel,
11634 tracker: &AuthTracker,
11635 auth_receivers: &mut Vec<AuthResultReceiver>,
11636 op: &ReconnectBufferTraceOp,
11637 ) -> Result<(), TestCaseError> {
11638 match op {
11639 ReconnectBufferTraceOp::BeginAuth => {
11640 auth_receivers.push(tracker.begin());
11641 model.auth_state = AuthState::Unauthenticated;
11642 }
11643 ReconnectBufferTraceOp::AuthSucceeds => {
11644 tracker.succeed();
11645 model.auth_state = AuthState::Authenticated;
11646 }
11647 ReconnectBufferTraceOp::AuthFails => {
11648 tracker.fail(AUTH_FAILED);
11649 model.auth_state = AuthState::Failed;
11650 }
11651 ReconnectBufferTraceOp::AuthInvalidates => {
11652 tracker.invalidate();
11653
11654 if model.auth_state == AuthState::Authenticated {
11655 model.auth_state = AuthState::Unauthenticated;
11656 }
11657 }
11658 ReconnectBufferTraceOp::ReconnectStarts => {
11659 tracker.invalidate();
11660
11661 if model.auth_state == AuthState::Authenticated {
11662 model.auth_state = AuthState::Unauthenticated;
11663 }
11664
11665 model.mode = ModelConnectionMode::Reconnect;
11666 }
11667 ReconnectBufferTraceOp::ReconnectCompletes => {
11668 model.mode = ModelConnectionMode::Active;
11669 model.handler_controls.push(RECONNECTED);
11670 }
11671 ReconnectBufferTraceOp::BufferedMessage(raw) => {
11672 let payload = model.next_payload(*raw);
11673 prop_assert_ne!(payload.as_str(), RECONNECTED);
11674
11675 if model.mode == ModelConnectionMode::Reconnect {
11676 model.buffer.push_back(payload);
11677 } else {
11678 model.live_sent.push(payload);
11679 }
11680 }
11681 }
11682
11683 Ok(())
11684 }
11685
11686 proptest! {
11687 #![proptest_config(ProptestConfig::with_cases(256))]
11688
11689 #[rstest]
11692 fn test_reconnect_buffer_trace_matches_auth_gate_model(
11693 waits_for_auth in any::<bool>(),
11694 ops in proptest::collection::vec(reconnect_buffer_trace_op_strategy(), 1..100)
11695 ) {
11696 let auth_tracker = Arc::new(OnceLock::new());
11697 let reconnect_buffer_waits_for_auth = AtomicBool::new(waits_for_auth);
11698 let tracker = AuthTracker::new();
11699 auth_tracker.set(tracker.clone()).unwrap();
11700 let mut auth_receivers = Vec::new();
11701 let mut model = ReconnectBufferModel::new();
11702
11703 for (step, op) in ops.iter().enumerate() {
11704 apply_reconnect_buffer_trace_op(
11705 &mut model,
11706 &tracker,
11707 &mut auth_receivers,
11708 op,
11709 )?;
11710
11711 prop_assert_eq!(
11712 tracker.auth_state(),
11713 model.auth_state,
11714 "auth state mismatch at step {}, op {:?}",
11715 step,
11716 op
11717 );
11718
11719 apply_ready_reconnect_buffer_action(
11720 &mut model,
11721 &reconnect_buffer_waits_for_auth,
11722 &auth_tracker,
11723 waits_for_auth,
11724 step,
11725 op,
11726 )?;
11727 assert_reconnected_control_stays_separate(&model, step)?;
11728 prop_assert_eq!(
11729 model.handler_controls.len(),
11730 ops[..=step]
11731 .iter()
11732 .filter(|op| matches!(op, ReconnectBufferTraceOp::ReconnectCompletes))
11733 .count(),
11734 "handler control count mismatch at step {}",
11735 step
11736 );
11737 assert_messages_accounted_once(&model, step)?;
11738 }
11739 }
11740
11741 #[rstest]
11744 fn test_reconnect_buffer_releases_after_auth_success_once(
11745 payloads in proptest::collection::vec(any::<u8>(), 1..32),
11746 extra_success_ticks in 0usize..16
11747 ) {
11748 let auth_tracker = Arc::new(OnceLock::new());
11749 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11750 let tracker = AuthTracker::new();
11751 auth_tracker.set(tracker.clone()).unwrap();
11752 let mut auth_receivers = Vec::new();
11753 let mut model = ReconnectBufferModel::new();
11754
11755 apply_reconnect_buffer_trace_op(
11756 &mut model,
11757 &tracker,
11758 &mut auth_receivers,
11759 &ReconnectBufferTraceOp::ReconnectStarts,
11760 )?;
11761 apply_reconnect_buffer_trace_op(
11762 &mut model,
11763 &tracker,
11764 &mut auth_receivers,
11765 &ReconnectBufferTraceOp::BeginAuth,
11766 )?;
11767
11768 for payload in payloads {
11769 apply_reconnect_buffer_trace_op(
11770 &mut model,
11771 &tracker,
11772 &mut auth_receivers,
11773 &ReconnectBufferTraceOp::BufferedMessage(payload),
11774 )?;
11775 }
11776
11777 let buffered_len = model.buffer.len();
11778 apply_reconnect_buffer_trace_op(
11779 &mut model,
11780 &tracker,
11781 &mut auth_receivers,
11782 &ReconnectBufferTraceOp::ReconnectCompletes,
11783 )?;
11784 apply_ready_reconnect_buffer_action(
11785 &mut model,
11786 &reconnect_buffer_waits_for_auth,
11787 &auth_tracker,
11788 true,
11789 0,
11790 &ReconnectBufferTraceOp::ReconnectCompletes,
11791 )?;
11792
11793 prop_assert_eq!(model.released.len(), 0);
11794 prop_assert_eq!(model.buffer.len(), buffered_len);
11795
11796 apply_reconnect_buffer_trace_op(
11797 &mut model,
11798 &tracker,
11799 &mut auth_receivers,
11800 &ReconnectBufferTraceOp::AuthSucceeds,
11801 )?;
11802 apply_ready_reconnect_buffer_action(
11803 &mut model,
11804 &reconnect_buffer_waits_for_auth,
11805 &auth_tracker,
11806 true,
11807 1,
11808 &ReconnectBufferTraceOp::AuthSucceeds,
11809 )?;
11810
11811 prop_assert_eq!(model.released.len(), buffered_len);
11812 prop_assert!(model.buffer.is_empty());
11813 assert_messages_accounted_once(&model, 1)?;
11814
11815 for tick in 0..extra_success_ticks {
11816 apply_reconnect_buffer_trace_op(
11817 &mut model,
11818 &tracker,
11819 &mut auth_receivers,
11820 &ReconnectBufferTraceOp::AuthSucceeds,
11821 )?;
11822 apply_ready_reconnect_buffer_action(
11823 &mut model,
11824 &reconnect_buffer_waits_for_auth,
11825 &auth_tracker,
11826 true,
11827 tick + 2,
11828 &ReconnectBufferTraceOp::AuthSucceeds,
11829 )?;
11830 prop_assert_eq!(
11831 model.released.len(),
11832 buffered_len,
11833 "buffered messages replayed more than once at tick {}",
11834 tick
11835 );
11836 }
11837 }
11838
11839 #[rstest]
11842 fn test_reconnect_buffer_discards_after_auth_failure(
11843 before_failure_payloads in proptest::collection::vec(any::<u8>(), 0..16),
11844 after_failure_payloads in proptest::collection::vec(any::<u8>(), 1..16),
11845 later_success_ticks in 0usize..16
11846 ) {
11847 let auth_tracker = Arc::new(OnceLock::new());
11848 let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11849 let tracker = AuthTracker::new();
11850 auth_tracker.set(tracker.clone()).unwrap();
11851 let mut auth_receivers = Vec::new();
11852 let mut model = ReconnectBufferModel::new();
11853
11854 apply_reconnect_buffer_trace_op(
11855 &mut model,
11856 &tracker,
11857 &mut auth_receivers,
11858 &ReconnectBufferTraceOp::ReconnectStarts,
11859 )?;
11860 apply_reconnect_buffer_trace_op(
11861 &mut model,
11862 &tracker,
11863 &mut auth_receivers,
11864 &ReconnectBufferTraceOp::BeginAuth,
11865 )?;
11866
11867 for payload in before_failure_payloads {
11868 apply_reconnect_buffer_trace_op(
11869 &mut model,
11870 &tracker,
11871 &mut auth_receivers,
11872 &ReconnectBufferTraceOp::BufferedMessage(payload),
11873 )?;
11874 }
11875
11876 apply_reconnect_buffer_trace_op(
11877 &mut model,
11878 &tracker,
11879 &mut auth_receivers,
11880 &ReconnectBufferTraceOp::AuthFails,
11881 )?;
11882
11883 for payload in after_failure_payloads {
11884 apply_reconnect_buffer_trace_op(
11885 &mut model,
11886 &tracker,
11887 &mut auth_receivers,
11888 &ReconnectBufferTraceOp::BufferedMessage(payload),
11889 )?;
11890 }
11891
11892 let buffered_len = model.buffer.len();
11893 apply_reconnect_buffer_trace_op(
11894 &mut model,
11895 &tracker,
11896 &mut auth_receivers,
11897 &ReconnectBufferTraceOp::ReconnectCompletes,
11898 )?;
11899 apply_ready_reconnect_buffer_action(
11900 &mut model,
11901 &reconnect_buffer_waits_for_auth,
11902 &auth_tracker,
11903 true,
11904 0,
11905 &ReconnectBufferTraceOp::ReconnectCompletes,
11906 )?;
11907
11908 prop_assert_eq!(model.discarded.len(), buffered_len);
11909 prop_assert!(model.released.is_empty());
11910 prop_assert!(model.buffer.is_empty());
11911 assert_messages_accounted_once(&model, 0)?;
11912
11913 for tick in 0..later_success_ticks {
11914 apply_reconnect_buffer_trace_op(
11915 &mut model,
11916 &tracker,
11917 &mut auth_receivers,
11918 &ReconnectBufferTraceOp::BeginAuth,
11919 )?;
11920 apply_reconnect_buffer_trace_op(
11921 &mut model,
11922 &tracker,
11923 &mut auth_receivers,
11924 &ReconnectBufferTraceOp::AuthSucceeds,
11925 )?;
11926 apply_ready_reconnect_buffer_action(
11927 &mut model,
11928 &reconnect_buffer_waits_for_auth,
11929 &auth_tracker,
11930 true,
11931 tick + 1,
11932 &ReconnectBufferTraceOp::AuthSucceeds,
11933 )?;
11934 prop_assert!(
11935 model.released.is_empty(),
11936 "discarded messages replayed after later auth success at tick {}",
11937 tick
11938 );
11939 }
11940 }
11941 }
11942}
11943
11944#[cfg(test)]
11945#[cfg(feature = "turmoil")]
11946mod turmoil_tests {
11947 use std::{sync::Arc, time::Duration};
11948
11949 use futures_util::{SinkExt, StreamExt};
11950 use nautilus_common::testing::wait_until_async;
11951 use rstest::rstest;
11952 use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
11953 use turmoil::{Builder, net};
11954
11955 use super::*;
11956 use crate::websocket::types::channel_message_handler;
11957
11958 const AUTH_BUFFER_WAIT_SEED: u64 = 0xA17B_0001;
11959 const AUTH_BUFFER_DISCARD_SEED: u64 = 0xA17B_0002;
11960
11961 fn seeded_turmoil_builder(seed: u64) -> Builder {
11962 let mut builder = Builder::new();
11963 builder.rng_seed(seed);
11964 builder
11965 }
11966
11967 #[rstest]
11968 fn test_turmoil_reconnect_buffer_waits_for_auth() {
11969 let mut sim = seeded_turmoil_builder(AUTH_BUFFER_WAIT_SEED).build();
11970 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
11971 let server_messages = Arc::clone(&messages);
11972 sim.host("server", move || {
11973 let messages = Arc::clone(&server_messages);
11974 auth_buffer_server(messages)
11975 });
11976
11977 sim.client("client", async move {
11978 let tracker = AuthTracker::new();
11979 let (handler, _rx) = channel_message_handler();
11980 let client = WebSocketClient::builder()
11981 .config(turmoil_websocket_config())
11982 .message_handler(handler)
11983 .connect()
11984 .await
11985 .expect("Should connect");
11986
11987 client.set_auth_tracker(tracker.clone(), true);
11988 assert!(client.is_active(), "Client should start active");
11989
11990 wait_until_async(
11991 || async { client.is_reconnecting() },
11992 Duration::from_secs(3),
11993 )
11994 .await;
11995
11996 client
11997 .writer_tx
11998 .send(WriterCommand::Send(Message::Text("stale".into())))
11999 .unwrap();
12000
12001 wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12002
12003 let _auth_receiver = tracker.begin();
12004
12005 tokio::time::sleep(Duration::from_millis(300)).await;
12006 assert!(
12007 messages.lock().await.is_empty(),
12008 "buffered messages should wait for auth after reconnect"
12009 );
12010
12011 tracker.succeed();
12012
12013 wait_until_async(
12014 || {
12015 let messages = Arc::clone(&messages);
12016 async move { messages.lock().await.as_slice() == ["stale"] }
12017 },
12018 Duration::from_secs(3),
12019 )
12020 .await;
12021
12022 assert_eq!(messages.lock().await.as_slice(), ["stale"]);
12023
12024 client.disconnect().await;
12025 assert!(client.is_disconnected());
12026
12027 Ok(())
12028 });
12029
12030 sim.run().unwrap();
12031 }
12032
12033 #[rstest]
12034 fn test_turmoil_reconnect_buffer_discards_after_auth_failure() {
12035 let mut sim = seeded_turmoil_builder(AUTH_BUFFER_DISCARD_SEED).build();
12036 let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
12037 let server_messages = Arc::clone(&messages);
12038 sim.host("server", move || {
12039 let messages = Arc::clone(&server_messages);
12040 auth_buffer_server(messages)
12041 });
12042
12043 sim.client("client", async move {
12044 let tracker = AuthTracker::new();
12045 let (handler, _rx) = channel_message_handler();
12046 let client = WebSocketClient::builder()
12047 .config(turmoil_websocket_config())
12048 .message_handler(handler)
12049 .connect()
12050 .await
12051 .expect("Should connect");
12052
12053 client.set_auth_tracker(tracker.clone(), true);
12054 assert!(client.is_active(), "Client should start active");
12055
12056 wait_until_async(
12057 || async { client.is_reconnecting() },
12058 Duration::from_secs(3),
12059 )
12060 .await;
12061
12062 client
12063 .writer_tx
12064 .send(WriterCommand::Send(Message::Text("stale".into())))
12065 .unwrap();
12066
12067 wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12068
12069 let _auth_receiver = tracker.begin();
12070 tracker.fail("rejected");
12071
12072 tokio::time::sleep(Duration::from_millis(300)).await;
12073 assert!(
12074 messages.lock().await.is_empty(),
12075 "buffered messages should be discarded after auth failure"
12076 );
12077
12078 let _retry_auth_receiver = tracker.begin();
12079 tracker.succeed();
12080
12081 tokio::time::sleep(Duration::from_millis(300)).await;
12082 assert!(
12083 messages.lock().await.is_empty(),
12084 "discarded messages should not replay on a later auth success"
12085 );
12086
12087 client.disconnect().await;
12088 assert!(client.is_disconnected());
12089
12090 Ok(())
12091 });
12092
12093 sim.run().unwrap();
12094 }
12095
12096 fn turmoil_websocket_config() -> WebSocketConfig {
12097 WebSocketConfig {
12098 url: "ws://server:8080".to_string(),
12099 headers: vec![],
12100 heartbeat_interval_secs: None,
12101 heartbeat_payload: None,
12102 connect_timeout_ms: Some(5_000),
12103 reconnect_delay_initial_ms: Some(50),
12104 reconnect_delay_max_ms: Some(200),
12105 reconnect_backoff_factor: Some(1.0),
12106 reconnect_jitter_ms: Some(0),
12107 reconnect_max_attempts: None,
12108 heartbeat_timeout_secs: None,
12109 idle_timeout_ms: None,
12110 writer_capacity: None,
12111 backend: TransportBackend::Tungstenite,
12112 proxy_url: None,
12113 max_message_size_bytes: None,
12114 max_frame_size_bytes: None,
12115 }
12116 }
12117
12118 async fn auth_buffer_server(
12119 messages: Arc<tokio::sync::Mutex<Vec<String>>>,
12120 ) -> Result<(), Box<dyn std::error::Error>> {
12121 let listener = net::TcpListener::bind("0.0.0.0:8080").await?;
12122
12123 let (stream, _) = listener.accept().await?;
12124 let mut websocket = accept_async(stream).await?;
12125 let _ = websocket.send(WsMessage::Text("first".into())).await;
12126 drop(websocket);
12127
12128 tokio::time::sleep(Duration::from_millis(200)).await;
12129
12130 let (stream, _) = listener.accept().await?;
12131 let mut websocket = accept_async(stream).await?;
12132
12133 while let Some(msg) = websocket.next().await {
12134 match msg {
12135 Ok(WsMessage::Text(text)) => {
12136 messages.lock().await.push(text.to_string());
12137 }
12138 Ok(WsMessage::Close(_)) => {
12139 let _ = websocket.close(None).await;
12140 break;
12141 }
12142 Ok(_) => {}
12143 Err(_) => break,
12144 }
12145 }
12146
12147 Ok(())
12148 }
12149}