Skip to main content

nautilus_network/websocket/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Connection lifecycle and task coordination for [`WebSocketClient`].
17//!
18//! # Execution model
19//!
20//! Handler mode runs a controller, reader, and writer task, plus an optional heartbeat task. The
21//! reader dispatches frames, the writer serializes all sink access, and the controller drives
22//! reconnect and shutdown from the shared [`ConnectionMode`]. Stream mode omits the managed
23//! reader task and automatic reconnect.
24//!
25//! A read failure ends the reader and wakes the controller; a write failure requests reconnect and
26//! also wakes it. The controller establishes a replacement transport with exponential backoff,
27//! hands its sink to the writer, replaces the reader, then publishes the reconnect notification.
28//! Terminal disconnect and close states stop all tasks instead of starting another reconnect.
29//!
30//! # Reconnect ordering
31//!
32//! An accepted reconnect request first transitions to `Reconnect` and invalidates registered
33//! authentication. The controller may establish the replacement while the synchronous loss
34//! callback runs, but cannot restore `Active` until loss publication completes. A reconnect through
35//! [`WebSocketReconnectHandle`] releases its controller request fence before invoking the callback,
36//! so dropping the client can terminate controller work even if the callback blocks. Terminal
37//! states always take precedence.
38//!
39//! # Send semantics
40//!
41//! Ordinary sends return after entering the writer channel. Failed ordinary writes and ordinary
42//! messages encountered during reconnect enter a FIFO buffer for replay on an active replacement
43//! connection. While that replay is pending, ordinary sends also enter the buffer and queue behind
44//! it instead of reaching the replacement sink early. Registered authentication state can hold or
45//! discard that replay before it reaches the sink. Control frames are never buffered, so a pong
46//! sent while replay waits still writes to the replacement sink directly.
47//! [`WebSocketConfig::writer_capacity`] bounds ordinary messages across the channel, in-flight
48//! writes, and replay. Full capacity rejects a send with [`SendError::BufferFull`] before
49//! enqueueing. Ownership-bound sends and control traffic use a separate bounded allowance so
50//! authentication can unblock replay.
51//!
52//! Ownership-bound sends wait for the writer result, require the expected connection epoch, and
53//! never enter the reconnect buffer. The writer advances the epoch only when it installs a
54//! replacement sink, so inbound attribution and bound sends share one transport ownership boundary.
55
56use std::{
57    collections::VecDeque,
58    fmt::{Debug, Display},
59    future::Future,
60    pin::pin,
61    sync::{
62        Arc, OnceLock,
63        atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering},
64    },
65    time::Duration,
66};
67
68use futures_util::{SinkExt, StreamExt};
69use http::HeaderName;
70use nautilus_core::string::secret::REDACTED;
71use nautilus_cryptography::providers::install_cryptographic_provider;
72use parking_lot::RwLock;
73#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
74use rustls::ClientConfig;
75#[cfg(feature = "transport-sockudo")]
76use sockudo_ws::{
77    Config as SockudoConfig, Http1, Role, Stream as SockudoStream,
78    WebSocketStream as SockudoWebSocketStream,
79};
80#[cfg(feature = "transport-sockudo")]
81use tokio::io::{AsyncRead, AsyncWrite};
82#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
83use tokio_rustls::TlsConnector;
84use tokio_tungstenite::tungstenite::{
85    client::IntoClientRequest, handshake::client::Request, http::HeaderValue, stream::Mode,
86};
87use tokio_util::sync::CancellationToken;
88use ustr::Ustr;
89
90#[cfg(not(feature = "turmoil"))]
91use super::proxy::{ProxyKind, WsTarget, tunnel_via_proxy};
92use super::{
93    auth::{AuthState, AuthTracker},
94    config::{InitialConnectRetryPolicy, TransportBackend, WebSocketConfig},
95    consts::{
96        CONNECTION_STATE_CHECK_INTERVAL_MS, GRACEFUL_SHUTDOWN_DELAY_MS,
97        GRACEFUL_SHUTDOWN_TIMEOUT_SECS,
98    },
99    frame::FrameStream,
100    types::{
101        CloseHandler, EpochMessageHandler, EpochPingHandler, MessageHandler, MessageReader,
102        MessageWriter, PingHandler, WriterCommand,
103    },
104};
105#[cfg(feature = "transport-sockudo")]
106use crate::net::TcpStream;
107#[cfg(feature = "transport-sockudo")]
108use crate::transport::sockudo::{
109    PrefixedIo, SockudoTransport, client_handshake_with_headers, validate_extra_headers,
110};
111use crate::{
112    RECONNECTED, SocketState, SocketStateSink,
113    backoff::{
114        ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
115    },
116    dst,
117    error::{SendError, is_connection_drop_io_error},
118    logging::{log_task_aborted, log_task_started, log_task_stopped},
119    mode::{
120        ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
121        ReconnectRequestOutcome,
122    },
123    ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota},
124    retry::{RetryConfig, RetryError, RetryManager},
125    transport::{
126        BoxedWsTransport, CloseFrame, Message, TransportError, error::retryable_status,
127        tungstenite::TungsteniteTransport,
128    },
129    writer::{DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
130};
131
132const WRITE_TIMEOUT_SECS: u64 = 5;
133const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
134const INITIAL_CONNECT_OPERATION: &str = "WebSocket initial connection";
135
136/// The RFC 6455 control-frame payload limit.
137const MAX_CONTROL_FRAME_PAYLOAD_BYTES: usize = 125;
138
139/// Owns the transport tasks and reconnect state used by [`WebSocketClient`].
140///
141/// # Connection ownership
142///
143/// The client uses one reader and supports concurrent senders. In handler mode, a reader task
144/// dispatches incoming messages while a writer task serializes sends received over a channel. The
145/// controller owns the connection lifecycle and replaces both transport halves during reconnects.
146///
147/// Stream mode returns the reader to the caller. The client cannot replace that reader, so stream
148/// mode disables automatic reconnection.
149///
150/// # Heartbeats
151///
152/// When configured, a dedicated task sends heartbeat messages at the requested interval. Configure
153/// the interval below the server's heartbeat deadline. Handler mode can also opt into a heartbeat
154/// timeout that reconnects when no frame arrives within a venue-specific duration. The timeout
155/// starts with each connection and resets on every inbound frame, including Ping and Pong.
156///
157/// # Reconnection
158///
159/// The writer task owns queued sends across reconnects. A successful reconnect installs the
160/// replacement writer and starts a reader for the new connection epoch. Depending on the
161/// configured authentication gate, buffered sends drain immediately or wait for the new session to
162/// authenticate. Failed authentication discards messages that remain buffered.
163pub struct WebSocketClientInner {
164    config: WebSocketConfig,
165    reconnect_headers: ReconnectHeaders,
166    handler: Option<IncomingHandler>,
167    ping_handler: Option<IncomingPingHandler>,
168    close_handler: Option<CloseHandler>,
169    read_task: Option<dst::task::JoinHandle<()>>,
170    read_fence: Option<ReadSessionFence>,
171    write_task: dst::task::JoinHandle<()>,
172    writer_tx: WriterSender<WriterCommand>,
173    heartbeat_task: Option<dst::task::JoinHandle<()>>,
174    connection_mode: Arc<AtomicU8>,
175    connection_epoch: Arc<AtomicU64>,
176    state_notify: Arc<tokio::sync::Notify>,
177    controller_notify: Arc<tokio::sync::Notify>,
178    reconnect_published: Arc<AtomicBool>,
179    connect_timeout: Duration,
180    heartbeat_timeout: Option<Duration>,
181    backoff: ExponentialBackoff,
182    reconnect_throttle: ReconnectThrottle,
183    reconnect_max_attempts: Option<u32>,
184    reconnection_attempt_count: u32,
185    auth_tracker: Arc<OnceLock<AuthTracker>>,
186    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
187    state_sink: Option<SocketStateSink>,
188    connection_rate_limit: Option<ConnectionRateLimit>,
189}
190
191#[derive(Clone, Debug)]
192struct ConnectionRateLimit {
193    limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
194    keys: Arc<[Ustr]>,
195}
196
197#[derive(Default)]
198struct InitialConnectOptions {
199    retry_policy: Option<InitialConnectRetryPolicy>,
200    cancellation_token: Option<CancellationToken>,
201}
202
203impl WebSocketClientInner {
204    /// Creates an inner WebSocket client with an existing writer.
205    ///
206    /// This is used for stream mode where the reader is owned by the caller.
207    ///
208    /// # Errors
209    ///
210    /// Returns an error if the exponential backoff configuration is invalid.
211    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
212    #[expect(
213        clippy::unused_async,
214        clippy::unused_async_trait_impl,
215        reason = "async signature for consistency with connect-based constructors"
216    )]
217    pub async fn new_with_writer(
218        config: WebSocketConfig,
219        writer: MessageWriter,
220    ) -> Result<Self, TransportError> {
221        Self::new_with_writer_and_state_sink(config, writer, None)
222    }
223
224    fn new_with_writer_and_state_sink(
225        mut config: WebSocketConfig,
226        writer: MessageWriter,
227        state_sink: Option<SocketStateSink>,
228    ) -> Result<Self, TransportError> {
229        install_cryptographic_provider();
230
231        crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
232            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
233        })?;
234
235        if config.heartbeat_interval_secs == Some(0) {
236            return Err(TransportError::Io(std::io::Error::new(
237                std::io::ErrorKind::InvalidInput,
238                "Heartbeat interval cannot be zero",
239            )));
240        }
241
242        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
243        let connection_epoch = Arc::new(AtomicU64::new(0));
244        let state_notify = Arc::new(tokio::sync::Notify::new());
245        let controller_notify = Arc::new(tokio::sync::Notify::new());
246        let reconnect_published = Arc::new(AtomicBool::new(true));
247        let outcome =
248            ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
249        debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
250
251        // Note: We don't spawn a read task here since the reader is handled externally
252        let read_task = None;
253        let read_fence = None;
254
255        // Stream mode ignores reconnect settings, use harmless defaults
256        let backoff = ExponentialBackoff::new(
257            Duration::from_secs(2),
258            Duration::from_secs(30),
259            1.5,
260            100,
261            true,
262        )
263        .map_err(|e| {
264            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
265        })?;
266
267        let auth_tracker = Arc::new(OnceLock::new());
268        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
269
270        let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
271            config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
272        );
273        let write_task = Self::spawn_write_task(
274            Arc::clone(&connection_mode),
275            Arc::clone(&controller_notify),
276            Arc::clone(&reconnect_published),
277            writer,
278            writer_rx,
279            Arc::clone(&connection_epoch),
280            Arc::clone(&auth_tracker),
281            Arc::clone(&reconnect_buffer_waits_for_auth),
282            state_sink.clone(),
283        );
284
285        let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_interval| {
286            Self::spawn_heartbeat_task(
287                Arc::clone(&connection_mode),
288                heartbeat_interval,
289                config.heartbeat_payload.clone(),
290                writer_tx.clone(),
291            )
292        });
293
294        let reconnect_max_attempts = None; // Stream mode does not reconnect
295        let connect_timeout = Duration::from_secs(10);
296
297        let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
298
299        Ok(Self {
300            config,
301            reconnect_headers,
302            handler: None, // Stream mode has no handler
303            ping_handler: None,
304            close_handler: None,
305            writer_tx,
306            connection_mode,
307            connection_epoch,
308            state_notify,
309            controller_notify,
310            reconnect_published,
311            connect_timeout,
312            heartbeat_timeout: None,
313            heartbeat_task,
314            read_task,
315            read_fence,
316            write_task,
317            backoff,
318            reconnect_throttle: ReconnectThrottle::default(),
319            reconnect_max_attempts,
320            reconnection_attempt_count: 0,
321            auth_tracker,
322            reconnect_buffer_waits_for_auth,
323            state_sink,
324            connection_rate_limit: None,
325        })
326    }
327
328    /// Creates an inner WebSocket client.
329    ///
330    /// # Errors
331    ///
332    /// Returns an error if:
333    /// - The writer capacity is invalid.
334    /// - The connection to the server fails.
335    /// - The exponential backoff configuration is invalid.
336    pub async fn connect_url(
337        config: WebSocketConfig,
338        message_handler: Option<MessageHandler>,
339        ping_handler: Option<PingHandler>,
340    ) -> Result<Self, TransportError> {
341        Self::connect_url_with_handler(
342            config,
343            message_handler.map(IncomingHandler::Message),
344            ping_handler.map(IncomingPingHandler::Ping),
345            None,
346            None,
347            None,
348            InitialConnectOptions::default(),
349        )
350        .await
351    }
352
353    async fn connect_url_with_handler(
354        config: WebSocketConfig,
355        handler: Option<IncomingHandler>,
356        ping_handler: Option<IncomingPingHandler>,
357        close_handler: Option<CloseHandler>,
358        state_sink: Option<SocketStateSink>,
359        connection_rate_limit: Option<ConnectionRateLimit>,
360        initial_connect_options: InitialConnectOptions,
361    ) -> Result<Self, TransportError> {
362        install_cryptographic_provider();
363
364        let is_stream_mode = handler.is_none();
365
366        // Adapters build this config by struct literal, bypassing the builder, so this is the only
367        // place the field invariants are enforced for them. Stream mode documents the reconnect and
368        // liveness fields as ignored and permits zero for them, so it checks only what it honors.
369        if is_stream_mode {
370            crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
371                TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
372            })?;
373
374            if config.heartbeat_interval_secs == Some(0) {
375                return Err(TransportError::Io(std::io::Error::new(
376                    std::io::ErrorKind::InvalidInput,
377                    "Heartbeat interval cannot be zero",
378                )));
379            }
380
381            validate_honored_inbound_limits(&config)?;
382        } else {
383            config.validate().map_err(|e| {
384                TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
385            })?;
386        }
387
388        let heartbeat_timeout = config.resolved_heartbeat_timeout().map(Duration::from_secs);
389        let frame_activity = (!is_stream_mode && heartbeat_timeout.is_some())
390            .then(|| Arc::new(AtomicBool::new(false)));
391        let reconnect_max_attempts = config.reconnect_max_attempts;
392
393        // Stream mode documents reconnect_* fields as ignored (callers may pass Some(0))
394        let connect_timeout = if is_stream_mode {
395            Duration::from_secs(10)
396        } else {
397            Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000))
398        };
399
400        let backoff = ExponentialBackoff::new(
401            Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
402            Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
403            config.reconnect_backoff_factor.unwrap_or(1.5),
404            config.reconnect_jitter_ms.unwrap_or(100),
405            true, // immediate-first
406        )
407        .map_err(|e| {
408            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
409        })?;
410
411        let cancellation_token = initial_connect_options
412            .cancellation_token
413            .unwrap_or_default();
414        let retry_policy = initial_connect_options.retry_policy;
415        let max_attempts = retry_policy
416            .as_ref()
417            .map_or(1, |policy| policy.max_attempts.get());
418        let retry_manager = RetryManager::new(initial_connect_retry_config(retry_policy.as_ref())?);
419        let attempt = AtomicU32::new(0);
420        let operation = || {
421            attempt.fetch_add(1, Ordering::Relaxed);
422
423            async {
424                await_initial_connect_attempt(&cancellation_token, async {
425                    if let Some(rate_limit) = &connection_rate_limit {
426                        rate_limit
427                            .limiter
428                            .await_keys_ready(Some(&rate_limit.keys))
429                            .await;
430                    }
431
432                    // Bound only the dial: the connection rate-limit wait has its own venue timing
433                    dst::time::timeout(
434                        connect_timeout,
435                        Box::pin(Self::connect_with_server_activity(
436                            &config.url,
437                            config.headers.clone(),
438                            config.backend,
439                            config.proxy_url.as_deref(),
440                            config.max_message_size_bytes,
441                            config.max_frame_size_bytes,
442                            frame_activity.clone(),
443                        )),
444                    )
445                    .await
446                    .unwrap_or_else(|_| {
447                        Err(TransportError::Io(std::io::Error::new(
448                            std::io::ErrorKind::TimedOut,
449                            format!(
450                                "connection timed out after {}s",
451                                connect_timeout.as_secs_f64()
452                            ),
453                        )))
454                    })
455                })
456                .await
457                .map_err(InitialConnectError)
458            }
459        };
460
461        let classify = |error: &InitialConnectError| {
462            let error = &error.0;
463            let retryable = is_retryable_initial_connect_error(error);
464            let attempt = attempt.load(Ordering::Relaxed);
465            if retryable && attempt < max_attempts && !cancellation_token.is_cancelled() {
466                log::warn!(
467                    "WebSocket connection attempt {attempt}/{max_attempts} to {REDACTED} failed: {}",
468                    error.redacted_message()
469                );
470            }
471
472            retryable
473        };
474
475        let transport = retry_manager
476            .invocation(INITIAL_CONNECT_OPERATION, operation, classify, |e| {
477                InitialConnectError(initial_connect_retry_error(e))
478            })
479            .cancellation_token(&cancellation_token)
480            .execute()
481            .await
482            .map_err(|e| e.0)?;
483
484        let attempt = attempt.load(Ordering::Relaxed);
485        if attempt > 1 {
486            log::info!("WebSocket connection established after {attempt} attempts");
487        }
488
489        let (writer, reader) = transport;
490        let mut config = config;
491        let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
492
493        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
494        let connection_epoch = Arc::new(AtomicU64::new(0));
495        let state_notify = Arc::new(tokio::sync::Notify::new());
496        let controller_notify = Arc::new(tokio::sync::Notify::new());
497        let reconnect_published = Arc::new(AtomicBool::new(true));
498        let outcome =
499            ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
500        debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
501
502        let (read_task, read_fence) = if is_stream_mode {
503            (None, None)
504        } else {
505            let read_fence = ReadSessionFence::new();
506            let read_task = Self::spawn_message_handler_task(
507                Arc::clone(&connection_mode),
508                Arc::clone(&state_notify),
509                read_fence.clone(),
510                reader,
511                0,
512                handler.as_ref(),
513                ping_handler.as_ref(),
514                close_handler.as_ref(),
515                state_sink.as_ref(),
516                config.idle_timeout_ms,
517                heartbeat_timeout,
518                frame_activity,
519            );
520            (Some(read_task), Some(read_fence))
521        };
522
523        let auth_tracker = Arc::new(OnceLock::new());
524        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
525
526        let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
527            config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
528        );
529        let write_task = Self::spawn_write_task(
530            Arc::clone(&connection_mode),
531            Arc::clone(&controller_notify),
532            Arc::clone(&reconnect_published),
533            writer,
534            writer_rx,
535            Arc::clone(&connection_epoch),
536            Arc::clone(&auth_tracker),
537            Arc::clone(&reconnect_buffer_waits_for_auth),
538            state_sink.clone(),
539        );
540
541        // Optionally spawn a heartbeat task to periodically ping server
542        let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_secs| {
543            Self::spawn_heartbeat_task(
544                Arc::clone(&connection_mode),
545                heartbeat_secs,
546                config.heartbeat_payload.clone(),
547                writer_tx.clone(),
548            )
549        });
550
551        Ok(Self {
552            config,
553            reconnect_headers,
554            handler,
555            ping_handler,
556            close_handler,
557            read_task,
558            read_fence,
559            write_task,
560            writer_tx,
561            heartbeat_task,
562            connection_mode,
563            connection_epoch,
564            state_notify,
565            controller_notify,
566            reconnect_published,
567            connect_timeout,
568            heartbeat_timeout,
569            backoff,
570            reconnect_throttle: ReconnectThrottle::default(),
571            reconnect_max_attempts,
572            reconnection_attempt_count: 0,
573            auth_tracker,
574            reconnect_buffer_waits_for_auth,
575            state_sink,
576            connection_rate_limit,
577        })
578    }
579
580    /// Connects to the server and returns the split halves of the active transport.
581    ///
582    /// Dispatches on `backend` to the matching transport implementation. The
583    /// [`TransportBackend::Tungstenite`] backend is always available; the
584    /// [`TransportBackend::Sockudo`] backend requires the `transport-sockudo`
585    /// Cargo feature (enabled by default) and uses a custom HTTP/1.1 handshake
586    /// path for upgrade headers.
587    ///
588    /// When `proxy_url` is `Some`, both backends establish an HTTP `CONNECT`
589    /// tunnel through the proxy before performing the WebSocket handshake, and
590    /// each keeps its own handshake path over the resulting stream.
591    ///
592    /// `max_message_size_bytes` and `max_frame_size_bytes` override the selected backend's inbound
593    /// caps. `None` passes that backend's current default config.
594    ///
595    /// # Errors
596    ///
597    /// Returns an error if the URL is invalid, headers fail to parse, the TCP / TLS layer cannot be
598    /// established, the proxy refuses the tunnel, or the WebSocket handshake is rejected by the
599    /// peer. Also returns an error if the Sockudo backend is selected without the
600    /// `transport-sockudo` feature.
601    #[inline]
602    pub async fn connect_with_server(
603        url: &str,
604        headers: Vec<(String, String)>,
605        backend: TransportBackend,
606        proxy_url: Option<&str>,
607        max_message_size_bytes: Option<usize>,
608        max_frame_size_bytes: Option<usize>,
609    ) -> Result<(MessageWriter, MessageReader), TransportError> {
610        Self::connect_with_server_activity(
611            url,
612            headers,
613            backend,
614            proxy_url,
615            max_message_size_bytes,
616            max_frame_size_bytes,
617            None,
618        )
619        .await
620    }
621
622    async fn connect_with_server_activity(
623        url: &str,
624        headers: Vec<(String, String)>,
625        backend: TransportBackend,
626        proxy_url: Option<&str>,
627        max_message_size_bytes: Option<usize>,
628        max_frame_size_bytes: Option<usize>,
629        frame_activity: Option<Arc<AtomicBool>>,
630    ) -> Result<(MessageWriter, MessageReader), TransportError> {
631        #[cfg(all(feature = "simulation", madsim))]
632        if backend == TransportBackend::Sockudo {
633            return Err(TransportError::Other(
634                "Sockudo timers are unsupported under simulation; select Tungstenite".into(),
635            ));
636        }
637
638        #[cfg(all(feature = "simulation", madsim))]
639        if proxy_url.is_some() || !url.starts_with("ws://") {
640            return Err(TransportError::InvalidUrl(
641                "WebSocket simulation requires plaintext ws:// endpoints without a proxy".into(),
642            ));
643        }
644
645        let limits = InboundConfig {
646            max_message_size_bytes,
647            max_frame_size_bytes,
648            frame_activity,
649        };
650
651        match backend {
652            TransportBackend::Tungstenite => match proxy_url {
653                Some(proxy) => {
654                    Box::pin(Self::connect_tungstenite_via_proxy(
655                        url, headers, proxy, limits,
656                    ))
657                    .await
658                }
659                None => Self::connect_tungstenite(url, headers, limits).await,
660            },
661            TransportBackend::Sockudo => {
662                #[cfg(feature = "transport-sockudo")]
663                {
664                    match proxy_url {
665                        Some(proxy) => {
666                            Box::pin(Self::connect_sockudo_via_proxy(url, headers, proxy, limits))
667                                .await
668                        }
669                        None => Self::connect_sockudo(url, headers, limits).await,
670                    }
671                }
672
673                #[cfg(not(feature = "transport-sockudo"))]
674                {
675                    Err(TransportError::Other(
676                        "sockudo backend selected but the transport-sockudo \
677                         Cargo feature is not enabled"
678                            .to_string(),
679                    ))
680                }
681            }
682        }
683    }
684
685    async fn connect_tungstenite(
686        url: &str,
687        headers: Vec<(String, String)>,
688        limits: InboundConfig,
689    ) -> Result<(MessageWriter, MessageReader), TransportError> {
690        let request = tungstenite_request(url, headers)?;
691        let mode = tokio_tungstenite::tungstenite::client::uri_mode(request.uri())?;
692        let host = request
693            .uri()
694            .host()
695            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".into()))?;
696        let port = request
697            .uri()
698            .port_u16()
699            .unwrap_or(if matches!(mode, Mode::Tls) { 443 } else { 80 });
700        let address = format!("{host}:{port}");
701        let socket = dst::net::TcpStream::connect(address.as_str()).await?;
702        #[cfg(not(all(feature = "simulation", madsim)))]
703        crate::net::apply_socket_options(&socket);
704        let socket = crate::tls::tcp_tls(&request, mode, socket, None).await?;
705        let socket = FrameStream::new(socket, limits.frame_activity.clone());
706        let (stream, _) = tokio_tungstenite::client_async_with_config(
707            request,
708            socket,
709            tungstenite_protocol_config(&limits),
710        )
711        .await?;
712        let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
713        Ok(transport.split())
714    }
715
716    /// Connects via an HTTP `CONNECT` proxy and performs the WebSocket
717    /// handshake over the resulting tunnel.
718    ///
719    /// Recognized but unsupported proxy schemes (currently SOCKS) log a
720    /// warning and fall back to a direct connection so existing REST proxy
721    /// configs remain usable. Only available in production builds; the
722    /// turmoil simulator does not model arbitrary outbound TCP via a proxy.
723    #[inline]
724    #[cfg(not(feature = "turmoil"))]
725    async fn connect_tungstenite_via_proxy(
726        url: &str,
727        headers: Vec<(String, String)>,
728        proxy_url: &str,
729        limits: InboundConfig,
730    ) -> Result<(MessageWriter, MessageReader), TransportError> {
731        let proxy = match ProxyKind::parse(proxy_url)? {
732            ProxyKind::Http(target) => target,
733            ProxyKind::Unsupported { scheme } => {
734                log::warn!(
735                    "WebSocket proxy_url scheme '{scheme}' is not yet supported; \
736                     connecting without a WebSocket proxy"
737                );
738                return Self::connect_tungstenite(url, headers, limits).await;
739            }
740        };
741
742        let request = tungstenite_request(url, headers)?;
743
744        let target = WsTarget::parse(url)?;
745        let stream = tunnel_via_proxy(&target, &proxy).await?;
746
747        // `ProxiedStream` implements the IO traits over all four variants, so one
748        // instantiation covers every tunnel shape. The future is boxed because
749        // `client_async` produces a large state machine.
750        let transport: BoxedWsTransport =
751            Box::pin(proxied_ws_handshake(request, stream, limits)).await?;
752
753        Ok(transport.split())
754    }
755
756    /// Turmoil simulator variant: HTTP `CONNECT` tunneling is not supported
757    /// under the simulator so any proxy URL is rejected up front.
758    #[inline]
759    #[cfg(feature = "turmoil")]
760    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
761    #[expect(
762        clippy::unused_async,
763        clippy::unused_async_trait_impl,
764        reason = "signature mirrors the production variant; both are awaited in the dispatcher"
765    )]
766    async fn connect_tungstenite_via_proxy(
767        _url: &str,
768        _headers: Vec<(String, String)>,
769        _proxy_url: &str,
770        _limits: InboundConfig,
771    ) -> Result<(MessageWriter, MessageReader), TransportError> {
772        Err(TransportError::Other(
773            "proxy_url is not supported under the turmoil simulator".to_string(),
774        ))
775    }
776
777    /// Connects with the server using the sockudo-ws backend.
778    ///
779    /// Uses a local HTTP/1.1 handshake path so error logging and stream
780    /// construction stay in our hands regardless of header count.
781    ///
782    /// Under the turmoil simulator, only plaintext `ws://` is supported (the
783    /// simulator does not model TLS), so a `wss://` URL returns
784    /// [`TransportError::Tls`] up front.
785    #[inline]
786    #[cfg(feature = "transport-sockudo")]
787    async fn connect_sockudo(
788        url: &str,
789        headers: Vec<(String, String)>,
790        limits: InboundConfig,
791    ) -> Result<(MessageWriter, MessageReader), TransportError> {
792        let target = SockudoTarget::parse(url)?;
793        validate_extra_headers(&headers).map_err(TransportError::from)?;
794
795        #[cfg(feature = "turmoil")]
796        if target.is_tls {
797            return Err(TransportError::Tls(
798                "wss:// is not supported under the turmoil simulator; use ws://".to_string(),
799            ));
800        }
801
802        let tcp_stream = TcpStream::connect((target.host.as_str(), target.port))
803            .await
804            .map_err(TransportError::Io)?;
805
806        crate::net::apply_socket_options(&tcp_stream);
807
808        #[cfg(not(feature = "turmoil"))]
809        if target.is_tls {
810            let mut root_store = rustls::RootCertStore::empty();
811            root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
812            let config = ClientConfig::builder()
813                .with_root_certificates(root_store)
814                .with_no_client_auth();
815            let connector = TlsConnector::from(std::sync::Arc::new(config));
816            let domain = rustls::pki_types::ServerName::try_from(target.host.clone())
817                .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
818            let tls_stream = connector
819                .connect(domain, tcp_stream)
820                .await
821                .map_err(TransportError::Io)?;
822            return Self::finish_sockudo_handshake(tls_stream, &target, &headers, limits).await;
823        }
824
825        Self::finish_sockudo_handshake(tcp_stream, &target, &headers, limits).await
826    }
827
828    /// Connects via an HTTP `CONNECT` proxy and performs the sockudo WebSocket
829    /// handshake over the resulting tunnel.
830    ///
831    /// Recognized but unsupported proxy schemes (currently SOCKS) log a warning
832    /// and fall back to a direct connection, matching the Tungstenite path.
833    #[inline]
834    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
835    async fn connect_sockudo_via_proxy(
836        url: &str,
837        headers: Vec<(String, String)>,
838        proxy_url: &str,
839        limits: InboundConfig,
840    ) -> Result<(MessageWriter, MessageReader), TransportError> {
841        let proxy = match ProxyKind::parse(proxy_url)? {
842            ProxyKind::Http(target) => target,
843            ProxyKind::Unsupported { scheme } => {
844                log::warn!(
845                    "WebSocket proxy_url scheme '{scheme}' is not yet supported; \
846                     connecting without a WebSocket proxy"
847                );
848                return Self::connect_sockudo(url, headers, limits).await;
849            }
850        };
851
852        let target = SockudoTarget::parse(url)?;
853        validate_extra_headers(&headers).map_err(TransportError::from)?;
854
855        // `tunnel_via_proxy` establishes upstream TLS inside the tunnel when the
856        // target is `wss://`, so the handshake below runs over the finished stream
857        // regardless of which of the four tunnel shapes it returned.
858        let ws_target = WsTarget::parse(url)?;
859        let stream = tunnel_via_proxy(&ws_target, &proxy).await?;
860
861        Self::finish_sockudo_handshake(stream, &target, &headers, limits).await
862    }
863
864    /// Turmoil simulator variant: HTTP `CONNECT` tunneling is not modeled under
865    /// the simulator so any proxy URL is rejected up front.
866    #[inline]
867    #[cfg(all(feature = "transport-sockudo", feature = "turmoil"))]
868    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
869    #[expect(
870        clippy::unused_async,
871        clippy::unused_async_trait_impl,
872        reason = "signature mirrors the production variant; both are awaited in the dispatcher"
873    )]
874    async fn connect_sockudo_via_proxy(
875        _url: &str,
876        _headers: Vec<(String, String)>,
877        _proxy_url: &str,
878        _limits: InboundConfig,
879    ) -> Result<(MessageWriter, MessageReader), TransportError> {
880        Err(TransportError::Other(
881            "proxy_url is not supported under the turmoil simulator".to_string(),
882        ))
883    }
884
885    #[cfg(feature = "transport-sockudo")]
886    async fn finish_sockudo_handshake<S>(
887        stream: S,
888        target: &SockudoTarget,
889        headers: &[(String, String)],
890        limits: InboundConfig,
891    ) -> Result<(MessageWriter, MessageReader), TransportError>
892    where
893        S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
894    {
895        // Use one path for uniform error logging and ownership of stream
896        // construction; sockudo's handshake error drops the rejection status
897        // the retry policy needs.
898        let mut stream = FrameStream::new(stream, limits.frame_activity.clone());
899        let handshake =
900            client_handshake_with_headers(&mut stream, &target.host_header, &target.path, headers)
901                .await?;
902
903        // Reading the HTTP 101 may also read the first WebSocket frame prefix;
904        // replay it only when present so the ordinary path stays unwrapped.
905        let stream = match handshake.leftover {
906            Some(prefix) => SockudoStream::<Http1>::new(PrefixedIo::new(stream, prefix)),
907            None => SockudoStream::<Http1>::new(stream),
908        };
909
910        let ws = SockudoWebSocketStream::from_raw(
911            stream,
912            Role::Client,
913            sockudo_protocol_config(&limits),
914        );
915        let transport: BoxedWsTransport = Box::pin(SockudoTransport::new(ws));
916        Ok(transport.split())
917    }
918}
919
920#[derive(Clone, Debug, Default)]
921struct InboundConfig {
922    max_message_size_bytes: Option<usize>,
923    max_frame_size_bytes: Option<usize>,
924    frame_activity: Option<Arc<AtomicBool>>,
925}
926
927fn validate_honored_inbound_limits(config: &WebSocketConfig) -> Result<(), TransportError> {
928    if config.max_message_size_bytes == Some(0) {
929        return Err(TransportError::Io(std::io::Error::new(
930            std::io::ErrorKind::InvalidInput,
931            "max_message_size_bytes must be positive",
932        )));
933    }
934
935    if config.max_frame_size_bytes == Some(0) {
936        return Err(TransportError::Io(std::io::Error::new(
937            std::io::ErrorKind::InvalidInput,
938            "max_frame_size_bytes must be positive",
939        )));
940    }
941
942    Ok(())
943}
944
945fn tungstenite_protocol_config(
946    limits: &InboundConfig,
947) -> Option<tokio_tungstenite::tungstenite::protocol::WebSocketConfig> {
948    if limits.max_message_size_bytes.is_none() && limits.max_frame_size_bytes.is_none() {
949        return None;
950    }
951
952    let mut config = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
953    if let Some(size) = limits.max_message_size_bytes {
954        config.max_message_size = Some(size);
955    }
956
957    if let Some(size) = limits.max_frame_size_bytes {
958        config.max_frame_size = Some(size);
959    }
960
961    Some(config)
962}
963
964#[cfg(feature = "transport-sockudo")]
965fn sockudo_protocol_config(limits: &InboundConfig) -> SockudoConfig {
966    let mut config = SockudoConfig::builder()
967        .auto_ping(false)
968        .idle_timeout(0)
969        .build();
970
971    if let Some(size) = limits.max_message_size_bytes {
972        config.max_message_size = size;
973    }
974
975    if let Some(size) = limits.max_frame_size_bytes {
976        config.max_frame_size = size;
977    }
978
979    config
980}
981
982fn tungstenite_request(
983    url: &str,
984    headers: Vec<(String, String)>,
985) -> Result<Request, TransportError> {
986    let mut request = url.into_client_request().map_err(TransportError::from)?;
987
988    for (key, value) in headers {
989        let value = HeaderValue::from_str(&value)
990            .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
991        let name: HeaderName = key
992            .parse()
993            .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
994        request.headers_mut().insert(name, value);
995    }
996
997    Ok(request)
998}
999
1000fn is_connection_drop_transport_error(err: &TransportError) -> bool {
1001    err.is_closed() || matches!(err, TransportError::Io(e) if is_connection_drop_io_error(e))
1002}
1003
1004fn is_retryable_initial_connect_error(err: &TransportError) -> bool {
1005    match err {
1006        TransportError::ConnectionClosed
1007        | TransportError::ConnectionReset
1008        | TransportError::ClosedByPeer(_) => true,
1009        TransportError::Io(error) => !matches!(
1010            error.kind(),
1011            std::io::ErrorKind::InvalidInput
1012                | std::io::ErrorKind::InvalidData
1013                | std::io::ErrorKind::Unsupported
1014                | std::io::ErrorKind::PermissionDenied
1015        ),
1016        TransportError::UpgradeRejected(status) | TransportError::ProxyConnectRejected(status) => {
1017            retryable_status(*status)
1018        }
1019        TransportError::InvalidUrl(_)
1020        | TransportError::Handshake(_)
1021        | TransportError::Tls(_)
1022        | TransportError::Protocol(_)
1023        | TransportError::MessageTooLarge
1024        | TransportError::FrameTooLarge
1025        | TransportError::InvalidUtf8
1026        | TransportError::Other(_) => false,
1027    }
1028}
1029
1030fn initial_connect_retry_config(
1031    policy: Option<&InitialConnectRetryPolicy>,
1032) -> Result<RetryConfig, TransportError> {
1033    if let Some(policy) = policy {
1034        ExponentialBackoff::new(
1035            policy.delay_initial,
1036            policy.delay_max,
1037            policy.backoff_factor,
1038            policy.jitter_ms,
1039            false,
1040        )
1041        .map_err(|e| {
1042            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
1043        })?;
1044
1045        Ok(RetryConfig {
1046            max_retries: policy.max_attempts.get() - 1,
1047            initial_delay_ms: duration_to_millis("delay_initial", policy.delay_initial)?,
1048            max_delay_ms: duration_to_millis("delay_max", policy.delay_max)?,
1049            backoff_factor: policy.backoff_factor,
1050            jitter_ms: policy.jitter_ms,
1051            operation_timeout_ms: None,
1052            immediate_first: false,
1053            max_elapsed_ms: None,
1054        })
1055    } else {
1056        Ok(RetryConfig {
1057            max_retries: 0,
1058            initial_delay_ms: 1,
1059            max_delay_ms: 1,
1060            backoff_factor: 1.0,
1061            jitter_ms: 0,
1062            operation_timeout_ms: None,
1063            immediate_first: false,
1064            max_elapsed_ms: None,
1065        })
1066    }
1067}
1068
1069fn duration_to_millis(field: &str, duration: Duration) -> Result<u64, TransportError> {
1070    let nanoseconds = duration.as_nanos();
1071    if nanoseconds > u128::from(u64::MAX) {
1072        return Err(TransportError::Io(std::io::Error::new(
1073            std::io::ErrorKind::InvalidInput,
1074            format!("{field} exceeds the maximum backoff duration"),
1075        )));
1076    }
1077
1078    let milliseconds = nanoseconds
1079        .div_ceil(1_000_000)
1080        .min(u128::from(u64::MAX / 1_000_000));
1081    u64::try_from(milliseconds).map_err(|_| {
1082        TransportError::Io(std::io::Error::new(
1083            std::io::ErrorKind::InvalidInput,
1084            format!("{field} exceeds the maximum backoff duration"),
1085        ))
1086    })
1087}
1088
1089async fn await_initial_connect_attempt<F, T>(
1090    cancellation_token: &CancellationToken,
1091    attempt: F,
1092) -> Result<T, TransportError>
1093where
1094    F: Future<Output = Result<T, TransportError>>,
1095{
1096    tokio::select! {
1097        biased;
1098        () = cancellation_token.cancelled() => Err(initial_connect_cancelled()),
1099        result = attempt => result,
1100    }
1101}
1102
1103struct InitialConnectError(TransportError);
1104
1105impl std::error::Error for InitialConnectError {}
1106
1107impl Display for InitialConnectError {
1108    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1109        f.write_str(&self.0.redacted_message())
1110    }
1111}
1112
1113impl Debug for InitialConnectError {
1114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1115        Display::fmt(self, f)
1116    }
1117}
1118
1119fn initial_connect_retry_error(error: RetryError) -> TransportError {
1120    let kind = match error {
1121        RetryError::Canceled => return initial_connect_cancelled(),
1122        RetryError::InvalidConfiguration { .. } => std::io::ErrorKind::InvalidInput,
1123        RetryError::OperationTimeout { .. } | RetryError::ElapsedBudgetExceeded { .. } => {
1124            std::io::ErrorKind::TimedOut
1125        }
1126    };
1127
1128    TransportError::Io(std::io::Error::new(kind, error))
1129}
1130
1131fn initial_connect_cancelled() -> TransportError {
1132    TransportError::Io(std::io::Error::new(
1133        std::io::ErrorKind::Interrupted,
1134        "initial WebSocket connection cancelled",
1135    ))
1136}
1137
1138// Debug when we asked to disconnect (Disconnect/Closed), else Warn for a peer close
1139fn read_termination_log_level(connection_state: &AtomicU8) -> log::Level {
1140    let mode = ConnectionMode::from_atomic(connection_state);
1141    if mode.is_disconnect() || mode.is_closed() {
1142        log::Level::Debug
1143    } else {
1144        log::Level::Warn
1145    }
1146}
1147
1148#[cfg(test)]
1149mod connection_error_tests {
1150    use std::io;
1151
1152    use rstest::rstest;
1153
1154    use super::*;
1155    use crate::transport::CloseFrame;
1156
1157    #[rstest]
1158    #[case(TransportError::ConnectionClosed, true)]
1159    #[case(TransportError::ConnectionReset, true)]
1160    #[case(TransportError::ClosedByPeer(Some(CloseFrame::new(1000, "bye"))), true)]
1161    #[case(TransportError::ClosedByPeer(None), true)]
1162    #[case(TransportError::Io(io::Error::from(io::ErrorKind::BrokenPipe)), true)]
1163    #[case(
1164        TransportError::Io(io::Error::from(io::ErrorKind::ConnectionReset)),
1165        true
1166    )]
1167    #[case(TransportError::Io(io::Error::from(io::ErrorKind::TimedOut)), true)]
1168    #[case(
1169        TransportError::Io(io::Error::from(io::ErrorKind::UnexpectedEof)),
1170        true
1171    )]
1172    #[case(
1173        TransportError::Io(io::Error::from(io::ErrorKind::InvalidInput)),
1174        false
1175    )]
1176    #[case(TransportError::InvalidUrl("http://example.com".into()), false)]
1177    #[case(TransportError::Handshake("bad".into()), false)]
1178    #[case(TransportError::Protocol("bad opcode".into()), false)]
1179    #[case(TransportError::Tls("bad certificate".into()), false)]
1180    #[case(TransportError::MessageTooLarge, false)]
1181    #[case(TransportError::FrameTooLarge, false)]
1182    #[case(TransportError::InvalidUtf8, false)]
1183    #[case(TransportError::Other("backend protocol mismatch".into()), false)]
1184    fn connection_drop_transport_error_classification(
1185        #[case] err: TransportError,
1186        #[case] expected: bool,
1187    ) {
1188        assert_eq!(is_connection_drop_transport_error(&err), expected);
1189    }
1190
1191    #[rstest]
1192    #[case(Duration::ZERO, 0)]
1193    #[case(Duration::from_nanos(1), 1)]
1194    #[case(Duration::from_micros(500), 1)]
1195    #[case(Duration::from_nanos(1_000_001), 2)]
1196    #[case(Duration::from_millis(500), 500)]
1197    #[case(Duration::from_secs(5), 5_000)]
1198    fn duration_to_millis_rounds_up(#[case] duration: Duration, #[case] expected: u64) {
1199        assert_eq!(duration_to_millis("delay", duration).unwrap(), expected);
1200    }
1201
1202    #[rstest]
1203    fn duration_to_millis_caps_at_backoff_maximum() {
1204        assert_eq!(
1205            duration_to_millis("delay", Duration::from_nanos(u64::MAX)).unwrap(),
1206            u64::MAX / 1_000_000
1207        );
1208    }
1209
1210    #[rstest]
1211    fn duration_to_millis_rejects_excessive_duration() {
1212        let error = duration_to_millis("delay", Duration::from_secs(u64::MAX)).unwrap_err();
1213
1214        let TransportError::Io(error) = error else {
1215            panic!("expected an I/O error, was: {error:?}");
1216        };
1217
1218        assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1219        assert_eq!(
1220            error.to_string(),
1221            "delay exceeds the maximum backoff duration"
1222        );
1223    }
1224
1225    #[rstest]
1226    fn initial_connect_retry_config_preserves_duration_validation() {
1227        let policy = InitialConnectRetryPolicy {
1228            max_attempts: std::num::NonZeroU32::new(2).unwrap(),
1229            delay_initial: Duration::from_micros(900),
1230            delay_max: Duration::from_micros(500),
1231            backoff_factor: 2.0,
1232            jitter_ms: 0,
1233        };
1234
1235        let error = initial_connect_retry_config(Some(&policy)).unwrap_err();
1236
1237        let TransportError::Io(error) = error else {
1238            panic!("expected an I/O error, was: {error:?}");
1239        };
1240
1241        assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1242        assert!(
1243            error
1244                .to_string()
1245                .contains("delay_max must be >= delay_initial")
1246        );
1247    }
1248
1249    #[rstest]
1250    #[tokio::test]
1251    async fn initial_connect_attempt_prefers_cancellation_over_ready_result() {
1252        let token = CancellationToken::new();
1253        token.cancel();
1254
1255        let error = await_initial_connect_attempt(&token, std::future::ready(Ok(())))
1256            .await
1257            .expect_err("cancellation should take priority");
1258
1259        assert!(
1260            matches!(error, TransportError::Io(ref error) if error.kind() == io::ErrorKind::Interrupted)
1261        );
1262    }
1263
1264    #[rstest]
1265    fn unset_inbound_limits_preserve_backend_defaults() {
1266        let limits = InboundConfig::default();
1267        assert!(tungstenite_protocol_config(&limits).is_none());
1268
1269        #[cfg(feature = "transport-sockudo")]
1270        {
1271            let config = sockudo_protocol_config(&limits);
1272            let default = SockudoConfig::default();
1273            assert_eq!(config.max_message_size, default.max_message_size);
1274            assert_eq!(config.max_frame_size, default.max_frame_size);
1275            assert!(!config.auto_ping);
1276            assert_eq!(config.idle_timeout, 0);
1277        }
1278    }
1279
1280    #[rstest]
1281    fn configured_inbound_limit_overrides_only_that_cap() {
1282        let limits = InboundConfig {
1283            max_message_size_bytes: Some(32),
1284            max_frame_size_bytes: None,
1285            frame_activity: None,
1286        };
1287
1288        let config = tungstenite_protocol_config(&limits).expect("override should build a config");
1289        let default = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
1290
1291        assert_eq!(config.max_message_size, Some(32));
1292        assert_eq!(config.max_frame_size, default.max_frame_size);
1293
1294        #[cfg(feature = "transport-sockudo")]
1295        {
1296            let config = sockudo_protocol_config(&limits);
1297            let default = SockudoConfig::default();
1298            assert_eq!(config.max_message_size, 32);
1299            assert_eq!(config.max_frame_size, default.max_frame_size);
1300            assert!(!config.auto_ping);
1301            assert_eq!(config.idle_timeout, 0);
1302        }
1303    }
1304}
1305
1306/// Complete the WebSocket handshake over a stream that has already been
1307/// tunneled through an HTTP `CONNECT` proxy. Generic over the concrete
1308/// stream type so the four [`super::proxy::ProxiedStream`] variants share
1309/// a single body.
1310#[cfg(not(feature = "turmoil"))]
1311async fn proxied_ws_handshake<S>(
1312    request: tokio_tungstenite::tungstenite::handshake::client::Request,
1313    stream: S,
1314    limits: InboundConfig,
1315) -> Result<BoxedWsTransport, TransportError>
1316where
1317    S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
1318{
1319    let stream = FrameStream::new(stream, limits.frame_activity.clone());
1320    let (ws, _resp) = tokio_tungstenite::client_async_with_config(
1321        request,
1322        stream,
1323        tungstenite_protocol_config(&limits),
1324    )
1325    .await
1326    .map_err(TransportError::from)?;
1327    Ok(Box::pin(TungsteniteTransport::new(ws)))
1328}
1329
1330/// Parsed components of a `ws://` / `wss://` URL needed by the sockudo backend.
1331///
1332/// Sockudo's HTTP/1.1 client passes the `host` argument verbatim as the
1333/// HTTP `Host:` header, so it must include the explicit port when one is
1334/// present in the URL (RFC 7230 section 5.4). The DNS / SNI lookup uses the bare
1335/// host without the port.
1336#[cfg(feature = "transport-sockudo")]
1337#[derive(Debug, PartialEq, Eq)]
1338struct SockudoTarget {
1339    host: String,
1340    host_header: String,
1341    port: u16,
1342    path: String,
1343    is_tls: bool,
1344}
1345
1346#[cfg(feature = "transport-sockudo")]
1347impl SockudoTarget {
1348    fn parse(url: &str) -> Result<Self, TransportError> {
1349        let parsed = url::Url::parse(url)
1350            .map_err(|e| TransportError::InvalidUrl(format!("invalid WebSocket URL: {e}")))?;
1351
1352        let scheme = parsed.scheme();
1353
1354        let is_tls = match scheme {
1355            "ws" => false,
1356            "wss" => true,
1357            other => {
1358                return Err(TransportError::InvalidUrl(format!(
1359                    "expected ws:// or wss:// scheme, was {other}"
1360                )));
1361            }
1362        };
1363
1364        let raw_host = parsed
1365            .host_str()
1366            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
1367
1368        // url::Url stores IPv6 hosts in their bracketed form (e.g. `[::1]`).
1369        // Brackets are correct for the HTTP `Host:` header but invalid for
1370        // DNS/TCP and TLS SNI, so we keep two representations: a bracketed
1371        // `host_header` for the upgrade, and a bare `host` for the socket connection.
1372        let is_bracketed = raw_host.starts_with('[') && raw_host.ends_with(']');
1373        let host = if is_bracketed {
1374            raw_host[1..raw_host.len() - 1].to_string()
1375        } else {
1376            raw_host.to_string()
1377        };
1378
1379        let explicit_port = parsed.port();
1380        let port = explicit_port.unwrap_or(if is_tls { 443 } else { 80 });
1381
1382        let host_header = match explicit_port {
1383            Some(p) => format!("{raw_host}:{p}"),
1384            None => raw_host.to_string(),
1385        };
1386
1387        let path = if parsed.path().is_empty() {
1388            "/".to_string()
1389        } else {
1390            let mut p = parsed.path().to_string();
1391            if let Some(query) = parsed.query() {
1392                p.push('?');
1393                p.push_str(query);
1394            }
1395
1396            p
1397        };
1398
1399        Ok(Self {
1400            host,
1401            host_header,
1402            port,
1403            path,
1404            is_tls,
1405        })
1406    }
1407}
1408
1409impl WebSocketClientInner {
1410    /// Reconnect with server.
1411    ///
1412    /// Make a new connection with server. Use the new read and write halves
1413    /// to update self writer and read and heartbeat tasks.
1414    ///
1415    /// For stream-based clients (created via [`WebSocketClient::stream_builder`]), reconnection is
1416    /// disabled because the reader is owned by the caller and cannot be replaced. Stream users
1417    /// should handle disconnections by creating a new connection.
1418    ///
1419    /// The reconnect timeout bounds only connection establishment. Once the
1420    /// new writer is handed to the writer task the swap runs to completion,
1421    /// so buffered messages can never drain into a connection that lost its
1422    /// reader to a timeout; the post-connect steps are individually bounded
1423    /// by the writer task's graceful-shutdown timeout.
1424    ///
1425    /// # Errors
1426    ///
1427    /// Returns an error if:
1428    /// - The reconnection attempt times out.
1429    /// - The connection to the server fails.
1430    pub async fn reconnect(&mut self) -> Result<(), TransportError> {
1431        Box::pin(self.reconnect_with_outcome()).await.map(|_| ())
1432    }
1433
1434    async fn wait_for_reconnect_publication(&self) -> bool {
1435        let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1436
1437        loop {
1438            let mut notified = pin!(self.controller_notify.notified());
1439            notified.as_mut().enable();
1440
1441            if !ConnectionMode::from_atomic(&self.connection_mode).is_reconnect() {
1442                return false;
1443            }
1444
1445            if self.reconnect_published.load(Ordering::SeqCst) {
1446                return true;
1447            }
1448
1449            tokio::select! {
1450                biased;
1451                () = notified => {}
1452                () = dst::time::sleep(fallback_interval) => {}
1453            }
1454        }
1455    }
1456
1457    async fn reconnect_with_outcome(&mut self) -> Result<ReconnectOutcome, TransportError> {
1458        log::info!("Reconnecting");
1459
1460        if self.handler.is_none() {
1461            log::warn!(
1462                "Auto-reconnect disabled for stream-based WebSocket client; \
1463                stream users must manually reconnect by creating a new connection"
1464            );
1465            // Transition to CLOSED state to stop reconnection attempts
1466            self.connection_mode
1467                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1468            fail_registered_auth(
1469                self.auth_tracker.as_ref(),
1470                "WebSocket stream mode cannot reconnect",
1471            );
1472            // Publish the terminal state only once its bookkeeping is complete; the
1473            // controller used to notify after `reconnect` returned, so a waiter must
1474            // not observe `Closed` before pending authentication has been failed.
1475            self.state_notify.notify_waiters();
1476            return Ok(ReconnectOutcome::Aborted);
1477        }
1478
1479        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1480            log::debug!("Reconnect aborted due to disconnect state");
1481            return Ok(ReconnectOutcome::Aborted);
1482        }
1483
1484        if let Some(rate_limit) = &self.connection_rate_limit {
1485            rate_limit
1486                .limiter
1487                .await_keys_ready(Some(&rate_limit.keys))
1488                .await;
1489        }
1490
1491        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1492            log::debug!("Reconnect aborted during connection rate-limit wait");
1493            return Ok(ReconnectOutcome::Aborted);
1494        }
1495
1496        let frame_activity = self
1497            .heartbeat_timeout
1498            .map(|_| Arc::new(AtomicBool::new(false)));
1499
1500        // Bound only connection establishment; the swap below must run to completion
1501        let (new_writer, reader) = dst::time::timeout(
1502            self.connect_timeout,
1503            Box::pin(Self::connect_with_server_activity(
1504                &self.config.url,
1505                self.reconnect_headers.snapshot(),
1506                self.config.backend,
1507                self.config.proxy_url.as_deref(),
1508                self.config.max_message_size_bytes,
1509                self.config.max_frame_size_bytes,
1510                frame_activity.clone(),
1511            )),
1512        )
1513        .await
1514        .map_err(|_| {
1515            TransportError::Io(std::io::Error::new(
1516                std::io::ErrorKind::TimedOut,
1517                format!(
1518                    "reconnection timed out after {}s",
1519                    self.connect_timeout.as_secs_f64()
1520                ),
1521            ))
1522        })??;
1523
1524        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1525            log::debug!("Reconnect aborted mid-flight (after connect)");
1526            return Ok(ReconnectOutcome::Aborted);
1527        }
1528
1529        // Use a oneshot channel to synchronize the writer swap before transitioning
1530        // back to ACTIVE. Buffered messages stay in the writer task and replay later.
1531        let (tx, rx) = tokio::sync::oneshot::channel();
1532
1533        if let Err(e) = self
1534            .writer_tx
1535            .send_update(WriterCommand::Update(new_writer, tx))
1536        {
1537            log::error!("{e}");
1538            return Err(TransportError::Io(std::io::Error::new(
1539                std::io::ErrorKind::BrokenPipe,
1540                format!("Failed to send update command: {e}"),
1541            )));
1542        }
1543
1544        // Wait for writer to confirm it accepted the new socket
1545        let connection_epoch = match rx.await {
1546            Ok(connection_epoch) => {
1547                log::debug!("Writer confirmed socket update: epoch={connection_epoch}");
1548                connection_epoch
1549            }
1550            Err(e) => {
1551                log::error!("Writer dropped update channel: {e}");
1552                return Err(TransportError::Io(std::io::Error::new(
1553                    std::io::ErrorKind::BrokenPipe,
1554                    "Writer task dropped response channel",
1555                )));
1556            }
1557        };
1558
1559        // Delay before closing connection
1560        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1561
1562        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1563            log::debug!("Reconnect aborted mid-flight (after delay)");
1564            return Ok(ReconnectOutcome::Aborted);
1565        }
1566
1567        if let Some(read_fence) = self.read_fence.take() {
1568            read_fence.invalidate();
1569        }
1570
1571        if let Some(ref read_task) = self.read_task.take()
1572            && !read_task.is_finished()
1573        {
1574            read_task.abort();
1575            log_task_aborted("read");
1576        }
1577
1578        if !self.wait_for_reconnect_publication().await {
1579            log::debug!("Reconnect aborted before state publication completed");
1580            return Ok(ReconnectOutcome::Aborted);
1581        }
1582
1583        // Atomically transition from Reconnect to Active
1584        // This prevents race condition where disconnect could be requested between check and store
1585        if ConnectionMode::complete_reconnect_with_sink(
1586            &self.connection_mode,
1587            self.state_sink.as_ref(),
1588        ) == ReconnectOutcome::Aborted
1589        {
1590            log::debug!("Reconnect aborted (state changed during reconnect)");
1591            return Ok(ReconnectOutcome::Aborted);
1592        }
1593
1594        let read_fence = ReadSessionFence::new();
1595        self.read_task = Some(Self::spawn_message_handler_task(
1596            Arc::clone(&self.connection_mode),
1597            Arc::clone(&self.state_notify),
1598            read_fence.clone(),
1599            reader,
1600            connection_epoch,
1601            self.handler.as_ref(),
1602            self.ping_handler.as_ref(),
1603            self.close_handler.as_ref(),
1604            self.state_sink.as_ref(),
1605            self.config.idle_timeout_ms,
1606            self.heartbeat_timeout,
1607            frame_activity,
1608        ));
1609        self.read_fence = Some(read_fence);
1610
1611        log::info!("Reconnect succeeded");
1612        Ok(ReconnectOutcome::Reconnected)
1613    }
1614
1615    /// Returns whether the client's transport tasks are still running.
1616    ///
1617    /// Returns `true` if both the read and write tasks are still running.
1618    /// There may be some delay between the connection closing and the
1619    /// client detecting it.
1620    #[inline]
1621    #[must_use]
1622    pub fn is_alive(&self) -> bool {
1623        match &self.read_task {
1624            Some(read_task) => !read_task.is_finished() && !self.write_task.is_finished(),
1625            None => !self.write_task.is_finished(),
1626        }
1627    }
1628
1629    #[expect(
1630        clippy::too_many_arguments,
1631        reason = "both handler modes share the same reader lifecycle"
1632    )]
1633    fn spawn_message_handler_task(
1634        connection_state: Arc<AtomicU8>,
1635        state_notify: Arc<tokio::sync::Notify>,
1636        read_fence: ReadSessionFence,
1637        mut reader: MessageReader,
1638        connection_epoch: u64,
1639        handler: Option<&IncomingHandler>,
1640        ping_handler: Option<&IncomingPingHandler>,
1641        close_handler: Option<&CloseHandler>,
1642        state_sink: Option<&SocketStateSink>,
1643        idle_timeout_ms: Option<u64>,
1644        heartbeat_timeout: Option<Duration>,
1645        frame_activity: Option<Arc<AtomicBool>>,
1646    ) -> dst::task::JoinHandle<()> {
1647        log::debug!("Started message handler task 'read'");
1648
1649        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1650        let idle_timeout = idle_timeout_ms.map(Duration::from_millis);
1651
1652        let handler = handler.cloned();
1653        let ping_handler = ping_handler.cloned();
1654        let close_handler = close_handler.cloned();
1655        let state_sink = state_sink.cloned();
1656
1657        dst::task::spawn(async move {
1658            let mut last_data_time = dst::time::Instant::now();
1659            let mut last_frame_time = dst::time::Instant::now();
1660
1661            loop {
1662                if !ConnectionMode::from_atomic(&connection_state).is_active()
1663                    || !read_fence.is_valid()
1664                {
1665                    break;
1666                }
1667
1668                let read_result = dst::time::timeout(check_interval, reader.next()).await;
1669
1670                if let Ok(Some(Ok(ref message))) = read_result
1671                    && (!ConnectionMode::from_atomic(&connection_state).is_active()
1672                        || !read_fence.is_valid())
1673                {
1674                    log::debug!(
1675                        "Dropping WebSocket message with {} bytes after session ended",
1676                        message.as_bytes().len()
1677                    );
1678                    break;
1679                }
1680
1681                if matches!(&read_result, Ok(Some(Ok(_)))) {
1682                    last_frame_time = dst::time::Instant::now();
1683                }
1684
1685                match read_result {
1686                    Ok(Some(Ok(Message::Binary(data)))) => {
1687                        log::trace!("Received message <binary> {} bytes", data.len());
1688                        last_data_time = dst::time::Instant::now();
1689
1690                        if !ConnectionMode::from_atomic(&connection_state).is_active()
1691                            || !read_fence.is_valid()
1692                        {
1693                            log::debug!(
1694                                "Dropping WebSocket message with {} bytes after session ended",
1695                                data.len()
1696                            );
1697                            break;
1698                        }
1699
1700                        if let Some(ref handler) = handler {
1701                            handler.handle(connection_epoch, Message::Binary(data));
1702                        }
1703                    }
1704                    Ok(Some(Ok(Message::Text(data)))) => {
1705                        log::trace!("Received text frame ({} bytes)", data.len());
1706                        last_data_time = dst::time::Instant::now();
1707
1708                        if !ConnectionMode::from_atomic(&connection_state).is_active()
1709                            || !read_fence.is_valid()
1710                        {
1711                            log::debug!(
1712                                "Dropping WebSocket message with {} bytes after session ended",
1713                                data.len()
1714                            );
1715                            break;
1716                        }
1717
1718                        if let Some(ref handler) = handler {
1719                            handler.handle(connection_epoch, Message::Text(data));
1720                        }
1721                    }
1722                    Ok(Some(Ok(Message::Ping(ping_data)))) => {
1723                        log::trace!("Received ping frame ({} bytes)", ping_data.len());
1724                        // Do not reset last_data_time: pings are keep-alive frames, not application
1725                        // data, so a peer that emits only pings must still trip the idle timeout.
1726                        // Checked here too: a ping flood faster than the check interval starves the timeout branch
1727
1728                        if let Some(ref handler) = ping_handler {
1729                            if !ConnectionMode::from_atomic(&connection_state).is_active()
1730                                || !read_fence.is_valid()
1731                            {
1732                                log::debug!(
1733                                    "Dropping WebSocket ping with {} bytes after session ended",
1734                                    ping_data.len()
1735                                );
1736                                break;
1737                            }
1738
1739                            handler.handle(connection_epoch, ping_data.to_vec());
1740                        }
1741
1742                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1743                            break;
1744                        }
1745                    }
1746                    Ok(Some(Ok(Message::Pong(_)))) => {
1747                        log::trace!("Received pong");
1748                        // Do not reset last_data_time: pongs are keep-alive replies (not data)
1749
1750                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1751                            break;
1752                        }
1753                    }
1754                    Ok(Some(Ok(Message::Close(frame)))) => {
1755                        Self::dispatch_close_frame(
1756                            &connection_state,
1757                            &read_fence,
1758                            close_handler.as_ref(),
1759                            state_sink.as_ref(),
1760                            frame,
1761                        );
1762                        break;
1763                    }
1764                    Ok(Some(Err(e))) => {
1765                        if is_connection_drop_transport_error(&e) {
1766                            log::warn!("Received connection error, terminating: {e}");
1767                        } else {
1768                            log::error!("Received transport error, terminating: {e}");
1769                        }
1770
1771                        break;
1772                    }
1773                    Ok(None) => {
1774                        log::log!(
1775                            read_termination_log_level(&connection_state),
1776                            "Connection closed by peer (no close frame), terminating"
1777                        );
1778                        break;
1779                    }
1780                    Err(_) => {
1781                        if heartbeat_timeout_exceeded(
1782                            &mut last_frame_time,
1783                            heartbeat_timeout,
1784                            frame_activity.as_deref(),
1785                        ) {
1786                            break;
1787                        }
1788
1789                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1790                            break;
1791                        }
1792                    }
1793                }
1794            }
1795
1796            // Wake the controller immediately so it detects the dead read task
1797            state_notify.notify_one();
1798        })
1799    }
1800
1801    fn dispatch_close_frame(
1802        connection_state: &AtomicU8,
1803        read_fence: &ReadSessionFence,
1804        close_handler: Option<&CloseHandler>,
1805        state_sink: Option<&SocketStateSink>,
1806        frame: Option<CloseFrame>,
1807    ) {
1808        match &frame {
1809            Some(frame) => log::log!(
1810                read_termination_log_level(connection_state),
1811                "Received close frame, terminating: code={}, reason={REDACTED}",
1812                frame.code
1813            ),
1814            None => log::log!(
1815                read_termination_log_level(connection_state),
1816                "Received close frame with no code or reason, terminating"
1817            ),
1818        }
1819
1820        if !ConnectionMode::from_atomic(connection_state).is_active() || !read_fence.is_valid() {
1821            return;
1822        }
1823
1824        if let Some(handler) = close_handler
1825            && !handler(frame)
1826        {
1827            ConnectionMode::close_websocket_on_loss(connection_state, state_sink);
1828        }
1829    }
1830
1831    /// Queues a message for replay on the replacement connection.
1832    ///
1833    /// A keepalive belongs to the connection it was issued on: a Pong answers that
1834    /// connection's Ping, a heartbeat probes it, and a Close terminates it. None
1835    /// carries its meaning on a replacement connection, so those messages are dropped
1836    /// instead. Text keepalives reach the writer as [`WriterCommand::Heartbeat`] and
1837    /// never enter this buffer.
1838    fn buffer_for_replay(
1839        buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1840        msg: Message,
1841        permit: tokio::sync::OwnedSemaphorePermit,
1842    ) {
1843        if msg.is_control() {
1844            return;
1845        }
1846
1847        log::debug!(
1848            "Buffering message for replay (buffer size: {})",
1849            buffer.len() + 1
1850        );
1851
1852        buffer.push_back((msg, permit));
1853    }
1854
1855    /// Attempts to send all buffered messages after reconnection.
1856    ///
1857    /// Returns `true` if a send error occurred (caller should trigger reconnection).
1858    /// Messages remain in buffer if send fails, preserving them for the next reconnection attempt.
1859    async fn drain_reconnect_buffer(
1860        buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1861        writer: &mut MessageWriter,
1862        connection_state: &AtomicU8,
1863        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1864        reconnect_buffer_waits_for_auth: &AtomicBool,
1865    ) -> bool {
1866        if buffer.is_empty() {
1867            return false;
1868        }
1869
1870        let initial_buffer_len = buffer.len();
1871        log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
1872
1873        while !buffer.is_empty() {
1874            match Self::reconnect_buffer_action(
1875                reconnect_buffer_waits_for_auth,
1876                auth_tracker,
1877                connection_state,
1878            ) {
1879                ReconnectBufferAction::Drain => {}
1880                ReconnectBufferAction::Wait => return false,
1881                ReconnectBufferAction::Discard => {
1882                    log::warn!(
1883                        "Discarding {} buffered messages after authentication failed",
1884                        buffer.len()
1885                    );
1886                    buffer.clear();
1887                    return false;
1888                }
1889            }
1890
1891            // Clone message before attempting send (to keep in buffer if send fails)
1892            let msg_to_send = buffer
1893                .front()
1894                .expect("reconnect buffer should not be empty")
1895                .0
1896                .clone();
1897
1898            if let Err(e) = writer.send(msg_to_send).await {
1899                if is_connection_drop_transport_error(&e) {
1900                    log::warn!(
1901                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1902                        buffer.len()
1903                    );
1904                } else {
1905                    log::error!(
1906                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1907                        buffer.len()
1908                    );
1909                }
1910
1911                return true;
1912            }
1913
1914            // Only remove from buffer after successful send
1915            buffer.pop_front();
1916        }
1917
1918        if buffer.is_empty() {
1919            log::info!("Successfully sent all {initial_buffer_len} buffered messages");
1920        }
1921
1922        false
1923    }
1924
1925    fn can_drain_reconnect_buffer(
1926        reconnect_buffer_waits_for_auth: &AtomicBool,
1927        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1928    ) -> ReconnectBufferAction {
1929        if !reconnect_buffer_waits_for_auth.load(Ordering::Acquire) {
1930            return ReconnectBufferAction::Drain;
1931        }
1932
1933        match auth_tracker.get().map(AuthTracker::auth_state) {
1934            Some(AuthState::Authenticated) => ReconnectBufferAction::Drain,
1935            Some(AuthState::Failed) => ReconnectBufferAction::Discard,
1936            Some(AuthState::Unauthenticated) | None => ReconnectBufferAction::Wait,
1937        }
1938    }
1939
1940    fn reconnect_buffer_action(
1941        reconnect_buffer_waits_for_auth: &AtomicBool,
1942        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1943        connection_state: &AtomicU8,
1944    ) -> ReconnectBufferAction {
1945        let action =
1946            Self::can_drain_reconnect_buffer(reconnect_buffer_waits_for_auth, auth_tracker);
1947
1948        // Keep mode as the final admission check so an accepted reconnect stops the next send.
1949        if action == ReconnectBufferAction::Drain
1950            && !ConnectionMode::from_atomic(connection_state).is_active()
1951        {
1952            ReconnectBufferAction::Wait
1953        } else {
1954            action
1955        }
1956    }
1957
1958    #[expect(
1959        clippy::too_many_arguments,
1960        reason = "writer task owns the transport and shared lifecycle coordination state"
1961    )]
1962    fn spawn_write_task(
1963        connection_state: Arc<AtomicU8>,
1964        controller_notify: Arc<tokio::sync::Notify>,
1965        reconnect_published: Arc<AtomicBool>,
1966        writer: MessageWriter,
1967        mut writer_rx: WriterReceiver<WriterCommand>,
1968        connection_epoch: Arc<AtomicU64>,
1969        auth_tracker: Arc<OnceLock<AuthTracker>>,
1970        reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1971        state_sink: Option<SocketStateSink>,
1972    ) -> dst::task::JoinHandle<()> {
1973        log_task_started("write");
1974
1975        // Interval between checking the connection mode
1976        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1977
1978        dst::task::spawn(async move {
1979            let mut active_writer = writer;
1980            // Buffer for messages received during reconnection
1981            // VecDeque for efficient pop_front() operations
1982            let mut reconnect_buffer: VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)> =
1983                VecDeque::new();
1984
1985            loop {
1986                let mode = ConnectionMode::from_atomic(&connection_state);
1987
1988                match mode {
1989                    ConnectionMode::Disconnect => {
1990                        // Log any buffered messages that will be lost
1991                        if !reconnect_buffer.is_empty() {
1992                            log::warn!(
1993                                "Discarding {} buffered messages due to disconnect",
1994                                reconnect_buffer.len()
1995                            );
1996                            reconnect_buffer.clear();
1997                        }
1998
1999                        // Attempt to close the writer gracefully before exiting,
2000                        // we ignore any error as the writer may already be closed.
2001                        _ = dst::time::timeout(
2002                            Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2003                            active_writer.close(),
2004                        )
2005                        .await;
2006                        break;
2007                    }
2008                    ConnectionMode::Closed => {
2009                        // Log any buffered messages that will be lost
2010                        if !reconnect_buffer.is_empty() {
2011                            log::warn!(
2012                                "Discarding {} buffered messages due to closed connection",
2013                                reconnect_buffer.len()
2014                            );
2015                            reconnect_buffer.clear();
2016                        }
2017
2018                        break;
2019                    }
2020                    _ => {}
2021                }
2022
2023                if mode.is_active() && !reconnect_buffer.is_empty() {
2024                    match Self::reconnect_buffer_action(
2025                        reconnect_buffer_waits_for_auth.as_ref(),
2026                        &auth_tracker,
2027                        &connection_state,
2028                    ) {
2029                        ReconnectBufferAction::Drain => {
2030                            let drain_result = dst::time::timeout(
2031                                Duration::from_secs(WRITE_TIMEOUT_SECS),
2032                                Self::drain_reconnect_buffer(
2033                                    &mut reconnect_buffer,
2034                                    &mut active_writer,
2035                                    &connection_state,
2036                                    &auth_tracker,
2037                                    reconnect_buffer_waits_for_auth.as_ref(),
2038                                ),
2039                            )
2040                            .await;
2041
2042                            let send_error = drain_result.unwrap_or_else(|_| {
2043                                log::warn!(
2044                                    "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
2045                                    reconnect_buffer.len()
2046                                );
2047                                true
2048                            });
2049
2050                            // CAS: a disconnect landing mid-drain must not be overwritten
2051                            if send_error {
2052                                _ = request_websocket_reconnect(
2053                                    &connection_state,
2054                                    &reconnect_published,
2055                                    state_sink.as_ref(),
2056                                    &auth_tracker,
2057                                    &controller_notify,
2058                                    || {},
2059                                );
2060                            }
2061
2062                            continue;
2063                        }
2064                        ReconnectBufferAction::Discard => {
2065                            log::warn!(
2066                                "Discarding {} buffered messages after authentication failed",
2067                                reconnect_buffer.len()
2068                            );
2069                            reconnect_buffer.clear();
2070                            continue;
2071                        }
2072                        ReconnectBufferAction::Wait => {}
2073                    }
2074                }
2075
2076                match dst::time::timeout(check_interval, writer_rx.recv()).await {
2077                    Ok(Some((msg, permit))) => {
2078                        // Re-check connection mode after receiving a message
2079                        let mode = ConnectionMode::from_atomic(&connection_state);
2080
2081                        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
2082                            break;
2083                        }
2084
2085                        match msg {
2086                            WriterCommand::Update(new_writer, tx) => {
2087                                log::debug!("Received new writer");
2088
2089                                // Delay before closing connection
2090                                dst::time::sleep(Duration::from_millis(100)).await;
2091
2092                                // Attempt to close the writer gracefully on update,
2093                                // we ignore any error as the writer may already be closed.
2094                                _ = dst::time::timeout(
2095                                    Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2096                                    active_writer.close(),
2097                                )
2098                                .await;
2099
2100                                active_writer = new_writer;
2101                                let epoch = connection_epoch.fetch_add(1, Ordering::AcqRel) + 1;
2102                                log::debug!("Updated writer: epoch={epoch}");
2103
2104                                if let Err(e) = tx.send(epoch) {
2105                                    log::error!(
2106                                        "Failed to report writer update to controller: {e:?}"
2107                                    );
2108                                }
2109                            }
2110                            WriterCommand::Send(msg)
2111                                if mode.is_reconnect()
2112                                    || (!reconnect_buffer.is_empty() && !msg.is_control()) =>
2113                            {
2114                                Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2115                            }
2116                            WriterCommand::Heartbeat(_)
2117                            | WriterCommand::SendPongOnConnection { .. }
2118                                if mode.is_reconnect() => {}
2119                            WriterCommand::SendOnConnection { response_tx, .. }
2120                                if mode.is_reconnect() =>
2121                            {
2122                                _ = response_tx.send(Err(SendError::ConnectionChanged));
2123                            }
2124                            WriterCommand::SendPongOnConnection {
2125                                data,
2126                                connection_epoch: expected_epoch,
2127                            } => {
2128                                let epoch = connection_epoch.load(Ordering::Acquire);
2129                                if epoch != expected_epoch {
2130                                    continue;
2131                                }
2132
2133                                let send_result = dst::time::timeout(
2134                                    Duration::from_secs(WRITE_TIMEOUT_SECS),
2135                                    active_writer.send(Message::Pong(data.into())),
2136                                )
2137                                .await;
2138
2139                                let send_failed = match send_result {
2140                                    Ok(Ok(())) => false,
2141                                    Ok(Err(e)) => {
2142                                        if is_connection_drop_transport_error(&e) {
2143                                            log::warn!("Failed to send pong: {e}");
2144                                        } else {
2145                                            log::error!("Failed to send pong: {e}");
2146                                        }
2147
2148                                        true
2149                                    }
2150                                    Err(_) => {
2151                                        log::warn!(
2152                                            "Timed out sending pong after {WRITE_TIMEOUT_SECS}s"
2153                                        );
2154                                        true
2155                                    }
2156                                };
2157
2158                                if send_failed
2159                                    && request_websocket_reconnect(
2160                                        &connection_state,
2161                                        &reconnect_published,
2162                                        state_sink.as_ref(),
2163                                        &auth_tracker,
2164                                        &controller_notify,
2165                                        || {},
2166                                    ) == ReconnectRequestOutcome::Accepted
2167                                {
2168                                    log::warn!("Writer triggering reconnect");
2169                                }
2170                            }
2171                            WriterCommand::SendOnConnection {
2172                                message,
2173                                connection_epoch: expected_epoch,
2174                                response_tx,
2175                            } => {
2176                                let epoch = connection_epoch.load(Ordering::Acquire);
2177                                if epoch != expected_epoch {
2178                                    _ = response_tx.send(Err(SendError::ConnectionChanged));
2179                                    continue;
2180                                }
2181
2182                                let send_result = dst::time::timeout(
2183                                    Duration::from_secs(WRITE_TIMEOUT_SECS),
2184                                    active_writer.send(message),
2185                                )
2186                                .await;
2187
2188                                // An ownership-bound message is never replayed, so an expired
2189                                // deadline reports failure to the caller instead of buffering
2190                                // the message as the ordinary send path does.
2191                                let result = match send_result {
2192                                    Ok(Ok(())) => Ok(()),
2193                                    Ok(Err(e)) => {
2194                                        if is_connection_drop_transport_error(&e) {
2195                                            log::warn!("Failed to send message: {e}");
2196                                        } else {
2197                                            log::error!("Failed to send message: {e}");
2198                                        }
2199
2200                                        Err(SendError::BrokenPipe(e.to_string()))
2201                                    }
2202                                    Err(_) => {
2203                                        log::warn!(
2204                                            "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
2205                                        );
2206
2207                                        Err(SendError::WriteTimeout)
2208                                    }
2209                                };
2210
2211                                let send_failed = result.is_err();
2212                                _ = response_tx.send(result);
2213
2214                                if send_failed
2215                                    && request_websocket_reconnect(
2216                                        &connection_state,
2217                                        &reconnect_published,
2218                                        state_sink.as_ref(),
2219                                        &auth_tracker,
2220                                        &controller_notify,
2221                                        || {},
2222                                    ) == ReconnectRequestOutcome::Accepted
2223                                {
2224                                    log::warn!("Writer triggering reconnect");
2225                                }
2226                            }
2227                            WriterCommand::Send(msg) => {
2228                                let send_failed =
2229                                    Self::write_outbound(&mut active_writer, msg.clone()).await;
2230                                if send_failed {
2231                                    Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2232
2233                                    // CAS: a disconnect landing mid-send must not be overwritten
2234                                    if request_websocket_reconnect(
2235                                        &connection_state,
2236                                        &reconnect_published,
2237                                        state_sink.as_ref(),
2238                                        &auth_tracker,
2239                                        &controller_notify,
2240                                        || {},
2241                                    ) == ReconnectRequestOutcome::Accepted
2242                                    {
2243                                        log::warn!("Writer triggering reconnect");
2244                                    }
2245                                }
2246                            }
2247                            WriterCommand::Heartbeat(msg) => {
2248                                let send_failed =
2249                                    Self::write_outbound(&mut active_writer, msg).await;
2250
2251                                if send_failed
2252                                    && request_websocket_reconnect(
2253                                        &connection_state,
2254                                        &reconnect_published,
2255                                        state_sink.as_ref(),
2256                                        &auth_tracker,
2257                                        &controller_notify,
2258                                        || {},
2259                                    ) == ReconnectRequestOutcome::Accepted
2260                                {
2261                                    log::warn!("Writer triggering reconnect");
2262                                }
2263                            }
2264                        }
2265                    }
2266                    Ok(None) => {
2267                        // Channel closed - writer task should terminate
2268                        log::debug!("Writer channel closed, terminating writer task");
2269                        break;
2270                    }
2271                    Err(_) => {
2272                        // Timeout - just continue the loop
2273                    }
2274                }
2275            }
2276
2277            // Attempt to close the writer gracefully before exiting,
2278            // we ignore any error as the writer may already be closed.
2279            _ = dst::time::timeout(
2280                Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2281                active_writer.close(),
2282            )
2283            .await;
2284
2285            log_task_stopped("write");
2286        })
2287    }
2288
2289    async fn write_outbound(writer: &mut MessageWriter, msg: Message) -> bool {
2290        let send_result =
2291            dst::time::timeout(Duration::from_secs(WRITE_TIMEOUT_SECS), writer.send(msg)).await;
2292
2293        match send_result {
2294            Ok(Ok(())) => false,
2295            Ok(Err(e)) => {
2296                if is_connection_drop_transport_error(&e) {
2297                    log::warn!("Failed to send message: {e}");
2298                } else {
2299                    log::error!("Failed to send message: {e}");
2300                }
2301
2302                true
2303            }
2304            Err(_) => {
2305                log::warn!("Timed out sending message after {WRITE_TIMEOUT_SECS}s");
2306                true
2307            }
2308        }
2309    }
2310
2311    fn spawn_heartbeat_task(
2312        connection_state: Arc<AtomicU8>,
2313        heartbeat_secs: u64,
2314        message: Option<String>,
2315        writer_tx: WriterSender<WriterCommand>,
2316    ) -> dst::task::JoinHandle<()> {
2317        log_task_started("heartbeat");
2318
2319        dst::task::spawn(async move {
2320            let interval = Duration::from_secs(heartbeat_secs);
2321            loop {
2322                dst::time::sleep(interval).await;
2323
2324                match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
2325                    ConnectionMode::Active => {
2326                        let msg = match &message {
2327                            Some(text) => {
2328                                WriterCommand::Heartbeat(Message::Text(text.clone().into()))
2329                            }
2330                            None => WriterCommand::Heartbeat(Message::Ping(vec![].into())),
2331                        };
2332
2333                        match writer_tx.send_control(msg) {
2334                            Ok(()) => log::trace!("Sent heartbeat to writer task"),
2335                            Err(e) => {
2336                                log::warn!("Failed to send heartbeat to writer task: {e}");
2337                            }
2338                        }
2339                    }
2340                    ConnectionMode::Reconnect => {}
2341                    ConnectionMode::Disconnect | ConnectionMode::Closed => break,
2342                }
2343            }
2344
2345            log_task_stopped("heartbeat");
2346        })
2347    }
2348}
2349
2350fn heartbeat_timeout_exceeded(
2351    last_frame_time: &mut dst::time::Instant,
2352    timeout: Option<Duration>,
2353    frame_activity: Option<&AtomicBool>,
2354) -> bool {
2355    if frame_activity.is_some_and(|activity| activity.swap(false, Ordering::Relaxed)) {
2356        *last_frame_time = dst::time::Instant::now();
2357    }
2358
2359    if let Some(timeout) = timeout {
2360        let elapsed = last_frame_time.elapsed();
2361        if elapsed >= timeout {
2362            log::warn!(
2363                "Heartbeat timeout: no frame received for {:.1}s",
2364                elapsed.as_secs_f64()
2365            );
2366            return true;
2367        }
2368    }
2369
2370    false
2371}
2372
2373fn idle_timeout_exceeded(
2374    last_data_time: dst::time::Instant,
2375    idle_timeout: Option<Duration>,
2376) -> bool {
2377    if let Some(timeout) = idle_timeout {
2378        let idle_duration = last_data_time.elapsed();
2379        if idle_duration >= timeout {
2380            log::warn!(
2381                "Read idle timeout: no data received for {:.1}s",
2382                idle_duration.as_secs_f64()
2383            );
2384            return true;
2385        }
2386    }
2387
2388    false
2389}
2390
2391impl Drop for WebSocketClientInner {
2392    fn drop(&mut self) {
2393        if let Some(read_fence) = self.read_fence.take() {
2394            read_fence.invalidate();
2395        }
2396
2397        if let Some(ref read_task) = self.read_task.take()
2398            && !read_task.is_finished()
2399        {
2400            read_task.abort();
2401            log_task_aborted("read");
2402        }
2403
2404        if !self.write_task.is_finished() {
2405            self.write_task.abort();
2406            log_task_aborted("write");
2407        }
2408
2409        if let Some(ref handle) = self.heartbeat_task.take()
2410            && !handle.is_finished()
2411        {
2412            handle.abort();
2413            log_task_aborted("heartbeat");
2414        }
2415    }
2416}
2417
2418#[expect(
2419    clippy::missing_fields_in_debug,
2420    reason = "handler closures and internal task handles are intentionally omitted"
2421)]
2422impl Debug for WebSocketClientInner {
2423    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2424        f.debug_struct(stringify!(WebSocketClientInner))
2425            .field("config", &self.config)
2426            .field(
2427                "connection_mode",
2428                &ConnectionMode::from_atomic(&self.connection_mode),
2429            )
2430            .field("connect_timeout", &self.connect_timeout)
2431            .field("is_stream_mode", &self.handler.is_none())
2432            .finish()
2433    }
2434}
2435
2436#[derive(Clone)]
2437enum IncomingHandler {
2438    Message(MessageHandler),
2439    Epoch(EpochMessageHandler),
2440}
2441
2442impl IncomingHandler {
2443    fn handle(&self, connection_epoch: u64, message: Message) {
2444        match self {
2445            Self::Message(handler) => handler(message),
2446            Self::Epoch(handler) => handler(connection_epoch, message),
2447        }
2448    }
2449}
2450
2451#[derive(Clone)]
2452enum IncomingPingHandler {
2453    Ping(PingHandler),
2454    Epoch(EpochPingHandler),
2455}
2456
2457impl IncomingPingHandler {
2458    fn handle(&self, connection_epoch: u64, data: Vec<u8>) {
2459        match self {
2460            Self::Ping(handler) => handler(data),
2461            Self::Epoch(handler) => handler(connection_epoch, data),
2462        }
2463    }
2464}
2465
2466#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2467enum ReconnectBufferAction {
2468    Drain,
2469    Wait,
2470    Discard,
2471}
2472
2473/// A WebSocket client with rate limiting, heartbeats, and automatic reconnection in handler mode.
2474///
2475/// Handler mode owns the reader and writer tasks, buffers sends during reconnection, and replays
2476/// them against the replacement connection. Stream mode returns the reader to the caller and does
2477/// not reconnect automatically. See [`crate::websocket`] for connection ownership, replay, and
2478/// epoch guarantees.
2479pub struct WebSocketClient {
2480    pub(crate) controller_task: dst::task::JoinHandle<()>,
2481    pub(crate) connection_mode: Arc<AtomicU8>,
2482    pub(crate) connection_epoch: Arc<AtomicU64>,
2483    pub(crate) state_notify: Arc<tokio::sync::Notify>,
2484    pub(crate) connect_timeout: Duration,
2485    pub(crate) rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
2486    pub(crate) writer_tx: WriterSender<WriterCommand>,
2487    auth_tracker: Arc<OnceLock<AuthTracker>>,
2488    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
2489    reconnect_headers: ReconnectHeaders,
2490    state_sink: Option<SocketStateSink>,
2491    controller_lifecycle: Arc<ControllerLifecycle>,
2492    controller_notify: Arc<tokio::sync::Notify>,
2493    reconnect_published: Arc<AtomicBool>,
2494    reconnect_supported: bool,
2495}
2496
2497/// Shared headers used by future automatic WebSocket reconnects.
2498///
2499/// Updating these headers does not affect the active connection or trigger a reconnect.
2500#[derive(Clone)]
2501pub struct ReconnectHeaders {
2502    inner: Arc<RwLock<Vec<(String, String)>>>,
2503}
2504
2505impl ReconnectHeaders {
2506    fn new(headers: Vec<(String, String)>) -> Self {
2507        Self {
2508            inner: Arc::new(RwLock::new(headers)),
2509        }
2510    }
2511
2512    /// Replaces a header used by future automatic reconnect attempts.
2513    ///
2514    /// # Errors
2515    ///
2516    /// Returns an error if the header name or value is invalid.
2517    pub fn update(&self, name: &str, value: &str) -> Result<(), TransportError> {
2518        let name = HeaderName::from_bytes(name.as_bytes()).map_err(|e| {
2519            TransportError::Io(std::io::Error::new(
2520                std::io::ErrorKind::InvalidInput,
2521                format!("Invalid WebSocket reconnect header name: {e}"),
2522            ))
2523        })?;
2524
2525        HeaderValue::from_str(value).map_err(|e| {
2526            TransportError::Io(std::io::Error::new(
2527                std::io::ErrorKind::InvalidInput,
2528                format!("Invalid WebSocket reconnect header value: {e}"),
2529            ))
2530        })?;
2531
2532        let name = name.as_str();
2533        let mut headers = self.inner.write();
2534        headers.retain(|(existing, _)| !existing.eq_ignore_ascii_case(name));
2535        headers.push((name.to_string(), value.to_string()));
2536        Ok(())
2537    }
2538
2539    fn snapshot(&self) -> Vec<(String, String)> {
2540        self.inner.read().clone()
2541    }
2542}
2543
2544impl Debug for ReconnectHeaders {
2545    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2546        f.debug_struct(stringify!(ReconnectHeaders))
2547            .finish_non_exhaustive()
2548    }
2549}
2550
2551/// Cloneable controller handle for requesting one transport reconnect.
2552#[derive(Clone)]
2553pub struct WebSocketReconnectHandle {
2554    connection_mode: Arc<AtomicU8>,
2555    auth_tracker: Arc<OnceLock<AuthTracker>>,
2556    state_sink: Option<SocketStateSink>,
2557    controller_lifecycle: Arc<ControllerLifecycle>,
2558    controller_notify: Arc<tokio::sync::Notify>,
2559    reconnect_published: Arc<AtomicBool>,
2560    supported: bool,
2561}
2562
2563impl Debug for WebSocketReconnectHandle {
2564    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2565        f.debug_struct(stringify!(WebSocketReconnectHandle))
2566            .field(
2567                "connection_mode",
2568                &ConnectionMode::from_atomic(&self.connection_mode),
2569            )
2570            .field("supported", &self.supported)
2571            .finish_non_exhaustive()
2572    }
2573}
2574
2575impl WebSocketReconnectHandle {
2576    /// Requests that the controller replace the active transport.
2577    ///
2578    /// An accepted request invalidates registered authentication state, wakes the controller, and
2579    /// publishes the loss before the replacement can become active. Rejected requests leave the
2580    /// transport and authentication state unchanged. Handler-mode handles return
2581    /// [`ReconnectRequestOutcome::Closed`] after the client drops; stream-mode handles always
2582    /// return [`ReconnectRequestOutcome::Unsupported`].
2583    #[must_use]
2584    pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
2585        if !self.supported {
2586            return ReconnectRequestOutcome::Unsupported;
2587        }
2588
2589        let Some(request) = self.controller_lifecycle.enter_request() else {
2590            return ReconnectRequestOutcome::Closed;
2591        };
2592
2593        let mut request = Some(request);
2594        request_websocket_reconnect(
2595            &self.connection_mode,
2596            &self.reconnect_published,
2597            self.state_sink.as_ref(),
2598            &self.auth_tracker,
2599            &self.controller_notify,
2600            || drop(request.take()),
2601        )
2602    }
2603}
2604
2605struct ReconnectPublication<'a> {
2606    published: &'a AtomicBool,
2607    controller_notify: &'a tokio::sync::Notify,
2608}
2609
2610impl Drop for ReconnectPublication<'_> {
2611    fn drop(&mut self) {
2612        self.published.store(true, Ordering::SeqCst);
2613        self.controller_notify.notify_one();
2614    }
2615}
2616
2617fn request_websocket_reconnect<F>(
2618    connection_mode: &AtomicU8,
2619    reconnect_published: &AtomicBool,
2620    state_sink: Option<&SocketStateSink>,
2621    auth_tracker: &OnceLock<AuthTracker>,
2622    controller_notify: &tokio::sync::Notify,
2623    on_handoff: F,
2624) -> ReconnectRequestOutcome
2625where
2626    F: FnOnce(),
2627{
2628    if reconnect_published
2629        .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst)
2630        .is_err()
2631    {
2632        return match ConnectionMode::from_atomic(connection_mode) {
2633            ConnectionMode::Active | ConnectionMode::Reconnect => {
2634                ReconnectRequestOutcome::AlreadyReconnecting
2635            }
2636            ConnectionMode::Disconnect => ReconnectRequestOutcome::Disconnected,
2637            ConnectionMode::Closed => ReconnectRequestOutcome::Closed,
2638        };
2639    }
2640
2641    let outcome = ConnectionMode::request_reconnect_outcome(connection_mode);
2642    if outcome != ReconnectRequestOutcome::Accepted {
2643        reconnect_published.store(true, Ordering::SeqCst);
2644        return outcome;
2645    }
2646
2647    let _publication = ReconnectPublication {
2648        published: reconnect_published,
2649        controller_notify,
2650    };
2651
2652    if let Some(tracker) = auth_tracker.get() {
2653        tracker.invalidate();
2654    }
2655
2656    controller_notify.notify_one();
2657    on_handoff();
2658
2659    if let Some(sink) = state_sink {
2660        sink.publish_websocket(SocketState::Disconnected);
2661    }
2662
2663    ReconnectRequestOutcome::Accepted
2664}
2665
2666#[cfg(test)]
2667mod reconnect_request_tests {
2668    use std::sync::{Arc, OnceLock, atomic::AtomicU8};
2669
2670    use rstest::rstest;
2671
2672    use super::*;
2673
2674    fn handle(
2675        mode: ConnectionMode,
2676        supported: bool,
2677    ) -> (
2678        WebSocketReconnectHandle,
2679        AuthTracker,
2680        Arc<tokio::sync::Notify>,
2681    ) {
2682        let tracker = AuthTracker::new();
2683        let _receiver = tracker.begin();
2684        tracker.succeed();
2685        let auth_tracker = Arc::new(OnceLock::new());
2686        auth_tracker
2687            .set(tracker.clone())
2688            .expect("auth tracker should be unset");
2689        let notify = Arc::new(tokio::sync::Notify::new());
2690
2691        let handle = WebSocketReconnectHandle {
2692            connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
2693            auth_tracker,
2694            state_sink: None,
2695            controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2696            controller_notify: Arc::clone(&notify),
2697            reconnect_published: Arc::new(AtomicBool::new(true)),
2698            supported,
2699        };
2700
2701        (handle, tracker, notify)
2702    }
2703
2704    #[rstest]
2705    #[tokio::test]
2706    async fn accepted_request_invalidates_auth_and_wakes_controller_once() {
2707        let (handle, tracker, notify) = handle(ConnectionMode::Active, true);
2708
2709        assert_eq!(
2710            handle.request_reconnect(),
2711            ReconnectRequestOutcome::Accepted
2712        );
2713        assert!(!tracker.is_authenticated());
2714        tokio::time::timeout(Duration::from_millis(10), notify.notified())
2715            .await
2716            .expect("accepted request should notify controller");
2717
2718        let _receiver = tracker.begin();
2719        tracker.succeed();
2720        assert_eq!(
2721            handle.request_reconnect(),
2722            ReconnectRequestOutcome::AlreadyReconnecting
2723        );
2724        assert!(tracker.is_authenticated());
2725        assert!(
2726            tokio::time::timeout(Duration::from_millis(10), notify.notified())
2727                .await
2728                .is_err(),
2729            "duplicate request should not notify controller",
2730        );
2731
2732        handle
2733            .connection_mode
2734            .store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
2735        assert_eq!(
2736            handle.request_reconnect(),
2737            ReconnectRequestOutcome::Accepted
2738        );
2739        assert!(!tracker.is_authenticated());
2740        tokio::time::timeout(Duration::from_millis(10), notify.notified())
2741            .await
2742            .expect("a later reconnect cycle should notify the controller");
2743    }
2744
2745    #[rstest]
2746    #[case(
2747        ConnectionMode::Disconnect,
2748        true,
2749        ReconnectRequestOutcome::Disconnected
2750    )]
2751    #[case(
2752        ConnectionMode::Reconnect,
2753        true,
2754        ReconnectRequestOutcome::AlreadyReconnecting
2755    )]
2756    #[case(ConnectionMode::Closed, true, ReconnectRequestOutcome::Closed)]
2757    #[case(ConnectionMode::Active, false, ReconnectRequestOutcome::Unsupported)]
2758    #[tokio::test]
2759    async fn rejected_request_preserves_auth_and_does_not_wake_controller(
2760        #[case] mode: ConnectionMode,
2761        #[case] supported: bool,
2762        #[case] expected: ReconnectRequestOutcome,
2763    ) {
2764        let (handle, tracker, notify) = handle(mode, supported);
2765
2766        assert_eq!(handle.request_reconnect(), expected);
2767        assert!(tracker.is_authenticated());
2768        assert!(
2769            tokio::time::timeout(Duration::from_millis(10), notify.notified())
2770                .await
2771                .is_err(),
2772            "rejected request should not notify controller",
2773        );
2774    }
2775
2776    #[rstest]
2777    #[tokio::test]
2778    async fn reconnect_loss_callback_rejects_nested_request() {
2779        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2780        let controller_notify = Arc::new(tokio::sync::Notify::new());
2781        let handle_slot = Arc::new(OnceLock::<WebSocketReconnectHandle>::new());
2782        let handle_slot_callback = Arc::clone(&handle_slot);
2783        let nested_outcomes = Arc::new(parking_lot::Mutex::new(Vec::new()));
2784        let nested_outcomes_callback = Arc::clone(&nested_outcomes);
2785        let states = Arc::new(parking_lot::Mutex::new(Vec::new()));
2786        let states_callback = Arc::clone(&states);
2787        let sink = SocketStateSink::new(move |state| {
2788            states_callback.lock().push(state);
2789            nested_outcomes_callback
2790                .lock()
2791                .push(handle_slot_callback.get().unwrap().request_reconnect());
2792        });
2793
2794        let handle = WebSocketReconnectHandle {
2795            connection_mode,
2796            auth_tracker: Arc::new(OnceLock::new()),
2797            state_sink: Some(sink),
2798            controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2799            controller_notify,
2800            reconnect_published: Arc::new(AtomicBool::new(true)),
2801            supported: true,
2802        };
2803
2804        handle_slot.set(handle.clone()).unwrap();
2805        let (result_tx, result_rx) = tokio::sync::oneshot::channel();
2806        std::thread::spawn(move || {
2807            _ = result_tx.send(handle.request_reconnect());
2808        });
2809
2810        assert_eq!(
2811            tokio::time::timeout(Duration::from_secs(1), result_rx)
2812                .await
2813                .expect("reentrant reconnect callback deadlocked")
2814                .unwrap(),
2815            ReconnectRequestOutcome::Accepted
2816        );
2817        assert_eq!(
2818            *nested_outcomes.lock(),
2819            vec![ReconnectRequestOutcome::AlreadyReconnecting]
2820        );
2821        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2822    }
2823
2824    #[rstest]
2825    fn closed_stream_handle_remains_unsupported() {
2826        let (handle, tracker, _notify) = handle(ConnectionMode::Closed, false);
2827        handle.controller_lifecycle.close_and_abort();
2828
2829        assert_eq!(
2830            handle.request_reconnect(),
2831            ReconnectRequestOutcome::Unsupported
2832        );
2833        assert!(tracker.is_authenticated());
2834    }
2835}
2836
2837impl Debug for WebSocketClient {
2838    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2839        f.debug_struct(stringify!(WebSocketClient)).finish()
2840    }
2841}
2842
2843#[bon::bon]
2844impl WebSocketClient {
2845    /// Returns a builder for a websocket client in **stream mode**.
2846    ///
2847    /// Calling `connect` returns a stream that the caller owns and reads from directly. Automatic
2848    /// reconnection is **disabled** because the reader cannot be replaced internally. On
2849    /// disconnection, the client transitions to CLOSED state and the caller must manually create a
2850    /// new connection.
2851    ///
2852    /// Use stream mode when you need custom reconnection logic, direct control over message
2853    /// reading, or fine-grained backpressure handling.
2854    ///
2855    /// `default_quota` and `keyed_quotas` limit outgoing messages. `state_sink` reports transport
2856    /// availability changes.
2857    ///
2858    /// See [`WebSocketConfig`] documentation for comparison with handler mode.
2859    ///
2860    /// # Errors
2861    ///
2862    /// Returns an error if the connection cannot be established.
2863    #[builder(
2864        builder_type = WebSocketClientStreamBuilder,
2865        finish_fn = connect
2866    )]
2867    pub async fn stream_builder(
2868        config: WebSocketConfig,
2869        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2870        default_quota: Option<Quota>,
2871        state_sink: Option<SocketStateSink>,
2872    ) -> Result<(MessageReader, Self), TransportError> {
2873        install_cryptographic_provider();
2874        validate_honored_inbound_limits(&config)?;
2875
2876        // Create a single connection and split it, respecting configured headers.
2877        // The connection attempt bound is a fixed default: stream mode documents reconnect_* fields as ignored
2878        let connect_timeout = Duration::from_secs(10);
2879
2880        let (writer, reader) = dst::time::timeout(
2881            connect_timeout,
2882            Box::pin(WebSocketClientInner::connect_with_server(
2883                &config.url,
2884                config.headers.clone(),
2885                config.backend,
2886                config.proxy_url.as_deref(),
2887                config.max_message_size_bytes,
2888                config.max_frame_size_bytes,
2889            )),
2890        )
2891        .await
2892        .map_err(|_| {
2893            TransportError::Io(std::io::Error::new(
2894                std::io::ErrorKind::TimedOut,
2895                format!(
2896                    "connection timed out after {}s",
2897                    connect_timeout.as_secs_f64()
2898                ),
2899            ))
2900        })??;
2901
2902        // Create inner without connecting (we'll provide the writer)
2903        let inner =
2904            WebSocketClientInner::new_with_writer_and_state_sink(config, writer, state_sink)?;
2905
2906        let connection_mode = Arc::clone(&inner.connection_mode);
2907        let connection_epoch = Arc::clone(&inner.connection_epoch);
2908        let state_notify = Arc::clone(&inner.state_notify);
2909        let controller_notify = Arc::clone(&inner.controller_notify);
2910        let reconnect_published = Arc::clone(&inner.reconnect_published);
2911        let connect_timeout = inner.connect_timeout;
2912        let auth_tracker = Arc::clone(&inner.auth_tracker);
2913        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
2914        let reconnect_headers = inner.reconnect_headers.clone();
2915        let state_sink = inner.state_sink.clone();
2916        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, None)?;
2917        let writer_tx = inner.writer_tx.clone();
2918        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2919
2920        let controller_task = Self::spawn_controller_task(
2921            inner,
2922            Arc::clone(&connection_mode),
2923            Arc::clone(&state_notify),
2924            Arc::clone(&auth_tracker),
2925            Arc::clone(&controller_lifecycle),
2926            Arc::clone(&controller_notify),
2927            Arc::clone(&reconnect_published),
2928        );
2929        let abort_handle = controller_task.abort_handle();
2930        controller_lifecycle.set_abort(move || abort_handle.abort());
2931
2932        Ok((
2933            reader,
2934            Self {
2935                controller_task,
2936                connection_mode,
2937                connection_epoch,
2938                state_notify,
2939                connect_timeout,
2940                rate_limiter,
2941                writer_tx,
2942                auth_tracker,
2943                reconnect_buffer_waits_for_auth,
2944                reconnect_headers,
2945                state_sink,
2946                controller_lifecycle,
2947                controller_notify,
2948                reconnect_published,
2949                reconnect_supported: false,
2950            },
2951        ))
2952    }
2953
2954    /// Returns a builder for a websocket client in **handler mode**.
2955    ///
2956    /// The handler is called for each incoming message on an internal task.
2957    /// Automatic reconnection is **enabled** with exponential backoff. On disconnection,
2958    /// the client automatically attempts to reconnect and replaces the internal reader
2959    /// (the handler continues working seamlessly).
2960    ///
2961    /// Use handler mode for simplified connection management, automatic reconnection, or
2962    /// callback-based message handling.
2963    ///
2964    /// See [`WebSocketConfig`] documentation for comparison with stream mode.
2965    ///
2966    /// Set `rate_limiter` to share message quota state across clients. Otherwise, the client
2967    /// creates one from `default_quota` and `keyed_quotas`. `connection_rate_limiter` gates the
2968    /// initial connection and reconnects using `connection_rate_keys`.
2969    ///
2970    /// Without `initial_connect_retry_policy` the builder makes exactly one connection attempt.
2971    /// With one, failures classified as retryable are retried up to its `max_attempts`; see
2972    /// [`InitialConnectRetryPolicy`] for which failures return before that bound is reached.
2973    ///
2974    /// `cancellation_token` aborts the initial connection only, and is observed during the
2975    /// connection rate-limit wait, the dial itself, and each backoff delay. It has no effect once
2976    /// this function returns a client: use [`WebSocketClient::disconnect`] to stop an established one, whose
2977    /// reconnect loop the token does not govern.
2978    ///
2979    /// Set `close_handler` to observe peer close frames and decide whether automatic reconnection
2980    /// continues. Close frames are not delivered to the message handler.
2981    ///
2982    /// The message handler is required:
2983    ///
2984    /// ```compile_fail
2985    /// use nautilus_network::websocket::{WebSocketClient, WebSocketConfig};
2986    ///
2987    /// let config: WebSocketConfig = unimplemented!();
2988    /// let _ = WebSocketClient::builder().config(config).connect();
2989    /// ```
2990    ///
2991    /// # Errors
2992    ///
2993    /// Returns an error if:
2994    /// - The configuration is invalid or the connection cannot be established.
2995    /// - A shared rate limiter is combined with quota configuration.
2996    /// - The connection rate limiter and its keys are not configured together.
2997    #[builder(finish_fn = connect)]
2998    pub async fn builder(
2999        config: WebSocketConfig,
3000        message_handler: MessageHandler,
3001        ping_handler: Option<PingHandler>,
3002        close_handler: Option<CloseHandler>,
3003        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3004        default_quota: Option<Quota>,
3005        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3006        state_sink: Option<SocketStateSink>,
3007        connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3008        #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3009        initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3010        cancellation_token: Option<CancellationToken>,
3011    ) -> Result<Self, TransportError> {
3012        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3013        let connection_rate_limit =
3014            Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3015        Self::connect_with_handler_scoped(
3016            config,
3017            IncomingHandler::Message(message_handler),
3018            ping_handler.map(IncomingPingHandler::Ping),
3019            close_handler,
3020            rate_limiter,
3021            state_sink,
3022            connection_rate_limit,
3023            InitialConnectOptions {
3024                retry_policy: initial_connect_retry_policy,
3025                cancellation_token,
3026            },
3027        )
3028        .await
3029    }
3030
3031    /// Returns a builder for a handler-mode client whose messages carry connection ownership.
3032    ///
3033    /// The initial connection has epoch `0`. Each replacement connection increments the epoch,
3034    /// and both its incoming messages and `RECONNECTED` notification carry that new value. Use
3035    /// [`Self::send_text_on_connection`] or [`Self::send_bytes_on_connection`] to bind an
3036    /// outgoing message to one of those epochs.
3037    /// Rate-limit, state, initial-connect retry, and cancellation options match [`Self::builder`].
3038    /// Set either `ping_handler` or `epoch_ping_handler` when custom ping handling is required.
3039    /// `close_handler` has the same behavior as [`Self::builder`].
3040    ///
3041    /// The epoch handler is required:
3042    ///
3043    /// ```compile_fail
3044    /// use nautilus_network::websocket::{WebSocketClient, WebSocketConfig};
3045    ///
3046    /// let config: WebSocketConfig = unimplemented!();
3047    /// let _ = WebSocketClient::epoch_builder().config(config).connect();
3048    /// ```
3049    ///
3050    /// # Errors
3051    ///
3052    /// Returns an error if:
3053    /// - The configuration is invalid or the connection cannot be established.
3054    /// - A shared rate limiter is combined with quota configuration.
3055    /// - The connection rate limiter and its keys are not configured together.
3056    /// - Both `ping_handler` and `epoch_ping_handler` are configured.
3057    #[builder(
3058        builder_type = WebSocketClientEpochBuilder,
3059        finish_fn = connect
3060    )]
3061    pub async fn epoch_builder(
3062        config: WebSocketConfig,
3063        epoch_handler: EpochMessageHandler,
3064        ping_handler: Option<PingHandler>,
3065        epoch_ping_handler: Option<EpochPingHandler>,
3066        close_handler: Option<CloseHandler>,
3067        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3068        default_quota: Option<Quota>,
3069        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3070        state_sink: Option<SocketStateSink>,
3071        connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3072        #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3073        initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3074        cancellation_token: Option<CancellationToken>,
3075    ) -> Result<Self, TransportError> {
3076        let ping_handler = match (ping_handler, epoch_ping_handler) {
3077            (Some(_), Some(_)) => {
3078                return Err(TransportError::Io(std::io::Error::new(
3079                    std::io::ErrorKind::InvalidInput,
3080                    "Cannot configure both ping_handler and epoch_ping_handler",
3081                )));
3082            }
3083            (Some(handler), None) => Some(IncomingPingHandler::Ping(handler)),
3084            (None, Some(handler)) => Some(IncomingPingHandler::Epoch(handler)),
3085            (None, None) => None,
3086        };
3087
3088        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3089        let connection_rate_limit =
3090            Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3091        Self::connect_with_handler_scoped(
3092            config,
3093            IncomingHandler::Epoch(epoch_handler),
3094            ping_handler,
3095            close_handler,
3096            rate_limiter,
3097            state_sink,
3098            connection_rate_limit,
3099            InitialConnectOptions {
3100                retry_policy: initial_connect_retry_policy,
3101                cancellation_token,
3102            },
3103        )
3104        .await
3105    }
3106
3107    fn resolve_connection_rate_limit(
3108        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3109        keys: Arc<[Ustr]>,
3110    ) -> Result<Option<ConnectionRateLimit>, TransportError> {
3111        if rate_limiter.is_none() && !keys.is_empty() {
3112            return Err(TransportError::Io(std::io::Error::new(
3113                std::io::ErrorKind::InvalidInput,
3114                "Connection rate keys require a connection rate limiter",
3115            )));
3116        }
3117
3118        if rate_limiter.is_some() && keys.is_empty() {
3119            return Err(TransportError::Io(std::io::Error::new(
3120                std::io::ErrorKind::InvalidInput,
3121                "Connection rate limiter requires at least one connection rate key",
3122            )));
3123        }
3124
3125        Ok(rate_limiter.map(|limiter| ConnectionRateLimit { limiter, keys }))
3126    }
3127
3128    fn resolve_rate_limiter(
3129        default_quota: Option<Quota>,
3130        keyed_quotas: Vec<(String, Quota)>,
3131        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3132    ) -> Result<Arc<RateLimiter<Ustr, MonotonicClock>>, TransportError> {
3133        if let Some(rate_limiter) = rate_limiter {
3134            if default_quota.is_some() || !keyed_quotas.is_empty() {
3135                return Err(TransportError::Io(std::io::Error::new(
3136                    std::io::ErrorKind::InvalidInput,
3137                    "Cannot combine a shared rate limiter with quota configuration",
3138                )));
3139            }
3140
3141            return Ok(rate_limiter);
3142        }
3143
3144        let keyed_quotas = keyed_quotas
3145            .into_iter()
3146            .map(|(key, quota)| (Ustr::from(&key), quota))
3147            .collect();
3148        Ok(Arc::new(RateLimiter::new_with_quota(
3149            default_quota,
3150            keyed_quotas,
3151        )))
3152    }
3153
3154    #[expect(
3155        clippy::too_many_arguments,
3156        reason = "handler delivery and connection controls are independent options"
3157    )]
3158    async fn connect_with_handler_scoped(
3159        config: WebSocketConfig,
3160        handler: IncomingHandler,
3161        ping_handler: Option<IncomingPingHandler>,
3162        close_handler: Option<CloseHandler>,
3163        rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
3164        state_sink: Option<SocketStateSink>,
3165        connection_rate_limit: Option<ConnectionRateLimit>,
3166        initial_connect_options: InitialConnectOptions,
3167    ) -> Result<Self, TransportError> {
3168        log::debug!("Connecting");
3169        let inner = WebSocketClientInner::connect_url_with_handler(
3170            config,
3171            Some(handler),
3172            ping_handler,
3173            close_handler,
3174            state_sink,
3175            connection_rate_limit,
3176            initial_connect_options,
3177        )
3178        .await?;
3179        let connection_mode = Arc::clone(&inner.connection_mode);
3180        let connection_epoch = Arc::clone(&inner.connection_epoch);
3181        let state_notify = Arc::clone(&inner.state_notify);
3182        let controller_notify = Arc::clone(&inner.controller_notify);
3183        let reconnect_published = Arc::clone(&inner.reconnect_published);
3184        let writer_tx = inner.writer_tx.clone();
3185        let connect_timeout = inner.connect_timeout;
3186        let auth_tracker = Arc::clone(&inner.auth_tracker);
3187        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
3188        let reconnect_headers = inner.reconnect_headers.clone();
3189        let state_sink = inner.state_sink.clone();
3190        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
3191
3192        let controller_task = Self::spawn_controller_task(
3193            inner,
3194            Arc::clone(&connection_mode),
3195            Arc::clone(&state_notify),
3196            Arc::clone(&auth_tracker),
3197            Arc::clone(&controller_lifecycle),
3198            Arc::clone(&controller_notify),
3199            Arc::clone(&reconnect_published),
3200        );
3201        let abort_handle = controller_task.abort_handle();
3202        controller_lifecycle.set_abort(move || abort_handle.abort());
3203
3204        Ok(Self {
3205            controller_task,
3206            connection_mode,
3207            connection_epoch,
3208            state_notify,
3209            connect_timeout,
3210            rate_limiter,
3211            writer_tx,
3212            auth_tracker,
3213            reconnect_buffer_waits_for_auth,
3214            reconnect_headers,
3215            state_sink,
3216            controller_lifecycle,
3217            controller_notify,
3218            reconnect_published,
3219            reconnect_supported: true,
3220        })
3221    }
3222
3223    /// Returns shared headers used by future automatic reconnect attempts.
3224    #[must_use]
3225    pub fn reconnect_headers(&self) -> ReconnectHeaders {
3226        self.reconnect_headers.clone()
3227    }
3228
3229    /// Returns a cloneable handle to this client's reconnect controller.
3230    #[must_use]
3231    pub fn reconnect_handle(&self) -> WebSocketReconnectHandle {
3232        WebSocketReconnectHandle {
3233            connection_mode: Arc::clone(&self.connection_mode),
3234            auth_tracker: Arc::clone(&self.auth_tracker),
3235            state_sink: self.state_sink.clone(),
3236            controller_lifecycle: Arc::clone(&self.controller_lifecycle),
3237            controller_notify: Arc::clone(&self.controller_notify),
3238            reconnect_published: Arc::clone(&self.reconnect_published),
3239            supported: self.reconnect_supported,
3240        }
3241    }
3242
3243    /// Requests that the controller replace the active transport.
3244    ///
3245    /// Returns `true` only when this call transitions a handler-mode client from active to
3246    /// reconnecting. Stream-mode, duplicate, disconnecting, and closed requests return `false`.
3247    #[must_use]
3248    pub fn request_reconnect(&self) -> bool {
3249        self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
3250    }
3251
3252    /// Returns the current connection mode.
3253    #[must_use]
3254    pub fn connection_mode(&self) -> ConnectionMode {
3255        ConnectionMode::from_atomic(&self.connection_mode)
3256    }
3257
3258    /// Returns the ownership epoch of the current connection.
3259    ///
3260    /// A connection keeps one epoch for its lifetime. The value increments when the writer swaps
3261    /// to a replacement connection.
3262    #[must_use]
3263    pub fn connection_epoch(&self) -> u64 {
3264        self.connection_epoch.load(Ordering::Acquire)
3265    }
3266
3267    /// Returns a clone of the connection mode atomic for external state tracking.
3268    ///
3269    /// This allows adapter clients to track connection state across reconnections
3270    /// without message-passing delays.
3271    #[must_use]
3272    pub fn connection_mode_atomic(&self) -> Arc<AtomicU8> {
3273        Arc::clone(&self.connection_mode)
3274    }
3275
3276    /// Returns shared read access to the current connection epoch.
3277    ///
3278    /// Callers must treat the returned atomic as read-only. The WebSocket writer task exclusively
3279    /// advances it when installing a replacement connection.
3280    #[must_use]
3281    pub fn connection_epoch_atomic(&self) -> Arc<AtomicU64> {
3282        Arc::clone(&self.connection_epoch)
3283    }
3284
3285    /// Returns whether the client connection is active.
3286    ///
3287    /// Returns `true` if the client is connected and has not been signaled to disconnect.
3288    /// The client will automatically retry connection based on its configuration.
3289    #[inline]
3290    #[must_use]
3291    pub fn is_active(&self) -> bool {
3292        self.connection_mode().is_active()
3293    }
3294
3295    /// Returns whether the controller task has stopped.
3296    #[must_use]
3297    pub fn is_disconnected(&self) -> bool {
3298        self.controller_task.is_finished()
3299    }
3300
3301    /// Returns whether the client is reconnecting.
3302    ///
3303    /// Returns `true` if the client lost connection and is attempting to reestablish it.
3304    /// The client will automatically retry connection based on its configuration.
3305    #[inline]
3306    #[must_use]
3307    pub fn is_reconnecting(&self) -> bool {
3308        self.connection_mode().is_reconnect()
3309    }
3310
3311    /// Registers an [`AuthTracker`] with the client.
3312    ///
3313    /// When the controller detects a dead connection and transitions to
3314    /// `Reconnect`, it calls `invalidate()` on the tracker so that any
3315    /// pending authenticated sends see the state change immediately. Terminal
3316    /// transitions fail the tracker so pending auth waits can terminate.
3317    /// Set `reconnect_buffer_waits_for_auth` for clients that must not replay
3318    /// buffered messages until the next session authenticates.
3319    ///
3320    /// Call this once after construction, before any authenticated sends.
3321    pub fn set_auth_tracker(&self, tracker: AuthTracker, reconnect_buffer_waits_for_auth: bool) {
3322        let _ = self.auth_tracker.set(tracker);
3323        self.reconnect_buffer_waits_for_auth
3324            .store(reconnect_buffer_waits_for_auth, Ordering::Release);
3325    }
3326
3327    /// Returns whether the client is disconnecting.
3328    ///
3329    /// Returns `true` if the client is in disconnect mode.
3330    #[inline]
3331    #[must_use]
3332    pub fn is_disconnecting(&self) -> bool {
3333        self.connection_mode().is_disconnect()
3334    }
3335
3336    /// Returns whether the client is closed.
3337    ///
3338    /// Returns `true` if the client has been explicitly disconnected or reached
3339    /// maximum reconnection attempts. In this state, the client cannot be reused
3340    /// and a new client must be created for further connections.
3341    #[inline]
3342    #[must_use]
3343    pub fn is_closed(&self) -> bool {
3344        self.connection_mode().is_closed()
3345    }
3346
3347    /// Checks whether the connection is in a terminal state (disconnecting or closed).
3348    ///
3349    /// Single atomic load to fail fast before rate limiting or waiting.
3350    #[inline]
3351    fn check_not_terminal(&self) -> Result<(), SendError> {
3352        match self.connection_mode() {
3353            ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
3354            _ => Ok(()),
3355        }
3356    }
3357
3358    /// Waits for rate limiter quota, aborting early if connection enters a terminal state.
3359    async fn await_rate_limit_or_closed(&self, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3360        const CHECK_INTERVAL_MS: u64 = 100;
3361
3362        tokio::select! {
3363            biased;
3364            () = self.rate_limiter.await_keys_ready(keys) => Ok(()),
3365            () = async {
3366                loop {
3367                    // Enable before the state check: an unpolled Notified is unregistered and misses notifies
3368                    let mut notified = pin!(self.state_notify.notified());
3369                    notified.as_mut().enable();
3370
3371                    if matches!(self.connection_mode(), ConnectionMode::Disconnect | ConnectionMode::Closed) {
3372                        break;
3373                    }
3374                    tokio::select! {
3375                        biased;
3376                        () = notified => {}
3377                        () = dst::time::sleep(Duration::from_millis(CHECK_INTERVAL_MS)) => {}
3378                    }
3379                }
3380            } => Err(SendError::Closed),
3381        }
3382    }
3383
3384    /// Waits for the client to become active before sending.
3385    ///
3386    /// Uses `state_notify` for event-driven wakeup so sends resume immediately
3387    /// after reconnection completes. A fallback interval guards against missed
3388    /// notifications.
3389    async fn wait_for_active(&self) -> Result<(), SendError> {
3390        const FALLBACK_INTERVAL_MS: u64 = 100;
3391
3392        let mode = self.connection_mode();
3393        if mode.is_active() {
3394            return Ok(());
3395        }
3396
3397        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3398            return Err(SendError::Closed);
3399        }
3400
3401        log::debug!("Waiting for client to become ACTIVE before sending...");
3402
3403        let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
3404
3405        dst::time::timeout(self.connect_timeout, async {
3406            loop {
3407                // Enable before the state check: an unpolled Notified is unregistered and misses notifies
3408                let mut notified = pin!(self.state_notify.notified());
3409                notified.as_mut().enable();
3410
3411                let mode = self.connection_mode();
3412                if mode.is_active() {
3413                    return Ok(());
3414                }
3415
3416                if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3417                    return Err(());
3418                }
3419
3420                tokio::select! {
3421                    biased;
3422                    () = notified => {}
3423                    () = dst::time::sleep(fallback_interval) => {}
3424                }
3425            }
3426        })
3427        .await
3428        .map_err(|_| SendError::Timeout)?
3429        .map_err(|()| SendError::Closed)
3430    }
3431
3432    /// Signals that the caller's reader has observed EOF or a fatal error.
3433    ///
3434    /// In stream mode the controller has no visibility into the caller-owned reader.
3435    /// Call this method when `reader.next().await` returns `None` or an unrecoverable
3436    /// error so the controller transitions to `Closed` and dependent tasks shut down.
3437    ///
3438    /// For peer-initiated close frames (`Message::Close`), use [`disconnect`](Self::disconnect)
3439    /// instead so the writer can send the close reply before shutting down.
3440    ///
3441    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
3442    ///
3443    /// This is a no-op if the connection is already closed or disconnecting.
3444    pub fn notify_closed(&self) {
3445        let mode = self.connection_mode();
3446        if mode.is_disconnect() || mode.is_closed() {
3447            return;
3448        }
3449
3450        log::debug!("Stream reader signalled EOF, transitioning to CLOSED");
3451
3452        if ConnectionMode::close_websocket_on_loss(&self.connection_mode, self.state_sink.as_ref())
3453        {
3454            fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3455            self.state_notify.notify_waiters();
3456        }
3457    }
3458
3459    /// Disconnects the client and waits for the controller task to stop.
3460    ///
3461    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
3462    pub async fn disconnect(&self) {
3463        log::debug!("Disconnecting");
3464
3465        // A CLOSED client keeps its terminal state; its tracker is already failed
3466        if ConnectionMode::request_disconnect(&self.connection_mode)
3467            && let Some(tracker) = self.auth_tracker.get()
3468        {
3469            tracker.fail("WebSocket client disconnected");
3470        }
3471
3472        self.state_notify.notify_waiters();
3473
3474        if dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
3475            while !self.is_disconnected() {
3476                dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
3477            }
3478
3479            if !self.controller_task.is_finished() {
3480                self.controller_task.abort();
3481                log_task_aborted("controller");
3482            }
3483        })
3484        .await
3485            == Ok(())
3486        {
3487            log::debug!("Controller task finished");
3488        } else {
3489            log::warn!("Timeout waiting for controller task to finish");
3490
3491            if !self.controller_task.is_finished() {
3492                self.controller_task.abort();
3493                log_task_aborted("controller");
3494            }
3495
3496            self.connection_mode
3497                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3498        }
3499    }
3500
3501    /// Sends the given text `data` to the server.
3502    ///
3503    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does not
3504    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
3505    /// message. During reconnection, messages are buffered and replayed on the new connection.
3506    ///
3507    /// # Errors
3508    ///
3509    /// Returns an error if the writer capacity is exhausted, or if the client cannot send.
3510    #[allow(unused_variables)]
3511    pub async fn send_text(&self, data: String, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3512        self.check_not_terminal()?;
3513
3514        self.await_rate_limit_or_closed(keys).await?;
3515        self.wait_for_active().await?;
3516
3517        log::trace!("Sending text frame ({} bytes)", data.len());
3518
3519        let msg = Message::Text(data.into());
3520        self.writer_tx.send(WriterCommand::Send(msg))
3521    }
3522
3523    /// Sends text once if the active connection matches `connection_epoch`.
3524    ///
3525    /// The writer rejects the send if reconnection changes ownership before the write. The
3526    /// message is never replayed on another connection.
3527    ///
3528    /// # Errors
3529    ///
3530    /// Returns an error if:
3531    /// - The shared control allowance is full.
3532    /// - The client closes.
3533    /// - The active wait times out before the write starts.
3534    /// - The expected connection no longer owns the writer.
3535    /// - The command or transport write fails.
3536    /// - The write starts but does not complete within the write
3537    ///   deadline. Delivery is undetermined in that case: the message is not replayed, and it
3538    ///   must not be resent blindly.
3539    pub async fn send_text_on_connection(
3540        &self,
3541        data: String,
3542        keys: Option<&[Ustr]>,
3543        connection_epoch: u64,
3544    ) -> Result<(), SendError> {
3545        self.check_not_terminal()?;
3546        self.await_rate_limit_or_closed(keys).await?;
3547        self.wait_for_active().await?;
3548
3549        log::trace!(
3550            "Sending text frame once: epoch={connection_epoch} ({} bytes)",
3551            data.len()
3552        );
3553
3554        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3555        self.writer_tx
3556            .send_control(WriterCommand::SendOnConnection {
3557                message: Message::Text(data.into()),
3558                connection_epoch,
3559                response_tx,
3560            })?;
3561
3562        response_rx
3563            .await
3564            .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3565    }
3566
3567    /// Sends a pong frame back to the server when the connection is active.
3568    ///
3569    /// The pong is skipped silently if the connection is not active when called:
3570    /// a pong belongs to the connection whose ping caused it, so this method does
3571    /// not wait for reconnection before enqueueing it.
3572    ///
3573    /// # Errors
3574    ///
3575    /// Returns an error if:
3576    /// - The payload exceeds 125 bytes, the RFC 6455 control-frame limit.
3577    /// - The writer channel is broken or its control allowance is full.
3578    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3579    #[expect(
3580        clippy::unused_async,
3581        clippy::unused_async_trait_impl,
3582        reason = "skipping instead of waiting removes the only await; the signature is public API shared with the other send methods"
3583    )]
3584    pub async fn send_pong(&self, data: Vec<u8>) -> Result<(), SendError> {
3585        validate_pong_payload(&data)?;
3586
3587        if !self.connection_mode().is_active() {
3588            log::debug!("Skipping pong: connection not active");
3589            return Ok(());
3590        }
3591
3592        log::trace!("Sending pong frame ({} bytes)", data.len());
3593
3594        self.writer_tx
3595            .send_control(WriterCommand::Send(Message::Pong(data.into())))
3596    }
3597
3598    /// Sends a pong if the connection that received its ping is still active.
3599    ///
3600    /// The pong is skipped silently if the connection is inactive or its epoch has changed.
3601    ///
3602    /// # Errors
3603    ///
3604    /// Returns an error if:
3605    /// - The payload exceeds 125 bytes, the RFC 6455 control-frame limit.
3606    /// - The writer channel is broken or its control allowance is full.
3607    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3608    #[expect(
3609        clippy::unused_async,
3610        clippy::unused_async_trait_impl,
3611        reason = "the public send API is async even though this method only enqueues"
3612    )]
3613    pub async fn send_pong_on_connection(
3614        &self,
3615        data: Vec<u8>,
3616        connection_epoch: u64,
3617    ) -> Result<(), SendError> {
3618        validate_pong_payload(&data)?;
3619
3620        if !self.connection_mode().is_active() {
3621            log::debug!("Skipping pong: connection not active");
3622            return Ok(());
3623        }
3624
3625        log::trace!(
3626            "Sending pong frame once: epoch={connection_epoch} ({} bytes)",
3627            data.len()
3628        );
3629
3630        self.writer_tx
3631            .send_control(WriterCommand::SendPongOnConnection {
3632                data,
3633                connection_epoch,
3634            })
3635    }
3636
3637    /// Sends the given bytes `data` to the server.
3638    ///
3639    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does not
3640    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
3641    /// message. During reconnection, messages are buffered and replayed on the new connection.
3642    ///
3643    /// # Errors
3644    ///
3645    /// Returns an error if the writer capacity is exhausted, or if the client cannot send.
3646    #[allow(unused_variables)]
3647    pub async fn send_bytes(&self, data: Vec<u8>, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3648        self.check_not_terminal()?;
3649
3650        self.await_rate_limit_or_closed(keys).await?;
3651        self.wait_for_active().await?;
3652
3653        log::trace!("Sending binary frame ({} bytes)", data.len());
3654
3655        let msg = Message::Binary(data.into());
3656        self.writer_tx.send(WriterCommand::Send(msg))
3657    }
3658
3659    /// Sends bytes once if the active connection matches `connection_epoch`.
3660    ///
3661    /// The writer rejects the send if reconnection changes ownership before the write. The
3662    /// message is never replayed on another connection.
3663    ///
3664    /// # Errors
3665    ///
3666    /// Returns an error if:
3667    /// - The shared control allowance is full.
3668    /// - The client closes.
3669    /// - The active wait times out before the write starts.
3670    /// - The expected connection no longer owns the writer.
3671    /// - The command or transport write fails.
3672    /// - The write starts but does not complete within the write
3673    ///   deadline. Delivery is undetermined in that case: the message is not replayed, and it
3674    ///   must not be resent blindly.
3675    pub async fn send_bytes_on_connection(
3676        &self,
3677        data: Vec<u8>,
3678        keys: Option<&[Ustr]>,
3679        connection_epoch: u64,
3680    ) -> Result<(), SendError> {
3681        self.check_not_terminal()?;
3682        self.await_rate_limit_or_closed(keys).await?;
3683        self.wait_for_active().await?;
3684
3685        log::trace!(
3686            "Sending binary frame once: epoch={connection_epoch} ({} bytes)",
3687            data.len()
3688        );
3689
3690        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3691        self.writer_tx
3692            .send_control(WriterCommand::SendOnConnection {
3693                message: Message::Binary(data.into()),
3694                connection_epoch,
3695                response_tx,
3696            })?;
3697
3698        response_rx
3699            .await
3700            .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3701    }
3702
3703    /// Sends a close message to the server.
3704    ///
3705    /// # Errors
3706    ///
3707    /// Returns an error if the shared control allowance is full, or if the client cannot send.
3708    pub async fn send_close_message(&self) -> Result<(), SendError> {
3709        self.wait_for_active().await?;
3710
3711        let msg = Message::Close(None);
3712        self.writer_tx.send_control(WriterCommand::Send(msg))
3713    }
3714
3715    fn spawn_controller_task(
3716        mut inner: WebSocketClientInner,
3717        connection_mode: Arc<AtomicU8>,
3718        state_notify: Arc<tokio::sync::Notify>,
3719        auth_tracker: Arc<OnceLock<AuthTracker>>,
3720        controller_lifecycle: Arc<ControllerLifecycle>,
3721        controller_notify: Arc<tokio::sync::Notify>,
3722        reconnect_published: Arc<AtomicBool>,
3723    ) -> dst::task::JoinHandle<()> {
3724        dst::task::spawn(async move {
3725            let _activity = controller_lifecycle.activity();
3726            log_task_started("controller");
3727
3728            let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
3729            let mut reconnected_at = None;
3730
3731            loop {
3732                tokio::select! {
3733                    biased;
3734                    () = controller_notify.notified() => {}
3735                    () = state_notify.notified() => {}
3736                    () = dst::time::sleep(fallback_interval) => {}
3737                }
3738
3739                let mut mode = ConnectionMode::from_atomic(&connection_mode);
3740
3741                if mode.is_disconnect() {
3742                    log::debug!("Disconnecting");
3743
3744                    let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
3745                    if dst::time::timeout(timeout, async {
3746                        // Delay awaiting graceful shutdown
3747                        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
3748
3749                        if let Some(read_fence) = inner.read_fence.take() {
3750                            read_fence.invalidate();
3751                        }
3752
3753                        if let Some(task) = &inner.read_task
3754                            && !task.is_finished()
3755                        {
3756                            task.abort();
3757                            log_task_aborted("read");
3758                        }
3759
3760                        if let Some(task) = &inner.heartbeat_task
3761                            && !task.is_finished()
3762                        {
3763                            task.abort();
3764                            log_task_aborted("heartbeat");
3765                        }
3766                    })
3767                    .await
3768                    .is_err()
3769                    {
3770                        log::warn!("Shutdown timed out after {}s", timeout.as_secs());
3771                    }
3772
3773                    log::debug!("Closed");
3774                    break; // Controller finished
3775                }
3776
3777                if mode.is_closed() {
3778                    log::debug!("Connection closed");
3779                    fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3780                    state_notify.notify_waiters();
3781                    break;
3782                }
3783
3784                if mode.is_active() && !inner.is_alive() {
3785                    let target = if inner.handler.is_none() {
3786                        ConnectionMode::Closed
3787                    } else {
3788                        ConnectionMode::Reconnect
3789                    };
3790
3791                    let transitioned = if target.is_closed() {
3792                        ConnectionMode::close_websocket_on_loss(
3793                            &connection_mode,
3794                            inner.state_sink.as_ref(),
3795                        )
3796                    } else {
3797                        request_websocket_reconnect(
3798                            &connection_mode,
3799                            &reconnect_published,
3800                            inner.state_sink.as_ref(),
3801                            &auth_tracker,
3802                            &controller_notify,
3803                            || {},
3804                        ) == ReconnectRequestOutcome::Accepted
3805                    };
3806
3807                    if transitioned {
3808                        if target.is_closed() {
3809                            fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3810                        }
3811
3812                        log::info!("Detected dead connection, transitioning to {target:?}");
3813                    }
3814
3815                    mode = ConnectionMode::from_atomic(&connection_mode);
3816                }
3817
3818                if mode.is_reconnect() {
3819                    if let Some(tracker) = auth_tracker.get() {
3820                        tracker.invalidate();
3821                    }
3822
3823                    let reconnect_uptime = reconnected_at
3824                        .take()
3825                        .map(|started: dst::time::Instant| started.elapsed());
3826                    let previous_reconnect_stable = reconnect_uptime
3827                        .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
3828
3829                    if previous_reconnect_stable {
3830                        inner.backoff.reset();
3831                        inner.reconnection_attempt_count = 0;
3832                        log::debug!(
3833                            "WebSocket remained active for at least {}s, resetting reconnect cycle",
3834                            RECONNECT_STABILITY_THRESHOLD.as_secs()
3835                        );
3836                    }
3837
3838                    if let Some(max_attempts) = inner.reconnect_max_attempts
3839                        && inner.reconnection_attempt_count >= max_attempts
3840                    {
3841                        log::error!(
3842                            "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
3843                        );
3844                        connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3845                        fail_registered_auth(
3846                            auth_tracker.as_ref(),
3847                            "WebSocket reconnect attempts exhausted",
3848                        );
3849                        state_notify.notify_waiters();
3850                        break;
3851                    }
3852
3853                    let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
3854                    {
3855                        inner.backoff.next_duration()
3856                    } else {
3857                        Duration::ZERO
3858                    };
3859
3860                    let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
3861                    if !duration.is_zero() {
3862                        log::warn!("Backing off for {}s...", duration.as_secs_f64());
3863
3864                        if !wait_reconnect_delay(
3865                            duration,
3866                            connection_mode.as_ref(),
3867                            state_notify.as_ref(),
3868                        )
3869                        .await
3870                        {
3871                            log::debug!("Backoff interrupted by terminal state");
3872                            continue;
3873                        }
3874                    }
3875
3876                    inner.reconnection_attempt_count += 1;
3877                    inner.reconnect_throttle.record_attempt();
3878                    log::debug!(
3879                        "Reconnection attempt {} of {}",
3880                        inner.reconnection_attempt_count,
3881                        inner
3882                            .reconnect_max_attempts
3883                            .map_or_else(|| "unlimited".to_string(), |m| m.to_string())
3884                    );
3885
3886                    // Race reconnect against disconnect notification
3887                    let reconnect_result = tokio::select! {
3888                        biased;
3889                        result = inner.reconnect_with_outcome() => Some(result),
3890                        () = async {
3891                            loop {
3892                                // Enable before the check so a disconnect notify between iterations is not missed
3893                                let mut notified = pin!(state_notify.notified());
3894                                notified.as_mut().enable();
3895
3896                                if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
3897                                    break;
3898                                }
3899                                notified.await;
3900                            }
3901                        } => None,
3902                    };
3903
3904                    match reconnect_result {
3905                        None => {
3906                            log::debug!("Reconnect interrupted by disconnect");
3907                        }
3908                        Some(Ok(ReconnectOutcome::Reconnected)) => {
3909                            reconnected_at = Some(dst::time::Instant::now());
3910
3911                            state_notify.notify_waiters();
3912
3913                            // The outcome records a completed reconnection; emit recovery
3914                            // callbacks only while the replacement is still `Active`, not
3915                            // after a teardown or another drop.
3916                            if ConnectionMode::from_atomic(&connection_mode).is_active() {
3917                                if let Some(ref handler) = inner.handler {
3918                                    let connection_epoch =
3919                                        inner.connection_epoch.load(Ordering::Acquire);
3920                                    let reconnected_msg =
3921                                        Message::Text(RECONNECTED.to_string().into());
3922                                    handler.handle(connection_epoch, reconnected_msg);
3923                                    match handler {
3924                                        IncomingHandler::Message(_) => {
3925                                            log::debug!("Sent reconnected message to handler");
3926                                        }
3927                                        IncomingHandler::Epoch(_) => {
3928                                            log::debug!(
3929                                                "Sent reconnected message to epoch handler: \
3930                                                 epoch={connection_epoch}",
3931                                            );
3932                                        }
3933                                    }
3934                                }
3935
3936                                log::debug!("Reconnected successfully");
3937                            } else {
3938                                log::debug!("Skipping reconnect handlers due to disconnect state");
3939                            }
3940                        }
3941                        Some(Ok(ReconnectOutcome::Aborted)) => {
3942                            log::debug!("Reconnect aborted");
3943                        }
3944                        Some(Err(e)) => {
3945                            let duration = inner.backoff.next_duration();
3946                            log::warn!(
3947                                "Reconnect attempt {} failed: {}",
3948                                inner.reconnection_attempt_count,
3949                                e.redacted_message()
3950                            );
3951
3952                            if !duration.is_zero() {
3953                                log::warn!("Backing off for {}s...", duration.as_secs_f64());
3954
3955                                if !wait_reconnect_delay(
3956                                    duration,
3957                                    connection_mode.as_ref(),
3958                                    state_notify.as_ref(),
3959                                )
3960                                .await
3961                                {
3962                                    log::debug!("Backoff interrupted by terminal state");
3963                                }
3964                            }
3965                        }
3966                    }
3967                }
3968            }
3969
3970            inner
3971                .connection_mode
3972                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3973
3974            log_task_stopped("controller");
3975        })
3976    }
3977}
3978
3979fn fail_registered_auth(auth_tracker: &OnceLock<AuthTracker>, reason: &str) {
3980    if let Some(tracker) = auth_tracker.get() {
3981        tracker.fail(reason);
3982    }
3983}
3984
3985fn validate_pong_payload(data: &[u8]) -> Result<(), SendError> {
3986    if data.len() > MAX_CONTROL_FRAME_PAYLOAD_BYTES {
3987        return Err(SendError::InvalidInput(format!(
3988            "pong payload exceeds {MAX_CONTROL_FRAME_PAYLOAD_BYTES} bytes"
3989        )));
3990    }
3991
3992    Ok(())
3993}
3994
3995impl Drop for WebSocketClient {
3996    fn drop(&mut self) {
3997        self.connection_mode
3998            .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3999        fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
4000        self.state_notify.notify_waiters();
4001        self.controller_notify.notify_waiters();
4002        self.controller_lifecycle.close_and_abort();
4003    }
4004}
4005
4006#[cfg(test)]
4007#[cfg(not(feature = "turmoil"))]
4008#[cfg(not(all(feature = "simulation", madsim)))] // These tests drive real localhost sockets
4009#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
4010mod tests {
4011    use std::{
4012        collections::HashMap,
4013        num::NonZeroU32,
4014        sync::{
4015            Arc,
4016            atomic::{AtomicU8, AtomicUsize, Ordering},
4017        },
4018        time::Duration,
4019    };
4020
4021    use axum::{Router, routing::post};
4022    use futures_util::{SinkExt, StreamExt};
4023    use log::Level;
4024    use nautilus_common::testing::wait_until_async;
4025    use nautilus_core::string::secret::REDACTED;
4026    use parking_lot::Mutex;
4027    use rstest::rstest;
4028    use tokio::{
4029        io::{AsyncReadExt, AsyncWriteExt},
4030        net::TcpListener,
4031        sync::{mpsc, oneshot},
4032        task::{self, JoinHandle},
4033    };
4034    use tokio_tungstenite::{
4035        accept_async, accept_hdr_async,
4036        tungstenite::{
4037            Message as WsMessage,
4038            handshake::server::{self, Callback},
4039            http::HeaderValue,
4040            protocol::frame::{CloseFrame as WsCloseFrame, coding::CloseCode},
4041        },
4042    };
4043
4044    use crate::{
4045        RECONNECTED, SocketState, SocketStateSink,
4046        error::SendError,
4047        http::{HttpClient, Method},
4048        logging::tests::capture_logs_for,
4049        mode::{ConnectionMode, ReadSessionFence},
4050        ratelimiter::quota::Quota,
4051        transport::{CloseFrame, Message, TransportError},
4052        websocket::{
4053            AuthTracker, CloseHandler, InitialConnectRetryPolicy, ReconnectHeaders,
4054            TransportBackend, WebSocketClient, WebSocketClientInner, WebSocketConfig,
4055            auth::AuthState,
4056            types::{WriterCommand, channel_epoch_message_handler, channel_message_handler},
4057        },
4058    };
4059
4060    const SECRET_MARKER: &str = "OUTBOUND_SECRET_MARKER";
4061    const PING_TRIGGER: &str = "send-test-ping";
4062    const HOSTILE_CLOSE_TRIGGER: &str = "close-hostile";
4063    const CODED_CLOSE_TRIGGER: &str = "close-coded";
4064    const CODED_CLOSE_CODE: u16 = 4003;
4065    const CODED_CLOSE_REASON: &str = "draining";
4066    const HOSTILE_CLOSE_REASON: &str = "injected\r\n\u{1b}[31mforged\u{1b}[0m";
4067    const NETWORK_LOG_TARGETS: &[&str] = &[
4068        "nautilus_network::http::client",
4069        "nautilus_network::websocket::client",
4070    ];
4071
4072    struct TestServer {
4073        task: JoinHandle<()>,
4074        port: u16,
4075    }
4076
4077    #[derive(Debug, Clone)]
4078    struct TestCallback {
4079        key: String,
4080        value: HeaderValue,
4081    }
4082
4083    impl Callback for TestCallback {
4084        #[expect(clippy::panic_in_result_fn)]
4085        fn on_request(
4086            self,
4087            request: &server::Request,
4088            response: server::Response,
4089        ) -> Result<server::Response, server::ErrorResponse> {
4090            let _ = response;
4091            let value = request.headers().get(&self.key);
4092            assert!(value.is_some());
4093
4094            if let Some(value) = request.headers().get(&self.key) {
4095                assert_eq!(value, self.value);
4096            }
4097
4098            Ok(response)
4099        }
4100    }
4101
4102    impl TestServer {
4103        async fn setup() -> Self {
4104            let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4105            let port = TcpListener::local_addr(&server).unwrap().port();
4106
4107            let header_key = "test".to_string();
4108            let header_value = "test".to_string();
4109
4110            let test_call_back = TestCallback {
4111                key: header_key,
4112                value: HeaderValue::from_str(&header_value).unwrap(),
4113            };
4114
4115            let task = task::spawn(async move {
4116                // Keep accepting connections
4117                loop {
4118                    let (conn, _) = server.accept().await.unwrap();
4119                    let mut websocket = accept_hdr_async(conn, test_call_back.clone())
4120                        .await
4121                        .unwrap();
4122
4123                    task::spawn(async move {
4124                        while let Some(Ok(msg)) = websocket.next().await {
4125                            match msg {
4126                                WsMessage::Text(txt) if txt == "close-now" => {
4127                                    log::debug!("Forcibly closing from server side");
4128                                    // This sends a close frame, then stops reading
4129                                    let _ = websocket.close(None).await;
4130                                    break;
4131                                }
4132                                WsMessage::Text(txt) if txt == HOSTILE_CLOSE_TRIGGER => {
4133                                    // Close reasons are server-controlled: this one carries a
4134                                    // forged line break and ANSI color sequences for the log
4135                                    // sanitization test.
4136                                    let _ = websocket
4137                                        .send(WsMessage::Close(Some(WsCloseFrame {
4138                                            code: CloseCode::Error,
4139                                            reason: HOSTILE_CLOSE_REASON.into(),
4140                                        })))
4141                                        .await;
4142
4143                                    break;
4144                                }
4145                                WsMessage::Text(txt) if txt == CODED_CLOSE_TRIGGER => {
4146                                    let _ = websocket
4147                                        .send(WsMessage::Close(Some(WsCloseFrame {
4148                                            code: CloseCode::from(CODED_CLOSE_CODE),
4149                                            reason: CODED_CLOSE_REASON.into(),
4150                                        })))
4151                                        .await;
4152
4153                                    break;
4154                                }
4155                                WsMessage::Text(txt) if txt == PING_TRIGGER => {
4156                                    let ping = format!("{SECRET_MARKER}:ping");
4157                                    if websocket.send(WsMessage::Ping(ping.into())).await.is_err() {
4158                                        break;
4159                                    }
4160                                }
4161                                // Echo text/binary frames
4162                                WsMessage::Text(_) | WsMessage::Binary(_) => {
4163                                    if websocket.send(msg).await.is_err() {
4164                                        break;
4165                                    }
4166                                }
4167                                // If the client closes, we also break
4168                                WsMessage::Close(_frame) => {
4169                                    let _ = websocket.close(None).await;
4170                                    break;
4171                                }
4172                                // Ignore pings/pongs
4173                                _ => {}
4174                            }
4175                        }
4176                    });
4177                }
4178            });
4179
4180            Self { task, port }
4181        }
4182    }
4183
4184    impl Drop for TestServer {
4185        fn drop(&mut self) {
4186            self.task.abort();
4187        }
4188    }
4189
4190    async fn setup_test_client(port: u16) -> WebSocketClient {
4191        let config = WebSocketConfig {
4192            url: format!("ws://127.0.0.1:{port}"),
4193            headers: vec![("test".into(), "test".into())],
4194            heartbeat_interval_secs: None,
4195            heartbeat_payload: None,
4196            connect_timeout_ms: None,
4197            reconnect_delay_initial_ms: None,
4198            reconnect_backoff_factor: None,
4199            reconnect_delay_max_ms: None,
4200            reconnect_jitter_ms: None,
4201            reconnect_max_attempts: None,
4202            heartbeat_timeout_secs: None,
4203            idle_timeout_ms: None,
4204            writer_capacity: None,
4205            backend: TransportBackend::Tungstenite,
4206            proxy_url: None,
4207            max_message_size_bytes: None,
4208            max_frame_size_bytes: None,
4209        };
4210
4211        WebSocketClient::builder()
4212            .config(config)
4213            .message_handler(Arc::new(|_| {}))
4214            .connect()
4215            .await
4216            .expect("Failed to connect")
4217    }
4218
4219    #[rstest]
4220    #[case::handler(false)]
4221    #[case::stream(true)]
4222    #[tokio::test]
4223    async fn test_writer_capacity_from_config(#[case] stream_mode: bool) {
4224        let server = TestServer::setup().await;
4225        let config = WebSocketConfig::builder()
4226            .url(format!("ws://127.0.0.1:{}", server.port))
4227            .headers(vec![("test".to_string(), "test".to_string())])
4228            .backend(TransportBackend::Tungstenite)
4229            .writer_capacity(2)
4230            .build()
4231            .unwrap();
4232
4233        let (client, _reader) = if stream_mode {
4234            let (reader, client) = WebSocketClient::stream_builder()
4235                .config(config)
4236                .connect()
4237                .await
4238                .unwrap();
4239            (client, Some(reader))
4240        } else {
4241            let client = WebSocketClient::builder()
4242                .config(config)
4243                .message_handler(Arc::new(|_| {}))
4244                .connect()
4245                .await
4246                .unwrap();
4247            (client, None)
4248        };
4249
4250        // Ready sends do not yield on this current-thread runtime, keeping the queue occupied
4251        client.send_text("first".to_string(), None).await.unwrap();
4252        client.send_bytes(vec![2], None).await.unwrap();
4253        let overflow = client.send_text("overflow".to_string(), None).await;
4254
4255        assert_eq!(client.connection_mode(), ConnectionMode::Active);
4256        assert!(matches!(overflow, Err(SendError::BufferFull)));
4257        client.disconnect().await;
4258    }
4259
4260    #[rstest]
4261    #[tokio::test]
4262    async fn test_send_reports_writer_buffer_full() {
4263        let server = TestServer::setup().await;
4264        let mut client = setup_test_client(server.port).await;
4265        let (writer_tx, mut writer_rx) = crate::writer::channel(1);
4266        client.writer_tx = writer_tx;
4267
4268        client.send_text("first".to_string(), None).await.unwrap();
4269        let result = client.send_bytes(vec![2], None).await;
4270        assert!(matches!(result, Err(SendError::BufferFull)));
4271        let (command, permit) = writer_rx.recv().await.unwrap();
4272        assert!(
4273            matches!(command, WriterCommand::Send(message) if message == Message::text("first"))
4274        );
4275        assert!(matches!(
4276            client.send_text("overflow".to_string(), None).await,
4277            Err(SendError::BufferFull)
4278        ));
4279        drop(permit);
4280        client.send_bytes(vec![3], None).await.unwrap();
4281        let (command, _) = writer_rx.recv().await.unwrap();
4282        assert!(
4283            matches!(command, WriterCommand::Send(message) if message == Message::Binary(vec![3].into()))
4284        );
4285        client.disconnect().await;
4286    }
4287
4288    async fn setup_reconnecting_client(port: u16) -> WebSocketClient {
4289        let config = WebSocketConfig {
4290            url: format!("ws://127.0.0.1:{port}"),
4291            headers: vec![],
4292            heartbeat_interval_secs: None,
4293            heartbeat_payload: None,
4294            connect_timeout_ms: Some(5_000),
4295            reconnect_delay_initial_ms: Some(1),
4296            reconnect_backoff_factor: Some(1.0),
4297            reconnect_delay_max_ms: Some(1),
4298            reconnect_jitter_ms: Some(0),
4299            reconnect_max_attempts: None,
4300            heartbeat_timeout_secs: None,
4301            idle_timeout_ms: None,
4302            writer_capacity: None,
4303            backend: TransportBackend::Tungstenite,
4304            proxy_url: None,
4305            max_message_size_bytes: None,
4306            max_frame_size_bytes: None,
4307        };
4308
4309        WebSocketClient::builder()
4310            .config(config)
4311            .message_handler(Arc::new(|_| {}))
4312            .connect()
4313            .await
4314            .expect("client should connect")
4315    }
4316
4317    async fn wait_for_mode(client: &WebSocketClient, expected: ConnectionMode) {
4318        crate::dst::time::timeout(Duration::from_secs(5), async {
4319            loop {
4320                if ConnectionMode::from_atomic(&client.connection_mode) == expected {
4321                    break;
4322                }
4323
4324                crate::dst::time::sleep(Duration::from_millis(1)).await;
4325            }
4326        })
4327        .await
4328        .expect("client should reach expected connection mode");
4329    }
4330
4331    async fn setup_http_test_server() -> (JoinHandle<()>, u16) {
4332        let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4333        let port = server.local_addr().unwrap().port();
4334
4335        let app = Router::new().route(
4336            "/logging",
4337            post(|| async {
4338                (
4339                    [("x-secret-response", SECRET_MARKER)],
4340                    format!("{SECRET_MARKER}:response"),
4341                )
4342            }),
4343        );
4344
4345        let task = task::spawn(async move {
4346            axum::serve(server, app).await.unwrap();
4347        });
4348
4349        (task, port)
4350    }
4351
4352    #[rstest]
4353    #[tokio::test]
4354    async fn test_network_logs_omit_payload_bodies() {
4355        let server = TestServer::setup().await;
4356        let client = setup_test_client(server.port).await;
4357        let capture = capture_logs_for(NETWORK_LOG_TARGETS).await;
4358        let binary = format!("{SECRET_MARKER}:binary").into_bytes();
4359        let binary_marker = format!("{binary:?}");
4360
4361        client
4362            .send_text(format!("{SECRET_MARKER}:café"), None)
4363            .await
4364            .unwrap();
4365        client
4366            .send_text_on_connection(
4367                format!("{SECRET_MARKER}:owned-é"),
4368                None,
4369                client.connection_epoch(),
4370            )
4371            .await
4372            .unwrap();
4373        client.send_bytes(binary.clone(), None).await.unwrap();
4374        client
4375            .send_bytes_on_connection(binary, None, client.connection_epoch())
4376            .await
4377            .unwrap();
4378        client
4379            .send_text(PING_TRIGGER.to_string(), None)
4380            .await
4381            .unwrap();
4382
4383        tokio::time::timeout(Duration::from_secs(2), async {
4384            loop {
4385                if capture.messages().iter().any(|(level, message)| {
4386                    matches!(level, Level::Trace | Level::Warn)
4387                        && message == "Received ping frame (27 bytes)"
4388                }) {
4389                    break;
4390                }
4391
4392                tokio::task::yield_now().await;
4393            }
4394        })
4395        .await
4396        .expect("timed out waiting for inbound WebSocket metadata log");
4397
4398        let (http_task, http_port) = setup_http_test_server().await;
4399        let invalid_headers =
4400            HashMap::from([("x-secret-default".to_string(), format!("{SECRET_MARKER}\n"))]);
4401        let invalid_header_error = HttpClient::builder()
4402            .headers(invalid_headers)
4403            .build()
4404            .unwrap_err();
4405        let http_client = HttpClient::builder().build().unwrap();
4406        let params = HashMap::from([("secret".to_string(), vec![SECRET_MARKER.to_string()])]);
4407        let headers = HashMap::from([
4408            (
4409                "X-Secret-Request".to_string(),
4410                format!("{SECRET_MARKER}:first"),
4411            ),
4412            (
4413                "x-secret-request".to_string(),
4414                format!("{SECRET_MARKER}:second"),
4415            ),
4416        ]);
4417        let http_body = format!("{SECRET_MARKER}:http-body").into_bytes();
4418        http_client
4419            .request(
4420                Method::POST,
4421                format!("http://127.0.0.1:{http_port}/logging"),
4422                Some(&params),
4423                Some(headers),
4424                Some(http_body),
4425                None,
4426                None,
4427            )
4428            .await
4429            .unwrap();
4430
4431        // The initial-connect retry warning names the endpoint, which can carry credentials or
4432        // signed query data, so it must report the redaction placeholder rather than the URL.
4433        let rejecting = TcpListener::bind("127.0.0.1:0").await.unwrap();
4434        let rejecting_port = rejecting.local_addr().unwrap().port();
4435
4436        let rejecting_server = task::spawn(async move {
4437            loop {
4438                let (mut stream, _) = rejecting.accept().await.unwrap();
4439                let mut request = [0; 1024];
4440                let _ = stream.read(&mut request).await.unwrap();
4441                stream
4442                    .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
4443                    .await
4444                    .unwrap();
4445            }
4446        });
4447
4448        let retry_config = WebSocketConfig {
4449            url: format!("ws://127.0.0.1:{rejecting_port}/?token={SECRET_MARKER}"),
4450            headers: vec![],
4451            heartbeat_interval_secs: None,
4452            heartbeat_payload: None,
4453            connect_timeout_ms: Some(5_000),
4454            reconnect_delay_initial_ms: None,
4455            reconnect_backoff_factor: None,
4456            reconnect_delay_max_ms: None,
4457            reconnect_jitter_ms: None,
4458            reconnect_max_attempts: None,
4459            heartbeat_timeout_secs: None,
4460            idle_timeout_ms: None,
4461            writer_capacity: None,
4462            backend: TransportBackend::Tungstenite,
4463            proxy_url: None,
4464            max_message_size_bytes: None,
4465            max_frame_size_bytes: None,
4466        };
4467
4468        let retry_error = WebSocketClient::builder()
4469            .config(retry_config)
4470            .message_handler(Arc::new(|_| {}))
4471            .initial_connect_retry_policy(InitialConnectRetryPolicy {
4472                max_attempts: NonZeroU32::new(2).unwrap(),
4473                delay_initial: Duration::from_millis(1),
4474                delay_max: Duration::from_millis(1),
4475                backoff_factor: 1.0,
4476                jitter_ms: 0,
4477            })
4478            .connect()
4479            .await
4480            .expect_err("server always rejects the upgrade");
4481
4482        rejecting_server.abort();
4483
4484        assert!(
4485            matches!(retry_error, TransportError::UpgradeRejected(503)),
4486            "expected a 503 upgrade rejection, was: {retry_error:?}"
4487        );
4488
4489        let messages: Vec<_> = capture
4490            .messages()
4491            .into_iter()
4492            .filter(|(level, message)| {
4493                matches!(level, Level::Trace | Level::Warn)
4494                    && (message.starts_with("Sending ")
4495                        || message.starts_with("Received ")
4496                        || message.starts_with("Replaced ")
4497                        || message.starts_with("WebSocket connection attempt "))
4498            })
4499            .map(|(_, message)| message)
4500            .collect();
4501
4502        let invalid_header_message = invalid_header_error.to_string();
4503
4504        // Asserted positively so the blanket secret assertion below cannot pass vacuously by
4505        // capturing no warning at all.
4506        let retry_warning = messages
4507            .iter()
4508            .find(|message| message.starts_with("WebSocket connection attempt 1/2 "))
4509            .expect("initial-connect retry warning was not captured");
4510        assert!(
4511            retry_warning.contains(REDACTED),
4512            "retry warning omitted the redaction placeholder: {retry_warning}"
4513        );
4514
4515        assert!(
4516            messages.iter().all(|message| {
4517                !message.contains(SECRET_MARKER) && !message.contains(&binary_marker)
4518            }),
4519            "network logs exposed the secret marker: {messages:?}"
4520        );
4521        assert!(
4522            !invalid_header_message.contains(SECRET_MARKER),
4523            "invalid header error exposed the secret marker: {invalid_header_message}"
4524        );
4525        assert!(
4526            invalid_header_message.contains("x-secret-default"),
4527            "invalid header error omitted safe header metadata: {invalid_header_message}"
4528        );
4529        assert!(
4530            messages
4531                .iter()
4532                .any(|message| message == "Sending text frame (28 bytes)"),
4533            "text send metadata missing or inaccurate: {messages:?}"
4534        );
4535        assert!(
4536            messages
4537                .iter()
4538                .any(|message| { message == "Sending text frame once: epoch=0 (31 bytes)" }),
4539            "ownership-bound text metadata missing or inaccurate: {messages:?}"
4540        );
4541        assert!(
4542            messages
4543                .iter()
4544                .any(|message| message == "Sending binary frame (29 bytes)"),
4545            "binary send metadata missing or inaccurate: {messages:?}"
4546        );
4547        assert!(
4548            messages
4549                .iter()
4550                .any(|message| message == "Sending binary frame once: epoch=0 (29 bytes)"),
4551            "ownership-bound binary metadata missing or inaccurate: {messages:?}"
4552        );
4553        assert!(
4554            messages
4555                .iter()
4556                .any(|message| message == "Received text frame (28 bytes)"),
4557            "text receive metadata missing or inaccurate: {messages:?}"
4558        );
4559        assert!(
4560            messages
4561                .iter()
4562                .any(|message| message == "Received text frame (31 bytes)"),
4563            "ownership-bound text receive metadata missing or inaccurate: {messages:?}"
4564        );
4565        assert!(
4566            messages
4567                .iter()
4568                .any(|message| message == "Received message <binary> 29 bytes"),
4569            "binary receive metadata missing or inaccurate: {messages:?}"
4570        );
4571        assert!(
4572            messages
4573                .iter()
4574                .any(|message| message == "Received ping frame (27 bytes)"),
4575            "ping receive metadata missing or inaccurate: {messages:?}"
4576        );
4577        // Two caller headers plus the `accept-encoding: gzip` added for buffered requests
4578        assert!(
4579            messages.iter().any(|message| {
4580                message
4581                    == "Sending HTTP request: method=POST extra_headers=3 query_bytes=29 \
4582                        body_bytes=32"
4583            }),
4584            "HTTP request metadata missing or inaccurate: {messages:?}"
4585        );
4586        assert!(
4587            messages
4588                .iter()
4589                .any(|message| message == "Replaced duplicate request header 'x-secret-request'"),
4590            "duplicate header metadata missing: {messages:?}"
4591        );
4592        assert!(
4593            messages.iter().any(|message| {
4594                message.starts_with("Received HTTP response: status=200 OK headers=")
4595                    && message.ends_with(" body_bytes=31")
4596            }),
4597            "HTTP response metadata missing or inaccurate: {messages:?}"
4598        );
4599
4600        client.disconnect().await;
4601        http_task.abort();
4602    }
4603
4604    // A pong belongs to the connection whose ping caused it: while the
4605    // replacement handshake is gated the pong must be dropped rather than
4606    // waited on, and the next pong must ride the new connection.
4607    #[rstest]
4608    #[tokio::test]
4609    async fn test_send_pong_skips_gated_reconnect() {
4610        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4611        let port = listener.local_addr().unwrap().port();
4612        let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
4613        let (handshake_gate_tx, handshake_gate_rx) = oneshot::channel();
4614        let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
4615
4616        let server_task = task::spawn(async move {
4617            let (first, _) = listener.accept().await.unwrap();
4618            let first_websocket = accept_async(first).await.unwrap();
4619            drop(first_websocket);
4620
4621            let (second, _) = listener.accept().await.unwrap();
4622            second_accepted_tx.send(()).unwrap();
4623            handshake_gate_rx.await.unwrap();
4624            let mut replacement = accept_async(second).await.unwrap();
4625
4626            while let Some(message) = replacement.next().await {
4627                match message.unwrap() {
4628                    WsMessage::Pong(data) => observed_tx.send(data.to_vec()).unwrap(),
4629                    WsMessage::Close(_) => {
4630                        let _ = replacement.close(None).await;
4631                        break;
4632                    }
4633                    _ => {}
4634                }
4635            }
4636        });
4637
4638        let client = setup_reconnecting_client(port).await;
4639
4640        crate::dst::time::timeout(Duration::from_secs(5), second_accepted_rx)
4641            .await
4642            .expect("replacement connection should be accepted")
4643            .unwrap();
4644        wait_for_mode(&client, ConnectionMode::Reconnect).await;
4645
4646        let rejected = client.send_pong(vec![2; 126]).await;
4647        assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4648
4649        // The gate is still shut, so a pong that waited for the connection to
4650        // become active could not complete here.
4651        crate::dst::time::timeout(
4652            Duration::from_secs(1),
4653            client.send_pong(b"stale-pong".to_vec()),
4654        )
4655        .await
4656        .expect("pong raised during reconnect should not wait for the replacement")
4657        .expect("skipped pong should report success");
4658
4659        handshake_gate_tx.send(()).unwrap();
4660        wait_for_mode(&client, ConnectionMode::Active).await;
4661
4662        let fresh_payload = b"fresh-pong".to_vec();
4663        client.send_pong(fresh_payload.clone()).await.unwrap();
4664        assert_eq!(
4665            crate::dst::time::timeout(Duration::from_secs(5), observed_rx.recv())
4666                .await
4667                .expect("replacement connection should receive the fresh pong"),
4668            Some(fresh_payload)
4669        );
4670
4671        client.disconnect().await;
4672        server_task.await.unwrap();
4673        assert_eq!(
4674            observed_rx.try_recv(),
4675            Err(mpsc::error::TryRecvError::Disconnected),
4676            "replacement connection should receive exactly one pong"
4677        );
4678    }
4679
4680    // Validation must precede the inactive skip: an oversized pong during
4681    // reconnect is a caller error, not a silently skipped frame.
4682    #[rstest]
4683    #[tokio::test]
4684    async fn test_pong_validation_precedes_inactive_skip() {
4685        let server = TestServer::setup().await;
4686        let client = setup_test_client(server.port).await;
4687        client
4688            .connection_mode
4689            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4690
4691        let result = client.send_pong(vec![2; 126]).await;
4692        let epoch_result = client.send_pong_on_connection(vec![2; 126], 0).await;
4693
4694        assert!(matches!(result, Err(SendError::InvalidInput(_))));
4695        assert!(matches!(epoch_result, Err(SendError::InvalidInput(_))));
4696    }
4697
4698    // A rejected oversized pong must not disturb the connection: the frame is
4699    // never enqueued, so the next in-range pong still rides the same socket.
4700    #[rstest]
4701    #[tokio::test]
4702    async fn test_pong_payload_limit_preserves_connection() {
4703        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4704        let port = listener.local_addr().unwrap().port();
4705        let (pong_tx, mut pong_rx) = mpsc::unbounded_channel();
4706
4707        let server_task = task::spawn(async move {
4708            let (stream, _) = listener.accept().await.unwrap();
4709            let mut websocket = accept_async(stream).await.unwrap();
4710
4711            while let Some(message) = websocket.next().await {
4712                match message.unwrap() {
4713                    WsMessage::Pong(data) => pong_tx.send(data.to_vec()).unwrap(),
4714                    WsMessage::Close(_) => {
4715                        let _ = websocket.close(None).await;
4716                        break;
4717                    }
4718                    _ => {}
4719                }
4720            }
4721        });
4722
4723        let client = setup_reconnecting_client(port).await;
4724        let accepted = vec![1; 125];
4725        let follow_up = vec![3; 125];
4726
4727        client.send_pong(accepted.clone()).await.unwrap();
4728        assert_eq!(
4729            crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4730                .await
4731                .unwrap(),
4732            Some(accepted)
4733        );
4734
4735        let rejected = client.send_pong(vec![2; 126]).await;
4736        assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4737
4738        client.send_pong(follow_up.clone()).await.unwrap();
4739        assert_eq!(
4740            crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4741                .await
4742                .unwrap(),
4743            Some(follow_up)
4744        );
4745
4746        client.disconnect().await;
4747        server_task.await.unwrap();
4748    }
4749
4750    #[tokio::test]
4751    async fn test_websocket_basic() {
4752        let server = TestServer::setup().await;
4753        let client = setup_test_client(server.port).await;
4754
4755        assert!(!client.is_disconnected());
4756
4757        client.disconnect().await;
4758        assert!(client.is_disconnected());
4759    }
4760
4761    #[rstest]
4762    #[tokio::test]
4763    async fn test_drop_sets_shared_connection_mode_closed() {
4764        let server = TestServer::setup().await;
4765        let client = setup_test_client(server.port).await;
4766        let connection_mode = client.connection_mode_atomic();
4767
4768        drop(client);
4769
4770        assert_eq!(
4771            ConnectionMode::from_atomic(&connection_mode),
4772            ConnectionMode::Closed
4773        );
4774    }
4775
4776    #[rstest]
4777    #[tokio::test]
4778    async fn test_notify_closed_closes_reconnecting_client() {
4779        let server = TestServer::setup().await;
4780        let client = setup_test_client(server.port).await;
4781        client
4782            .connection_mode
4783            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4784
4785        client.notify_closed();
4786
4787        assert!(client.is_closed());
4788    }
4789
4790    #[tokio::test]
4791    async fn test_websocket_heartbeat() {
4792        let server = TestServer::setup().await;
4793        let client = setup_test_client(server.port).await;
4794
4795        // Wait ~3s => server should see multiple "ping"
4796        tokio::time::sleep(std::time::Duration::from_secs(3)).await;
4797
4798        // Cleanup
4799        client.disconnect().await;
4800        assert!(client.is_disconnected());
4801    }
4802
4803    #[rstest]
4804    #[tokio::test]
4805    async fn test_websocket_reconnect_exhausted() {
4806        let config = WebSocketConfig {
4807            url: "ws://127.0.0.1:9997".into(), // <-- No server
4808            headers: vec![],
4809            heartbeat_interval_secs: None,
4810            heartbeat_payload: None,
4811            connect_timeout_ms: None,
4812            reconnect_delay_initial_ms: None,
4813            reconnect_backoff_factor: None,
4814            reconnect_delay_max_ms: None,
4815            reconnect_jitter_ms: None,
4816            reconnect_max_attempts: None,
4817            heartbeat_timeout_secs: None,
4818            idle_timeout_ms: None,
4819            writer_capacity: None,
4820            backend: TransportBackend::Tungstenite,
4821            proxy_url: None,
4822            max_message_size_bytes: None,
4823            max_frame_size_bytes: None,
4824        };
4825
4826        let states = Arc::new(Mutex::new(Vec::new()));
4827        let states_callback = Arc::clone(&states);
4828        let sink = SocketStateSink::new(move |state| {
4829            states_callback.lock().push(state);
4830        });
4831
4832        let res = WebSocketClient::builder()
4833            .config(config)
4834            .message_handler(Arc::new(|_| {}))
4835            .state_sink(sink)
4836            .connect()
4837            .await;
4838        assert!(res.is_err(), "Should fail quickly with no server");
4839        assert_eq!(*states.lock(), Vec::new());
4840    }
4841
4842    #[tokio::test]
4843    async fn test_websocket_forced_close_reconnect() {
4844        let server = TestServer::setup().await;
4845        let client = setup_test_client(server.port).await;
4846
4847        // 1) Send normal message
4848        client.send_text("Hello".into(), None).await.unwrap();
4849
4850        // 2) Trigger forced close from server
4851        client.send_text("close-now".into(), None).await.unwrap();
4852
4853        // 3) Wait a bit => read loop sees close => reconnect
4854        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
4855
4856        // Confirm not disconnected
4857        assert!(!client.is_disconnected());
4858
4859        // Cleanup
4860        client.disconnect().await;
4861        assert!(client.is_disconnected());
4862    }
4863
4864    #[rstest]
4865    #[case::plain(false)]
4866    #[case::epoch(true)]
4867    #[tokio::test]
4868    async fn test_peer_close_preserves_forwarded_messages(#[case] epoch_aware: bool) {
4869        let timeout = Duration::from_secs(5);
4870        let subscription = "subscribe-retained";
4871        let server = TestServer::setup().await;
4872        let config = WebSocketConfig::builder()
4873            .url(format!("ws://127.0.0.1:{}", server.port))
4874            .headers(vec![("test".to_string(), "test".to_string())])
4875            .backend(TransportBackend::Tungstenite)
4876            .reconnect_delay_initial_ms(50)
4877            .reconnect_jitter_ms(0)
4878            .build()
4879            .unwrap();
4880        let (handler, mut raw_rx) = channel_epoch_message_handler();
4881
4882        let ws_client = if epoch_aware {
4883            WebSocketClient::epoch_builder()
4884                .config(config)
4885                .epoch_handler(handler)
4886                .connect()
4887                .await
4888        } else {
4889            WebSocketClient::builder()
4890                .config(config)
4891                .message_handler(Arc::new(move |message| handler(0, message)))
4892                .connect()
4893                .await
4894        }
4895        .unwrap();
4896
4897        let ws_client = Arc::new(ws_client);
4898        let (bytes_tx, mut bytes_rx) = tokio::sync::mpsc::unbounded_channel();
4899        let forwarding_client = Arc::clone(&ws_client);
4900
4901        let forwarding_task = tokio::spawn(async move {
4902            while let Some((epoch, message)) = raw_rx.recv().await {
4903                let data = match message {
4904                    WsMessage::Text(text) => text.as_bytes().to_vec(),
4905                    WsMessage::Binary(data) => data.to_vec(),
4906                    WsMessage::Close(_) => break,
4907                    _ => continue,
4908                };
4909
4910                if data == RECONNECTED.as_bytes() {
4911                    forwarding_client
4912                        .send_text(subscription.into(), None)
4913                        .await
4914                        .unwrap();
4915                }
4916
4917                if bytes_tx.send((epoch, data)).is_err() {
4918                    break;
4919                }
4920            }
4921        });
4922
4923        ws_client
4924            .send_text(subscription.into(), None)
4925            .await
4926            .unwrap();
4927        let initial = tokio::time::timeout(timeout, bytes_rx.recv())
4928            .await
4929            .expect("initial subscription response timed out")
4930            .expect("forwarding task ended before peer close");
4931
4932        ws_client.send_text("close-now".into(), None).await.unwrap();
4933        let reconnected = tokio::time::timeout(timeout, bytes_rx.recv())
4934            .await
4935            .expect("reconnect notification timed out")
4936            .expect("peer close terminated the forwarding task");
4937        let replayed = tokio::time::timeout(timeout, bytes_rx.recv())
4938            .await
4939            .expect("subscription replay response timed out")
4940            .expect("forwarding task ended before subscription replay");
4941        let payload = vec![3, 7, 11];
4942        ws_client.send_bytes(payload.clone(), None).await.unwrap();
4943        let live = tokio::time::timeout(timeout, bytes_rx.recv())
4944            .await
4945            .expect("replacement connection data timed out")
4946            .expect("forwarding task ended before replacement data");
4947
4948        let expected_epoch = u64::from(epoch_aware);
4949        assert_eq!(initial, (0, subscription.as_bytes().to_vec()));
4950        assert_eq!(
4951            reconnected,
4952            (expected_epoch, RECONNECTED.as_bytes().to_vec())
4953        );
4954        assert_eq!(replayed, (expected_epoch, subscription.as_bytes().to_vec()));
4955        assert_eq!(live, (expected_epoch, payload));
4956        assert_eq!(ws_client.connection_epoch(), 1);
4957        assert!(!forwarding_task.is_finished());
4958
4959        ws_client.disconnect().await;
4960        forwarding_task.abort();
4961    }
4962
4963    #[rstest]
4964    #[case(true)]
4965    #[case(false)]
4966    #[tokio::test]
4967    async fn test_peer_close_frame_is_forwarded_to_close_handler(#[case] reconnect: bool) {
4968        let server = TestServer::setup().await;
4969        let (handler, mut handler_rx) = channel_message_handler();
4970        let (state_tx, mut state_rx) = tokio::sync::mpsc::unbounded_channel();
4971
4972        let client = WebSocketClient::builder()
4973            .config(WebSocketConfig {
4974                url: format!("ws://127.0.0.1:{}", server.port),
4975                headers: vec![("test".into(), "test".into())],
4976                heartbeat_interval_secs: None,
4977                heartbeat_payload: None,
4978                connect_timeout_ms: Some(1_000),
4979                reconnect_delay_initial_ms: Some(1_000),
4980                reconnect_backoff_factor: None,
4981                reconnect_delay_max_ms: None,
4982                reconnect_jitter_ms: Some(0),
4983                reconnect_max_attempts: None,
4984                heartbeat_timeout_secs: None,
4985                idle_timeout_ms: None,
4986                writer_capacity: None,
4987                backend: TransportBackend::Tungstenite,
4988                proxy_url: None,
4989                max_message_size_bytes: None,
4990                max_frame_size_bytes: None,
4991            })
4992            .message_handler(Arc::new(|_| {}))
4993            .state_sink(SocketStateSink::new(move |state| {
4994                state_tx.send(state).expect("state channel closed");
4995            }))
4996            .close_handler(Arc::new(move |frame| {
4997                handler(Message::Close(frame));
4998                reconnect
4999            }))
5000            .connect()
5001            .await
5002            .expect("Failed to connect");
5003
5004        let tracker = AuthTracker::new();
5005        let pending_auth = tracker.begin();
5006        client.set_auth_tracker(tracker.clone(), false);
5007
5008        client
5009            .send_text(CODED_CLOSE_TRIGGER.to_string(), None)
5010            .await
5011            .unwrap();
5012
5013        let message = tokio::time::timeout(Duration::from_secs(10), handler_rx.recv())
5014            .await
5015            .expect("close frame was not forwarded")
5016            .expect("handler channel closed");
5017
5018        let WsMessage::Close(Some(frame)) = message else {
5019            panic!("expected a close frame, was {message:?}");
5020        };
5021
5022        let connected = state_rx.recv().await.expect("initial socket state");
5023        let disconnected = tokio::time::timeout(Duration::from_secs(5), state_rx.recv())
5024            .await
5025            .expect("peer close must publish socket loss")
5026            .expect("state channel closed");
5027
5028        assert_eq!(u16::from(frame.code), CODED_CLOSE_CODE);
5029        assert_eq!(frame.reason.as_str(), CODED_CLOSE_REASON);
5030        assert_eq!(connected, SocketState::Connected);
5031        assert_eq!(disconnected, SocketState::Disconnected);
5032
5033        if !reconnect {
5034            let auth_result = tokio::time::timeout(Duration::from_secs(1), pending_auth)
5035                .await
5036                .expect("terminal close should resolve pending authentication")
5037                .expect("authentication sender should report its terminal result");
5038
5039            assert_eq!(auth_result, Err("WebSocket client closed".to_string()));
5040            assert_eq!(tracker.auth_state(), AuthState::Failed);
5041        }
5042
5043        client.disconnect().await;
5044    }
5045
5046    #[rstest]
5047    #[case(true, ConnectionMode::Active)]
5048    #[case(false, ConnectionMode::Closed)]
5049    fn test_peer_close_callback_controls_reconnect(
5050        #[case] reconnect: bool,
5051        #[case] expected_mode: ConnectionMode,
5052    ) {
5053        let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5054        let fence = ReadSessionFence::new();
5055        let received = Arc::new(Mutex::new(Vec::new()));
5056        let states = Arc::new(Mutex::new(Vec::new()));
5057        let sink_states = Arc::clone(&states);
5058        let sink = SocketStateSink::new(move |state| sink_states.lock().push(state));
5059        let callback_received = Arc::clone(&received);
5060        let callback: CloseHandler = Arc::new(move |frame| {
5061            callback_received.lock().push(frame);
5062            reconnect
5063        });
5064
5065        let frame = CloseFrame::new(CODED_CLOSE_CODE, CODED_CLOSE_REASON);
5066
5067        WebSocketClientInner::dispatch_close_frame(
5068            &mode,
5069            &fence,
5070            Some(&callback),
5071            Some(&sink),
5072            Some(frame.clone()),
5073        );
5074
5075        assert_eq!(*received.lock(), vec![Some(frame)]);
5076        assert_eq!(ConnectionMode::from_atomic(&mode), expected_mode);
5077        assert_eq!(
5078            *states.lock(),
5079            if reconnect {
5080                Vec::new()
5081            } else {
5082                vec![SocketState::Disconnected]
5083            },
5084        );
5085    }
5086
5087    #[rstest]
5088    fn test_peer_close_callback_ignores_retired_reader() {
5089        let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5090        let fence = ReadSessionFence::new();
5091        fence.invalidate();
5092        let calls = Arc::new(AtomicUsize::new(0));
5093        let callback_calls = Arc::clone(&calls);
5094        let callback: CloseHandler = Arc::new(move |_| {
5095            callback_calls.fetch_add(1, Ordering::SeqCst);
5096            false
5097        });
5098
5099        WebSocketClientInner::dispatch_close_frame(&mode, &fence, Some(&callback), None, None);
5100
5101        assert_eq!(calls.load(Ordering::SeqCst), 0);
5102        assert_eq!(ConnectionMode::from_atomic(&mode), ConnectionMode::Active);
5103    }
5104
5105    // Server-controlled reasons can contain credentials and forged log lines
5106    #[rstest]
5107    #[tokio::test]
5108    async fn test_close_frame_reason_is_redacted_in_logs() {
5109        let server = TestServer::setup().await;
5110        let client = setup_test_client(server.port).await;
5111        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5112
5113        client
5114            .send_text(HOSTILE_CLOSE_TRIGGER.to_string(), None)
5115            .await
5116            .unwrap();
5117
5118        tokio::time::timeout(Duration::from_secs(5), async {
5119            loop {
5120                if capture.messages().iter().any(|(_, message)| {
5121                    message.starts_with("Received close frame, terminating: code=")
5122                }) {
5123                    break;
5124                }
5125
5126                tokio::task::yield_now().await;
5127            }
5128        })
5129        .await
5130        .expect("timed out waiting for the close frame log");
5131
5132        let close_frame_logs: Vec<String> = capture
5133            .messages()
5134            .iter()
5135            .map(|(_, message)| message.clone())
5136            .filter(|message| message.starts_with("Received close frame, terminating: code="))
5137            .collect();
5138
5139        let expected = "Received close frame, terminating: code=1011, reason=<redacted>";
5140        assert!(
5141            close_frame_logs.iter().any(|message| message == expected),
5142            "close frame reason was not redacted: {close_frame_logs:?}"
5143        );
5144        assert!(
5145            close_frame_logs.iter().all(|message| {
5146                !message.contains('\r') && !message.contains('\n') && !message.contains('\u{1b}')
5147            }),
5148            "close frame log forged line breaks or escape sequences: {close_frame_logs:?}"
5149        );
5150
5151        assert!(
5152            close_frame_logs
5153                .iter()
5154                .all(|message| !message.contains("injected") && !message.contains("forged"))
5155        );
5156
5157        client.disconnect().await;
5158    }
5159
5160    #[rstest]
5161    #[tokio::test]
5162    async fn test_state_sink_reports_initial_loss_and_recovery() {
5163        let server = TestServer::setup().await;
5164
5165        let config = WebSocketConfig {
5166            url: format!("ws://127.0.0.1:{}", server.port),
5167            headers: vec![("test".into(), "test".into())],
5168            heartbeat_interval_secs: None,
5169            heartbeat_payload: None,
5170            connect_timeout_ms: Some(1_000),
5171            reconnect_delay_initial_ms: Some(1),
5172            reconnect_backoff_factor: Some(1.0),
5173            reconnect_delay_max_ms: Some(1),
5174            reconnect_jitter_ms: Some(0),
5175            reconnect_max_attempts: Some(3),
5176            heartbeat_timeout_secs: None,
5177            idle_timeout_ms: None,
5178            writer_capacity: None,
5179            backend: TransportBackend::Tungstenite,
5180            proxy_url: None,
5181            max_message_size_bytes: None,
5182            max_frame_size_bytes: None,
5183        };
5184
5185        let states = Arc::new(Mutex::new(Vec::new()));
5186        let states_callback = Arc::clone(&states);
5187        let sink = SocketStateSink::new(move |state| {
5188            states_callback.lock().push(state);
5189        });
5190
5191        let client = WebSocketClient::builder()
5192            .config(config)
5193            .message_handler(Arc::new(|_| {}))
5194            .state_sink(sink)
5195            .connect()
5196            .await
5197            .unwrap();
5198
5199        assert_eq!(*states.lock(), vec![SocketState::Connected]);
5200
5201        client.send_text("close-now".into(), None).await.unwrap();
5202        wait_until_async(
5203            || {
5204                let states = Arc::clone(&states);
5205                async move { states.lock().len() == 3 }
5206            },
5207            Duration::from_secs(5),
5208        )
5209        .await;
5210
5211        assert_eq!(
5212            *states.lock(),
5213            vec![
5214                SocketState::Connected,
5215                SocketState::Disconnected,
5216                SocketState::Connected,
5217            ]
5218        );
5219
5220        client.disconnect().await;
5221        assert_eq!(states.lock().len(), 3);
5222    }
5223
5224    #[rstest]
5225    #[tokio::test]
5226    async fn test_drop_suppresses_socket_state_event() {
5227        let server = TestServer::setup().await;
5228
5229        let config = WebSocketConfig {
5230            url: format!("ws://127.0.0.1:{}", server.port),
5231            headers: vec![("test".into(), "test".into())],
5232            heartbeat_interval_secs: None,
5233            heartbeat_payload: None,
5234            connect_timeout_ms: Some(1_000),
5235            reconnect_delay_initial_ms: Some(1),
5236            reconnect_backoff_factor: Some(1.0),
5237            reconnect_delay_max_ms: Some(1),
5238            reconnect_jitter_ms: Some(0),
5239            reconnect_max_attempts: Some(3),
5240            heartbeat_timeout_secs: None,
5241            idle_timeout_ms: None,
5242            writer_capacity: None,
5243            backend: TransportBackend::Tungstenite,
5244            proxy_url: None,
5245            max_message_size_bytes: None,
5246            max_frame_size_bytes: None,
5247        };
5248
5249        let states = Arc::new(Mutex::new(Vec::new()));
5250        let states_callback = Arc::clone(&states);
5251        let sink = SocketStateSink::new(move |state| {
5252            states_callback.lock().push(state);
5253        });
5254
5255        let client = WebSocketClient::builder()
5256            .config(config)
5257            .message_handler(Arc::new(|_| {}))
5258            .state_sink(sink)
5259            .connect()
5260            .await
5261            .unwrap();
5262
5263        drop(client);
5264        crate::dst::time::sleep(Duration::from_millis(25)).await;
5265
5266        assert_eq!(*states.lock(), vec![SocketState::Connected]);
5267    }
5268
5269    #[rstest]
5270    #[tokio::test]
5271    async fn test_stream_state_sink_reports_reader_loss() {
5272        let server = TestServer::setup().await;
5273
5274        let config = WebSocketConfig {
5275            url: format!("ws://127.0.0.1:{}", server.port),
5276            headers: vec![("test".into(), "test".into())],
5277            heartbeat_interval_secs: None,
5278            heartbeat_payload: None,
5279            connect_timeout_ms: None,
5280            reconnect_delay_initial_ms: None,
5281            reconnect_backoff_factor: None,
5282            reconnect_delay_max_ms: None,
5283            reconnect_jitter_ms: None,
5284            reconnect_max_attempts: None,
5285            heartbeat_timeout_secs: None,
5286            idle_timeout_ms: None,
5287            writer_capacity: None,
5288            backend: TransportBackend::Tungstenite,
5289            proxy_url: None,
5290            max_message_size_bytes: None,
5291            max_frame_size_bytes: None,
5292        };
5293
5294        let states = Arc::new(Mutex::new(Vec::new()));
5295        let states_callback = Arc::clone(&states);
5296        let sink = SocketStateSink::new(move |state| {
5297            states_callback.lock().push(state);
5298        });
5299
5300        let (_reader, client) = WebSocketClient::stream_builder()
5301            .config(config)
5302            .state_sink(sink)
5303            .connect()
5304            .await
5305            .unwrap();
5306
5307        client.notify_closed();
5308
5309        assert!(client.is_closed());
5310        assert_eq!(
5311            *states.lock(),
5312            vec![SocketState::Connected, SocketState::Disconnected]
5313        );
5314    }
5315
5316    #[rstest]
5317    #[tokio::test]
5318    async fn test_state_sink_emits_no_retry_or_exhaustion_events() {
5319        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5320        let port = listener.local_addr().unwrap().port();
5321        let server_task = task::spawn(async move {
5322            let (connection, _) = listener.accept().await.unwrap();
5323            let mut websocket = accept_async(connection).await.unwrap();
5324            while let Some(Ok(message)) = websocket.next().await {
5325                if matches!(&message, WsMessage::Text(text) if text.as_str() == "close-now") {
5326                    websocket.close(None).await.unwrap();
5327                    break;
5328                }
5329            }
5330        });
5331
5332        let config = WebSocketConfig {
5333            url: format!("ws://127.0.0.1:{port}"),
5334            headers: vec![],
5335            heartbeat_interval_secs: None,
5336            heartbeat_payload: None,
5337            connect_timeout_ms: Some(100),
5338            reconnect_delay_initial_ms: Some(1),
5339            reconnect_backoff_factor: Some(1.0),
5340            reconnect_delay_max_ms: Some(1),
5341            reconnect_jitter_ms: Some(0),
5342            reconnect_max_attempts: Some(2),
5343            heartbeat_timeout_secs: None,
5344            idle_timeout_ms: None,
5345            writer_capacity: None,
5346            backend: TransportBackend::Tungstenite,
5347            proxy_url: None,
5348            max_message_size_bytes: None,
5349            max_frame_size_bytes: None,
5350        };
5351
5352        let states = Arc::new(Mutex::new(Vec::new()));
5353        let states_callback = Arc::clone(&states);
5354        let sink = SocketStateSink::new(move |state| {
5355            states_callback.lock().push(state);
5356        });
5357
5358        let client = WebSocketClient::builder()
5359            .config(config)
5360            .message_handler(Arc::new(|_| {}))
5361            .state_sink(sink)
5362            .connect()
5363            .await
5364            .unwrap();
5365
5366        client.send_text("close-now".into(), None).await.unwrap();
5367        wait_until_async(
5368            || async { client.is_disconnected() },
5369            Duration::from_secs(5),
5370        )
5371        .await;
5372        assert_eq!(
5373            *states.lock(),
5374            vec![SocketState::Connected, SocketState::Disconnected]
5375        );
5376
5377        server_task.await.unwrap();
5378    }
5379
5380    #[rstest]
5381    #[case(
5382        "invalid header",
5383        "value",
5384        "Invalid WebSocket reconnect header name: invalid HTTP header name"
5385    )]
5386    #[case(
5387        "x-test",
5388        "invalid\nvalue",
5389        "Invalid WebSocket reconnect header value: failed to parse header value"
5390    )]
5391    fn test_reconnect_headers_reject_invalid_update(
5392        #[case] name: &str,
5393        #[case] value: &str,
5394        #[case] expected_message: &str,
5395    ) {
5396        let initial = vec![("x-existing".to_string(), "initial".to_string())];
5397        let headers = ReconnectHeaders::new(initial.clone());
5398
5399        let error = headers.update(name, value).unwrap_err();
5400
5401        let TransportError::Io(error) = error else {
5402            panic!("expected I/O error, was {error:?}");
5403        };
5404
5405        assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
5406        assert_eq!(error.to_string(), expected_message);
5407        assert_eq!(headers.snapshot(), initial);
5408    }
5409
5410    #[tokio::test]
5411    #[allow(clippy::result_large_err)]
5412    async fn test_reconnect_uses_updated_headers_without_interrupting_active_connection() {
5413        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5414        let port = listener.local_addr().unwrap().port();
5415        let (header_tx, mut header_rx) = tokio::sync::mpsc::unbounded_channel();
5416
5417        let server_task = task::spawn(async move {
5418            loop {
5419                let (conn, _) = listener.accept().await.unwrap();
5420                let header_tx = header_tx.clone();
5421
5422                let mut websocket = accept_hdr_async(
5423                    conn,
5424                    move |request: &server::Request, response: server::Response| {
5425                        let values = request
5426                            .headers()
5427                            .get_all("authorization")
5428                            .iter()
5429                            .map(|value| value.to_str().unwrap().to_string())
5430                            .collect::<Vec<_>>();
5431                        header_tx.send(values).unwrap();
5432                        Ok(response)
5433                    },
5434                )
5435                .await
5436                .unwrap();
5437
5438                task::spawn(async move {
5439                    while let Some(Ok(msg)) = websocket.next().await {
5440                        if matches!(&msg, WsMessage::Text(text) if text.as_str() == "close-now") {
5441                            let _ = websocket.close(None).await;
5442                            break;
5443                        }
5444                    }
5445                });
5446            }
5447        });
5448
5449        let config = WebSocketConfig {
5450            url: format!("ws://127.0.0.1:{port}"),
5451            headers: vec![("Authorization".into(), "Bearer initial".into())],
5452            heartbeat_interval_secs: None,
5453            heartbeat_payload: None,
5454            connect_timeout_ms: Some(1_000),
5455            reconnect_delay_initial_ms: Some(50),
5456            reconnect_delay_max_ms: Some(50),
5457            reconnect_backoff_factor: Some(1.0),
5458            reconnect_jitter_ms: Some(0),
5459            reconnect_max_attempts: None,
5460            heartbeat_timeout_secs: None,
5461            idle_timeout_ms: None,
5462            writer_capacity: None,
5463            backend: TransportBackend::Tungstenite,
5464            proxy_url: None,
5465            max_message_size_bytes: None,
5466            max_frame_size_bytes: None,
5467        };
5468
5469        let client = WebSocketClient::builder()
5470            .config(config)
5471            .message_handler(Arc::new(|_| {}))
5472            .connect()
5473            .await
5474            .unwrap();
5475
5476        let initial = header_rx.recv().await.unwrap();
5477        let reconnect_headers = client.reconnect_headers();
5478        reconnect_headers
5479            .update("authorization", "Bearer refreshed")
5480            .unwrap();
5481
5482        tokio::time::sleep(Duration::from_millis(100)).await;
5483        assert_eq!(initial, vec!["Bearer initial"]);
5484        assert!(client.is_active());
5485        assert!(header_rx.try_recv().is_err());
5486        assert!(!format!("{reconnect_headers:?}").contains("refreshed"));
5487
5488        client.send_text("close-now".into(), None).await.unwrap();
5489        let refreshed = tokio::time::timeout(Duration::from_secs(3), header_rx.recv())
5490            .await
5491            .unwrap()
5492            .unwrap();
5493
5494        assert_eq!(refreshed, vec!["Bearer refreshed"]);
5495
5496        client.disconnect().await;
5497        server_task.abort();
5498    }
5499
5500    #[tokio::test]
5501    async fn test_rate_limiter() {
5502        let server = TestServer::setup().await;
5503        let quota = Quota::per_second(NonZeroU32::new(2).unwrap()).unwrap();
5504
5505        let config = WebSocketConfig {
5506            url: format!("ws://127.0.0.1:{}", server.port),
5507            headers: vec![("test".into(), "test".into())],
5508            heartbeat_interval_secs: None,
5509            heartbeat_payload: None,
5510            connect_timeout_ms: None,
5511            reconnect_delay_initial_ms: None,
5512            reconnect_backoff_factor: None,
5513            reconnect_delay_max_ms: None,
5514            reconnect_jitter_ms: None,
5515            reconnect_max_attempts: None,
5516            heartbeat_timeout_secs: None,
5517            idle_timeout_ms: None,
5518            writer_capacity: None,
5519            backend: TransportBackend::Tungstenite,
5520            proxy_url: None,
5521            max_message_size_bytes: None,
5522            max_frame_size_bytes: None,
5523        };
5524
5525        let client = WebSocketClient::builder()
5526            .config(config)
5527            .message_handler(Arc::new(|_| {}))
5528            .keyed_quotas(vec![("default".into(), quota)])
5529            .connect()
5530            .await
5531            .unwrap();
5532
5533        // Burst of 2 passes immediately; the third send must wait for the
5534        // ~500ms replenish interval (keys=None would bypass the limiter)
5535        let keys: [ustr::Ustr; 1] = [ustr::Ustr::from("default")];
5536        let start = std::time::Instant::now();
5537        client
5538            .send_text("test1".into(), Some(keys.as_slice()))
5539            .await
5540            .unwrap();
5541        client
5542            .send_text("test2".into(), Some(keys.as_slice()))
5543            .await
5544            .unwrap();
5545        let after_burst = start.elapsed();
5546        client
5547            .send_text("test3".into(), Some(keys.as_slice()))
5548            .await
5549            .unwrap();
5550        let after_third = start.elapsed();
5551
5552        assert!(
5553            after_burst < std::time::Duration::from_millis(300),
5554            "Burst sends should not be rate limited, took {after_burst:?}"
5555        );
5556        assert!(
5557            after_third >= std::time::Duration::from_millis(400),
5558            "Third send should wait for quota replenishment, took {after_third:?}"
5559        );
5560
5561        // Cleanup
5562        client.disconnect().await;
5563        assert!(client.is_disconnected());
5564    }
5565
5566    #[tokio::test]
5567    async fn test_concurrent_writers() {
5568        let server = TestServer::setup().await;
5569        let client = Arc::new(setup_test_client(server.port).await);
5570
5571        let mut handles = vec![];
5572
5573        for i in 0..10 {
5574            let client = Arc::clone(&client);
5575            handles.push(task::spawn(async move {
5576                client.send_text(format!("test{i}"), None).await.unwrap();
5577            }));
5578        }
5579
5580        for handle in handles {
5581            handle.await.unwrap();
5582        }
5583
5584        // Cleanup
5585        client.disconnect().await;
5586        assert!(client.is_disconnected());
5587    }
5588}
5589
5590#[cfg(test)]
5591#[cfg(not(feature = "turmoil"))]
5592#[cfg(not(all(feature = "simulation", madsim)))] // These tests drive real localhost sockets
5593mod rust_tests {
5594    use std::{
5595        pin::Pin,
5596        sync::{
5597            Arc, OnceLock,
5598            atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
5599        },
5600        task::{Context, Poll},
5601    };
5602
5603    use futures_util::{SinkExt, StreamExt};
5604    #[cfg(target_os = "linux")]
5605    use log::Level;
5606    use nautilus_common::testing::wait_until_async;
5607    use parking_lot::{Condvar, Mutex};
5608    use rstest::rstest;
5609    #[cfg(feature = "transport-sockudo")]
5610    use sockudo_ws::handshake as sockudo_handshake;
5611    #[cfg(feature = "transport-sockudo")]
5612    use tokio::io::AsyncRead;
5613    use tokio::{
5614        io::{AsyncReadExt, AsyncWriteExt},
5615        net::TcpListener,
5616        sync::oneshot,
5617        task::{self, JoinHandle},
5618        time::{Duration, sleep},
5619    };
5620    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
5621    #[cfg(feature = "transport-sockudo")]
5622    use tokio_tungstenite::{
5623        accept_hdr_async,
5624        tungstenite::{
5625            handshake::server::{self, Callback},
5626            http::HeaderValue,
5627        },
5628    };
5629
5630    use super::*;
5631    #[cfg(target_os = "linux")]
5632    use crate::logging::tests::capture_logs_for;
5633    use crate::{
5634        SocketState,
5635        websocket::types::{channel_epoch_message_handler, channel_message_handler},
5636    };
5637
5638    const TEST_TIMEOUT: Duration = Duration::from_secs(10);
5639
5640    // A retryable upgrade rejection must stay below ERROR on both the transport that reports it
5641    // and the client that retries it.
5642    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5643    const RECONNECT_LOG_TARGETS: &[&str] = &[
5644        "nautilus_network::transport::sockudo",
5645        "nautilus_network::websocket::client",
5646    ];
5647
5648    #[cfg(target_os = "linux")]
5649    #[rstest]
5650    #[case::tungstenite(TransportBackend::Tungstenite)]
5651    #[cfg_attr(
5652        feature = "transport-sockudo",
5653        case::sockudo(TransportBackend::Sockudo)
5654    )]
5655    #[tokio::test]
5656    async fn test_initial_connect_redacts_error_text(#[case] backend: TransportBackend) {
5657        let capture = capture_logs_for(&["nautilus_network::retry"]).await;
5658        let socket = tokio::net::TcpSocket::new_v4().unwrap();
5659        socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
5660        let mut config = reconnect_test_config(socket.local_addr().unwrap().port());
5661        config.backend = backend;
5662        let error = WebSocketClient::builder()
5663            .config(config)
5664            .message_handler(Arc::new(|_| {}))
5665            .connect()
5666            .await
5667            .unwrap_err();
5668        let messages = capture.messages();
5669        let raw_message = error.to_string();
5670        let redacted_message = error.redacted_message();
5671
5672        assert!(
5673            matches!(error, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::ConnectionRefused)
5674        );
5675        assert_ne!(raw_message, redacted_message);
5676        assert_eq!(redacted_message, "I/O error: ConnectionRefused");
5677        assert!(messages.contains(&(
5678            Level::Trace,
5679            format!(
5680                "Operation '{INITIAL_CONNECT_OPERATION}' retries exhausted after 1 attempts: {redacted_message}"
5681            ),
5682        )), "{messages:?}");
5683        assert!(
5684            messages
5685                .iter()
5686                .all(|(_, message)| !message.contains(&raw_message)),
5687            "{messages:?}"
5688        );
5689    }
5690
5691    #[cfg(target_os = "linux")]
5692    #[rstest]
5693    #[case::tungstenite(TransportBackend::Tungstenite)]
5694    #[cfg_attr(
5695        feature = "transport-sockudo",
5696        case::sockudo(TransportBackend::Sockudo)
5697    )]
5698    #[tokio::test]
5699    async fn test_reconnect_redacts_error_text(#[case] backend: TransportBackend) {
5700        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5701        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5702        let port = listener.local_addr().unwrap().port();
5703        let (close_tx, close_rx) = oneshot::channel();
5704
5705        let server = tokio::spawn(async move {
5706            let (stream, _) = listener.accept().await.unwrap();
5707            let ws_client = accept_async(stream).await.unwrap();
5708            close_rx.await.unwrap();
5709            drop(ws_client);
5710            let (_stream, _) = listener.accept().await.unwrap();
5711            std::future::pending::<()>().await;
5712        });
5713
5714        let mut config = reconnect_test_config(port);
5715        config.backend = backend;
5716        config.reconnect_max_attempts = Some(1);
5717        config.reconnect_delay_initial_ms = Some(1);
5718        config.reconnect_delay_max_ms = Some(1);
5719        config.reconnect_jitter_ms = Some(0);
5720        let client = WebSocketClient::builder()
5721            .config(config)
5722            .message_handler(Arc::new(|_| {}))
5723            .connect()
5724            .await
5725            .unwrap();
5726        close_tx.send(()).unwrap();
5727
5728        wait_until_async(
5729            || async {
5730                capture.messages().iter().any(|(level, message)| {
5731                    *level == Level::Warn && message.starts_with("Reconnect attempt 1 failed:")
5732                })
5733            },
5734            TEST_TIMEOUT,
5735        )
5736        .await;
5737
5738        client.disconnect().await;
5739        server.abort();
5740        await_task_termination(server, "redaction peer").await;
5741        let messages = capture.messages();
5742        let warnings: Vec<_> = messages
5743            .iter()
5744            .filter(|(_, message)| message.starts_with("Reconnect attempt "))
5745            .cloned()
5746            .collect();
5747
5748        assert_eq!(
5749            warnings,
5750            vec![(
5751                Level::Warn,
5752                "Reconnect attempt 1 failed: I/O error: TimedOut".into()
5753            )],
5754        );
5755    }
5756
5757    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5758    #[tokio::test]
5759    async fn test_sockudo_handshake_trace_redacts_authentication() {
5760        let capture = capture_logs_for(&[
5761            "tungstenite::handshake::client",
5762            "nautilus_network::transport::sockudo",
5763            "nautilus_network::retry",
5764            "nautilus_network::websocket::client",
5765        ])
5766        .await;
5767        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5768        let addr = listener.local_addr().unwrap();
5769        let peer = tokio::spawn(async move {
5770            let (mut stream, _) = listener.accept().await.unwrap();
5771            let request = read_http_request(&mut stream).await;
5772            stream
5773                .write_all(
5774                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
5775                )
5776                .await
5777                .unwrap();
5778            stream.flush().await.unwrap();
5779            request
5780        });
5781
5782        let config = WebSocketConfig::builder()
5783            .url(format!(
5784                "ws://{addr}/TRACE_PATH_SECRET?token=TRACE_QUERY_SECRET"
5785            ))
5786            .headers(vec![
5787                ("KALSHI-ACCESS-KEY".into(), "TRACE_KEY_SECRET".into()),
5788                (
5789                    "KALSHI-ACCESS-SIGNATURE".into(),
5790                    "TRACE_SIGNATURE_SECRET".into(),
5791                ),
5792            ])
5793            .backend(TransportBackend::Sockudo)
5794            .build()
5795            .unwrap();
5796        let error = WebSocketClient::builder()
5797            .config(config)
5798            .message_handler(Arc::new(|_| {}))
5799            .connect()
5800            .await
5801            .unwrap_err();
5802        let request = String::from_utf8(peer.await.unwrap()).unwrap();
5803        let messages = capture.messages();
5804
5805        assert!(matches!(error, TransportError::UpgradeRejected(503)));
5806
5807        for secret in [
5808            "TRACE_KEY_SECRET",
5809            "TRACE_SIGNATURE_SECRET",
5810            "TRACE_PATH_SECRET",
5811            "TRACE_QUERY_SECRET",
5812        ] {
5813            assert!(request.contains(secret));
5814            assert!(
5815                messages
5816                    .iter()
5817                    .all(|(_, message)| !message.contains(secret)),
5818                "{messages:?}"
5819            );
5820        }
5821
5822        assert!(
5823            messages
5824                .iter()
5825                .any(|(level, message)| *level == log::Level::Trace
5826                    && message.contains("retries exhausted after 1 attempts")),
5827            "{messages:?}"
5828        );
5829    }
5830
5831    struct CondvarReleaseGuard<'a> {
5832        release: &'a (Mutex<bool>, Condvar),
5833    }
5834
5835    impl<'a> CondvarReleaseGuard<'a> {
5836        fn new(release: &'a (Mutex<bool>, Condvar)) -> Self {
5837            Self { release }
5838        }
5839
5840        fn release(&self) {
5841            let (lock, condvar) = self.release;
5842            let mut released = lock.lock();
5843            *released = true;
5844            condvar.notify_all();
5845        }
5846    }
5847
5848    impl Drop for CondvarReleaseGuard<'_> {
5849        fn drop(&mut self) {
5850            self.release();
5851        }
5852    }
5853
5854    async fn recv_rendezvous<T: Send + 'static>(
5855        receiver: std::sync::mpsc::Receiver<T>,
5856        name: &'static str,
5857    ) -> T {
5858        let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
5859
5860        match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
5861            Ok(Ok(Ok(value))) => value,
5862            Ok(Ok(Err(e))) => {
5863                panic!("{name} did not arrive within the test timeout: {e}")
5864            }
5865            Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
5866            Err(e) => panic!("{name} receive task did not finish: {e}"),
5867        }
5868    }
5869
5870    async fn await_task_termination(task: tokio::task::JoinHandle<()>, name: &'static str) {
5871        match tokio::time::timeout(TEST_TIMEOUT, task).await {
5872            Ok(Ok(())) => {}
5873            Ok(Err(e)) if e.is_cancelled() => {}
5874            Ok(Err(e)) => panic!("{name} failed: {e}"),
5875            Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
5876        }
5877    }
5878
5879    fn reconnect_test_config(port: u16) -> WebSocketConfig {
5880        WebSocketConfig {
5881            url: format!("ws://127.0.0.1:{port}"),
5882            headers: vec![],
5883            heartbeat_interval_secs: None,
5884            heartbeat_payload: None,
5885            connect_timeout_ms: Some(1_000),
5886            reconnect_delay_initial_ms: None,
5887            reconnect_delay_max_ms: None,
5888            reconnect_backoff_factor: None,
5889            reconnect_jitter_ms: None,
5890            reconnect_max_attempts: None,
5891            heartbeat_timeout_secs: None,
5892            idle_timeout_ms: None,
5893            writer_capacity: None,
5894            backend: TransportBackend::Tungstenite,
5895            proxy_url: None,
5896            max_message_size_bytes: None,
5897            max_frame_size_bytes: None,
5898        }
5899    }
5900
5901    fn initial_connect_retry_policy(
5902        max_attempts: u32,
5903        initial_delay_ms: u64,
5904    ) -> InitialConnectRetryPolicy {
5905        InitialConnectRetryPolicy {
5906            max_attempts: std::num::NonZeroU32::new(max_attempts).unwrap(),
5907            delay_initial: Duration::from_millis(initial_delay_ms),
5908            delay_max: Duration::from_millis(initial_delay_ms),
5909            backoff_factor: 2.0,
5910            jitter_ms: 0,
5911        }
5912    }
5913
5914    #[rstest]
5915    #[case(TransportError::UpgradeRejected(408), true)]
5916    #[case(TransportError::UpgradeRejected(425), true)]
5917    #[case(TransportError::UpgradeRejected(429), true)]
5918    #[case(TransportError::UpgradeRejected(500), true)]
5919    #[case(TransportError::UpgradeRejected(599), true)]
5920    #[case(TransportError::UpgradeRejected(404), false)]
5921    #[case(TransportError::UpgradeRejected(600), false)]
5922    #[case(TransportError::ProxyConnectRejected(429), true)]
5923    #[case(TransportError::ProxyConnectRejected(407), false)]
5924    #[case(TransportError::ConnectionClosed, true)]
5925    #[case(
5926        TransportError::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
5927        true
5928    )]
5929    #[case(
5930        TransportError::Io(std::io::Error::from(std::io::ErrorKind::Interrupted)),
5931        true
5932    )]
5933    #[case(
5934        TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidInput)),
5935        false
5936    )]
5937    #[case(
5938        TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidData)),
5939        false
5940    )]
5941    #[case(
5942        TransportError::Io(std::io::Error::from(std::io::ErrorKind::Unsupported)),
5943        false
5944    )]
5945    #[case(
5946        TransportError::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
5947        false
5948    )]
5949    #[case(TransportError::Handshake("malformed".to_string()), false)]
5950    fn initial_connect_error_classification(#[case] error: TransportError, #[case] expected: bool) {
5951        assert_eq!(is_retryable_initial_connect_error(&error), expected);
5952    }
5953
5954    async fn rejected_upgrade_attempts(
5955        status: u16,
5956        backend: TransportBackend,
5957        retry_policy: Option<InitialConnectRetryPolicy>,
5958    ) -> u64 {
5959        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5960        let port = listener.local_addr().unwrap().port();
5961        let accepted = Arc::new(AtomicU64::new(0));
5962        let accepted_server = Arc::clone(&accepted);
5963
5964        let server = tokio::spawn(async move {
5965            loop {
5966                let (mut stream, _) = listener.accept().await.unwrap();
5967                accepted_server.fetch_add(1, Ordering::SeqCst);
5968                let mut request = Vec::new();
5969
5970                loop {
5971                    let mut chunk = [0; 1024];
5972                    let n = stream.read(&mut chunk).await.unwrap();
5973                    if n == 0 {
5974                        break;
5975                    }
5976
5977                    request.extend_from_slice(&chunk[..n]);
5978                    if request.windows(4).any(|window| window == b"\r\n\r\n") {
5979                        break;
5980                    }
5981                }
5982
5983                stream
5984                    .write_all(format!("HTTP/1.1 {status}\r\n\r\n").as_bytes())
5985                    .await
5986                    .unwrap();
5987            }
5988        });
5989
5990        let (handler, _rx) = channel_message_handler();
5991        let mut config = reconnect_test_config(port);
5992        config.backend = backend;
5993        WebSocketClient::builder()
5994            .config(config)
5995            .message_handler(handler)
5996            .maybe_initial_connect_retry_policy(retry_policy)
5997            .connect()
5998            .await
5999            .expect_err("server always rejects the upgrade");
6000
6001        let attempts = accepted.load(Ordering::SeqCst);
6002        server.abort();
6003        attempts
6004    }
6005
6006    async fn closed_proxy_attempts() -> u64 {
6007        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6008        let proxy_addr = listener.local_addr().unwrap();
6009        let accepted = Arc::new(AtomicU64::new(0));
6010        let accepted_server = Arc::clone(&accepted);
6011
6012        let server = tokio::spawn(async move {
6013            loop {
6014                let (mut stream, _) = listener.accept().await.unwrap();
6015                accepted_server.fetch_add(1, Ordering::SeqCst);
6016                let mut request = [0; 1024];
6017                let _ = stream.read(&mut request).await.unwrap();
6018            }
6019        });
6020
6021        let (handler, _rx) = channel_message_handler();
6022        let mut config = reconnect_test_config(9);
6023        config.proxy_url = Some(format!("http://{proxy_addr}"));
6024        WebSocketClient::builder()
6025            .config(config)
6026            .message_handler(handler)
6027            .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6028            .connect()
6029            .await
6030            .expect_err("proxy always closes before its CONNECT response");
6031
6032        let attempts = accepted.load(Ordering::SeqCst);
6033        server.abort();
6034        attempts
6035    }
6036
6037    #[rstest]
6038    #[tokio::test]
6039    async fn initial_connect_cancellation_interrupts_in_flight_handshake() {
6040        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6041        let port = listener.local_addr().unwrap().port();
6042        let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
6043
6044        let server = tokio::spawn(async move {
6045            let (_stream, _) = listener.accept().await.unwrap();
6046            let _ = accepted_tx.send(());
6047            std::future::pending::<()>().await;
6048        });
6049
6050        let token = CancellationToken::new();
6051        let (handler, _rx) = channel_message_handler();
6052        let mut config = reconnect_test_config(port);
6053        config.connect_timeout_ms = Some(10_000);
6054        let connect = WebSocketClient::builder()
6055            .config(config)
6056            .message_handler(handler)
6057            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6058            .cancellation_token(token.clone())
6059            .connect();
6060        tokio::pin!(connect);
6061
6062        tokio::select! {
6063            biased;
6064            result = &mut connect => panic!("connect completed before cancellation: {result:?}"),
6065            result = accepted_rx => result.unwrap(),
6066        }
6067        token.cancel();
6068
6069        let err = tokio::time::timeout(Duration::from_millis(250), connect)
6070            .await
6071            .expect("cancellation should interrupt the handshake")
6072            .expect_err("cancelled connect should fail");
6073        assert!(
6074            matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::Interrupted)
6075        );
6076        server.abort();
6077    }
6078
6079    #[rstest]
6080    #[tokio::test]
6081    async fn timed_out_initial_connect_attempt_is_retried() {
6082        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6083        let port = listener.local_addr().unwrap().port();
6084        let accepted = Arc::new(AtomicU64::new(0));
6085        let accepted_server = Arc::clone(&accepted);
6086
6087        // Accept the TCP connection but never complete the upgrade, so every attempt
6088        // reaches the connect timeout rather than failing fast.
6089        let server = tokio::spawn(async move {
6090            let mut held = Vec::new();
6091
6092            loop {
6093                let (stream, _) = listener.accept().await.unwrap();
6094                accepted_server.fetch_add(1, Ordering::SeqCst);
6095                held.push(stream);
6096            }
6097        });
6098
6099        let (handler, _rx) = channel_message_handler();
6100        let mut config = reconnect_test_config(port);
6101        config.connect_timeout_ms = Some(50);
6102        let err = WebSocketClient::builder()
6103            .config(config)
6104            .message_handler(handler)
6105            .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6106            .connect()
6107            .await
6108            .expect_err("every attempt times out");
6109
6110        assert!(
6111            matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::TimedOut)
6112        );
6113
6114        // The ladder must exhaust its attempts: a timeout is transient, so returning after
6115        // the first one leaves this at 1. Wait for the count rather than sampling it, since
6116        // the final connection can complete through the listen backlog before the accepting
6117        // task is scheduled to record it.
6118        let counted = tokio::time::timeout(Duration::from_secs(1), async {
6119            while accepted.load(Ordering::SeqCst) < 3 {
6120                tokio::time::sleep(Duration::from_millis(5)).await;
6121            }
6122        })
6123        .await;
6124
6125        assert!(
6126            counted.is_ok(),
6127            "expected 3 connection attempts, observed {}",
6128            accepted.load(Ordering::SeqCst)
6129        );
6130        // Exactly three: the wait above resolves the scheduling race, but the ladder must
6131        // also stop at its configured maximum rather than exceeding it.
6132        assert_eq!(accepted.load(Ordering::SeqCst), 3);
6133        server.abort();
6134    }
6135
6136    #[rstest]
6137    #[case::too_many_requests(429)]
6138    #[case::server_error(503)]
6139    #[tokio::test]
6140    async fn transient_upgrade_rejection_is_retried(#[case] status: u16) {
6141        assert_eq!(
6142            rejected_upgrade_attempts(
6143                status,
6144                TransportBackend::Tungstenite,
6145                Some(initial_connect_retry_policy(3, 1)),
6146            )
6147            .await,
6148            3
6149        );
6150    }
6151
6152    #[rstest]
6153    #[tokio::test]
6154    async fn initial_connect_without_retry_policy_makes_one_attempt() {
6155        assert_eq!(
6156            rejected_upgrade_attempts(503, TransportBackend::Tungstenite, None).await,
6157            1
6158        );
6159    }
6160
6161    #[cfg(feature = "transport-sockudo")]
6162    #[rstest]
6163    #[tokio::test]
6164    async fn sockudo_too_many_requests_upgrade_rejection_is_retried() {
6165        assert_eq!(
6166            rejected_upgrade_attempts(
6167                429,
6168                TransportBackend::Sockudo,
6169                Some(initial_connect_retry_policy(3, 1)),
6170            )
6171            .await,
6172            3
6173        );
6174    }
6175
6176    #[rstest]
6177    #[tokio::test]
6178    async fn permanent_upgrade_rejection_is_not_retried() {
6179        assert_eq!(
6180            rejected_upgrade_attempts(
6181                404,
6182                TransportBackend::Tungstenite,
6183                Some(initial_connect_retry_policy(3, 1)),
6184            )
6185            .await,
6186            1
6187        );
6188    }
6189
6190    #[rstest]
6191    #[tokio::test]
6192    async fn proxy_close_before_connect_response_is_retried() {
6193        assert_eq!(closed_proxy_attempts().await, 3);
6194    }
6195
6196    #[rstest]
6197    #[tokio::test]
6198    async fn initial_connect_cancellation_interrupts_retry_backoff() {
6199        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6200        let port = listener.local_addr().unwrap().port();
6201        let accepted = Arc::new(AtomicU64::new(0));
6202        let accepted_server = Arc::clone(&accepted);
6203        let (rejected_tx, rejected_rx) = oneshot::channel();
6204        let server = tokio::spawn(async move {
6205            let mut rejected_tx = Some(rejected_tx);
6206
6207            loop {
6208                let (mut stream, _) = listener.accept().await.unwrap();
6209                accepted_server.fetch_add(1, Ordering::SeqCst);
6210                let mut request = [0; 1024];
6211                let _ = stream.read(&mut request).await.unwrap();
6212                stream
6213                    .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
6214                    .await
6215                    .unwrap();
6216
6217                if let Some(rejected_tx) = rejected_tx.take() {
6218                    rejected_tx.send(()).unwrap();
6219                }
6220            }
6221        });
6222
6223        let token = CancellationToken::new();
6224        let (handler, _rx) = channel_message_handler();
6225        let connect = WebSocketClient::builder()
6226            .config(reconnect_test_config(port))
6227            .message_handler(handler)
6228            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6229            .cancellation_token(token.clone())
6230            .connect();
6231        tokio::pin!(connect);
6232
6233        tokio::select! {
6234            biased;
6235            result = &mut connect => panic!("connect completed before backoff: {result:?}"),
6236            result = rejected_rx => result.unwrap(),
6237        }
6238        assert!(futures_util::poll!(&mut connect).is_pending());
6239        token.cancel();
6240
6241        let error = tokio::time::timeout(Duration::from_millis(250), connect)
6242            .await
6243            .expect("cancellation should interrupt retry backoff")
6244            .expect_err("cancelled connect should fail");
6245
6246        assert!(
6247            matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6248        );
6249        assert_eq!(accepted.load(Ordering::SeqCst), 1);
6250        server.abort();
6251    }
6252
6253    #[rstest]
6254    #[tokio::test]
6255    async fn permanent_initial_connect_error_does_not_wait_for_retry_ladder() {
6256        let (handler, _rx) = channel_message_handler();
6257        let mut config = reconnect_test_config(1);
6258        config.url = "not a websocket URL".to_string();
6259        let connect = WebSocketClient::builder()
6260            .config(config)
6261            .message_handler(handler)
6262            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6263            .connect();
6264
6265        let err = tokio::time::timeout(Duration::from_millis(250), connect)
6266            .await
6267            .expect("permanent error should not enter retry backoff")
6268            .expect_err("invalid URL should fail");
6269        assert!(matches!(
6270            err,
6271            TransportError::InvalidUrl(_) | TransportError::Handshake(_)
6272        ));
6273    }
6274
6275    #[rstest]
6276    #[tokio::test(start_paused = true)]
6277    async fn connection_rate_limit_gates_initial_connect_and_reconnect() {
6278        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6279        let port = listener.local_addr().unwrap().port();
6280        let server = tokio::spawn(async move {
6281            let (stream, _) = listener.accept().await.unwrap();
6282            let _first = accept_async(stream).await.unwrap();
6283            let (stream, _) = listener.accept().await.unwrap();
6284            let _second = accept_async(stream).await.unwrap();
6285            std::future::pending::<()>().await;
6286        });
6287
6288        let key = Ustr::from("connection-attempt");
6289
6290        let limiter = Arc::new(RateLimiter::new_with_quota(
6291            None,
6292            vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6293        ));
6294        limiter.await_keys_ready(Some(&[key])).await;
6295
6296        let rate_limit = ConnectionRateLimit {
6297            limiter,
6298            keys: Arc::from([key]),
6299        };
6300
6301        let (handler, _rx) = channel_message_handler();
6302        let mut config = reconnect_test_config(port);
6303        config.connect_timeout_ms = Some(10_000);
6304        let initial = WebSocketClientInner::connect_url_with_handler(
6305            config,
6306            Some(IncomingHandler::Message(handler)),
6307            None,
6308            None,
6309            None,
6310            Some(rate_limit),
6311            InitialConnectOptions::default(),
6312        );
6313        tokio::pin!(initial);
6314        assert!(futures_util::poll!(&mut initial).is_pending());
6315        tokio::time::advance(Duration::from_secs(1)).await;
6316        tokio::task::yield_now().await;
6317
6318        // Paused time auto-advances while the runtime waits on socket I/O, which
6319        // would fire the connect timeout, so run the handshake on the real clock
6320        tokio::time::resume();
6321        let mut inner = initial.await.unwrap();
6322        tokio::time::pause();
6323
6324        inner
6325            .connection_mode
6326            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6327        let reconnect = inner.reconnect_with_outcome();
6328        tokio::pin!(reconnect);
6329        assert!(futures_util::poll!(&mut reconnect).is_pending());
6330        tokio::time::advance(Duration::from_secs(1)).await;
6331        tokio::task::yield_now().await;
6332        tokio::time::resume();
6333        assert_eq!(reconnect.await.unwrap(), ReconnectOutcome::Reconnected);
6334
6335        server.abort();
6336    }
6337
6338    #[rstest]
6339    #[tokio::test(start_paused = true)]
6340    async fn initial_connect_cancellation_interrupts_connection_rate_limit_wait() {
6341        let key = Ustr::from("connection-attempt");
6342
6343        let limiter = Arc::new(RateLimiter::new_with_quota(
6344            None,
6345            vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6346        ));
6347        limiter.await_keys_ready(Some(&[key])).await;
6348
6349        let rate_limit = ConnectionRateLimit {
6350            limiter,
6351            keys: Arc::from([key]),
6352        };
6353
6354        let token = CancellationToken::new();
6355        let (handler, _rx) = channel_message_handler();
6356
6357        let connect = WebSocketClientInner::connect_url_with_handler(
6358            reconnect_test_config(1),
6359            Some(IncomingHandler::Message(handler)),
6360            None,
6361            None,
6362            None,
6363            Some(rate_limit),
6364            InitialConnectOptions {
6365                retry_policy: None,
6366                cancellation_token: Some(token.clone()),
6367            },
6368        );
6369
6370        tokio::pin!(connect);
6371        assert!(futures_util::poll!(&mut connect).is_pending());
6372
6373        token.cancel();
6374        let error = connect.await.expect_err("cancelled connect should fail");
6375
6376        assert!(
6377            matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6378        );
6379    }
6380
6381    #[rstest]
6382    #[tokio::test]
6383    async fn test_reconnect_outcome_is_aborted_before_connect() {
6384        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6385        let port = listener.local_addr().unwrap().port();
6386        let server = tokio::spawn(async move {
6387            let (stream, _) = listener.accept().await.unwrap();
6388            let _websocket = accept_async(stream).await.unwrap();
6389            std::future::pending::<()>().await;
6390        });
6391
6392        let (handler, _rx) = channel_message_handler();
6393        let mut inner =
6394            WebSocketClientInner::connect_url(reconnect_test_config(port), Some(handler), None)
6395                .await
6396                .unwrap();
6397        inner
6398            .connection_mode
6399            .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
6400
6401        let outcome = inner.reconnect_with_outcome().await.unwrap();
6402
6403        assert_eq!(outcome, ReconnectOutcome::Aborted);
6404        server.abort();
6405    }
6406
6407    #[rstest]
6408    #[tokio::test]
6409    async fn test_stream_reconnect_outcome_is_aborted_and_notifies_closed() {
6410        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6411        let port = listener.local_addr().unwrap().port();
6412        let server = tokio::spawn(async move {
6413            let (stream, _) = listener.accept().await.unwrap();
6414            let _websocket = accept_async(stream).await.unwrap();
6415            std::future::pending::<()>().await;
6416        });
6417
6418        let mut inner = WebSocketClientInner::connect_url(reconnect_test_config(port), None, None)
6419            .await
6420            .unwrap();
6421        let state_notify = Arc::clone(&inner.state_notify);
6422        let mut notified = std::pin::pin!(state_notify.notified());
6423        notified.as_mut().enable();
6424
6425        let outcome = inner.reconnect_with_outcome().await.unwrap();
6426
6427        assert_eq!(outcome, ReconnectOutcome::Aborted);
6428        assert_eq!(
6429            ConnectionMode::from_atomic(&inner.connection_mode),
6430            ConnectionMode::Closed
6431        );
6432        tokio::time::timeout(TEST_TIMEOUT, notified)
6433            .await
6434            .expect("stream close notification was not published");
6435        server.abort();
6436    }
6437
6438    #[rstest]
6439    #[tokio::test]
6440    async fn test_reconnect_outcome_is_reconnected_with_handler() {
6441        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6442        let port = listener.local_addr().unwrap().port();
6443        let server = tokio::spawn(async move {
6444            let (stream, _) = listener.accept().await.unwrap();
6445            let _first = accept_async(stream).await.unwrap();
6446            let (stream, _) = listener.accept().await.unwrap();
6447            let mut second = accept_async(stream).await.unwrap();
6448            second
6449                .send(WsMessage::Text("replacement".into()))
6450                .await
6451                .unwrap();
6452            std::future::pending::<()>().await;
6453        });
6454
6455        let (epoch_handler, mut epoch_rx) = channel_epoch_message_handler();
6456        let mut inner = WebSocketClientInner::connect_url_with_handler(
6457            reconnect_test_config(port),
6458            Some(IncomingHandler::Epoch(epoch_handler)),
6459            None,
6460            None,
6461            None,
6462            None,
6463            InitialConnectOptions::default(),
6464        )
6465        .await
6466        .unwrap();
6467        inner
6468            .connection_mode
6469            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6470
6471        let outcome = inner.reconnect_with_outcome().await.unwrap();
6472
6473        assert_eq!(outcome, ReconnectOutcome::Reconnected);
6474        assert_eq!(
6475            ConnectionMode::from_atomic(&inner.connection_mode),
6476            ConnectionMode::Active
6477        );
6478        let (epoch, message) = tokio::time::timeout(TEST_TIMEOUT, epoch_rx.recv())
6479            .await
6480            .expect("replacement epoch message was not delivered")
6481            .expect("epoch handler channel closed");
6482        assert_eq!(epoch, 1);
6483        assert_eq!(message, WsMessage::Text("replacement".into()));
6484        server.abort();
6485    }
6486
6487    #[rstest]
6488    #[tokio::test]
6489    async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
6490        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6491        let port = listener.local_addr().unwrap().port();
6492        let server = tokio::spawn(async move {
6493            let (stream, _) = listener.accept().await.unwrap();
6494            let _websocket = accept_async(stream).await.unwrap();
6495            std::future::pending::<()>().await;
6496        });
6497
6498        let mut config = reconnect_test_config(port);
6499        config.heartbeat_interval_secs = Some(60);
6500        let (handler, _handler_rx) = channel_message_handler();
6501        let inner = WebSocketClientInner::connect_url(config, Some(handler), None)
6502            .await
6503            .unwrap();
6504        let read_fence = inner
6505            .read_fence
6506            .clone()
6507            .expect("read fence should exist in handler mode");
6508        let read_abort = inner
6509            .read_task
6510            .as_ref()
6511            .expect("read task should be spawned in handler mode")
6512            .abort_handle();
6513        let write_abort = inner.write_task.abort_handle();
6514        let heartbeat_abort = inner
6515            .heartbeat_task
6516            .as_ref()
6517            .expect("heartbeat task should be spawned for a configured heartbeat")
6518            .abort_handle();
6519
6520        assert!(read_fence.is_valid(), "read fence should start valid");
6521        assert!(
6522            !read_abort.is_finished(),
6523            "read task should be running before drop"
6524        );
6525        assert!(
6526            !write_abort.is_finished(),
6527            "write task should be running before drop"
6528        );
6529        assert!(
6530            !heartbeat_abort.is_finished(),
6531            "heartbeat task should be running before drop"
6532        );
6533
6534        drop(inner);
6535        wait_until_async(
6536            || async {
6537                read_abort.is_finished()
6538                    && write_abort.is_finished()
6539                    && heartbeat_abort.is_finished()
6540            },
6541            TEST_TIMEOUT,
6542        )
6543        .await;
6544
6545        assert!(!read_fence.is_valid(), "read fence was not invalidated");
6546        assert!(read_abort.is_finished(), "read task was not aborted");
6547        assert!(write_abort.is_finished(), "write task was not aborted");
6548        assert!(
6549            heartbeat_abort.is_finished(),
6550            "heartbeat task was not aborted"
6551        );
6552        server.abort();
6553    }
6554
6555    struct RecordingServer {
6556        task: JoinHandle<()>,
6557        port: u16,
6558        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
6559        binary_frames: Arc<tokio::sync::Mutex<Vec<Vec<u8>>>>,
6560        control_frames: Arc<tokio::sync::Mutex<Vec<String>>>,
6561        connections: Arc<AtomicUsize>,
6562    }
6563
6564    #[cfg(feature = "transport-sockudo")]
6565    async fn read_http_request<S>(stream: &mut S) -> Vec<u8>
6566    where
6567        S: AsyncRead + Unpin,
6568    {
6569        let mut buf = Vec::new();
6570        let mut chunk = [0u8; 256];
6571        loop {
6572            let n = stream.read(&mut chunk).await.unwrap();
6573            assert!(n > 0, "HTTP request closed before headers completed");
6574            buf.extend_from_slice(&chunk[..n]);
6575            if buf.windows(4).any(|window| window == b"\r\n\r\n") {
6576                return buf;
6577            }
6578        }
6579    }
6580
6581    #[cfg(feature = "transport-sockudo")]
6582    fn extract_header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
6583        request.lines().find_map(|line| {
6584            let (header_name, header_value) = line.split_once(':')?;
6585            if header_name.eq_ignore_ascii_case(name) {
6586                Some(header_value.trim())
6587            } else {
6588                None
6589            }
6590        })
6591    }
6592
6593    #[cfg(feature = "transport-sockudo")]
6594    #[derive(Debug, Clone)]
6595    struct HeaderAssertCallback {
6596        key: String,
6597        value: HeaderValue,
6598    }
6599
6600    #[cfg(feature = "transport-sockudo")]
6601    impl Callback for HeaderAssertCallback {
6602        #[expect(
6603            clippy::panic_in_result_fn,
6604            reason = "assertion failures should fail the test"
6605        )]
6606        fn on_request(
6607            self,
6608            request: &server::Request,
6609            response: server::Response,
6610        ) -> Result<server::Response, server::ErrorResponse> {
6611            assert_eq!(request.headers().get(&self.key), Some(&self.value));
6612            Ok(response)
6613        }
6614    }
6615
6616    impl RecordingServer {
6617        async fn setup() -> Self {
6618            let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6619            let port = listener.local_addr().unwrap().port();
6620            let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6621            let messages_clone = Arc::clone(&messages);
6622            let binary_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6623            let binary_frames_clone = Arc::clone(&binary_frames);
6624            let control_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6625            let control_frames_clone = Arc::clone(&control_frames);
6626            let connections = Arc::new(AtomicUsize::new(0));
6627            let connections_clone = Arc::clone(&connections);
6628
6629            let task = task::spawn(async move {
6630                loop {
6631                    let (stream, _) = listener.accept().await.unwrap();
6632                    let mut websocket = accept_async(stream).await.unwrap();
6633                    connections_clone.fetch_add(1, Ordering::SeqCst);
6634                    let messages = Arc::clone(&messages_clone);
6635                    let binary_frames = Arc::clone(&binary_frames_clone);
6636                    let control_frames = Arc::clone(&control_frames_clone);
6637
6638                    task::spawn(async move {
6639                        while let Some(Ok(msg)) = websocket.next().await {
6640                            match msg {
6641                                WsMessage::Text(text) => {
6642                                    messages.lock().await.push(text.to_string());
6643                                }
6644                                WsMessage::Binary(data) => {
6645                                    binary_frames.lock().await.push(data.to_vec());
6646                                }
6647                                WsMessage::Pong(data) => {
6648                                    control_frames
6649                                        .lock()
6650                                        .await
6651                                        .push(format!("pong:{}", String::from_utf8_lossy(&data)));
6652                                }
6653                                WsMessage::Close(_) => {
6654                                    let _ = websocket.close(None).await;
6655                                    break;
6656                                }
6657                                _ => {}
6658                            }
6659                        }
6660                    });
6661                }
6662            });
6663
6664            Self {
6665                task,
6666                port,
6667                messages,
6668                binary_frames,
6669                control_frames,
6670                connections,
6671            }
6672        }
6673
6674        async fn messages(&self) -> Vec<String> {
6675            self.messages.lock().await.clone()
6676        }
6677
6678        async fn binary_frames(&self) -> Vec<Vec<u8>> {
6679            self.binary_frames.lock().await.clone()
6680        }
6681
6682        async fn control_frames(&self) -> Vec<String> {
6683            self.control_frames.lock().await.clone()
6684        }
6685
6686        async fn wait_for_connections(&self, expected: usize) {
6687            wait_until_async(
6688                || async { self.connections.load(Ordering::SeqCst) == expected },
6689                TEST_TIMEOUT,
6690            )
6691            .await;
6692        }
6693    }
6694
6695    impl Drop for RecordingServer {
6696        fn drop(&mut self) {
6697            self.task.abort();
6698        }
6699    }
6700
6701    #[rstest]
6702    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6703    async fn test_manual_reconnect_waits_for_slow_loss_callback_and_new_auth() {
6704        let server = RecordingServer::setup().await;
6705        let tracker = AuthTracker::new();
6706        let _initial_auth = tracker.begin();
6707        tracker.succeed();
6708        let states = Arc::new(Mutex::new(Vec::new()));
6709        let states_callback = Arc::clone(&states);
6710        let auth_at_loss = Arc::new(Mutex::new(Vec::new()));
6711        let auth_at_loss_callback = Arc::clone(&auth_at_loss);
6712        let tracker_callback = tracker.clone();
6713        let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6714        let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6715        let callback_release_clone = Arc::clone(&callback_release);
6716        let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6717
6718        let sink = SocketStateSink::new(move |state| {
6719            states_callback.lock().push(state);
6720            if state == SocketState::Disconnected {
6721                auth_at_loss_callback
6722                    .lock()
6723                    .push(tracker_callback.auth_state());
6724                callback_entered_tx.send(()).unwrap();
6725                let (lock, condvar) = callback_release_clone.as_ref();
6726                let mut released = lock.lock();
6727                while !*released {
6728                    condvar.wait(&mut released);
6729                }
6730            }
6731        });
6732
6733        let (handler, mut handler_rx) = channel_message_handler();
6734        let client = WebSocketClient::builder()
6735            .config(reconnect_test_config(server.port))
6736            .message_handler(handler)
6737            .state_sink(sink)
6738            .connect()
6739            .await
6740            .unwrap();
6741        client.set_auth_tracker(tracker.clone(), true);
6742        server.wait_for_connections(1).await;
6743
6744        let handle = client.reconnect_handle();
6745        let (request_tx, request_rx) = std::sync::mpsc::channel();
6746        let request_thread = std::thread::spawn(move || {
6747            request_tx.send(handle.request_reconnect()).unwrap();
6748        });
6749
6750        recv_rendezvous(callback_entered_rx, "slow reconnect callback entry").await;
6751        client
6752            .writer_tx
6753            .send(WriterCommand::Send(Message::text("buffered")))
6754            .unwrap();
6755        server.wait_for_connections(2).await;
6756        tokio::time::sleep(Duration::from_millis(250)).await;
6757
6758        assert_eq!(client.connection_mode(), ConnectionMode::Reconnect);
6759        assert!(!client.reconnect_published.load(Ordering::SeqCst));
6760        assert_eq!(tracker.auth_state(), AuthState::Unauthenticated);
6761        assert_eq!(*auth_at_loss.lock(), vec![AuthState::Unauthenticated]);
6762        assert_eq!(
6763            *states.lock(),
6764            vec![SocketState::Connected, SocketState::Disconnected]
6765        );
6766        assert!(server.messages().await.is_empty());
6767
6768        callback_release_guard.release();
6769        assert_eq!(
6770            recv_rendezvous(request_rx, "manual reconnect result").await,
6771            ReconnectRequestOutcome::Accepted
6772        );
6773        request_thread.join().unwrap();
6774        wait_until_async(|| async { client.is_active() }, TEST_TIMEOUT).await;
6775        wait_until_async(
6776            || {
6777                let states = Arc::clone(&states);
6778                async move { states.lock().len() == 3 }
6779            },
6780            TEST_TIMEOUT,
6781        )
6782        .await;
6783
6784        let notification = tokio::time::timeout(TEST_TIMEOUT, handler_rx.recv())
6785            .await
6786            .expect("reconnect notification was not delivered")
6787            .expect("handler channel closed");
6788        assert_eq!(notification, WsMessage::Text(RECONNECTED.into()));
6789        assert!(handler_rx.try_recv().is_err());
6790        tokio::time::sleep(Duration::from_millis(200)).await;
6791        assert!(server.messages().await.is_empty());
6792
6793        // The auth frame is bound to the replacement connection, so it is written while the
6794        // replay waits for that authentication.
6795        let _replacement_auth = tracker.begin();
6796        client
6797            .send_bytes_on_connection(b"auth".to_vec(), None, client.connection_epoch())
6798            .await
6799            .unwrap();
6800        wait_until_async(
6801            || {
6802                let binary_frames = Arc::clone(&server.binary_frames);
6803                async move { binary_frames.lock().await.len() == 1 }
6804            },
6805            TEST_TIMEOUT,
6806        )
6807        .await;
6808        assert_eq!(server.binary_frames().await, vec![b"auth".to_vec()]);
6809        assert!(server.messages().await.is_empty());
6810
6811        tracker.succeed();
6812        wait_until_async(
6813            || {
6814                let messages = Arc::clone(&server.messages);
6815                async move { messages.lock().await.len() == 1 }
6816            },
6817            TEST_TIMEOUT,
6818        )
6819        .await;
6820
6821        client.send_text("live".into(), None).await.unwrap();
6822        wait_until_async(
6823            || {
6824                let messages = Arc::clone(&server.messages);
6825                async move { messages.lock().await.len() == 2 }
6826            },
6827            TEST_TIMEOUT,
6828        )
6829        .await;
6830
6831        assert_eq!(server.messages().await, vec!["buffered", "live"]);
6832        assert_eq!(server.connections.load(Ordering::SeqCst), 2);
6833        assert_eq!(
6834            *states.lock(),
6835            vec![
6836                SocketState::Connected,
6837                SocketState::Disconnected,
6838                SocketState::Connected,
6839            ]
6840        );
6841
6842        client.disconnect().await;
6843    }
6844
6845    #[rstest]
6846    #[tokio::test]
6847    async fn test_reconnect_handle_is_closed_after_client_drop() {
6848        let server = RecordingServer::setup().await;
6849        let (handler, _handler_rx) = channel_message_handler();
6850        let client = WebSocketClient::builder()
6851            .config(reconnect_test_config(server.port))
6852            .message_handler(handler)
6853            .connect()
6854            .await
6855            .unwrap();
6856        let tracker = AuthTracker::new();
6857        let pending_auth = tracker.begin();
6858        client.set_auth_tracker(tracker.clone(), true);
6859        let controller_abort = client.controller_task.abort_handle();
6860        let handle = client.reconnect_handle();
6861
6862        drop(client);
6863        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6864
6865        assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
6866        assert_eq!(tracker.auth_state(), AuthState::Failed);
6867        assert_eq!(
6868            tokio::time::timeout(TEST_TIMEOUT, pending_auth)
6869                .await
6870                .expect("client drop should resolve pending authentication")
6871                .expect("authentication sender should report its terminal result"),
6872            Err("WebSocket client closed".to_string())
6873        );
6874    }
6875
6876    #[rstest]
6877    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6878    async fn test_concurrent_drop_closes_accepted_reconnect() {
6879        let server = RecordingServer::setup().await;
6880        let tracker = AuthTracker::new();
6881        let _initial_auth = tracker.begin();
6882        tracker.succeed();
6883        let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6884        let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6885        let callback_release_clone = Arc::clone(&callback_release);
6886        let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6887        let states = Arc::new(Mutex::new(Vec::new()));
6888        let states_callback = Arc::clone(&states);
6889        let sink = SocketStateSink::new(move |state| {
6890            states_callback.lock().push(state);
6891            if state == SocketState::Disconnected {
6892                callback_entered_tx.send(()).unwrap();
6893                let (lock, condvar) = callback_release_clone.as_ref();
6894                let mut released = lock.lock();
6895                while !*released {
6896                    condvar.wait(&mut released);
6897                }
6898            }
6899        });
6900
6901        let (handler, _handler_rx) = channel_message_handler();
6902        let client = WebSocketClient::builder()
6903            .config(reconnect_test_config(server.port))
6904            .message_handler(handler)
6905            .state_sink(sink)
6906            .connect()
6907            .await
6908            .unwrap();
6909        client.set_auth_tracker(tracker.clone(), true);
6910        let controller_abort = client.controller_task.abort_handle();
6911        let connection_mode = Arc::clone(&client.connection_mode);
6912        let handle = client.reconnect_handle();
6913        let surviving_handle = handle.clone();
6914        let (request_tx, request_rx) = std::sync::mpsc::channel();
6915        let request_thread = std::thread::spawn(move || {
6916            request_tx.send(handle.request_reconnect()).unwrap();
6917        });
6918
6919        recv_rendezvous(callback_entered_rx, "concurrent drop callback entry").await;
6920        drop(client);
6921        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6922        assert_eq!(
6923            ConnectionMode::from_atomic(&connection_mode),
6924            ConnectionMode::Closed
6925        );
6926        assert_eq!(tracker.auth_state(), AuthState::Failed);
6927
6928        callback_release_guard.release();
6929        assert_eq!(
6930            recv_rendezvous(request_rx, "concurrent drop reconnect result").await,
6931            ReconnectRequestOutcome::Accepted
6932        );
6933        request_thread.join().unwrap();
6934        assert_eq!(
6935            surviving_handle.request_reconnect(),
6936            ReconnectRequestOutcome::Closed
6937        );
6938        assert_eq!(
6939            *states.lock(),
6940            vec![SocketState::Connected, SocketState::Disconnected]
6941        );
6942    }
6943
6944    #[rstest]
6945    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6946    async fn test_reconnect_callback_can_drop_client() {
6947        let server = RecordingServer::setup().await;
6948        let client_slot = Arc::new(Mutex::new(None::<WebSocketClient>));
6949        let client_slot_callback = Arc::clone(&client_slot);
6950        let states = Arc::new(Mutex::new(Vec::new()));
6951        let states_callback = Arc::clone(&states);
6952        let sink = SocketStateSink::new(move |state| {
6953            states_callback.lock().push(state);
6954            if state == SocketState::Disconnected {
6955                drop(client_slot_callback.lock().take());
6956            }
6957        });
6958
6959        let (handler, _handler_rx) = channel_message_handler();
6960        let client = WebSocketClient::builder()
6961            .config(reconnect_test_config(server.port))
6962            .message_handler(handler)
6963            .state_sink(sink)
6964            .connect()
6965            .await
6966            .unwrap();
6967        let tracker = AuthTracker::new();
6968        let _initial_auth = tracker.begin();
6969        tracker.succeed();
6970        client.set_auth_tracker(tracker.clone(), true);
6971        let controller_abort = client.controller_task.abort_handle();
6972        let connection_mode = Arc::clone(&client.connection_mode);
6973        let handle = client.reconnect_handle();
6974        let surviving_handle = handle.clone();
6975        *client_slot.lock() = Some(client);
6976        let (result_tx, result_rx) = std::sync::mpsc::channel();
6977        std::thread::spawn(move || {
6978            result_tx.send(handle.request_reconnect()).unwrap();
6979        });
6980
6981        assert_eq!(
6982            recv_rendezvous(result_rx, "callback drop reconnect result").await,
6983            ReconnectRequestOutcome::Accepted
6984        );
6985        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6986
6987        assert!(client_slot.lock().is_none());
6988        assert_eq!(
6989            ConnectionMode::from_atomic(&connection_mode),
6990            ConnectionMode::Closed
6991        );
6992        assert_eq!(tracker.auth_state(), AuthState::Failed);
6993        assert_eq!(
6994            surviving_handle.request_reconnect(),
6995            ReconnectRequestOutcome::Closed
6996        );
6997        assert_eq!(
6998            *states.lock(),
6999            vec![SocketState::Connected, SocketState::Disconnected]
7000        );
7001    }
7002
7003    #[rstest]
7004    #[tokio::test]
7005    async fn test_reconnect_then_disconnect() {
7006        // Bind an ephemeral port
7007        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7008        let port = listener.local_addr().unwrap().port();
7009
7010        // Server task: accept one ws connection then close it
7011        let server = task::spawn(async move {
7012            let (stream, _) = listener.accept().await.unwrap();
7013            let ws = accept_async(stream).await.unwrap();
7014            drop(ws);
7015            // Keep alive briefly
7016            sleep(Duration::from_secs(1)).await;
7017        });
7018
7019        // Build a channel-based message handler for incoming messages (unused here)
7020        let (handler, _rx) = channel_message_handler();
7021
7022        // Configure client with short reconnect backoff
7023        let config = WebSocketConfig {
7024            url: format!("ws://127.0.0.1:{port}"),
7025            headers: vec![],
7026            heartbeat_interval_secs: None,
7027            heartbeat_payload: None,
7028            connect_timeout_ms: Some(1_000),
7029            reconnect_delay_initial_ms: Some(50),
7030            reconnect_delay_max_ms: Some(100),
7031            reconnect_backoff_factor: Some(1.0),
7032            reconnect_jitter_ms: Some(0),
7033            reconnect_max_attempts: None,
7034            heartbeat_timeout_secs: None,
7035            idle_timeout_ms: None,
7036            writer_capacity: None,
7037            backend: TransportBackend::Tungstenite,
7038            proxy_url: None,
7039            max_message_size_bytes: None,
7040            max_frame_size_bytes: None,
7041        };
7042
7043        // Connect the client
7044        let client = WebSocketClient::builder()
7045            .config(config)
7046            .message_handler(handler)
7047            .connect()
7048            .await
7049            .unwrap();
7050
7051        // Allow server to drop connection and client to detect
7052        sleep(Duration::from_millis(100)).await;
7053        // Now immediately disconnect the client
7054        client.disconnect().await;
7055        assert!(client.is_disconnected());
7056        server.abort();
7057    }
7058
7059    #[rstest]
7060    #[tokio::test]
7061    async fn test_reconnect_state_flips_when_reader_stops() {
7062        // Bind an ephemeral port and accept a single websocket connection which we drop.
7063        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7064        let port = listener.local_addr().unwrap().port();
7065        let server = task::spawn(async move {
7066            if let Ok((stream, _)) = listener.accept().await
7067                && let Ok(ws) = accept_async(stream).await
7068            {
7069                drop(ws);
7070            }
7071
7072            sleep(Duration::from_millis(50)).await;
7073        });
7074
7075        let (handler, _rx) = channel_message_handler();
7076
7077        let config = WebSocketConfig {
7078            url: format!("ws://127.0.0.1:{port}"),
7079            headers: vec![],
7080            heartbeat_interval_secs: None,
7081            heartbeat_payload: None,
7082            connect_timeout_ms: Some(1_000),
7083            reconnect_delay_initial_ms: Some(50),
7084            reconnect_delay_max_ms: Some(100),
7085            reconnect_backoff_factor: Some(1.0),
7086            reconnect_jitter_ms: Some(0),
7087            reconnect_max_attempts: None,
7088            heartbeat_timeout_secs: None,
7089            idle_timeout_ms: None,
7090            writer_capacity: None,
7091            backend: TransportBackend::Tungstenite,
7092            proxy_url: None,
7093            max_message_size_bytes: None,
7094            max_frame_size_bytes: None,
7095        };
7096
7097        let client = WebSocketClient::builder()
7098            .config(config)
7099            .message_handler(handler)
7100            .connect()
7101            .await
7102            .unwrap();
7103
7104        tokio::time::timeout(Duration::from_secs(2), async {
7105            loop {
7106                if client.is_reconnecting() {
7107                    break;
7108                }
7109
7110                tokio::time::sleep(Duration::from_millis(10)).await;
7111            }
7112        })
7113        .await
7114        .expect("client did not enter RECONNECT state");
7115
7116        client.disconnect().await;
7117        server.abort();
7118    }
7119
7120    #[rstest]
7121    #[tokio::test]
7122    async fn test_stream_mode_disables_auto_reconnect() {
7123        // Test that stream-based clients do not retain an internal message handler
7124        // and that reconnect() transitions to CLOSED state for stream mode
7125        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7126        let port = listener.local_addr().unwrap().port();
7127        let server = task::spawn(async move {
7128            if let Ok((stream, _)) = listener.accept().await
7129                && let Ok(_ws) = accept_async(stream).await
7130            {
7131                // Keep connection alive briefly
7132                sleep(Duration::from_millis(100)).await;
7133            }
7134        });
7135
7136        let config = WebSocketConfig {
7137            url: format!("ws://127.0.0.1:{port}"),
7138            headers: vec![],
7139            heartbeat_interval_secs: None,
7140            heartbeat_payload: None,
7141            connect_timeout_ms: Some(1_000),
7142            reconnect_delay_initial_ms: Some(50),
7143            reconnect_delay_max_ms: Some(100),
7144            reconnect_backoff_factor: Some(1.0),
7145            reconnect_jitter_ms: Some(0),
7146            reconnect_max_attempts: None,
7147            heartbeat_timeout_secs: None,
7148            idle_timeout_ms: None,
7149            writer_capacity: None,
7150            backend: TransportBackend::Tungstenite,
7151            proxy_url: None,
7152            max_message_size_bytes: None,
7153            max_frame_size_bytes: None,
7154        };
7155
7156        let (_reader, _client) = WebSocketClient::stream_builder()
7157            .config(config)
7158            .connect()
7159            .await
7160            .unwrap();
7161
7162        // Note: We can't easily test the reconnect behavior from the outside since
7163        // the inner client is private. The key fix is that WebSocketClientInner
7164        // now has no internal handler in stream mode, and reconnect() will
7165        // transition to CLOSED state instead of creating a new reader that gets dropped.
7166        // This is tested implicitly by the fact that stream users won't get stuck
7167        // in an infinite reconnect loop.
7168
7169        server.abort();
7170    }
7171
7172    #[rstest]
7173    #[tokio::test]
7174    async fn test_message_handler_mode_allows_auto_reconnect() {
7175        // Test that regular clients (with message handler) can auto-reconnect
7176        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7177        let port = listener.local_addr().unwrap().port();
7178
7179        let server = task::spawn(async move {
7180            // Accept first connection and close it
7181            if let Ok((stream, _)) = listener.accept().await
7182                && let Ok(ws) = accept_async(stream).await
7183            {
7184                drop(ws);
7185            }
7186
7187            sleep(Duration::from_millis(50)).await;
7188        });
7189
7190        let (handler, _rx) = channel_message_handler();
7191
7192        let config = WebSocketConfig {
7193            url: format!("ws://127.0.0.1:{port}"),
7194            headers: vec![],
7195            heartbeat_interval_secs: None,
7196            heartbeat_payload: None,
7197            connect_timeout_ms: Some(1_000),
7198            reconnect_delay_initial_ms: Some(50),
7199            reconnect_delay_max_ms: Some(100),
7200            reconnect_backoff_factor: Some(1.0),
7201            reconnect_jitter_ms: Some(0),
7202            reconnect_max_attempts: None,
7203            heartbeat_timeout_secs: None,
7204            idle_timeout_ms: None,
7205            writer_capacity: None,
7206            backend: TransportBackend::Tungstenite,
7207            proxy_url: None,
7208            max_message_size_bytes: None,
7209            max_frame_size_bytes: None,
7210        };
7211
7212        let client = WebSocketClient::builder()
7213            .config(config)
7214            .message_handler(handler)
7215            .connect()
7216            .await
7217            .unwrap();
7218
7219        // Wait for the connection to be dropped and reconnection to be attempted
7220        tokio::time::timeout(Duration::from_secs(2), async {
7221            loop {
7222                if client.is_reconnecting() || client.is_closed() {
7223                    break;
7224                }
7225
7226                tokio::time::sleep(Duration::from_millis(10)).await;
7227            }
7228        })
7229        .await
7230        .expect("client should attempt reconnection or close");
7231
7232        // Should either be reconnecting or closed (depending on timing)
7233        // The important thing is it's not staying active forever
7234        assert!(
7235            client.is_reconnecting() || client.is_closed(),
7236            "Client with message handler should attempt reconnection"
7237        );
7238
7239        client.disconnect().await;
7240        server.abort();
7241    }
7242
7243    #[rstest]
7244    #[tokio::test]
7245    async fn test_handler_mode_reconnect_with_new_connection() {
7246        // Test that handler mode successfully reconnects and messages continue flowing
7247        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7248        let port = listener.local_addr().unwrap().port();
7249
7250        let server = task::spawn(async move {
7251            // First connection - accept and immediately close
7252            if let Ok((stream, _)) = listener.accept().await
7253                && let Ok(ws) = accept_async(stream).await
7254            {
7255                drop(ws);
7256            }
7257
7258            // Small delay to let client detect disconnection
7259            sleep(Duration::from_millis(100)).await;
7260
7261            // Second connection - accept, send a message, then keep alive
7262            if let Ok((stream, _)) = listener.accept().await
7263                && let Ok(mut ws) = accept_async(stream).await
7264            {
7265                use futures_util::SinkExt;
7266                let _ = ws
7267                    .send(WsMessage::Text("reconnected".to_string().into()))
7268                    .await;
7269                sleep(Duration::from_secs(1)).await;
7270            }
7271        });
7272
7273        let (handler, mut rx) = channel_message_handler();
7274
7275        let config = WebSocketConfig {
7276            url: format!("ws://127.0.0.1:{port}"),
7277            headers: vec![],
7278            heartbeat_interval_secs: None,
7279            heartbeat_payload: None,
7280            connect_timeout_ms: Some(2_000),
7281            reconnect_delay_initial_ms: Some(50),
7282            reconnect_delay_max_ms: Some(200),
7283            reconnect_backoff_factor: Some(1.5),
7284            reconnect_jitter_ms: Some(10),
7285            reconnect_max_attempts: None,
7286            heartbeat_timeout_secs: None,
7287            idle_timeout_ms: None,
7288            writer_capacity: None,
7289            backend: TransportBackend::Tungstenite,
7290            proxy_url: None,
7291            max_message_size_bytes: None,
7292            max_frame_size_bytes: None,
7293        };
7294
7295        let client = WebSocketClient::builder()
7296            .config(config)
7297            .message_handler(handler)
7298            .connect()
7299            .await
7300            .unwrap();
7301
7302        // Wait for reconnection to happen and message to arrive
7303        let result = tokio::time::timeout(Duration::from_secs(5), async {
7304            loop {
7305                if let Ok(msg) = rx.try_recv()
7306                    && matches!(msg, WsMessage::Text(ref text) if AsRef::<str>::as_ref(text) == "reconnected")
7307                {
7308                    return true;
7309                }
7310
7311                tokio::time::sleep(Duration::from_millis(10)).await;
7312            }
7313        })
7314        .await;
7315
7316        assert!(
7317            result.is_ok(),
7318            "Should receive message after reconnection within timeout"
7319        );
7320
7321        client.disconnect().await;
7322        server.abort();
7323    }
7324
7325    #[rstest]
7326    #[tokio::test]
7327    async fn test_stream_mode_no_auto_reconnect() {
7328        // Test that stream mode does not automatically reconnect when connection is lost
7329        // The caller owns the reader and is responsible for detecting disconnection
7330        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7331        let port = listener.local_addr().unwrap().port();
7332
7333        let server = task::spawn(async move {
7334            // Accept connection and send one message, then close
7335            if let Ok((stream, _)) = listener.accept().await
7336                && let Ok(mut ws) = accept_async(stream).await
7337            {
7338                use futures_util::SinkExt;
7339                let _ = ws.send(WsMessage::Text("hello".to_string().into())).await;
7340                sleep(Duration::from_millis(50)).await;
7341                // Connection closes when ws is dropped
7342            }
7343        });
7344
7345        let config = WebSocketConfig {
7346            url: format!("ws://127.0.0.1:{port}"),
7347            headers: vec![],
7348            heartbeat_interval_secs: None,
7349            heartbeat_payload: None,
7350            connect_timeout_ms: Some(1_000),
7351            reconnect_delay_initial_ms: Some(50),
7352            reconnect_delay_max_ms: Some(100),
7353            reconnect_backoff_factor: Some(1.0),
7354            reconnect_jitter_ms: Some(0),
7355            reconnect_max_attempts: None,
7356            heartbeat_timeout_secs: None,
7357            idle_timeout_ms: None,
7358            writer_capacity: None,
7359            backend: TransportBackend::Tungstenite,
7360            proxy_url: None,
7361            max_message_size_bytes: None,
7362            max_frame_size_bytes: None,
7363        };
7364
7365        let (mut reader, client) = WebSocketClient::stream_builder()
7366            .config(config)
7367            .connect()
7368            .await
7369            .unwrap();
7370
7371        // Initially active
7372        assert!(client.is_active(), "Client should start as active");
7373
7374        // Read the hello message
7375        let msg = reader.next().await;
7376        assert!(
7377            matches!(&msg, Some(Ok(Message::Text(bytes))) if bytes.as_ref() == b"hello"),
7378            "Should receive initial message"
7379        );
7380
7381        // Read until connection closes (reader will return None or error)
7382        while let Some(msg) = reader.next().await {
7383            if msg.is_err() || matches!(msg, Ok(Message::Close(_))) {
7384                break;
7385            }
7386        }
7387
7388        // Controller cannot detect reader EOF (reader is owned by caller),
7389        // so the client stays ACTIVE until the caller signals.
7390        sleep(Duration::from_millis(200)).await;
7391        assert!(
7392            client.is_active(),
7393            "Stream mode client stays ACTIVE before notify_closed()"
7394        );
7395
7396        // Caller signals EOF via notify_closed()
7397        client.notify_closed();
7398
7399        assert!(
7400            client.is_closed(),
7401            "Stream mode client should be CLOSED after notify_closed()"
7402        );
7403        assert!(
7404            !client.is_reconnecting(),
7405            "Stream mode client should never attempt reconnection"
7406        );
7407
7408        client.disconnect().await;
7409        server.abort();
7410    }
7411
7412    #[rstest]
7413    #[tokio::test]
7414    async fn test_send_timeout_uses_configured_connect_timeout() {
7415        // Test that send operations respect the configured connect_timeout.
7416        // When a client is stuck in RECONNECT longer than the timeout, sends should fail with Timeout.
7417        use nautilus_common::testing::wait_until_async;
7418
7419        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7420        let port = listener.local_addr().unwrap().port();
7421
7422        let server = task::spawn(async move {
7423            // Accept first connection and immediately close it
7424            if let Ok((stream, _)) = listener.accept().await
7425                && let Ok(ws) = accept_async(stream).await
7426            {
7427                drop(ws);
7428            }
7429
7430            // Don't accept second connection - client will be stuck in RECONNECT
7431            sleep(Duration::from_mins(1)).await;
7432        });
7433
7434        let (handler, _rx) = channel_message_handler();
7435
7436        // Configure with SHORT 2s reconnect timeout
7437        let config = WebSocketConfig {
7438            url: format!("ws://127.0.0.1:{port}"),
7439            headers: vec![],
7440            heartbeat_interval_secs: None,
7441            heartbeat_payload: None,
7442            connect_timeout_ms: Some(2_000), // 2s timeout
7443            reconnect_delay_initial_ms: Some(50),
7444            reconnect_delay_max_ms: Some(100),
7445            reconnect_backoff_factor: Some(1.0),
7446            reconnect_jitter_ms: Some(0),
7447            reconnect_max_attempts: None,
7448            heartbeat_timeout_secs: None,
7449            idle_timeout_ms: None,
7450            writer_capacity: None,
7451            backend: TransportBackend::Tungstenite,
7452            proxy_url: None,
7453            max_message_size_bytes: None,
7454            max_frame_size_bytes: None,
7455        };
7456
7457        let client = WebSocketClient::builder()
7458            .config(config)
7459            .message_handler(handler)
7460            .connect()
7461            .await
7462            .unwrap();
7463
7464        // Wait for client to enter RECONNECT state
7465        wait_until_async(
7466            || async { client.is_reconnecting() },
7467            Duration::from_secs(3),
7468        )
7469        .await;
7470
7471        // Attempt send while stuck in RECONNECT - should timeout after 2s (configured timeout)
7472        let start = std::time::Instant::now();
7473        let send_result = client.send_text("test".to_string(), None).await;
7474        let elapsed = start.elapsed();
7475
7476        assert!(
7477            send_result.is_err(),
7478            "Send should fail when client stuck in RECONNECT"
7479        );
7480        assert!(
7481            matches!(send_result, Err(crate::error::SendError::Timeout)),
7482            "Send should return Timeout error, was: {send_result:?}"
7483        );
7484        // Verify timeout respects configured value (2s), but don't check upper bound
7485        // as CI scheduler jitter can cause legitimate delays beyond the timeout
7486        assert!(
7487            elapsed >= Duration::from_millis(1800),
7488            "Send should timeout after at least 2s (configured timeout), took {elapsed:?}"
7489        );
7490
7491        client.disconnect().await;
7492        server.abort();
7493    }
7494
7495    #[rstest]
7496    #[tokio::test]
7497    async fn test_send_waits_during_reconnection() {
7498        // Test that send operations wait for reconnection to complete (up to timeout)
7499        use nautilus_common::testing::wait_until_async;
7500
7501        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7502        let port = listener.local_addr().unwrap().port();
7503
7504        let server = task::spawn(async move {
7505            // First connection - accept and immediately close
7506            if let Ok((stream, _)) = listener.accept().await
7507                && let Ok(ws) = accept_async(stream).await
7508            {
7509                drop(ws);
7510            }
7511
7512            // Wait a bit before accepting second connection
7513            sleep(Duration::from_millis(500)).await;
7514
7515            // Second connection - accept and keep alive
7516            if let Ok((stream, _)) = listener.accept().await
7517                && let Ok(mut ws) = accept_async(stream).await
7518            {
7519                // Echo messages
7520                while let Some(Ok(msg)) = ws.next().await {
7521                    if ws.send(msg).await.is_err() {
7522                        break;
7523                    }
7524                }
7525            }
7526        });
7527
7528        let (handler, _rx) = channel_message_handler();
7529
7530        let config = WebSocketConfig {
7531            url: format!("ws://127.0.0.1:{port}"),
7532            headers: vec![],
7533            heartbeat_interval_secs: None,
7534            heartbeat_payload: None,
7535            connect_timeout_ms: Some(5_000), // 5s timeout - enough for reconnect
7536            reconnect_delay_initial_ms: Some(100),
7537            reconnect_delay_max_ms: Some(200),
7538            reconnect_backoff_factor: Some(1.0),
7539            reconnect_jitter_ms: Some(0),
7540            reconnect_max_attempts: None,
7541            heartbeat_timeout_secs: None,
7542            idle_timeout_ms: None,
7543            writer_capacity: None,
7544            backend: TransportBackend::Tungstenite,
7545            proxy_url: None,
7546            max_message_size_bytes: None,
7547            max_frame_size_bytes: None,
7548        };
7549
7550        let client = WebSocketClient::builder()
7551            .config(config)
7552            .message_handler(handler)
7553            .connect()
7554            .await
7555            .unwrap();
7556
7557        // Wait for reconnection to trigger
7558        wait_until_async(
7559            || async { client.is_reconnecting() },
7560            Duration::from_secs(2),
7561        )
7562        .await;
7563
7564        // Try to send while reconnecting - should wait and succeed after reconnect
7565        let send_result = tokio::time::timeout(
7566            Duration::from_secs(3),
7567            client.send_text("test_message".to_string(), None),
7568        )
7569        .await;
7570
7571        assert!(
7572            send_result.is_ok() && send_result.unwrap().is_ok(),
7573            "Send should succeed after waiting for reconnection"
7574        );
7575
7576        client.disconnect().await;
7577        server.abort();
7578    }
7579
7580    #[rstest]
7581    #[tokio::test]
7582    async fn test_rate_limiter_before_active_wait() {
7583        // Test that rate limiting happens BEFORE active state check.
7584        // This prevents race conditions where connection state changes during rate limit wait.
7585        // We verify this by: (1) exhausting rate limit, (2) ensuring client is RECONNECTING,
7586        // (3) sending again and confirming it waits for rate limit THEN reconnection.
7587        use std::{num::NonZeroU32, sync::Arc};
7588
7589        use nautilus_common::testing::wait_until_async;
7590
7591        use crate::ratelimiter::quota::Quota;
7592
7593        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7594        let port = listener.local_addr().unwrap().port();
7595
7596        let server = task::spawn(async move {
7597            // First connection - accept and close after receiving one message
7598            if let Ok((stream, _)) = listener.accept().await
7599                && let Ok(mut ws) = accept_async(stream).await
7600            {
7601                // Receive first message then close
7602                if let Some(Ok(_)) = ws.next().await {
7603                    drop(ws);
7604                }
7605            }
7606
7607            // Wait before accepting reconnection
7608            sleep(Duration::from_millis(500)).await;
7609
7610            // Second connection - accept and keep alive
7611            if let Ok((stream, _)) = listener.accept().await
7612                && let Ok(mut ws) = accept_async(stream).await
7613            {
7614                while let Some(Ok(msg)) = ws.next().await {
7615                    if ws.send(msg).await.is_err() {
7616                        break;
7617                    }
7618                }
7619            }
7620        });
7621
7622        let (handler, _rx) = channel_message_handler();
7623
7624        let config = WebSocketConfig {
7625            url: format!("ws://127.0.0.1:{port}"),
7626            headers: vec![],
7627            heartbeat_interval_secs: None,
7628            heartbeat_payload: None,
7629            connect_timeout_ms: Some(5_000),
7630            reconnect_delay_initial_ms: Some(50),
7631            reconnect_delay_max_ms: Some(100),
7632            reconnect_backoff_factor: Some(1.0),
7633            reconnect_jitter_ms: Some(0),
7634            reconnect_max_attempts: None,
7635            heartbeat_timeout_secs: None,
7636            idle_timeout_ms: None,
7637            writer_capacity: None,
7638            backend: TransportBackend::Tungstenite,
7639            proxy_url: None,
7640            max_message_size_bytes: None,
7641            max_frame_size_bytes: None,
7642        };
7643
7644        // Very restrictive rate limit: 1 request per second, burst of 1
7645        let quota = Quota::per_second(NonZeroU32::new(1).unwrap())
7646            .unwrap()
7647            .allow_burst(NonZeroU32::new(1).unwrap());
7648
7649        let client = Arc::new(
7650            WebSocketClient::builder()
7651                .config(config)
7652                .message_handler(handler)
7653                .keyed_quotas(vec![("test_key".to_string(), quota)])
7654                .connect()
7655                .await
7656                .unwrap(),
7657        );
7658
7659        // First send exhausts burst capacity and triggers connection close
7660        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7661        client
7662            .send_text("msg1".to_string(), Some(test_key.as_slice()))
7663            .await
7664            .unwrap();
7665
7666        // Wait for client to enter RECONNECT state
7667        wait_until_async(
7668            || async { client.is_reconnecting() },
7669            Duration::from_secs(2),
7670        )
7671        .await;
7672
7673        // Second send: will hit rate limit (~1s) THEN wait for reconnection (~0.5s)
7674        let start = std::time::Instant::now();
7675        let send_result = client
7676            .send_text("msg2".to_string(), Some(test_key.as_slice()))
7677            .await;
7678        let elapsed = start.elapsed();
7679
7680        // Should succeed after both rate limit AND reconnection
7681        assert!(
7682            send_result.is_ok(),
7683            "Send should succeed after rate limit + reconnection, was: {send_result:?}"
7684        );
7685        // Total wait should be at least rate limit time (~1s)
7686        // The reconnection completes while rate limiting or after
7687        // Use 850ms threshold to account for timing jitter in CI
7688        assert!(
7689            elapsed >= Duration::from_millis(850),
7690            "Should wait for rate limit (~1s), waited {elapsed:?}"
7691        );
7692
7693        client.disconnect().await;
7694        server.abort();
7695    }
7696
7697    #[rstest]
7698    #[tokio::test]
7699    async fn test_disconnect_during_reconnect_exits_cleanly() {
7700        // Test CAS race condition: disconnect called during reconnection
7701        // Should exit cleanly without spawning new tasks
7702        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7703        let port = listener.local_addr().unwrap().port();
7704
7705        let server = task::spawn(async move {
7706            // Accept first connection and immediately close
7707            if let Ok((stream, _)) = listener.accept().await
7708                && let Ok(ws) = accept_async(stream).await
7709            {
7710                drop(ws);
7711            }
7712
7713            // Don't accept second connection - let reconnect hang
7714            sleep(Duration::from_mins(1)).await;
7715        });
7716
7717        let (handler, _rx) = channel_message_handler();
7718
7719        let config = WebSocketConfig {
7720            url: format!("ws://127.0.0.1:{port}"),
7721            headers: vec![],
7722            heartbeat_interval_secs: None,
7723            heartbeat_payload: None,
7724            connect_timeout_ms: Some(2_000), // 2s timeout - shorter than disconnect timeout
7725            reconnect_delay_initial_ms: Some(100),
7726            reconnect_delay_max_ms: Some(200),
7727            reconnect_backoff_factor: Some(1.0),
7728            reconnect_jitter_ms: Some(0),
7729            reconnect_max_attempts: None,
7730            heartbeat_timeout_secs: None,
7731            idle_timeout_ms: None,
7732            writer_capacity: None,
7733            backend: TransportBackend::Tungstenite,
7734            proxy_url: None,
7735            max_message_size_bytes: None,
7736            max_frame_size_bytes: None,
7737        };
7738
7739        let client = WebSocketClient::builder()
7740            .config(config)
7741            .message_handler(handler)
7742            .connect()
7743            .await
7744            .unwrap();
7745
7746        // Wait for reconnection to start
7747        tokio::time::timeout(Duration::from_secs(2), async {
7748            while !client.is_reconnecting() {
7749                sleep(Duration::from_millis(10)).await;
7750            }
7751        })
7752        .await
7753        .expect("Client should enter RECONNECT state");
7754
7755        // Disconnect while reconnecting
7756        client.disconnect().await;
7757
7758        // Should be cleanly closed
7759        assert!(
7760            client.is_disconnected(),
7761            "Client should be cleanly disconnected"
7762        );
7763
7764        server.abort();
7765    }
7766
7767    #[rstest]
7768    #[tokio::test]
7769    async fn test_send_fails_fast_when_closed_before_rate_limit() {
7770        // Test that send operations check connection state BEFORE rate limiting,
7771        // preventing unnecessary delays when the connection is already closed.
7772        use std::{num::NonZeroU32, sync::Arc};
7773
7774        use nautilus_common::testing::wait_until_async;
7775
7776        use crate::ratelimiter::quota::Quota;
7777
7778        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7779        let port = listener.local_addr().unwrap().port();
7780
7781        let server = task::spawn(async move {
7782            // Accept connection and immediately close
7783            if let Ok((stream, _)) = listener.accept().await
7784                && let Ok(ws) = accept_async(stream).await
7785            {
7786                drop(ws);
7787            }
7788
7789            sleep(Duration::from_mins(1)).await;
7790        });
7791
7792        let (handler, _rx) = channel_message_handler();
7793
7794        let config = WebSocketConfig {
7795            url: format!("ws://127.0.0.1:{port}"),
7796            headers: vec![],
7797            heartbeat_interval_secs: None,
7798            heartbeat_payload: None,
7799            connect_timeout_ms: Some(5_000),
7800            reconnect_delay_initial_ms: Some(50),
7801            reconnect_delay_max_ms: Some(100),
7802            reconnect_backoff_factor: Some(1.0),
7803            reconnect_jitter_ms: Some(0),
7804            reconnect_max_attempts: None,
7805            heartbeat_timeout_secs: None,
7806            idle_timeout_ms: None,
7807            writer_capacity: None,
7808            backend: TransportBackend::Tungstenite,
7809            proxy_url: None,
7810            max_message_size_bytes: None,
7811            max_frame_size_bytes: None,
7812        };
7813
7814        // Very restrictive rate limit: 1 request per 10 seconds
7815        // This ensures that if we wait for rate limit, the test will timeout
7816        let quota = Quota::with_period(Duration::from_secs(10))
7817            .unwrap()
7818            .allow_burst(NonZeroU32::new(1).unwrap());
7819
7820        let client = Arc::new(
7821            WebSocketClient::builder()
7822                .config(config)
7823                .message_handler(handler)
7824                .keyed_quotas(vec![("test_key".to_string(), quota)])
7825                .connect()
7826                .await
7827                .unwrap(),
7828        );
7829
7830        // Wait for disconnection
7831        wait_until_async(
7832            || async { client.is_reconnecting() || client.is_closed() },
7833            Duration::from_secs(2),
7834        )
7835        .await;
7836
7837        // Explicitly disconnect to move away from ACTIVE state
7838        client.disconnect().await;
7839        assert!(
7840            !client.is_active(),
7841            "Client should not be active after disconnect"
7842        );
7843
7844        // Attempt send - should fail IMMEDIATELY without waiting for rate limit
7845        let start = std::time::Instant::now();
7846        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7847        let result = client
7848            .send_text("test".to_string(), Some(test_key.as_slice()))
7849            .await;
7850        let elapsed = start.elapsed();
7851
7852        // Should fail with Closed error
7853        assert!(result.is_err(), "Send should fail when client is closed");
7854        assert!(
7855            matches!(result, Err(crate::error::SendError::Closed)),
7856            "Send should return Closed error, was: {result:?}"
7857        );
7858
7859        // Should fail FAST (< 100ms) without waiting for rate limit (10s)
7860        assert!(
7861            elapsed < Duration::from_millis(100),
7862            "Send should fail fast without rate limiting, took {elapsed:?}"
7863        );
7864
7865        server.abort();
7866    }
7867
7868    #[rstest]
7869    #[tokio::test]
7870    async fn test_builder_rejects_shared_rate_limiter_with_quotas() {
7871        let (handler, _rx) = channel_message_handler();
7872        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7873        let result = WebSocketClient::builder()
7874            .config(reconnect_test_config(1))
7875            .message_handler(handler)
7876            .default_quota(Quota::with_period(Duration::from_secs(1)).unwrap())
7877            .rate_limiter(rate_limiter)
7878            .connect()
7879            .await;
7880
7881        assert_eq!(
7882            result.unwrap_err().to_string(),
7883            "I/O error: Cannot combine a shared rate limiter with quota configuration"
7884        );
7885    }
7886
7887    #[rstest]
7888    #[tokio::test]
7889    async fn test_builder_rejects_connection_keys_without_limiter() {
7890        let (handler, _rx) = channel_message_handler();
7891        let result = WebSocketClient::builder()
7892            .config(reconnect_test_config(1))
7893            .message_handler(handler)
7894            .connection_rate_keys(Arc::from([Ustr::from("connect")]))
7895            .connect()
7896            .await;
7897
7898        assert_eq!(
7899            result.unwrap_err().to_string(),
7900            "I/O error: Connection rate keys require a connection rate limiter"
7901        );
7902    }
7903
7904    #[rstest]
7905    #[tokio::test]
7906    async fn test_builder_rejects_connection_limiter_without_keys() {
7907        let (handler, _rx) = channel_message_handler();
7908        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7909        let result = WebSocketClient::builder()
7910            .config(reconnect_test_config(1))
7911            .message_handler(handler)
7912            .connection_rate_limiter(rate_limiter)
7913            .connect()
7914            .await;
7915
7916        assert_eq!(
7917            result.unwrap_err().to_string(),
7918            "I/O error: Connection rate limiter requires at least one connection rate key"
7919        );
7920    }
7921
7922    #[rstest]
7923    #[tokio::test]
7924    async fn test_epoch_builder_rejects_both_ping_handler_types() {
7925        let (handler, _rx) = channel_epoch_message_handler();
7926        let ping_handler: PingHandler = Arc::new(|_| {});
7927        let epoch_ping_handler: EpochPingHandler = Arc::new(|_, _| {});
7928        let result = WebSocketClient::epoch_builder()
7929            .config(reconnect_test_config(1))
7930            .epoch_handler(handler)
7931            .ping_handler(ping_handler)
7932            .epoch_ping_handler(epoch_ping_handler)
7933            .connect()
7934            .await;
7935
7936        assert_eq!(
7937            result.unwrap_err().to_string(),
7938            "I/O error: Cannot configure both ping_handler and epoch_ping_handler"
7939        );
7940    }
7941
7942    #[rstest]
7943    #[tokio::test]
7944    async fn test_connect_url_rejects_invalid_reconnect_timing_before_connect() {
7945        let (handler, _rx) = channel_message_handler();
7946
7947        let config = WebSocketConfig {
7948            url: "ws://127.0.0.1:1".to_string(),
7949            headers: vec![],
7950            heartbeat_interval_secs: None,
7951            heartbeat_payload: None,
7952            connect_timeout_ms: Some(0),
7953            reconnect_delay_initial_ms: Some(100),
7954            reconnect_delay_max_ms: Some(500),
7955            reconnect_backoff_factor: Some(1.5),
7956            reconnect_jitter_ms: Some(0),
7957            reconnect_max_attempts: None,
7958            heartbeat_timeout_secs: None,
7959            idle_timeout_ms: None,
7960            writer_capacity: None,
7961            backend: TransportBackend::Tungstenite,
7962            proxy_url: None,
7963            max_message_size_bytes: None,
7964            max_frame_size_bytes: None,
7965        };
7966
7967        let err = WebSocketClientInner::connect_url(config, Some(handler), None)
7968            .await
7969            .expect_err("invalid reconnect timing should be rejected");
7970
7971        match err {
7972            TransportError::Io(error) => {
7973                assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
7974                assert!(
7975                    error.to_string().contains("connect_timeout_ms"),
7976                    "error should mention zero reconnect timeout, was: {error}"
7977                );
7978            }
7979            other => panic!("expected InvalidInput IO error, was: {other:?}"),
7980        }
7981    }
7982
7983    #[rstest]
7984    #[tokio::test]
7985    async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
7986        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7987        let port = listener.local_addr().unwrap().port();
7988        let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
7989        let accepted_clone = Arc::clone(&accepted);
7990
7991        let server = task::spawn(async move {
7992            let (stream, _) = listener.accept().await.unwrap();
7993            accepted_clone.store(true, Ordering::SeqCst);
7994            accept_async(stream).await.unwrap();
7995        });
7996
7997        let (handler, _rx) = channel_message_handler();
7998
7999        let config = WebSocketConfig {
8000            url: format!("ws://127.0.0.1:{port}"),
8001            headers: vec![],
8002            heartbeat_interval_secs: None,
8003            heartbeat_payload: None,
8004            connect_timeout_ms: Some(1_000),
8005            reconnect_delay_initial_ms: Some(50),
8006            reconnect_delay_max_ms: Some(100),
8007            reconnect_backoff_factor: Some(100.1),
8008            reconnect_jitter_ms: Some(0),
8009            reconnect_max_attempts: None,
8010            heartbeat_timeout_secs: None,
8011            idle_timeout_ms: None,
8012            writer_capacity: None,
8013            backend: TransportBackend::Tungstenite,
8014            proxy_url: None,
8015            max_message_size_bytes: None,
8016            max_frame_size_bytes: None,
8017        };
8018
8019        let error = WebSocketClientInner::connect_url(config, Some(handler), None)
8020            .await
8021            .expect_err("invalid reconnect backoff should be rejected");
8022
8023        match error {
8024            TransportError::Io(error) => {
8025                assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
8026                assert!(
8027                    error.to_string().contains("factor"),
8028                    "error should mention the invalid factor, was: {error}"
8029                );
8030            }
8031            other => panic!("expected InvalidInput IO error, was: {other:?}"),
8032        }
8033
8034        assert!(
8035            !accepted.load(Ordering::SeqCst),
8036            "invalid reconnect backoff must be rejected before connecting"
8037        );
8038        server.abort();
8039    }
8040
8041    #[rstest]
8042    #[tokio::test]
8043    async fn test_client_without_handler_sets_stream_mode() {
8044        // Test that if a client is created without a handler via connect_url,
8045        // it keeps the internal handler empty to prevent zombie connections
8046
8047        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8048        let port = listener.local_addr().unwrap().port();
8049
8050        let server = task::spawn(async move {
8051            // Accept and immediately close to simulate server disconnect
8052            if let Ok((stream, _)) = listener.accept().await
8053                && let Ok(ws) = accept_async(stream).await
8054            {
8055                drop(ws); // Drop connection immediately
8056            }
8057        });
8058
8059        let config = WebSocketConfig {
8060            url: format!("ws://127.0.0.1:{port}"),
8061            headers: vec![],
8062            heartbeat_interval_secs: None,
8063            heartbeat_payload: None,
8064            connect_timeout_ms: Some(1_000),
8065            reconnect_delay_initial_ms: Some(100),
8066            reconnect_delay_max_ms: Some(500),
8067            reconnect_backoff_factor: Some(1.5),
8068            reconnect_jitter_ms: Some(0),
8069            reconnect_max_attempts: None,
8070            heartbeat_timeout_secs: None,
8071            idle_timeout_ms: None,
8072            writer_capacity: None,
8073            backend: TransportBackend::Tungstenite,
8074            proxy_url: None,
8075            max_message_size_bytes: None,
8076            max_frame_size_bytes: None,
8077        };
8078
8079        // Create client directly via connect_url with no handler (stream mode)
8080        let inner = WebSocketClientInner::connect_url(config, None, None)
8081            .await
8082            .unwrap();
8083
8084        // Verify stream mode does not retain an internal handler
8085        assert!(
8086            inner.handler.is_none(),
8087            "Client without handler should not retain an internal handler"
8088        );
8089
8090        // Verify that when stream mode is enabled, reconnection is disabled
8091        // (documented behavior - stream mode clients close instead of reconnecting)
8092
8093        server.abort();
8094    }
8095
8096    #[rstest]
8097    #[tokio::test]
8098    async fn test_idle_timeout_triggers_reconnect() {
8099        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8100        let port = listener.local_addr().unwrap().port();
8101
8102        // Server accepts WS connection but sends nothing (simulates silent death)
8103        let server = task::spawn(async move {
8104            let (stream, _) = listener.accept().await.unwrap();
8105            let _ws = accept_async(stream).await.unwrap();
8106            // Hold connection open but send nothing
8107            sleep(Duration::from_secs(5)).await;
8108        });
8109
8110        let (handler, _rx) = channel_message_handler();
8111
8112        let config = WebSocketConfig {
8113            url: format!("ws://127.0.0.1:{port}"),
8114            headers: vec![],
8115            heartbeat_interval_secs: None,
8116            heartbeat_payload: None,
8117            connect_timeout_ms: Some(2_000),
8118            reconnect_delay_initial_ms: Some(50),
8119            reconnect_delay_max_ms: Some(100),
8120            reconnect_backoff_factor: Some(1.0),
8121            reconnect_jitter_ms: Some(0),
8122            reconnect_max_attempts: Some(1),
8123            heartbeat_timeout_secs: None,
8124            idle_timeout_ms: Some(500),
8125            writer_capacity: None,
8126            backend: TransportBackend::Tungstenite,
8127            proxy_url: None,
8128            max_message_size_bytes: None,
8129            max_frame_size_bytes: None,
8130        };
8131
8132        let client = WebSocketClient::builder()
8133            .config(config)
8134            .message_handler(handler)
8135            .connect()
8136            .await
8137            .unwrap();
8138
8139        assert!(client.is_active());
8140
8141        // Wait for idle timeout to fire and client to enter reconnect/closed
8142        wait_until_async(
8143            || async { client.is_reconnecting() || client.is_disconnected() },
8144            Duration::from_secs(3),
8145        )
8146        .await;
8147
8148        assert!(
8149            !client.is_active(),
8150            "Client should not be active after idle timeout"
8151        );
8152
8153        client.disconnect().await;
8154        server.abort();
8155    }
8156
8157    #[rstest]
8158    #[tokio::test]
8159    async fn test_idle_timeout_resets_on_data() {
8160        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8161        let port = listener.local_addr().unwrap().port();
8162
8163        // Server sends a message every 200ms (well within 1s idle timeout)
8164        let server = task::spawn(async move {
8165            let (stream, _) = listener.accept().await.unwrap();
8166            let mut ws = accept_async(stream).await.unwrap();
8167
8168            for _ in 0..10 {
8169                sleep(Duration::from_millis(200)).await;
8170
8171                if ws.send(WsMessage::Text("ping".into())).await.is_err() {
8172                    break;
8173                }
8174            }
8175        });
8176
8177        let (handler, _rx) = channel_message_handler();
8178
8179        let config = WebSocketConfig {
8180            url: format!("ws://127.0.0.1:{port}"),
8181            headers: vec![],
8182            heartbeat_interval_secs: None,
8183            heartbeat_payload: None,
8184            connect_timeout_ms: Some(2_000),
8185            reconnect_delay_initial_ms: Some(50),
8186            reconnect_delay_max_ms: Some(100),
8187            reconnect_backoff_factor: Some(1.0),
8188            reconnect_jitter_ms: Some(0),
8189            reconnect_max_attempts: Some(1),
8190            heartbeat_timeout_secs: None,
8191            idle_timeout_ms: Some(1_000),
8192            writer_capacity: None,
8193            backend: TransportBackend::Tungstenite,
8194            proxy_url: None,
8195            max_message_size_bytes: None,
8196            max_frame_size_bytes: None,
8197        };
8198
8199        let client = WebSocketClient::builder()
8200            .config(config)
8201            .message_handler(handler)
8202            .connect()
8203            .await
8204            .unwrap();
8205
8206        assert!(client.is_active());
8207
8208        // Wait 1.5s - data arrives every 200ms so idle timeout (1s) should NOT fire
8209        sleep(Duration::from_millis(1_500)).await;
8210
8211        assert!(
8212            client.is_active(),
8213            "Client should remain active when data is flowing"
8214        );
8215
8216        client.disconnect().await;
8217        server.abort();
8218    }
8219
8220    #[rstest]
8221    #[tokio::test]
8222    async fn test_idle_timeout_fires_when_only_pings_received() {
8223        // Regression: pings and pongs are keep-alive frames, not application data,
8224        // so a peer that only emits control frames must still trip the idle timeout.
8225        // The peer keeps pinging for well past the observation window so the
8226        // pre-fix behavior (reset-on-ping) would keep the client active; under the
8227        // fix the idle timer never resets and fires after ~500ms.
8228        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8229        let port = listener.local_addr().unwrap().port();
8230        let server = task::spawn(async move {
8231            let (stream, _) = listener.accept().await.unwrap();
8232            let mut ws = accept_async(stream).await.unwrap();
8233
8234            for _ in 0..60 {
8235                sleep(Duration::from_millis(100)).await;
8236
8237                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
8238                    break;
8239                }
8240            }
8241        });
8242
8243        let (handler, _rx) = channel_message_handler();
8244
8245        let config = WebSocketConfig {
8246            url: format!("ws://127.0.0.1:{port}"),
8247            headers: vec![],
8248            heartbeat_interval_secs: None,
8249            heartbeat_payload: None,
8250            connect_timeout_ms: Some(2_000),
8251            reconnect_delay_initial_ms: Some(50),
8252            reconnect_delay_max_ms: Some(100),
8253            reconnect_backoff_factor: Some(1.0),
8254            reconnect_jitter_ms: Some(0),
8255            reconnect_max_attempts: Some(1),
8256            heartbeat_timeout_secs: None,
8257            idle_timeout_ms: Some(500),
8258            writer_capacity: None,
8259            backend: TransportBackend::Tungstenite,
8260            proxy_url: None,
8261            max_message_size_bytes: None,
8262            max_frame_size_bytes: None,
8263        };
8264
8265        let client = WebSocketClient::builder()
8266            .config(config)
8267            .message_handler(handler)
8268            .connect()
8269            .await
8270            .unwrap();
8271
8272        assert!(client.is_active());
8273
8274        // Observation window is shorter than the ping stream (6s). If the idle
8275        // timer mistakenly reset on every ping the client would still be active
8276        // here; under the fix it goes inactive at ~500ms.
8277        wait_until_async(
8278            || async { client.is_reconnecting() || client.is_disconnected() },
8279            Duration::from_millis(1_500),
8280        )
8281        .await;
8282
8283        assert!(
8284            !client.is_active(),
8285            "Client should not be active after idle timeout when only pings/pongs flow"
8286        );
8287
8288        client.disconnect().await;
8289        server.abort();
8290    }
8291
8292    #[rstest]
8293    #[tokio::test]
8294    async fn test_idle_timeout_fires_when_only_pongs_received() {
8295        // Regression for the heartbeat-reply path. When the client heartbeat is
8296        // enabled, the peer auto-replies with pongs for every outgoing ping. If
8297        // those pongs refreshed last_data_time the idle timer would never fire on
8298        // a zombie connection (the motivating Polymarket scenario).
8299        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8300        let port = listener.local_addr().unwrap().port();
8301        let server = task::spawn(async move {
8302            let (stream, _) = listener.accept().await.unwrap();
8303            let mut ws = accept_async(stream).await.unwrap();
8304
8305            // Drain incoming frames so tungstenite's internal pong replies are
8306            // actually flushed to the client. Hold the connection open well past
8307            // the observation window.
8308            let deadline = tokio::time::Instant::now() + Duration::from_secs(6);
8309            while tokio::time::Instant::now() < deadline {
8310                if let Ok(Some(Err(_)) | None) =
8311                    tokio::time::timeout(Duration::from_millis(100), ws.next()).await
8312                {
8313                    break;
8314                }
8315            }
8316        });
8317
8318        let (handler, _rx) = channel_message_handler();
8319
8320        let config = WebSocketConfig {
8321            url: format!("ws://127.0.0.1:{port}"),
8322            headers: vec![],
8323            heartbeat_interval_secs: Some(1),
8324            heartbeat_payload: None,
8325            connect_timeout_ms: Some(2_000),
8326            reconnect_delay_initial_ms: Some(50),
8327            reconnect_delay_max_ms: Some(100),
8328            reconnect_backoff_factor: Some(1.0),
8329            reconnect_jitter_ms: Some(0),
8330            reconnect_max_attempts: Some(1),
8331            heartbeat_timeout_secs: None,
8332            idle_timeout_ms: Some(1_500),
8333            writer_capacity: None,
8334            backend: TransportBackend::Tungstenite,
8335            proxy_url: None,
8336            max_message_size_bytes: None,
8337            max_frame_size_bytes: None,
8338        };
8339
8340        let client = WebSocketClient::builder()
8341            .config(config)
8342            .message_handler(handler)
8343            .connect()
8344            .await
8345            .unwrap();
8346
8347        assert!(client.is_active());
8348
8349        // Heartbeat cadence is 1s; each ping draws a pong reply. Under the fix
8350        // the idle timer ignores those pongs and fires at ~1.5s. Under the bug
8351        // every pong reset the timer and the client would stay active.
8352        wait_until_async(
8353            || async { client.is_reconnecting() || client.is_disconnected() },
8354            Duration::from_millis(2_500),
8355        )
8356        .await;
8357
8358        assert!(
8359            !client.is_active(),
8360            "Client should not be active after idle timeout when only pongs flow"
8361        );
8362
8363        client.disconnect().await;
8364        server.abort();
8365    }
8366
8367    #[rstest]
8368    #[tokio::test]
8369    async fn test_disconnect_during_backoff_exits_promptly() {
8370        // Verify that disconnect interrupts backoff sleep (Finding 1).
8371        // Server accepts then drops, no second listener -> reconnect fails -> enters backoff.
8372        // We disconnect while backing off and assert the client shuts down quickly.
8373        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8374        let port = listener.local_addr().unwrap().port();
8375
8376        let server = task::spawn(async move {
8377            // Accept first connection, close immediately
8378            if let Ok((stream, _)) = listener.accept().await {
8379                let _ = accept_async(stream).await;
8380            }
8381
8382            // Don't accept again so reconnect fails and enters backoff
8383            sleep(Duration::from_mins(1)).await;
8384        });
8385
8386        let (handler, _rx) = channel_message_handler();
8387
8388        let config = WebSocketConfig {
8389            url: format!("ws://127.0.0.1:{port}"),
8390            headers: vec![],
8391            heartbeat_interval_secs: None,
8392            heartbeat_payload: None,
8393            connect_timeout_ms: Some(1_000),
8394            reconnect_delay_initial_ms: Some(10_000), // 10s backoff to ensure we're sleeping
8395            reconnect_delay_max_ms: Some(10_000),
8396            reconnect_backoff_factor: Some(1.0),
8397            reconnect_jitter_ms: Some(0),
8398            reconnect_max_attempts: None,
8399            heartbeat_timeout_secs: None,
8400            idle_timeout_ms: None,
8401            writer_capacity: None,
8402            backend: TransportBackend::Tungstenite,
8403            proxy_url: None,
8404            max_message_size_bytes: None,
8405            max_frame_size_bytes: None,
8406        };
8407
8408        let client = WebSocketClient::builder()
8409            .config(config)
8410            .message_handler(handler)
8411            .connect()
8412            .await
8413            .unwrap();
8414
8415        // Wait for client to enter reconnect
8416        wait_until_async(
8417            || async { client.is_reconnecting() },
8418            Duration::from_secs(3),
8419        )
8420        .await;
8421
8422        // Wait a bit more for the reconnect attempt to fail and enter backoff sleep
8423        sleep(Duration::from_millis(1_500)).await;
8424
8425        // Disconnect while backing off
8426        let start = std::time::Instant::now();
8427        client.disconnect().await;
8428        let elapsed = start.elapsed();
8429
8430        assert!(client.is_disconnected(), "Client should be disconnected");
8431        // Should exit well before the 10s backoff sleep completes
8432        assert!(
8433            elapsed < Duration::from_secs(2),
8434            "Disconnect should interrupt backoff sleep, took {elapsed:?}"
8435        );
8436
8437        server.abort();
8438    }
8439
8440    #[rstest]
8441    #[tokio::test]
8442    async fn test_rate_limit_cancelled_on_disconnect() {
8443        // Verify that a send blocked on rate limiting returns Closed when
8444        // the client disconnects (Finding 6).
8445        use std::{num::NonZeroU32, sync::Arc};
8446
8447        use crate::ratelimiter::quota::Quota;
8448
8449        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8450        let port = listener.local_addr().unwrap().port();
8451        let server = task::spawn(async move {
8452            if let Ok((stream, _)) = listener.accept().await {
8453                let mut ws = accept_async(stream).await.unwrap();
8454                // Keep alive and echo
8455                while let Some(Ok(msg)) = ws.next().await {
8456                    if ws.send(msg).await.is_err() {
8457                        break;
8458                    }
8459                }
8460            }
8461        });
8462
8463        let (handler, _rx) = channel_message_handler();
8464
8465        let config = WebSocketConfig {
8466            url: format!("ws://127.0.0.1:{port}"),
8467            headers: vec![],
8468            heartbeat_interval_secs: None,
8469            heartbeat_payload: None,
8470            connect_timeout_ms: Some(5_000),
8471            reconnect_delay_initial_ms: Some(100),
8472            reconnect_delay_max_ms: Some(500),
8473            reconnect_backoff_factor: Some(1.5),
8474            reconnect_jitter_ms: Some(0),
8475            reconnect_max_attempts: None,
8476            heartbeat_timeout_secs: None,
8477            idle_timeout_ms: None,
8478            writer_capacity: None,
8479            backend: TransportBackend::Tungstenite,
8480            proxy_url: None,
8481            max_message_size_bytes: None,
8482            max_frame_size_bytes: None,
8483        };
8484
8485        // Very restrictive: 1 req per 60 seconds
8486        let quota = Quota::with_period(Duration::from_mins(1))
8487            .unwrap()
8488            .allow_burst(NonZeroU32::new(1).unwrap());
8489
8490        let client = Arc::new(
8491            WebSocketClient::builder()
8492                .config(config)
8493                .message_handler(handler)
8494                .keyed_quotas(vec![("rate_key".to_string(), quota)])
8495                .connect()
8496                .await
8497                .unwrap(),
8498        );
8499
8500        let test_key: [Ustr; 1] = [Ustr::from("rate_key")];
8501
8502        // Exhaust the burst quota
8503        client
8504            .send_text("exhaust".to_string(), Some(test_key.as_slice()))
8505            .await
8506            .unwrap();
8507
8508        // Spawn a send that will block on rate limiter
8509        let client_clone = Arc::clone(&client);
8510        let send_handle = task::spawn(async move {
8511            client_clone
8512                .send_text("blocked".to_string(), Some(&[Ustr::from("rate_key")]))
8513                .await
8514        });
8515
8516        // Let the send block on rate limit
8517        sleep(Duration::from_millis(200)).await;
8518
8519        // Disconnect while send is blocked
8520        let start = std::time::Instant::now();
8521        client.disconnect().await;
8522        let elapsed_disconnect = start.elapsed();
8523
8524        // The blocked send should return Closed
8525        let result = tokio::time::timeout(Duration::from_secs(2), send_handle)
8526            .await
8527            .expect("Send task should complete quickly")
8528            .expect("Send task should not panic");
8529
8530        assert!(
8531            matches!(result, Err(crate::error::SendError::Closed)),
8532            "Blocked send should return Closed, was: {result:?}"
8533        );
8534
8535        // Disconnect should be fast, not waiting for the 60s rate limit
8536        assert!(
8537            elapsed_disconnect < Duration::from_secs(3),
8538            "Disconnect should not wait for rate limiter, took {elapsed_disconnect:?}"
8539        );
8540
8541        server.abort();
8542    }
8543
8544    #[rstest]
8545    #[tokio::test]
8546    async fn test_stream_mode_transitions_to_closed_on_dead_write_task() {
8547        // Verify that stream mode transitions to CLOSED (not RECONNECT) when
8548        // the write task dies (Finding 4). We force write failure by sending
8549        // after the server closes the connection.
8550        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8551        let port = listener.local_addr().unwrap().port();
8552        let server = task::spawn(async move {
8553            if let Ok((stream, _)) = listener.accept().await
8554                && let Ok(ws) = accept_async(stream).await
8555            {
8556                // Close immediately to cause write errors
8557                drop(ws);
8558            }
8559        });
8560
8561        let config = WebSocketConfig {
8562            url: format!("ws://127.0.0.1:{port}"),
8563            headers: vec![],
8564            heartbeat_interval_secs: None,
8565            heartbeat_payload: None,
8566            connect_timeout_ms: Some(1_000),
8567            reconnect_delay_initial_ms: Some(50),
8568            reconnect_delay_max_ms: Some(100),
8569            reconnect_backoff_factor: Some(1.0),
8570            reconnect_jitter_ms: Some(0),
8571            reconnect_max_attempts: None,
8572            heartbeat_timeout_secs: None,
8573            idle_timeout_ms: None,
8574            writer_capacity: None,
8575            backend: TransportBackend::Tungstenite,
8576            proxy_url: None,
8577            max_message_size_bytes: None,
8578            max_frame_size_bytes: None,
8579        };
8580
8581        let (_reader, client) = WebSocketClient::stream_builder()
8582            .config(config)
8583            .connect()
8584            .await
8585            .unwrap();
8586
8587        assert!(client.is_active(), "Client should start active");
8588
8589        // Wait for server to close, then send to trigger write task failure
8590        sleep(Duration::from_millis(100)).await;
8591
8592        // Keep sending until the write task detects the broken connection
8593        for _ in 0..20 {
8594            let _ = client.send_text("ping".to_string(), None).await;
8595            sleep(Duration::from_millis(50)).await;
8596
8597            if !client.is_active() {
8598                break;
8599            }
8600        }
8601
8602        // Wait for controller to process the state change
8603        wait_until_async(|| async { !client.is_active() }, Duration::from_secs(5)).await;
8604
8605        // Stream mode should go to CLOSED, not RECONNECT
8606        assert!(
8607            client.is_closed() || client.is_disconnected(),
8608            "Stream mode should transition to CLOSED, not RECONNECT. \
8609             is_reconnecting={}, is_closed={}, is_disconnected={}",
8610            client.is_reconnecting(),
8611            client.is_closed(),
8612            client.is_disconnected(),
8613        );
8614        assert!(
8615            !client.is_reconnecting(),
8616            "Stream mode should never attempt reconnection"
8617        );
8618
8619        server.abort();
8620    }
8621
8622    #[derive(Default)]
8623    struct BlockingFailState {
8624        send_entered: AtomicBool,
8625        send_entered_notify: tokio::sync::Notify,
8626        released: AtomicBool,
8627        fail: AtomicBool,
8628        waker: parking_lot::Mutex<Option<std::task::Waker>>,
8629    }
8630
8631    impl BlockingFailState {
8632        fn trigger_failure(&self) {
8633            self.fail.store(true, Ordering::SeqCst);
8634            self.release_send();
8635        }
8636
8637        fn release_send(&self) {
8638            self.released.store(true, Ordering::SeqCst);
8639
8640            if let Some(waker) = self.waker.lock().take() {
8641                waker.wake();
8642            }
8643        }
8644    }
8645
8646    /// Transport whose sends block until released and can fail on demand.
8647    struct BlockingFailTransport {
8648        state: Arc<BlockingFailState>,
8649    }
8650
8651    impl futures_util::Stream for BlockingFailTransport {
8652        type Item = Result<Message, TransportError>;
8653
8654        fn poll_next(
8655            self: std::pin::Pin<&mut Self>,
8656            _cx: &mut std::task::Context<'_>,
8657        ) -> std::task::Poll<Option<Self::Item>> {
8658            std::task::Poll::Pending
8659        }
8660    }
8661
8662    impl futures_util::Sink<Message> for BlockingFailTransport {
8663        type Error = TransportError;
8664
8665        fn poll_ready(
8666            self: std::pin::Pin<&mut Self>,
8667            _cx: &mut std::task::Context<'_>,
8668        ) -> std::task::Poll<Result<(), Self::Error>> {
8669            std::task::Poll::Ready(Ok(()))
8670        }
8671
8672        fn start_send(self: std::pin::Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8673            Ok(())
8674        }
8675
8676        fn poll_flush(
8677            self: std::pin::Pin<&mut Self>,
8678            cx: &mut std::task::Context<'_>,
8679        ) -> std::task::Poll<Result<(), Self::Error>> {
8680            // Store the waker before checking the flag so trigger_failure
8681            // cannot slip between the check and the registration
8682            *self.state.waker.lock() = Some(cx.waker().clone());
8683            self.state.send_entered.store(true, Ordering::SeqCst);
8684            self.state.send_entered_notify.notify_one();
8685
8686            if !self.state.released.load(Ordering::SeqCst) {
8687                std::task::Poll::Pending
8688            } else if self.state.fail.load(Ordering::SeqCst) {
8689                std::task::Poll::Ready(Err(TransportError::ConnectionReset))
8690            } else {
8691                std::task::Poll::Ready(Ok(()))
8692            }
8693        }
8694
8695        fn poll_close(
8696            self: std::pin::Pin<&mut Self>,
8697            _cx: &mut std::task::Context<'_>,
8698        ) -> std::task::Poll<Result<(), Self::Error>> {
8699            std::task::Poll::Ready(Ok(()))
8700        }
8701    }
8702
8703    struct BlockingMessageState {
8704        polled_tx: Mutex<Option<std::sync::mpsc::Sender<()>>>,
8705        release: (Mutex<bool>, parking_lot::Condvar),
8706        message: Mutex<Option<Message>>,
8707    }
8708
8709    struct BlockingMessageTransport {
8710        state: Arc<BlockingMessageState>,
8711    }
8712
8713    impl futures_util::Stream for BlockingMessageTransport {
8714        type Item = Result<Message, TransportError>;
8715
8716        fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8717            if let Some(polled_tx) = self.state.polled_tx.lock().take() {
8718                polled_tx.send(()).unwrap();
8719            }
8720
8721            let (lock, condvar) = &self.state.release;
8722            let mut released = lock.lock();
8723            while !*released {
8724                condvar.wait(&mut released);
8725            }
8726
8727            Poll::Ready(self.state.message.lock().take().map(Ok))
8728        }
8729    }
8730
8731    impl futures_util::Sink<Message> for BlockingMessageTransport {
8732        type Error = TransportError;
8733
8734        fn poll_ready(
8735            self: Pin<&mut Self>,
8736            _cx: &mut Context<'_>,
8737        ) -> Poll<Result<(), Self::Error>> {
8738            Poll::Ready(Ok(()))
8739        }
8740
8741        fn start_send(self: Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8742            Ok(())
8743        }
8744
8745        fn poll_flush(
8746            self: Pin<&mut Self>,
8747            _cx: &mut Context<'_>,
8748        ) -> Poll<Result<(), Self::Error>> {
8749            Poll::Ready(Ok(()))
8750        }
8751
8752        fn poll_close(
8753            self: Pin<&mut Self>,
8754            _cx: &mut Context<'_>,
8755        ) -> Poll<Result<(), Self::Error>> {
8756            Poll::Ready(Ok(()))
8757        }
8758    }
8759
8760    struct RecordingState {
8761        messages: Arc<Mutex<Vec<Message>>>,
8762        recorded_notify: tokio::sync::Notify,
8763    }
8764
8765    struct RecordingTransport {
8766        state: Arc<RecordingState>,
8767    }
8768
8769    impl futures_util::Stream for RecordingTransport {
8770        type Item = Result<Message, TransportError>;
8771
8772        fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8773            Poll::Pending
8774        }
8775    }
8776
8777    impl futures_util::Sink<Message> for RecordingTransport {
8778        type Error = TransportError;
8779
8780        fn poll_ready(
8781            self: Pin<&mut Self>,
8782            _cx: &mut Context<'_>,
8783        ) -> Poll<Result<(), Self::Error>> {
8784            Poll::Ready(Ok(()))
8785        }
8786
8787        fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> {
8788            self.state.messages.lock().push(item);
8789            self.state.recorded_notify.notify_one();
8790            Ok(())
8791        }
8792
8793        fn poll_flush(
8794            self: Pin<&mut Self>,
8795            _cx: &mut Context<'_>,
8796        ) -> Poll<Result<(), Self::Error>> {
8797            Poll::Ready(Ok(()))
8798        }
8799
8800        fn poll_close(
8801            self: Pin<&mut Self>,
8802            _cx: &mut Context<'_>,
8803        ) -> Poll<Result<(), Self::Error>> {
8804            Poll::Ready(Ok(()))
8805        }
8806    }
8807
8808    #[rstest]
8809    #[tokio::test(start_paused = true)]
8810    async fn test_pong_is_bound_to_connection_epoch() {
8811        let initial_state = Arc::new(RecordingState {
8812            messages: Arc::new(Mutex::new(Vec::new())),
8813            recorded_notify: tokio::sync::Notify::new(),
8814        });
8815
8816        let initial_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8817            state: initial_state,
8818        });
8819
8820        let (writer, _reader) = initial_transport.split();
8821        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8822        let state_notify = Arc::new(tokio::sync::Notify::new());
8823        let auth_tracker = Arc::new(OnceLock::new());
8824        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
8825        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
8826        let write_task = WebSocketClientInner::spawn_write_task(
8827            Arc::clone(&connection_state),
8828            Arc::clone(&state_notify),
8829            Arc::new(AtomicBool::new(true)),
8830            writer,
8831            writer_rx,
8832            Arc::new(AtomicU64::new(0)),
8833            auth_tracker,
8834            reconnect_buffer_waits_for_auth,
8835            None,
8836        );
8837
8838        let recorded = Arc::new(Mutex::new(Vec::new()));
8839
8840        let replacement_state = Arc::new(RecordingState {
8841            messages: Arc::clone(&recorded),
8842            recorded_notify: tokio::sync::Notify::new(),
8843        });
8844
8845        let replacement_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8846            state: replacement_state,
8847        });
8848
8849        let (replacement_writer, _reader) = replacement_transport.split();
8850        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
8851        writer_tx
8852            .send(WriterCommand::Update(replacement_writer, update_tx))
8853            .unwrap();
8854        writer_tx
8855            .send_control(WriterCommand::SendPongOnConnection {
8856                data: b"stale-pong".to_vec(),
8857                connection_epoch: 0,
8858            })
8859            .unwrap();
8860
8861        tokio::time::advance(Duration::from_millis(100)).await;
8862        assert_eq!(update_rx.await.unwrap(), 1);
8863
8864        let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8865        writer_tx
8866            .send_control(WriterCommand::SendOnConnection {
8867                message: Message::text("sentinel-1"),
8868                connection_epoch: 1,
8869                response_tx: sentinel_tx,
8870            })
8871            .unwrap();
8872
8873        sentinel_rx.await.unwrap().unwrap();
8874        assert_eq!(recorded.lock().as_slice(), &[Message::text("sentinel-1")]);
8875
8876        writer_tx
8877            .send_control(WriterCommand::SendPongOnConnection {
8878                data: b"fresh-pong".to_vec(),
8879                connection_epoch: 1,
8880            })
8881            .unwrap();
8882
8883        let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8884        writer_tx
8885            .send_control(WriterCommand::SendOnConnection {
8886                message: Message::text("sentinel-2"),
8887                connection_epoch: 1,
8888                response_tx: sentinel_tx,
8889            })
8890            .unwrap();
8891
8892        sentinel_rx.await.unwrap().unwrap();
8893        assert_eq!(
8894            recorded.lock().as_slice(),
8895            &[
8896                Message::text("sentinel-1"),
8897                Message::Pong(b"fresh-pong".to_vec().into()),
8898                Message::text("sentinel-2"),
8899            ]
8900        );
8901
8902        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
8903        state_notify.notify_waiters();
8904        drop(writer_tx);
8905        write_task.await.unwrap();
8906    }
8907
8908    #[rstest]
8909    #[case(Message::text("stale"))]
8910    #[case(Message::Binary(vec![1, 2, 3].into()))]
8911    #[case(Message::Ping(vec![1, 2, 3].into()))]
8912    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8913    async fn test_message_handler_drops_old_session_message(#[case] message: Message) {
8914        let (polled_tx, polled_rx) = std::sync::mpsc::channel();
8915
8916        let state = Arc::new(BlockingMessageState {
8917            polled_tx: Mutex::new(Some(polled_tx)),
8918            release: (Mutex::new(false), Condvar::new()),
8919            message: Mutex::new(Some(message)),
8920        });
8921
8922        let release_guard = CondvarReleaseGuard::new(&state.release);
8923
8924        let transport: BoxedWsTransport = Box::pin(BlockingMessageTransport {
8925            state: Arc::clone(&state),
8926        });
8927
8928        let (_writer, reader) = transport.split();
8929        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8930        let state_notify = Arc::new(tokio::sync::Notify::new());
8931        let read_fence = ReadSessionFence::new();
8932        let message_count = Arc::new(AtomicUsize::new(0));
8933        let ping_count = Arc::new(AtomicUsize::new(0));
8934        let message_count_clone = Arc::clone(&message_count);
8935        let ping_count_clone = Arc::clone(&ping_count);
8936        let message_handler: MessageHandler =
8937            Arc::new(move |_| _ = message_count_clone.fetch_add(1, Ordering::SeqCst));
8938        let message_handler = IncomingHandler::Message(message_handler);
8939        let ping_handler: PingHandler =
8940            Arc::new(move |_| _ = ping_count_clone.fetch_add(1, Ordering::SeqCst));
8941        let ping_handler = IncomingPingHandler::Ping(ping_handler);
8942
8943        let read_task = WebSocketClientInner::spawn_message_handler_task(
8944            Arc::clone(&connection_state),
8945            state_notify,
8946            read_fence.clone(),
8947            reader,
8948            0,
8949            Some(&message_handler),
8950            Some(&ping_handler),
8951            None,
8952            None,
8953            None,
8954            None,
8955            None,
8956        );
8957
8958        recv_rendezvous(polled_rx, "WebSocket reader poll entry").await;
8959        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8960        read_fence.invalidate();
8961        read_task.abort();
8962        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
8963        release_guard.release();
8964        await_task_termination(read_task, "old WebSocket read task").await;
8965
8966        assert_eq!(message_count.load(Ordering::SeqCst), 0);
8967        assert_eq!(ping_count.load(Ordering::SeqCst), 0);
8968    }
8969
8970    #[rstest]
8971    #[tokio::test]
8972    async fn test_reconnect_buffer_drain_stops_after_reconnect_request() {
8973        let state = Arc::new(BlockingFailState::default());
8974
8975        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
8976            state: Arc::clone(&state),
8977        });
8978
8979        let (mut writer, _reader) = transport.split();
8980        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8981        let task_connection_state = Arc::clone(&connection_state);
8982
8983        let drain_task = tokio::spawn(async move {
8984            let auth_tracker = Arc::new(OnceLock::new());
8985            let reconnect_buffer_waits_for_auth = AtomicBool::new(false);
8986            let (sender, mut receiver) = crate::writer::channel(2);
8987            sender.send(Message::text("admitted")).unwrap();
8988            sender.send(Message::text("held-for-reconnect")).unwrap();
8989            let mut buffer = VecDeque::from([
8990                receiver.recv().await.unwrap(),
8991                receiver.recv().await.unwrap(),
8992            ]);
8993            let send_error = WebSocketClientInner::drain_reconnect_buffer(
8994                &mut buffer,
8995                &mut writer,
8996                &task_connection_state,
8997                &auth_tracker,
8998                &reconnect_buffer_waits_for_auth,
8999            )
9000            .await;
9001            (buffer, send_error)
9002        });
9003
9004        state.send_entered_notify.notified().await;
9005        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
9006        state.release_send();
9007
9008        let (buffer, send_error) = tokio::time::timeout(TEST_TIMEOUT, drain_task)
9009            .await
9010            .expect("buffer drain should stop after reconnect acceptance")
9011            .unwrap();
9012        assert!(!send_error);
9013        assert_eq!(
9014            buffer
9015                .into_iter()
9016                .map(|(message, _)| message)
9017                .collect::<Vec<_>>(),
9018            vec![Message::text("held-for-reconnect")]
9019        );
9020    }
9021
9022    #[rstest]
9023    #[case(0)]
9024    #[case(usize::MAX)]
9025    #[tokio::test]
9026    async fn test_stream_writer_rejects_invalid_capacity(#[case] capacity: usize) {
9027        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9028            state: Arc::new(BlockingFailState::default()),
9029        });
9030
9031        let (writer, _reader) = transport.split();
9032        let mut config = WebSocketConfig::builder()
9033            .url("ws://localhost".to_string())
9034            .build()
9035            .unwrap();
9036        config.writer_capacity = Some(capacity);
9037
9038        let result = WebSocketClientInner::new_with_writer(config, writer).await;
9039
9040        assert!(
9041            matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9042        );
9043    }
9044
9045    #[rstest]
9046    #[case(0)]
9047    #[case(usize::MAX)]
9048    #[tokio::test]
9049    async fn test_handlerless_connect_rejects_invalid_capacity(#[case] capacity: usize) {
9050        let mut config = WebSocketConfig::builder()
9051            .url("ws://127.0.0.1:1".to_string())
9052            .build()
9053            .unwrap();
9054        config.writer_capacity = Some(capacity);
9055
9056        let result = WebSocketClientInner::connect_url(config, None, None).await;
9057
9058        assert!(
9059            matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9060        );
9061    }
9062
9063    #[rstest]
9064    #[tokio::test(start_paused = true)]
9065    async fn test_stalled_websocket_send_reconnects_and_replays() {
9066        let state = Arc::new(BlockingFailState::default());
9067
9068        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9069            state: Arc::clone(&state),
9070        });
9071
9072        let (writer, _reader) = transport.split();
9073        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9074        let state_notify = Arc::new(tokio::sync::Notify::new());
9075        let auth_tracker = Arc::new(OnceLock::new());
9076        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9077        let (writer_tx, writer_rx) = crate::writer::channel(2);
9078        let states = Arc::new(Mutex::new(Vec::new()));
9079        let states_callback = Arc::clone(&states);
9080        let sink = SocketStateSink::new(move |state| {
9081            states_callback.lock().push(state);
9082        });
9083
9084        let write_task = WebSocketClientInner::spawn_write_task(
9085            Arc::clone(&connection_state),
9086            Arc::clone(&state_notify),
9087            Arc::new(AtomicBool::new(true)),
9088            writer,
9089            writer_rx,
9090            Arc::new(AtomicU64::new(0)),
9091            Arc::clone(&auth_tracker),
9092            reconnect_buffer_waits_for_auth,
9093            Some(sink),
9094        );
9095
9096        writer_tx
9097            .send(WriterCommand::Send(Message::text("complete-message")))
9098            .unwrap();
9099        state.send_entered_notify.notified().await;
9100        writer_tx
9101            .send(WriterCommand::Send(Message::text("second-message")))
9102            .unwrap();
9103        assert_eq!(
9104            ConnectionMode::from_atomic(&connection_state),
9105            ConnectionMode::Active
9106        );
9107
9108        for _ in 0..100 {
9109            assert!(matches!(
9110                writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9111                Err(SendError::BufferFull)
9112            ));
9113        }
9114
9115        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9116        tokio::task::yield_now().await;
9117        assert_eq!(
9118            ConnectionMode::from_atomic(&connection_state),
9119            ConnectionMode::Reconnect
9120        );
9121        assert!(matches!(
9122            writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9123            Err(SendError::BufferFull)
9124        ));
9125
9126        let recorded = Arc::new(Mutex::new(Vec::new()));
9127
9128        let recording_state = Arc::new(RecordingState {
9129            messages: Arc::clone(&recorded),
9130            recorded_notify: tokio::sync::Notify::new(),
9131        });
9132
9133        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9134            state: Arc::clone(&recording_state),
9135        });
9136
9137        let (new_writer, _reader) = transport.split();
9138        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9139        writer_tx
9140            .send_update(WriterCommand::Update(new_writer, update_tx))
9141            .unwrap();
9142
9143        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9144        assert_eq!(
9145            tokio::time::timeout(Duration::from_secs(1), update_rx)
9146                .await
9147                .expect("writer update should not remain queued behind a stalled send")
9148                .unwrap(),
9149            1,
9150            "the replacement sink should install as connection epoch 1"
9151        );
9152        assert_eq!(
9153            ConnectionMode::from_atomic(&connection_state),
9154            ConnectionMode::Reconnect
9155        );
9156
9157        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9158        state_notify.notify_waiters();
9159        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9160        recording_state.recorded_notify.notified().await;
9161        assert_eq!(
9162            recorded.lock().as_slice(),
9163            &[
9164                Message::text("complete-message"),
9165                Message::text("second-message")
9166            ]
9167        );
9168
9169        writer_tx
9170            .send(WriterCommand::Send(Message::text("second-message")))
9171            .unwrap();
9172
9173        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9174        state_notify.notify_waiters();
9175        drop(writer_tx);
9176        write_task.await.unwrap();
9177
9178        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
9179    }
9180
9181    #[rstest]
9182    #[case(Message::Ping(vec![1, 2, 3].into()))]
9183    #[case(Message::Pong(vec![4, 5, 6].into()))]
9184    #[case(Message::Close(None))]
9185    #[tokio::test(start_paused = true)]
9186    async fn test_stalled_control_frame_is_not_replayed(#[case] control: Message) {
9187        // A control frame belongs to the connection it was issued on, so a failed write must
9188        // drop it instead of replaying it onto the replacement connection.
9189        let state = Arc::new(BlockingFailState::default());
9190
9191        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9192            state: Arc::clone(&state),
9193        });
9194
9195        let (writer, _reader) = transport.split();
9196        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9197        let state_notify = Arc::new(tokio::sync::Notify::new());
9198        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9199        let write_task = WebSocketClientInner::spawn_write_task(
9200            Arc::clone(&connection_state),
9201            Arc::clone(&state_notify),
9202            Arc::new(AtomicBool::new(true)),
9203            writer,
9204            writer_rx,
9205            Arc::new(AtomicU64::new(0)),
9206            Arc::new(OnceLock::new()),
9207            Arc::new(AtomicBool::new(false)),
9208            None,
9209        );
9210
9211        writer_tx.send(WriterCommand::Send(control)).unwrap();
9212        state.send_entered_notify.notified().await;
9213
9214        let recorded = Arc::new(Mutex::new(Vec::new()));
9215
9216        let recording_state = Arc::new(RecordingState {
9217            messages: Arc::clone(&recorded),
9218            recorded_notify: tokio::sync::Notify::new(),
9219        });
9220
9221        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9222            state: recording_state,
9223        });
9224
9225        let (new_writer, _reader) = transport.split();
9226        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9227        writer_tx
9228            .send(WriterCommand::Update(new_writer, update_tx))
9229            .unwrap();
9230
9231        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9232        assert_eq!(
9233            tokio::time::timeout(Duration::from_secs(1), update_rx)
9234                .await
9235                .expect("writer update should not remain queued behind a stalled send")
9236                .unwrap(),
9237            1,
9238            "the replacement sink should install as connection epoch 1"
9239        );
9240        assert_eq!(
9241            ConnectionMode::from_atomic(&connection_state),
9242            ConnectionMode::Reconnect,
9243            "a failed control-frame write should still trigger a reconnect"
9244        );
9245
9246        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9247        state_notify.notify_waiters();
9248
9249        // Each pass returns the writer to the loop top, where a buffered frame would drain
9250        for _ in 0..5 {
9251            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9252        }
9253
9254        let replayed = recorded.lock().clone();
9255        assert!(
9256            replayed.is_empty(),
9257            "a failed control frame must not reach the replacement connection, was {replayed:?}"
9258        );
9259
9260        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9261        state_notify.notify_waiters();
9262        drop(writer_tx);
9263        write_task.await.unwrap();
9264    }
9265
9266    #[rstest]
9267    #[case(Message::Ping(vec![1, 2, 3].into()))]
9268    #[case(Message::Pong(vec![4, 5, 6].into()))]
9269    #[case(Message::Close(None))]
9270    #[tokio::test(start_paused = true)]
9271    async fn test_control_frame_enqueued_during_reconnect_is_not_replayed(
9272        #[case] control: Message,
9273    ) {
9274        // `send_pong` and the heartbeat task check for an active connection before enqueueing,
9275        // so a mode flip can land their frame on the writer's reconnect-mode branch instead.
9276        let recorded = Arc::new(Mutex::new(Vec::new()));
9277
9278        let recording_state = Arc::new(RecordingState {
9279            messages: Arc::clone(&recorded),
9280            recorded_notify: tokio::sync::Notify::new(),
9281        });
9282
9283        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9284            state: recording_state,
9285        });
9286
9287        let (writer, _reader) = transport.split();
9288        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9289        let state_notify = Arc::new(tokio::sync::Notify::new());
9290        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9291        let write_task = WebSocketClientInner::spawn_write_task(
9292            Arc::clone(&connection_state),
9293            Arc::clone(&state_notify),
9294            Arc::new(AtomicBool::new(true)),
9295            writer,
9296            writer_rx,
9297            Arc::new(AtomicU64::new(0)),
9298            Arc::new(OnceLock::new()),
9299            Arc::new(AtomicBool::new(false)),
9300            None,
9301        );
9302
9303        writer_tx.send(WriterCommand::Send(control)).unwrap();
9304        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9305
9306        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9307        state_notify.notify_waiters();
9308
9309        // Each pass returns the writer to the loop top, where a buffered frame would drain
9310        for _ in 0..5 {
9311            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9312        }
9313
9314        let replayed = recorded.lock().clone();
9315        assert!(
9316            replayed.is_empty(),
9317            "a control frame enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9318        );
9319
9320        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9321        state_notify.notify_waiters();
9322        drop(writer_tx);
9323        write_task.await.unwrap();
9324    }
9325
9326    #[rstest]
9327    #[tokio::test(start_paused = true)]
9328    async fn test_stalled_text_heartbeat_is_not_replayed() {
9329        let state = Arc::new(BlockingFailState::default());
9330
9331        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9332            state: Arc::clone(&state),
9333        });
9334
9335        let (writer, _reader) = transport.split();
9336        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9337        let state_notify = Arc::new(tokio::sync::Notify::new());
9338        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9339        let write_task = WebSocketClientInner::spawn_write_task(
9340            Arc::clone(&connection_state),
9341            Arc::clone(&state_notify),
9342            Arc::new(AtomicBool::new(true)),
9343            writer,
9344            writer_rx,
9345            Arc::new(AtomicU64::new(0)),
9346            Arc::new(OnceLock::new()),
9347            Arc::new(AtomicBool::new(false)),
9348            None,
9349        );
9350
9351        writer_tx
9352            .send(WriterCommand::Heartbeat(Message::text(
9353                "{\"op\":\"heartbeat\"}",
9354            )))
9355            .unwrap();
9356        state.send_entered_notify.notified().await;
9357
9358        let recorded = Arc::new(Mutex::new(Vec::new()));
9359
9360        let recording_state = Arc::new(RecordingState {
9361            messages: Arc::clone(&recorded),
9362            recorded_notify: tokio::sync::Notify::new(),
9363        });
9364
9365        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9366            state: recording_state,
9367        });
9368
9369        let (new_writer, _reader) = transport.split();
9370        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9371        writer_tx
9372            .send(WriterCommand::Update(new_writer, update_tx))
9373            .unwrap();
9374
9375        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9376        assert_eq!(
9377            tokio::time::timeout(Duration::from_secs(1), update_rx)
9378                .await
9379                .expect("writer update should not remain queued behind a stalled send")
9380                .unwrap(),
9381            1,
9382            "the replacement sink should install as connection epoch 1"
9383        );
9384        assert_eq!(
9385            ConnectionMode::from_atomic(&connection_state),
9386            ConnectionMode::Reconnect,
9387            "a failed heartbeat write should still trigger a reconnect"
9388        );
9389
9390        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9391        state_notify.notify_waiters();
9392
9393        for _ in 0..5 {
9394            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9395        }
9396
9397        let replayed = recorded.lock().clone();
9398        assert!(
9399            replayed.is_empty(),
9400            "a failed text heartbeat must not reach the replacement connection, was {replayed:?}"
9401        );
9402
9403        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9404        state_notify.notify_waiters();
9405        drop(writer_tx);
9406        write_task.await.unwrap();
9407    }
9408
9409    #[rstest]
9410    #[tokio::test(start_paused = true)]
9411    async fn test_text_heartbeat_enqueued_during_reconnect_is_not_replayed() {
9412        let recorded = Arc::new(Mutex::new(Vec::new()));
9413
9414        let recording_state = Arc::new(RecordingState {
9415            messages: Arc::clone(&recorded),
9416            recorded_notify: tokio::sync::Notify::new(),
9417        });
9418
9419        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9420            state: recording_state,
9421        });
9422
9423        let (writer, _reader) = transport.split();
9424        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9425        let state_notify = Arc::new(tokio::sync::Notify::new());
9426        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9427        let write_task = WebSocketClientInner::spawn_write_task(
9428            Arc::clone(&connection_state),
9429            Arc::clone(&state_notify),
9430            Arc::new(AtomicBool::new(true)),
9431            writer,
9432            writer_rx,
9433            Arc::new(AtomicU64::new(0)),
9434            Arc::new(OnceLock::new()),
9435            Arc::new(AtomicBool::new(false)),
9436            None,
9437        );
9438
9439        writer_tx
9440            .send(WriterCommand::Heartbeat(Message::text(
9441                "{\"op\":\"heartbeat\"}",
9442            )))
9443            .unwrap();
9444        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9445
9446        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9447        state_notify.notify_waiters();
9448
9449        for _ in 0..5 {
9450            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9451        }
9452
9453        let replayed = recorded.lock().clone();
9454        assert!(
9455            replayed.is_empty(),
9456            "a text heartbeat enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9457        );
9458
9459        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9460        state_notify.notify_waiters();
9461        drop(writer_tx);
9462        write_task.await.unwrap();
9463    }
9464
9465    #[rstest]
9466    #[case::text(
9467        Some("{\"op\":\"heartbeat\"}"),
9468        Message::text("{\"op\":\"heartbeat\"}")
9469    )]
9470    #[case::ping(None, Message::Ping(vec![].into()))]
9471    #[tokio::test(start_paused = true)]
9472    async fn test_heartbeat_task_enqueues_writer_heartbeat_command(
9473        #[case] payload: Option<&str>,
9474        #[case] expected: Message,
9475    ) {
9476        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9477        let (writer_tx, mut writer_rx) =
9478            crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9479        let task = WebSocketClientInner::spawn_heartbeat_task(
9480            Arc::clone(&connection_state),
9481            1,
9482            payload.map(ToString::to_string),
9483            writer_tx,
9484        );
9485
9486        tokio::time::advance(Duration::from_secs(1)).await;
9487        let cmd = writer_rx
9488            .recv()
9489            .await
9490            .expect("heartbeat task should enqueue");
9491
9492        match cmd.0 {
9493            WriterCommand::Heartbeat(msg) => assert_eq!(msg, expected),
9494            other => panic!("expected Heartbeat, was {other:?}"),
9495        }
9496
9497        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9498        tokio::time::advance(Duration::from_secs(1)).await;
9499        task.await.unwrap();
9500    }
9501
9502    #[cfg(target_os = "linux")]
9503    #[rstest]
9504    #[tokio::test]
9505    async fn test_heartbeat_task_warns_when_writer_channel_closed() {
9506        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
9507        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9508        let (writer_tx, mut writer_rx) =
9509            crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9510        let task = WebSocketClientInner::spawn_heartbeat_task(
9511            Arc::clone(&connection_state),
9512            1,
9513            None,
9514            writer_tx,
9515        );
9516
9517        let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
9518            .await
9519            .expect("timed out waiting for the first heartbeat")
9520            .expect("heartbeat channel closed before the first heartbeat");
9521        drop(writer_rx);
9522        tokio::time::sleep(Duration::from_millis(1_200)).await;
9523
9524        let messages = capture.messages();
9525        assert!(
9526            messages.iter().any(|(level, message)| {
9527                *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
9528            }),
9529            "expected WARN when the writer channel is closed, was {messages:?}"
9530        );
9531        assert!(
9532            !messages.iter().any(|(level, _)| *level == Level::Error),
9533            "a closed writer channel must not log ERROR, was {messages:?}"
9534        );
9535
9536        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9537        tokio::time::timeout(Duration::from_secs(2), task)
9538            .await
9539            .expect("heartbeat task should stop after close")
9540            .unwrap();
9541    }
9542
9543    #[rstest]
9544    #[tokio::test(start_paused = true)]
9545    async fn test_stalled_ownership_bound_send_times_out_without_replay() {
9546        let state = Arc::new(BlockingFailState::default());
9547
9548        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9549            state: Arc::clone(&state),
9550        });
9551
9552        let (writer, _reader) = transport.split();
9553        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9554        let state_notify = Arc::new(tokio::sync::Notify::new());
9555        let auth_tracker = Arc::new(OnceLock::new());
9556        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9557        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9558        let write_task = WebSocketClientInner::spawn_write_task(
9559            Arc::clone(&connection_state),
9560            Arc::clone(&state_notify),
9561            Arc::new(AtomicBool::new(true)),
9562            writer,
9563            writer_rx,
9564            Arc::new(AtomicU64::new(0)),
9565            Arc::clone(&auth_tracker),
9566            reconnect_buffer_waits_for_auth,
9567            None,
9568        );
9569
9570        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
9571        writer_tx
9572            .send_control(WriterCommand::SendOnConnection {
9573                message: Message::text("ownership-bound"),
9574                connection_epoch: 0,
9575                response_tx,
9576            })
9577            .unwrap();
9578
9579        state.send_entered_notify.notified().await;
9580
9581        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9582        let outcome = tokio::time::timeout(Duration::from_secs(1), response_rx)
9583            .await
9584            .expect("a stalled ownership-bound send must not wedge the writer task")
9585            .unwrap();
9586        assert!(
9587            matches!(outcome, Err(SendError::WriteTimeout)),
9588            "expected the write deadline to be reported, was {outcome:?}"
9589        );
9590        assert_eq!(
9591            ConnectionMode::from_atomic(&connection_state),
9592            ConnectionMode::Reconnect
9593        );
9594
9595        // The timed-out message is ownership-bound, so unlike an ordinary send it must
9596        // NOT be buffered for replay onto the replacement sink.
9597        let recorded = Arc::new(Mutex::new(Vec::new()));
9598
9599        let recording_state = Arc::new(RecordingState {
9600            messages: Arc::clone(&recorded),
9601            recorded_notify: tokio::sync::Notify::new(),
9602        });
9603
9604        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9605            state: Arc::clone(&recording_state),
9606        });
9607
9608        let (new_writer, _reader) = transport.split();
9609        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9610        writer_tx
9611            .send(WriterCommand::Update(new_writer, update_tx))
9612            .unwrap();
9613
9614        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9615        assert_eq!(
9616            tokio::time::timeout(Duration::from_secs(1), update_rx)
9617                .await
9618                .expect("writer update should not remain queued behind a stalled send")
9619                .unwrap(),
9620            1
9621        );
9622
9623        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9624        state_notify.notify_waiters();
9625        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9626
9627        // Barrier: two sentinels bound to the NEW epoch, the second sent only after the first
9628        // is acknowledged. The writer can accept the first from a `recv()` that was already
9629        // pending when the mode flipped, so only the second is necessarily processed after the
9630        // intervening loop-top drain - which is where a mistakenly buffered message would be
9631        // replayed. One sentinel alone would leave the ordering to chance.
9632        for name in ["sentinel-1", "sentinel-2"] {
9633            let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
9634            writer_tx
9635                .send_control(WriterCommand::SendOnConnection {
9636                    message: Message::text(name),
9637                    connection_epoch: 1,
9638                    response_tx: sentinel_tx,
9639                })
9640                .unwrap();
9641
9642            recording_state.recorded_notify.notified().await;
9643            sentinel_rx
9644                .await
9645                .unwrap()
9646                .expect("the sentinel should send on the replacement connection");
9647        }
9648
9649        assert_eq!(
9650            recorded.lock().as_slice(),
9651            &[Message::text("sentinel-1"), Message::text("sentinel-2")],
9652            "an ownership-bound message must never be replayed after its deadline expires"
9653        );
9654
9655        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9656        state_notify.notify_waiters();
9657        drop(writer_tx);
9658        write_task.await.unwrap();
9659    }
9660
9661    #[rstest]
9662    #[tokio::test(start_paused = true)]
9663    async fn test_stalled_websocket_replay_reconnects_and_retries_buffer() {
9664        let initial_messages = Arc::new(Mutex::new(Vec::new()));
9665
9666        let initial_recording_state = Arc::new(RecordingState {
9667            messages: initial_messages,
9668            recorded_notify: tokio::sync::Notify::new(),
9669        });
9670
9671        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9672            state: initial_recording_state,
9673        });
9674
9675        let (writer, _reader) = transport.split();
9676        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9677        let state_notify = Arc::new(tokio::sync::Notify::new());
9678        let auth_tracker = Arc::new(OnceLock::new());
9679        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9680        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9681        let write_task = WebSocketClientInner::spawn_write_task(
9682            Arc::clone(&connection_state),
9683            Arc::clone(&state_notify),
9684            Arc::new(AtomicBool::new(true)),
9685            writer,
9686            writer_rx,
9687            Arc::new(AtomicU64::new(0)),
9688            Arc::clone(&auth_tracker),
9689            reconnect_buffer_waits_for_auth,
9690            None,
9691        );
9692
9693        writer_tx
9694            .send(WriterCommand::Send(Message::text("buffered-message")))
9695            .unwrap();
9696
9697        let blocking_state = Arc::new(BlockingFailState::default());
9698
9699        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9700            state: Arc::clone(&blocking_state),
9701        });
9702
9703        let (blocking_writer, _reader) = transport.split();
9704        let (blocking_tx, blocking_rx) = tokio::sync::oneshot::channel();
9705        writer_tx
9706            .send(WriterCommand::Update(blocking_writer, blocking_tx))
9707            .unwrap();
9708        assert_eq!(blocking_rx.await.unwrap(), 1);
9709
9710        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9711        state_notify.notify_waiters();
9712        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9713        blocking_state.send_entered_notify.notified().await;
9714
9715        let recorded = Arc::new(Mutex::new(Vec::new()));
9716
9717        let recording_state = Arc::new(RecordingState {
9718            messages: Arc::clone(&recorded),
9719            recorded_notify: tokio::sync::Notify::new(),
9720        });
9721
9722        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9723            state: Arc::clone(&recording_state),
9724        });
9725
9726        let (new_writer, _reader) = transport.split();
9727        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9728        writer_tx
9729            .send(WriterCommand::Update(new_writer, update_tx))
9730            .unwrap();
9731
9732        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9733        assert_eq!(
9734            tokio::time::timeout(Duration::from_secs(1), update_rx)
9735                .await
9736                .expect("writer update should not remain queued behind stalled replay")
9737                .unwrap(),
9738            2,
9739            "the second replacement sink should install as connection epoch 2"
9740        );
9741        assert_eq!(
9742            ConnectionMode::from_atomic(&connection_state),
9743            ConnectionMode::Reconnect
9744        );
9745
9746        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9747        state_notify.notify_waiters();
9748        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9749        recording_state.recorded_notify.notified().await;
9750        assert_eq!(
9751            recorded.lock().as_slice(),
9752            &[Message::text("buffered-message")]
9753        );
9754
9755        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9756        state_notify.notify_waiters();
9757        drop(writer_tx);
9758        write_task.await.unwrap();
9759    }
9760
9761    #[rstest]
9762    #[tokio::test]
9763    async fn test_new_with_writer_rejects_zero_heartbeat() {
9764        // Stream mode shares connect_url's validation: a zero heartbeat would
9765        // spawn a busy-loop ping flood
9766        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9767            state: Arc::new(BlockingFailState::default()),
9768        });
9769
9770        let (writer, _reader) = transport.split();
9771
9772        let config = WebSocketConfig {
9773            url: "ws://127.0.0.1:1".to_string(),
9774            headers: vec![],
9775            heartbeat_interval_secs: Some(0),
9776            heartbeat_payload: None,
9777            connect_timeout_ms: None,
9778            reconnect_delay_initial_ms: None,
9779            reconnect_delay_max_ms: None,
9780            reconnect_backoff_factor: None,
9781            reconnect_jitter_ms: None,
9782            reconnect_max_attempts: None,
9783            heartbeat_timeout_secs: None,
9784            idle_timeout_ms: None,
9785            writer_capacity: None,
9786            backend: TransportBackend::Tungstenite,
9787            proxy_url: None,
9788            max_message_size_bytes: None,
9789            max_frame_size_bytes: None,
9790        };
9791
9792        let err = WebSocketClientInner::new_with_writer(config, writer)
9793            .await
9794            .expect_err("zero heartbeat should be rejected in stream mode");
9795        assert!(
9796            err.to_string()
9797                .contains("Heartbeat interval cannot be zero"),
9798            "error should mention zero heartbeat, was: {err}"
9799        );
9800    }
9801
9802    #[rstest]
9803    #[tokio::test]
9804    async fn test_connect_times_out_on_silent_server() {
9805        // A server that accepts TCP but never completes the WebSocket upgrade
9806        // must not hang connect() indefinitely
9807        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9808        let port = listener.local_addr().unwrap().port();
9809
9810        let server = task::spawn(async move {
9811            // Accept and hold the socket open without responding
9812            if let Ok((_stream, _)) = listener.accept().await {
9813                sleep(Duration::from_secs(30)).await;
9814            }
9815        });
9816
9817        let (handler, _rx) = channel_message_handler();
9818
9819        let config = WebSocketConfig {
9820            url: format!("ws://127.0.0.1:{port}"),
9821            headers: vec![],
9822            heartbeat_interval_secs: None,
9823            heartbeat_payload: None,
9824            connect_timeout_ms: Some(500),
9825            reconnect_delay_initial_ms: Some(50),
9826            reconnect_delay_max_ms: Some(100),
9827            reconnect_backoff_factor: Some(1.0),
9828            reconnect_jitter_ms: Some(0),
9829            reconnect_max_attempts: None,
9830            heartbeat_timeout_secs: None,
9831            idle_timeout_ms: None,
9832            writer_capacity: None,
9833            backend: TransportBackend::Tungstenite,
9834            proxy_url: None,
9835            max_message_size_bytes: None,
9836            max_frame_size_bytes: None,
9837        };
9838
9839        let result = tokio::time::timeout(
9840            Duration::from_secs(5),
9841            WebSocketClient::builder()
9842                .config(config)
9843                .message_handler(handler)
9844                .connect(),
9845        )
9846        .await
9847        .expect("connect should not hang on a silent server");
9848
9849        assert!(result.is_err(), "connect should fail with a timeout error");
9850        let err_msg = result.unwrap_err().to_string();
9851        assert!(
9852            err_msg.contains("timed out"),
9853            "error should mention the timeout, was: {err_msg}"
9854        );
9855
9856        server.abort();
9857    }
9858
9859    #[rstest]
9860    #[tokio::test]
9861    async fn test_reconnect_succeeds_with_timeout_shorter_than_swap_ceremony() {
9862        // Regression: the reconnect timeout used to cover the writer swap and
9863        // graceful-shutdown delays (~200ms minimum), so a short timeout caused
9864        // every otherwise-successful reconnect to be discarded mid-swap and the
9865        // already-swapped writer to be orphaned. The timeout now bounds only
9866        // connection establishment.
9867        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9868        let port = listener.local_addr().unwrap().port();
9869        let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel();
9870
9871        let server = task::spawn(async move {
9872            // Keep the first socket until connect() returns; an immediate drop
9873            // can fail the initial handshake
9874            if let Ok((stream, _)) = listener.accept().await
9875                && let Ok(ws) = accept_async(stream).await
9876            {
9877                let _ = release_first_rx.await;
9878                drop(ws);
9879            }
9880
9881            // A timed-out reconnect can consume an accept without completing the
9882            // handshake, so keep accepting and announce on every replacement
9883            loop {
9884                let Ok((stream, _)) = listener.accept().await else {
9885                    break;
9886                };
9887
9888                if let Ok(mut ws) = accept_async(stream).await {
9889                    loop {
9890                        if ws
9891                            .send(WsMessage::Text("reconnected-msg".to_string().into()))
9892                            .await
9893                            .is_err()
9894                        {
9895                            break;
9896                        }
9897
9898                        sleep(Duration::from_millis(50)).await;
9899                    }
9900                }
9901            }
9902        });
9903
9904        let (handler, mut rx) = channel_message_handler();
9905
9906        let config = WebSocketConfig {
9907            url: format!("ws://127.0.0.1:{port}"),
9908            headers: vec![],
9909            heartbeat_interval_secs: None,
9910            heartbeat_payload: None,
9911            connect_timeout_ms: Some(150), // Shorter than the ~200ms swap ceremony
9912            reconnect_delay_initial_ms: Some(25),
9913            reconnect_delay_max_ms: Some(50),
9914            reconnect_backoff_factor: Some(1.0),
9915            reconnect_jitter_ms: Some(0),
9916            reconnect_max_attempts: None,
9917            heartbeat_timeout_secs: None,
9918            idle_timeout_ms: None,
9919            writer_capacity: None,
9920            backend: TransportBackend::Tungstenite,
9921            proxy_url: None,
9922            max_message_size_bytes: None,
9923            max_frame_size_bytes: None,
9924        };
9925
9926        let client = WebSocketClient::builder()
9927            .config(config)
9928            .message_handler(handler)
9929            .connect()
9930            .await
9931            .expect("Initial WebSocket connection should succeed before triggering reconnect");
9932        wait_until_async(|| async { client.is_active() }, Duration::from_secs(2)).await;
9933        release_first_tx
9934            .send(())
9935            .expect("server should still be holding the first connection");
9936
9937        let received = tokio::time::timeout(Duration::from_secs(5), async {
9938            loop {
9939                match rx.recv().await {
9940                    Some(WsMessage::Text(text)) if text.as_str() == "reconnected-msg" => {
9941                        return true;
9942                    }
9943                    Some(_) => {}
9944                    None => return false,
9945                }
9946            }
9947        })
9948        .await
9949        .expect("Timed out waiting for a message from the replacement WebSocket connection");
9950
9951        assert!(
9952            received,
9953            "Replacement WebSocket message channel closed before delivering the reconnect message"
9954        );
9955        assert!(
9956            client.is_active(),
9957            "WebSocket client should be active after delivering the reconnect message"
9958        );
9959
9960        client.disconnect().await;
9961        server.abort();
9962    }
9963
9964    #[rstest]
9965    #[tokio::test]
9966    async fn test_idle_timeout_fires_under_ping_flood() {
9967        // Regression: the idle check used to run only when nothing arrived for a
9968        // full check interval (10ms), so pings flooding faster than that starved
9969        // it and a ping-only zombie connection never tripped the timeout
9970        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9971        let port = listener.local_addr().unwrap().port();
9972        let server = task::spawn(async move {
9973            let (stream, _) = listener.accept().await.unwrap();
9974            let mut ws = accept_async(stream).await.unwrap();
9975
9976            for _ in 0..600 {
9977                sleep(Duration::from_millis(5)).await;
9978
9979                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
9980                    break;
9981                }
9982            }
9983        });
9984
9985        let (handler, _rx) = channel_message_handler();
9986
9987        let config = WebSocketConfig {
9988            url: format!("ws://127.0.0.1:{port}"),
9989            headers: vec![],
9990            heartbeat_interval_secs: None,
9991            heartbeat_payload: None,
9992            connect_timeout_ms: Some(2_000),
9993            reconnect_delay_initial_ms: Some(50),
9994            reconnect_delay_max_ms: Some(100),
9995            reconnect_backoff_factor: Some(1.0),
9996            reconnect_jitter_ms: Some(0),
9997            reconnect_max_attempts: Some(1),
9998            heartbeat_timeout_secs: None,
9999            idle_timeout_ms: Some(500),
10000            writer_capacity: None,
10001            backend: TransportBackend::Tungstenite,
10002            proxy_url: None,
10003            max_message_size_bytes: None,
10004            max_frame_size_bytes: None,
10005        };
10006
10007        let client = WebSocketClient::builder()
10008            .config(config)
10009            .message_handler(handler)
10010            .connect()
10011            .await
10012            .unwrap();
10013
10014        assert!(client.is_active());
10015
10016        wait_until_async(
10017            || async { client.is_reconnecting() || client.is_disconnected() },
10018            Duration::from_millis(1_500),
10019        )
10020        .await;
10021
10022        assert!(
10023            !client.is_active(),
10024            "Client should not be active after idle timeout under a ping flood"
10025        );
10026
10027        client.disconnect().await;
10028        server.abort();
10029    }
10030
10031    #[rstest]
10032    #[tokio::test]
10033    async fn test_send_failure_does_not_overwrite_disconnect() {
10034        let state = Arc::new(BlockingFailState::default());
10035
10036        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10037            state: Arc::clone(&state),
10038        });
10039
10040        let (writer, _reader) = transport.split();
10041
10042        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10043        let state_notify = Arc::new(tokio::sync::Notify::new());
10044        let auth_tracker = Arc::new(OnceLock::new());
10045        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
10046        let connection_epoch = Arc::new(AtomicU64::new(0));
10047
10048        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10049        let write_task = WebSocketClientInner::spawn_write_task(
10050            Arc::clone(&connection_state),
10051            Arc::clone(&state_notify),
10052            Arc::new(AtomicBool::new(true)),
10053            writer,
10054            writer_rx,
10055            connection_epoch,
10056            Arc::clone(&auth_tracker),
10057            Arc::clone(&reconnect_buffer_waits_for_auth),
10058            None,
10059        );
10060
10061        writer_tx
10062            .send(WriterCommand::Send(Message::text("doomed")))
10063            .unwrap();
10064
10065        // Wait until the writer task is blocked inside the transport send
10066        wait_until_async(
10067            || {
10068                let state = Arc::clone(&state);
10069                async move { state.send_entered.load(Ordering::SeqCst) }
10070            },
10071            Duration::from_secs(2),
10072        )
10073        .await;
10074
10075        // Disconnect lands while the send is in flight, then the send fails;
10076        // the writer error path must not overwrite DISCONNECT with RECONNECT
10077        connection_state.store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
10078        state.trigger_failure();
10079
10080        tokio::time::timeout(Duration::from_secs(2), write_task)
10081            .await
10082            .expect("write task should exit after disconnect")
10083            .unwrap();
10084
10085        assert_eq!(
10086            ConnectionMode::from_atomic(&connection_state),
10087            ConnectionMode::Disconnect,
10088            "Send failure must not resurrect a disconnecting client into RECONNECT"
10089        );
10090    }
10091
10092    #[tokio::test]
10093    async fn send_on_connection_write_failure_reports_broken_pipe_and_reconnects() {
10094        let state = Arc::new(BlockingFailState::default());
10095
10096        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10097            state: Arc::clone(&state),
10098        });
10099
10100        let (writer, _reader) = transport.split();
10101        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10102        let state_notify = Arc::new(tokio::sync::Notify::new());
10103        let connection_epoch = Arc::new(AtomicU64::new(0));
10104        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10105        let write_task = WebSocketClientInner::spawn_write_task(
10106            Arc::clone(&connection_state),
10107            Arc::clone(&state_notify),
10108            Arc::new(AtomicBool::new(true)),
10109            writer,
10110            writer_rx,
10111            Arc::clone(&connection_epoch),
10112            Arc::new(OnceLock::new()),
10113            Arc::new(AtomicBool::new(false)),
10114            None,
10115        );
10116
10117        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10118        writer_tx
10119            .send_control(WriterCommand::SendOnConnection {
10120                message: Message::text("doomed"),
10121                connection_epoch: 0,
10122                response_tx,
10123            })
10124            .unwrap();
10125
10126        wait_until_async(
10127            || {
10128                let state = Arc::clone(&state);
10129                async move { state.send_entered.load(Ordering::SeqCst) }
10130            },
10131            Duration::from_secs(2),
10132        )
10133        .await;
10134
10135        state.trigger_failure();
10136
10137        match response_rx.await.unwrap().unwrap_err() {
10138            SendError::BrokenPipe(message) => assert_eq!(message, "connection reset"),
10139            other => panic!("expected broken-pipe send error, was {other:?}"),
10140        }
10141
10142        wait_until_async(
10143            || async {
10144                ConnectionMode::from_atomic(&connection_state) == ConnectionMode::Reconnect
10145            },
10146            Duration::from_secs(2),
10147        )
10148        .await;
10149
10150        assert_eq!(
10151            ConnectionMode::from_atomic(&connection_state),
10152            ConnectionMode::Reconnect,
10153        );
10154        assert_eq!(connection_epoch.load(Ordering::Acquire), 0);
10155
10156        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10157        state_notify.notify_waiters();
10158        drop(writer_tx);
10159        write_task.await.unwrap();
10160    }
10161
10162    #[rstest]
10163    #[tokio::test]
10164    async fn test_send_bytes_on_connection_rejects_stale_epoch_without_replay() {
10165        let server = RecordingServer::setup().await;
10166        let (handler, _handler_rx) = channel_message_handler();
10167        let client = WebSocketClient::builder()
10168            .config(reconnect_test_config(server.port))
10169            .message_handler(handler)
10170            .connect()
10171            .await
10172            .unwrap();
10173        let stale_epoch = client.connection_epoch();
10174
10175        client
10176            .send_bytes_on_connection(vec![0x00, 0x7f, 0x80, 0xff], None, stale_epoch)
10177            .await
10178            .unwrap();
10179        assert!(client.request_reconnect());
10180        wait_until_async(
10181            || async { client.is_active() && client.connection_epoch() == stale_epoch + 1 },
10182            TEST_TIMEOUT,
10183        )
10184        .await;
10185
10186        let stale_result = client
10187            .send_bytes_on_connection(vec![0xde, 0xad], None, stale_epoch)
10188            .await;
10189        assert!(matches!(stale_result, Err(SendError::ConnectionChanged)));
10190
10191        client
10192            .send_bytes_on_connection(
10193                vec![0xfe, 0xed, 0xfa, 0xce],
10194                None,
10195                client.connection_epoch(),
10196            )
10197            .await
10198            .unwrap();
10199        wait_until_async(
10200            || {
10201                let binary_frames = Arc::clone(&server.binary_frames);
10202                async move { binary_frames.lock().await.len() == 2 }
10203            },
10204            TEST_TIMEOUT,
10205        )
10206        .await;
10207
10208        assert_eq!(
10209            server.binary_frames().await,
10210            vec![vec![0x00, 0x7f, 0x80, 0xff], vec![0xfe, 0xed, 0xfa, 0xce]],
10211        );
10212        assert!(server.messages().await.is_empty());
10213
10214        client.disconnect().await;
10215    }
10216
10217    #[tokio::test]
10218    async fn send_on_connection_rejects_stale_epoch_without_replay() {
10219        let server = RecordingServer::setup().await;
10220        let url = format!("ws://127.0.0.1:{}", server.port);
10221        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10222            &url,
10223            vec![],
10224            TransportBackend::Tungstenite,
10225            None,
10226            None,
10227            None,
10228        )
10229        .await
10230        .unwrap();
10231        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10232        let state_notify = Arc::new(tokio::sync::Notify::new());
10233        let auth_tracker = Arc::new(OnceLock::new());
10234        let connection_epoch = Arc::new(AtomicU64::new(0));
10235        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10236        let write_task = WebSocketClientInner::spawn_write_task(
10237            Arc::clone(&connection_state),
10238            Arc::clone(&state_notify),
10239            Arc::new(AtomicBool::new(true)),
10240            writer,
10241            writer_rx,
10242            Arc::clone(&connection_epoch),
10243            auth_tracker,
10244            Arc::new(AtomicBool::new(false)),
10245            None,
10246        );
10247
10248        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10249        writer_tx
10250            .send_control(WriterCommand::SendOnConnection {
10251                message: Message::text("epoch-0"),
10252                connection_epoch: 0,
10253                response_tx,
10254            })
10255            .unwrap();
10256
10257        response_rx.await.unwrap().unwrap();
10258
10259        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10260        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10261        writer_tx
10262            .send_control(WriterCommand::SendOnConnection {
10263                message: Message::text("during-reconnect"),
10264                connection_epoch: 0,
10265                response_tx,
10266            })
10267            .unwrap();
10268
10269        assert!(matches!(
10270            response_rx.await.unwrap(),
10271            Err(SendError::ConnectionChanged),
10272        ));
10273
10274        let (replacement, _reader) = WebSocketClientInner::connect_with_server(
10275            &url,
10276            vec![],
10277            TransportBackend::Tungstenite,
10278            None,
10279            None,
10280            None,
10281        )
10282        .await
10283        .unwrap();
10284        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10285        writer_tx
10286            .send(WriterCommand::Update(replacement, update_tx))
10287            .unwrap();
10288        assert_eq!(update_rx.await.unwrap(), 1);
10289        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10290
10291        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10292        writer_tx
10293            .send_control(WriterCommand::SendOnConnection {
10294                message: Message::text("stale"),
10295                connection_epoch: 0,
10296                response_tx,
10297            })
10298            .unwrap();
10299
10300        assert!(matches!(
10301            response_rx.await.unwrap(),
10302            Err(SendError::ConnectionChanged),
10303        ));
10304
10305        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10306        writer_tx
10307            .send_control(WriterCommand::SendOnConnection {
10308                message: Message::text("epoch-1"),
10309                connection_epoch: 1,
10310                response_tx,
10311            })
10312            .unwrap();
10313
10314        response_rx.await.unwrap().unwrap();
10315
10316        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10317        let (second_replacement, _reader) = WebSocketClientInner::connect_with_server(
10318            &url,
10319            vec![],
10320            TransportBackend::Tungstenite,
10321            None,
10322            None,
10323            None,
10324        )
10325        .await
10326        .unwrap();
10327        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10328        writer_tx
10329            .send(WriterCommand::Update(second_replacement, update_tx))
10330            .unwrap();
10331        assert_eq!(update_rx.await.unwrap(), 2);
10332        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10333
10334        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10335        writer_tx
10336            .send_control(WriterCommand::SendOnConnection {
10337                message: Message::text("stale-after-second-reconnect"),
10338                connection_epoch: 1,
10339                response_tx,
10340            })
10341            .unwrap();
10342
10343        assert!(matches!(
10344            response_rx.await.unwrap(),
10345            Err(SendError::ConnectionChanged),
10346        ));
10347
10348        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10349        writer_tx
10350            .send_control(WriterCommand::SendOnConnection {
10351                message: Message::text("epoch-2"),
10352                connection_epoch: 2,
10353                response_tx,
10354            })
10355            .unwrap();
10356
10357        response_rx.await.unwrap().unwrap();
10358
10359        wait_until_async(
10360            || {
10361                let messages = Arc::clone(&server.messages);
10362                async move { messages.lock().await.len() == 3 }
10363            },
10364            Duration::from_secs(2),
10365        )
10366        .await;
10367
10368        assert_eq!(connection_epoch.load(Ordering::Acquire), 2);
10369        assert_eq!(
10370            server.messages().await,
10371            vec![
10372                "epoch-0".to_string(),
10373                "epoch-1".to_string(),
10374                "epoch-2".to_string(),
10375            ],
10376        );
10377
10378        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10379        state_notify.notify_waiters();
10380        drop(writer_tx);
10381        write_task.abort();
10382    }
10383
10384    #[rstest]
10385    fn test_reconnect_buffer_action_requires_active_mode() {
10386        let connection_state = AtomicU8::new(ConnectionMode::Active.as_u8());
10387        let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
10388        let auth_tracker = Arc::new(OnceLock::new());
10389        let tracker = AuthTracker::new();
10390        tracker.succeed();
10391        auth_tracker.set(tracker).unwrap();
10392
10393        assert_eq!(
10394            WebSocketClientInner::reconnect_buffer_action(
10395                &reconnect_buffer_waits_for_auth,
10396                &auth_tracker,
10397                &connection_state,
10398            ),
10399            ReconnectBufferAction::Drain,
10400        );
10401
10402        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10403        assert_eq!(
10404            WebSocketClientInner::reconnect_buffer_action(
10405                &reconnect_buffer_waits_for_auth,
10406                &auth_tracker,
10407                &connection_state,
10408            ),
10409            ReconnectBufferAction::Wait,
10410        );
10411    }
10412
10413    #[tokio::test]
10414    async fn test_write_task_waits_for_auth_before_replaying_buffer() {
10415        use nautilus_common::testing::wait_until_async;
10416
10417        let server = RecordingServer::setup().await;
10418        let url = format!("ws://127.0.0.1:{}", server.port);
10419        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10420            &url,
10421            vec![],
10422            TransportBackend::Tungstenite,
10423            None,
10424            None,
10425            None,
10426        )
10427        .await
10428        .unwrap();
10429
10430        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10431        let state_notify = Arc::new(tokio::sync::Notify::new());
10432        let auth_tracker = Arc::new(OnceLock::new());
10433        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10434        let connection_epoch = Arc::new(AtomicU64::new(0));
10435        let tracker = AuthTracker::new();
10436        auth_tracker.set(tracker.clone()).unwrap();
10437
10438        let (writer_tx, writer_rx) = crate::writer::channel(1);
10439        let write_task = WebSocketClientInner::spawn_write_task(
10440            Arc::clone(&connection_state),
10441            Arc::clone(&state_notify),
10442            Arc::new(AtomicBool::new(true)),
10443            writer,
10444            writer_rx,
10445            Arc::clone(&connection_epoch),
10446            Arc::clone(&auth_tracker),
10447            Arc::clone(&reconnect_buffer_waits_for_auth),
10448            None,
10449        );
10450
10451        writer_tx
10452            .send(WriterCommand::Send(Message::Text("stale".into())))
10453            .unwrap();
10454
10455        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10456            &url,
10457            vec![],
10458            TransportBackend::Tungstenite,
10459            None,
10460            None,
10461            None,
10462        )
10463        .await
10464        .unwrap();
10465        let (tx, rx) = tokio::sync::oneshot::channel();
10466        writer_tx
10467            .send_update(WriterCommand::Update(new_writer, tx))
10468            .unwrap();
10469        assert_eq!(rx.await.unwrap(), 1);
10470
10471        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10472
10473        tokio::time::sleep(Duration::from_millis(300)).await;
10474        assert!(
10475            server.messages().await.is_empty(),
10476            "buffered messages should wait for re-authentication"
10477        );
10478
10479        assert!(matches!(
10480            writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
10481            Err(SendError::BufferFull)
10482        ));
10483        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10484        writer_tx
10485            .send_control(WriterCommand::SendOnConnection {
10486                message: Message::text("authenticate"),
10487                connection_epoch: 1,
10488                response_tx,
10489            })
10490            .unwrap();
10491
10492        response_rx.await.unwrap().unwrap();
10493        tracker.succeed();
10494
10495        wait_until_async(
10496            || {
10497                let messages = Arc::clone(&server.messages);
10498                async move { messages.lock().await.len() == 2 }
10499            },
10500            Duration::from_secs(3),
10501        )
10502        .await;
10503
10504        assert_eq!(
10505            server.messages().await,
10506            vec!["authenticate".to_string(), "stale".to_string()]
10507        );
10508
10509        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10510        state_notify.notify_waiters();
10511        drop(writer_tx);
10512        write_task.abort();
10513    }
10514
10515    #[rstest]
10516    #[tokio::test]
10517    async fn test_write_task_buffers_send_behind_pending_replay() {
10518        use nautilus_common::testing::wait_until_async;
10519
10520        let server = RecordingServer::setup().await;
10521        let url = format!("ws://127.0.0.1:{}", server.port);
10522        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10523            &url,
10524            vec![],
10525            TransportBackend::Tungstenite,
10526            None,
10527            None,
10528            None,
10529        )
10530        .await
10531        .unwrap();
10532
10533        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10534        let state_notify = Arc::new(tokio::sync::Notify::new());
10535        let auth_tracker = Arc::new(OnceLock::new());
10536        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10537        let connection_epoch = Arc::new(AtomicU64::new(0));
10538        let tracker = AuthTracker::new();
10539        auth_tracker.set(tracker.clone()).unwrap();
10540
10541        // Two slots: the buffered message holds one while replay waits for auth, so a second
10542        // ordinary send is still admissible and reaches the writer loop.
10543        let (writer_tx, writer_rx) = crate::writer::channel(2);
10544        let write_task = WebSocketClientInner::spawn_write_task(
10545            Arc::clone(&connection_state),
10546            Arc::clone(&state_notify),
10547            Arc::new(AtomicBool::new(true)),
10548            writer,
10549            writer_rx,
10550            Arc::clone(&connection_epoch),
10551            Arc::clone(&auth_tracker),
10552            Arc::clone(&reconnect_buffer_waits_for_auth),
10553            None,
10554        );
10555
10556        writer_tx
10557            .send(WriterCommand::Send(Message::Text("stale".into())))
10558            .unwrap();
10559
10560        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10561            &url,
10562            vec![],
10563            TransportBackend::Tungstenite,
10564            None,
10565            None,
10566            None,
10567        )
10568        .await
10569        .unwrap();
10570        let (tx, rx) = tokio::sync::oneshot::channel();
10571        writer_tx
10572            .send_update(WriterCommand::Update(new_writer, tx))
10573            .unwrap();
10574        assert_eq!(rx.await.unwrap(), 1);
10575
10576        // Active with a non-empty replay buffer: the drain is gated on authentication.
10577        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10578        tokio::time::sleep(Duration::from_millis(300)).await;
10579        assert!(
10580            server.messages().await.is_empty(),
10581            "buffered message should wait for re-authentication"
10582        );
10583
10584        // A later ordinary send must not overtake the still-buffered message, nor bypass
10585        // the authentication gate by writing to the replacement connection.
10586        writer_tx
10587            .send(WriterCommand::Send(Message::Text("fresh".into())))
10588            .unwrap();
10589        tokio::time::sleep(Duration::from_millis(300)).await;
10590        assert!(
10591            server.messages().await.is_empty(),
10592            "a send arriving before the gate opens must not reach the connection, was {:?}",
10593            server.messages().await
10594        );
10595
10596        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10597        writer_tx
10598            .send_control(WriterCommand::SendOnConnection {
10599                message: Message::text("authenticate"),
10600                connection_epoch: 1,
10601                response_tx,
10602            })
10603            .unwrap();
10604
10605        response_rx.await.unwrap().unwrap();
10606        tracker.succeed();
10607
10608        wait_until_async(
10609            || {
10610                let messages = Arc::clone(&server.messages);
10611                async move { messages.lock().await.len() == 3 }
10612            },
10613            Duration::from_secs(3),
10614        )
10615        .await;
10616
10617        assert_eq!(
10618            server.messages().await,
10619            vec![
10620                "authenticate".to_string(),
10621                "stale".to_string(),
10622                "fresh".to_string(),
10623            ],
10624            "replay must preserve FIFO order behind the authentication handshake"
10625        );
10626
10627        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10628        state_notify.notify_waiters();
10629        drop(writer_tx);
10630        write_task.abort();
10631    }
10632
10633    #[rstest]
10634    #[tokio::test]
10635    async fn test_write_task_control_send_reaches_connection_behind_pending_replay() {
10636        use nautilus_common::testing::wait_until_async;
10637
10638        let server = RecordingServer::setup().await;
10639        let url = format!("ws://127.0.0.1:{}", server.port);
10640        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10641            &url,
10642            vec![],
10643            TransportBackend::Tungstenite,
10644            None,
10645            None,
10646            None,
10647        )
10648        .await
10649        .unwrap();
10650
10651        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10652        let state_notify = Arc::new(tokio::sync::Notify::new());
10653        let auth_tracker = Arc::new(OnceLock::new());
10654        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10655        let connection_epoch = Arc::new(AtomicU64::new(0));
10656        let tracker = AuthTracker::new();
10657        auth_tracker.set(tracker.clone()).unwrap();
10658
10659        let (writer_tx, writer_rx) = crate::writer::channel(2);
10660        let write_task = WebSocketClientInner::spawn_write_task(
10661            Arc::clone(&connection_state),
10662            Arc::clone(&state_notify),
10663            Arc::new(AtomicBool::new(true)),
10664            writer,
10665            writer_rx,
10666            Arc::clone(&connection_epoch),
10667            Arc::clone(&auth_tracker),
10668            Arc::clone(&reconnect_buffer_waits_for_auth),
10669            None,
10670        );
10671
10672        writer_tx
10673            .send(WriterCommand::Send(Message::Text("stale".into())))
10674            .unwrap();
10675
10676        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10677            &url,
10678            vec![],
10679            TransportBackend::Tungstenite,
10680            None,
10681            None,
10682            None,
10683        )
10684        .await
10685        .unwrap();
10686        let (tx, rx) = tokio::sync::oneshot::channel();
10687        writer_tx
10688            .send_update(WriterCommand::Update(new_writer, tx))
10689            .unwrap();
10690        assert_eq!(rx.await.unwrap(), 1);
10691
10692        // Active with a non-empty replay buffer: the drain is gated on authentication.
10693        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10694        tokio::time::sleep(Duration::from_millis(300)).await;
10695        assert!(
10696            server.messages().await.is_empty(),
10697            "buffered message should wait for re-authentication"
10698        );
10699
10700        // A control frame enqueued through the ordinary send path in the same window must not be
10701        // buffered with the replay: it has to reach the replacement connection while the gate is
10702        // still closed.
10703        writer_tx
10704            .send_control(WriterCommand::Send(Message::Pong(
10705                b"keepalive".to_vec().into(),
10706            )))
10707            .unwrap();
10708
10709        wait_until_async(
10710            || {
10711                let control_frames = Arc::clone(&server.control_frames);
10712                async move { !control_frames.lock().await.is_empty() }
10713            },
10714            Duration::from_secs(3),
10715        )
10716        .await;
10717
10718        assert_eq!(
10719            server.control_frames().await,
10720            vec!["pong:keepalive".to_string()],
10721            "a control frame sent during the wait must reach the new connection without replay"
10722        );
10723        assert!(
10724            server.messages().await.is_empty(),
10725            "ordinary sends must still wait for re-authentication, was {:?}",
10726            server.messages().await
10727        );
10728
10729        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10730        writer_tx
10731            .send_control(WriterCommand::SendOnConnection {
10732                message: Message::text("authenticate"),
10733                connection_epoch: 1,
10734                response_tx,
10735            })
10736            .unwrap();
10737
10738        response_rx.await.unwrap().unwrap();
10739        tracker.succeed();
10740
10741        wait_until_async(
10742            || {
10743                let messages = Arc::clone(&server.messages);
10744                async move { messages.lock().await.len() == 2 }
10745            },
10746            Duration::from_secs(3),
10747        )
10748        .await;
10749
10750        assert_eq!(
10751            server.messages().await,
10752            vec!["authenticate".to_string(), "stale".to_string()],
10753            "the control frame must not disturb ordinary replay ordering"
10754        );
10755
10756        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10757        state_notify.notify_waiters();
10758        drop(writer_tx);
10759        write_task.abort();
10760    }
10761
10762    #[tokio::test]
10763    async fn test_write_task_discards_buffer_after_auth_failure() {
10764        let server = RecordingServer::setup().await;
10765        let url = format!("ws://127.0.0.1:{}", server.port);
10766        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10767            &url,
10768            vec![],
10769            TransportBackend::Tungstenite,
10770            None,
10771            None,
10772            None,
10773        )
10774        .await
10775        .unwrap();
10776
10777        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10778        let state_notify = Arc::new(tokio::sync::Notify::new());
10779        let auth_tracker = Arc::new(OnceLock::new());
10780        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10781        let connection_epoch = Arc::new(AtomicU64::new(0));
10782        let tracker = AuthTracker::new();
10783        auth_tracker.set(tracker.clone()).unwrap();
10784
10785        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10786        let write_task = WebSocketClientInner::spawn_write_task(
10787            Arc::clone(&connection_state),
10788            Arc::clone(&state_notify),
10789            Arc::new(AtomicBool::new(true)),
10790            writer,
10791            writer_rx,
10792            Arc::clone(&connection_epoch),
10793            Arc::clone(&auth_tracker),
10794            Arc::clone(&reconnect_buffer_waits_for_auth),
10795            None,
10796        );
10797
10798        writer_tx
10799            .send(WriterCommand::Send(Message::Text("stale".into())))
10800            .unwrap();
10801
10802        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10803            &url,
10804            vec![],
10805            TransportBackend::Tungstenite,
10806            None,
10807            None,
10808            None,
10809        )
10810        .await
10811        .unwrap();
10812        let (tx, rx) = tokio::sync::oneshot::channel();
10813        writer_tx
10814            .send(WriterCommand::Update(new_writer, tx))
10815            .unwrap();
10816        assert_eq!(rx.await.unwrap(), 1);
10817
10818        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10819        tracker.fail("rejected");
10820        tracker.invalidate();
10821        assert_eq!(tracker.auth_state(), AuthState::Failed);
10822        tokio::time::sleep(Duration::from_millis(300)).await;
10823        assert!(
10824            server.messages().await.is_empty(),
10825            "buffered messages should be discarded after authentication failure"
10826        );
10827
10828        let _auth_receiver = tracker.begin();
10829        tracker.succeed();
10830        tokio::time::sleep(Duration::from_millis(300)).await;
10831        assert!(
10832            server.messages().await.is_empty(),
10833            "discarded buffered messages should not replay on a later auth success"
10834        );
10835
10836        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10837        state_notify.notify_waiters();
10838        drop(writer_tx);
10839        write_task.abort();
10840    }
10841
10842    #[rstest]
10843    #[tokio::test]
10844    async fn test_zero_idle_timeout_rejected() {
10845        let (handler, _rx) = channel_message_handler();
10846
10847        let config = WebSocketConfig {
10848            url: "ws://127.0.0.1:9999".to_string(),
10849            headers: vec![],
10850            heartbeat_interval_secs: None,
10851            heartbeat_payload: None,
10852            connect_timeout_ms: None,
10853            reconnect_delay_initial_ms: None,
10854            reconnect_delay_max_ms: None,
10855            reconnect_backoff_factor: None,
10856            reconnect_jitter_ms: None,
10857            reconnect_max_attempts: None,
10858            heartbeat_timeout_secs: None,
10859            idle_timeout_ms: Some(0),
10860            writer_capacity: None,
10861            backend: TransportBackend::Tungstenite,
10862            proxy_url: None,
10863            max_message_size_bytes: None,
10864            max_frame_size_bytes: None,
10865        };
10866
10867        let result = WebSocketClient::builder()
10868            .config(config)
10869            .message_handler(handler)
10870            .connect()
10871            .await;
10872
10873        assert!(result.is_err(), "Zero idle timeout should be rejected");
10874        let err_msg = result.unwrap_err().to_string();
10875        assert!(
10876            err_msg.contains("idle_timeout_ms"),
10877            "Error should name the offending field, was: {err_msg}"
10878        );
10879    }
10880
10881    #[rstest]
10882    #[tokio::test]
10883    async fn test_zero_heartbeat_timeout_rejected() {
10884        let (handler, _rx) = channel_message_handler();
10885
10886        let config = WebSocketConfig {
10887            url: "ws://127.0.0.1:9999".to_string(),
10888            headers: vec![],
10889            heartbeat_interval_secs: Some(30),
10890            heartbeat_payload: None,
10891            connect_timeout_ms: None,
10892            reconnect_delay_initial_ms: None,
10893            reconnect_delay_max_ms: None,
10894            reconnect_backoff_factor: None,
10895            reconnect_jitter_ms: None,
10896            reconnect_max_attempts: None,
10897            heartbeat_timeout_secs: Some(0),
10898            idle_timeout_ms: None,
10899            writer_capacity: None,
10900            backend: TransportBackend::Tungstenite,
10901            proxy_url: None,
10902            max_message_size_bytes: None,
10903            max_frame_size_bytes: None,
10904        };
10905
10906        let result = WebSocketClient::builder()
10907            .config(config)
10908            .message_handler(handler)
10909            .connect()
10910            .await;
10911
10912        assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
10913        let err_msg = result.unwrap_err().to_string();
10914        assert!(
10915            err_msg.contains("heartbeat_timeout_secs"),
10916            "Error should name the offending field, was: {err_msg}"
10917        );
10918    }
10919
10920    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
10921    #[rstest]
10922    #[tokio::test]
10923    async fn test_sockudo_backend_rejects_reserved_headers_before_connect() {
10924        let (handler, _rx) = channel_message_handler();
10925
10926        let config = WebSocketConfig {
10927            url: "ws://127.0.0.1:1".to_string(),
10928            headers: vec![("Host".to_string(), "example.com".to_string())],
10929            heartbeat_interval_secs: None,
10930            heartbeat_payload: None,
10931            connect_timeout_ms: None,
10932            reconnect_delay_initial_ms: None,
10933            reconnect_delay_max_ms: None,
10934            reconnect_backoff_factor: None,
10935            reconnect_jitter_ms: None,
10936            reconnect_max_attempts: None,
10937            heartbeat_timeout_secs: None,
10938            idle_timeout_ms: None,
10939            writer_capacity: None,
10940            backend: TransportBackend::Sockudo,
10941            proxy_url: None,
10942            max_message_size_bytes: None,
10943            max_frame_size_bytes: None,
10944        };
10945
10946        let err = WebSocketClient::builder()
10947            .config(config)
10948            .message_handler(handler)
10949            .connect()
10950            .await
10951            .expect_err("reserved header should fail before TCP connect");
10952
10953        assert!(
10954            err.to_string()
10955                .contains("reserved upgrade header not allowed in extra_headers"),
10956            "expected reserved-header failure, was: {err}"
10957        );
10958    }
10959
10960    // The log-capture harness is Linux-only for CI stability.
10961    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
10962    #[rstest]
10963    #[tokio::test]
10964    async fn test_sockudo_reconnect_recovers_from_retryable_upgrade_rejection() {
10965        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
10966        let port = listener.local_addr().unwrap().port();
10967
10968        let server = task::spawn(async move {
10969            // First connection: accept the upgrade, then drop it to force a reconnect.
10970            let (stream, _) = listener.accept().await.unwrap();
10971            drop(accept_async(stream).await.unwrap());
10972
10973            // Second connection: reject the reconnect handshake with a retryable status.
10974            let (mut stream, _) = listener.accept().await.unwrap();
10975            let _request = read_http_request(&mut stream).await;
10976            stream
10977                .write_all(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
10978                .await
10979                .unwrap();
10980            stream.flush().await.unwrap();
10981            sleep(Duration::from_millis(50)).await;
10982            drop(stream);
10983
10984            // Third connection: accept the upgrade so the reconnect loop completes recovery.
10985            let (stream, _) = listener.accept().await.unwrap();
10986            let mut ws = accept_async(stream).await.unwrap();
10987            ws.send(WsMessage::Text("recovered".to_string().into()))
10988                .await
10989                .unwrap();
10990            sleep(Duration::from_secs(1)).await;
10991        });
10992
10993        let (handler, mut rx) = channel_message_handler();
10994
10995        let config = WebSocketConfig {
10996            url: format!(
10997                "ws://user-secret:password-secret@127.0.0.1:{port}/path-secret?token=query-secret"
10998            ),
10999            headers: vec![],
11000            heartbeat_interval_secs: None,
11001            heartbeat_payload: None,
11002            connect_timeout_ms: Some(2_000),
11003            reconnect_delay_initial_ms: Some(50),
11004            reconnect_delay_max_ms: Some(100),
11005            reconnect_backoff_factor: Some(1.0),
11006            reconnect_jitter_ms: Some(0),
11007            reconnect_max_attempts: None,
11008            heartbeat_timeout_secs: None,
11009            idle_timeout_ms: None,
11010            writer_capacity: None,
11011            backend: TransportBackend::Sockudo,
11012            proxy_url: None,
11013            max_message_size_bytes: None,
11014            max_frame_size_bytes: None,
11015        };
11016
11017        // Capture before connecting so the whole reconnect cycle is observed. The harness owns
11018        // the global logger, so the Nautilus logger that arms `shutdown_on_error` on an ERROR
11019        // record cannot run here; the absence of ERROR records is what rules the trigger out.
11020        let capture = capture_logs_for(RECONNECT_LOG_TARGETS).await;
11021
11022        let client = WebSocketClient::builder()
11023            .config(config)
11024            .message_handler(handler)
11025            .connect()
11026            .await
11027            .expect("sockudo connect should succeed");
11028
11029        let recovered = tokio::time::timeout(Duration::from_secs(10), async {
11030            loop {
11031                if let Ok(WsMessage::Text(text)) = rx.try_recv()
11032                    && text.as_str() == "recovered"
11033                {
11034                    return;
11035                }
11036
11037                tokio::time::sleep(Duration::from_millis(10)).await;
11038            }
11039        })
11040        .await;
11041
11042        let messages = capture.messages();
11043        client.disconnect().await;
11044        server.abort();
11045
11046        recovered.expect("client should recover after the retryable upgrade rejection");
11047
11048        let errors: Vec<_> = messages
11049            .iter()
11050            .filter(|(level, _)| *level == Level::Error)
11051            .collect();
11052        assert!(
11053            errors.is_empty(),
11054            "a retryable rejection must not log at ERROR, was: {errors:?}"
11055        );
11056
11057        // Asserted positively so the ERROR assertion above cannot pass by capturing nothing.
11058        assert!(
11059            messages.iter().any(|(level, message)| *level == Level::Warn
11060                && message.contains(&format!(
11061                    "Sockudo handshake rejected by 127.0.0.1:{port} with retryable status 502"
11062                ))),
11063            "expected a warning naming the retryable rejection, was: {messages:?}"
11064        );
11065
11066        for secret in [
11067            "user-secret",
11068            "password-secret",
11069            "path-secret",
11070            "query-secret",
11071        ] {
11072            assert!(
11073                messages
11074                    .iter()
11075                    .all(|(_, message)| !message.contains(secret)),
11076                "logs must not contain {secret}, was: {messages:?}"
11077            );
11078        }
11079    }
11080
11081    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11082    #[rstest]
11083    #[tokio::test]
11084    async fn test_sockudo_backend_replays_leftover_without_custom_headers() {
11085        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11086        let port = listener.local_addr().unwrap().port();
11087        let server = task::spawn(async move {
11088            if let Ok((mut stream, _)) = listener.accept().await {
11089                let request = read_http_request(&mut stream).await;
11090                let request = String::from_utf8(request).unwrap();
11091                let sec_websocket_key = extract_header(&request, "Sec-WebSocket-Key").unwrap();
11092                let accept = sockudo_handshake::generate_accept_key(sec_websocket_key);
11093                let mut response = format!(
11094                    concat!(
11095                        "HTTP/1.1 101 Switching Protocols\r\n",
11096                        "Upgrade: websocket\r\n",
11097                        "Connection: Upgrade\r\n",
11098                        "Sec-WebSocket-Accept: {}\r\n",
11099                        "\r\n",
11100                    ),
11101                    accept
11102                )
11103                .into_bytes();
11104                response.extend_from_slice(b"\x81\x05hello");
11105                stream.write_all(&response).await.unwrap();
11106            }
11107        });
11108
11109        let (handler, mut rx) = channel_message_handler();
11110
11111        let config = WebSocketConfig {
11112            url: format!("ws://127.0.0.1:{port}/ws"),
11113            headers: vec![],
11114            heartbeat_interval_secs: None,
11115            heartbeat_payload: None,
11116            connect_timeout_ms: Some(2_000),
11117            reconnect_delay_initial_ms: Some(50),
11118            reconnect_delay_max_ms: Some(100),
11119            reconnect_backoff_factor: Some(1.0),
11120            reconnect_jitter_ms: Some(0),
11121            reconnect_max_attempts: None,
11122            heartbeat_timeout_secs: None,
11123            idle_timeout_ms: None,
11124            writer_capacity: None,
11125            backend: TransportBackend::Sockudo,
11126            proxy_url: None,
11127            max_message_size_bytes: None,
11128            max_frame_size_bytes: None,
11129        };
11130
11131        let client = WebSocketClient::builder()
11132            .config(config)
11133            .message_handler(handler)
11134            .connect()
11135            .await
11136            .expect("sockudo connect without custom headers");
11137
11138        let received = tokio::time::timeout(Duration::from_secs(3), async {
11139            loop {
11140                if let Ok(msg) = rx.try_recv() {
11141                    return msg;
11142                }
11143
11144                tokio::time::sleep(Duration::from_millis(10)).await;
11145            }
11146        })
11147        .await
11148        .expect("did not receive leftover frame before timeout");
11149
11150        match received {
11151            WsMessage::Text(t) => assert_eq!(t.as_str(), "hello"),
11152            other => panic!("expected text, was {other:?}"),
11153        }
11154
11155        client.disconnect().await;
11156        tokio::time::timeout(Duration::from_secs(3), server)
11157            .await
11158            .expect("server did not close before timeout")
11159            .unwrap();
11160    }
11161
11162    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11163    #[rstest]
11164    #[tokio::test]
11165    async fn test_sockudo_backend_sends_custom_headers() {
11166        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11167        let port = listener.local_addr().unwrap().port();
11168        let server = task::spawn(async move {
11169            if let Ok((stream, _)) = listener.accept().await {
11170                let callback = HeaderAssertCallback {
11171                    key: "X-Test".to_string(),
11172                    value: HeaderValue::from_static("value"),
11173                };
11174
11175                if let Ok(mut ws) = accept_hdr_async(stream, callback).await {
11176                    while let Some(Ok(msg)) = ws.next().await {
11177                        if msg.is_text() || msg.is_binary() {
11178                            if ws.send(msg).await.is_err() {
11179                                break;
11180                            }
11181
11182                            continue;
11183                        }
11184
11185                        if msg.is_close() {
11186                            let _ = ws.close(None).await;
11187                            break;
11188                        }
11189                    }
11190                }
11191            }
11192        });
11193
11194        let (handler, mut rx) = channel_message_handler();
11195
11196        let config = WebSocketConfig {
11197            url: format!("ws://127.0.0.1:{port}"),
11198            headers: vec![("X-Test".to_string(), "value".to_string())],
11199            heartbeat_interval_secs: None,
11200            heartbeat_payload: None,
11201            connect_timeout_ms: Some(2_000),
11202            reconnect_delay_initial_ms: Some(50),
11203            reconnect_delay_max_ms: Some(100),
11204            reconnect_backoff_factor: Some(1.0),
11205            reconnect_jitter_ms: Some(0),
11206            reconnect_max_attempts: None,
11207            heartbeat_timeout_secs: None,
11208            idle_timeout_ms: None,
11209            writer_capacity: None,
11210            backend: TransportBackend::Sockudo,
11211            proxy_url: None,
11212            max_message_size_bytes: None,
11213            max_frame_size_bytes: None,
11214        };
11215
11216        let client = WebSocketClient::builder()
11217            .config(config)
11218            .message_handler(handler)
11219            .connect()
11220            .await
11221            .expect("sockudo connect with custom headers");
11222
11223        client.send_text("ping".to_string(), None).await.unwrap();
11224
11225        let received = tokio::time::timeout(Duration::from_secs(3), async {
11226            loop {
11227                if let Ok(msg) = rx.try_recv() {
11228                    return msg;
11229                }
11230
11231                tokio::time::sleep(Duration::from_millis(10)).await;
11232            }
11233        })
11234        .await
11235        .expect("did not receive echo before timeout");
11236
11237        match received {
11238            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11239            other => panic!("expected text, was {other:?}"),
11240        }
11241
11242        client.disconnect().await;
11243        tokio::time::timeout(Duration::from_secs(3), server)
11244            .await
11245            .expect("server did not close before timeout")
11246            .unwrap();
11247    }
11248
11249    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11250    #[rstest]
11251    #[tokio::test]
11252    async fn test_sockudo_backend_round_trip_text() {
11253        // tokio-tungstenite test peer paired with a sockudo client.
11254        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11255        let port = listener.local_addr().unwrap().port();
11256        let server = task::spawn(async move {
11257            if let Ok((stream, _)) = listener.accept().await
11258                && let Ok(mut ws) = accept_async(stream).await
11259            {
11260                while let Some(Ok(msg)) = ws.next().await {
11261                    match msg {
11262                        WsMessage::Text(_) | WsMessage::Binary(_) => {
11263                            if ws.send(msg).await.is_err() {
11264                                break;
11265                            }
11266                        }
11267                        WsMessage::Close(_) => {
11268                            let _ = ws.close(None).await;
11269                            break;
11270                        }
11271                        _ => {}
11272                    }
11273                }
11274            }
11275        });
11276
11277        let (handler, mut rx) = channel_message_handler();
11278
11279        let config = WebSocketConfig {
11280            url: format!("ws://127.0.0.1:{port}"),
11281            headers: vec![],
11282            heartbeat_interval_secs: None,
11283            heartbeat_payload: None,
11284            connect_timeout_ms: Some(2_000),
11285            reconnect_delay_initial_ms: Some(50),
11286            reconnect_delay_max_ms: Some(100),
11287            reconnect_backoff_factor: Some(1.0),
11288            reconnect_jitter_ms: Some(0),
11289            reconnect_max_attempts: None,
11290            heartbeat_timeout_secs: None,
11291            idle_timeout_ms: None,
11292            writer_capacity: None,
11293            backend: TransportBackend::Sockudo,
11294            proxy_url: None,
11295            max_message_size_bytes: None,
11296            max_frame_size_bytes: None,
11297        };
11298
11299        let client = WebSocketClient::builder()
11300            .config(config)
11301            .message_handler(handler)
11302            .connect()
11303            .await
11304            .expect("sockudo connect");
11305
11306        client.send_text("ping".to_string(), None).await.unwrap();
11307
11308        let received = tokio::time::timeout(Duration::from_secs(3), async {
11309            loop {
11310                if let Ok(msg) = rx.try_recv() {
11311                    return msg;
11312                }
11313
11314                tokio::time::sleep(Duration::from_millis(10)).await;
11315            }
11316        })
11317        .await
11318        .expect("did not receive echo before timeout");
11319
11320        match received {
11321            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11322            other => panic!("expected text, was {other:?}"),
11323        }
11324
11325        client.disconnect().await;
11326        server.abort();
11327    }
11328
11329    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11330    #[rstest]
11331    #[case::ws_default_port("ws://example.com/ws", "example.com", "example.com", 80, "/ws", false)]
11332    #[case::wss_default_port(
11333        "wss://example.com/ws",
11334        "example.com",
11335        "example.com",
11336        443,
11337        "/ws",
11338        true
11339    )]
11340    // url::Url normalizes explicit default ports (`:80` for ws, `:443` for wss)
11341    // away, so `parsed.port()` reports `None` here and Host stays unqualified.
11342    #[case::ws_explicit_default(
11343        "ws://example.com:80/ws",
11344        "example.com",
11345        "example.com",
11346        80,
11347        "/ws",
11348        false
11349    )]
11350    #[case::ws_non_default(
11351        "ws://example.com:8443/feed",
11352        "example.com",
11353        "example.com:8443",
11354        8443,
11355        "/feed",
11356        false
11357    )]
11358    #[case::wss_non_default(
11359        "wss://example.com:9443/feed",
11360        "example.com",
11361        "example.com:9443",
11362        9443,
11363        "/feed",
11364        true
11365    )]
11366    #[case::root_path(
11367        "ws://example.com:9000/",
11368        "example.com",
11369        "example.com:9000",
11370        9000,
11371        "/",
11372        false
11373    )]
11374    #[case::query_string(
11375        "ws://example.com/feed?token=abc&channel=trades",
11376        "example.com",
11377        "example.com",
11378        80,
11379        "/feed?token=abc&channel=trades",
11380        false
11381    )]
11382    // IPv6: bare host strips brackets for DNS/TCP/SNI; Host header keeps them.
11383    #[case::ipv6_default("ws://[::1]/feed", "::1", "[::1]", 80, "/feed", false)]
11384    #[case::ipv6_explicit_port("ws://[::1]:9000/feed", "::1", "[::1]:9000", 9000, "/feed", false)]
11385    #[case::ipv6_wss(
11386        "wss://[2001:db8::1]:8443/",
11387        "2001:db8::1",
11388        "[2001:db8::1]:8443",
11389        8443,
11390        "/",
11391        true
11392    )]
11393    fn sockudo_target_parses_url(
11394        #[case] url: &str,
11395        #[case] host: &str,
11396        #[case] host_header: &str,
11397        #[case] port: u16,
11398        #[case] path: &str,
11399        #[case] is_tls: bool,
11400    ) {
11401        let target = super::SockudoTarget::parse(url).expect("parse should succeed");
11402        assert_eq!(target.host, host);
11403        assert_eq!(target.host_header, host_header);
11404        assert_eq!(target.port, port);
11405        assert_eq!(target.path, path);
11406        assert_eq!(target.is_tls, is_tls);
11407    }
11408
11409    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11410    #[rstest]
11411    fn sockudo_target_rejects_unsupported_scheme() {
11412        let err = super::SockudoTarget::parse("http://example.com/feed").expect_err("not a ws URL");
11413        let msg = err.to_string();
11414        assert!(
11415            msg.contains("expected ws:// or wss://"),
11416            "unexpected error: {msg}"
11417        );
11418    }
11419
11420    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11421    #[rstest]
11422    fn sockudo_target_rejects_malformed_url() {
11423        const SECRET: &str = "malformed-websocket-url-secret";
11424        let url = format!("not a url {SECRET}");
11425        let err = super::SockudoTarget::parse(&url).expect_err("malformed URL");
11426        let message = err.to_string();
11427
11428        assert!(
11429            matches!(err, super::TransportError::InvalidUrl(_)),
11430            "expected InvalidUrl, was: {err:?}"
11431        );
11432        assert!(!message.contains(SECRET));
11433        assert!(!message.contains(&url));
11434    }
11435}
11436
11437#[cfg(test)]
11438mod property_tests {
11439    use std::{
11440        collections::{HashSet, VecDeque},
11441        sync::{Arc, OnceLock, atomic::AtomicBool},
11442    };
11443
11444    use proptest::prelude::*;
11445    use rstest::rstest;
11446
11447    use super::{super::auth::AuthResultReceiver, *};
11448
11449    const AUTH_FAILED: &str = "model auth failed";
11450
11451    #[derive(Debug, Clone)]
11452    enum ReconnectBufferTraceOp {
11453        BeginAuth,
11454        AuthSucceeds,
11455        AuthFails,
11456        AuthInvalidates,
11457        ReconnectStarts,
11458        ReconnectCompletes,
11459        BufferedMessage(u8),
11460    }
11461
11462    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
11463    enum ModelConnectionMode {
11464        Active,
11465        Reconnect,
11466    }
11467
11468    #[derive(Debug, Clone, Copy)]
11469    enum ExpectedReconnectBufferAction {
11470        Drain,
11471        Wait,
11472        Discard,
11473    }
11474
11475    #[derive(Debug)]
11476    struct ReconnectBufferModel {
11477        mode: ModelConnectionMode,
11478        auth_state: AuthState,
11479        buffer: VecDeque<String>,
11480        released: Vec<String>,
11481        discarded: Vec<String>,
11482        live_sent: Vec<String>,
11483        handler_controls: Vec<&'static str>,
11484        next_message_index: usize,
11485    }
11486
11487    impl ReconnectBufferModel {
11488        fn new() -> Self {
11489            Self {
11490                mode: ModelConnectionMode::Active,
11491                auth_state: AuthState::Unauthenticated,
11492                buffer: VecDeque::new(),
11493                released: Vec::new(),
11494                discarded: Vec::new(),
11495                live_sent: Vec::new(),
11496                handler_controls: Vec::new(),
11497                next_message_index: 0,
11498            }
11499        }
11500
11501        fn next_payload(&mut self, raw: u8) -> String {
11502            let payload = format!("message-{}-{raw}", self.next_message_index);
11503            self.next_message_index += 1;
11504            payload
11505        }
11506
11507        fn expected_action(&self, waits_for_auth: bool) -> ExpectedReconnectBufferAction {
11508            if !waits_for_auth {
11509                return ExpectedReconnectBufferAction::Drain;
11510            }
11511
11512            match self.auth_state {
11513                AuthState::Authenticated => ExpectedReconnectBufferAction::Drain,
11514                AuthState::Failed => ExpectedReconnectBufferAction::Discard,
11515                AuthState::Unauthenticated => ExpectedReconnectBufferAction::Wait,
11516            }
11517        }
11518    }
11519
11520    fn reconnect_buffer_trace_op_strategy() -> impl Strategy<Value = ReconnectBufferTraceOp> {
11521        prop_oneof![
11522            Just(ReconnectBufferTraceOp::BeginAuth),
11523            Just(ReconnectBufferTraceOp::AuthSucceeds),
11524            Just(ReconnectBufferTraceOp::AuthFails),
11525            Just(ReconnectBufferTraceOp::AuthInvalidates),
11526            Just(ReconnectBufferTraceOp::ReconnectStarts),
11527            Just(ReconnectBufferTraceOp::ReconnectCompletes),
11528            any::<u8>().prop_map(ReconnectBufferTraceOp::BufferedMessage),
11529        ]
11530    }
11531
11532    fn reconnect_buffer_actions_match(
11533        actual: ReconnectBufferAction,
11534        expected: ExpectedReconnectBufferAction,
11535    ) -> bool {
11536        matches!(
11537            (actual, expected),
11538            (
11539                ReconnectBufferAction::Drain,
11540                ExpectedReconnectBufferAction::Drain
11541            ) | (
11542                ReconnectBufferAction::Wait,
11543                ExpectedReconnectBufferAction::Wait
11544            ) | (
11545                ReconnectBufferAction::Discard,
11546                ExpectedReconnectBufferAction::Discard
11547            )
11548        )
11549    }
11550
11551    fn apply_ready_reconnect_buffer_action(
11552        model: &mut ReconnectBufferModel,
11553        reconnect_buffer_waits_for_auth: &AtomicBool,
11554        auth_tracker: &Arc<OnceLock<AuthTracker>>,
11555        waits_for_auth: bool,
11556        step: usize,
11557        op: &ReconnectBufferTraceOp,
11558    ) -> Result<(), TestCaseError> {
11559        if model.mode != ModelConnectionMode::Active || model.buffer.is_empty() {
11560            return Ok(());
11561        }
11562
11563        let expected = model.expected_action(waits_for_auth);
11564        let actual = WebSocketClientInner::can_drain_reconnect_buffer(
11565            reconnect_buffer_waits_for_auth,
11566            auth_tracker,
11567        );
11568
11569        prop_assert!(
11570            reconnect_buffer_actions_match(actual, expected),
11571            "reconnect buffer action mismatch at step {}, op {:?}, waits_for_auth={}, auth_state={:?}",
11572            step,
11573            op,
11574            waits_for_auth,
11575            model.auth_state
11576        );
11577
11578        match expected {
11579            ExpectedReconnectBufferAction::Drain => {
11580                model.released.extend(model.buffer.drain(..));
11581            }
11582            ExpectedReconnectBufferAction::Wait => {}
11583            ExpectedReconnectBufferAction::Discard => {
11584                model.discarded.extend(model.buffer.drain(..));
11585            }
11586        }
11587
11588        Ok(())
11589    }
11590
11591    fn assert_reconnected_control_stays_separate(
11592        model: &ReconnectBufferModel,
11593        step: usize,
11594    ) -> Result<(), TestCaseError> {
11595        prop_assert!(
11596            model
11597                .handler_controls
11598                .iter()
11599                .all(|message| *message == RECONNECTED),
11600            "handler control stream contained a non-RECONNECTED message at step {}",
11601            step
11602        );
11603        prop_assert!(
11604            !model.buffer.iter().any(|message| message == RECONNECTED),
11605            "RECONNECTED control message entered reconnect buffer at step {}",
11606            step
11607        );
11608        prop_assert!(
11609            !model.released.iter().any(|message| message == RECONNECTED),
11610            "RECONNECTED control message entered replayed messages at step {}",
11611            step
11612        );
11613        prop_assert!(
11614            !model.discarded.iter().any(|message| message == RECONNECTED),
11615            "RECONNECTED control message entered discarded messages at step {}",
11616            step
11617        );
11618        prop_assert!(
11619            !model.live_sent.iter().any(|message| message == RECONNECTED),
11620            "RECONNECTED control message entered application sends at step {}",
11621            step
11622        );
11623
11624        Ok(())
11625    }
11626
11627    fn assert_messages_accounted_once(
11628        model: &ReconnectBufferModel,
11629        step: usize,
11630    ) -> Result<(), TestCaseError> {
11631        let mut seen = HashSet::new();
11632
11633        for message in model
11634            .released
11635            .iter()
11636            .chain(model.discarded.iter())
11637            .chain(model.buffer.iter())
11638            .chain(model.live_sent.iter())
11639        {
11640            prop_assert!(
11641                seen.insert(message.as_str()),
11642                "message {} appeared more than once at step {}",
11643                message,
11644                step
11645            );
11646        }
11647
11648        Ok(())
11649    }
11650
11651    fn apply_reconnect_buffer_trace_op(
11652        model: &mut ReconnectBufferModel,
11653        tracker: &AuthTracker,
11654        auth_receivers: &mut Vec<AuthResultReceiver>,
11655        op: &ReconnectBufferTraceOp,
11656    ) -> Result<(), TestCaseError> {
11657        match op {
11658            ReconnectBufferTraceOp::BeginAuth => {
11659                auth_receivers.push(tracker.begin());
11660                model.auth_state = AuthState::Unauthenticated;
11661            }
11662            ReconnectBufferTraceOp::AuthSucceeds => {
11663                tracker.succeed();
11664                model.auth_state = AuthState::Authenticated;
11665            }
11666            ReconnectBufferTraceOp::AuthFails => {
11667                tracker.fail(AUTH_FAILED);
11668                model.auth_state = AuthState::Failed;
11669            }
11670            ReconnectBufferTraceOp::AuthInvalidates => {
11671                tracker.invalidate();
11672
11673                if model.auth_state == AuthState::Authenticated {
11674                    model.auth_state = AuthState::Unauthenticated;
11675                }
11676            }
11677            ReconnectBufferTraceOp::ReconnectStarts => {
11678                tracker.invalidate();
11679
11680                if model.auth_state == AuthState::Authenticated {
11681                    model.auth_state = AuthState::Unauthenticated;
11682                }
11683
11684                model.mode = ModelConnectionMode::Reconnect;
11685            }
11686            ReconnectBufferTraceOp::ReconnectCompletes => {
11687                model.mode = ModelConnectionMode::Active;
11688                model.handler_controls.push(RECONNECTED);
11689            }
11690            ReconnectBufferTraceOp::BufferedMessage(raw) => {
11691                let payload = model.next_payload(*raw);
11692                prop_assert_ne!(payload.as_str(), RECONNECTED);
11693
11694                if model.mode == ModelConnectionMode::Reconnect {
11695                    model.buffer.push_back(payload);
11696                } else {
11697                    model.live_sent.push(payload);
11698                }
11699            }
11700        }
11701
11702        Ok(())
11703    }
11704
11705    proptest! {
11706        #![proptest_config(ProptestConfig::with_cases(256))]
11707
11708        /// Property: reconnect-buffer traces match the auth-gated release and
11709        /// discard model, and `RECONNECTED` remains a separate control signal.
11710        #[rstest]
11711        fn test_reconnect_buffer_trace_matches_auth_gate_model(
11712            waits_for_auth in any::<bool>(),
11713            ops in proptest::collection::vec(reconnect_buffer_trace_op_strategy(), 1..100)
11714        ) {
11715            let auth_tracker = Arc::new(OnceLock::new());
11716            let reconnect_buffer_waits_for_auth = AtomicBool::new(waits_for_auth);
11717            let tracker = AuthTracker::new();
11718            auth_tracker.set(tracker.clone()).unwrap();
11719            let mut auth_receivers = Vec::new();
11720            let mut model = ReconnectBufferModel::new();
11721
11722            for (step, op) in ops.iter().enumerate() {
11723                apply_reconnect_buffer_trace_op(
11724                    &mut model,
11725                    &tracker,
11726                    &mut auth_receivers,
11727                    op,
11728                )?;
11729
11730                prop_assert_eq!(
11731                    tracker.auth_state(),
11732                    model.auth_state,
11733                    "auth state mismatch at step {}, op {:?}",
11734                    step,
11735                    op
11736                );
11737
11738                apply_ready_reconnect_buffer_action(
11739                    &mut model,
11740                    &reconnect_buffer_waits_for_auth,
11741                    &auth_tracker,
11742                    waits_for_auth,
11743                    step,
11744                    op,
11745                )?;
11746                assert_reconnected_control_stays_separate(&model, step)?;
11747                prop_assert_eq!(
11748                    model.handler_controls.len(),
11749                    ops[..=step]
11750                        .iter()
11751                        .filter(|op| matches!(op, ReconnectBufferTraceOp::ReconnectCompletes))
11752                        .count(),
11753                    "handler control count mismatch at step {}",
11754                    step
11755                );
11756                assert_messages_accounted_once(&model, step)?;
11757            }
11758        }
11759
11760        /// Property: successful re-authentication releases buffered messages
11761        /// exactly once when replay is configured to wait for auth.
11762        #[rstest]
11763        fn test_reconnect_buffer_releases_after_auth_success_once(
11764            payloads in proptest::collection::vec(any::<u8>(), 1..32),
11765            extra_success_ticks in 0usize..16
11766        ) {
11767            let auth_tracker = Arc::new(OnceLock::new());
11768            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11769            let tracker = AuthTracker::new();
11770            auth_tracker.set(tracker.clone()).unwrap();
11771            let mut auth_receivers = Vec::new();
11772            let mut model = ReconnectBufferModel::new();
11773
11774            apply_reconnect_buffer_trace_op(
11775                &mut model,
11776                &tracker,
11777                &mut auth_receivers,
11778                &ReconnectBufferTraceOp::ReconnectStarts,
11779            )?;
11780            apply_reconnect_buffer_trace_op(
11781                &mut model,
11782                &tracker,
11783                &mut auth_receivers,
11784                &ReconnectBufferTraceOp::BeginAuth,
11785            )?;
11786
11787            for payload in payloads {
11788                apply_reconnect_buffer_trace_op(
11789                    &mut model,
11790                    &tracker,
11791                    &mut auth_receivers,
11792                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11793                )?;
11794            }
11795
11796            let buffered_len = model.buffer.len();
11797            apply_reconnect_buffer_trace_op(
11798                &mut model,
11799                &tracker,
11800                &mut auth_receivers,
11801                &ReconnectBufferTraceOp::ReconnectCompletes,
11802            )?;
11803            apply_ready_reconnect_buffer_action(
11804                &mut model,
11805                &reconnect_buffer_waits_for_auth,
11806                &auth_tracker,
11807                true,
11808                0,
11809                &ReconnectBufferTraceOp::ReconnectCompletes,
11810            )?;
11811
11812            prop_assert_eq!(model.released.len(), 0);
11813            prop_assert_eq!(model.buffer.len(), buffered_len);
11814
11815            apply_reconnect_buffer_trace_op(
11816                &mut model,
11817                &tracker,
11818                &mut auth_receivers,
11819                &ReconnectBufferTraceOp::AuthSucceeds,
11820            )?;
11821            apply_ready_reconnect_buffer_action(
11822                &mut model,
11823                &reconnect_buffer_waits_for_auth,
11824                &auth_tracker,
11825                true,
11826                1,
11827                &ReconnectBufferTraceOp::AuthSucceeds,
11828            )?;
11829
11830            prop_assert_eq!(model.released.len(), buffered_len);
11831            prop_assert!(model.buffer.is_empty());
11832            assert_messages_accounted_once(&model, 1)?;
11833
11834            for tick in 0..extra_success_ticks {
11835                apply_reconnect_buffer_trace_op(
11836                    &mut model,
11837                    &tracker,
11838                    &mut auth_receivers,
11839                    &ReconnectBufferTraceOp::AuthSucceeds,
11840                )?;
11841                apply_ready_reconnect_buffer_action(
11842                    &mut model,
11843                    &reconnect_buffer_waits_for_auth,
11844                    &auth_tracker,
11845                    true,
11846                    tick + 2,
11847                    &ReconnectBufferTraceOp::AuthSucceeds,
11848                )?;
11849                prop_assert_eq!(
11850                    model.released.len(),
11851                    buffered_len,
11852                    "buffered messages replayed more than once at tick {}",
11853                    tick
11854                );
11855            }
11856        }
11857
11858        /// Property: auth failure discards messages buffered before or after
11859        /// that failure, and later auth success does not replay discarded data.
11860        #[rstest]
11861        fn test_reconnect_buffer_discards_after_auth_failure(
11862            before_failure_payloads in proptest::collection::vec(any::<u8>(), 0..16),
11863            after_failure_payloads in proptest::collection::vec(any::<u8>(), 1..16),
11864            later_success_ticks in 0usize..16
11865        ) {
11866            let auth_tracker = Arc::new(OnceLock::new());
11867            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11868            let tracker = AuthTracker::new();
11869            auth_tracker.set(tracker.clone()).unwrap();
11870            let mut auth_receivers = Vec::new();
11871            let mut model = ReconnectBufferModel::new();
11872
11873            apply_reconnect_buffer_trace_op(
11874                &mut model,
11875                &tracker,
11876                &mut auth_receivers,
11877                &ReconnectBufferTraceOp::ReconnectStarts,
11878            )?;
11879            apply_reconnect_buffer_trace_op(
11880                &mut model,
11881                &tracker,
11882                &mut auth_receivers,
11883                &ReconnectBufferTraceOp::BeginAuth,
11884            )?;
11885
11886            for payload in before_failure_payloads {
11887                apply_reconnect_buffer_trace_op(
11888                    &mut model,
11889                    &tracker,
11890                    &mut auth_receivers,
11891                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11892                )?;
11893            }
11894
11895            apply_reconnect_buffer_trace_op(
11896                &mut model,
11897                &tracker,
11898                &mut auth_receivers,
11899                &ReconnectBufferTraceOp::AuthFails,
11900            )?;
11901
11902            for payload in after_failure_payloads {
11903                apply_reconnect_buffer_trace_op(
11904                    &mut model,
11905                    &tracker,
11906                    &mut auth_receivers,
11907                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11908                )?;
11909            }
11910
11911            let buffered_len = model.buffer.len();
11912            apply_reconnect_buffer_trace_op(
11913                &mut model,
11914                &tracker,
11915                &mut auth_receivers,
11916                &ReconnectBufferTraceOp::ReconnectCompletes,
11917            )?;
11918            apply_ready_reconnect_buffer_action(
11919                &mut model,
11920                &reconnect_buffer_waits_for_auth,
11921                &auth_tracker,
11922                true,
11923                0,
11924                &ReconnectBufferTraceOp::ReconnectCompletes,
11925            )?;
11926
11927            prop_assert_eq!(model.discarded.len(), buffered_len);
11928            prop_assert!(model.released.is_empty());
11929            prop_assert!(model.buffer.is_empty());
11930            assert_messages_accounted_once(&model, 0)?;
11931
11932            for tick in 0..later_success_ticks {
11933                apply_reconnect_buffer_trace_op(
11934                    &mut model,
11935                    &tracker,
11936                    &mut auth_receivers,
11937                    &ReconnectBufferTraceOp::BeginAuth,
11938                )?;
11939                apply_reconnect_buffer_trace_op(
11940                    &mut model,
11941                    &tracker,
11942                    &mut auth_receivers,
11943                    &ReconnectBufferTraceOp::AuthSucceeds,
11944                )?;
11945                apply_ready_reconnect_buffer_action(
11946                    &mut model,
11947                    &reconnect_buffer_waits_for_auth,
11948                    &auth_tracker,
11949                    true,
11950                    tick + 1,
11951                    &ReconnectBufferTraceOp::AuthSucceeds,
11952                )?;
11953                prop_assert!(
11954                    model.released.is_empty(),
11955                    "discarded messages replayed after later auth success at tick {}",
11956                    tick
11957                );
11958            }
11959        }
11960    }
11961}
11962
11963#[cfg(test)]
11964#[cfg(feature = "turmoil")]
11965mod turmoil_tests {
11966    use std::{sync::Arc, time::Duration};
11967
11968    use futures_util::{SinkExt, StreamExt};
11969    use nautilus_common::testing::wait_until_async;
11970    use rstest::rstest;
11971    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
11972    use turmoil::{Builder, net};
11973
11974    use super::*;
11975    use crate::websocket::types::channel_message_handler;
11976
11977    const AUTH_BUFFER_WAIT_SEED: u64 = 0xA17B_0001;
11978    const AUTH_BUFFER_DISCARD_SEED: u64 = 0xA17B_0002;
11979
11980    fn seeded_turmoil_builder(seed: u64) -> Builder {
11981        let mut builder = Builder::new();
11982        builder.rng_seed(seed);
11983        builder
11984    }
11985
11986    #[rstest]
11987    fn test_turmoil_reconnect_buffer_waits_for_auth() {
11988        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_WAIT_SEED).build();
11989        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
11990        let server_messages = Arc::clone(&messages);
11991        sim.host("server", move || {
11992            let messages = Arc::clone(&server_messages);
11993            auth_buffer_server(messages)
11994        });
11995
11996        sim.client("client", async move {
11997            let tracker = AuthTracker::new();
11998            let (handler, _rx) = channel_message_handler();
11999            let client = WebSocketClient::builder()
12000                .config(turmoil_websocket_config())
12001                .message_handler(handler)
12002                .connect()
12003                .await
12004                .expect("Should connect");
12005
12006            client.set_auth_tracker(tracker.clone(), true);
12007            assert!(client.is_active(), "Client should start active");
12008
12009            wait_until_async(
12010                || async { client.is_reconnecting() },
12011                Duration::from_secs(3),
12012            )
12013            .await;
12014
12015            client
12016                .writer_tx
12017                .send(WriterCommand::Send(Message::Text("stale".into())))
12018                .unwrap();
12019
12020            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12021
12022            let _auth_receiver = tracker.begin();
12023
12024            tokio::time::sleep(Duration::from_millis(300)).await;
12025            assert!(
12026                messages.lock().await.is_empty(),
12027                "buffered messages should wait for auth after reconnect"
12028            );
12029
12030            tracker.succeed();
12031
12032            wait_until_async(
12033                || {
12034                    let messages = Arc::clone(&messages);
12035                    async move { messages.lock().await.as_slice() == ["stale"] }
12036                },
12037                Duration::from_secs(3),
12038            )
12039            .await;
12040
12041            assert_eq!(messages.lock().await.as_slice(), ["stale"]);
12042
12043            client.disconnect().await;
12044            assert!(client.is_disconnected());
12045
12046            Ok(())
12047        });
12048
12049        sim.run().unwrap();
12050    }
12051
12052    #[rstest]
12053    fn test_turmoil_reconnect_buffer_discards_after_auth_failure() {
12054        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_DISCARD_SEED).build();
12055        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
12056        let server_messages = Arc::clone(&messages);
12057        sim.host("server", move || {
12058            let messages = Arc::clone(&server_messages);
12059            auth_buffer_server(messages)
12060        });
12061
12062        sim.client("client", async move {
12063            let tracker = AuthTracker::new();
12064            let (handler, _rx) = channel_message_handler();
12065            let client = WebSocketClient::builder()
12066                .config(turmoil_websocket_config())
12067                .message_handler(handler)
12068                .connect()
12069                .await
12070                .expect("Should connect");
12071
12072            client.set_auth_tracker(tracker.clone(), true);
12073            assert!(client.is_active(), "Client should start active");
12074
12075            wait_until_async(
12076                || async { client.is_reconnecting() },
12077                Duration::from_secs(3),
12078            )
12079            .await;
12080
12081            client
12082                .writer_tx
12083                .send(WriterCommand::Send(Message::Text("stale".into())))
12084                .unwrap();
12085
12086            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12087
12088            let _auth_receiver = tracker.begin();
12089            tracker.fail("rejected");
12090
12091            tokio::time::sleep(Duration::from_millis(300)).await;
12092            assert!(
12093                messages.lock().await.is_empty(),
12094                "buffered messages should be discarded after auth failure"
12095            );
12096
12097            let _retry_auth_receiver = tracker.begin();
12098            tracker.succeed();
12099
12100            tokio::time::sleep(Duration::from_millis(300)).await;
12101            assert!(
12102                messages.lock().await.is_empty(),
12103                "discarded messages should not replay on a later auth success"
12104            );
12105
12106            client.disconnect().await;
12107            assert!(client.is_disconnected());
12108
12109            Ok(())
12110        });
12111
12112        sim.run().unwrap();
12113    }
12114
12115    fn turmoil_websocket_config() -> WebSocketConfig {
12116        WebSocketConfig {
12117            url: "ws://server:8080".to_string(),
12118            headers: vec![],
12119            heartbeat_interval_secs: None,
12120            heartbeat_payload: None,
12121            connect_timeout_ms: Some(5_000),
12122            reconnect_delay_initial_ms: Some(50),
12123            reconnect_delay_max_ms: Some(200),
12124            reconnect_backoff_factor: Some(1.0),
12125            reconnect_jitter_ms: Some(0),
12126            reconnect_max_attempts: None,
12127            heartbeat_timeout_secs: None,
12128            idle_timeout_ms: None,
12129            writer_capacity: None,
12130            backend: TransportBackend::Tungstenite,
12131            proxy_url: None,
12132            max_message_size_bytes: None,
12133            max_frame_size_bytes: None,
12134        }
12135    }
12136
12137    async fn auth_buffer_server(
12138        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
12139    ) -> Result<(), Box<dyn std::error::Error>> {
12140        let listener = net::TcpListener::bind("0.0.0.0:8080").await?;
12141
12142        let (stream, _) = listener.accept().await?;
12143        let mut websocket = accept_async(stream).await?;
12144        let _ = websocket.send(WsMessage::Text("first".into())).await;
12145        drop(websocket);
12146
12147        tokio::time::sleep(Duration::from_millis(200)).await;
12148
12149        let (stream, _) = listener.accept().await?;
12150        let mut websocket = accept_async(stream).await?;
12151
12152        while let Some(msg) = websocket.next().await {
12153            match msg {
12154                Ok(WsMessage::Text(text)) => {
12155                    messages.lock().await.push(text.to_string());
12156                }
12157                Ok(WsMessage::Close(_)) => {
12158                    let _ = websocket.close(None).await;
12159                    break;
12160                }
12161                Ok(_) => {}
12162                Err(_) => break,
12163            }
12164        }
12165
12166        Ok(())
12167    }
12168}