Skip to main content

nautilus_network/websocket/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Connection lifecycle and task coordination for [`WebSocketClient`].
17//!
18//! # Execution model
19//!
20//! Handler mode runs a controller, reader, and writer task, plus an optional heartbeat task. The
21//! reader dispatches frames, the writer serializes all sink access, and the controller drives
22//! reconnect and shutdown from the shared [`ConnectionMode`]. Stream mode omits the managed
23//! reader task and automatic reconnect.
24//!
25//! A read failure ends the reader and wakes the controller; a write failure requests reconnect and
26//! also wakes it. The controller establishes a replacement transport with exponential backoff,
27//! hands its sink to the writer, replaces the reader, then publishes the reconnect notification.
28//! Terminal disconnect and close states stop all tasks instead of starting another reconnect.
29//!
30//! # Reconnect ordering
31//!
32//! An accepted reconnect request first transitions to `Reconnect` and invalidates registered
33//! authentication. The controller may establish the replacement while the synchronous loss
34//! callback runs, but cannot restore `Active` until loss publication completes. A reconnect through
35//! [`WebSocketReconnectHandle`] releases its controller request fence before invoking the callback,
36//! so dropping the client can terminate controller work even if the callback blocks. Terminal
37//! states always take precedence.
38//!
39//! # Send semantics
40//!
41//! Ordinary sends return after entering the writer channel. Failed ordinary writes and ordinary
42//! messages encountered during reconnect enter a FIFO buffer for replay on an active replacement
43//! connection. While that replay is pending, ordinary sends also enter the buffer and queue behind
44//! it instead of reaching the replacement sink early. Registered authentication state can hold or
45//! discard that replay before it reaches the sink. Control frames are never buffered, so a pong
46//! sent while replay waits still writes to the replacement sink directly.
47//! [`WebSocketConfig::writer_capacity`] bounds ordinary messages across the channel, in-flight
48//! writes, and replay. Full capacity rejects a send with [`SendError::BufferFull`] before
49//! enqueueing. Ownership-bound sends and control traffic use a separate bounded allowance so
50//! authentication can unblock replay.
51//!
52//! Ownership-bound sends wait for the writer result, require the expected connection epoch, and
53//! never enter the reconnect buffer. The writer advances the epoch only when it installs a
54//! replacement sink, so inbound attribution and bound sends share one transport ownership boundary.
55
56use std::{
57    collections::VecDeque,
58    fmt::{Debug, Display},
59    future::Future,
60    pin::pin,
61    sync::{
62        Arc, OnceLock,
63        atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering},
64    },
65    time::Duration,
66};
67
68use futures_util::{SinkExt, StreamExt};
69use http::HeaderName;
70use nautilus_core::string::secret::REDACTED;
71use nautilus_cryptography::providers::install_cryptographic_provider;
72use parking_lot::RwLock;
73#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
74use rustls::ClientConfig;
75#[cfg(feature = "transport-sockudo")]
76use sockudo_ws::{
77    Config as SockudoConfig, Http1, Role, Stream as SockudoStream,
78    WebSocketStream as SockudoWebSocketStream,
79};
80#[cfg(feature = "transport-sockudo")]
81use tokio::io::{AsyncRead, AsyncWrite};
82#[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
83use tokio_rustls::TlsConnector;
84use tokio_tungstenite::tungstenite::{
85    client::IntoClientRequest, handshake::client::Request, http::HeaderValue, stream::Mode,
86};
87use tokio_util::sync::CancellationToken;
88use ustr::Ustr;
89
90#[cfg(not(feature = "turmoil"))]
91use super::proxy::{ProxyTarget, WsTarget, tunnel_via_proxy};
92use super::{
93    auth::{AuthState, AuthTracker},
94    config::{InitialConnectRetryPolicy, TransportBackend, WebSocketConfig},
95    consts::{
96        CONNECTION_STATE_CHECK_INTERVAL_MS, GRACEFUL_SHUTDOWN_DELAY_MS,
97        GRACEFUL_SHUTDOWN_TIMEOUT_SECS,
98    },
99    frame::FrameStream,
100    types::{
101        CloseHandler, EpochMessageHandler, EpochPingHandler, MessageHandler, MessageReader,
102        MessageWriter, PingHandler, WriterCommand,
103    },
104};
105#[cfg(feature = "transport-sockudo")]
106use crate::net::TcpStream;
107#[cfg(feature = "transport-sockudo")]
108use crate::transport::sockudo::{
109    PrefixedIo, SockudoTransport, client_handshake_with_headers, validate_extra_headers,
110};
111use crate::{
112    RECONNECTED, SocketState, SocketStateSink,
113    backoff::{
114        ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
115    },
116    dst,
117    error::{SendError, is_connection_drop_io_error},
118    logging::{log_task_aborted, log_task_started, log_task_stopped},
119    mode::{
120        ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
121        ReconnectRequestOutcome,
122    },
123    ratelimiter::{RateLimiter, clock::MonotonicClock, quota::Quota},
124    retry::{RetryConfig, RetryError, RetryManager},
125    transport::{
126        BoxedWsTransport, CloseFrame, Message, TransportError, error::retryable_status,
127        tungstenite::TungsteniteTransport,
128    },
129    writer::{DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
130};
131
132const WRITE_TIMEOUT_SECS: u64 = 5;
133const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
134const INITIAL_CONNECT_OPERATION: &str = "WebSocket initial connection";
135
136/// The RFC 6455 control-frame payload limit.
137const MAX_CONTROL_FRAME_PAYLOAD_BYTES: usize = 125;
138
139/// Owns the transport tasks and reconnect state used by [`WebSocketClient`].
140///
141/// # Connection ownership
142///
143/// The client uses one reader and supports concurrent senders. In handler mode, a reader task
144/// dispatches incoming messages while a writer task serializes sends received over a channel. The
145/// controller owns the connection lifecycle and replaces both transport halves during reconnects.
146///
147/// Stream mode returns the reader to the caller. The client cannot replace that reader, so stream
148/// mode disables automatic reconnection.
149///
150/// # Heartbeats
151///
152/// When configured, a dedicated task sends heartbeat messages at the requested interval. Configure
153/// the interval below the server's heartbeat deadline. Handler mode can also opt into a heartbeat
154/// timeout that reconnects when no frame arrives within a venue-specific duration. The timeout
155/// starts with each connection and resets on every inbound frame, including Ping and Pong.
156///
157/// # Reconnection
158///
159/// The writer task owns queued sends across reconnects. A successful reconnect installs the
160/// replacement writer and starts a reader for the new connection epoch. Depending on the
161/// configured authentication gate, buffered sends drain immediately or wait for the new session to
162/// authenticate. Failed authentication discards messages that remain buffered.
163pub struct WebSocketClientInner {
164    config: WebSocketConfig,
165    reconnect_headers: ReconnectHeaders,
166    handler: Option<IncomingHandler>,
167    ping_handler: Option<IncomingPingHandler>,
168    close_handler: Option<CloseHandler>,
169    read_task: Option<dst::task::JoinHandle<()>>,
170    read_fence: Option<ReadSessionFence>,
171    write_task: dst::task::JoinHandle<()>,
172    writer_tx: WriterSender<WriterCommand>,
173    heartbeat_task: Option<dst::task::JoinHandle<()>>,
174    connection_mode: Arc<AtomicU8>,
175    connection_epoch: Arc<AtomicU64>,
176    state_notify: Arc<tokio::sync::Notify>,
177    controller_notify: Arc<tokio::sync::Notify>,
178    reconnect_published: Arc<AtomicBool>,
179    connect_timeout: Duration,
180    heartbeat_timeout: Option<Duration>,
181    backoff: ExponentialBackoff,
182    reconnect_throttle: ReconnectThrottle,
183    reconnect_max_attempts: Option<u32>,
184    reconnection_attempt_count: u32,
185    auth_tracker: Arc<OnceLock<AuthTracker>>,
186    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
187    state_sink: Option<SocketStateSink>,
188    connection_rate_limit: Option<ConnectionRateLimit>,
189}
190
191#[derive(Clone, Debug)]
192struct ConnectionRateLimit {
193    limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
194    keys: Arc<[Ustr]>,
195}
196
197#[derive(Default)]
198struct InitialConnectOptions {
199    retry_policy: Option<InitialConnectRetryPolicy>,
200    cancellation_token: Option<CancellationToken>,
201}
202
203impl WebSocketClientInner {
204    /// Creates an inner WebSocket client with an existing writer.
205    ///
206    /// This is used for stream mode where the reader is owned by the caller.
207    ///
208    /// # Errors
209    ///
210    /// Returns an error if the exponential backoff configuration is invalid.
211    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
212    #[expect(
213        clippy::unused_async,
214        clippy::unused_async_trait_impl,
215        reason = "async signature for consistency with connect-based constructors"
216    )]
217    pub async fn new_with_writer(
218        config: WebSocketConfig,
219        writer: MessageWriter,
220    ) -> Result<Self, TransportError> {
221        Self::new_with_writer_and_state_sink(config, writer, None)
222    }
223
224    fn new_with_writer_and_state_sink(
225        mut config: WebSocketConfig,
226        writer: MessageWriter,
227        state_sink: Option<SocketStateSink>,
228    ) -> Result<Self, TransportError> {
229        install_cryptographic_provider();
230
231        crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
232            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
233        })?;
234
235        if config.heartbeat_interval_secs == Some(0) {
236            return Err(TransportError::Io(std::io::Error::new(
237                std::io::ErrorKind::InvalidInput,
238                "Heartbeat interval cannot be zero",
239            )));
240        }
241
242        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
243        let connection_epoch = Arc::new(AtomicU64::new(0));
244        let state_notify = Arc::new(tokio::sync::Notify::new());
245        let controller_notify = Arc::new(tokio::sync::Notify::new());
246        let reconnect_published = Arc::new(AtomicBool::new(true));
247        let outcome =
248            ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
249        debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
250
251        // Note: We don't spawn a read task here since the reader is handled externally
252        let read_task = None;
253        let read_fence = None;
254
255        // Stream mode ignores reconnect settings, use harmless defaults
256        let backoff = ExponentialBackoff::new(
257            Duration::from_secs(2),
258            Duration::from_secs(30),
259            1.5,
260            100,
261            true,
262        )
263        .map_err(|e| {
264            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
265        })?;
266
267        let auth_tracker = Arc::new(OnceLock::new());
268        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
269
270        let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
271            config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
272        );
273        let write_task = Self::spawn_write_task(
274            Arc::clone(&connection_mode),
275            Arc::clone(&controller_notify),
276            Arc::clone(&reconnect_published),
277            writer,
278            writer_rx,
279            Arc::clone(&connection_epoch),
280            Arc::clone(&auth_tracker),
281            Arc::clone(&reconnect_buffer_waits_for_auth),
282            state_sink.clone(),
283        );
284
285        let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_interval| {
286            Self::spawn_heartbeat_task(
287                Arc::clone(&connection_mode),
288                heartbeat_interval,
289                config.heartbeat_payload.clone(),
290                writer_tx.clone(),
291            )
292        });
293
294        let reconnect_max_attempts = None; // Stream mode does not reconnect
295        let connect_timeout = Duration::from_secs(10);
296
297        let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
298
299        Ok(Self {
300            config,
301            reconnect_headers,
302            handler: None, // Stream mode has no handler
303            ping_handler: None,
304            close_handler: None,
305            writer_tx,
306            connection_mode,
307            connection_epoch,
308            state_notify,
309            controller_notify,
310            reconnect_published,
311            connect_timeout,
312            heartbeat_timeout: None,
313            heartbeat_task,
314            read_task,
315            read_fence,
316            write_task,
317            backoff,
318            reconnect_throttle: ReconnectThrottle::default(),
319            reconnect_max_attempts,
320            reconnection_attempt_count: 0,
321            auth_tracker,
322            reconnect_buffer_waits_for_auth,
323            state_sink,
324            connection_rate_limit: None,
325        })
326    }
327
328    /// Creates an inner WebSocket client.
329    ///
330    /// # Errors
331    ///
332    /// Returns an error if:
333    /// - The writer capacity is invalid.
334    /// - The connection to the server fails.
335    /// - The exponential backoff configuration is invalid.
336    pub async fn connect_url(
337        config: WebSocketConfig,
338        message_handler: Option<MessageHandler>,
339        ping_handler: Option<PingHandler>,
340    ) -> Result<Self, TransportError> {
341        Self::connect_url_with_handler(
342            config,
343            message_handler.map(IncomingHandler::Message),
344            ping_handler.map(IncomingPingHandler::Ping),
345            None,
346            None,
347            None,
348            InitialConnectOptions::default(),
349        )
350        .await
351    }
352
353    async fn connect_url_with_handler(
354        config: WebSocketConfig,
355        handler: Option<IncomingHandler>,
356        ping_handler: Option<IncomingPingHandler>,
357        close_handler: Option<CloseHandler>,
358        state_sink: Option<SocketStateSink>,
359        connection_rate_limit: Option<ConnectionRateLimit>,
360        initial_connect_options: InitialConnectOptions,
361    ) -> Result<Self, TransportError> {
362        install_cryptographic_provider();
363
364        let is_stream_mode = handler.is_none();
365
366        // Adapters build this config by struct literal, bypassing the builder, so this is the only
367        // place the field invariants are enforced for them. Stream mode documents the reconnect and
368        // liveness fields as ignored and permits zero for them, so it checks only what it honors.
369        if is_stream_mode {
370            crate::writer::validate_capacity(config.writer_capacity).map_err(|e| {
371                TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
372            })?;
373
374            if config.heartbeat_interval_secs == Some(0) {
375                return Err(TransportError::Io(std::io::Error::new(
376                    std::io::ErrorKind::InvalidInput,
377                    "Heartbeat interval cannot be zero",
378                )));
379            }
380
381            validate_honored_inbound_limits(&config)?;
382        } else {
383            config.validate().map_err(|e| {
384                TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
385            })?;
386        }
387
388        let heartbeat_timeout = config.resolved_heartbeat_timeout().map(Duration::from_secs);
389        let frame_activity = (!is_stream_mode && heartbeat_timeout.is_some())
390            .then(|| Arc::new(AtomicBool::new(false)));
391        let reconnect_max_attempts = config.reconnect_max_attempts;
392
393        // Stream mode documents reconnect_* fields as ignored (callers may pass Some(0))
394        let connect_timeout = if is_stream_mode {
395            Duration::from_secs(10)
396        } else {
397            Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000))
398        };
399
400        let backoff = ExponentialBackoff::new(
401            Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
402            Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
403            config.reconnect_backoff_factor.unwrap_or(1.5),
404            config.reconnect_jitter_ms.unwrap_or(100),
405            true, // immediate-first
406        )
407        .map_err(|e| {
408            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
409        })?;
410
411        let cancellation_token = initial_connect_options
412            .cancellation_token
413            .unwrap_or_default();
414        let retry_policy = initial_connect_options.retry_policy;
415        let max_attempts = retry_policy
416            .as_ref()
417            .map_or(1, |policy| policy.max_attempts.get());
418        let retry_manager = RetryManager::new(initial_connect_retry_config(retry_policy.as_ref())?);
419        let attempt = AtomicU32::new(0);
420        let operation = || {
421            attempt.fetch_add(1, Ordering::Relaxed);
422
423            async {
424                await_initial_connect_attempt(&cancellation_token, async {
425                    if let Some(rate_limit) = &connection_rate_limit {
426                        rate_limit
427                            .limiter
428                            .await_keys_ready(Some(&rate_limit.keys))
429                            .await;
430                    }
431
432                    // Bound only the dial: the connection rate-limit wait has its own venue timing
433                    dst::time::timeout(
434                        connect_timeout,
435                        Box::pin(Self::connect_with_server_activity(
436                            &config.url,
437                            config.headers.clone(),
438                            config.backend,
439                            config.proxy_url.as_deref(),
440                            config.max_message_size_bytes,
441                            config.max_frame_size_bytes,
442                            frame_activity.clone(),
443                        )),
444                    )
445                    .await
446                    .unwrap_or_else(|_| {
447                        Err(TransportError::Io(std::io::Error::new(
448                            std::io::ErrorKind::TimedOut,
449                            format!(
450                                "connection timed out after {}s",
451                                connect_timeout.as_secs_f64()
452                            ),
453                        )))
454                    })
455                })
456                .await
457                .map_err(InitialConnectError)
458            }
459        };
460
461        let classify = |error: &InitialConnectError| {
462            let error = &error.0;
463            let retryable = is_retryable_initial_connect_error(error);
464            let attempt = attempt.load(Ordering::Relaxed);
465            if retryable && attempt < max_attempts && !cancellation_token.is_cancelled() {
466                log::warn!(
467                    "WebSocket connection attempt {attempt}/{max_attempts} to {REDACTED} failed: {}",
468                    error.redacted_message()
469                );
470            }
471
472            retryable
473        };
474
475        let transport = retry_manager
476            .invocation(INITIAL_CONNECT_OPERATION, operation, classify, |e| {
477                InitialConnectError(initial_connect_retry_error(e))
478            })
479            .cancellation_token(&cancellation_token)
480            .execute()
481            .await
482            .map_err(|e| e.0)?;
483
484        let attempt = attempt.load(Ordering::Relaxed);
485        if attempt > 1 {
486            log::info!("WebSocket connection established after {attempt} attempts");
487        }
488
489        let (writer, reader) = transport;
490        let mut config = config;
491        let reconnect_headers = ReconnectHeaders::new(std::mem::take(&mut config.headers));
492
493        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
494        let connection_epoch = Arc::new(AtomicU64::new(0));
495        let state_notify = Arc::new(tokio::sync::Notify::new());
496        let controller_notify = Arc::new(tokio::sync::Notify::new());
497        let reconnect_published = Arc::new(AtomicBool::new(true));
498        let outcome =
499            ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
500        debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
501
502        let (read_task, read_fence) = if is_stream_mode {
503            (None, None)
504        } else {
505            let read_fence = ReadSessionFence::new();
506            let read_task = Self::spawn_message_handler_task(
507                Arc::clone(&connection_mode),
508                Arc::clone(&state_notify),
509                read_fence.clone(),
510                reader,
511                0,
512                handler.as_ref(),
513                ping_handler.as_ref(),
514                close_handler.as_ref(),
515                state_sink.as_ref(),
516                config.idle_timeout_ms,
517                heartbeat_timeout,
518                frame_activity,
519            );
520            (Some(read_task), Some(read_fence))
521        };
522
523        let auth_tracker = Arc::new(OnceLock::new());
524        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
525
526        let (writer_tx, writer_rx) = crate::writer::channel::<WriterCommand>(
527            config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
528        );
529        let write_task = Self::spawn_write_task(
530            Arc::clone(&connection_mode),
531            Arc::clone(&controller_notify),
532            Arc::clone(&reconnect_published),
533            writer,
534            writer_rx,
535            Arc::clone(&connection_epoch),
536            Arc::clone(&auth_tracker),
537            Arc::clone(&reconnect_buffer_waits_for_auth),
538            state_sink.clone(),
539        );
540
541        // Optionally spawn a heartbeat task to periodically ping server
542        let heartbeat_task = config.heartbeat_interval_secs.map(|heartbeat_secs| {
543            Self::spawn_heartbeat_task(
544                Arc::clone(&connection_mode),
545                heartbeat_secs,
546                config.heartbeat_payload.clone(),
547                writer_tx.clone(),
548            )
549        });
550
551        Ok(Self {
552            config,
553            reconnect_headers,
554            handler,
555            ping_handler,
556            close_handler,
557            read_task,
558            read_fence,
559            write_task,
560            writer_tx,
561            heartbeat_task,
562            connection_mode,
563            connection_epoch,
564            state_notify,
565            controller_notify,
566            reconnect_published,
567            connect_timeout,
568            heartbeat_timeout,
569            backoff,
570            reconnect_throttle: ReconnectThrottle::default(),
571            reconnect_max_attempts,
572            reconnection_attempt_count: 0,
573            auth_tracker,
574            reconnect_buffer_waits_for_auth,
575            state_sink,
576            connection_rate_limit,
577        })
578    }
579
580    /// Connects to the server and returns the split halves of the active transport.
581    ///
582    /// Dispatches on `backend` to the matching transport implementation. The
583    /// [`TransportBackend::Tungstenite`] backend is always available; the
584    /// [`TransportBackend::Sockudo`] backend requires the `transport-sockudo`
585    /// Cargo feature (enabled by default) and uses a custom HTTP/1.1 handshake
586    /// path for upgrade headers.
587    ///
588    /// When `proxy_url` is `Some`, both backends establish an HTTP `CONNECT`
589    /// tunnel through the proxy before performing the WebSocket handshake, and
590    /// each keeps its own handshake path over the resulting stream.
591    ///
592    /// `max_message_size_bytes` and `max_frame_size_bytes` override the selected backend's inbound
593    /// caps. `None` passes that backend's current default config.
594    ///
595    /// # Errors
596    ///
597    /// Returns an error if the URL is invalid, headers fail to parse, the TCP / TLS layer cannot be
598    /// established, the proxy refuses the tunnel, or the WebSocket handshake is rejected by the
599    /// peer. Also returns an error if the Sockudo backend is selected without the
600    /// `transport-sockudo` feature.
601    #[inline]
602    pub async fn connect_with_server(
603        url: &str,
604        headers: Vec<(String, String)>,
605        backend: TransportBackend,
606        proxy_url: Option<&str>,
607        max_message_size_bytes: Option<usize>,
608        max_frame_size_bytes: Option<usize>,
609    ) -> Result<(MessageWriter, MessageReader), TransportError> {
610        Self::connect_with_server_activity(
611            url,
612            headers,
613            backend,
614            proxy_url,
615            max_message_size_bytes,
616            max_frame_size_bytes,
617            None,
618        )
619        .await
620    }
621
622    async fn connect_with_server_activity(
623        url: &str,
624        headers: Vec<(String, String)>,
625        backend: TransportBackend,
626        proxy_url: Option<&str>,
627        max_message_size_bytes: Option<usize>,
628        max_frame_size_bytes: Option<usize>,
629        frame_activity: Option<Arc<AtomicBool>>,
630    ) -> Result<(MessageWriter, MessageReader), TransportError> {
631        #[cfg(all(feature = "simulation", madsim))]
632        if backend == TransportBackend::Sockudo {
633            return Err(TransportError::Other(
634                "Sockudo timers are unsupported under simulation; select Tungstenite".into(),
635            ));
636        }
637
638        #[cfg(all(feature = "simulation", madsim))]
639        if proxy_url.is_some() || !url.starts_with("ws://") {
640            return Err(TransportError::InvalidUrl(
641                "WebSocket simulation requires plaintext ws:// endpoints without a proxy".into(),
642            ));
643        }
644
645        let limits = InboundConfig {
646            max_message_size_bytes,
647            max_frame_size_bytes,
648            frame_activity,
649        };
650
651        match backend {
652            TransportBackend::Tungstenite => match proxy_url {
653                Some(proxy) => {
654                    Box::pin(Self::connect_tungstenite_via_proxy(
655                        url, headers, proxy, limits,
656                    ))
657                    .await
658                }
659                None => Self::connect_tungstenite(url, headers, limits).await,
660            },
661            TransportBackend::Sockudo => {
662                #[cfg(feature = "transport-sockudo")]
663                {
664                    match proxy_url {
665                        Some(proxy) => {
666                            Box::pin(Self::connect_sockudo_via_proxy(url, headers, proxy, limits))
667                                .await
668                        }
669                        None => Self::connect_sockudo(url, headers, limits).await,
670                    }
671                }
672
673                #[cfg(not(feature = "transport-sockudo"))]
674                {
675                    Err(TransportError::Other(
676                        "sockudo backend selected but the transport-sockudo \
677                         Cargo feature is not enabled"
678                            .to_string(),
679                    ))
680                }
681            }
682        }
683    }
684
685    async fn connect_tungstenite(
686        url: &str,
687        headers: Vec<(String, String)>,
688        limits: InboundConfig,
689    ) -> Result<(MessageWriter, MessageReader), TransportError> {
690        let request = tungstenite_request(url, headers)?;
691        let mode = tokio_tungstenite::tungstenite::client::uri_mode(request.uri())?;
692        let host = request
693            .uri()
694            .host()
695            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".into()))?;
696        let port = request
697            .uri()
698            .port_u16()
699            .unwrap_or(if matches!(mode, Mode::Tls) { 443 } else { 80 });
700        let address = format!("{host}:{port}");
701        let socket = dst::net::TcpStream::connect(address.as_str()).await?;
702        #[cfg(not(all(feature = "simulation", madsim)))]
703        crate::net::apply_socket_options(&socket);
704        let socket = crate::tls::tcp_tls(&request, mode, socket, None).await?;
705        let socket = FrameStream::new(socket, limits.frame_activity.clone());
706        let (stream, _) = tokio_tungstenite::client_async_with_config(
707            request,
708            socket,
709            tungstenite_protocol_config(&limits),
710        )
711        .await?;
712        let transport: BoxedWsTransport = Box::pin(TungsteniteTransport::new(stream));
713        Ok(transport.split())
714    }
715
716    /// Connects via an HTTP `CONNECT` proxy and performs the WebSocket
717    /// handshake over the resulting tunnel.
718    ///
719    /// A SOCKS or other unsupported proxy scheme is an error; the client never
720    /// falls back to a direct connection. Only available in production builds; the
721    /// turmoil simulator does not model arbitrary outbound TCP via a proxy.
722    #[inline]
723    #[cfg(not(feature = "turmoil"))]
724    async fn connect_tungstenite_via_proxy(
725        url: &str,
726        headers: Vec<(String, String)>,
727        proxy_url: &str,
728        limits: InboundConfig,
729    ) -> Result<(MessageWriter, MessageReader), TransportError> {
730        let proxy = ProxyTarget::parse(proxy_url)?;
731
732        let request = tungstenite_request(url, headers)?;
733
734        let target = WsTarget::parse(url)?;
735        let stream = tunnel_via_proxy(&target, &proxy).await?;
736
737        // `ProxiedStream` implements the IO traits over all four variants, so one
738        // instantiation covers every tunnel shape. The future is boxed because
739        // `client_async` produces a large state machine.
740        let transport: BoxedWsTransport =
741            Box::pin(proxied_ws_handshake(request, stream, limits)).await?;
742
743        Ok(transport.split())
744    }
745
746    /// Turmoil simulator variant: HTTP `CONNECT` tunneling is not supported
747    /// under the simulator so any proxy URL is rejected up front.
748    #[inline]
749    #[cfg(feature = "turmoil")]
750    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
751    #[expect(
752        clippy::unused_async,
753        clippy::unused_async_trait_impl,
754        reason = "signature mirrors the production variant; both are awaited in the dispatcher"
755    )]
756    async fn connect_tungstenite_via_proxy(
757        _url: &str,
758        _headers: Vec<(String, String)>,
759        _proxy_url: &str,
760        _limits: InboundConfig,
761    ) -> Result<(MessageWriter, MessageReader), TransportError> {
762        Err(TransportError::Other(
763            "proxy_url is not supported under the turmoil simulator".to_string(),
764        ))
765    }
766
767    /// Connects with the server using the sockudo-ws backend.
768    ///
769    /// Uses a local HTTP/1.1 handshake path so error logging and stream
770    /// construction stay in our hands regardless of header count.
771    ///
772    /// Under the turmoil simulator, only plaintext `ws://` is supported (the
773    /// simulator does not model TLS), so a `wss://` URL returns
774    /// [`TransportError::Tls`] up front.
775    #[inline]
776    #[cfg(feature = "transport-sockudo")]
777    async fn connect_sockudo(
778        url: &str,
779        headers: Vec<(String, String)>,
780        limits: InboundConfig,
781    ) -> Result<(MessageWriter, MessageReader), TransportError> {
782        let target = SockudoTarget::parse(url)?;
783        validate_extra_headers(&headers).map_err(TransportError::from)?;
784
785        #[cfg(feature = "turmoil")]
786        if target.is_tls {
787            return Err(TransportError::Tls(
788                "wss:// is not supported under the turmoil simulator; use ws://".to_string(),
789            ));
790        }
791
792        let tcp_stream = TcpStream::connect((target.host.as_str(), target.port))
793            .await
794            .map_err(TransportError::Io)?;
795
796        crate::net::apply_socket_options(&tcp_stream);
797
798        #[cfg(not(feature = "turmoil"))]
799        if target.is_tls {
800            let mut root_store = rustls::RootCertStore::empty();
801            root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
802            let config = ClientConfig::builder()
803                .with_root_certificates(root_store)
804                .with_no_client_auth();
805            let connector = TlsConnector::from(std::sync::Arc::new(config));
806            let domain = rustls::pki_types::ServerName::try_from(target.host.clone())
807                .map_err(|e| TransportError::Tls(format!("Invalid DNS name: {e}")))?;
808            let tls_stream = connector
809                .connect(domain, tcp_stream)
810                .await
811                .map_err(TransportError::Io)?;
812            return Self::finish_sockudo_handshake(tls_stream, &target, &headers, limits).await;
813        }
814
815        Self::finish_sockudo_handshake(tcp_stream, &target, &headers, limits).await
816    }
817
818    /// Connects via an HTTP `CONNECT` proxy and performs the sockudo WebSocket
819    /// handshake over the resulting tunnel.
820    ///
821    /// A SOCKS or other unsupported proxy scheme is an error, matching the
822    /// Tungstenite path.
823    #[inline]
824    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
825    async fn connect_sockudo_via_proxy(
826        url: &str,
827        headers: Vec<(String, String)>,
828        proxy_url: &str,
829        limits: InboundConfig,
830    ) -> Result<(MessageWriter, MessageReader), TransportError> {
831        let proxy = ProxyTarget::parse(proxy_url)?;
832
833        let target = SockudoTarget::parse(url)?;
834        validate_extra_headers(&headers).map_err(TransportError::from)?;
835
836        // `tunnel_via_proxy` establishes upstream TLS inside the tunnel when the
837        // target is `wss://`, so the handshake below runs over the finished stream
838        // regardless of which of the four tunnel shapes it returned.
839        let ws_target = WsTarget::parse(url)?;
840        let stream = tunnel_via_proxy(&ws_target, &proxy).await?;
841
842        Self::finish_sockudo_handshake(stream, &target, &headers, limits).await
843    }
844
845    /// Turmoil simulator variant: HTTP `CONNECT` tunneling is not modeled under
846    /// the simulator so any proxy URL is rejected up front.
847    #[inline]
848    #[cfg(all(feature = "transport-sockudo", feature = "turmoil"))]
849    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
850    #[expect(
851        clippy::unused_async,
852        clippy::unused_async_trait_impl,
853        reason = "signature mirrors the production variant; both are awaited in the dispatcher"
854    )]
855    async fn connect_sockudo_via_proxy(
856        _url: &str,
857        _headers: Vec<(String, String)>,
858        _proxy_url: &str,
859        _limits: InboundConfig,
860    ) -> Result<(MessageWriter, MessageReader), TransportError> {
861        Err(TransportError::Other(
862            "proxy_url is not supported under the turmoil simulator".to_string(),
863        ))
864    }
865
866    #[cfg(feature = "transport-sockudo")]
867    async fn finish_sockudo_handshake<S>(
868        stream: S,
869        target: &SockudoTarget,
870        headers: &[(String, String)],
871        limits: InboundConfig,
872    ) -> Result<(MessageWriter, MessageReader), TransportError>
873    where
874        S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
875    {
876        // Use one path for uniform error logging and ownership of stream
877        // construction; sockudo's handshake error drops the rejection status
878        // the retry policy needs.
879        let mut stream = FrameStream::new(stream, limits.frame_activity.clone());
880        let handshake =
881            client_handshake_with_headers(&mut stream, &target.host_header, &target.path, headers)
882                .await?;
883
884        // Reading the HTTP 101 may also read the first WebSocket frame prefix;
885        // replay it only when present so the ordinary path stays unwrapped.
886        let stream = match handshake.leftover {
887            Some(prefix) => SockudoStream::<Http1>::new(PrefixedIo::new(stream, prefix)),
888            None => SockudoStream::<Http1>::new(stream),
889        };
890
891        let ws = SockudoWebSocketStream::from_raw(
892            stream,
893            Role::Client,
894            sockudo_protocol_config(&limits),
895        );
896        let transport: BoxedWsTransport = Box::pin(SockudoTransport::new(ws));
897        Ok(transport.split())
898    }
899}
900
901#[derive(Clone, Debug, Default)]
902struct InboundConfig {
903    max_message_size_bytes: Option<usize>,
904    max_frame_size_bytes: Option<usize>,
905    frame_activity: Option<Arc<AtomicBool>>,
906}
907
908fn validate_honored_inbound_limits(config: &WebSocketConfig) -> Result<(), TransportError> {
909    if config.max_message_size_bytes == Some(0) {
910        return Err(TransportError::Io(std::io::Error::new(
911            std::io::ErrorKind::InvalidInput,
912            "max_message_size_bytes must be positive",
913        )));
914    }
915
916    if config.max_frame_size_bytes == Some(0) {
917        return Err(TransportError::Io(std::io::Error::new(
918            std::io::ErrorKind::InvalidInput,
919            "max_frame_size_bytes must be positive",
920        )));
921    }
922
923    Ok(())
924}
925
926fn tungstenite_protocol_config(
927    limits: &InboundConfig,
928) -> Option<tokio_tungstenite::tungstenite::protocol::WebSocketConfig> {
929    if limits.max_message_size_bytes.is_none() && limits.max_frame_size_bytes.is_none() {
930        return None;
931    }
932
933    let mut config = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
934    if let Some(size) = limits.max_message_size_bytes {
935        config.max_message_size = Some(size);
936    }
937
938    if let Some(size) = limits.max_frame_size_bytes {
939        config.max_frame_size = Some(size);
940    }
941
942    Some(config)
943}
944
945#[cfg(feature = "transport-sockudo")]
946fn sockudo_protocol_config(limits: &InboundConfig) -> SockudoConfig {
947    let mut config = SockudoConfig::builder()
948        .auto_ping(false)
949        .idle_timeout(0)
950        .build();
951
952    if let Some(size) = limits.max_message_size_bytes {
953        config.max_message_size = size;
954    }
955
956    if let Some(size) = limits.max_frame_size_bytes {
957        config.max_frame_size = size;
958    }
959
960    config
961}
962
963fn tungstenite_request(
964    url: &str,
965    headers: Vec<(String, String)>,
966) -> Result<Request, TransportError> {
967    let mut request = url.into_client_request().map_err(TransportError::from)?;
968
969    for (key, value) in headers {
970        let value = HeaderValue::from_str(&value)
971            .map_err(|e| TransportError::Handshake(format!("invalid header value: {e}")))?;
972        let name: HeaderName = key
973            .parse()
974            .map_err(|e| TransportError::Handshake(format!("invalid header name: {e}")))?;
975        request.headers_mut().insert(name, value);
976    }
977
978    Ok(request)
979}
980
981fn is_connection_drop_transport_error(err: &TransportError) -> bool {
982    err.is_closed() || matches!(err, TransportError::Io(e) if is_connection_drop_io_error(e))
983}
984
985fn is_retryable_initial_connect_error(err: &TransportError) -> bool {
986    match err {
987        TransportError::ConnectionClosed
988        | TransportError::ConnectionReset
989        | TransportError::ClosedByPeer(_) => true,
990        TransportError::Io(error) => !matches!(
991            error.kind(),
992            std::io::ErrorKind::InvalidInput
993                | std::io::ErrorKind::InvalidData
994                | std::io::ErrorKind::Unsupported
995                | std::io::ErrorKind::PermissionDenied
996        ),
997        TransportError::UpgradeRejected(status) | TransportError::ProxyConnectRejected(status) => {
998            retryable_status(*status)
999        }
1000        TransportError::InvalidUrl(_)
1001        | TransportError::Handshake(_)
1002        | TransportError::Tls(_)
1003        | TransportError::Protocol(_)
1004        | TransportError::MessageTooLarge
1005        | TransportError::FrameTooLarge
1006        | TransportError::InvalidUtf8
1007        | TransportError::Other(_) => false,
1008    }
1009}
1010
1011fn initial_connect_retry_config(
1012    policy: Option<&InitialConnectRetryPolicy>,
1013) -> Result<RetryConfig, TransportError> {
1014    if let Some(policy) = policy {
1015        ExponentialBackoff::new(
1016            policy.delay_initial,
1017            policy.delay_max,
1018            policy.backoff_factor,
1019            policy.jitter_ms,
1020            false,
1021        )
1022        .map_err(|e| {
1023            TransportError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
1024        })?;
1025
1026        Ok(RetryConfig {
1027            max_retries: policy.max_attempts.get() - 1,
1028            initial_delay_ms: duration_to_millis("delay_initial", policy.delay_initial)?,
1029            max_delay_ms: duration_to_millis("delay_max", policy.delay_max)?,
1030            backoff_factor: policy.backoff_factor,
1031            jitter_ms: policy.jitter_ms,
1032            operation_timeout_ms: None,
1033            immediate_first: false,
1034            max_elapsed_ms: None,
1035        })
1036    } else {
1037        Ok(RetryConfig {
1038            max_retries: 0,
1039            initial_delay_ms: 1,
1040            max_delay_ms: 1,
1041            backoff_factor: 1.0,
1042            jitter_ms: 0,
1043            operation_timeout_ms: None,
1044            immediate_first: false,
1045            max_elapsed_ms: None,
1046        })
1047    }
1048}
1049
1050fn duration_to_millis(field: &str, duration: Duration) -> Result<u64, TransportError> {
1051    let nanoseconds = duration.as_nanos();
1052    if nanoseconds > u128::from(u64::MAX) {
1053        return Err(TransportError::Io(std::io::Error::new(
1054            std::io::ErrorKind::InvalidInput,
1055            format!("{field} exceeds the maximum backoff duration"),
1056        )));
1057    }
1058
1059    let milliseconds = nanoseconds
1060        .div_ceil(1_000_000)
1061        .min(u128::from(u64::MAX / 1_000_000));
1062    u64::try_from(milliseconds).map_err(|_| {
1063        TransportError::Io(std::io::Error::new(
1064            std::io::ErrorKind::InvalidInput,
1065            format!("{field} exceeds the maximum backoff duration"),
1066        ))
1067    })
1068}
1069
1070async fn await_initial_connect_attempt<F, T>(
1071    cancellation_token: &CancellationToken,
1072    attempt: F,
1073) -> Result<T, TransportError>
1074where
1075    F: Future<Output = Result<T, TransportError>>,
1076{
1077    tokio::select! {
1078        biased;
1079        () = cancellation_token.cancelled() => Err(initial_connect_cancelled()),
1080        result = attempt => result,
1081    }
1082}
1083
1084struct InitialConnectError(TransportError);
1085
1086impl std::error::Error for InitialConnectError {}
1087
1088impl Display for InitialConnectError {
1089    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1090        f.write_str(&self.0.redacted_message())
1091    }
1092}
1093
1094impl Debug for InitialConnectError {
1095    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1096        Display::fmt(self, f)
1097    }
1098}
1099
1100fn initial_connect_retry_error(error: RetryError) -> TransportError {
1101    let kind = match error {
1102        RetryError::Canceled => return initial_connect_cancelled(),
1103        RetryError::InvalidConfiguration { .. } => std::io::ErrorKind::InvalidInput,
1104        RetryError::OperationTimeout { .. } | RetryError::ElapsedBudgetExceeded { .. } => {
1105            std::io::ErrorKind::TimedOut
1106        }
1107    };
1108
1109    TransportError::Io(std::io::Error::new(kind, error))
1110}
1111
1112fn initial_connect_cancelled() -> TransportError {
1113    TransportError::Io(std::io::Error::new(
1114        std::io::ErrorKind::Interrupted,
1115        "initial WebSocket connection cancelled",
1116    ))
1117}
1118
1119// Debug when we asked to disconnect (Disconnect/Closed), else Warn for a peer close
1120fn read_termination_log_level(connection_state: &AtomicU8) -> log::Level {
1121    let mode = ConnectionMode::from_atomic(connection_state);
1122    if mode.is_disconnect() || mode.is_closed() {
1123        log::Level::Debug
1124    } else {
1125        log::Level::Warn
1126    }
1127}
1128
1129#[cfg(test)]
1130mod connection_error_tests {
1131    use std::io;
1132
1133    use rstest::rstest;
1134
1135    use super::*;
1136    use crate::transport::CloseFrame;
1137
1138    #[rstest]
1139    #[case(TransportError::ConnectionClosed, true)]
1140    #[case(TransportError::ConnectionReset, true)]
1141    #[case(TransportError::ClosedByPeer(Some(CloseFrame::new(1000, "bye"))), true)]
1142    #[case(TransportError::ClosedByPeer(None), true)]
1143    #[case(TransportError::Io(io::Error::from(io::ErrorKind::BrokenPipe)), true)]
1144    #[case(
1145        TransportError::Io(io::Error::from(io::ErrorKind::ConnectionReset)),
1146        true
1147    )]
1148    #[case(TransportError::Io(io::Error::from(io::ErrorKind::TimedOut)), true)]
1149    #[case(
1150        TransportError::Io(io::Error::from(io::ErrorKind::UnexpectedEof)),
1151        true
1152    )]
1153    #[case(
1154        TransportError::Io(io::Error::from(io::ErrorKind::InvalidInput)),
1155        false
1156    )]
1157    #[case(TransportError::InvalidUrl("http://example.com".into()), false)]
1158    #[case(TransportError::Handshake("bad".into()), false)]
1159    #[case(TransportError::Protocol("bad opcode".into()), false)]
1160    #[case(TransportError::Tls("bad certificate".into()), false)]
1161    #[case(TransportError::MessageTooLarge, false)]
1162    #[case(TransportError::FrameTooLarge, false)]
1163    #[case(TransportError::InvalidUtf8, false)]
1164    #[case(TransportError::Other("backend protocol mismatch".into()), false)]
1165    fn connection_drop_transport_error_classification(
1166        #[case] err: TransportError,
1167        #[case] expected: bool,
1168    ) {
1169        assert_eq!(is_connection_drop_transport_error(&err), expected);
1170    }
1171
1172    #[rstest]
1173    #[case(Duration::ZERO, 0)]
1174    #[case(Duration::from_nanos(1), 1)]
1175    #[case(Duration::from_micros(500), 1)]
1176    #[case(Duration::from_nanos(1_000_001), 2)]
1177    #[case(Duration::from_millis(500), 500)]
1178    #[case(Duration::from_secs(5), 5_000)]
1179    fn duration_to_millis_rounds_up(#[case] duration: Duration, #[case] expected: u64) {
1180        assert_eq!(duration_to_millis("delay", duration).unwrap(), expected);
1181    }
1182
1183    #[rstest]
1184    fn duration_to_millis_caps_at_backoff_maximum() {
1185        assert_eq!(
1186            duration_to_millis("delay", Duration::from_nanos(u64::MAX)).unwrap(),
1187            u64::MAX / 1_000_000
1188        );
1189    }
1190
1191    #[rstest]
1192    fn duration_to_millis_rejects_excessive_duration() {
1193        let error = duration_to_millis("delay", Duration::from_secs(u64::MAX)).unwrap_err();
1194
1195        let TransportError::Io(error) = error else {
1196            panic!("expected an I/O error, was: {error:?}");
1197        };
1198
1199        assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1200        assert_eq!(
1201            error.to_string(),
1202            "delay exceeds the maximum backoff duration"
1203        );
1204    }
1205
1206    #[rstest]
1207    fn initial_connect_retry_config_preserves_duration_validation() {
1208        let policy = InitialConnectRetryPolicy {
1209            max_attempts: std::num::NonZeroU32::new(2).unwrap(),
1210            delay_initial: Duration::from_micros(900),
1211            delay_max: Duration::from_micros(500),
1212            backoff_factor: 2.0,
1213            jitter_ms: 0,
1214        };
1215
1216        let error = initial_connect_retry_config(Some(&policy)).unwrap_err();
1217
1218        let TransportError::Io(error) = error else {
1219            panic!("expected an I/O error, was: {error:?}");
1220        };
1221
1222        assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1223        assert!(
1224            error
1225                .to_string()
1226                .contains("delay_max must be >= delay_initial")
1227        );
1228    }
1229
1230    #[rstest]
1231    #[tokio::test]
1232    async fn initial_connect_attempt_prefers_cancellation_over_ready_result() {
1233        let token = CancellationToken::new();
1234        token.cancel();
1235
1236        let error = await_initial_connect_attempt(&token, std::future::ready(Ok(())))
1237            .await
1238            .expect_err("cancellation should take priority");
1239
1240        assert!(
1241            matches!(error, TransportError::Io(ref error) if error.kind() == io::ErrorKind::Interrupted)
1242        );
1243    }
1244
1245    #[rstest]
1246    fn unset_inbound_limits_preserve_backend_defaults() {
1247        let limits = InboundConfig::default();
1248        assert!(tungstenite_protocol_config(&limits).is_none());
1249
1250        #[cfg(feature = "transport-sockudo")]
1251        {
1252            let config = sockudo_protocol_config(&limits);
1253            let default = SockudoConfig::default();
1254            assert_eq!(config.max_message_size, default.max_message_size);
1255            assert_eq!(config.max_frame_size, default.max_frame_size);
1256            assert!(!config.auto_ping);
1257            assert_eq!(config.idle_timeout, 0);
1258        }
1259    }
1260
1261    #[rstest]
1262    fn configured_inbound_limit_overrides_only_that_cap() {
1263        let limits = InboundConfig {
1264            max_message_size_bytes: Some(32),
1265            max_frame_size_bytes: None,
1266            frame_activity: None,
1267        };
1268
1269        let config = tungstenite_protocol_config(&limits).expect("override should build a config");
1270        let default = tokio_tungstenite::tungstenite::protocol::WebSocketConfig::default();
1271
1272        assert_eq!(config.max_message_size, Some(32));
1273        assert_eq!(config.max_frame_size, default.max_frame_size);
1274
1275        #[cfg(feature = "transport-sockudo")]
1276        {
1277            let config = sockudo_protocol_config(&limits);
1278            let default = SockudoConfig::default();
1279            assert_eq!(config.max_message_size, 32);
1280            assert_eq!(config.max_frame_size, default.max_frame_size);
1281            assert!(!config.auto_ping);
1282            assert_eq!(config.idle_timeout, 0);
1283        }
1284    }
1285}
1286
1287/// Complete the WebSocket handshake over a stream that has already been
1288/// tunneled through an HTTP `CONNECT` proxy. Generic over the concrete
1289/// stream type so the four [`super::proxy::ProxiedStream`] variants share
1290/// a single body.
1291#[cfg(not(feature = "turmoil"))]
1292async fn proxied_ws_handshake<S>(
1293    request: tokio_tungstenite::tungstenite::handshake::client::Request,
1294    stream: S,
1295    limits: InboundConfig,
1296) -> Result<BoxedWsTransport, TransportError>
1297where
1298    S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
1299{
1300    let stream = FrameStream::new(stream, limits.frame_activity.clone());
1301    let (ws, _resp) = tokio_tungstenite::client_async_with_config(
1302        request,
1303        stream,
1304        tungstenite_protocol_config(&limits),
1305    )
1306    .await
1307    .map_err(TransportError::from)?;
1308    Ok(Box::pin(TungsteniteTransport::new(ws)))
1309}
1310
1311/// Parsed components of a `ws://` / `wss://` URL needed by the sockudo backend.
1312///
1313/// Sockudo's HTTP/1.1 client passes the `host` argument verbatim as the
1314/// HTTP `Host:` header, so it must include the explicit port when one is
1315/// present in the URL (RFC 7230 section 5.4). The DNS / SNI lookup uses the bare
1316/// host without the port.
1317#[cfg(feature = "transport-sockudo")]
1318#[derive(Debug, PartialEq, Eq)]
1319struct SockudoTarget {
1320    host: String,
1321    host_header: String,
1322    port: u16,
1323    path: String,
1324    is_tls: bool,
1325}
1326
1327#[cfg(feature = "transport-sockudo")]
1328impl SockudoTarget {
1329    fn parse(url: &str) -> Result<Self, TransportError> {
1330        let parsed = url::Url::parse(url)
1331            .map_err(|e| TransportError::InvalidUrl(format!("invalid WebSocket URL: {e}")))?;
1332
1333        let scheme = parsed.scheme();
1334
1335        let is_tls = match scheme {
1336            "ws" => false,
1337            "wss" => true,
1338            other => {
1339                return Err(TransportError::InvalidUrl(format!(
1340                    "expected ws:// or wss:// scheme, was {other}"
1341                )));
1342            }
1343        };
1344
1345        let raw_host = parsed
1346            .host_str()
1347            .ok_or_else(|| TransportError::InvalidUrl("missing hostname".to_string()))?;
1348
1349        // url::Url stores IPv6 hosts in their bracketed form (e.g. `[::1]`).
1350        // Brackets are correct for the HTTP `Host:` header but invalid for
1351        // DNS/TCP and TLS SNI, so we keep two representations: a bracketed
1352        // `host_header` for the upgrade, and a bare `host` for the socket connection.
1353        let is_bracketed = raw_host.starts_with('[') && raw_host.ends_with(']');
1354        let host = if is_bracketed {
1355            raw_host[1..raw_host.len() - 1].to_string()
1356        } else {
1357            raw_host.to_string()
1358        };
1359
1360        let explicit_port = parsed.port();
1361        let port = explicit_port.unwrap_or(if is_tls { 443 } else { 80 });
1362
1363        let host_header = match explicit_port {
1364            Some(p) => format!("{raw_host}:{p}"),
1365            None => raw_host.to_string(),
1366        };
1367
1368        let path = if parsed.path().is_empty() {
1369            "/".to_string()
1370        } else {
1371            let mut p = parsed.path().to_string();
1372            if let Some(query) = parsed.query() {
1373                p.push('?');
1374                p.push_str(query);
1375            }
1376
1377            p
1378        };
1379
1380        Ok(Self {
1381            host,
1382            host_header,
1383            port,
1384            path,
1385            is_tls,
1386        })
1387    }
1388}
1389
1390impl WebSocketClientInner {
1391    /// Reconnect with server.
1392    ///
1393    /// Make a new connection with server. Use the new read and write halves
1394    /// to update self writer and read and heartbeat tasks.
1395    ///
1396    /// For stream-based clients (created via [`WebSocketClient::stream_builder`]), reconnection is
1397    /// disabled because the reader is owned by the caller and cannot be replaced. Stream users
1398    /// should handle disconnections by creating a new connection.
1399    ///
1400    /// The reconnect timeout bounds only connection establishment. Once the
1401    /// new writer is handed to the writer task the swap runs to completion,
1402    /// so buffered messages can never drain into a connection that lost its
1403    /// reader to a timeout; the post-connect steps are individually bounded
1404    /// by the writer task's graceful-shutdown timeout.
1405    ///
1406    /// # Errors
1407    ///
1408    /// Returns an error if:
1409    /// - The reconnection attempt times out.
1410    /// - The connection to the server fails.
1411    pub async fn reconnect(&mut self) -> Result<(), TransportError> {
1412        Box::pin(self.reconnect_with_outcome()).await.map(|_| ())
1413    }
1414
1415    async fn wait_for_reconnect_publication(&self) -> bool {
1416        let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1417
1418        loop {
1419            let mut notified = pin!(self.controller_notify.notified());
1420            notified.as_mut().enable();
1421
1422            if !ConnectionMode::from_atomic(&self.connection_mode).is_reconnect() {
1423                return false;
1424            }
1425
1426            if self.reconnect_published.load(Ordering::SeqCst) {
1427                return true;
1428            }
1429
1430            tokio::select! {
1431                biased;
1432                () = notified => {}
1433                () = dst::time::sleep(fallback_interval) => {}
1434            }
1435        }
1436    }
1437
1438    async fn reconnect_with_outcome(&mut self) -> Result<ReconnectOutcome, TransportError> {
1439        log::info!("Reconnecting");
1440
1441        if self.handler.is_none() {
1442            log::warn!(
1443                "Auto-reconnect disabled for stream-based WebSocket client; \
1444                stream users must manually reconnect by creating a new connection"
1445            );
1446            // Transition to CLOSED state to stop reconnection attempts
1447            self.connection_mode
1448                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1449            fail_registered_auth(
1450                self.auth_tracker.as_ref(),
1451                "WebSocket stream mode cannot reconnect",
1452            );
1453            // Publish the terminal state only once its bookkeeping is complete; the
1454            // controller used to notify after `reconnect` returned, so a waiter must
1455            // not observe `Closed` before pending authentication has been failed.
1456            self.state_notify.notify_waiters();
1457            return Ok(ReconnectOutcome::Aborted);
1458        }
1459
1460        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1461            log::debug!("Reconnect aborted due to disconnect state");
1462            return Ok(ReconnectOutcome::Aborted);
1463        }
1464
1465        if let Some(rate_limit) = &self.connection_rate_limit {
1466            rate_limit
1467                .limiter
1468                .await_keys_ready(Some(&rate_limit.keys))
1469                .await;
1470        }
1471
1472        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1473            log::debug!("Reconnect aborted during connection rate-limit wait");
1474            return Ok(ReconnectOutcome::Aborted);
1475        }
1476
1477        let frame_activity = self
1478            .heartbeat_timeout
1479            .map(|_| Arc::new(AtomicBool::new(false)));
1480
1481        // Bound only connection establishment; the swap below must run to completion
1482        let (new_writer, reader) = dst::time::timeout(
1483            self.connect_timeout,
1484            Box::pin(Self::connect_with_server_activity(
1485                &self.config.url,
1486                self.reconnect_headers.snapshot(),
1487                self.config.backend,
1488                self.config.proxy_url.as_deref(),
1489                self.config.max_message_size_bytes,
1490                self.config.max_frame_size_bytes,
1491                frame_activity.clone(),
1492            )),
1493        )
1494        .await
1495        .map_err(|_| {
1496            TransportError::Io(std::io::Error::new(
1497                std::io::ErrorKind::TimedOut,
1498                format!(
1499                    "reconnection timed out after {}s",
1500                    self.connect_timeout.as_secs_f64()
1501                ),
1502            ))
1503        })??;
1504
1505        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1506            log::debug!("Reconnect aborted mid-flight (after connect)");
1507            return Ok(ReconnectOutcome::Aborted);
1508        }
1509
1510        // Use a oneshot channel to synchronize the writer swap before transitioning
1511        // back to ACTIVE. Buffered messages stay in the writer task and replay later.
1512        let (tx, rx) = tokio::sync::oneshot::channel();
1513
1514        if let Err(e) = self
1515            .writer_tx
1516            .send_update(WriterCommand::Update(new_writer, tx))
1517        {
1518            log::error!("{e}");
1519            return Err(TransportError::Io(std::io::Error::new(
1520                std::io::ErrorKind::BrokenPipe,
1521                format!("Failed to send update command: {e}"),
1522            )));
1523        }
1524
1525        // Wait for writer to confirm it accepted the new socket
1526        let connection_epoch = match rx.await {
1527            Ok(connection_epoch) => {
1528                log::debug!("Writer confirmed socket update: epoch={connection_epoch}");
1529                connection_epoch
1530            }
1531            Err(e) => {
1532                log::error!("Writer dropped update channel: {e}");
1533                return Err(TransportError::Io(std::io::Error::new(
1534                    std::io::ErrorKind::BrokenPipe,
1535                    "Writer task dropped response channel",
1536                )));
1537            }
1538        };
1539
1540        // Delay before closing connection
1541        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1542
1543        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
1544            log::debug!("Reconnect aborted mid-flight (after delay)");
1545            return Ok(ReconnectOutcome::Aborted);
1546        }
1547
1548        if let Some(read_fence) = self.read_fence.take() {
1549            read_fence.invalidate();
1550        }
1551
1552        if let Some(ref read_task) = self.read_task.take()
1553            && !read_task.is_finished()
1554        {
1555            read_task.abort();
1556            log_task_aborted("read");
1557        }
1558
1559        if !self.wait_for_reconnect_publication().await {
1560            log::debug!("Reconnect aborted before state publication completed");
1561            return Ok(ReconnectOutcome::Aborted);
1562        }
1563
1564        // Atomically transition from Reconnect to Active
1565        // This prevents race condition where disconnect could be requested between check and store
1566        if ConnectionMode::complete_reconnect_with_sink(
1567            &self.connection_mode,
1568            self.state_sink.as_ref(),
1569        ) == ReconnectOutcome::Aborted
1570        {
1571            log::debug!("Reconnect aborted (state changed during reconnect)");
1572            return Ok(ReconnectOutcome::Aborted);
1573        }
1574
1575        let read_fence = ReadSessionFence::new();
1576        self.read_task = Some(Self::spawn_message_handler_task(
1577            Arc::clone(&self.connection_mode),
1578            Arc::clone(&self.state_notify),
1579            read_fence.clone(),
1580            reader,
1581            connection_epoch,
1582            self.handler.as_ref(),
1583            self.ping_handler.as_ref(),
1584            self.close_handler.as_ref(),
1585            self.state_sink.as_ref(),
1586            self.config.idle_timeout_ms,
1587            self.heartbeat_timeout,
1588            frame_activity,
1589        ));
1590        self.read_fence = Some(read_fence);
1591
1592        log::info!("Reconnect succeeded");
1593        Ok(ReconnectOutcome::Reconnected)
1594    }
1595
1596    /// Returns whether the client's transport tasks are still running.
1597    ///
1598    /// Returns `true` if both the read and write tasks are still running.
1599    /// There may be some delay between the connection closing and the
1600    /// client detecting it.
1601    #[inline]
1602    #[must_use]
1603    pub fn is_alive(&self) -> bool {
1604        match &self.read_task {
1605            Some(read_task) => !read_task.is_finished() && !self.write_task.is_finished(),
1606            None => !self.write_task.is_finished(),
1607        }
1608    }
1609
1610    #[expect(
1611        clippy::too_many_arguments,
1612        reason = "both handler modes share the same reader lifecycle"
1613    )]
1614    fn spawn_message_handler_task(
1615        connection_state: Arc<AtomicU8>,
1616        state_notify: Arc<tokio::sync::Notify>,
1617        read_fence: ReadSessionFence,
1618        mut reader: MessageReader,
1619        connection_epoch: u64,
1620        handler: Option<&IncomingHandler>,
1621        ping_handler: Option<&IncomingPingHandler>,
1622        close_handler: Option<&CloseHandler>,
1623        state_sink: Option<&SocketStateSink>,
1624        idle_timeout_ms: Option<u64>,
1625        heartbeat_timeout: Option<Duration>,
1626        frame_activity: Option<Arc<AtomicBool>>,
1627    ) -> dst::task::JoinHandle<()> {
1628        log::debug!("Started message handler task 'read'");
1629
1630        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1631        let idle_timeout = idle_timeout_ms.map(Duration::from_millis);
1632
1633        let handler = handler.cloned();
1634        let ping_handler = ping_handler.cloned();
1635        let close_handler = close_handler.cloned();
1636        let state_sink = state_sink.cloned();
1637
1638        dst::task::spawn(async move {
1639            let mut last_data_time = dst::time::Instant::now();
1640            let mut last_frame_time = dst::time::Instant::now();
1641
1642            loop {
1643                if !ConnectionMode::from_atomic(&connection_state).is_active()
1644                    || !read_fence.is_valid()
1645                {
1646                    break;
1647                }
1648
1649                let read_result = dst::time::timeout(check_interval, reader.next()).await;
1650
1651                if let Ok(Some(Ok(ref message))) = read_result
1652                    && (!ConnectionMode::from_atomic(&connection_state).is_active()
1653                        || !read_fence.is_valid())
1654                {
1655                    log::debug!(
1656                        "Dropping WebSocket message with {} bytes after session ended",
1657                        message.as_bytes().len()
1658                    );
1659                    break;
1660                }
1661
1662                if matches!(&read_result, Ok(Some(Ok(_)))) {
1663                    last_frame_time = dst::time::Instant::now();
1664                }
1665
1666                match read_result {
1667                    Ok(Some(Ok(Message::Binary(data)))) => {
1668                        log::trace!("Received message <binary> {} bytes", data.len());
1669                        last_data_time = dst::time::Instant::now();
1670
1671                        if !ConnectionMode::from_atomic(&connection_state).is_active()
1672                            || !read_fence.is_valid()
1673                        {
1674                            log::debug!(
1675                                "Dropping WebSocket message with {} bytes after session ended",
1676                                data.len()
1677                            );
1678                            break;
1679                        }
1680
1681                        if let Some(ref handler) = handler {
1682                            handler.handle(connection_epoch, Message::Binary(data));
1683                        }
1684                    }
1685                    Ok(Some(Ok(Message::Text(data)))) => {
1686                        log::trace!("Received text frame ({} bytes)", data.len());
1687                        last_data_time = dst::time::Instant::now();
1688
1689                        if !ConnectionMode::from_atomic(&connection_state).is_active()
1690                            || !read_fence.is_valid()
1691                        {
1692                            log::debug!(
1693                                "Dropping WebSocket message with {} bytes after session ended",
1694                                data.len()
1695                            );
1696                            break;
1697                        }
1698
1699                        if let Some(ref handler) = handler {
1700                            handler.handle(connection_epoch, Message::Text(data));
1701                        }
1702                    }
1703                    Ok(Some(Ok(Message::Ping(ping_data)))) => {
1704                        log::trace!("Received ping frame ({} bytes)", ping_data.len());
1705                        // Do not reset last_data_time: pings are keep-alive frames, not application
1706                        // data, so a peer that emits only pings must still trip the idle timeout.
1707                        // Checked here too: a ping flood faster than the check interval starves the timeout branch
1708
1709                        if let Some(ref handler) = ping_handler {
1710                            if !ConnectionMode::from_atomic(&connection_state).is_active()
1711                                || !read_fence.is_valid()
1712                            {
1713                                log::debug!(
1714                                    "Dropping WebSocket ping with {} bytes after session ended",
1715                                    ping_data.len()
1716                                );
1717                                break;
1718                            }
1719
1720                            handler.handle(connection_epoch, ping_data.to_vec());
1721                        }
1722
1723                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1724                            break;
1725                        }
1726                    }
1727                    Ok(Some(Ok(Message::Pong(_)))) => {
1728                        log::trace!("Received pong");
1729                        // Do not reset last_data_time: pongs are keep-alive replies (not data)
1730
1731                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1732                            break;
1733                        }
1734                    }
1735                    Ok(Some(Ok(Message::Close(frame)))) => {
1736                        Self::dispatch_close_frame(
1737                            &connection_state,
1738                            &read_fence,
1739                            close_handler.as_ref(),
1740                            state_sink.as_ref(),
1741                            frame,
1742                        );
1743                        break;
1744                    }
1745                    Ok(Some(Err(e))) => {
1746                        if is_connection_drop_transport_error(&e) {
1747                            log::warn!("Received connection error, terminating: {e}");
1748                        } else {
1749                            log::error!("Received transport error, terminating: {e}");
1750                        }
1751
1752                        break;
1753                    }
1754                    Ok(None) => {
1755                        log::log!(
1756                            read_termination_log_level(&connection_state),
1757                            "Connection closed by peer (no close frame), terminating"
1758                        );
1759                        break;
1760                    }
1761                    Err(_) => {
1762                        if heartbeat_timeout_exceeded(
1763                            &mut last_frame_time,
1764                            heartbeat_timeout,
1765                            frame_activity.as_deref(),
1766                        ) {
1767                            break;
1768                        }
1769
1770                        if idle_timeout_exceeded(last_data_time, idle_timeout) {
1771                            break;
1772                        }
1773                    }
1774                }
1775            }
1776
1777            // Wake the controller immediately so it detects the dead read task
1778            state_notify.notify_one();
1779        })
1780    }
1781
1782    fn dispatch_close_frame(
1783        connection_state: &AtomicU8,
1784        read_fence: &ReadSessionFence,
1785        close_handler: Option<&CloseHandler>,
1786        state_sink: Option<&SocketStateSink>,
1787        frame: Option<CloseFrame>,
1788    ) {
1789        match &frame {
1790            Some(frame) => log::log!(
1791                read_termination_log_level(connection_state),
1792                "Received close frame, terminating: code={}, reason={REDACTED}",
1793                frame.code
1794            ),
1795            None => log::log!(
1796                read_termination_log_level(connection_state),
1797                "Received close frame with no code or reason, terminating"
1798            ),
1799        }
1800
1801        if !ConnectionMode::from_atomic(connection_state).is_active() || !read_fence.is_valid() {
1802            return;
1803        }
1804
1805        if let Some(handler) = close_handler
1806            && !handler(frame)
1807        {
1808            ConnectionMode::close_websocket_on_loss(connection_state, state_sink);
1809        }
1810    }
1811
1812    /// Queues a message for replay on the replacement connection.
1813    ///
1814    /// A keepalive belongs to the connection it was issued on: a Pong answers that
1815    /// connection's Ping, a heartbeat probes it, and a Close terminates it. None
1816    /// carries its meaning on a replacement connection, so those messages are dropped
1817    /// instead. Text keepalives reach the writer as [`WriterCommand::Heartbeat`] and
1818    /// never enter this buffer.
1819    fn buffer_for_replay(
1820        buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1821        msg: Message,
1822        permit: tokio::sync::OwnedSemaphorePermit,
1823    ) {
1824        if msg.is_control() {
1825            return;
1826        }
1827
1828        log::debug!(
1829            "Buffering message for replay (buffer size: {})",
1830            buffer.len() + 1
1831        );
1832
1833        buffer.push_back((msg, permit));
1834    }
1835
1836    /// Attempts to send all buffered messages after reconnection.
1837    ///
1838    /// Returns `true` if a send error occurred (caller should trigger reconnection).
1839    /// Messages remain in buffer if send fails, preserving them for the next reconnection attempt.
1840    async fn drain_reconnect_buffer(
1841        buffer: &mut VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)>,
1842        writer: &mut MessageWriter,
1843        connection_state: &AtomicU8,
1844        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1845        reconnect_buffer_waits_for_auth: &AtomicBool,
1846    ) -> bool {
1847        if buffer.is_empty() {
1848            return false;
1849        }
1850
1851        let initial_buffer_len = buffer.len();
1852        log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
1853
1854        while !buffer.is_empty() {
1855            match Self::reconnect_buffer_action(
1856                reconnect_buffer_waits_for_auth,
1857                auth_tracker,
1858                connection_state,
1859            ) {
1860                ReconnectBufferAction::Drain => {}
1861                ReconnectBufferAction::Wait => return false,
1862                ReconnectBufferAction::Discard => {
1863                    log::warn!(
1864                        "Discarding {} buffered messages after authentication failed",
1865                        buffer.len()
1866                    );
1867                    buffer.clear();
1868                    return false;
1869                }
1870            }
1871
1872            // Clone message before attempting send (to keep in buffer if send fails)
1873            let msg_to_send = buffer
1874                .front()
1875                .expect("reconnect buffer should not be empty")
1876                .0
1877                .clone();
1878
1879            if let Err(e) = writer.send(msg_to_send).await {
1880                if is_connection_drop_transport_error(&e) {
1881                    log::warn!(
1882                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1883                        buffer.len()
1884                    );
1885                } else {
1886                    log::error!(
1887                        "Failed to send buffered message after reconnection: {e}, {} messages remain in buffer",
1888                        buffer.len()
1889                    );
1890                }
1891
1892                return true;
1893            }
1894
1895            // Only remove from buffer after successful send
1896            buffer.pop_front();
1897        }
1898
1899        if buffer.is_empty() {
1900            log::info!("Successfully sent all {initial_buffer_len} buffered messages");
1901        }
1902
1903        false
1904    }
1905
1906    fn can_drain_reconnect_buffer(
1907        reconnect_buffer_waits_for_auth: &AtomicBool,
1908        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1909    ) -> ReconnectBufferAction {
1910        if !reconnect_buffer_waits_for_auth.load(Ordering::Acquire) {
1911            return ReconnectBufferAction::Drain;
1912        }
1913
1914        match auth_tracker.get().map(AuthTracker::auth_state) {
1915            Some(AuthState::Authenticated) => ReconnectBufferAction::Drain,
1916            Some(AuthState::Failed) => ReconnectBufferAction::Discard,
1917            Some(AuthState::Unauthenticated) | None => ReconnectBufferAction::Wait,
1918        }
1919    }
1920
1921    fn reconnect_buffer_action(
1922        reconnect_buffer_waits_for_auth: &AtomicBool,
1923        auth_tracker: &Arc<OnceLock<AuthTracker>>,
1924        connection_state: &AtomicU8,
1925    ) -> ReconnectBufferAction {
1926        let action =
1927            Self::can_drain_reconnect_buffer(reconnect_buffer_waits_for_auth, auth_tracker);
1928
1929        // Keep mode as the final admission check so an accepted reconnect stops the next send.
1930        if action == ReconnectBufferAction::Drain
1931            && !ConnectionMode::from_atomic(connection_state).is_active()
1932        {
1933            ReconnectBufferAction::Wait
1934        } else {
1935            action
1936        }
1937    }
1938
1939    #[expect(
1940        clippy::too_many_arguments,
1941        reason = "writer task owns the transport and shared lifecycle coordination state"
1942    )]
1943    fn spawn_write_task(
1944        connection_state: Arc<AtomicU8>,
1945        controller_notify: Arc<tokio::sync::Notify>,
1946        reconnect_published: Arc<AtomicBool>,
1947        writer: MessageWriter,
1948        mut writer_rx: WriterReceiver<WriterCommand>,
1949        connection_epoch: Arc<AtomicU64>,
1950        auth_tracker: Arc<OnceLock<AuthTracker>>,
1951        reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
1952        state_sink: Option<SocketStateSink>,
1953    ) -> dst::task::JoinHandle<()> {
1954        log_task_started("write");
1955
1956        // Interval between checking the connection mode
1957        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
1958
1959        dst::task::spawn(async move {
1960            let mut active_writer = writer;
1961            // Buffer for messages received during reconnection
1962            // VecDeque for efficient pop_front() operations
1963            let mut reconnect_buffer: VecDeque<(Message, tokio::sync::OwnedSemaphorePermit)> =
1964                VecDeque::new();
1965
1966            loop {
1967                let mode = ConnectionMode::from_atomic(&connection_state);
1968
1969                match mode {
1970                    ConnectionMode::Disconnect => {
1971                        // Log any buffered messages that will be lost
1972                        if !reconnect_buffer.is_empty() {
1973                            log::warn!(
1974                                "Discarding {} buffered messages due to disconnect",
1975                                reconnect_buffer.len()
1976                            );
1977                            reconnect_buffer.clear();
1978                        }
1979
1980                        // Attempt to close the writer gracefully before exiting,
1981                        // we ignore any error as the writer may already be closed.
1982                        _ = dst::time::timeout(
1983                            Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
1984                            active_writer.close(),
1985                        )
1986                        .await;
1987                        break;
1988                    }
1989                    ConnectionMode::Closed => {
1990                        // Log any buffered messages that will be lost
1991                        if !reconnect_buffer.is_empty() {
1992                            log::warn!(
1993                                "Discarding {} buffered messages due to closed connection",
1994                                reconnect_buffer.len()
1995                            );
1996                            reconnect_buffer.clear();
1997                        }
1998
1999                        break;
2000                    }
2001                    _ => {}
2002                }
2003
2004                if mode.is_active() && !reconnect_buffer.is_empty() {
2005                    match Self::reconnect_buffer_action(
2006                        reconnect_buffer_waits_for_auth.as_ref(),
2007                        &auth_tracker,
2008                        &connection_state,
2009                    ) {
2010                        ReconnectBufferAction::Drain => {
2011                            let drain_result = dst::time::timeout(
2012                                Duration::from_secs(WRITE_TIMEOUT_SECS),
2013                                Self::drain_reconnect_buffer(
2014                                    &mut reconnect_buffer,
2015                                    &mut active_writer,
2016                                    &connection_state,
2017                                    &auth_tracker,
2018                                    reconnect_buffer_waits_for_auth.as_ref(),
2019                                ),
2020                            )
2021                            .await;
2022
2023                            let send_error = drain_result.unwrap_or_else(|_| {
2024                                log::warn!(
2025                                    "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
2026                                    reconnect_buffer.len()
2027                                );
2028                                true
2029                            });
2030
2031                            // CAS: a disconnect landing mid-drain must not be overwritten
2032                            if send_error {
2033                                _ = request_websocket_reconnect(
2034                                    &connection_state,
2035                                    &reconnect_published,
2036                                    state_sink.as_ref(),
2037                                    &auth_tracker,
2038                                    &controller_notify,
2039                                    || {},
2040                                );
2041                            }
2042
2043                            continue;
2044                        }
2045                        ReconnectBufferAction::Discard => {
2046                            log::warn!(
2047                                "Discarding {} buffered messages after authentication failed",
2048                                reconnect_buffer.len()
2049                            );
2050                            reconnect_buffer.clear();
2051                            continue;
2052                        }
2053                        ReconnectBufferAction::Wait => {}
2054                    }
2055                }
2056
2057                match dst::time::timeout(check_interval, writer_rx.recv()).await {
2058                    Ok(Some((msg, permit))) => {
2059                        // Re-check connection mode after receiving a message
2060                        let mode = ConnectionMode::from_atomic(&connection_state);
2061
2062                        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
2063                            break;
2064                        }
2065
2066                        match msg {
2067                            WriterCommand::Update(new_writer, tx) => {
2068                                log::debug!("Received new writer");
2069
2070                                // Delay before closing connection
2071                                dst::time::sleep(Duration::from_millis(100)).await;
2072
2073                                // Attempt to close the writer gracefully on update,
2074                                // we ignore any error as the writer may already be closed.
2075                                _ = dst::time::timeout(
2076                                    Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2077                                    active_writer.close(),
2078                                )
2079                                .await;
2080
2081                                active_writer = new_writer;
2082                                let epoch = connection_epoch.fetch_add(1, Ordering::AcqRel) + 1;
2083                                log::debug!("Updated writer: epoch={epoch}");
2084
2085                                if let Err(e) = tx.send(epoch) {
2086                                    log::error!(
2087                                        "Failed to report writer update to controller: {e:?}"
2088                                    );
2089                                }
2090                            }
2091                            WriterCommand::Send(msg)
2092                                if mode.is_reconnect()
2093                                    || (!reconnect_buffer.is_empty() && !msg.is_control()) =>
2094                            {
2095                                Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2096                            }
2097                            WriterCommand::Heartbeat(_)
2098                            | WriterCommand::SendPongOnConnection { .. }
2099                                if mode.is_reconnect() => {}
2100                            WriterCommand::SendOnConnection { response_tx, .. }
2101                                if mode.is_reconnect() =>
2102                            {
2103                                _ = response_tx.send(Err(SendError::ConnectionChanged));
2104                            }
2105                            WriterCommand::SendPongOnConnection {
2106                                data,
2107                                connection_epoch: expected_epoch,
2108                            } => {
2109                                let epoch = connection_epoch.load(Ordering::Acquire);
2110                                if epoch != expected_epoch {
2111                                    continue;
2112                                }
2113
2114                                let send_result = dst::time::timeout(
2115                                    Duration::from_secs(WRITE_TIMEOUT_SECS),
2116                                    active_writer.send(Message::Pong(data.into())),
2117                                )
2118                                .await;
2119
2120                                let send_failed = match send_result {
2121                                    Ok(Ok(())) => false,
2122                                    Ok(Err(e)) => {
2123                                        if is_connection_drop_transport_error(&e) {
2124                                            log::warn!("Failed to send pong: {e}");
2125                                        } else {
2126                                            log::error!("Failed to send pong: {e}");
2127                                        }
2128
2129                                        true
2130                                    }
2131                                    Err(_) => {
2132                                        log::warn!(
2133                                            "Timed out sending pong after {WRITE_TIMEOUT_SECS}s"
2134                                        );
2135                                        true
2136                                    }
2137                                };
2138
2139                                if send_failed
2140                                    && request_websocket_reconnect(
2141                                        &connection_state,
2142                                        &reconnect_published,
2143                                        state_sink.as_ref(),
2144                                        &auth_tracker,
2145                                        &controller_notify,
2146                                        || {},
2147                                    ) == ReconnectRequestOutcome::Accepted
2148                                {
2149                                    log::warn!("Writer triggering reconnect");
2150                                }
2151                            }
2152                            WriterCommand::SendOnConnection {
2153                                message,
2154                                connection_epoch: expected_epoch,
2155                                response_tx,
2156                            } => {
2157                                let epoch = connection_epoch.load(Ordering::Acquire);
2158                                if epoch != expected_epoch {
2159                                    _ = response_tx.send(Err(SendError::ConnectionChanged));
2160                                    continue;
2161                                }
2162
2163                                let send_result = dst::time::timeout(
2164                                    Duration::from_secs(WRITE_TIMEOUT_SECS),
2165                                    active_writer.send(message),
2166                                )
2167                                .await;
2168
2169                                // An ownership-bound message is never replayed, so an expired
2170                                // deadline reports failure to the caller instead of buffering
2171                                // the message as the ordinary send path does.
2172                                let result = match send_result {
2173                                    Ok(Ok(())) => Ok(()),
2174                                    Ok(Err(e)) => {
2175                                        if is_connection_drop_transport_error(&e) {
2176                                            log::warn!("Failed to send message: {e}");
2177                                        } else {
2178                                            log::error!("Failed to send message: {e}");
2179                                        }
2180
2181                                        Err(SendError::BrokenPipe(e.to_string()))
2182                                    }
2183                                    Err(_) => {
2184                                        log::warn!(
2185                                            "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
2186                                        );
2187
2188                                        Err(SendError::WriteTimeout)
2189                                    }
2190                                };
2191
2192                                let send_failed = result.is_err();
2193                                _ = response_tx.send(result);
2194
2195                                if send_failed
2196                                    && request_websocket_reconnect(
2197                                        &connection_state,
2198                                        &reconnect_published,
2199                                        state_sink.as_ref(),
2200                                        &auth_tracker,
2201                                        &controller_notify,
2202                                        || {},
2203                                    ) == ReconnectRequestOutcome::Accepted
2204                                {
2205                                    log::warn!("Writer triggering reconnect");
2206                                }
2207                            }
2208                            WriterCommand::Send(msg) => {
2209                                let send_failed =
2210                                    Self::write_outbound(&mut active_writer, msg.clone()).await;
2211                                if send_failed {
2212                                    Self::buffer_for_replay(&mut reconnect_buffer, msg, permit);
2213
2214                                    // CAS: a disconnect landing mid-send must not be overwritten
2215                                    if request_websocket_reconnect(
2216                                        &connection_state,
2217                                        &reconnect_published,
2218                                        state_sink.as_ref(),
2219                                        &auth_tracker,
2220                                        &controller_notify,
2221                                        || {},
2222                                    ) == ReconnectRequestOutcome::Accepted
2223                                    {
2224                                        log::warn!("Writer triggering reconnect");
2225                                    }
2226                                }
2227                            }
2228                            WriterCommand::Heartbeat(msg) => {
2229                                let send_failed =
2230                                    Self::write_outbound(&mut active_writer, msg).await;
2231
2232                                if send_failed
2233                                    && request_websocket_reconnect(
2234                                        &connection_state,
2235                                        &reconnect_published,
2236                                        state_sink.as_ref(),
2237                                        &auth_tracker,
2238                                        &controller_notify,
2239                                        || {},
2240                                    ) == ReconnectRequestOutcome::Accepted
2241                                {
2242                                    log::warn!("Writer triggering reconnect");
2243                                }
2244                            }
2245                        }
2246                    }
2247                    Ok(None) => {
2248                        // Channel closed - writer task should terminate
2249                        log::debug!("Writer channel closed, terminating writer task");
2250                        break;
2251                    }
2252                    Err(_) => {
2253                        // Timeout - just continue the loop
2254                    }
2255                }
2256            }
2257
2258            // Attempt to close the writer gracefully before exiting,
2259            // we ignore any error as the writer may already be closed.
2260            _ = dst::time::timeout(
2261                Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
2262                active_writer.close(),
2263            )
2264            .await;
2265
2266            log_task_stopped("write");
2267        })
2268    }
2269
2270    async fn write_outbound(writer: &mut MessageWriter, msg: Message) -> bool {
2271        let send_result =
2272            dst::time::timeout(Duration::from_secs(WRITE_TIMEOUT_SECS), writer.send(msg)).await;
2273
2274        match send_result {
2275            Ok(Ok(())) => false,
2276            Ok(Err(e)) => {
2277                if is_connection_drop_transport_error(&e) {
2278                    log::warn!("Failed to send message: {e}");
2279                } else {
2280                    log::error!("Failed to send message: {e}");
2281                }
2282
2283                true
2284            }
2285            Err(_) => {
2286                log::warn!("Timed out sending message after {WRITE_TIMEOUT_SECS}s");
2287                true
2288            }
2289        }
2290    }
2291
2292    fn spawn_heartbeat_task(
2293        connection_state: Arc<AtomicU8>,
2294        heartbeat_secs: u64,
2295        message: Option<String>,
2296        writer_tx: WriterSender<WriterCommand>,
2297    ) -> dst::task::JoinHandle<()> {
2298        log_task_started("heartbeat");
2299
2300        dst::task::spawn(async move {
2301            let interval = Duration::from_secs(heartbeat_secs);
2302            loop {
2303                dst::time::sleep(interval).await;
2304
2305                match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
2306                    ConnectionMode::Active => {
2307                        let msg = match &message {
2308                            Some(text) => {
2309                                WriterCommand::Heartbeat(Message::Text(text.clone().into()))
2310                            }
2311                            None => WriterCommand::Heartbeat(Message::Ping(vec![].into())),
2312                        };
2313
2314                        match writer_tx.send_control(msg) {
2315                            Ok(()) => log::trace!("Sent heartbeat to writer task"),
2316                            Err(e) => {
2317                                log::warn!("Failed to send heartbeat to writer task: {e}");
2318                            }
2319                        }
2320                    }
2321                    ConnectionMode::Reconnect => {}
2322                    ConnectionMode::Disconnect | ConnectionMode::Closed => break,
2323                }
2324            }
2325
2326            log_task_stopped("heartbeat");
2327        })
2328    }
2329}
2330
2331fn heartbeat_timeout_exceeded(
2332    last_frame_time: &mut dst::time::Instant,
2333    timeout: Option<Duration>,
2334    frame_activity: Option<&AtomicBool>,
2335) -> bool {
2336    if frame_activity.is_some_and(|activity| activity.swap(false, Ordering::Relaxed)) {
2337        *last_frame_time = dst::time::Instant::now();
2338    }
2339
2340    if let Some(timeout) = timeout {
2341        let elapsed = last_frame_time.elapsed();
2342        if elapsed >= timeout {
2343            log::warn!(
2344                "Heartbeat timeout: no frame received for {:.1}s",
2345                elapsed.as_secs_f64()
2346            );
2347            return true;
2348        }
2349    }
2350
2351    false
2352}
2353
2354fn idle_timeout_exceeded(
2355    last_data_time: dst::time::Instant,
2356    idle_timeout: Option<Duration>,
2357) -> bool {
2358    if let Some(timeout) = idle_timeout {
2359        let idle_duration = last_data_time.elapsed();
2360        if idle_duration >= timeout {
2361            log::warn!(
2362                "Read idle timeout: no data received for {:.1}s",
2363                idle_duration.as_secs_f64()
2364            );
2365            return true;
2366        }
2367    }
2368
2369    false
2370}
2371
2372impl Drop for WebSocketClientInner {
2373    fn drop(&mut self) {
2374        if let Some(read_fence) = self.read_fence.take() {
2375            read_fence.invalidate();
2376        }
2377
2378        if let Some(ref read_task) = self.read_task.take()
2379            && !read_task.is_finished()
2380        {
2381            read_task.abort();
2382            log_task_aborted("read");
2383        }
2384
2385        if !self.write_task.is_finished() {
2386            self.write_task.abort();
2387            log_task_aborted("write");
2388        }
2389
2390        if let Some(ref handle) = self.heartbeat_task.take()
2391            && !handle.is_finished()
2392        {
2393            handle.abort();
2394            log_task_aborted("heartbeat");
2395        }
2396    }
2397}
2398
2399#[expect(
2400    clippy::missing_fields_in_debug,
2401    reason = "handler closures and internal task handles are intentionally omitted"
2402)]
2403impl Debug for WebSocketClientInner {
2404    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2405        f.debug_struct(stringify!(WebSocketClientInner))
2406            .field("config", &self.config)
2407            .field(
2408                "connection_mode",
2409                &ConnectionMode::from_atomic(&self.connection_mode),
2410            )
2411            .field("connect_timeout", &self.connect_timeout)
2412            .field("is_stream_mode", &self.handler.is_none())
2413            .finish()
2414    }
2415}
2416
2417#[derive(Clone)]
2418enum IncomingHandler {
2419    Message(MessageHandler),
2420    Epoch(EpochMessageHandler),
2421}
2422
2423impl IncomingHandler {
2424    fn handle(&self, connection_epoch: u64, message: Message) {
2425        match self {
2426            Self::Message(handler) => handler(message),
2427            Self::Epoch(handler) => handler(connection_epoch, message),
2428        }
2429    }
2430}
2431
2432#[derive(Clone)]
2433enum IncomingPingHandler {
2434    Ping(PingHandler),
2435    Epoch(EpochPingHandler),
2436}
2437
2438impl IncomingPingHandler {
2439    fn handle(&self, connection_epoch: u64, data: Vec<u8>) {
2440        match self {
2441            Self::Ping(handler) => handler(data),
2442            Self::Epoch(handler) => handler(connection_epoch, data),
2443        }
2444    }
2445}
2446
2447#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2448enum ReconnectBufferAction {
2449    Drain,
2450    Wait,
2451    Discard,
2452}
2453
2454/// A WebSocket client with rate limiting, heartbeats, and automatic reconnection in handler mode.
2455///
2456/// Handler mode owns the reader and writer tasks, buffers sends during reconnection, and replays
2457/// them against the replacement connection. Stream mode returns the reader to the caller and does
2458/// not reconnect automatically. See [`crate::websocket`] for connection ownership, replay, and
2459/// epoch guarantees.
2460pub struct WebSocketClient {
2461    pub(crate) controller_task: dst::task::JoinHandle<()>,
2462    pub(crate) connection_mode: Arc<AtomicU8>,
2463    pub(crate) connection_epoch: Arc<AtomicU64>,
2464    pub(crate) state_notify: Arc<tokio::sync::Notify>,
2465    pub(crate) connect_timeout: Duration,
2466    pub(crate) rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
2467    pub(crate) writer_tx: WriterSender<WriterCommand>,
2468    auth_tracker: Arc<OnceLock<AuthTracker>>,
2469    reconnect_buffer_waits_for_auth: Arc<AtomicBool>,
2470    reconnect_headers: ReconnectHeaders,
2471    state_sink: Option<SocketStateSink>,
2472    controller_lifecycle: Arc<ControllerLifecycle>,
2473    controller_notify: Arc<tokio::sync::Notify>,
2474    reconnect_published: Arc<AtomicBool>,
2475    reconnect_supported: bool,
2476}
2477
2478/// Shared headers used by future automatic WebSocket reconnects.
2479///
2480/// Updating these headers does not affect the active connection or trigger a reconnect.
2481#[derive(Clone)]
2482pub struct ReconnectHeaders {
2483    inner: Arc<RwLock<Vec<(String, String)>>>,
2484}
2485
2486impl ReconnectHeaders {
2487    fn new(headers: Vec<(String, String)>) -> Self {
2488        Self {
2489            inner: Arc::new(RwLock::new(headers)),
2490        }
2491    }
2492
2493    /// Replaces a header used by future automatic reconnect attempts.
2494    ///
2495    /// # Errors
2496    ///
2497    /// Returns an error if the header name or value is invalid.
2498    pub fn update(&self, name: &str, value: &str) -> Result<(), TransportError> {
2499        let name = HeaderName::from_bytes(name.as_bytes()).map_err(|e| {
2500            TransportError::Io(std::io::Error::new(
2501                std::io::ErrorKind::InvalidInput,
2502                format!("Invalid WebSocket reconnect header name: {e}"),
2503            ))
2504        })?;
2505
2506        HeaderValue::from_str(value).map_err(|e| {
2507            TransportError::Io(std::io::Error::new(
2508                std::io::ErrorKind::InvalidInput,
2509                format!("Invalid WebSocket reconnect header value: {e}"),
2510            ))
2511        })?;
2512
2513        let name = name.as_str();
2514        let mut headers = self.inner.write();
2515        headers.retain(|(existing, _)| !existing.eq_ignore_ascii_case(name));
2516        headers.push((name.to_string(), value.to_string()));
2517        Ok(())
2518    }
2519
2520    fn snapshot(&self) -> Vec<(String, String)> {
2521        self.inner.read().clone()
2522    }
2523}
2524
2525impl Debug for ReconnectHeaders {
2526    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2527        f.debug_struct(stringify!(ReconnectHeaders))
2528            .finish_non_exhaustive()
2529    }
2530}
2531
2532/// Cloneable controller handle for requesting one transport reconnect.
2533#[derive(Clone)]
2534pub struct WebSocketReconnectHandle {
2535    connection_mode: Arc<AtomicU8>,
2536    auth_tracker: Arc<OnceLock<AuthTracker>>,
2537    state_sink: Option<SocketStateSink>,
2538    controller_lifecycle: Arc<ControllerLifecycle>,
2539    controller_notify: Arc<tokio::sync::Notify>,
2540    reconnect_published: Arc<AtomicBool>,
2541    supported: bool,
2542}
2543
2544impl Debug for WebSocketReconnectHandle {
2545    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2546        f.debug_struct(stringify!(WebSocketReconnectHandle))
2547            .field(
2548                "connection_mode",
2549                &ConnectionMode::from_atomic(&self.connection_mode),
2550            )
2551            .field("supported", &self.supported)
2552            .finish_non_exhaustive()
2553    }
2554}
2555
2556impl WebSocketReconnectHandle {
2557    /// Requests that the controller replace the active transport.
2558    ///
2559    /// An accepted request invalidates registered authentication state, wakes the controller, and
2560    /// publishes the loss before the replacement can become active. Rejected requests leave the
2561    /// transport and authentication state unchanged. Handler-mode handles return
2562    /// [`ReconnectRequestOutcome::Closed`] after the client drops; stream-mode handles always
2563    /// return [`ReconnectRequestOutcome::Unsupported`].
2564    #[must_use]
2565    pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
2566        if !self.supported {
2567            return ReconnectRequestOutcome::Unsupported;
2568        }
2569
2570        let Some(request) = self.controller_lifecycle.enter_request() else {
2571            return ReconnectRequestOutcome::Closed;
2572        };
2573
2574        let mut request = Some(request);
2575        request_websocket_reconnect(
2576            &self.connection_mode,
2577            &self.reconnect_published,
2578            self.state_sink.as_ref(),
2579            &self.auth_tracker,
2580            &self.controller_notify,
2581            || drop(request.take()),
2582        )
2583    }
2584}
2585
2586struct ReconnectPublication<'a> {
2587    published: &'a AtomicBool,
2588    controller_notify: &'a tokio::sync::Notify,
2589}
2590
2591impl Drop for ReconnectPublication<'_> {
2592    fn drop(&mut self) {
2593        self.published.store(true, Ordering::SeqCst);
2594        self.controller_notify.notify_one();
2595    }
2596}
2597
2598fn request_websocket_reconnect<F>(
2599    connection_mode: &AtomicU8,
2600    reconnect_published: &AtomicBool,
2601    state_sink: Option<&SocketStateSink>,
2602    auth_tracker: &OnceLock<AuthTracker>,
2603    controller_notify: &tokio::sync::Notify,
2604    on_handoff: F,
2605) -> ReconnectRequestOutcome
2606where
2607    F: FnOnce(),
2608{
2609    if reconnect_published
2610        .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst)
2611        .is_err()
2612    {
2613        return match ConnectionMode::from_atomic(connection_mode) {
2614            ConnectionMode::Active | ConnectionMode::Reconnect => {
2615                ReconnectRequestOutcome::AlreadyReconnecting
2616            }
2617            ConnectionMode::Disconnect => ReconnectRequestOutcome::Disconnected,
2618            ConnectionMode::Closed => ReconnectRequestOutcome::Closed,
2619        };
2620    }
2621
2622    let outcome = ConnectionMode::request_reconnect_outcome(connection_mode);
2623    if outcome != ReconnectRequestOutcome::Accepted {
2624        reconnect_published.store(true, Ordering::SeqCst);
2625        return outcome;
2626    }
2627
2628    let _publication = ReconnectPublication {
2629        published: reconnect_published,
2630        controller_notify,
2631    };
2632
2633    if let Some(tracker) = auth_tracker.get() {
2634        tracker.invalidate();
2635    }
2636
2637    controller_notify.notify_one();
2638    on_handoff();
2639
2640    if let Some(sink) = state_sink {
2641        sink.publish_websocket(SocketState::Disconnected);
2642    }
2643
2644    ReconnectRequestOutcome::Accepted
2645}
2646
2647#[cfg(test)]
2648mod reconnect_request_tests {
2649    use std::sync::{Arc, OnceLock, atomic::AtomicU8};
2650
2651    use rstest::rstest;
2652
2653    use super::*;
2654
2655    fn handle(
2656        mode: ConnectionMode,
2657        supported: bool,
2658    ) -> (
2659        WebSocketReconnectHandle,
2660        AuthTracker,
2661        Arc<tokio::sync::Notify>,
2662    ) {
2663        let tracker = AuthTracker::new();
2664        let _receiver = tracker.begin();
2665        tracker.succeed();
2666        let auth_tracker = Arc::new(OnceLock::new());
2667        auth_tracker
2668            .set(tracker.clone())
2669            .expect("auth tracker should be unset");
2670        let notify = Arc::new(tokio::sync::Notify::new());
2671
2672        let handle = WebSocketReconnectHandle {
2673            connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
2674            auth_tracker,
2675            state_sink: None,
2676            controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2677            controller_notify: Arc::clone(&notify),
2678            reconnect_published: Arc::new(AtomicBool::new(true)),
2679            supported,
2680        };
2681
2682        (handle, tracker, notify)
2683    }
2684
2685    #[rstest]
2686    #[tokio::test]
2687    async fn accepted_request_invalidates_auth_and_wakes_controller_once() {
2688        let (handle, tracker, notify) = handle(ConnectionMode::Active, true);
2689
2690        assert_eq!(
2691            handle.request_reconnect(),
2692            ReconnectRequestOutcome::Accepted
2693        );
2694        assert!(!tracker.is_authenticated());
2695        tokio::time::timeout(Duration::from_millis(10), notify.notified())
2696            .await
2697            .expect("accepted request should notify controller");
2698
2699        let _receiver = tracker.begin();
2700        tracker.succeed();
2701        assert_eq!(
2702            handle.request_reconnect(),
2703            ReconnectRequestOutcome::AlreadyReconnecting
2704        );
2705        assert!(tracker.is_authenticated());
2706        assert!(
2707            tokio::time::timeout(Duration::from_millis(10), notify.notified())
2708                .await
2709                .is_err(),
2710            "duplicate request should not notify controller",
2711        );
2712
2713        handle
2714            .connection_mode
2715            .store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
2716        assert_eq!(
2717            handle.request_reconnect(),
2718            ReconnectRequestOutcome::Accepted
2719        );
2720        assert!(!tracker.is_authenticated());
2721        tokio::time::timeout(Duration::from_millis(10), notify.notified())
2722            .await
2723            .expect("a later reconnect cycle should notify the controller");
2724    }
2725
2726    #[rstest]
2727    #[case(
2728        ConnectionMode::Disconnect,
2729        true,
2730        ReconnectRequestOutcome::Disconnected
2731    )]
2732    #[case(
2733        ConnectionMode::Reconnect,
2734        true,
2735        ReconnectRequestOutcome::AlreadyReconnecting
2736    )]
2737    #[case(ConnectionMode::Closed, true, ReconnectRequestOutcome::Closed)]
2738    #[case(ConnectionMode::Active, false, ReconnectRequestOutcome::Unsupported)]
2739    #[tokio::test]
2740    async fn rejected_request_preserves_auth_and_does_not_wake_controller(
2741        #[case] mode: ConnectionMode,
2742        #[case] supported: bool,
2743        #[case] expected: ReconnectRequestOutcome,
2744    ) {
2745        let (handle, tracker, notify) = handle(mode, supported);
2746
2747        assert_eq!(handle.request_reconnect(), expected);
2748        assert!(tracker.is_authenticated());
2749        assert!(
2750            tokio::time::timeout(Duration::from_millis(10), notify.notified())
2751                .await
2752                .is_err(),
2753            "rejected request should not notify controller",
2754        );
2755    }
2756
2757    #[rstest]
2758    #[tokio::test]
2759    async fn reconnect_loss_callback_rejects_nested_request() {
2760        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2761        let controller_notify = Arc::new(tokio::sync::Notify::new());
2762        let handle_slot = Arc::new(OnceLock::<WebSocketReconnectHandle>::new());
2763        let handle_slot_callback = Arc::clone(&handle_slot);
2764        let nested_outcomes = Arc::new(parking_lot::Mutex::new(Vec::new()));
2765        let nested_outcomes_callback = Arc::clone(&nested_outcomes);
2766        let states = Arc::new(parking_lot::Mutex::new(Vec::new()));
2767        let states_callback = Arc::clone(&states);
2768        let sink = SocketStateSink::new(move |state| {
2769            states_callback.lock().push(state);
2770            nested_outcomes_callback
2771                .lock()
2772                .push(handle_slot_callback.get().unwrap().request_reconnect());
2773        });
2774
2775        let handle = WebSocketReconnectHandle {
2776            connection_mode,
2777            auth_tracker: Arc::new(OnceLock::new()),
2778            state_sink: Some(sink),
2779            controller_lifecycle: Arc::new(ControllerLifecycle::new()),
2780            controller_notify,
2781            reconnect_published: Arc::new(AtomicBool::new(true)),
2782            supported: true,
2783        };
2784
2785        handle_slot.set(handle.clone()).unwrap();
2786        let (result_tx, result_rx) = tokio::sync::oneshot::channel();
2787        std::thread::spawn(move || {
2788            _ = result_tx.send(handle.request_reconnect());
2789        });
2790
2791        assert_eq!(
2792            tokio::time::timeout(Duration::from_secs(1), result_rx)
2793                .await
2794                .expect("reentrant reconnect callback deadlocked")
2795                .unwrap(),
2796            ReconnectRequestOutcome::Accepted
2797        );
2798        assert_eq!(
2799            *nested_outcomes.lock(),
2800            vec![ReconnectRequestOutcome::AlreadyReconnecting]
2801        );
2802        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2803    }
2804
2805    #[rstest]
2806    fn closed_stream_handle_remains_unsupported() {
2807        let (handle, tracker, _notify) = handle(ConnectionMode::Closed, false);
2808        handle.controller_lifecycle.close_and_abort();
2809
2810        assert_eq!(
2811            handle.request_reconnect(),
2812            ReconnectRequestOutcome::Unsupported
2813        );
2814        assert!(tracker.is_authenticated());
2815    }
2816}
2817
2818impl Debug for WebSocketClient {
2819    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2820        f.debug_struct(stringify!(WebSocketClient)).finish()
2821    }
2822}
2823
2824#[bon::bon]
2825impl WebSocketClient {
2826    /// Returns a builder for a websocket client in **stream mode**.
2827    ///
2828    /// Calling `connect` returns a stream that the caller owns and reads from directly. Automatic
2829    /// reconnection is **disabled** because the reader cannot be replaced internally. On
2830    /// disconnection, the client transitions to CLOSED state and the caller must manually create a
2831    /// new connection.
2832    ///
2833    /// Use stream mode when you need custom reconnection logic, direct control over message
2834    /// reading, or fine-grained backpressure handling.
2835    ///
2836    /// `default_quota` and `keyed_quotas` limit outgoing messages. `state_sink` reports transport
2837    /// availability changes.
2838    ///
2839    /// See [`WebSocketConfig`] documentation for comparison with handler mode.
2840    ///
2841    /// # Errors
2842    ///
2843    /// Returns an error if the connection cannot be established.
2844    #[builder(
2845        builder_type = WebSocketClientStreamBuilder,
2846        finish_fn = connect
2847    )]
2848    pub async fn stream_builder(
2849        config: WebSocketConfig,
2850        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2851        default_quota: Option<Quota>,
2852        state_sink: Option<SocketStateSink>,
2853    ) -> Result<(MessageReader, Self), TransportError> {
2854        install_cryptographic_provider();
2855        validate_honored_inbound_limits(&config)?;
2856
2857        // Create a single connection and split it, respecting configured headers.
2858        // The connection attempt bound is a fixed default: stream mode documents reconnect_* fields as ignored
2859        let connect_timeout = Duration::from_secs(10);
2860
2861        let (writer, reader) = dst::time::timeout(
2862            connect_timeout,
2863            Box::pin(WebSocketClientInner::connect_with_server(
2864                &config.url,
2865                config.headers.clone(),
2866                config.backend,
2867                config.proxy_url.as_deref(),
2868                config.max_message_size_bytes,
2869                config.max_frame_size_bytes,
2870            )),
2871        )
2872        .await
2873        .map_err(|_| {
2874            TransportError::Io(std::io::Error::new(
2875                std::io::ErrorKind::TimedOut,
2876                format!(
2877                    "connection timed out after {}s",
2878                    connect_timeout.as_secs_f64()
2879                ),
2880            ))
2881        })??;
2882
2883        // Create inner without connecting (we'll provide the writer)
2884        let inner =
2885            WebSocketClientInner::new_with_writer_and_state_sink(config, writer, state_sink)?;
2886
2887        let connection_mode = Arc::clone(&inner.connection_mode);
2888        let connection_epoch = Arc::clone(&inner.connection_epoch);
2889        let state_notify = Arc::clone(&inner.state_notify);
2890        let controller_notify = Arc::clone(&inner.controller_notify);
2891        let reconnect_published = Arc::clone(&inner.reconnect_published);
2892        let connect_timeout = inner.connect_timeout;
2893        let auth_tracker = Arc::clone(&inner.auth_tracker);
2894        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
2895        let reconnect_headers = inner.reconnect_headers.clone();
2896        let state_sink = inner.state_sink.clone();
2897        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, None)?;
2898        let writer_tx = inner.writer_tx.clone();
2899        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2900
2901        let controller_task = Self::spawn_controller_task(
2902            inner,
2903            Arc::clone(&connection_mode),
2904            Arc::clone(&state_notify),
2905            Arc::clone(&auth_tracker),
2906            Arc::clone(&controller_lifecycle),
2907            Arc::clone(&controller_notify),
2908            Arc::clone(&reconnect_published),
2909        );
2910        let abort_handle = controller_task.abort_handle();
2911        controller_lifecycle.set_abort(move || abort_handle.abort());
2912
2913        Ok((
2914            reader,
2915            Self {
2916                controller_task,
2917                connection_mode,
2918                connection_epoch,
2919                state_notify,
2920                connect_timeout,
2921                rate_limiter,
2922                writer_tx,
2923                auth_tracker,
2924                reconnect_buffer_waits_for_auth,
2925                reconnect_headers,
2926                state_sink,
2927                controller_lifecycle,
2928                controller_notify,
2929                reconnect_published,
2930                reconnect_supported: false,
2931            },
2932        ))
2933    }
2934
2935    /// Returns a builder for a websocket client in **handler mode**.
2936    ///
2937    /// The handler is called for each incoming message on an internal task.
2938    /// Automatic reconnection is **enabled** with exponential backoff. On disconnection,
2939    /// the client automatically attempts to reconnect and replaces the internal reader
2940    /// (the handler continues working seamlessly).
2941    ///
2942    /// Use handler mode for simplified connection management, automatic reconnection, or
2943    /// callback-based message handling.
2944    ///
2945    /// See [`WebSocketConfig`] documentation for comparison with stream mode.
2946    ///
2947    /// Set `rate_limiter` to share message quota state across clients. Otherwise, the client
2948    /// creates one from `default_quota` and `keyed_quotas`. `connection_rate_limiter` gates the
2949    /// initial connection and reconnects using `connection_rate_keys`.
2950    ///
2951    /// Without `initial_connect_retry_policy` the builder makes exactly one connection attempt.
2952    /// With one, failures classified as retryable are retried up to its `max_attempts`; see
2953    /// [`InitialConnectRetryPolicy`] for which failures return before that bound is reached.
2954    ///
2955    /// `cancellation_token` aborts the initial connection only, and is observed during the
2956    /// connection rate-limit wait, the dial itself, and each backoff delay. It has no effect once
2957    /// this function returns a client: use [`WebSocketClient::disconnect`] to stop an established one, whose
2958    /// reconnect loop the token does not govern.
2959    ///
2960    /// Set `close_handler` to observe peer close frames and decide whether automatic reconnection
2961    /// continues. Close frames are not delivered to the message handler.
2962    ///
2963    /// The message handler is required:
2964    ///
2965    /// ```compile_fail
2966    /// use nautilus_network::websocket::{WebSocketClient, WebSocketConfig};
2967    ///
2968    /// let config: WebSocketConfig = unimplemented!();
2969    /// let _ = WebSocketClient::builder().config(config).connect();
2970    /// ```
2971    ///
2972    /// # Errors
2973    ///
2974    /// Returns an error if:
2975    /// - The configuration is invalid or the connection cannot be established.
2976    /// - A shared rate limiter is combined with quota configuration.
2977    /// - The connection rate limiter and its keys are not configured together.
2978    #[builder(finish_fn = connect)]
2979    pub async fn builder(
2980        config: WebSocketConfig,
2981        message_handler: MessageHandler,
2982        ping_handler: Option<PingHandler>,
2983        close_handler: Option<CloseHandler>,
2984        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
2985        default_quota: Option<Quota>,
2986        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
2987        state_sink: Option<SocketStateSink>,
2988        connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
2989        #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
2990        initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
2991        cancellation_token: Option<CancellationToken>,
2992    ) -> Result<Self, TransportError> {
2993        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
2994        let connection_rate_limit =
2995            Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
2996        Self::connect_with_handler_scoped(
2997            config,
2998            IncomingHandler::Message(message_handler),
2999            ping_handler.map(IncomingPingHandler::Ping),
3000            close_handler,
3001            rate_limiter,
3002            state_sink,
3003            connection_rate_limit,
3004            InitialConnectOptions {
3005                retry_policy: initial_connect_retry_policy,
3006                cancellation_token,
3007            },
3008        )
3009        .await
3010    }
3011
3012    /// Returns a builder for a handler-mode client whose messages carry connection ownership.
3013    ///
3014    /// The initial connection has epoch `0`. Each replacement connection increments the epoch,
3015    /// and both its incoming messages and `RECONNECTED` notification carry that new value. Use
3016    /// [`Self::send_text_on_connection`] or [`Self::send_bytes_on_connection`] to bind an
3017    /// outgoing message to one of those epochs.
3018    /// Rate-limit, state, initial-connect retry, and cancellation options match [`Self::builder`].
3019    /// Set either `ping_handler` or `epoch_ping_handler` when custom ping handling is required.
3020    /// `close_handler` has the same behavior as [`Self::builder`].
3021    ///
3022    /// The epoch handler is required:
3023    ///
3024    /// ```compile_fail
3025    /// use nautilus_network::websocket::{WebSocketClient, WebSocketConfig};
3026    ///
3027    /// let config: WebSocketConfig = unimplemented!();
3028    /// let _ = WebSocketClient::epoch_builder().config(config).connect();
3029    /// ```
3030    ///
3031    /// # Errors
3032    ///
3033    /// Returns an error if:
3034    /// - The configuration is invalid or the connection cannot be established.
3035    /// - A shared rate limiter is combined with quota configuration.
3036    /// - The connection rate limiter and its keys are not configured together.
3037    /// - Both `ping_handler` and `epoch_ping_handler` are configured.
3038    #[builder(
3039        builder_type = WebSocketClientEpochBuilder,
3040        finish_fn = connect
3041    )]
3042    pub async fn epoch_builder(
3043        config: WebSocketConfig,
3044        epoch_handler: EpochMessageHandler,
3045        ping_handler: Option<PingHandler>,
3046        epoch_ping_handler: Option<EpochPingHandler>,
3047        close_handler: Option<CloseHandler>,
3048        #[builder(default)] keyed_quotas: Vec<(String, Quota)>,
3049        default_quota: Option<Quota>,
3050        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3051        state_sink: Option<SocketStateSink>,
3052        connection_rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3053        #[builder(default)] connection_rate_keys: Arc<[Ustr]>,
3054        initial_connect_retry_policy: Option<InitialConnectRetryPolicy>,
3055        cancellation_token: Option<CancellationToken>,
3056    ) -> Result<Self, TransportError> {
3057        let ping_handler = match (ping_handler, epoch_ping_handler) {
3058            (Some(_), Some(_)) => {
3059                return Err(TransportError::Io(std::io::Error::new(
3060                    std::io::ErrorKind::InvalidInput,
3061                    "Cannot configure both ping_handler and epoch_ping_handler",
3062                )));
3063            }
3064            (Some(handler), None) => Some(IncomingPingHandler::Ping(handler)),
3065            (None, Some(handler)) => Some(IncomingPingHandler::Epoch(handler)),
3066            (None, None) => None,
3067        };
3068
3069        let rate_limiter = Self::resolve_rate_limiter(default_quota, keyed_quotas, rate_limiter)?;
3070        let connection_rate_limit =
3071            Self::resolve_connection_rate_limit(connection_rate_limiter, connection_rate_keys)?;
3072        Self::connect_with_handler_scoped(
3073            config,
3074            IncomingHandler::Epoch(epoch_handler),
3075            ping_handler,
3076            close_handler,
3077            rate_limiter,
3078            state_sink,
3079            connection_rate_limit,
3080            InitialConnectOptions {
3081                retry_policy: initial_connect_retry_policy,
3082                cancellation_token,
3083            },
3084        )
3085        .await
3086    }
3087
3088    fn resolve_connection_rate_limit(
3089        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3090        keys: Arc<[Ustr]>,
3091    ) -> Result<Option<ConnectionRateLimit>, TransportError> {
3092        if rate_limiter.is_none() && !keys.is_empty() {
3093            return Err(TransportError::Io(std::io::Error::new(
3094                std::io::ErrorKind::InvalidInput,
3095                "Connection rate keys require a connection rate limiter",
3096            )));
3097        }
3098
3099        if rate_limiter.is_some() && keys.is_empty() {
3100            return Err(TransportError::Io(std::io::Error::new(
3101                std::io::ErrorKind::InvalidInput,
3102                "Connection rate limiter requires at least one connection rate key",
3103            )));
3104        }
3105
3106        Ok(rate_limiter.map(|limiter| ConnectionRateLimit { limiter, keys }))
3107    }
3108
3109    fn resolve_rate_limiter(
3110        default_quota: Option<Quota>,
3111        keyed_quotas: Vec<(String, Quota)>,
3112        rate_limiter: Option<Arc<RateLimiter<Ustr, MonotonicClock>>>,
3113    ) -> Result<Arc<RateLimiter<Ustr, MonotonicClock>>, TransportError> {
3114        if let Some(rate_limiter) = rate_limiter {
3115            if default_quota.is_some() || !keyed_quotas.is_empty() {
3116                return Err(TransportError::Io(std::io::Error::new(
3117                    std::io::ErrorKind::InvalidInput,
3118                    "Cannot combine a shared rate limiter with quota configuration",
3119                )));
3120            }
3121
3122            return Ok(rate_limiter);
3123        }
3124
3125        let keyed_quotas = keyed_quotas
3126            .into_iter()
3127            .map(|(key, quota)| (Ustr::from(&key), quota))
3128            .collect();
3129        Ok(Arc::new(RateLimiter::new_with_quota(
3130            default_quota,
3131            keyed_quotas,
3132        )))
3133    }
3134
3135    #[expect(
3136        clippy::too_many_arguments,
3137        reason = "handler delivery and connection controls are independent options"
3138    )]
3139    async fn connect_with_handler_scoped(
3140        config: WebSocketConfig,
3141        handler: IncomingHandler,
3142        ping_handler: Option<IncomingPingHandler>,
3143        close_handler: Option<CloseHandler>,
3144        rate_limiter: Arc<RateLimiter<Ustr, MonotonicClock>>,
3145        state_sink: Option<SocketStateSink>,
3146        connection_rate_limit: Option<ConnectionRateLimit>,
3147        initial_connect_options: InitialConnectOptions,
3148    ) -> Result<Self, TransportError> {
3149        log::debug!("Connecting");
3150        let inner = WebSocketClientInner::connect_url_with_handler(
3151            config,
3152            Some(handler),
3153            ping_handler,
3154            close_handler,
3155            state_sink,
3156            connection_rate_limit,
3157            initial_connect_options,
3158        )
3159        .await?;
3160        let connection_mode = Arc::clone(&inner.connection_mode);
3161        let connection_epoch = Arc::clone(&inner.connection_epoch);
3162        let state_notify = Arc::clone(&inner.state_notify);
3163        let controller_notify = Arc::clone(&inner.controller_notify);
3164        let reconnect_published = Arc::clone(&inner.reconnect_published);
3165        let writer_tx = inner.writer_tx.clone();
3166        let connect_timeout = inner.connect_timeout;
3167        let auth_tracker = Arc::clone(&inner.auth_tracker);
3168        let reconnect_buffer_waits_for_auth = Arc::clone(&inner.reconnect_buffer_waits_for_auth);
3169        let reconnect_headers = inner.reconnect_headers.clone();
3170        let state_sink = inner.state_sink.clone();
3171        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
3172
3173        let controller_task = Self::spawn_controller_task(
3174            inner,
3175            Arc::clone(&connection_mode),
3176            Arc::clone(&state_notify),
3177            Arc::clone(&auth_tracker),
3178            Arc::clone(&controller_lifecycle),
3179            Arc::clone(&controller_notify),
3180            Arc::clone(&reconnect_published),
3181        );
3182        let abort_handle = controller_task.abort_handle();
3183        controller_lifecycle.set_abort(move || abort_handle.abort());
3184
3185        Ok(Self {
3186            controller_task,
3187            connection_mode,
3188            connection_epoch,
3189            state_notify,
3190            connect_timeout,
3191            rate_limiter,
3192            writer_tx,
3193            auth_tracker,
3194            reconnect_buffer_waits_for_auth,
3195            reconnect_headers,
3196            state_sink,
3197            controller_lifecycle,
3198            controller_notify,
3199            reconnect_published,
3200            reconnect_supported: true,
3201        })
3202    }
3203
3204    /// Returns shared headers used by future automatic reconnect attempts.
3205    #[must_use]
3206    pub fn reconnect_headers(&self) -> ReconnectHeaders {
3207        self.reconnect_headers.clone()
3208    }
3209
3210    /// Returns a cloneable handle to this client's reconnect controller.
3211    #[must_use]
3212    pub fn reconnect_handle(&self) -> WebSocketReconnectHandle {
3213        WebSocketReconnectHandle {
3214            connection_mode: Arc::clone(&self.connection_mode),
3215            auth_tracker: Arc::clone(&self.auth_tracker),
3216            state_sink: self.state_sink.clone(),
3217            controller_lifecycle: Arc::clone(&self.controller_lifecycle),
3218            controller_notify: Arc::clone(&self.controller_notify),
3219            reconnect_published: Arc::clone(&self.reconnect_published),
3220            supported: self.reconnect_supported,
3221        }
3222    }
3223
3224    /// Requests that the controller replace the active transport.
3225    ///
3226    /// Returns `true` only when this call transitions a handler-mode client from active to
3227    /// reconnecting. Stream-mode, duplicate, disconnecting, and closed requests return `false`.
3228    #[must_use]
3229    pub fn request_reconnect(&self) -> bool {
3230        self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
3231    }
3232
3233    /// Returns the current connection mode.
3234    #[must_use]
3235    pub fn connection_mode(&self) -> ConnectionMode {
3236        ConnectionMode::from_atomic(&self.connection_mode)
3237    }
3238
3239    /// Returns the ownership epoch of the current connection.
3240    ///
3241    /// A connection keeps one epoch for its lifetime. The value increments when the writer swaps
3242    /// to a replacement connection.
3243    #[must_use]
3244    pub fn connection_epoch(&self) -> u64 {
3245        self.connection_epoch.load(Ordering::Acquire)
3246    }
3247
3248    /// Returns a clone of the connection mode atomic for external state tracking.
3249    ///
3250    /// This allows adapter clients to track connection state across reconnections
3251    /// without message-passing delays.
3252    #[must_use]
3253    pub fn connection_mode_atomic(&self) -> Arc<AtomicU8> {
3254        Arc::clone(&self.connection_mode)
3255    }
3256
3257    /// Returns shared read access to the current connection epoch.
3258    ///
3259    /// Callers must treat the returned atomic as read-only. The WebSocket writer task exclusively
3260    /// advances it when installing a replacement connection.
3261    #[must_use]
3262    pub fn connection_epoch_atomic(&self) -> Arc<AtomicU64> {
3263        Arc::clone(&self.connection_epoch)
3264    }
3265
3266    /// Returns whether the client connection is active.
3267    ///
3268    /// Returns `true` if the client is connected and has not been signaled to disconnect.
3269    /// The client will automatically retry connection based on its configuration.
3270    #[inline]
3271    #[must_use]
3272    pub fn is_active(&self) -> bool {
3273        self.connection_mode().is_active()
3274    }
3275
3276    /// Returns whether the controller task has stopped.
3277    #[must_use]
3278    pub fn is_disconnected(&self) -> bool {
3279        self.controller_task.is_finished()
3280    }
3281
3282    /// Returns whether the client is reconnecting.
3283    ///
3284    /// Returns `true` if the client lost connection and is attempting to reestablish it.
3285    /// The client will automatically retry connection based on its configuration.
3286    #[inline]
3287    #[must_use]
3288    pub fn is_reconnecting(&self) -> bool {
3289        self.connection_mode().is_reconnect()
3290    }
3291
3292    /// Registers an [`AuthTracker`] with the client.
3293    ///
3294    /// When the controller detects a dead connection and transitions to
3295    /// `Reconnect`, it calls `invalidate()` on the tracker so that any
3296    /// pending authenticated sends see the state change immediately. Terminal
3297    /// transitions fail the tracker so pending auth waits can terminate.
3298    /// Set `reconnect_buffer_waits_for_auth` for clients that must not replay
3299    /// buffered messages until the next session authenticates.
3300    ///
3301    /// Call this once after construction, before any authenticated sends.
3302    pub fn set_auth_tracker(&self, tracker: AuthTracker, reconnect_buffer_waits_for_auth: bool) {
3303        let _ = self.auth_tracker.set(tracker);
3304        self.reconnect_buffer_waits_for_auth
3305            .store(reconnect_buffer_waits_for_auth, Ordering::Release);
3306    }
3307
3308    /// Returns whether the client is disconnecting.
3309    ///
3310    /// Returns `true` if the client is in disconnect mode.
3311    #[inline]
3312    #[must_use]
3313    pub fn is_disconnecting(&self) -> bool {
3314        self.connection_mode().is_disconnect()
3315    }
3316
3317    /// Returns whether the client is closed.
3318    ///
3319    /// Returns `true` if the client has been explicitly disconnected or reached
3320    /// maximum reconnection attempts. In this state, the client cannot be reused
3321    /// and a new client must be created for further connections.
3322    #[inline]
3323    #[must_use]
3324    pub fn is_closed(&self) -> bool {
3325        self.connection_mode().is_closed()
3326    }
3327
3328    /// Checks whether the connection is in a terminal state (disconnecting or closed).
3329    ///
3330    /// Single atomic load to fail fast before rate limiting or waiting.
3331    #[inline]
3332    fn check_not_terminal(&self) -> Result<(), SendError> {
3333        match self.connection_mode() {
3334            ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
3335            _ => Ok(()),
3336        }
3337    }
3338
3339    /// Waits for rate limiter quota, aborting early if connection enters a terminal state.
3340    async fn await_rate_limit_or_closed(&self, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3341        const CHECK_INTERVAL_MS: u64 = 100;
3342
3343        tokio::select! {
3344            biased;
3345            () = self.rate_limiter.await_keys_ready(keys) => Ok(()),
3346            () = async {
3347                loop {
3348                    // Enable before the state check: an unpolled Notified is unregistered and misses notifies
3349                    let mut notified = pin!(self.state_notify.notified());
3350                    notified.as_mut().enable();
3351
3352                    if matches!(self.connection_mode(), ConnectionMode::Disconnect | ConnectionMode::Closed) {
3353                        break;
3354                    }
3355                    tokio::select! {
3356                        biased;
3357                        () = notified => {}
3358                        () = dst::time::sleep(Duration::from_millis(CHECK_INTERVAL_MS)) => {}
3359                    }
3360                }
3361            } => Err(SendError::Closed),
3362        }
3363    }
3364
3365    /// Waits for the client to become active before sending.
3366    ///
3367    /// Uses `state_notify` for event-driven wakeup so sends resume immediately
3368    /// after reconnection completes. A fallback interval guards against missed
3369    /// notifications.
3370    async fn wait_for_active(&self) -> Result<(), SendError> {
3371        const FALLBACK_INTERVAL_MS: u64 = 100;
3372
3373        let mode = self.connection_mode();
3374        if mode.is_active() {
3375            return Ok(());
3376        }
3377
3378        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3379            return Err(SendError::Closed);
3380        }
3381
3382        log::debug!("Waiting for client to become ACTIVE before sending...");
3383
3384        let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
3385
3386        dst::time::timeout(self.connect_timeout, async {
3387            loop {
3388                // Enable before the state check: an unpolled Notified is unregistered and misses notifies
3389                let mut notified = pin!(self.state_notify.notified());
3390                notified.as_mut().enable();
3391
3392                let mode = self.connection_mode();
3393                if mode.is_active() {
3394                    return Ok(());
3395                }
3396
3397                if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
3398                    return Err(());
3399                }
3400
3401                tokio::select! {
3402                    biased;
3403                    () = notified => {}
3404                    () = dst::time::sleep(fallback_interval) => {}
3405                }
3406            }
3407        })
3408        .await
3409        .map_err(|_| SendError::Timeout)?
3410        .map_err(|()| SendError::Closed)
3411    }
3412
3413    /// Signals that the caller's reader has observed EOF or a fatal error.
3414    ///
3415    /// In stream mode the controller has no visibility into the caller-owned reader.
3416    /// Call this method when `reader.next().await` returns `None` or an unrecoverable
3417    /// error so the controller transitions to `Closed` and dependent tasks shut down.
3418    ///
3419    /// For peer-initiated close frames (`Message::Close`), use [`disconnect`](Self::disconnect)
3420    /// instead so the writer can send the close reply before shutting down.
3421    ///
3422    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
3423    ///
3424    /// This is a no-op if the connection is already closed or disconnecting.
3425    pub fn notify_closed(&self) {
3426        let mode = self.connection_mode();
3427        if mode.is_disconnect() || mode.is_closed() {
3428            return;
3429        }
3430
3431        log::debug!("Stream reader signalled EOF, transitioning to CLOSED");
3432
3433        if ConnectionMode::close_websocket_on_loss(&self.connection_mode, self.state_sink.as_ref())
3434        {
3435            fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3436            self.state_notify.notify_waiters();
3437        }
3438    }
3439
3440    /// Disconnects the client and waits for the controller task to stop.
3441    ///
3442    /// If an [`AuthTracker`] is registered, this fails pending auth waits.
3443    pub async fn disconnect(&self) {
3444        log::debug!("Disconnecting");
3445
3446        // A CLOSED client keeps its terminal state; its tracker is already failed
3447        if ConnectionMode::request_disconnect(&self.connection_mode)
3448            && let Some(tracker) = self.auth_tracker.get()
3449        {
3450            tracker.fail("WebSocket client disconnected");
3451        }
3452
3453        self.state_notify.notify_waiters();
3454
3455        if dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
3456            while !self.is_disconnected() {
3457                dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
3458            }
3459
3460            if !self.controller_task.is_finished() {
3461                self.controller_task.abort();
3462                log_task_aborted("controller");
3463            }
3464        })
3465        .await
3466            == Ok(())
3467        {
3468            log::debug!("Controller task finished");
3469        } else {
3470            log::warn!("Timeout waiting for controller task to finish");
3471
3472            if !self.controller_task.is_finished() {
3473                self.controller_task.abort();
3474                log_task_aborted("controller");
3475            }
3476
3477            self.connection_mode
3478                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3479        }
3480    }
3481
3482    /// Sends the given text `data` to the server.
3483    ///
3484    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does not
3485    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
3486    /// message. During reconnection, messages are buffered and replayed on the new connection.
3487    ///
3488    /// # Errors
3489    ///
3490    /// Returns an error if the writer capacity is exhausted, or if the client cannot send.
3491    #[allow(unused_variables)]
3492    pub async fn send_text(&self, data: String, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3493        self.check_not_terminal()?;
3494
3495        self.await_rate_limit_or_closed(keys).await?;
3496        self.wait_for_active().await?;
3497
3498        log::trace!("Sending text frame ({} bytes)", data.len());
3499
3500        let msg = Message::Text(data.into());
3501        self.writer_tx.send(WriterCommand::Send(msg))
3502    }
3503
3504    /// Sends text once if the active connection matches `connection_epoch`.
3505    ///
3506    /// The writer rejects the send if reconnection changes ownership before the write. The
3507    /// message is never replayed on another connection.
3508    ///
3509    /// # Errors
3510    ///
3511    /// Returns an error if:
3512    /// - The shared control allowance is full.
3513    /// - The client closes.
3514    /// - The active wait times out before the write starts.
3515    /// - The expected connection no longer owns the writer.
3516    /// - The command or transport write fails.
3517    /// - The write starts but does not complete within the write
3518    ///   deadline. Delivery is undetermined in that case: the message is not replayed, and it
3519    ///   must not be resent blindly.
3520    pub async fn send_text_on_connection(
3521        &self,
3522        data: String,
3523        keys: Option<&[Ustr]>,
3524        connection_epoch: u64,
3525    ) -> Result<(), SendError> {
3526        self.check_not_terminal()?;
3527        self.await_rate_limit_or_closed(keys).await?;
3528        self.wait_for_active().await?;
3529
3530        log::trace!(
3531            "Sending text frame once: epoch={connection_epoch} ({} bytes)",
3532            data.len()
3533        );
3534
3535        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3536        self.writer_tx
3537            .send_control(WriterCommand::SendOnConnection {
3538                message: Message::Text(data.into()),
3539                connection_epoch,
3540                response_tx,
3541            })?;
3542
3543        response_rx
3544            .await
3545            .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3546    }
3547
3548    /// Sends a pong frame back to the server when the connection is active.
3549    ///
3550    /// The pong is skipped silently if the connection is not active when called:
3551    /// a pong belongs to the connection whose ping caused it, so this method does
3552    /// not wait for reconnection before enqueueing it.
3553    ///
3554    /// # Errors
3555    ///
3556    /// Returns an error if:
3557    /// - The payload exceeds 125 bytes, the RFC 6455 control-frame limit.
3558    /// - The writer channel is broken or its control allowance is full.
3559    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3560    #[expect(
3561        clippy::unused_async,
3562        clippy::unused_async_trait_impl,
3563        reason = "skipping instead of waiting removes the only await; the signature is public API shared with the other send methods"
3564    )]
3565    pub async fn send_pong(&self, data: Vec<u8>) -> Result<(), SendError> {
3566        validate_pong_payload(&data)?;
3567
3568        if !self.connection_mode().is_active() {
3569            log::debug!("Skipping pong: connection not active");
3570            return Ok(());
3571        }
3572
3573        log::trace!("Sending pong frame ({} bytes)", data.len());
3574
3575        self.writer_tx
3576            .send_control(WriterCommand::Send(Message::Pong(data.into())))
3577    }
3578
3579    /// Sends a pong if the connection that received its ping is still active.
3580    ///
3581    /// The pong is skipped silently if the connection is inactive or its epoch has changed.
3582    ///
3583    /// # Errors
3584    ///
3585    /// Returns an error if:
3586    /// - The payload exceeds 125 bytes, the RFC 6455 control-frame limit.
3587    /// - The writer channel is broken or its control allowance is full.
3588    #[allow(unknown_lints, reason = "Clippy lint is unavailable on Rust 1.97")]
3589    #[expect(
3590        clippy::unused_async,
3591        clippy::unused_async_trait_impl,
3592        reason = "the public send API is async even though this method only enqueues"
3593    )]
3594    pub async fn send_pong_on_connection(
3595        &self,
3596        data: Vec<u8>,
3597        connection_epoch: u64,
3598    ) -> Result<(), SendError> {
3599        validate_pong_payload(&data)?;
3600
3601        if !self.connection_mode().is_active() {
3602            log::debug!("Skipping pong: connection not active");
3603            return Ok(());
3604        }
3605
3606        log::trace!(
3607            "Sending pong frame once: epoch={connection_epoch} ({} bytes)",
3608            data.len()
3609        );
3610
3611        self.writer_tx
3612            .send_control(WriterCommand::SendPongOnConnection {
3613                data,
3614                connection_epoch,
3615            })
3616    }
3617
3618    /// Sends the given bytes `data` to the server.
3619    ///
3620    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does not
3621    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
3622    /// message. During reconnection, messages are buffered and replayed on the new connection.
3623    ///
3624    /// # Errors
3625    ///
3626    /// Returns an error if the writer capacity is exhausted, or if the client cannot send.
3627    #[allow(unused_variables)]
3628    pub async fn send_bytes(&self, data: Vec<u8>, keys: Option<&[Ustr]>) -> Result<(), SendError> {
3629        self.check_not_terminal()?;
3630
3631        self.await_rate_limit_or_closed(keys).await?;
3632        self.wait_for_active().await?;
3633
3634        log::trace!("Sending binary frame ({} bytes)", data.len());
3635
3636        let msg = Message::Binary(data.into());
3637        self.writer_tx.send(WriterCommand::Send(msg))
3638    }
3639
3640    /// Sends bytes once if the active connection matches `connection_epoch`.
3641    ///
3642    /// The writer rejects the send if reconnection changes ownership before the write. The
3643    /// message is never replayed on another connection.
3644    ///
3645    /// # Errors
3646    ///
3647    /// Returns an error if:
3648    /// - The shared control allowance is full.
3649    /// - The client closes.
3650    /// - The active wait times out before the write starts.
3651    /// - The expected connection no longer owns the writer.
3652    /// - The command or transport write fails.
3653    /// - The write starts but does not complete within the write
3654    ///   deadline. Delivery is undetermined in that case: the message is not replayed, and it
3655    ///   must not be resent blindly.
3656    pub async fn send_bytes_on_connection(
3657        &self,
3658        data: Vec<u8>,
3659        keys: Option<&[Ustr]>,
3660        connection_epoch: u64,
3661    ) -> Result<(), SendError> {
3662        self.check_not_terminal()?;
3663        self.await_rate_limit_or_closed(keys).await?;
3664        self.wait_for_active().await?;
3665
3666        log::trace!(
3667            "Sending binary frame once: epoch={connection_epoch} ({} bytes)",
3668            data.len()
3669        );
3670
3671        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3672        self.writer_tx
3673            .send_control(WriterCommand::SendOnConnection {
3674                message: Message::Binary(data.into()),
3675                connection_epoch,
3676                response_tx,
3677            })?;
3678
3679        response_rx
3680            .await
3681            .map_err(|e| SendError::BrokenPipe(e.to_string()))?
3682    }
3683
3684    /// Sends a close message to the server.
3685    ///
3686    /// # Errors
3687    ///
3688    /// Returns an error if the shared control allowance is full, or if the client cannot send.
3689    pub async fn send_close_message(&self) -> Result<(), SendError> {
3690        self.wait_for_active().await?;
3691
3692        let msg = Message::Close(None);
3693        self.writer_tx.send_control(WriterCommand::Send(msg))
3694    }
3695
3696    fn spawn_controller_task(
3697        mut inner: WebSocketClientInner,
3698        connection_mode: Arc<AtomicU8>,
3699        state_notify: Arc<tokio::sync::Notify>,
3700        auth_tracker: Arc<OnceLock<AuthTracker>>,
3701        controller_lifecycle: Arc<ControllerLifecycle>,
3702        controller_notify: Arc<tokio::sync::Notify>,
3703        reconnect_published: Arc<AtomicBool>,
3704    ) -> dst::task::JoinHandle<()> {
3705        dst::task::spawn(async move {
3706            let _activity = controller_lifecycle.activity();
3707            log_task_started("controller");
3708
3709            let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
3710            let mut reconnected_at = None;
3711
3712            loop {
3713                tokio::select! {
3714                    biased;
3715                    () = controller_notify.notified() => {}
3716                    () = state_notify.notified() => {}
3717                    () = dst::time::sleep(fallback_interval) => {}
3718                }
3719
3720                let mut mode = ConnectionMode::from_atomic(&connection_mode);
3721
3722                if mode.is_disconnect() {
3723                    log::debug!("Disconnecting");
3724
3725                    let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
3726                    if dst::time::timeout(timeout, async {
3727                        // Delay awaiting graceful shutdown
3728                        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
3729
3730                        if let Some(read_fence) = inner.read_fence.take() {
3731                            read_fence.invalidate();
3732                        }
3733
3734                        if let Some(task) = &inner.read_task
3735                            && !task.is_finished()
3736                        {
3737                            task.abort();
3738                            log_task_aborted("read");
3739                        }
3740
3741                        if let Some(task) = &inner.heartbeat_task
3742                            && !task.is_finished()
3743                        {
3744                            task.abort();
3745                            log_task_aborted("heartbeat");
3746                        }
3747                    })
3748                    .await
3749                    .is_err()
3750                    {
3751                        log::warn!("Shutdown timed out after {}s", timeout.as_secs());
3752                    }
3753
3754                    log::debug!("Closed");
3755                    break; // Controller finished
3756                }
3757
3758                if mode.is_closed() {
3759                    log::debug!("Connection closed");
3760                    fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3761                    state_notify.notify_waiters();
3762                    break;
3763                }
3764
3765                if mode.is_active() && !inner.is_alive() {
3766                    let target = if inner.handler.is_none() {
3767                        ConnectionMode::Closed
3768                    } else {
3769                        ConnectionMode::Reconnect
3770                    };
3771
3772                    let transitioned = if target.is_closed() {
3773                        ConnectionMode::close_websocket_on_loss(
3774                            &connection_mode,
3775                            inner.state_sink.as_ref(),
3776                        )
3777                    } else {
3778                        request_websocket_reconnect(
3779                            &connection_mode,
3780                            &reconnect_published,
3781                            inner.state_sink.as_ref(),
3782                            &auth_tracker,
3783                            &controller_notify,
3784                            || {},
3785                        ) == ReconnectRequestOutcome::Accepted
3786                    };
3787
3788                    if transitioned {
3789                        if target.is_closed() {
3790                            fail_registered_auth(auth_tracker.as_ref(), "WebSocket client closed");
3791                        }
3792
3793                        log::info!("Detected dead connection, transitioning to {target:?}");
3794                    }
3795
3796                    mode = ConnectionMode::from_atomic(&connection_mode);
3797                }
3798
3799                if mode.is_reconnect() {
3800                    if let Some(tracker) = auth_tracker.get() {
3801                        tracker.invalidate();
3802                    }
3803
3804                    let reconnect_uptime = reconnected_at
3805                        .take()
3806                        .map(|started: dst::time::Instant| started.elapsed());
3807                    let previous_reconnect_stable = reconnect_uptime
3808                        .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
3809
3810                    if previous_reconnect_stable {
3811                        inner.backoff.reset();
3812                        inner.reconnection_attempt_count = 0;
3813                        log::debug!(
3814                            "WebSocket remained active for at least {}s, resetting reconnect cycle",
3815                            RECONNECT_STABILITY_THRESHOLD.as_secs()
3816                        );
3817                    }
3818
3819                    if let Some(max_attempts) = inner.reconnect_max_attempts
3820                        && inner.reconnection_attempt_count >= max_attempts
3821                    {
3822                        log::error!(
3823                            "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
3824                        );
3825                        connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3826                        fail_registered_auth(
3827                            auth_tracker.as_ref(),
3828                            "WebSocket reconnect attempts exhausted",
3829                        );
3830                        state_notify.notify_waiters();
3831                        break;
3832                    }
3833
3834                    let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
3835                    {
3836                        inner.backoff.next_duration()
3837                    } else {
3838                        Duration::ZERO
3839                    };
3840
3841                    let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
3842                    if !duration.is_zero() {
3843                        log::warn!("Backing off for {}s...", duration.as_secs_f64());
3844
3845                        if !wait_reconnect_delay(
3846                            duration,
3847                            connection_mode.as_ref(),
3848                            state_notify.as_ref(),
3849                        )
3850                        .await
3851                        {
3852                            log::debug!("Backoff interrupted by terminal state");
3853                            continue;
3854                        }
3855                    }
3856
3857                    inner.reconnection_attempt_count += 1;
3858                    inner.reconnect_throttle.record_attempt();
3859                    log::debug!(
3860                        "Reconnection attempt {} of {}",
3861                        inner.reconnection_attempt_count,
3862                        inner
3863                            .reconnect_max_attempts
3864                            .map_or_else(|| "unlimited".to_string(), |m| m.to_string())
3865                    );
3866
3867                    // Race reconnect against disconnect notification
3868                    let reconnect_result = tokio::select! {
3869                        biased;
3870                        result = inner.reconnect_with_outcome() => Some(result),
3871                        () = async {
3872                            loop {
3873                                // Enable before the check so a disconnect notify between iterations is not missed
3874                                let mut notified = pin!(state_notify.notified());
3875                                notified.as_mut().enable();
3876
3877                                if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
3878                                    break;
3879                                }
3880                                notified.await;
3881                            }
3882                        } => None,
3883                    };
3884
3885                    match reconnect_result {
3886                        None => {
3887                            log::debug!("Reconnect interrupted by disconnect");
3888                        }
3889                        Some(Ok(ReconnectOutcome::Reconnected)) => {
3890                            reconnected_at = Some(dst::time::Instant::now());
3891
3892                            state_notify.notify_waiters();
3893
3894                            // The outcome records a completed reconnection; emit recovery
3895                            // callbacks only while the replacement is still `Active`, not
3896                            // after a teardown or another drop.
3897                            if ConnectionMode::from_atomic(&connection_mode).is_active() {
3898                                if let Some(ref handler) = inner.handler {
3899                                    let connection_epoch =
3900                                        inner.connection_epoch.load(Ordering::Acquire);
3901                                    let reconnected_msg =
3902                                        Message::Text(RECONNECTED.to_string().into());
3903                                    handler.handle(connection_epoch, reconnected_msg);
3904                                    match handler {
3905                                        IncomingHandler::Message(_) => {
3906                                            log::debug!("Sent reconnected message to handler");
3907                                        }
3908                                        IncomingHandler::Epoch(_) => {
3909                                            log::debug!(
3910                                                "Sent reconnected message to epoch handler: \
3911                                                 epoch={connection_epoch}",
3912                                            );
3913                                        }
3914                                    }
3915                                }
3916
3917                                log::debug!("Reconnected successfully");
3918                            } else {
3919                                log::debug!("Skipping reconnect handlers due to disconnect state");
3920                            }
3921                        }
3922                        Some(Ok(ReconnectOutcome::Aborted)) => {
3923                            log::debug!("Reconnect aborted");
3924                        }
3925                        Some(Err(e)) => {
3926                            let duration = inner.backoff.next_duration();
3927                            log::warn!(
3928                                "Reconnect attempt {} failed: {}",
3929                                inner.reconnection_attempt_count,
3930                                e.redacted_message()
3931                            );
3932
3933                            if !duration.is_zero() {
3934                                log::warn!("Backing off for {}s...", duration.as_secs_f64());
3935
3936                                if !wait_reconnect_delay(
3937                                    duration,
3938                                    connection_mode.as_ref(),
3939                                    state_notify.as_ref(),
3940                                )
3941                                .await
3942                                {
3943                                    log::debug!("Backoff interrupted by terminal state");
3944                                }
3945                            }
3946                        }
3947                    }
3948                }
3949            }
3950
3951            inner
3952                .connection_mode
3953                .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3954
3955            log_task_stopped("controller");
3956        })
3957    }
3958}
3959
3960fn fail_registered_auth(auth_tracker: &OnceLock<AuthTracker>, reason: &str) {
3961    if let Some(tracker) = auth_tracker.get() {
3962        tracker.fail(reason);
3963    }
3964}
3965
3966fn validate_pong_payload(data: &[u8]) -> Result<(), SendError> {
3967    if data.len() > MAX_CONTROL_FRAME_PAYLOAD_BYTES {
3968        return Err(SendError::InvalidInput(format!(
3969            "pong payload exceeds {MAX_CONTROL_FRAME_PAYLOAD_BYTES} bytes"
3970        )));
3971    }
3972
3973    Ok(())
3974}
3975
3976impl Drop for WebSocketClient {
3977    fn drop(&mut self) {
3978        self.connection_mode
3979            .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3980        fail_registered_auth(self.auth_tracker.as_ref(), "WebSocket client closed");
3981        self.state_notify.notify_waiters();
3982        self.controller_notify.notify_waiters();
3983        self.controller_lifecycle.close_and_abort();
3984    }
3985}
3986
3987#[cfg(test)]
3988#[cfg(not(feature = "turmoil"))]
3989#[cfg(not(all(feature = "simulation", madsim)))] // These tests drive real localhost sockets
3990#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
3991mod tests {
3992    use std::{
3993        collections::HashMap,
3994        num::NonZeroU32,
3995        sync::{
3996            Arc,
3997            atomic::{AtomicU8, AtomicUsize, Ordering},
3998        },
3999        time::Duration,
4000    };
4001
4002    use axum::{Router, routing::post};
4003    use futures_util::{SinkExt, StreamExt};
4004    use log::Level;
4005    use nautilus_common::testing::wait_until_async;
4006    use nautilus_core::string::secret::REDACTED;
4007    use parking_lot::Mutex;
4008    use rstest::rstest;
4009    use tokio::{
4010        io::{AsyncReadExt, AsyncWriteExt},
4011        net::TcpListener,
4012        sync::{mpsc, oneshot},
4013        task::{self, JoinHandle},
4014    };
4015    use tokio_tungstenite::{
4016        accept_async, accept_hdr_async,
4017        tungstenite::{
4018            Message as WsMessage,
4019            handshake::server::{self, Callback},
4020            http::HeaderValue,
4021            protocol::frame::{CloseFrame as WsCloseFrame, coding::CloseCode},
4022        },
4023    };
4024
4025    use crate::{
4026        RECONNECTED, SocketState, SocketStateSink,
4027        error::SendError,
4028        http::{HttpClient, Method},
4029        logging::tests::capture_logs_for,
4030        mode::{ConnectionMode, ReadSessionFence},
4031        ratelimiter::quota::Quota,
4032        transport::{CloseFrame, Message, TransportError},
4033        websocket::{
4034            AuthTracker, CloseHandler, InitialConnectRetryPolicy, ReconnectHeaders,
4035            TransportBackend, WebSocketClient, WebSocketClientInner, WebSocketConfig,
4036            auth::AuthState,
4037            types::{WriterCommand, channel_epoch_message_handler, channel_message_handler},
4038        },
4039    };
4040
4041    const SECRET_MARKER: &str = "OUTBOUND_SECRET_MARKER";
4042    const PING_TRIGGER: &str = "send-test-ping";
4043    const HOSTILE_CLOSE_TRIGGER: &str = "close-hostile";
4044    const CODED_CLOSE_TRIGGER: &str = "close-coded";
4045    const CODED_CLOSE_CODE: u16 = 4003;
4046    const CODED_CLOSE_REASON: &str = "draining";
4047    const HOSTILE_CLOSE_REASON: &str = "injected\r\n\u{1b}[31mforged\u{1b}[0m";
4048    const NETWORK_LOG_TARGETS: &[&str] = &[
4049        "nautilus_network::http::client",
4050        "nautilus_network::websocket::client",
4051    ];
4052
4053    struct TestServer {
4054        task: JoinHandle<()>,
4055        port: u16,
4056    }
4057
4058    #[derive(Debug, Clone)]
4059    struct TestCallback {
4060        key: String,
4061        value: HeaderValue,
4062    }
4063
4064    impl Callback for TestCallback {
4065        #[expect(clippy::panic_in_result_fn)]
4066        fn on_request(
4067            self,
4068            request: &server::Request,
4069            response: server::Response,
4070        ) -> Result<server::Response, server::ErrorResponse> {
4071            let _ = response;
4072            let value = request.headers().get(&self.key);
4073            assert!(value.is_some());
4074
4075            if let Some(value) = request.headers().get(&self.key) {
4076                assert_eq!(value, self.value);
4077            }
4078
4079            Ok(response)
4080        }
4081    }
4082
4083    impl TestServer {
4084        async fn setup() -> Self {
4085            let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4086            let port = TcpListener::local_addr(&server).unwrap().port();
4087
4088            let header_key = "test".to_string();
4089            let header_value = "test".to_string();
4090
4091            let test_call_back = TestCallback {
4092                key: header_key,
4093                value: HeaderValue::from_str(&header_value).unwrap(),
4094            };
4095
4096            let task = task::spawn(async move {
4097                // Keep accepting connections
4098                loop {
4099                    let (conn, _) = server.accept().await.unwrap();
4100                    let mut websocket = accept_hdr_async(conn, test_call_back.clone())
4101                        .await
4102                        .unwrap();
4103
4104                    task::spawn(async move {
4105                        while let Some(Ok(msg)) = websocket.next().await {
4106                            match msg {
4107                                WsMessage::Text(txt) if txt == "close-now" => {
4108                                    log::debug!("Forcibly closing from server side");
4109                                    // This sends a close frame, then stops reading
4110                                    let _ = websocket.close(None).await;
4111                                    break;
4112                                }
4113                                WsMessage::Text(txt) if txt == HOSTILE_CLOSE_TRIGGER => {
4114                                    // Close reasons are server-controlled: this one carries a
4115                                    // forged line break and ANSI color sequences for the log
4116                                    // sanitization test.
4117                                    let _ = websocket
4118                                        .send(WsMessage::Close(Some(WsCloseFrame {
4119                                            code: CloseCode::Error,
4120                                            reason: HOSTILE_CLOSE_REASON.into(),
4121                                        })))
4122                                        .await;
4123
4124                                    break;
4125                                }
4126                                WsMessage::Text(txt) if txt == CODED_CLOSE_TRIGGER => {
4127                                    let _ = websocket
4128                                        .send(WsMessage::Close(Some(WsCloseFrame {
4129                                            code: CloseCode::from(CODED_CLOSE_CODE),
4130                                            reason: CODED_CLOSE_REASON.into(),
4131                                        })))
4132                                        .await;
4133
4134                                    break;
4135                                }
4136                                WsMessage::Text(txt) if txt == PING_TRIGGER => {
4137                                    let ping = format!("{SECRET_MARKER}:ping");
4138                                    if websocket.send(WsMessage::Ping(ping.into())).await.is_err() {
4139                                        break;
4140                                    }
4141                                }
4142                                // Echo text/binary frames
4143                                WsMessage::Text(_) | WsMessage::Binary(_) => {
4144                                    if websocket.send(msg).await.is_err() {
4145                                        break;
4146                                    }
4147                                }
4148                                // If the client closes, we also break
4149                                WsMessage::Close(_frame) => {
4150                                    let _ = websocket.close(None).await;
4151                                    break;
4152                                }
4153                                // Ignore pings/pongs
4154                                _ => {}
4155                            }
4156                        }
4157                    });
4158                }
4159            });
4160
4161            Self { task, port }
4162        }
4163    }
4164
4165    impl Drop for TestServer {
4166        fn drop(&mut self) {
4167            self.task.abort();
4168        }
4169    }
4170
4171    async fn setup_test_client(port: u16) -> WebSocketClient {
4172        let config = WebSocketConfig {
4173            url: format!("ws://127.0.0.1:{port}"),
4174            headers: vec![("test".into(), "test".into())],
4175            heartbeat_interval_secs: None,
4176            heartbeat_payload: None,
4177            connect_timeout_ms: None,
4178            reconnect_delay_initial_ms: None,
4179            reconnect_backoff_factor: None,
4180            reconnect_delay_max_ms: None,
4181            reconnect_jitter_ms: None,
4182            reconnect_max_attempts: None,
4183            heartbeat_timeout_secs: None,
4184            idle_timeout_ms: None,
4185            writer_capacity: None,
4186            backend: TransportBackend::Tungstenite,
4187            proxy_url: None,
4188            max_message_size_bytes: None,
4189            max_frame_size_bytes: None,
4190        };
4191
4192        WebSocketClient::builder()
4193            .config(config)
4194            .message_handler(Arc::new(|_| {}))
4195            .connect()
4196            .await
4197            .expect("Failed to connect")
4198    }
4199
4200    #[rstest]
4201    #[case::handler(false)]
4202    #[case::stream(true)]
4203    #[tokio::test]
4204    async fn test_writer_capacity_from_config(#[case] stream_mode: bool) {
4205        let server = TestServer::setup().await;
4206        let config = WebSocketConfig::builder()
4207            .url(format!("ws://127.0.0.1:{}", server.port))
4208            .headers(vec![("test".to_string(), "test".to_string())])
4209            .backend(TransportBackend::Tungstenite)
4210            .writer_capacity(2)
4211            .build()
4212            .unwrap();
4213
4214        let (client, _reader) = if stream_mode {
4215            let (reader, client) = WebSocketClient::stream_builder()
4216                .config(config)
4217                .connect()
4218                .await
4219                .unwrap();
4220            (client, Some(reader))
4221        } else {
4222            let client = WebSocketClient::builder()
4223                .config(config)
4224                .message_handler(Arc::new(|_| {}))
4225                .connect()
4226                .await
4227                .unwrap();
4228            (client, None)
4229        };
4230
4231        // Ready sends do not yield on this current-thread runtime, keeping the queue occupied
4232        client.send_text("first".to_string(), None).await.unwrap();
4233        client.send_bytes(vec![2], None).await.unwrap();
4234        let overflow = client.send_text("overflow".to_string(), None).await;
4235
4236        assert_eq!(client.connection_mode(), ConnectionMode::Active);
4237        assert!(matches!(overflow, Err(SendError::BufferFull)));
4238        client.disconnect().await;
4239    }
4240
4241    #[rstest]
4242    #[tokio::test]
4243    async fn test_send_reports_writer_buffer_full() {
4244        let server = TestServer::setup().await;
4245        let mut client = setup_test_client(server.port).await;
4246        let (writer_tx, mut writer_rx) = crate::writer::channel(1);
4247        client.writer_tx = writer_tx;
4248
4249        client.send_text("first".to_string(), None).await.unwrap();
4250        let result = client.send_bytes(vec![2], None).await;
4251        assert!(matches!(result, Err(SendError::BufferFull)));
4252        let (command, permit) = writer_rx.recv().await.unwrap();
4253        assert!(
4254            matches!(command, WriterCommand::Send(message) if message == Message::text("first"))
4255        );
4256        assert!(matches!(
4257            client.send_text("overflow".to_string(), None).await,
4258            Err(SendError::BufferFull)
4259        ));
4260        drop(permit);
4261        client.send_bytes(vec![3], None).await.unwrap();
4262        let (command, _) = writer_rx.recv().await.unwrap();
4263        assert!(
4264            matches!(command, WriterCommand::Send(message) if message == Message::Binary(vec![3].into()))
4265        );
4266        client.disconnect().await;
4267    }
4268
4269    async fn setup_reconnecting_client(port: u16) -> WebSocketClient {
4270        let config = WebSocketConfig {
4271            url: format!("ws://127.0.0.1:{port}"),
4272            headers: vec![],
4273            heartbeat_interval_secs: None,
4274            heartbeat_payload: None,
4275            connect_timeout_ms: Some(5_000),
4276            reconnect_delay_initial_ms: Some(1),
4277            reconnect_backoff_factor: Some(1.0),
4278            reconnect_delay_max_ms: Some(1),
4279            reconnect_jitter_ms: Some(0),
4280            reconnect_max_attempts: None,
4281            heartbeat_timeout_secs: None,
4282            idle_timeout_ms: None,
4283            writer_capacity: None,
4284            backend: TransportBackend::Tungstenite,
4285            proxy_url: None,
4286            max_message_size_bytes: None,
4287            max_frame_size_bytes: None,
4288        };
4289
4290        WebSocketClient::builder()
4291            .config(config)
4292            .message_handler(Arc::new(|_| {}))
4293            .connect()
4294            .await
4295            .expect("client should connect")
4296    }
4297
4298    async fn wait_for_mode(client: &WebSocketClient, expected: ConnectionMode) {
4299        crate::dst::time::timeout(Duration::from_secs(5), async {
4300            loop {
4301                if ConnectionMode::from_atomic(&client.connection_mode) == expected {
4302                    break;
4303                }
4304
4305                crate::dst::time::sleep(Duration::from_millis(1)).await;
4306            }
4307        })
4308        .await
4309        .expect("client should reach expected connection mode");
4310    }
4311
4312    async fn setup_http_test_server() -> (JoinHandle<()>, u16) {
4313        let server = TcpListener::bind("127.0.0.1:0").await.unwrap();
4314        let port = server.local_addr().unwrap().port();
4315
4316        let app = Router::new().route(
4317            "/logging",
4318            post(|| async {
4319                (
4320                    [("x-secret-response", SECRET_MARKER)],
4321                    format!("{SECRET_MARKER}:response"),
4322                )
4323            }),
4324        );
4325
4326        let task = task::spawn(async move {
4327            axum::serve(server, app).await.unwrap();
4328        });
4329
4330        (task, port)
4331    }
4332
4333    #[rstest]
4334    #[tokio::test]
4335    async fn test_network_logs_omit_payload_bodies() {
4336        let server = TestServer::setup().await;
4337        let client = setup_test_client(server.port).await;
4338        let capture = capture_logs_for(NETWORK_LOG_TARGETS).await;
4339        let binary = format!("{SECRET_MARKER}:binary").into_bytes();
4340        let binary_marker = format!("{binary:?}");
4341
4342        client
4343            .send_text(format!("{SECRET_MARKER}:café"), None)
4344            .await
4345            .unwrap();
4346        client
4347            .send_text_on_connection(
4348                format!("{SECRET_MARKER}:owned-é"),
4349                None,
4350                client.connection_epoch(),
4351            )
4352            .await
4353            .unwrap();
4354        client.send_bytes(binary.clone(), None).await.unwrap();
4355        client
4356            .send_bytes_on_connection(binary, None, client.connection_epoch())
4357            .await
4358            .unwrap();
4359        client
4360            .send_text(PING_TRIGGER.to_string(), None)
4361            .await
4362            .unwrap();
4363
4364        tokio::time::timeout(Duration::from_secs(2), async {
4365            loop {
4366                if capture.messages().iter().any(|(level, message)| {
4367                    matches!(level, Level::Trace | Level::Warn)
4368                        && message == "Received ping frame (27 bytes)"
4369                }) {
4370                    break;
4371                }
4372
4373                tokio::task::yield_now().await;
4374            }
4375        })
4376        .await
4377        .expect("timed out waiting for inbound WebSocket metadata log");
4378
4379        let (http_task, http_port) = setup_http_test_server().await;
4380        let invalid_headers =
4381            HashMap::from([("x-secret-default".to_string(), format!("{SECRET_MARKER}\n"))]);
4382        let invalid_header_error = HttpClient::builder()
4383            .headers(invalid_headers)
4384            .build()
4385            .unwrap_err();
4386        let http_client = HttpClient::builder().build().unwrap();
4387        let params = HashMap::from([("secret".to_string(), vec![SECRET_MARKER.to_string()])]);
4388        let headers = HashMap::from([
4389            (
4390                "X-Secret-Request".to_string(),
4391                format!("{SECRET_MARKER}:first"),
4392            ),
4393            (
4394                "x-secret-request".to_string(),
4395                format!("{SECRET_MARKER}:second"),
4396            ),
4397        ]);
4398        let http_body = format!("{SECRET_MARKER}:http-body").into_bytes();
4399        http_client
4400            .request(
4401                Method::POST,
4402                format!("http://127.0.0.1:{http_port}/logging"),
4403                Some(&params),
4404                Some(headers),
4405                Some(http_body),
4406                None,
4407                None,
4408            )
4409            .await
4410            .unwrap();
4411
4412        // The initial-connect retry warning names the endpoint, which can carry credentials or
4413        // signed query data, so it must report the redaction placeholder rather than the URL.
4414        let rejecting = TcpListener::bind("127.0.0.1:0").await.unwrap();
4415        let rejecting_port = rejecting.local_addr().unwrap().port();
4416
4417        let rejecting_server = task::spawn(async move {
4418            loop {
4419                let (mut stream, _) = rejecting.accept().await.unwrap();
4420                let mut request = [0; 1024];
4421                let _ = stream.read(&mut request).await.unwrap();
4422                stream
4423                    .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
4424                    .await
4425                    .unwrap();
4426            }
4427        });
4428
4429        let retry_config = WebSocketConfig {
4430            url: format!("ws://127.0.0.1:{rejecting_port}/?token={SECRET_MARKER}"),
4431            headers: vec![],
4432            heartbeat_interval_secs: None,
4433            heartbeat_payload: None,
4434            connect_timeout_ms: Some(5_000),
4435            reconnect_delay_initial_ms: None,
4436            reconnect_backoff_factor: None,
4437            reconnect_delay_max_ms: None,
4438            reconnect_jitter_ms: None,
4439            reconnect_max_attempts: None,
4440            heartbeat_timeout_secs: None,
4441            idle_timeout_ms: None,
4442            writer_capacity: None,
4443            backend: TransportBackend::Tungstenite,
4444            proxy_url: None,
4445            max_message_size_bytes: None,
4446            max_frame_size_bytes: None,
4447        };
4448
4449        let retry_error = WebSocketClient::builder()
4450            .config(retry_config)
4451            .message_handler(Arc::new(|_| {}))
4452            .initial_connect_retry_policy(InitialConnectRetryPolicy {
4453                max_attempts: NonZeroU32::new(2).unwrap(),
4454                delay_initial: Duration::from_millis(1),
4455                delay_max: Duration::from_millis(1),
4456                backoff_factor: 1.0,
4457                jitter_ms: 0,
4458            })
4459            .connect()
4460            .await
4461            .expect_err("server always rejects the upgrade");
4462
4463        rejecting_server.abort();
4464
4465        assert!(
4466            matches!(retry_error, TransportError::UpgradeRejected(503)),
4467            "expected a 503 upgrade rejection, was: {retry_error:?}"
4468        );
4469
4470        let messages: Vec<_> = capture
4471            .messages()
4472            .into_iter()
4473            .filter(|(level, message)| {
4474                matches!(level, Level::Trace | Level::Warn)
4475                    && (message.starts_with("Sending ")
4476                        || message.starts_with("Received ")
4477                        || message.starts_with("Replaced ")
4478                        || message.starts_with("WebSocket connection attempt "))
4479            })
4480            .map(|(_, message)| message)
4481            .collect();
4482
4483        let invalid_header_message = invalid_header_error.to_string();
4484
4485        // Asserted positively so the blanket secret assertion below cannot pass vacuously by
4486        // capturing no warning at all.
4487        let retry_warning = messages
4488            .iter()
4489            .find(|message| message.starts_with("WebSocket connection attempt 1/2 "))
4490            .expect("initial-connect retry warning was not captured");
4491        assert!(
4492            retry_warning.contains(REDACTED),
4493            "retry warning omitted the redaction placeholder: {retry_warning}"
4494        );
4495
4496        assert!(
4497            messages.iter().all(|message| {
4498                !message.contains(SECRET_MARKER) && !message.contains(&binary_marker)
4499            }),
4500            "network logs exposed the secret marker: {messages:?}"
4501        );
4502        assert!(
4503            !invalid_header_message.contains(SECRET_MARKER),
4504            "invalid header error exposed the secret marker: {invalid_header_message}"
4505        );
4506        assert!(
4507            invalid_header_message.contains("x-secret-default"),
4508            "invalid header error omitted safe header metadata: {invalid_header_message}"
4509        );
4510        assert!(
4511            messages
4512                .iter()
4513                .any(|message| message == "Sending text frame (28 bytes)"),
4514            "text send metadata missing or inaccurate: {messages:?}"
4515        );
4516        assert!(
4517            messages
4518                .iter()
4519                .any(|message| { message == "Sending text frame once: epoch=0 (31 bytes)" }),
4520            "ownership-bound text metadata missing or inaccurate: {messages:?}"
4521        );
4522        assert!(
4523            messages
4524                .iter()
4525                .any(|message| message == "Sending binary frame (29 bytes)"),
4526            "binary send metadata missing or inaccurate: {messages:?}"
4527        );
4528        assert!(
4529            messages
4530                .iter()
4531                .any(|message| message == "Sending binary frame once: epoch=0 (29 bytes)"),
4532            "ownership-bound binary metadata missing or inaccurate: {messages:?}"
4533        );
4534        assert!(
4535            messages
4536                .iter()
4537                .any(|message| message == "Received text frame (28 bytes)"),
4538            "text receive metadata missing or inaccurate: {messages:?}"
4539        );
4540        assert!(
4541            messages
4542                .iter()
4543                .any(|message| message == "Received text frame (31 bytes)"),
4544            "ownership-bound text receive metadata missing or inaccurate: {messages:?}"
4545        );
4546        assert!(
4547            messages
4548                .iter()
4549                .any(|message| message == "Received message <binary> 29 bytes"),
4550            "binary receive metadata missing or inaccurate: {messages:?}"
4551        );
4552        assert!(
4553            messages
4554                .iter()
4555                .any(|message| message == "Received ping frame (27 bytes)"),
4556            "ping receive metadata missing or inaccurate: {messages:?}"
4557        );
4558        // Two caller headers plus the `accept-encoding: gzip` added for buffered requests
4559        assert!(
4560            messages.iter().any(|message| {
4561                message
4562                    == "Sending HTTP request: method=POST extra_headers=3 query_bytes=29 \
4563                        body_bytes=32"
4564            }),
4565            "HTTP request metadata missing or inaccurate: {messages:?}"
4566        );
4567        assert!(
4568            messages
4569                .iter()
4570                .any(|message| message == "Replaced duplicate request header 'x-secret-request'"),
4571            "duplicate header metadata missing: {messages:?}"
4572        );
4573        assert!(
4574            messages.iter().any(|message| {
4575                message.starts_with("Received HTTP response: status=200 OK headers=")
4576                    && message.ends_with(" body_bytes=31")
4577            }),
4578            "HTTP response metadata missing or inaccurate: {messages:?}"
4579        );
4580
4581        client.disconnect().await;
4582        http_task.abort();
4583    }
4584
4585    // A pong belongs to the connection whose ping caused it: while the
4586    // replacement handshake is gated the pong must be dropped rather than
4587    // waited on, and the next pong must ride the new connection.
4588    #[rstest]
4589    #[tokio::test]
4590    async fn test_send_pong_skips_gated_reconnect() {
4591        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4592        let port = listener.local_addr().unwrap().port();
4593        let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
4594        let (handshake_gate_tx, handshake_gate_rx) = oneshot::channel();
4595        let (observed_tx, mut observed_rx) = mpsc::unbounded_channel();
4596
4597        let server_task = task::spawn(async move {
4598            let (first, _) = listener.accept().await.unwrap();
4599            let first_websocket = accept_async(first).await.unwrap();
4600            drop(first_websocket);
4601
4602            let (second, _) = listener.accept().await.unwrap();
4603            second_accepted_tx.send(()).unwrap();
4604            handshake_gate_rx.await.unwrap();
4605            let mut replacement = accept_async(second).await.unwrap();
4606
4607            while let Some(message) = replacement.next().await {
4608                match message.unwrap() {
4609                    WsMessage::Pong(data) => observed_tx.send(data.to_vec()).unwrap(),
4610                    WsMessage::Close(_) => {
4611                        let _ = replacement.close(None).await;
4612                        break;
4613                    }
4614                    _ => {}
4615                }
4616            }
4617        });
4618
4619        let client = setup_reconnecting_client(port).await;
4620
4621        crate::dst::time::timeout(Duration::from_secs(5), second_accepted_rx)
4622            .await
4623            .expect("replacement connection should be accepted")
4624            .unwrap();
4625        wait_for_mode(&client, ConnectionMode::Reconnect).await;
4626
4627        let rejected = client.send_pong(vec![2; 126]).await;
4628        assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4629
4630        // The gate is still shut, so a pong that waited for the connection to
4631        // become active could not complete here.
4632        crate::dst::time::timeout(
4633            Duration::from_secs(1),
4634            client.send_pong(b"stale-pong".to_vec()),
4635        )
4636        .await
4637        .expect("pong raised during reconnect should not wait for the replacement")
4638        .expect("skipped pong should report success");
4639
4640        handshake_gate_tx.send(()).unwrap();
4641        wait_for_mode(&client, ConnectionMode::Active).await;
4642
4643        let fresh_payload = b"fresh-pong".to_vec();
4644        client.send_pong(fresh_payload.clone()).await.unwrap();
4645        assert_eq!(
4646            crate::dst::time::timeout(Duration::from_secs(5), observed_rx.recv())
4647                .await
4648                .expect("replacement connection should receive the fresh pong"),
4649            Some(fresh_payload)
4650        );
4651
4652        client.disconnect().await;
4653        server_task.await.unwrap();
4654        assert_eq!(
4655            observed_rx.try_recv(),
4656            Err(mpsc::error::TryRecvError::Disconnected),
4657            "replacement connection should receive exactly one pong"
4658        );
4659    }
4660
4661    // Validation must precede the inactive skip: an oversized pong during
4662    // reconnect is a caller error, not a silently skipped frame.
4663    #[rstest]
4664    #[tokio::test]
4665    async fn test_pong_validation_precedes_inactive_skip() {
4666        let server = TestServer::setup().await;
4667        let client = setup_test_client(server.port).await;
4668        client
4669            .connection_mode
4670            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4671
4672        let result = client.send_pong(vec![2; 126]).await;
4673        let epoch_result = client.send_pong_on_connection(vec![2; 126], 0).await;
4674
4675        assert!(matches!(result, Err(SendError::InvalidInput(_))));
4676        assert!(matches!(epoch_result, Err(SendError::InvalidInput(_))));
4677    }
4678
4679    // A rejected oversized pong must not disturb the connection: the frame is
4680    // never enqueued, so the next in-range pong still rides the same socket.
4681    #[rstest]
4682    #[tokio::test]
4683    async fn test_pong_payload_limit_preserves_connection() {
4684        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4685        let port = listener.local_addr().unwrap().port();
4686        let (pong_tx, mut pong_rx) = mpsc::unbounded_channel();
4687
4688        let server_task = task::spawn(async move {
4689            let (stream, _) = listener.accept().await.unwrap();
4690            let mut websocket = accept_async(stream).await.unwrap();
4691
4692            while let Some(message) = websocket.next().await {
4693                match message.unwrap() {
4694                    WsMessage::Pong(data) => pong_tx.send(data.to_vec()).unwrap(),
4695                    WsMessage::Close(_) => {
4696                        let _ = websocket.close(None).await;
4697                        break;
4698                    }
4699                    _ => {}
4700                }
4701            }
4702        });
4703
4704        let client = setup_reconnecting_client(port).await;
4705        let accepted = vec![1; 125];
4706        let follow_up = vec![3; 125];
4707
4708        client.send_pong(accepted.clone()).await.unwrap();
4709        assert_eq!(
4710            crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4711                .await
4712                .unwrap(),
4713            Some(accepted)
4714        );
4715
4716        let rejected = client.send_pong(vec![2; 126]).await;
4717        assert!(matches!(rejected, Err(SendError::InvalidInput(_))));
4718
4719        client.send_pong(follow_up.clone()).await.unwrap();
4720        assert_eq!(
4721            crate::dst::time::timeout(Duration::from_secs(5), pong_rx.recv())
4722                .await
4723                .unwrap(),
4724            Some(follow_up)
4725        );
4726
4727        client.disconnect().await;
4728        server_task.await.unwrap();
4729    }
4730
4731    #[tokio::test]
4732    async fn test_websocket_basic() {
4733        let server = TestServer::setup().await;
4734        let client = setup_test_client(server.port).await;
4735
4736        assert!(!client.is_disconnected());
4737
4738        client.disconnect().await;
4739        assert!(client.is_disconnected());
4740    }
4741
4742    #[rstest]
4743    #[tokio::test]
4744    async fn test_drop_sets_shared_connection_mode_closed() {
4745        let server = TestServer::setup().await;
4746        let client = setup_test_client(server.port).await;
4747        let connection_mode = client.connection_mode_atomic();
4748
4749        drop(client);
4750
4751        assert_eq!(
4752            ConnectionMode::from_atomic(&connection_mode),
4753            ConnectionMode::Closed
4754        );
4755    }
4756
4757    #[rstest]
4758    #[tokio::test]
4759    async fn test_notify_closed_closes_reconnecting_client() {
4760        let server = TestServer::setup().await;
4761        let client = setup_test_client(server.port).await;
4762        client
4763            .connection_mode
4764            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
4765
4766        client.notify_closed();
4767
4768        assert!(client.is_closed());
4769    }
4770
4771    #[tokio::test]
4772    async fn test_websocket_heartbeat() {
4773        let server = TestServer::setup().await;
4774        let client = setup_test_client(server.port).await;
4775
4776        // Wait ~3s => server should see multiple "ping"
4777        tokio::time::sleep(std::time::Duration::from_secs(3)).await;
4778
4779        // Cleanup
4780        client.disconnect().await;
4781        assert!(client.is_disconnected());
4782    }
4783
4784    #[rstest]
4785    #[tokio::test]
4786    async fn test_websocket_reconnect_exhausted() {
4787        let config = WebSocketConfig {
4788            url: "ws://127.0.0.1:9997".into(), // <-- No server
4789            headers: vec![],
4790            heartbeat_interval_secs: None,
4791            heartbeat_payload: None,
4792            connect_timeout_ms: None,
4793            reconnect_delay_initial_ms: None,
4794            reconnect_backoff_factor: None,
4795            reconnect_delay_max_ms: None,
4796            reconnect_jitter_ms: None,
4797            reconnect_max_attempts: None,
4798            heartbeat_timeout_secs: None,
4799            idle_timeout_ms: None,
4800            writer_capacity: None,
4801            backend: TransportBackend::Tungstenite,
4802            proxy_url: None,
4803            max_message_size_bytes: None,
4804            max_frame_size_bytes: None,
4805        };
4806
4807        let states = Arc::new(Mutex::new(Vec::new()));
4808        let states_callback = Arc::clone(&states);
4809        let sink = SocketStateSink::new(move |state| {
4810            states_callback.lock().push(state);
4811        });
4812
4813        let res = WebSocketClient::builder()
4814            .config(config)
4815            .message_handler(Arc::new(|_| {}))
4816            .state_sink(sink)
4817            .connect()
4818            .await;
4819        assert!(res.is_err(), "Should fail quickly with no server");
4820        assert_eq!(*states.lock(), Vec::new());
4821    }
4822
4823    #[tokio::test]
4824    async fn test_websocket_forced_close_reconnect() {
4825        let server = TestServer::setup().await;
4826        let client = setup_test_client(server.port).await;
4827
4828        // 1) Send normal message
4829        client.send_text("Hello".into(), None).await.unwrap();
4830
4831        // 2) Trigger forced close from server
4832        client.send_text("close-now".into(), None).await.unwrap();
4833
4834        // 3) Wait a bit => read loop sees close => reconnect
4835        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
4836
4837        // Confirm not disconnected
4838        assert!(!client.is_disconnected());
4839
4840        // Cleanup
4841        client.disconnect().await;
4842        assert!(client.is_disconnected());
4843    }
4844
4845    #[rstest]
4846    #[case::plain(false)]
4847    #[case::epoch(true)]
4848    #[tokio::test]
4849    async fn test_peer_close_preserves_forwarded_messages(#[case] epoch_aware: bool) {
4850        let timeout = Duration::from_secs(5);
4851        let subscription = "subscribe-retained";
4852        let server = TestServer::setup().await;
4853        let config = WebSocketConfig::builder()
4854            .url(format!("ws://127.0.0.1:{}", server.port))
4855            .headers(vec![("test".to_string(), "test".to_string())])
4856            .backend(TransportBackend::Tungstenite)
4857            .reconnect_delay_initial_ms(50)
4858            .reconnect_jitter_ms(0)
4859            .build()
4860            .unwrap();
4861        let (handler, mut raw_rx) = channel_epoch_message_handler();
4862
4863        let ws_client = if epoch_aware {
4864            WebSocketClient::epoch_builder()
4865                .config(config)
4866                .epoch_handler(handler)
4867                .connect()
4868                .await
4869        } else {
4870            WebSocketClient::builder()
4871                .config(config)
4872                .message_handler(Arc::new(move |message| handler(0, message)))
4873                .connect()
4874                .await
4875        }
4876        .unwrap();
4877
4878        let ws_client = Arc::new(ws_client);
4879        let (bytes_tx, mut bytes_rx) = tokio::sync::mpsc::unbounded_channel();
4880        let forwarding_client = Arc::clone(&ws_client);
4881
4882        let forwarding_task = tokio::spawn(async move {
4883            while let Some((epoch, message)) = raw_rx.recv().await {
4884                let data = match message {
4885                    WsMessage::Text(text) => text.as_bytes().to_vec(),
4886                    WsMessage::Binary(data) => data.to_vec(),
4887                    WsMessage::Close(_) => break,
4888                    _ => continue,
4889                };
4890
4891                if data == RECONNECTED.as_bytes() {
4892                    forwarding_client
4893                        .send_text(subscription.into(), None)
4894                        .await
4895                        .unwrap();
4896                }
4897
4898                if bytes_tx.send((epoch, data)).is_err() {
4899                    break;
4900                }
4901            }
4902        });
4903
4904        ws_client
4905            .send_text(subscription.into(), None)
4906            .await
4907            .unwrap();
4908        let initial = tokio::time::timeout(timeout, bytes_rx.recv())
4909            .await
4910            .expect("initial subscription response timed out")
4911            .expect("forwarding task ended before peer close");
4912
4913        ws_client.send_text("close-now".into(), None).await.unwrap();
4914        let reconnected = tokio::time::timeout(timeout, bytes_rx.recv())
4915            .await
4916            .expect("reconnect notification timed out")
4917            .expect("peer close terminated the forwarding task");
4918        let replayed = tokio::time::timeout(timeout, bytes_rx.recv())
4919            .await
4920            .expect("subscription replay response timed out")
4921            .expect("forwarding task ended before subscription replay");
4922        let payload = vec![3, 7, 11];
4923        ws_client.send_bytes(payload.clone(), None).await.unwrap();
4924        let live = tokio::time::timeout(timeout, bytes_rx.recv())
4925            .await
4926            .expect("replacement connection data timed out")
4927            .expect("forwarding task ended before replacement data");
4928
4929        let expected_epoch = u64::from(epoch_aware);
4930        assert_eq!(initial, (0, subscription.as_bytes().to_vec()));
4931        assert_eq!(
4932            reconnected,
4933            (expected_epoch, RECONNECTED.as_bytes().to_vec())
4934        );
4935        assert_eq!(replayed, (expected_epoch, subscription.as_bytes().to_vec()));
4936        assert_eq!(live, (expected_epoch, payload));
4937        assert_eq!(ws_client.connection_epoch(), 1);
4938        assert!(!forwarding_task.is_finished());
4939
4940        ws_client.disconnect().await;
4941        forwarding_task.abort();
4942    }
4943
4944    #[rstest]
4945    #[case(true)]
4946    #[case(false)]
4947    #[tokio::test]
4948    async fn test_peer_close_frame_is_forwarded_to_close_handler(#[case] reconnect: bool) {
4949        let server = TestServer::setup().await;
4950        let (handler, mut handler_rx) = channel_message_handler();
4951        let (state_tx, mut state_rx) = tokio::sync::mpsc::unbounded_channel();
4952
4953        let client = WebSocketClient::builder()
4954            .config(WebSocketConfig {
4955                url: format!("ws://127.0.0.1:{}", server.port),
4956                headers: vec![("test".into(), "test".into())],
4957                heartbeat_interval_secs: None,
4958                heartbeat_payload: None,
4959                connect_timeout_ms: Some(1_000),
4960                reconnect_delay_initial_ms: Some(1_000),
4961                reconnect_backoff_factor: None,
4962                reconnect_delay_max_ms: None,
4963                reconnect_jitter_ms: Some(0),
4964                reconnect_max_attempts: None,
4965                heartbeat_timeout_secs: None,
4966                idle_timeout_ms: None,
4967                writer_capacity: None,
4968                backend: TransportBackend::Tungstenite,
4969                proxy_url: None,
4970                max_message_size_bytes: None,
4971                max_frame_size_bytes: None,
4972            })
4973            .message_handler(Arc::new(|_| {}))
4974            .state_sink(SocketStateSink::new(move |state| {
4975                state_tx.send(state).expect("state channel closed");
4976            }))
4977            .close_handler(Arc::new(move |frame| {
4978                handler(Message::Close(frame));
4979                reconnect
4980            }))
4981            .connect()
4982            .await
4983            .expect("Failed to connect");
4984
4985        let tracker = AuthTracker::new();
4986        let pending_auth = tracker.begin();
4987        client.set_auth_tracker(tracker.clone(), false);
4988
4989        client
4990            .send_text(CODED_CLOSE_TRIGGER.to_string(), None)
4991            .await
4992            .unwrap();
4993
4994        let message = tokio::time::timeout(Duration::from_secs(10), handler_rx.recv())
4995            .await
4996            .expect("close frame was not forwarded")
4997            .expect("handler channel closed");
4998
4999        let WsMessage::Close(Some(frame)) = message else {
5000            panic!("expected a close frame, was {message:?}");
5001        };
5002
5003        let connected = state_rx.recv().await.expect("initial socket state");
5004        let disconnected = tokio::time::timeout(Duration::from_secs(5), state_rx.recv())
5005            .await
5006            .expect("peer close must publish socket loss")
5007            .expect("state channel closed");
5008
5009        assert_eq!(u16::from(frame.code), CODED_CLOSE_CODE);
5010        assert_eq!(frame.reason.as_str(), CODED_CLOSE_REASON);
5011        assert_eq!(connected, SocketState::Connected);
5012        assert_eq!(disconnected, SocketState::Disconnected);
5013
5014        if !reconnect {
5015            let auth_result = tokio::time::timeout(Duration::from_secs(1), pending_auth)
5016                .await
5017                .expect("terminal close should resolve pending authentication")
5018                .expect("authentication sender should report its terminal result");
5019
5020            assert_eq!(auth_result, Err("WebSocket client closed".to_string()));
5021            assert_eq!(tracker.auth_state(), AuthState::Failed);
5022        }
5023
5024        client.disconnect().await;
5025    }
5026
5027    #[rstest]
5028    #[case(true, ConnectionMode::Active)]
5029    #[case(false, ConnectionMode::Closed)]
5030    fn test_peer_close_callback_controls_reconnect(
5031        #[case] reconnect: bool,
5032        #[case] expected_mode: ConnectionMode,
5033    ) {
5034        let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5035        let fence = ReadSessionFence::new();
5036        let received = Arc::new(Mutex::new(Vec::new()));
5037        let states = Arc::new(Mutex::new(Vec::new()));
5038        let sink_states = Arc::clone(&states);
5039        let sink = SocketStateSink::new(move |state| sink_states.lock().push(state));
5040        let callback_received = Arc::clone(&received);
5041        let callback: CloseHandler = Arc::new(move |frame| {
5042            callback_received.lock().push(frame);
5043            reconnect
5044        });
5045
5046        let frame = CloseFrame::new(CODED_CLOSE_CODE, CODED_CLOSE_REASON);
5047
5048        WebSocketClientInner::dispatch_close_frame(
5049            &mode,
5050            &fence,
5051            Some(&callback),
5052            Some(&sink),
5053            Some(frame.clone()),
5054        );
5055
5056        assert_eq!(*received.lock(), vec![Some(frame)]);
5057        assert_eq!(ConnectionMode::from_atomic(&mode), expected_mode);
5058        assert_eq!(
5059            *states.lock(),
5060            if reconnect {
5061                Vec::new()
5062            } else {
5063                vec![SocketState::Disconnected]
5064            },
5065        );
5066    }
5067
5068    #[rstest]
5069    fn test_peer_close_callback_ignores_retired_reader() {
5070        let mode = AtomicU8::new(ConnectionMode::Active.as_u8());
5071        let fence = ReadSessionFence::new();
5072        fence.invalidate();
5073        let calls = Arc::new(AtomicUsize::new(0));
5074        let callback_calls = Arc::clone(&calls);
5075        let callback: CloseHandler = Arc::new(move |_| {
5076            callback_calls.fetch_add(1, Ordering::SeqCst);
5077            false
5078        });
5079
5080        WebSocketClientInner::dispatch_close_frame(&mode, &fence, Some(&callback), None, None);
5081
5082        assert_eq!(calls.load(Ordering::SeqCst), 0);
5083        assert_eq!(ConnectionMode::from_atomic(&mode), ConnectionMode::Active);
5084    }
5085
5086    // Server-controlled reasons can contain credentials and forged log lines
5087    #[rstest]
5088    #[tokio::test]
5089    async fn test_close_frame_reason_is_redacted_in_logs() {
5090        let server = TestServer::setup().await;
5091        let client = setup_test_client(server.port).await;
5092        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5093
5094        client
5095            .send_text(HOSTILE_CLOSE_TRIGGER.to_string(), None)
5096            .await
5097            .unwrap();
5098
5099        tokio::time::timeout(Duration::from_secs(5), async {
5100            loop {
5101                if capture.messages().iter().any(|(_, message)| {
5102                    message.starts_with("Received close frame, terminating: code=")
5103                }) {
5104                    break;
5105                }
5106
5107                tokio::task::yield_now().await;
5108            }
5109        })
5110        .await
5111        .expect("timed out waiting for the close frame log");
5112
5113        let close_frame_logs: Vec<String> = capture
5114            .messages()
5115            .iter()
5116            .map(|(_, message)| message.clone())
5117            .filter(|message| message.starts_with("Received close frame, terminating: code="))
5118            .collect();
5119
5120        let expected = "Received close frame, terminating: code=1011, reason=<redacted>";
5121        assert!(
5122            close_frame_logs.iter().any(|message| message == expected),
5123            "close frame reason was not redacted: {close_frame_logs:?}"
5124        );
5125        assert!(
5126            close_frame_logs.iter().all(|message| {
5127                !message.contains('\r') && !message.contains('\n') && !message.contains('\u{1b}')
5128            }),
5129            "close frame log forged line breaks or escape sequences: {close_frame_logs:?}"
5130        );
5131
5132        assert!(
5133            close_frame_logs
5134                .iter()
5135                .all(|message| !message.contains("injected") && !message.contains("forged"))
5136        );
5137
5138        client.disconnect().await;
5139    }
5140
5141    #[rstest]
5142    #[tokio::test]
5143    async fn test_state_sink_reports_initial_loss_and_recovery() {
5144        let server = TestServer::setup().await;
5145
5146        let config = WebSocketConfig {
5147            url: format!("ws://127.0.0.1:{}", server.port),
5148            headers: vec![("test".into(), "test".into())],
5149            heartbeat_interval_secs: None,
5150            heartbeat_payload: None,
5151            connect_timeout_ms: Some(1_000),
5152            reconnect_delay_initial_ms: Some(1),
5153            reconnect_backoff_factor: Some(1.0),
5154            reconnect_delay_max_ms: Some(1),
5155            reconnect_jitter_ms: Some(0),
5156            reconnect_max_attempts: Some(3),
5157            heartbeat_timeout_secs: None,
5158            idle_timeout_ms: None,
5159            writer_capacity: None,
5160            backend: TransportBackend::Tungstenite,
5161            proxy_url: None,
5162            max_message_size_bytes: None,
5163            max_frame_size_bytes: None,
5164        };
5165
5166        let states = Arc::new(Mutex::new(Vec::new()));
5167        let states_callback = Arc::clone(&states);
5168        let sink = SocketStateSink::new(move |state| {
5169            states_callback.lock().push(state);
5170        });
5171
5172        let client = WebSocketClient::builder()
5173            .config(config)
5174            .message_handler(Arc::new(|_| {}))
5175            .state_sink(sink)
5176            .connect()
5177            .await
5178            .unwrap();
5179
5180        assert_eq!(*states.lock(), vec![SocketState::Connected]);
5181
5182        client.send_text("close-now".into(), None).await.unwrap();
5183        wait_until_async(
5184            || {
5185                let states = Arc::clone(&states);
5186                async move { states.lock().len() == 3 }
5187            },
5188            Duration::from_secs(5),
5189        )
5190        .await;
5191
5192        assert_eq!(
5193            *states.lock(),
5194            vec![
5195                SocketState::Connected,
5196                SocketState::Disconnected,
5197                SocketState::Connected,
5198            ]
5199        );
5200
5201        client.disconnect().await;
5202        assert_eq!(states.lock().len(), 3);
5203    }
5204
5205    #[rstest]
5206    #[tokio::test]
5207    async fn test_drop_suppresses_socket_state_event() {
5208        let server = TestServer::setup().await;
5209
5210        let config = WebSocketConfig {
5211            url: format!("ws://127.0.0.1:{}", server.port),
5212            headers: vec![("test".into(), "test".into())],
5213            heartbeat_interval_secs: None,
5214            heartbeat_payload: None,
5215            connect_timeout_ms: Some(1_000),
5216            reconnect_delay_initial_ms: Some(1),
5217            reconnect_backoff_factor: Some(1.0),
5218            reconnect_delay_max_ms: Some(1),
5219            reconnect_jitter_ms: Some(0),
5220            reconnect_max_attempts: Some(3),
5221            heartbeat_timeout_secs: None,
5222            idle_timeout_ms: None,
5223            writer_capacity: None,
5224            backend: TransportBackend::Tungstenite,
5225            proxy_url: None,
5226            max_message_size_bytes: None,
5227            max_frame_size_bytes: None,
5228        };
5229
5230        let states = Arc::new(Mutex::new(Vec::new()));
5231        let states_callback = Arc::clone(&states);
5232        let sink = SocketStateSink::new(move |state| {
5233            states_callback.lock().push(state);
5234        });
5235
5236        let client = WebSocketClient::builder()
5237            .config(config)
5238            .message_handler(Arc::new(|_| {}))
5239            .state_sink(sink)
5240            .connect()
5241            .await
5242            .unwrap();
5243
5244        drop(client);
5245        crate::dst::time::sleep(Duration::from_millis(25)).await;
5246
5247        assert_eq!(*states.lock(), vec![SocketState::Connected]);
5248    }
5249
5250    #[rstest]
5251    #[tokio::test]
5252    async fn test_stream_state_sink_reports_reader_loss() {
5253        let server = TestServer::setup().await;
5254
5255        let config = WebSocketConfig {
5256            url: format!("ws://127.0.0.1:{}", server.port),
5257            headers: vec![("test".into(), "test".into())],
5258            heartbeat_interval_secs: None,
5259            heartbeat_payload: None,
5260            connect_timeout_ms: None,
5261            reconnect_delay_initial_ms: None,
5262            reconnect_backoff_factor: None,
5263            reconnect_delay_max_ms: None,
5264            reconnect_jitter_ms: None,
5265            reconnect_max_attempts: None,
5266            heartbeat_timeout_secs: None,
5267            idle_timeout_ms: None,
5268            writer_capacity: None,
5269            backend: TransportBackend::Tungstenite,
5270            proxy_url: None,
5271            max_message_size_bytes: None,
5272            max_frame_size_bytes: None,
5273        };
5274
5275        let states = Arc::new(Mutex::new(Vec::new()));
5276        let states_callback = Arc::clone(&states);
5277        let sink = SocketStateSink::new(move |state| {
5278            states_callback.lock().push(state);
5279        });
5280
5281        let (_reader, client) = WebSocketClient::stream_builder()
5282            .config(config)
5283            .state_sink(sink)
5284            .connect()
5285            .await
5286            .unwrap();
5287
5288        client.notify_closed();
5289
5290        assert!(client.is_closed());
5291        assert_eq!(
5292            *states.lock(),
5293            vec![SocketState::Connected, SocketState::Disconnected]
5294        );
5295    }
5296
5297    #[rstest]
5298    #[tokio::test]
5299    async fn test_state_sink_emits_no_retry_or_exhaustion_events() {
5300        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5301        let port = listener.local_addr().unwrap().port();
5302        let server_task = task::spawn(async move {
5303            let (connection, _) = listener.accept().await.unwrap();
5304            let mut websocket = accept_async(connection).await.unwrap();
5305            while let Some(Ok(message)) = websocket.next().await {
5306                if matches!(&message, WsMessage::Text(text) if text.as_str() == "close-now") {
5307                    websocket.close(None).await.unwrap();
5308                    break;
5309                }
5310            }
5311        });
5312
5313        let config = WebSocketConfig {
5314            url: format!("ws://127.0.0.1:{port}"),
5315            headers: vec![],
5316            heartbeat_interval_secs: None,
5317            heartbeat_payload: None,
5318            connect_timeout_ms: Some(100),
5319            reconnect_delay_initial_ms: Some(1),
5320            reconnect_backoff_factor: Some(1.0),
5321            reconnect_delay_max_ms: Some(1),
5322            reconnect_jitter_ms: Some(0),
5323            reconnect_max_attempts: Some(2),
5324            heartbeat_timeout_secs: None,
5325            idle_timeout_ms: None,
5326            writer_capacity: None,
5327            backend: TransportBackend::Tungstenite,
5328            proxy_url: None,
5329            max_message_size_bytes: None,
5330            max_frame_size_bytes: None,
5331        };
5332
5333        let states = Arc::new(Mutex::new(Vec::new()));
5334        let states_callback = Arc::clone(&states);
5335        let sink = SocketStateSink::new(move |state| {
5336            states_callback.lock().push(state);
5337        });
5338
5339        let client = WebSocketClient::builder()
5340            .config(config)
5341            .message_handler(Arc::new(|_| {}))
5342            .state_sink(sink)
5343            .connect()
5344            .await
5345            .unwrap();
5346
5347        client.send_text("close-now".into(), None).await.unwrap();
5348        wait_until_async(
5349            || async { client.is_disconnected() },
5350            Duration::from_secs(5),
5351        )
5352        .await;
5353        assert_eq!(
5354            *states.lock(),
5355            vec![SocketState::Connected, SocketState::Disconnected]
5356        );
5357
5358        server_task.await.unwrap();
5359    }
5360
5361    #[rstest]
5362    #[case(
5363        "invalid header",
5364        "value",
5365        "Invalid WebSocket reconnect header name: invalid HTTP header name"
5366    )]
5367    #[case(
5368        "x-test",
5369        "invalid\nvalue",
5370        "Invalid WebSocket reconnect header value: failed to parse header value"
5371    )]
5372    fn test_reconnect_headers_reject_invalid_update(
5373        #[case] name: &str,
5374        #[case] value: &str,
5375        #[case] expected_message: &str,
5376    ) {
5377        let initial = vec![("x-existing".to_string(), "initial".to_string())];
5378        let headers = ReconnectHeaders::new(initial.clone());
5379
5380        let error = headers.update(name, value).unwrap_err();
5381
5382        let TransportError::Io(error) = error else {
5383            panic!("expected I/O error, was {error:?}");
5384        };
5385
5386        assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
5387        assert_eq!(error.to_string(), expected_message);
5388        assert_eq!(headers.snapshot(), initial);
5389    }
5390
5391    #[tokio::test]
5392    #[allow(clippy::result_large_err)]
5393    async fn test_reconnect_uses_updated_headers_without_interrupting_active_connection() {
5394        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5395        let port = listener.local_addr().unwrap().port();
5396        let (header_tx, mut header_rx) = tokio::sync::mpsc::unbounded_channel();
5397
5398        let server_task = task::spawn(async move {
5399            loop {
5400                let (conn, _) = listener.accept().await.unwrap();
5401                let header_tx = header_tx.clone();
5402
5403                let mut websocket = accept_hdr_async(
5404                    conn,
5405                    move |request: &server::Request, response: server::Response| {
5406                        let values = request
5407                            .headers()
5408                            .get_all("authorization")
5409                            .iter()
5410                            .map(|value| value.to_str().unwrap().to_string())
5411                            .collect::<Vec<_>>();
5412                        header_tx.send(values).unwrap();
5413                        Ok(response)
5414                    },
5415                )
5416                .await
5417                .unwrap();
5418
5419                task::spawn(async move {
5420                    while let Some(Ok(msg)) = websocket.next().await {
5421                        if matches!(&msg, WsMessage::Text(text) if text.as_str() == "close-now") {
5422                            let _ = websocket.close(None).await;
5423                            break;
5424                        }
5425                    }
5426                });
5427            }
5428        });
5429
5430        let config = WebSocketConfig {
5431            url: format!("ws://127.0.0.1:{port}"),
5432            headers: vec![("Authorization".into(), "Bearer initial".into())],
5433            heartbeat_interval_secs: None,
5434            heartbeat_payload: None,
5435            connect_timeout_ms: Some(1_000),
5436            reconnect_delay_initial_ms: Some(50),
5437            reconnect_delay_max_ms: Some(50),
5438            reconnect_backoff_factor: Some(1.0),
5439            reconnect_jitter_ms: Some(0),
5440            reconnect_max_attempts: None,
5441            heartbeat_timeout_secs: None,
5442            idle_timeout_ms: None,
5443            writer_capacity: None,
5444            backend: TransportBackend::Tungstenite,
5445            proxy_url: None,
5446            max_message_size_bytes: None,
5447            max_frame_size_bytes: None,
5448        };
5449
5450        let client = WebSocketClient::builder()
5451            .config(config)
5452            .message_handler(Arc::new(|_| {}))
5453            .connect()
5454            .await
5455            .unwrap();
5456
5457        let initial = header_rx.recv().await.unwrap();
5458        let reconnect_headers = client.reconnect_headers();
5459        reconnect_headers
5460            .update("authorization", "Bearer refreshed")
5461            .unwrap();
5462
5463        tokio::time::sleep(Duration::from_millis(100)).await;
5464        assert_eq!(initial, vec!["Bearer initial"]);
5465        assert!(client.is_active());
5466        assert!(header_rx.try_recv().is_err());
5467        assert!(!format!("{reconnect_headers:?}").contains("refreshed"));
5468
5469        client.send_text("close-now".into(), None).await.unwrap();
5470        let refreshed = tokio::time::timeout(Duration::from_secs(3), header_rx.recv())
5471            .await
5472            .unwrap()
5473            .unwrap();
5474
5475        assert_eq!(refreshed, vec!["Bearer refreshed"]);
5476
5477        client.disconnect().await;
5478        server_task.abort();
5479    }
5480
5481    #[tokio::test]
5482    async fn test_rate_limiter() {
5483        let server = TestServer::setup().await;
5484        let quota = Quota::per_second(NonZeroU32::new(2).unwrap()).unwrap();
5485
5486        let config = WebSocketConfig {
5487            url: format!("ws://127.0.0.1:{}", server.port),
5488            headers: vec![("test".into(), "test".into())],
5489            heartbeat_interval_secs: None,
5490            heartbeat_payload: None,
5491            connect_timeout_ms: None,
5492            reconnect_delay_initial_ms: None,
5493            reconnect_backoff_factor: None,
5494            reconnect_delay_max_ms: None,
5495            reconnect_jitter_ms: None,
5496            reconnect_max_attempts: None,
5497            heartbeat_timeout_secs: None,
5498            idle_timeout_ms: None,
5499            writer_capacity: None,
5500            backend: TransportBackend::Tungstenite,
5501            proxy_url: None,
5502            max_message_size_bytes: None,
5503            max_frame_size_bytes: None,
5504        };
5505
5506        let client = WebSocketClient::builder()
5507            .config(config)
5508            .message_handler(Arc::new(|_| {}))
5509            .keyed_quotas(vec![("default".into(), quota)])
5510            .connect()
5511            .await
5512            .unwrap();
5513
5514        // Burst of 2 passes immediately; the third send must wait for the
5515        // ~500ms replenish interval (keys=None would bypass the limiter)
5516        let keys: [ustr::Ustr; 1] = [ustr::Ustr::from("default")];
5517        let start = std::time::Instant::now();
5518        client
5519            .send_text("test1".into(), Some(keys.as_slice()))
5520            .await
5521            .unwrap();
5522        client
5523            .send_text("test2".into(), Some(keys.as_slice()))
5524            .await
5525            .unwrap();
5526        let after_burst = start.elapsed();
5527        client
5528            .send_text("test3".into(), Some(keys.as_slice()))
5529            .await
5530            .unwrap();
5531        let after_third = start.elapsed();
5532
5533        assert!(
5534            after_burst < std::time::Duration::from_millis(300),
5535            "Burst sends should not be rate limited, took {after_burst:?}"
5536        );
5537        assert!(
5538            after_third >= std::time::Duration::from_millis(400),
5539            "Third send should wait for quota replenishment, took {after_third:?}"
5540        );
5541
5542        // Cleanup
5543        client.disconnect().await;
5544        assert!(client.is_disconnected());
5545    }
5546
5547    #[tokio::test]
5548    async fn test_concurrent_writers() {
5549        let server = TestServer::setup().await;
5550        let client = Arc::new(setup_test_client(server.port).await);
5551
5552        let mut handles = vec![];
5553
5554        for i in 0..10 {
5555            let client = Arc::clone(&client);
5556            handles.push(task::spawn(async move {
5557                client.send_text(format!("test{i}"), None).await.unwrap();
5558            }));
5559        }
5560
5561        for handle in handles {
5562            handle.await.unwrap();
5563        }
5564
5565        // Cleanup
5566        client.disconnect().await;
5567        assert!(client.is_disconnected());
5568    }
5569}
5570
5571#[cfg(test)]
5572#[cfg(not(feature = "turmoil"))]
5573#[cfg(not(all(feature = "simulation", madsim)))] // These tests drive real localhost sockets
5574mod rust_tests {
5575    use std::{
5576        pin::Pin,
5577        sync::{
5578            Arc, OnceLock,
5579            atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
5580        },
5581        task::{Context, Poll},
5582    };
5583
5584    use futures_util::{SinkExt, StreamExt};
5585    #[cfg(target_os = "linux")]
5586    use log::Level;
5587    use nautilus_common::testing::wait_until_async;
5588    use parking_lot::{Condvar, Mutex};
5589    use rstest::rstest;
5590    #[cfg(feature = "transport-sockudo")]
5591    use sockudo_ws::handshake as sockudo_handshake;
5592    #[cfg(feature = "transport-sockudo")]
5593    use tokio::io::AsyncRead;
5594    use tokio::{
5595        io::{AsyncReadExt, AsyncWriteExt},
5596        net::TcpListener,
5597        sync::oneshot,
5598        task::{self, JoinHandle},
5599        time::{Duration, sleep},
5600    };
5601    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
5602    #[cfg(feature = "transport-sockudo")]
5603    use tokio_tungstenite::{
5604        accept_hdr_async,
5605        tungstenite::{
5606            handshake::server::{self, Callback},
5607            http::HeaderValue,
5608        },
5609    };
5610
5611    use super::*;
5612    #[cfg(target_os = "linux")]
5613    use crate::logging::tests::capture_logs_for;
5614    use crate::{
5615        SocketState,
5616        websocket::types::{channel_epoch_message_handler, channel_message_handler},
5617    };
5618
5619    const TEST_TIMEOUT: Duration = Duration::from_secs(10);
5620
5621    // A retryable upgrade rejection must stay below ERROR on both the transport that reports it
5622    // and the client that retries it.
5623    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5624    const RECONNECT_LOG_TARGETS: &[&str] = &[
5625        "nautilus_network::transport::sockudo",
5626        "nautilus_network::websocket::client",
5627    ];
5628
5629    #[cfg(target_os = "linux")]
5630    #[rstest]
5631    #[case::tungstenite(TransportBackend::Tungstenite)]
5632    #[cfg_attr(
5633        feature = "transport-sockudo",
5634        case::sockudo(TransportBackend::Sockudo)
5635    )]
5636    #[tokio::test]
5637    async fn test_initial_connect_redacts_error_text(#[case] backend: TransportBackend) {
5638        let capture = capture_logs_for(&["nautilus_network::retry"]).await;
5639        let socket = tokio::net::TcpSocket::new_v4().unwrap();
5640        socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
5641        let mut config = reconnect_test_config(socket.local_addr().unwrap().port());
5642        config.backend = backend;
5643        let error = WebSocketClient::builder()
5644            .config(config)
5645            .message_handler(Arc::new(|_| {}))
5646            .connect()
5647            .await
5648            .unwrap_err();
5649        let messages = capture.messages();
5650        let raw_message = error.to_string();
5651        let redacted_message = error.redacted_message();
5652
5653        assert!(
5654            matches!(error, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::ConnectionRefused)
5655        );
5656        assert_ne!(raw_message, redacted_message);
5657        assert_eq!(redacted_message, "I/O error: ConnectionRefused");
5658        assert!(messages.contains(&(
5659            Level::Trace,
5660            format!(
5661                "Operation '{INITIAL_CONNECT_OPERATION}' retries exhausted after 1 attempts: {redacted_message}"
5662            ),
5663        )), "{messages:?}");
5664        assert!(
5665            messages
5666                .iter()
5667                .all(|(_, message)| !message.contains(&raw_message)),
5668            "{messages:?}"
5669        );
5670    }
5671
5672    #[cfg(target_os = "linux")]
5673    #[rstest]
5674    #[case::tungstenite(TransportBackend::Tungstenite)]
5675    #[cfg_attr(
5676        feature = "transport-sockudo",
5677        case::sockudo(TransportBackend::Sockudo)
5678    )]
5679    #[tokio::test]
5680    async fn test_reconnect_redacts_error_text(#[case] backend: TransportBackend) {
5681        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
5682        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5683        let port = listener.local_addr().unwrap().port();
5684        let (close_tx, close_rx) = oneshot::channel();
5685
5686        let server = tokio::spawn(async move {
5687            let (stream, _) = listener.accept().await.unwrap();
5688            let ws_client = accept_async(stream).await.unwrap();
5689            close_rx.await.unwrap();
5690            drop(ws_client);
5691            let (_stream, _) = listener.accept().await.unwrap();
5692            std::future::pending::<()>().await;
5693        });
5694
5695        let mut config = reconnect_test_config(port);
5696        config.backend = backend;
5697        config.reconnect_max_attempts = Some(1);
5698        config.reconnect_delay_initial_ms = Some(1);
5699        config.reconnect_delay_max_ms = Some(1);
5700        config.reconnect_jitter_ms = Some(0);
5701        let client = WebSocketClient::builder()
5702            .config(config)
5703            .message_handler(Arc::new(|_| {}))
5704            .connect()
5705            .await
5706            .unwrap();
5707        close_tx.send(()).unwrap();
5708
5709        wait_until_async(
5710            || async {
5711                capture.messages().iter().any(|(level, message)| {
5712                    *level == Level::Warn && message.starts_with("Reconnect attempt 1 failed:")
5713                })
5714            },
5715            TEST_TIMEOUT,
5716        )
5717        .await;
5718
5719        client.disconnect().await;
5720        server.abort();
5721        await_task_termination(server, "redaction peer").await;
5722        let messages = capture.messages();
5723        let warnings: Vec<_> = messages
5724            .iter()
5725            .filter(|(_, message)| message.starts_with("Reconnect attempt "))
5726            .cloned()
5727            .collect();
5728
5729        assert_eq!(
5730            warnings,
5731            vec![(
5732                Level::Warn,
5733                "Reconnect attempt 1 failed: I/O error: TimedOut".into()
5734            )],
5735        );
5736    }
5737
5738    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
5739    #[tokio::test]
5740    async fn test_sockudo_handshake_trace_redacts_authentication() {
5741        let capture = capture_logs_for(&[
5742            "tungstenite::handshake::client",
5743            "nautilus_network::transport::sockudo",
5744            "nautilus_network::retry",
5745            "nautilus_network::websocket::client",
5746        ])
5747        .await;
5748        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5749        let addr = listener.local_addr().unwrap();
5750        let peer = tokio::spawn(async move {
5751            let (mut stream, _) = listener.accept().await.unwrap();
5752            let request = read_http_request(&mut stream).await;
5753            stream
5754                .write_all(
5755                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
5756                )
5757                .await
5758                .unwrap();
5759            stream.flush().await.unwrap();
5760            request
5761        });
5762
5763        let config = WebSocketConfig::builder()
5764            .url(format!(
5765                "ws://{addr}/TRACE_PATH_SECRET?token=TRACE_QUERY_SECRET"
5766            ))
5767            .headers(vec![
5768                ("KALSHI-ACCESS-KEY".into(), "TRACE_KEY_SECRET".into()),
5769                (
5770                    "KALSHI-ACCESS-SIGNATURE".into(),
5771                    "TRACE_SIGNATURE_SECRET".into(),
5772                ),
5773            ])
5774            .backend(TransportBackend::Sockudo)
5775            .build()
5776            .unwrap();
5777        let error = WebSocketClient::builder()
5778            .config(config)
5779            .message_handler(Arc::new(|_| {}))
5780            .connect()
5781            .await
5782            .unwrap_err();
5783        let request = String::from_utf8(peer.await.unwrap()).unwrap();
5784        let messages = capture.messages();
5785
5786        assert!(matches!(error, TransportError::UpgradeRejected(503)));
5787
5788        for secret in [
5789            "TRACE_KEY_SECRET",
5790            "TRACE_SIGNATURE_SECRET",
5791            "TRACE_PATH_SECRET",
5792            "TRACE_QUERY_SECRET",
5793        ] {
5794            assert!(request.contains(secret));
5795            assert!(
5796                messages
5797                    .iter()
5798                    .all(|(_, message)| !message.contains(secret)),
5799                "{messages:?}"
5800            );
5801        }
5802
5803        assert!(
5804            messages
5805                .iter()
5806                .any(|(level, message)| *level == log::Level::Trace
5807                    && message.contains("retries exhausted after 1 attempts")),
5808            "{messages:?}"
5809        );
5810    }
5811
5812    struct CondvarReleaseGuard<'a> {
5813        release: &'a (Mutex<bool>, Condvar),
5814    }
5815
5816    impl<'a> CondvarReleaseGuard<'a> {
5817        fn new(release: &'a (Mutex<bool>, Condvar)) -> Self {
5818            Self { release }
5819        }
5820
5821        fn release(&self) {
5822            let (lock, condvar) = self.release;
5823            let mut released = lock.lock();
5824            *released = true;
5825            condvar.notify_all();
5826        }
5827    }
5828
5829    impl Drop for CondvarReleaseGuard<'_> {
5830        fn drop(&mut self) {
5831            self.release();
5832        }
5833    }
5834
5835    async fn recv_rendezvous<T: Send + 'static>(
5836        receiver: std::sync::mpsc::Receiver<T>,
5837        name: &'static str,
5838    ) -> T {
5839        let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
5840
5841        match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
5842            Ok(Ok(Ok(value))) => value,
5843            Ok(Ok(Err(e))) => {
5844                panic!("{name} did not arrive within the test timeout: {e}")
5845            }
5846            Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
5847            Err(e) => panic!("{name} receive task did not finish: {e}"),
5848        }
5849    }
5850
5851    async fn await_task_termination(task: tokio::task::JoinHandle<()>, name: &'static str) {
5852        match tokio::time::timeout(TEST_TIMEOUT, task).await {
5853            Ok(Ok(())) => {}
5854            Ok(Err(e)) if e.is_cancelled() => {}
5855            Ok(Err(e)) => panic!("{name} failed: {e}"),
5856            Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
5857        }
5858    }
5859
5860    fn reconnect_test_config(port: u16) -> WebSocketConfig {
5861        WebSocketConfig {
5862            url: format!("ws://127.0.0.1:{port}"),
5863            headers: vec![],
5864            heartbeat_interval_secs: None,
5865            heartbeat_payload: None,
5866            connect_timeout_ms: Some(1_000),
5867            reconnect_delay_initial_ms: None,
5868            reconnect_delay_max_ms: None,
5869            reconnect_backoff_factor: None,
5870            reconnect_jitter_ms: None,
5871            reconnect_max_attempts: None,
5872            heartbeat_timeout_secs: None,
5873            idle_timeout_ms: None,
5874            writer_capacity: None,
5875            backend: TransportBackend::Tungstenite,
5876            proxy_url: None,
5877            max_message_size_bytes: None,
5878            max_frame_size_bytes: None,
5879        }
5880    }
5881
5882    fn initial_connect_retry_policy(
5883        max_attempts: u32,
5884        initial_delay_ms: u64,
5885    ) -> InitialConnectRetryPolicy {
5886        InitialConnectRetryPolicy {
5887            max_attempts: std::num::NonZeroU32::new(max_attempts).unwrap(),
5888            delay_initial: Duration::from_millis(initial_delay_ms),
5889            delay_max: Duration::from_millis(initial_delay_ms),
5890            backoff_factor: 2.0,
5891            jitter_ms: 0,
5892        }
5893    }
5894
5895    #[rstest]
5896    #[case(TransportError::UpgradeRejected(408), true)]
5897    #[case(TransportError::UpgradeRejected(425), true)]
5898    #[case(TransportError::UpgradeRejected(429), true)]
5899    #[case(TransportError::UpgradeRejected(500), true)]
5900    #[case(TransportError::UpgradeRejected(599), true)]
5901    #[case(TransportError::UpgradeRejected(404), false)]
5902    #[case(TransportError::UpgradeRejected(600), false)]
5903    #[case(TransportError::ProxyConnectRejected(429), true)]
5904    #[case(TransportError::ProxyConnectRejected(407), false)]
5905    #[case(TransportError::ConnectionClosed, true)]
5906    #[case(
5907        TransportError::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
5908        true
5909    )]
5910    #[case(
5911        TransportError::Io(std::io::Error::from(std::io::ErrorKind::Interrupted)),
5912        true
5913    )]
5914    #[case(
5915        TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidInput)),
5916        false
5917    )]
5918    #[case(
5919        TransportError::Io(std::io::Error::from(std::io::ErrorKind::InvalidData)),
5920        false
5921    )]
5922    #[case(
5923        TransportError::Io(std::io::Error::from(std::io::ErrorKind::Unsupported)),
5924        false
5925    )]
5926    #[case(
5927        TransportError::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
5928        false
5929    )]
5930    #[case(TransportError::Handshake("malformed".to_string()), false)]
5931    fn initial_connect_error_classification(#[case] error: TransportError, #[case] expected: bool) {
5932        assert_eq!(is_retryable_initial_connect_error(&error), expected);
5933    }
5934
5935    async fn rejected_upgrade_attempts(
5936        status: u16,
5937        backend: TransportBackend,
5938        retry_policy: Option<InitialConnectRetryPolicy>,
5939    ) -> u64 {
5940        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5941        let port = listener.local_addr().unwrap().port();
5942        let accepted = Arc::new(AtomicU64::new(0));
5943        let accepted_server = Arc::clone(&accepted);
5944
5945        let server = tokio::spawn(async move {
5946            loop {
5947                let (mut stream, _) = listener.accept().await.unwrap();
5948                accepted_server.fetch_add(1, Ordering::SeqCst);
5949                let mut request = Vec::new();
5950
5951                loop {
5952                    let mut chunk = [0; 1024];
5953                    let n = stream.read(&mut chunk).await.unwrap();
5954                    if n == 0 {
5955                        break;
5956                    }
5957
5958                    request.extend_from_slice(&chunk[..n]);
5959                    if request.windows(4).any(|window| window == b"\r\n\r\n") {
5960                        break;
5961                    }
5962                }
5963
5964                stream
5965                    .write_all(format!("HTTP/1.1 {status}\r\n\r\n").as_bytes())
5966                    .await
5967                    .unwrap();
5968            }
5969        });
5970
5971        let (handler, _rx) = channel_message_handler();
5972        let mut config = reconnect_test_config(port);
5973        config.backend = backend;
5974        WebSocketClient::builder()
5975            .config(config)
5976            .message_handler(handler)
5977            .maybe_initial_connect_retry_policy(retry_policy)
5978            .connect()
5979            .await
5980            .expect_err("server always rejects the upgrade");
5981
5982        let attempts = accepted.load(Ordering::SeqCst);
5983        server.abort();
5984        attempts
5985    }
5986
5987    async fn closed_proxy_attempts() -> u64 {
5988        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
5989        let proxy_addr = listener.local_addr().unwrap();
5990        let accepted = Arc::new(AtomicU64::new(0));
5991        let accepted_server = Arc::clone(&accepted);
5992
5993        let server = tokio::spawn(async move {
5994            loop {
5995                let (mut stream, _) = listener.accept().await.unwrap();
5996                accepted_server.fetch_add(1, Ordering::SeqCst);
5997                let mut request = [0; 1024];
5998                let _ = stream.read(&mut request).await.unwrap();
5999            }
6000        });
6001
6002        let (handler, _rx) = channel_message_handler();
6003        let mut config = reconnect_test_config(9);
6004        config.proxy_url = Some(format!("http://{proxy_addr}"));
6005        WebSocketClient::builder()
6006            .config(config)
6007            .message_handler(handler)
6008            .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6009            .connect()
6010            .await
6011            .expect_err("proxy always closes before its CONNECT response");
6012
6013        let attempts = accepted.load(Ordering::SeqCst);
6014        server.abort();
6015        attempts
6016    }
6017
6018    #[rstest]
6019    #[tokio::test]
6020    async fn initial_connect_cancellation_interrupts_in_flight_handshake() {
6021        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6022        let port = listener.local_addr().unwrap().port();
6023        let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
6024
6025        let server = tokio::spawn(async move {
6026            let (_stream, _) = listener.accept().await.unwrap();
6027            let _ = accepted_tx.send(());
6028            std::future::pending::<()>().await;
6029        });
6030
6031        let token = CancellationToken::new();
6032        let (handler, _rx) = channel_message_handler();
6033        let mut config = reconnect_test_config(port);
6034        config.connect_timeout_ms = Some(10_000);
6035        let connect = WebSocketClient::builder()
6036            .config(config)
6037            .message_handler(handler)
6038            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6039            .cancellation_token(token.clone())
6040            .connect();
6041        tokio::pin!(connect);
6042
6043        tokio::select! {
6044            biased;
6045            result = &mut connect => panic!("connect completed before cancellation: {result:?}"),
6046            result = accepted_rx => result.unwrap(),
6047        }
6048        token.cancel();
6049
6050        let err = tokio::time::timeout(Duration::from_millis(250), connect)
6051            .await
6052            .expect("cancellation should interrupt the handshake")
6053            .expect_err("cancelled connect should fail");
6054        assert!(
6055            matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::Interrupted)
6056        );
6057        server.abort();
6058    }
6059
6060    #[rstest]
6061    #[tokio::test]
6062    async fn timed_out_initial_connect_attempt_is_retried() {
6063        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6064        let port = listener.local_addr().unwrap().port();
6065        let accepted = Arc::new(AtomicU64::new(0));
6066        let accepted_server = Arc::clone(&accepted);
6067
6068        // Accept the TCP connection but never complete the upgrade, so every attempt
6069        // reaches the connect timeout rather than failing fast.
6070        let server = tokio::spawn(async move {
6071            let mut held = Vec::new();
6072
6073            loop {
6074                let (stream, _) = listener.accept().await.unwrap();
6075                accepted_server.fetch_add(1, Ordering::SeqCst);
6076                held.push(stream);
6077            }
6078        });
6079
6080        let (handler, _rx) = channel_message_handler();
6081        let mut config = reconnect_test_config(port);
6082        config.connect_timeout_ms = Some(50);
6083        let err = WebSocketClient::builder()
6084            .config(config)
6085            .message_handler(handler)
6086            .initial_connect_retry_policy(initial_connect_retry_policy(3, 1))
6087            .connect()
6088            .await
6089            .expect_err("every attempt times out");
6090
6091        assert!(
6092            matches!(err, TransportError::Io(ref e) if e.kind() == std::io::ErrorKind::TimedOut)
6093        );
6094
6095        // The ladder must exhaust its attempts: a timeout is transient, so returning after
6096        // the first one leaves this at 1. Wait for the count rather than sampling it, since
6097        // the final connection can complete through the listen backlog before the accepting
6098        // task is scheduled to record it.
6099        let counted = tokio::time::timeout(Duration::from_secs(1), async {
6100            while accepted.load(Ordering::SeqCst) < 3 {
6101                tokio::time::sleep(Duration::from_millis(5)).await;
6102            }
6103        })
6104        .await;
6105
6106        assert!(
6107            counted.is_ok(),
6108            "expected 3 connection attempts, observed {}",
6109            accepted.load(Ordering::SeqCst)
6110        );
6111        // Exactly three: the wait above resolves the scheduling race, but the ladder must
6112        // also stop at its configured maximum rather than exceeding it.
6113        assert_eq!(accepted.load(Ordering::SeqCst), 3);
6114        server.abort();
6115    }
6116
6117    #[rstest]
6118    #[case::too_many_requests(429)]
6119    #[case::server_error(503)]
6120    #[tokio::test]
6121    async fn transient_upgrade_rejection_is_retried(#[case] status: u16) {
6122        assert_eq!(
6123            rejected_upgrade_attempts(
6124                status,
6125                TransportBackend::Tungstenite,
6126                Some(initial_connect_retry_policy(3, 1)),
6127            )
6128            .await,
6129            3
6130        );
6131    }
6132
6133    #[rstest]
6134    #[tokio::test]
6135    async fn initial_connect_without_retry_policy_makes_one_attempt() {
6136        assert_eq!(
6137            rejected_upgrade_attempts(503, TransportBackend::Tungstenite, None).await,
6138            1
6139        );
6140    }
6141
6142    #[cfg(feature = "transport-sockudo")]
6143    #[rstest]
6144    #[tokio::test]
6145    async fn sockudo_too_many_requests_upgrade_rejection_is_retried() {
6146        assert_eq!(
6147            rejected_upgrade_attempts(
6148                429,
6149                TransportBackend::Sockudo,
6150                Some(initial_connect_retry_policy(3, 1)),
6151            )
6152            .await,
6153            3
6154        );
6155    }
6156
6157    #[rstest]
6158    #[tokio::test]
6159    async fn permanent_upgrade_rejection_is_not_retried() {
6160        assert_eq!(
6161            rejected_upgrade_attempts(
6162                404,
6163                TransportBackend::Tungstenite,
6164                Some(initial_connect_retry_policy(3, 1)),
6165            )
6166            .await,
6167            1
6168        );
6169    }
6170
6171    #[rstest]
6172    #[tokio::test]
6173    async fn proxy_close_before_connect_response_is_retried() {
6174        assert_eq!(closed_proxy_attempts().await, 3);
6175    }
6176
6177    #[rstest]
6178    #[tokio::test]
6179    async fn initial_connect_cancellation_interrupts_retry_backoff() {
6180        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6181        let port = listener.local_addr().unwrap().port();
6182        let accepted = Arc::new(AtomicU64::new(0));
6183        let accepted_server = Arc::clone(&accepted);
6184        let (rejected_tx, rejected_rx) = oneshot::channel();
6185        let server = tokio::spawn(async move {
6186            let mut rejected_tx = Some(rejected_tx);
6187
6188            loop {
6189                let (mut stream, _) = listener.accept().await.unwrap();
6190                accepted_server.fetch_add(1, Ordering::SeqCst);
6191                let mut request = [0; 1024];
6192                let _ = stream.read(&mut request).await.unwrap();
6193                stream
6194                    .write_all(b"HTTP/1.1 503 Service Unavailable\r\n\r\n")
6195                    .await
6196                    .unwrap();
6197
6198                if let Some(rejected_tx) = rejected_tx.take() {
6199                    rejected_tx.send(()).unwrap();
6200                }
6201            }
6202        });
6203
6204        let token = CancellationToken::new();
6205        let (handler, _rx) = channel_message_handler();
6206        let connect = WebSocketClient::builder()
6207            .config(reconnect_test_config(port))
6208            .message_handler(handler)
6209            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6210            .cancellation_token(token.clone())
6211            .connect();
6212        tokio::pin!(connect);
6213
6214        tokio::select! {
6215            biased;
6216            result = &mut connect => panic!("connect completed before backoff: {result:?}"),
6217            result = rejected_rx => result.unwrap(),
6218        }
6219        assert!(futures_util::poll!(&mut connect).is_pending());
6220        token.cancel();
6221
6222        let error = tokio::time::timeout(Duration::from_millis(250), connect)
6223            .await
6224            .expect("cancellation should interrupt retry backoff")
6225            .expect_err("cancelled connect should fail");
6226
6227        assert!(
6228            matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6229        );
6230        assert_eq!(accepted.load(Ordering::SeqCst), 1);
6231        server.abort();
6232    }
6233
6234    #[rstest]
6235    #[tokio::test]
6236    async fn permanent_initial_connect_error_does_not_wait_for_retry_ladder() {
6237        let (handler, _rx) = channel_message_handler();
6238        let mut config = reconnect_test_config(1);
6239        config.url = "not a websocket URL".to_string();
6240        let connect = WebSocketClient::builder()
6241            .config(config)
6242            .message_handler(handler)
6243            .initial_connect_retry_policy(initial_connect_retry_policy(5, 30_000))
6244            .connect();
6245
6246        let err = tokio::time::timeout(Duration::from_millis(250), connect)
6247            .await
6248            .expect("permanent error should not enter retry backoff")
6249            .expect_err("invalid URL should fail");
6250        assert!(matches!(
6251            err,
6252            TransportError::InvalidUrl(_) | TransportError::Handshake(_)
6253        ));
6254    }
6255
6256    #[rstest]
6257    #[tokio::test(start_paused = true)]
6258    async fn connection_rate_limit_gates_initial_connect_and_reconnect() {
6259        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6260        let port = listener.local_addr().unwrap().port();
6261        let server = tokio::spawn(async move {
6262            let (stream, _) = listener.accept().await.unwrap();
6263            let _first = accept_async(stream).await.unwrap();
6264            let (stream, _) = listener.accept().await.unwrap();
6265            let _second = accept_async(stream).await.unwrap();
6266            std::future::pending::<()>().await;
6267        });
6268
6269        let key = Ustr::from("connection-attempt");
6270
6271        let limiter = Arc::new(RateLimiter::new_with_quota(
6272            None,
6273            vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6274        ));
6275        limiter.await_keys_ready(Some(&[key])).await;
6276
6277        let rate_limit = ConnectionRateLimit {
6278            limiter,
6279            keys: Arc::from([key]),
6280        };
6281
6282        let (handler, _rx) = channel_message_handler();
6283        let mut config = reconnect_test_config(port);
6284        config.connect_timeout_ms = Some(10_000);
6285        let initial = WebSocketClientInner::connect_url_with_handler(
6286            config,
6287            Some(IncomingHandler::Message(handler)),
6288            None,
6289            None,
6290            None,
6291            Some(rate_limit),
6292            InitialConnectOptions::default(),
6293        );
6294        tokio::pin!(initial);
6295        assert!(futures_util::poll!(&mut initial).is_pending());
6296        tokio::time::advance(Duration::from_secs(1)).await;
6297        tokio::task::yield_now().await;
6298
6299        // Paused time auto-advances while the runtime waits on socket I/O, which
6300        // would fire the connect timeout, so run the handshake on the real clock
6301        tokio::time::resume();
6302        let mut inner = initial.await.unwrap();
6303        tokio::time::pause();
6304
6305        inner
6306            .connection_mode
6307            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6308        let reconnect = inner.reconnect_with_outcome();
6309        tokio::pin!(reconnect);
6310        assert!(futures_util::poll!(&mut reconnect).is_pending());
6311        tokio::time::advance(Duration::from_secs(1)).await;
6312        tokio::task::yield_now().await;
6313        tokio::time::resume();
6314        assert_eq!(reconnect.await.unwrap(), ReconnectOutcome::Reconnected);
6315
6316        server.abort();
6317    }
6318
6319    #[rstest]
6320    #[tokio::test(start_paused = true)]
6321    async fn initial_connect_cancellation_interrupts_connection_rate_limit_wait() {
6322        let key = Ustr::from("connection-attempt");
6323
6324        let limiter = Arc::new(RateLimiter::new_with_quota(
6325            None,
6326            vec![(key, Quota::with_period(Duration::from_secs(1)).unwrap())],
6327        ));
6328        limiter.await_keys_ready(Some(&[key])).await;
6329
6330        let rate_limit = ConnectionRateLimit {
6331            limiter,
6332            keys: Arc::from([key]),
6333        };
6334
6335        let token = CancellationToken::new();
6336        let (handler, _rx) = channel_message_handler();
6337
6338        let connect = WebSocketClientInner::connect_url_with_handler(
6339            reconnect_test_config(1),
6340            Some(IncomingHandler::Message(handler)),
6341            None,
6342            None,
6343            None,
6344            Some(rate_limit),
6345            InitialConnectOptions {
6346                retry_policy: None,
6347                cancellation_token: Some(token.clone()),
6348            },
6349        );
6350
6351        tokio::pin!(connect);
6352        assert!(futures_util::poll!(&mut connect).is_pending());
6353
6354        token.cancel();
6355        let error = connect.await.expect_err("cancelled connect should fail");
6356
6357        assert!(
6358            matches!(error, TransportError::Io(ref error) if error.kind() == std::io::ErrorKind::Interrupted)
6359        );
6360    }
6361
6362    #[rstest]
6363    #[tokio::test]
6364    async fn test_reconnect_outcome_is_aborted_before_connect() {
6365        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6366        let port = listener.local_addr().unwrap().port();
6367        let server = tokio::spawn(async move {
6368            let (stream, _) = listener.accept().await.unwrap();
6369            let _websocket = accept_async(stream).await.unwrap();
6370            std::future::pending::<()>().await;
6371        });
6372
6373        let (handler, _rx) = channel_message_handler();
6374        let mut inner =
6375            WebSocketClientInner::connect_url(reconnect_test_config(port), Some(handler), None)
6376                .await
6377                .unwrap();
6378        inner
6379            .connection_mode
6380            .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
6381
6382        let outcome = inner.reconnect_with_outcome().await.unwrap();
6383
6384        assert_eq!(outcome, ReconnectOutcome::Aborted);
6385        server.abort();
6386    }
6387
6388    #[rstest]
6389    #[tokio::test]
6390    async fn test_stream_reconnect_outcome_is_aborted_and_notifies_closed() {
6391        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6392        let port = listener.local_addr().unwrap().port();
6393        let server = tokio::spawn(async move {
6394            let (stream, _) = listener.accept().await.unwrap();
6395            let _websocket = accept_async(stream).await.unwrap();
6396            std::future::pending::<()>().await;
6397        });
6398
6399        let mut inner = WebSocketClientInner::connect_url(reconnect_test_config(port), None, None)
6400            .await
6401            .unwrap();
6402        let state_notify = Arc::clone(&inner.state_notify);
6403        let mut notified = std::pin::pin!(state_notify.notified());
6404        notified.as_mut().enable();
6405
6406        let outcome = inner.reconnect_with_outcome().await.unwrap();
6407
6408        assert_eq!(outcome, ReconnectOutcome::Aborted);
6409        assert_eq!(
6410            ConnectionMode::from_atomic(&inner.connection_mode),
6411            ConnectionMode::Closed
6412        );
6413        tokio::time::timeout(TEST_TIMEOUT, notified)
6414            .await
6415            .expect("stream close notification was not published");
6416        server.abort();
6417    }
6418
6419    #[rstest]
6420    #[tokio::test]
6421    async fn test_reconnect_outcome_is_reconnected_with_handler() {
6422        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6423        let port = listener.local_addr().unwrap().port();
6424        let server = tokio::spawn(async move {
6425            let (stream, _) = listener.accept().await.unwrap();
6426            let _first = accept_async(stream).await.unwrap();
6427            let (stream, _) = listener.accept().await.unwrap();
6428            let mut second = accept_async(stream).await.unwrap();
6429            second
6430                .send(WsMessage::Text("replacement".into()))
6431                .await
6432                .unwrap();
6433            std::future::pending::<()>().await;
6434        });
6435
6436        let (epoch_handler, mut epoch_rx) = channel_epoch_message_handler();
6437        let mut inner = WebSocketClientInner::connect_url_with_handler(
6438            reconnect_test_config(port),
6439            Some(IncomingHandler::Epoch(epoch_handler)),
6440            None,
6441            None,
6442            None,
6443            None,
6444            InitialConnectOptions::default(),
6445        )
6446        .await
6447        .unwrap();
6448        inner
6449            .connection_mode
6450            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
6451
6452        let outcome = inner.reconnect_with_outcome().await.unwrap();
6453
6454        assert_eq!(outcome, ReconnectOutcome::Reconnected);
6455        assert_eq!(
6456            ConnectionMode::from_atomic(&inner.connection_mode),
6457            ConnectionMode::Active
6458        );
6459        let (epoch, message) = tokio::time::timeout(TEST_TIMEOUT, epoch_rx.recv())
6460            .await
6461            .expect("replacement epoch message was not delivered")
6462            .expect("epoch handler channel closed");
6463        assert_eq!(epoch, 1);
6464        assert_eq!(message, WsMessage::Text("replacement".into()));
6465        server.abort();
6466    }
6467
6468    #[rstest]
6469    #[tokio::test]
6470    async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
6471        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6472        let port = listener.local_addr().unwrap().port();
6473        let server = tokio::spawn(async move {
6474            let (stream, _) = listener.accept().await.unwrap();
6475            let _websocket = accept_async(stream).await.unwrap();
6476            std::future::pending::<()>().await;
6477        });
6478
6479        let mut config = reconnect_test_config(port);
6480        config.heartbeat_interval_secs = Some(60);
6481        let (handler, _handler_rx) = channel_message_handler();
6482        let inner = WebSocketClientInner::connect_url(config, Some(handler), None)
6483            .await
6484            .unwrap();
6485        let read_fence = inner
6486            .read_fence
6487            .clone()
6488            .expect("read fence should exist in handler mode");
6489        let read_abort = inner
6490            .read_task
6491            .as_ref()
6492            .expect("read task should be spawned in handler mode")
6493            .abort_handle();
6494        let write_abort = inner.write_task.abort_handle();
6495        let heartbeat_abort = inner
6496            .heartbeat_task
6497            .as_ref()
6498            .expect("heartbeat task should be spawned for a configured heartbeat")
6499            .abort_handle();
6500
6501        assert!(read_fence.is_valid(), "read fence should start valid");
6502        assert!(
6503            !read_abort.is_finished(),
6504            "read task should be running before drop"
6505        );
6506        assert!(
6507            !write_abort.is_finished(),
6508            "write task should be running before drop"
6509        );
6510        assert!(
6511            !heartbeat_abort.is_finished(),
6512            "heartbeat task should be running before drop"
6513        );
6514
6515        drop(inner);
6516        wait_until_async(
6517            || async {
6518                read_abort.is_finished()
6519                    && write_abort.is_finished()
6520                    && heartbeat_abort.is_finished()
6521            },
6522            TEST_TIMEOUT,
6523        )
6524        .await;
6525
6526        assert!(!read_fence.is_valid(), "read fence was not invalidated");
6527        assert!(read_abort.is_finished(), "read task was not aborted");
6528        assert!(write_abort.is_finished(), "write task was not aborted");
6529        assert!(
6530            heartbeat_abort.is_finished(),
6531            "heartbeat task was not aborted"
6532        );
6533        server.abort();
6534    }
6535
6536    struct RecordingServer {
6537        task: JoinHandle<()>,
6538        port: u16,
6539        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
6540        binary_frames: Arc<tokio::sync::Mutex<Vec<Vec<u8>>>>,
6541        control_frames: Arc<tokio::sync::Mutex<Vec<String>>>,
6542        connections: Arc<AtomicUsize>,
6543    }
6544
6545    #[cfg(feature = "transport-sockudo")]
6546    async fn read_http_request<S>(stream: &mut S) -> Vec<u8>
6547    where
6548        S: AsyncRead + Unpin,
6549    {
6550        let mut buf = Vec::new();
6551        let mut chunk = [0u8; 256];
6552        loop {
6553            let n = stream.read(&mut chunk).await.unwrap();
6554            assert!(n > 0, "HTTP request closed before headers completed");
6555            buf.extend_from_slice(&chunk[..n]);
6556            if buf.windows(4).any(|window| window == b"\r\n\r\n") {
6557                return buf;
6558            }
6559        }
6560    }
6561
6562    #[cfg(feature = "transport-sockudo")]
6563    fn extract_header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
6564        request.lines().find_map(|line| {
6565            let (header_name, header_value) = line.split_once(':')?;
6566            if header_name.eq_ignore_ascii_case(name) {
6567                Some(header_value.trim())
6568            } else {
6569                None
6570            }
6571        })
6572    }
6573
6574    #[cfg(feature = "transport-sockudo")]
6575    #[derive(Debug, Clone)]
6576    struct HeaderAssertCallback {
6577        key: String,
6578        value: HeaderValue,
6579    }
6580
6581    #[cfg(feature = "transport-sockudo")]
6582    impl Callback for HeaderAssertCallback {
6583        #[expect(
6584            clippy::panic_in_result_fn,
6585            reason = "assertion failures should fail the test"
6586        )]
6587        fn on_request(
6588            self,
6589            request: &server::Request,
6590            response: server::Response,
6591        ) -> Result<server::Response, server::ErrorResponse> {
6592            assert_eq!(request.headers().get(&self.key), Some(&self.value));
6593            Ok(response)
6594        }
6595    }
6596
6597    impl RecordingServer {
6598        async fn setup() -> Self {
6599            let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6600            let port = listener.local_addr().unwrap().port();
6601            let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6602            let messages_clone = Arc::clone(&messages);
6603            let binary_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6604            let binary_frames_clone = Arc::clone(&binary_frames);
6605            let control_frames = Arc::new(tokio::sync::Mutex::new(Vec::new()));
6606            let control_frames_clone = Arc::clone(&control_frames);
6607            let connections = Arc::new(AtomicUsize::new(0));
6608            let connections_clone = Arc::clone(&connections);
6609
6610            let task = task::spawn(async move {
6611                loop {
6612                    let (stream, _) = listener.accept().await.unwrap();
6613                    let mut websocket = accept_async(stream).await.unwrap();
6614                    connections_clone.fetch_add(1, Ordering::SeqCst);
6615                    let messages = Arc::clone(&messages_clone);
6616                    let binary_frames = Arc::clone(&binary_frames_clone);
6617                    let control_frames = Arc::clone(&control_frames_clone);
6618
6619                    task::spawn(async move {
6620                        while let Some(Ok(msg)) = websocket.next().await {
6621                            match msg {
6622                                WsMessage::Text(text) => {
6623                                    messages.lock().await.push(text.to_string());
6624                                }
6625                                WsMessage::Binary(data) => {
6626                                    binary_frames.lock().await.push(data.to_vec());
6627                                }
6628                                WsMessage::Pong(data) => {
6629                                    control_frames
6630                                        .lock()
6631                                        .await
6632                                        .push(format!("pong:{}", String::from_utf8_lossy(&data)));
6633                                }
6634                                WsMessage::Close(_) => {
6635                                    let _ = websocket.close(None).await;
6636                                    break;
6637                                }
6638                                _ => {}
6639                            }
6640                        }
6641                    });
6642                }
6643            });
6644
6645            Self {
6646                task,
6647                port,
6648                messages,
6649                binary_frames,
6650                control_frames,
6651                connections,
6652            }
6653        }
6654
6655        async fn messages(&self) -> Vec<String> {
6656            self.messages.lock().await.clone()
6657        }
6658
6659        async fn binary_frames(&self) -> Vec<Vec<u8>> {
6660            self.binary_frames.lock().await.clone()
6661        }
6662
6663        async fn control_frames(&self) -> Vec<String> {
6664            self.control_frames.lock().await.clone()
6665        }
6666
6667        async fn wait_for_connections(&self, expected: usize) {
6668            wait_until_async(
6669                || async { self.connections.load(Ordering::SeqCst) == expected },
6670                TEST_TIMEOUT,
6671            )
6672            .await;
6673        }
6674    }
6675
6676    impl Drop for RecordingServer {
6677        fn drop(&mut self) {
6678            self.task.abort();
6679        }
6680    }
6681
6682    #[rstest]
6683    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6684    async fn test_manual_reconnect_waits_for_slow_loss_callback_and_new_auth() {
6685        let server = RecordingServer::setup().await;
6686        let tracker = AuthTracker::new();
6687        let _initial_auth = tracker.begin();
6688        tracker.succeed();
6689        let states = Arc::new(Mutex::new(Vec::new()));
6690        let states_callback = Arc::clone(&states);
6691        let auth_at_loss = Arc::new(Mutex::new(Vec::new()));
6692        let auth_at_loss_callback = Arc::clone(&auth_at_loss);
6693        let tracker_callback = tracker.clone();
6694        let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6695        let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6696        let callback_release_clone = Arc::clone(&callback_release);
6697        let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6698
6699        let sink = SocketStateSink::new(move |state| {
6700            states_callback.lock().push(state);
6701            if state == SocketState::Disconnected {
6702                auth_at_loss_callback
6703                    .lock()
6704                    .push(tracker_callback.auth_state());
6705                callback_entered_tx.send(()).unwrap();
6706                let (lock, condvar) = callback_release_clone.as_ref();
6707                let mut released = lock.lock();
6708                while !*released {
6709                    condvar.wait(&mut released);
6710                }
6711            }
6712        });
6713
6714        let (handler, mut handler_rx) = channel_message_handler();
6715        let client = WebSocketClient::builder()
6716            .config(reconnect_test_config(server.port))
6717            .message_handler(handler)
6718            .state_sink(sink)
6719            .connect()
6720            .await
6721            .unwrap();
6722        client.set_auth_tracker(tracker.clone(), true);
6723        server.wait_for_connections(1).await;
6724
6725        let handle = client.reconnect_handle();
6726        let (request_tx, request_rx) = std::sync::mpsc::channel();
6727        let request_thread = std::thread::spawn(move || {
6728            request_tx.send(handle.request_reconnect()).unwrap();
6729        });
6730
6731        recv_rendezvous(callback_entered_rx, "slow reconnect callback entry").await;
6732        client
6733            .writer_tx
6734            .send(WriterCommand::Send(Message::text("buffered")))
6735            .unwrap();
6736        server.wait_for_connections(2).await;
6737        tokio::time::sleep(Duration::from_millis(250)).await;
6738
6739        assert_eq!(client.connection_mode(), ConnectionMode::Reconnect);
6740        assert!(!client.reconnect_published.load(Ordering::SeqCst));
6741        assert_eq!(tracker.auth_state(), AuthState::Unauthenticated);
6742        assert_eq!(*auth_at_loss.lock(), vec![AuthState::Unauthenticated]);
6743        assert_eq!(
6744            *states.lock(),
6745            vec![SocketState::Connected, SocketState::Disconnected]
6746        );
6747        assert!(server.messages().await.is_empty());
6748
6749        callback_release_guard.release();
6750        assert_eq!(
6751            recv_rendezvous(request_rx, "manual reconnect result").await,
6752            ReconnectRequestOutcome::Accepted
6753        );
6754        request_thread.join().unwrap();
6755        wait_until_async(|| async { client.is_active() }, TEST_TIMEOUT).await;
6756        wait_until_async(
6757            || {
6758                let states = Arc::clone(&states);
6759                async move { states.lock().len() == 3 }
6760            },
6761            TEST_TIMEOUT,
6762        )
6763        .await;
6764
6765        let notification = tokio::time::timeout(TEST_TIMEOUT, handler_rx.recv())
6766            .await
6767            .expect("reconnect notification was not delivered")
6768            .expect("handler channel closed");
6769        assert_eq!(notification, WsMessage::Text(RECONNECTED.into()));
6770        assert!(handler_rx.try_recv().is_err());
6771        tokio::time::sleep(Duration::from_millis(200)).await;
6772        assert!(server.messages().await.is_empty());
6773
6774        // The auth frame is bound to the replacement connection, so it is written while the
6775        // replay waits for that authentication.
6776        let _replacement_auth = tracker.begin();
6777        client
6778            .send_bytes_on_connection(b"auth".to_vec(), None, client.connection_epoch())
6779            .await
6780            .unwrap();
6781        wait_until_async(
6782            || {
6783                let binary_frames = Arc::clone(&server.binary_frames);
6784                async move { binary_frames.lock().await.len() == 1 }
6785            },
6786            TEST_TIMEOUT,
6787        )
6788        .await;
6789        assert_eq!(server.binary_frames().await, vec![b"auth".to_vec()]);
6790        assert!(server.messages().await.is_empty());
6791
6792        tracker.succeed();
6793        wait_until_async(
6794            || {
6795                let messages = Arc::clone(&server.messages);
6796                async move { messages.lock().await.len() == 1 }
6797            },
6798            TEST_TIMEOUT,
6799        )
6800        .await;
6801
6802        client.send_text("live".into(), None).await.unwrap();
6803        wait_until_async(
6804            || {
6805                let messages = Arc::clone(&server.messages);
6806                async move { messages.lock().await.len() == 2 }
6807            },
6808            TEST_TIMEOUT,
6809        )
6810        .await;
6811
6812        assert_eq!(server.messages().await, vec!["buffered", "live"]);
6813        assert_eq!(server.connections.load(Ordering::SeqCst), 2);
6814        assert_eq!(
6815            *states.lock(),
6816            vec![
6817                SocketState::Connected,
6818                SocketState::Disconnected,
6819                SocketState::Connected,
6820            ]
6821        );
6822
6823        client.disconnect().await;
6824    }
6825
6826    #[rstest]
6827    #[tokio::test]
6828    async fn test_reconnect_handle_is_closed_after_client_drop() {
6829        let server = RecordingServer::setup().await;
6830        let (handler, _handler_rx) = channel_message_handler();
6831        let client = WebSocketClient::builder()
6832            .config(reconnect_test_config(server.port))
6833            .message_handler(handler)
6834            .connect()
6835            .await
6836            .unwrap();
6837        let tracker = AuthTracker::new();
6838        let pending_auth = tracker.begin();
6839        client.set_auth_tracker(tracker.clone(), true);
6840        let controller_abort = client.controller_task.abort_handle();
6841        let handle = client.reconnect_handle();
6842
6843        drop(client);
6844        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6845
6846        assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
6847        assert_eq!(tracker.auth_state(), AuthState::Failed);
6848        assert_eq!(
6849            tokio::time::timeout(TEST_TIMEOUT, pending_auth)
6850                .await
6851                .expect("client drop should resolve pending authentication")
6852                .expect("authentication sender should report its terminal result"),
6853            Err("WebSocket client closed".to_string())
6854        );
6855    }
6856
6857    #[rstest]
6858    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6859    async fn test_concurrent_drop_closes_accepted_reconnect() {
6860        let server = RecordingServer::setup().await;
6861        let tracker = AuthTracker::new();
6862        let _initial_auth = tracker.begin();
6863        tracker.succeed();
6864        let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
6865        let callback_release_guard = CondvarReleaseGuard::new(callback_release.as_ref());
6866        let callback_release_clone = Arc::clone(&callback_release);
6867        let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
6868        let states = Arc::new(Mutex::new(Vec::new()));
6869        let states_callback = Arc::clone(&states);
6870        let sink = SocketStateSink::new(move |state| {
6871            states_callback.lock().push(state);
6872            if state == SocketState::Disconnected {
6873                callback_entered_tx.send(()).unwrap();
6874                let (lock, condvar) = callback_release_clone.as_ref();
6875                let mut released = lock.lock();
6876                while !*released {
6877                    condvar.wait(&mut released);
6878                }
6879            }
6880        });
6881
6882        let (handler, _handler_rx) = channel_message_handler();
6883        let client = WebSocketClient::builder()
6884            .config(reconnect_test_config(server.port))
6885            .message_handler(handler)
6886            .state_sink(sink)
6887            .connect()
6888            .await
6889            .unwrap();
6890        client.set_auth_tracker(tracker.clone(), true);
6891        let controller_abort = client.controller_task.abort_handle();
6892        let connection_mode = Arc::clone(&client.connection_mode);
6893        let handle = client.reconnect_handle();
6894        let surviving_handle = handle.clone();
6895        let (request_tx, request_rx) = std::sync::mpsc::channel();
6896        let request_thread = std::thread::spawn(move || {
6897            request_tx.send(handle.request_reconnect()).unwrap();
6898        });
6899
6900        recv_rendezvous(callback_entered_rx, "concurrent drop callback entry").await;
6901        drop(client);
6902        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6903        assert_eq!(
6904            ConnectionMode::from_atomic(&connection_mode),
6905            ConnectionMode::Closed
6906        );
6907        assert_eq!(tracker.auth_state(), AuthState::Failed);
6908
6909        callback_release_guard.release();
6910        assert_eq!(
6911            recv_rendezvous(request_rx, "concurrent drop reconnect result").await,
6912            ReconnectRequestOutcome::Accepted
6913        );
6914        request_thread.join().unwrap();
6915        assert_eq!(
6916            surviving_handle.request_reconnect(),
6917            ReconnectRequestOutcome::Closed
6918        );
6919        assert_eq!(
6920            *states.lock(),
6921            vec![SocketState::Connected, SocketState::Disconnected]
6922        );
6923    }
6924
6925    #[rstest]
6926    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
6927    async fn test_reconnect_callback_can_drop_client() {
6928        let server = RecordingServer::setup().await;
6929        let client_slot = Arc::new(Mutex::new(None::<WebSocketClient>));
6930        let client_slot_callback = Arc::clone(&client_slot);
6931        let states = Arc::new(Mutex::new(Vec::new()));
6932        let states_callback = Arc::clone(&states);
6933        let sink = SocketStateSink::new(move |state| {
6934            states_callback.lock().push(state);
6935            if state == SocketState::Disconnected {
6936                drop(client_slot_callback.lock().take());
6937            }
6938        });
6939
6940        let (handler, _handler_rx) = channel_message_handler();
6941        let client = WebSocketClient::builder()
6942            .config(reconnect_test_config(server.port))
6943            .message_handler(handler)
6944            .state_sink(sink)
6945            .connect()
6946            .await
6947            .unwrap();
6948        let tracker = AuthTracker::new();
6949        let _initial_auth = tracker.begin();
6950        tracker.succeed();
6951        client.set_auth_tracker(tracker.clone(), true);
6952        let controller_abort = client.controller_task.abort_handle();
6953        let connection_mode = Arc::clone(&client.connection_mode);
6954        let handle = client.reconnect_handle();
6955        let surviving_handle = handle.clone();
6956        *client_slot.lock() = Some(client);
6957        let (result_tx, result_rx) = std::sync::mpsc::channel();
6958        std::thread::spawn(move || {
6959            result_tx.send(handle.request_reconnect()).unwrap();
6960        });
6961
6962        assert_eq!(
6963            recv_rendezvous(result_rx, "callback drop reconnect result").await,
6964            ReconnectRequestOutcome::Accepted
6965        );
6966        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
6967
6968        assert!(client_slot.lock().is_none());
6969        assert_eq!(
6970            ConnectionMode::from_atomic(&connection_mode),
6971            ConnectionMode::Closed
6972        );
6973        assert_eq!(tracker.auth_state(), AuthState::Failed);
6974        assert_eq!(
6975            surviving_handle.request_reconnect(),
6976            ReconnectRequestOutcome::Closed
6977        );
6978        assert_eq!(
6979            *states.lock(),
6980            vec![SocketState::Connected, SocketState::Disconnected]
6981        );
6982    }
6983
6984    #[rstest]
6985    #[tokio::test]
6986    async fn test_reconnect_then_disconnect() {
6987        // Bind an ephemeral port
6988        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
6989        let port = listener.local_addr().unwrap().port();
6990
6991        // Server task: accept one ws connection then close it
6992        let server = task::spawn(async move {
6993            let (stream, _) = listener.accept().await.unwrap();
6994            let ws = accept_async(stream).await.unwrap();
6995            drop(ws);
6996            // Keep alive briefly
6997            sleep(Duration::from_secs(1)).await;
6998        });
6999
7000        // Build a channel-based message handler for incoming messages (unused here)
7001        let (handler, _rx) = channel_message_handler();
7002
7003        // Configure client with short reconnect backoff
7004        let config = WebSocketConfig {
7005            url: format!("ws://127.0.0.1:{port}"),
7006            headers: vec![],
7007            heartbeat_interval_secs: None,
7008            heartbeat_payload: None,
7009            connect_timeout_ms: Some(1_000),
7010            reconnect_delay_initial_ms: Some(50),
7011            reconnect_delay_max_ms: Some(100),
7012            reconnect_backoff_factor: Some(1.0),
7013            reconnect_jitter_ms: Some(0),
7014            reconnect_max_attempts: None,
7015            heartbeat_timeout_secs: None,
7016            idle_timeout_ms: None,
7017            writer_capacity: None,
7018            backend: TransportBackend::Tungstenite,
7019            proxy_url: None,
7020            max_message_size_bytes: None,
7021            max_frame_size_bytes: None,
7022        };
7023
7024        // Connect the client
7025        let client = WebSocketClient::builder()
7026            .config(config)
7027            .message_handler(handler)
7028            .connect()
7029            .await
7030            .unwrap();
7031
7032        // Allow server to drop connection and client to detect
7033        sleep(Duration::from_millis(100)).await;
7034        // Now immediately disconnect the client
7035        client.disconnect().await;
7036        assert!(client.is_disconnected());
7037        server.abort();
7038    }
7039
7040    #[rstest]
7041    #[tokio::test]
7042    async fn test_reconnect_state_flips_when_reader_stops() {
7043        // Bind an ephemeral port and accept a single websocket connection which we drop.
7044        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7045        let port = listener.local_addr().unwrap().port();
7046        let server = task::spawn(async move {
7047            if let Ok((stream, _)) = listener.accept().await
7048                && let Ok(ws) = accept_async(stream).await
7049            {
7050                drop(ws);
7051            }
7052
7053            sleep(Duration::from_millis(50)).await;
7054        });
7055
7056        let (handler, _rx) = channel_message_handler();
7057
7058        let config = WebSocketConfig {
7059            url: format!("ws://127.0.0.1:{port}"),
7060            headers: vec![],
7061            heartbeat_interval_secs: None,
7062            heartbeat_payload: None,
7063            connect_timeout_ms: Some(1_000),
7064            reconnect_delay_initial_ms: Some(50),
7065            reconnect_delay_max_ms: Some(100),
7066            reconnect_backoff_factor: Some(1.0),
7067            reconnect_jitter_ms: Some(0),
7068            reconnect_max_attempts: None,
7069            heartbeat_timeout_secs: None,
7070            idle_timeout_ms: None,
7071            writer_capacity: None,
7072            backend: TransportBackend::Tungstenite,
7073            proxy_url: None,
7074            max_message_size_bytes: None,
7075            max_frame_size_bytes: None,
7076        };
7077
7078        let client = WebSocketClient::builder()
7079            .config(config)
7080            .message_handler(handler)
7081            .connect()
7082            .await
7083            .unwrap();
7084
7085        tokio::time::timeout(Duration::from_secs(2), async {
7086            loop {
7087                if client.is_reconnecting() {
7088                    break;
7089                }
7090
7091                tokio::time::sleep(Duration::from_millis(10)).await;
7092            }
7093        })
7094        .await
7095        .expect("client did not enter RECONNECT state");
7096
7097        client.disconnect().await;
7098        server.abort();
7099    }
7100
7101    #[rstest]
7102    #[tokio::test]
7103    async fn test_stream_mode_disables_auto_reconnect() {
7104        // Test that stream-based clients do not retain an internal message handler
7105        // and that reconnect() transitions to CLOSED state for stream mode
7106        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7107        let port = listener.local_addr().unwrap().port();
7108        let server = task::spawn(async move {
7109            if let Ok((stream, _)) = listener.accept().await
7110                && let Ok(_ws) = accept_async(stream).await
7111            {
7112                // Keep connection alive briefly
7113                sleep(Duration::from_millis(100)).await;
7114            }
7115        });
7116
7117        let config = WebSocketConfig {
7118            url: format!("ws://127.0.0.1:{port}"),
7119            headers: vec![],
7120            heartbeat_interval_secs: None,
7121            heartbeat_payload: None,
7122            connect_timeout_ms: Some(1_000),
7123            reconnect_delay_initial_ms: Some(50),
7124            reconnect_delay_max_ms: Some(100),
7125            reconnect_backoff_factor: Some(1.0),
7126            reconnect_jitter_ms: Some(0),
7127            reconnect_max_attempts: None,
7128            heartbeat_timeout_secs: None,
7129            idle_timeout_ms: None,
7130            writer_capacity: None,
7131            backend: TransportBackend::Tungstenite,
7132            proxy_url: None,
7133            max_message_size_bytes: None,
7134            max_frame_size_bytes: None,
7135        };
7136
7137        let (_reader, _client) = WebSocketClient::stream_builder()
7138            .config(config)
7139            .connect()
7140            .await
7141            .unwrap();
7142
7143        // Note: We can't easily test the reconnect behavior from the outside since
7144        // the inner client is private. The key fix is that WebSocketClientInner
7145        // now has no internal handler in stream mode, and reconnect() will
7146        // transition to CLOSED state instead of creating a new reader that gets dropped.
7147        // This is tested implicitly by the fact that stream users won't get stuck
7148        // in an infinite reconnect loop.
7149
7150        server.abort();
7151    }
7152
7153    #[rstest]
7154    #[tokio::test]
7155    async fn test_message_handler_mode_allows_auto_reconnect() {
7156        // Test that regular clients (with message handler) can auto-reconnect
7157        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7158        let port = listener.local_addr().unwrap().port();
7159
7160        let server = task::spawn(async move {
7161            // Accept first connection and close it
7162            if let Ok((stream, _)) = listener.accept().await
7163                && let Ok(ws) = accept_async(stream).await
7164            {
7165                drop(ws);
7166            }
7167
7168            sleep(Duration::from_millis(50)).await;
7169        });
7170
7171        let (handler, _rx) = channel_message_handler();
7172
7173        let config = WebSocketConfig {
7174            url: format!("ws://127.0.0.1:{port}"),
7175            headers: vec![],
7176            heartbeat_interval_secs: None,
7177            heartbeat_payload: None,
7178            connect_timeout_ms: Some(1_000),
7179            reconnect_delay_initial_ms: Some(50),
7180            reconnect_delay_max_ms: Some(100),
7181            reconnect_backoff_factor: Some(1.0),
7182            reconnect_jitter_ms: Some(0),
7183            reconnect_max_attempts: None,
7184            heartbeat_timeout_secs: None,
7185            idle_timeout_ms: None,
7186            writer_capacity: None,
7187            backend: TransportBackend::Tungstenite,
7188            proxy_url: None,
7189            max_message_size_bytes: None,
7190            max_frame_size_bytes: None,
7191        };
7192
7193        let client = WebSocketClient::builder()
7194            .config(config)
7195            .message_handler(handler)
7196            .connect()
7197            .await
7198            .unwrap();
7199
7200        // Wait for the connection to be dropped and reconnection to be attempted
7201        tokio::time::timeout(Duration::from_secs(2), async {
7202            loop {
7203                if client.is_reconnecting() || client.is_closed() {
7204                    break;
7205                }
7206
7207                tokio::time::sleep(Duration::from_millis(10)).await;
7208            }
7209        })
7210        .await
7211        .expect("client should attempt reconnection or close");
7212
7213        // Should either be reconnecting or closed (depending on timing)
7214        // The important thing is it's not staying active forever
7215        assert!(
7216            client.is_reconnecting() || client.is_closed(),
7217            "Client with message handler should attempt reconnection"
7218        );
7219
7220        client.disconnect().await;
7221        server.abort();
7222    }
7223
7224    #[rstest]
7225    #[tokio::test]
7226    async fn test_handler_mode_reconnect_with_new_connection() {
7227        // Test that handler mode successfully reconnects and messages continue flowing
7228        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7229        let port = listener.local_addr().unwrap().port();
7230
7231        let server = task::spawn(async move {
7232            // First connection - accept and immediately close
7233            if let Ok((stream, _)) = listener.accept().await
7234                && let Ok(ws) = accept_async(stream).await
7235            {
7236                drop(ws);
7237            }
7238
7239            // Small delay to let client detect disconnection
7240            sleep(Duration::from_millis(100)).await;
7241
7242            // Second connection - accept, send a message, then keep alive
7243            if let Ok((stream, _)) = listener.accept().await
7244                && let Ok(mut ws) = accept_async(stream).await
7245            {
7246                use futures_util::SinkExt;
7247                let _ = ws
7248                    .send(WsMessage::Text("reconnected".to_string().into()))
7249                    .await;
7250                sleep(Duration::from_secs(1)).await;
7251            }
7252        });
7253
7254        let (handler, mut rx) = channel_message_handler();
7255
7256        let config = WebSocketConfig {
7257            url: format!("ws://127.0.0.1:{port}"),
7258            headers: vec![],
7259            heartbeat_interval_secs: None,
7260            heartbeat_payload: None,
7261            connect_timeout_ms: Some(2_000),
7262            reconnect_delay_initial_ms: Some(50),
7263            reconnect_delay_max_ms: Some(200),
7264            reconnect_backoff_factor: Some(1.5),
7265            reconnect_jitter_ms: Some(10),
7266            reconnect_max_attempts: None,
7267            heartbeat_timeout_secs: None,
7268            idle_timeout_ms: None,
7269            writer_capacity: None,
7270            backend: TransportBackend::Tungstenite,
7271            proxy_url: None,
7272            max_message_size_bytes: None,
7273            max_frame_size_bytes: None,
7274        };
7275
7276        let client = WebSocketClient::builder()
7277            .config(config)
7278            .message_handler(handler)
7279            .connect()
7280            .await
7281            .unwrap();
7282
7283        // Wait for reconnection to happen and message to arrive
7284        let result = tokio::time::timeout(Duration::from_secs(5), async {
7285            loop {
7286                if let Ok(msg) = rx.try_recv()
7287                    && matches!(msg, WsMessage::Text(ref text) if AsRef::<str>::as_ref(text) == "reconnected")
7288                {
7289                    return true;
7290                }
7291
7292                tokio::time::sleep(Duration::from_millis(10)).await;
7293            }
7294        })
7295        .await;
7296
7297        assert!(
7298            result.is_ok(),
7299            "Should receive message after reconnection within timeout"
7300        );
7301
7302        client.disconnect().await;
7303        server.abort();
7304    }
7305
7306    #[rstest]
7307    #[tokio::test]
7308    async fn test_stream_mode_no_auto_reconnect() {
7309        // Test that stream mode does not automatically reconnect when connection is lost
7310        // The caller owns the reader and is responsible for detecting disconnection
7311        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7312        let port = listener.local_addr().unwrap().port();
7313
7314        let server = task::spawn(async move {
7315            // Accept connection and send one message, then close
7316            if let Ok((stream, _)) = listener.accept().await
7317                && let Ok(mut ws) = accept_async(stream).await
7318            {
7319                use futures_util::SinkExt;
7320                let _ = ws.send(WsMessage::Text("hello".to_string().into())).await;
7321                sleep(Duration::from_millis(50)).await;
7322                // Connection closes when ws is dropped
7323            }
7324        });
7325
7326        let config = WebSocketConfig {
7327            url: format!("ws://127.0.0.1:{port}"),
7328            headers: vec![],
7329            heartbeat_interval_secs: None,
7330            heartbeat_payload: None,
7331            connect_timeout_ms: Some(1_000),
7332            reconnect_delay_initial_ms: Some(50),
7333            reconnect_delay_max_ms: Some(100),
7334            reconnect_backoff_factor: Some(1.0),
7335            reconnect_jitter_ms: Some(0),
7336            reconnect_max_attempts: None,
7337            heartbeat_timeout_secs: None,
7338            idle_timeout_ms: None,
7339            writer_capacity: None,
7340            backend: TransportBackend::Tungstenite,
7341            proxy_url: None,
7342            max_message_size_bytes: None,
7343            max_frame_size_bytes: None,
7344        };
7345
7346        let (mut reader, client) = WebSocketClient::stream_builder()
7347            .config(config)
7348            .connect()
7349            .await
7350            .unwrap();
7351
7352        // Initially active
7353        assert!(client.is_active(), "Client should start as active");
7354
7355        // Read the hello message
7356        let msg = reader.next().await;
7357        assert!(
7358            matches!(&msg, Some(Ok(Message::Text(bytes))) if bytes.as_ref() == b"hello"),
7359            "Should receive initial message"
7360        );
7361
7362        // Read until connection closes (reader will return None or error)
7363        while let Some(msg) = reader.next().await {
7364            if msg.is_err() || matches!(msg, Ok(Message::Close(_))) {
7365                break;
7366            }
7367        }
7368
7369        // Controller cannot detect reader EOF (reader is owned by caller),
7370        // so the client stays ACTIVE until the caller signals.
7371        sleep(Duration::from_millis(200)).await;
7372        assert!(
7373            client.is_active(),
7374            "Stream mode client stays ACTIVE before notify_closed()"
7375        );
7376
7377        // Caller signals EOF via notify_closed()
7378        client.notify_closed();
7379
7380        assert!(
7381            client.is_closed(),
7382            "Stream mode client should be CLOSED after notify_closed()"
7383        );
7384        assert!(
7385            !client.is_reconnecting(),
7386            "Stream mode client should never attempt reconnection"
7387        );
7388
7389        client.disconnect().await;
7390        server.abort();
7391    }
7392
7393    #[rstest]
7394    #[tokio::test]
7395    async fn test_send_timeout_uses_configured_connect_timeout() {
7396        // Test that send operations respect the configured connect_timeout.
7397        // When a client is stuck in RECONNECT longer than the timeout, sends should fail with Timeout.
7398        use nautilus_common::testing::wait_until_async;
7399
7400        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7401        let port = listener.local_addr().unwrap().port();
7402
7403        let server = task::spawn(async move {
7404            // Accept first connection and immediately close it
7405            if let Ok((stream, _)) = listener.accept().await
7406                && let Ok(ws) = accept_async(stream).await
7407            {
7408                drop(ws);
7409            }
7410
7411            // Don't accept second connection - client will be stuck in RECONNECT
7412            sleep(Duration::from_mins(1)).await;
7413        });
7414
7415        let (handler, _rx) = channel_message_handler();
7416
7417        // Configure with SHORT 2s reconnect timeout
7418        let config = WebSocketConfig {
7419            url: format!("ws://127.0.0.1:{port}"),
7420            headers: vec![],
7421            heartbeat_interval_secs: None,
7422            heartbeat_payload: None,
7423            connect_timeout_ms: Some(2_000), // 2s timeout
7424            reconnect_delay_initial_ms: Some(50),
7425            reconnect_delay_max_ms: Some(100),
7426            reconnect_backoff_factor: Some(1.0),
7427            reconnect_jitter_ms: Some(0),
7428            reconnect_max_attempts: None,
7429            heartbeat_timeout_secs: None,
7430            idle_timeout_ms: None,
7431            writer_capacity: None,
7432            backend: TransportBackend::Tungstenite,
7433            proxy_url: None,
7434            max_message_size_bytes: None,
7435            max_frame_size_bytes: None,
7436        };
7437
7438        let client = WebSocketClient::builder()
7439            .config(config)
7440            .message_handler(handler)
7441            .connect()
7442            .await
7443            .unwrap();
7444
7445        // Wait for client to enter RECONNECT state
7446        wait_until_async(
7447            || async { client.is_reconnecting() },
7448            Duration::from_secs(3),
7449        )
7450        .await;
7451
7452        // Attempt send while stuck in RECONNECT - should timeout after 2s (configured timeout)
7453        let start = std::time::Instant::now();
7454        let send_result = client.send_text("test".to_string(), None).await;
7455        let elapsed = start.elapsed();
7456
7457        assert!(
7458            send_result.is_err(),
7459            "Send should fail when client stuck in RECONNECT"
7460        );
7461        assert!(
7462            matches!(send_result, Err(crate::error::SendError::Timeout)),
7463            "Send should return Timeout error, was: {send_result:?}"
7464        );
7465        // Verify timeout respects configured value (2s), but don't check upper bound
7466        // as CI scheduler jitter can cause legitimate delays beyond the timeout
7467        assert!(
7468            elapsed >= Duration::from_millis(1800),
7469            "Send should timeout after at least 2s (configured timeout), took {elapsed:?}"
7470        );
7471
7472        client.disconnect().await;
7473        server.abort();
7474    }
7475
7476    #[rstest]
7477    #[tokio::test]
7478    async fn test_send_waits_during_reconnection() {
7479        // Test that send operations wait for reconnection to complete (up to timeout)
7480        use nautilus_common::testing::wait_until_async;
7481
7482        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7483        let port = listener.local_addr().unwrap().port();
7484
7485        let server = task::spawn(async move {
7486            // First connection - accept and immediately close
7487            if let Ok((stream, _)) = listener.accept().await
7488                && let Ok(ws) = accept_async(stream).await
7489            {
7490                drop(ws);
7491            }
7492
7493            // Wait a bit before accepting second connection
7494            sleep(Duration::from_millis(500)).await;
7495
7496            // Second connection - accept and keep alive
7497            if let Ok((stream, _)) = listener.accept().await
7498                && let Ok(mut ws) = accept_async(stream).await
7499            {
7500                // Echo messages
7501                while let Some(Ok(msg)) = ws.next().await {
7502                    if ws.send(msg).await.is_err() {
7503                        break;
7504                    }
7505                }
7506            }
7507        });
7508
7509        let (handler, _rx) = channel_message_handler();
7510
7511        let config = WebSocketConfig {
7512            url: format!("ws://127.0.0.1:{port}"),
7513            headers: vec![],
7514            heartbeat_interval_secs: None,
7515            heartbeat_payload: None,
7516            connect_timeout_ms: Some(5_000), // 5s timeout - enough for reconnect
7517            reconnect_delay_initial_ms: Some(100),
7518            reconnect_delay_max_ms: Some(200),
7519            reconnect_backoff_factor: Some(1.0),
7520            reconnect_jitter_ms: Some(0),
7521            reconnect_max_attempts: None,
7522            heartbeat_timeout_secs: None,
7523            idle_timeout_ms: None,
7524            writer_capacity: None,
7525            backend: TransportBackend::Tungstenite,
7526            proxy_url: None,
7527            max_message_size_bytes: None,
7528            max_frame_size_bytes: None,
7529        };
7530
7531        let client = WebSocketClient::builder()
7532            .config(config)
7533            .message_handler(handler)
7534            .connect()
7535            .await
7536            .unwrap();
7537
7538        // Wait for reconnection to trigger
7539        wait_until_async(
7540            || async { client.is_reconnecting() },
7541            Duration::from_secs(2),
7542        )
7543        .await;
7544
7545        // Try to send while reconnecting - should wait and succeed after reconnect
7546        let send_result = tokio::time::timeout(
7547            Duration::from_secs(3),
7548            client.send_text("test_message".to_string(), None),
7549        )
7550        .await;
7551
7552        assert!(
7553            send_result.is_ok() && send_result.unwrap().is_ok(),
7554            "Send should succeed after waiting for reconnection"
7555        );
7556
7557        client.disconnect().await;
7558        server.abort();
7559    }
7560
7561    #[rstest]
7562    #[tokio::test]
7563    async fn test_rate_limiter_before_active_wait() {
7564        // Test that rate limiting happens BEFORE active state check.
7565        // This prevents race conditions where connection state changes during rate limit wait.
7566        // We verify this by: (1) exhausting rate limit, (2) ensuring client is RECONNECTING,
7567        // (3) sending again and confirming it waits for rate limit THEN reconnection.
7568        use std::{num::NonZeroU32, sync::Arc};
7569
7570        use nautilus_common::testing::wait_until_async;
7571
7572        use crate::ratelimiter::quota::Quota;
7573
7574        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7575        let port = listener.local_addr().unwrap().port();
7576
7577        let server = task::spawn(async move {
7578            // First connection - accept and close after receiving one message
7579            if let Ok((stream, _)) = listener.accept().await
7580                && let Ok(mut ws) = accept_async(stream).await
7581            {
7582                // Receive first message then close
7583                if let Some(Ok(_)) = ws.next().await {
7584                    drop(ws);
7585                }
7586            }
7587
7588            // Wait before accepting reconnection
7589            sleep(Duration::from_millis(500)).await;
7590
7591            // Second connection - accept and keep alive
7592            if let Ok((stream, _)) = listener.accept().await
7593                && let Ok(mut ws) = accept_async(stream).await
7594            {
7595                while let Some(Ok(msg)) = ws.next().await {
7596                    if ws.send(msg).await.is_err() {
7597                        break;
7598                    }
7599                }
7600            }
7601        });
7602
7603        let (handler, _rx) = channel_message_handler();
7604
7605        let config = WebSocketConfig {
7606            url: format!("ws://127.0.0.1:{port}"),
7607            headers: vec![],
7608            heartbeat_interval_secs: None,
7609            heartbeat_payload: None,
7610            connect_timeout_ms: Some(5_000),
7611            reconnect_delay_initial_ms: Some(50),
7612            reconnect_delay_max_ms: Some(100),
7613            reconnect_backoff_factor: Some(1.0),
7614            reconnect_jitter_ms: Some(0),
7615            reconnect_max_attempts: None,
7616            heartbeat_timeout_secs: None,
7617            idle_timeout_ms: None,
7618            writer_capacity: None,
7619            backend: TransportBackend::Tungstenite,
7620            proxy_url: None,
7621            max_message_size_bytes: None,
7622            max_frame_size_bytes: None,
7623        };
7624
7625        // Very restrictive rate limit: 1 request per second, burst of 1
7626        let quota = Quota::per_second(NonZeroU32::new(1).unwrap())
7627            .unwrap()
7628            .allow_burst(NonZeroU32::new(1).unwrap());
7629
7630        let client = Arc::new(
7631            WebSocketClient::builder()
7632                .config(config)
7633                .message_handler(handler)
7634                .keyed_quotas(vec![("test_key".to_string(), quota)])
7635                .connect()
7636                .await
7637                .unwrap(),
7638        );
7639
7640        // First send exhausts burst capacity and triggers connection close
7641        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7642        client
7643            .send_text("msg1".to_string(), Some(test_key.as_slice()))
7644            .await
7645            .unwrap();
7646
7647        // Wait for client to enter RECONNECT state
7648        wait_until_async(
7649            || async { client.is_reconnecting() },
7650            Duration::from_secs(2),
7651        )
7652        .await;
7653
7654        // Second send: will hit rate limit (~1s) THEN wait for reconnection (~0.5s)
7655        let start = std::time::Instant::now();
7656        let send_result = client
7657            .send_text("msg2".to_string(), Some(test_key.as_slice()))
7658            .await;
7659        let elapsed = start.elapsed();
7660
7661        // Should succeed after both rate limit AND reconnection
7662        assert!(
7663            send_result.is_ok(),
7664            "Send should succeed after rate limit + reconnection, was: {send_result:?}"
7665        );
7666        // Total wait should be at least rate limit time (~1s)
7667        // The reconnection completes while rate limiting or after
7668        // Use 850ms threshold to account for timing jitter in CI
7669        assert!(
7670            elapsed >= Duration::from_millis(850),
7671            "Should wait for rate limit (~1s), waited {elapsed:?}"
7672        );
7673
7674        client.disconnect().await;
7675        server.abort();
7676    }
7677
7678    #[rstest]
7679    #[tokio::test]
7680    async fn test_disconnect_during_reconnect_exits_cleanly() {
7681        // Test CAS race condition: disconnect called during reconnection
7682        // Should exit cleanly without spawning new tasks
7683        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7684        let port = listener.local_addr().unwrap().port();
7685
7686        let server = task::spawn(async move {
7687            // Accept first connection and immediately close
7688            if let Ok((stream, _)) = listener.accept().await
7689                && let Ok(ws) = accept_async(stream).await
7690            {
7691                drop(ws);
7692            }
7693
7694            // Don't accept second connection - let reconnect hang
7695            sleep(Duration::from_mins(1)).await;
7696        });
7697
7698        let (handler, _rx) = channel_message_handler();
7699
7700        let config = WebSocketConfig {
7701            url: format!("ws://127.0.0.1:{port}"),
7702            headers: vec![],
7703            heartbeat_interval_secs: None,
7704            heartbeat_payload: None,
7705            connect_timeout_ms: Some(2_000), // 2s timeout - shorter than disconnect timeout
7706            reconnect_delay_initial_ms: Some(100),
7707            reconnect_delay_max_ms: Some(200),
7708            reconnect_backoff_factor: Some(1.0),
7709            reconnect_jitter_ms: Some(0),
7710            reconnect_max_attempts: None,
7711            heartbeat_timeout_secs: None,
7712            idle_timeout_ms: None,
7713            writer_capacity: None,
7714            backend: TransportBackend::Tungstenite,
7715            proxy_url: None,
7716            max_message_size_bytes: None,
7717            max_frame_size_bytes: None,
7718        };
7719
7720        let client = WebSocketClient::builder()
7721            .config(config)
7722            .message_handler(handler)
7723            .connect()
7724            .await
7725            .unwrap();
7726
7727        // Wait for reconnection to start
7728        tokio::time::timeout(Duration::from_secs(2), async {
7729            while !client.is_reconnecting() {
7730                sleep(Duration::from_millis(10)).await;
7731            }
7732        })
7733        .await
7734        .expect("Client should enter RECONNECT state");
7735
7736        // Disconnect while reconnecting
7737        client.disconnect().await;
7738
7739        // Should be cleanly closed
7740        assert!(
7741            client.is_disconnected(),
7742            "Client should be cleanly disconnected"
7743        );
7744
7745        server.abort();
7746    }
7747
7748    #[rstest]
7749    #[tokio::test]
7750    async fn test_send_fails_fast_when_closed_before_rate_limit() {
7751        // Test that send operations check connection state BEFORE rate limiting,
7752        // preventing unnecessary delays when the connection is already closed.
7753        use std::{num::NonZeroU32, sync::Arc};
7754
7755        use nautilus_common::testing::wait_until_async;
7756
7757        use crate::ratelimiter::quota::Quota;
7758
7759        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7760        let port = listener.local_addr().unwrap().port();
7761
7762        let server = task::spawn(async move {
7763            // Accept connection and immediately close
7764            if let Ok((stream, _)) = listener.accept().await
7765                && let Ok(ws) = accept_async(stream).await
7766            {
7767                drop(ws);
7768            }
7769
7770            sleep(Duration::from_mins(1)).await;
7771        });
7772
7773        let (handler, _rx) = channel_message_handler();
7774
7775        let config = WebSocketConfig {
7776            url: format!("ws://127.0.0.1:{port}"),
7777            headers: vec![],
7778            heartbeat_interval_secs: None,
7779            heartbeat_payload: None,
7780            connect_timeout_ms: Some(5_000),
7781            reconnect_delay_initial_ms: Some(50),
7782            reconnect_delay_max_ms: Some(100),
7783            reconnect_backoff_factor: Some(1.0),
7784            reconnect_jitter_ms: Some(0),
7785            reconnect_max_attempts: None,
7786            heartbeat_timeout_secs: None,
7787            idle_timeout_ms: None,
7788            writer_capacity: None,
7789            backend: TransportBackend::Tungstenite,
7790            proxy_url: None,
7791            max_message_size_bytes: None,
7792            max_frame_size_bytes: None,
7793        };
7794
7795        // Very restrictive rate limit: 1 request per 10 seconds
7796        // This ensures that if we wait for rate limit, the test will timeout
7797        let quota = Quota::with_period(Duration::from_secs(10))
7798            .unwrap()
7799            .allow_burst(NonZeroU32::new(1).unwrap());
7800
7801        let client = Arc::new(
7802            WebSocketClient::builder()
7803                .config(config)
7804                .message_handler(handler)
7805                .keyed_quotas(vec![("test_key".to_string(), quota)])
7806                .connect()
7807                .await
7808                .unwrap(),
7809        );
7810
7811        // Wait for disconnection
7812        wait_until_async(
7813            || async { client.is_reconnecting() || client.is_closed() },
7814            Duration::from_secs(2),
7815        )
7816        .await;
7817
7818        // Explicitly disconnect to move away from ACTIVE state
7819        client.disconnect().await;
7820        assert!(
7821            !client.is_active(),
7822            "Client should not be active after disconnect"
7823        );
7824
7825        // Attempt send - should fail IMMEDIATELY without waiting for rate limit
7826        let start = std::time::Instant::now();
7827        let test_key: [Ustr; 1] = [Ustr::from("test_key")];
7828        let result = client
7829            .send_text("test".to_string(), Some(test_key.as_slice()))
7830            .await;
7831        let elapsed = start.elapsed();
7832
7833        // Should fail with Closed error
7834        assert!(result.is_err(), "Send should fail when client is closed");
7835        assert!(
7836            matches!(result, Err(crate::error::SendError::Closed)),
7837            "Send should return Closed error, was: {result:?}"
7838        );
7839
7840        // Should fail FAST (< 100ms) without waiting for rate limit (10s)
7841        assert!(
7842            elapsed < Duration::from_millis(100),
7843            "Send should fail fast without rate limiting, took {elapsed:?}"
7844        );
7845
7846        server.abort();
7847    }
7848
7849    #[rstest]
7850    #[tokio::test]
7851    async fn test_builder_rejects_shared_rate_limiter_with_quotas() {
7852        let (handler, _rx) = channel_message_handler();
7853        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7854        let result = WebSocketClient::builder()
7855            .config(reconnect_test_config(1))
7856            .message_handler(handler)
7857            .default_quota(Quota::with_period(Duration::from_secs(1)).unwrap())
7858            .rate_limiter(rate_limiter)
7859            .connect()
7860            .await;
7861
7862        assert_eq!(
7863            result.unwrap_err().to_string(),
7864            "I/O error: Cannot combine a shared rate limiter with quota configuration"
7865        );
7866    }
7867
7868    #[rstest]
7869    #[tokio::test]
7870    async fn test_builder_rejects_connection_keys_without_limiter() {
7871        let (handler, _rx) = channel_message_handler();
7872        let result = WebSocketClient::builder()
7873            .config(reconnect_test_config(1))
7874            .message_handler(handler)
7875            .connection_rate_keys(Arc::from([Ustr::from("connect")]))
7876            .connect()
7877            .await;
7878
7879        assert_eq!(
7880            result.unwrap_err().to_string(),
7881            "I/O error: Connection rate keys require a connection rate limiter"
7882        );
7883    }
7884
7885    #[rstest]
7886    #[tokio::test]
7887    async fn test_builder_rejects_connection_limiter_without_keys() {
7888        let (handler, _rx) = channel_message_handler();
7889        let rate_limiter = Arc::new(RateLimiter::new_with_quota(None, Vec::new()));
7890        let result = WebSocketClient::builder()
7891            .config(reconnect_test_config(1))
7892            .message_handler(handler)
7893            .connection_rate_limiter(rate_limiter)
7894            .connect()
7895            .await;
7896
7897        assert_eq!(
7898            result.unwrap_err().to_string(),
7899            "I/O error: Connection rate limiter requires at least one connection rate key"
7900        );
7901    }
7902
7903    #[rstest]
7904    #[tokio::test]
7905    async fn test_epoch_builder_rejects_both_ping_handler_types() {
7906        let (handler, _rx) = channel_epoch_message_handler();
7907        let ping_handler: PingHandler = Arc::new(|_| {});
7908        let epoch_ping_handler: EpochPingHandler = Arc::new(|_, _| {});
7909        let result = WebSocketClient::epoch_builder()
7910            .config(reconnect_test_config(1))
7911            .epoch_handler(handler)
7912            .ping_handler(ping_handler)
7913            .epoch_ping_handler(epoch_ping_handler)
7914            .connect()
7915            .await;
7916
7917        assert_eq!(
7918            result.unwrap_err().to_string(),
7919            "I/O error: Cannot configure both ping_handler and epoch_ping_handler"
7920        );
7921    }
7922
7923    #[rstest]
7924    #[tokio::test]
7925    async fn test_connect_url_rejects_invalid_reconnect_timing_before_connect() {
7926        let (handler, _rx) = channel_message_handler();
7927
7928        let config = WebSocketConfig {
7929            url: "ws://127.0.0.1:1".to_string(),
7930            headers: vec![],
7931            heartbeat_interval_secs: None,
7932            heartbeat_payload: None,
7933            connect_timeout_ms: Some(0),
7934            reconnect_delay_initial_ms: Some(100),
7935            reconnect_delay_max_ms: Some(500),
7936            reconnect_backoff_factor: Some(1.5),
7937            reconnect_jitter_ms: Some(0),
7938            reconnect_max_attempts: None,
7939            heartbeat_timeout_secs: None,
7940            idle_timeout_ms: None,
7941            writer_capacity: None,
7942            backend: TransportBackend::Tungstenite,
7943            proxy_url: None,
7944            max_message_size_bytes: None,
7945            max_frame_size_bytes: None,
7946        };
7947
7948        let err = WebSocketClientInner::connect_url(config, Some(handler), None)
7949            .await
7950            .expect_err("invalid reconnect timing should be rejected");
7951
7952        match err {
7953            TransportError::Io(error) => {
7954                assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
7955                assert!(
7956                    error.to_string().contains("connect_timeout_ms"),
7957                    "error should mention zero reconnect timeout, was: {error}"
7958                );
7959            }
7960            other => panic!("expected InvalidInput IO error, was: {other:?}"),
7961        }
7962    }
7963
7964    #[rstest]
7965    #[tokio::test]
7966    async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
7967        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
7968        let port = listener.local_addr().unwrap().port();
7969        let accepted = Arc::new(std::sync::atomic::AtomicBool::new(false));
7970        let accepted_clone = Arc::clone(&accepted);
7971
7972        let server = task::spawn(async move {
7973            let (stream, _) = listener.accept().await.unwrap();
7974            accepted_clone.store(true, Ordering::SeqCst);
7975            accept_async(stream).await.unwrap();
7976        });
7977
7978        let (handler, _rx) = channel_message_handler();
7979
7980        let config = WebSocketConfig {
7981            url: format!("ws://127.0.0.1:{port}"),
7982            headers: vec![],
7983            heartbeat_interval_secs: None,
7984            heartbeat_payload: None,
7985            connect_timeout_ms: Some(1_000),
7986            reconnect_delay_initial_ms: Some(50),
7987            reconnect_delay_max_ms: Some(100),
7988            reconnect_backoff_factor: Some(100.1),
7989            reconnect_jitter_ms: Some(0),
7990            reconnect_max_attempts: None,
7991            heartbeat_timeout_secs: None,
7992            idle_timeout_ms: None,
7993            writer_capacity: None,
7994            backend: TransportBackend::Tungstenite,
7995            proxy_url: None,
7996            max_message_size_bytes: None,
7997            max_frame_size_bytes: None,
7998        };
7999
8000        let error = WebSocketClientInner::connect_url(config, Some(handler), None)
8001            .await
8002            .expect_err("invalid reconnect backoff should be rejected");
8003
8004        match error {
8005            TransportError::Io(error) => {
8006                assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
8007                assert!(
8008                    error.to_string().contains("factor"),
8009                    "error should mention the invalid factor, was: {error}"
8010                );
8011            }
8012            other => panic!("expected InvalidInput IO error, was: {other:?}"),
8013        }
8014
8015        assert!(
8016            !accepted.load(Ordering::SeqCst),
8017            "invalid reconnect backoff must be rejected before connecting"
8018        );
8019        server.abort();
8020    }
8021
8022    #[rstest]
8023    #[tokio::test]
8024    async fn test_client_without_handler_sets_stream_mode() {
8025        // Test that if a client is created without a handler via connect_url,
8026        // it keeps the internal handler empty to prevent zombie connections
8027
8028        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8029        let port = listener.local_addr().unwrap().port();
8030
8031        let server = task::spawn(async move {
8032            // Accept and immediately close to simulate server disconnect
8033            if let Ok((stream, _)) = listener.accept().await
8034                && let Ok(ws) = accept_async(stream).await
8035            {
8036                drop(ws); // Drop connection immediately
8037            }
8038        });
8039
8040        let config = WebSocketConfig {
8041            url: format!("ws://127.0.0.1:{port}"),
8042            headers: vec![],
8043            heartbeat_interval_secs: None,
8044            heartbeat_payload: None,
8045            connect_timeout_ms: Some(1_000),
8046            reconnect_delay_initial_ms: Some(100),
8047            reconnect_delay_max_ms: Some(500),
8048            reconnect_backoff_factor: Some(1.5),
8049            reconnect_jitter_ms: Some(0),
8050            reconnect_max_attempts: None,
8051            heartbeat_timeout_secs: None,
8052            idle_timeout_ms: None,
8053            writer_capacity: None,
8054            backend: TransportBackend::Tungstenite,
8055            proxy_url: None,
8056            max_message_size_bytes: None,
8057            max_frame_size_bytes: None,
8058        };
8059
8060        // Create client directly via connect_url with no handler (stream mode)
8061        let inner = WebSocketClientInner::connect_url(config, None, None)
8062            .await
8063            .unwrap();
8064
8065        // Verify stream mode does not retain an internal handler
8066        assert!(
8067            inner.handler.is_none(),
8068            "Client without handler should not retain an internal handler"
8069        );
8070
8071        // Verify that when stream mode is enabled, reconnection is disabled
8072        // (documented behavior - stream mode clients close instead of reconnecting)
8073
8074        server.abort();
8075    }
8076
8077    #[rstest]
8078    #[tokio::test]
8079    async fn test_idle_timeout_triggers_reconnect() {
8080        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8081        let port = listener.local_addr().unwrap().port();
8082
8083        // Server accepts WS connection but sends nothing (simulates silent death)
8084        let server = task::spawn(async move {
8085            let (stream, _) = listener.accept().await.unwrap();
8086            let _ws = accept_async(stream).await.unwrap();
8087            // Hold connection open but send nothing
8088            sleep(Duration::from_secs(5)).await;
8089        });
8090
8091        let (handler, _rx) = channel_message_handler();
8092
8093        let config = WebSocketConfig {
8094            url: format!("ws://127.0.0.1:{port}"),
8095            headers: vec![],
8096            heartbeat_interval_secs: None,
8097            heartbeat_payload: None,
8098            connect_timeout_ms: Some(2_000),
8099            reconnect_delay_initial_ms: Some(50),
8100            reconnect_delay_max_ms: Some(100),
8101            reconnect_backoff_factor: Some(1.0),
8102            reconnect_jitter_ms: Some(0),
8103            reconnect_max_attempts: Some(1),
8104            heartbeat_timeout_secs: None,
8105            idle_timeout_ms: Some(500),
8106            writer_capacity: None,
8107            backend: TransportBackend::Tungstenite,
8108            proxy_url: None,
8109            max_message_size_bytes: None,
8110            max_frame_size_bytes: None,
8111        };
8112
8113        let client = WebSocketClient::builder()
8114            .config(config)
8115            .message_handler(handler)
8116            .connect()
8117            .await
8118            .unwrap();
8119
8120        assert!(client.is_active());
8121
8122        // Wait for idle timeout to fire and client to enter reconnect/closed
8123        wait_until_async(
8124            || async { client.is_reconnecting() || client.is_disconnected() },
8125            Duration::from_secs(3),
8126        )
8127        .await;
8128
8129        assert!(
8130            !client.is_active(),
8131            "Client should not be active after idle timeout"
8132        );
8133
8134        client.disconnect().await;
8135        server.abort();
8136    }
8137
8138    #[rstest]
8139    #[tokio::test]
8140    async fn test_idle_timeout_resets_on_data() {
8141        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8142        let port = listener.local_addr().unwrap().port();
8143
8144        // Server sends a message every 200ms (well within 1s idle timeout)
8145        let server = task::spawn(async move {
8146            let (stream, _) = listener.accept().await.unwrap();
8147            let mut ws = accept_async(stream).await.unwrap();
8148
8149            for _ in 0..10 {
8150                sleep(Duration::from_millis(200)).await;
8151
8152                if ws.send(WsMessage::Text("ping".into())).await.is_err() {
8153                    break;
8154                }
8155            }
8156        });
8157
8158        let (handler, _rx) = channel_message_handler();
8159
8160        let config = WebSocketConfig {
8161            url: format!("ws://127.0.0.1:{port}"),
8162            headers: vec![],
8163            heartbeat_interval_secs: None,
8164            heartbeat_payload: None,
8165            connect_timeout_ms: Some(2_000),
8166            reconnect_delay_initial_ms: Some(50),
8167            reconnect_delay_max_ms: Some(100),
8168            reconnect_backoff_factor: Some(1.0),
8169            reconnect_jitter_ms: Some(0),
8170            reconnect_max_attempts: Some(1),
8171            heartbeat_timeout_secs: None,
8172            idle_timeout_ms: Some(1_000),
8173            writer_capacity: None,
8174            backend: TransportBackend::Tungstenite,
8175            proxy_url: None,
8176            max_message_size_bytes: None,
8177            max_frame_size_bytes: None,
8178        };
8179
8180        let client = WebSocketClient::builder()
8181            .config(config)
8182            .message_handler(handler)
8183            .connect()
8184            .await
8185            .unwrap();
8186
8187        assert!(client.is_active());
8188
8189        // Wait 1.5s - data arrives every 200ms so idle timeout (1s) should NOT fire
8190        sleep(Duration::from_millis(1_500)).await;
8191
8192        assert!(
8193            client.is_active(),
8194            "Client should remain active when data is flowing"
8195        );
8196
8197        client.disconnect().await;
8198        server.abort();
8199    }
8200
8201    #[rstest]
8202    #[tokio::test]
8203    async fn test_idle_timeout_fires_when_only_pings_received() {
8204        // Regression: pings and pongs are keep-alive frames, not application data,
8205        // so a peer that only emits control frames must still trip the idle timeout.
8206        // The peer keeps pinging for well past the observation window so the
8207        // pre-fix behavior (reset-on-ping) would keep the client active; under the
8208        // fix the idle timer never resets and fires after ~500ms.
8209        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8210        let port = listener.local_addr().unwrap().port();
8211        let server = task::spawn(async move {
8212            let (stream, _) = listener.accept().await.unwrap();
8213            let mut ws = accept_async(stream).await.unwrap();
8214
8215            for _ in 0..60 {
8216                sleep(Duration::from_millis(100)).await;
8217
8218                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
8219                    break;
8220                }
8221            }
8222        });
8223
8224        let (handler, _rx) = channel_message_handler();
8225
8226        let config = WebSocketConfig {
8227            url: format!("ws://127.0.0.1:{port}"),
8228            headers: vec![],
8229            heartbeat_interval_secs: None,
8230            heartbeat_payload: None,
8231            connect_timeout_ms: Some(2_000),
8232            reconnect_delay_initial_ms: Some(50),
8233            reconnect_delay_max_ms: Some(100),
8234            reconnect_backoff_factor: Some(1.0),
8235            reconnect_jitter_ms: Some(0),
8236            reconnect_max_attempts: Some(1),
8237            heartbeat_timeout_secs: None,
8238            idle_timeout_ms: Some(500),
8239            writer_capacity: None,
8240            backend: TransportBackend::Tungstenite,
8241            proxy_url: None,
8242            max_message_size_bytes: None,
8243            max_frame_size_bytes: None,
8244        };
8245
8246        let client = WebSocketClient::builder()
8247            .config(config)
8248            .message_handler(handler)
8249            .connect()
8250            .await
8251            .unwrap();
8252
8253        assert!(client.is_active());
8254
8255        // Observation window is shorter than the ping stream (6s). If the idle
8256        // timer mistakenly reset on every ping the client would still be active
8257        // here; under the fix it goes inactive at ~500ms.
8258        wait_until_async(
8259            || async { client.is_reconnecting() || client.is_disconnected() },
8260            Duration::from_millis(1_500),
8261        )
8262        .await;
8263
8264        assert!(
8265            !client.is_active(),
8266            "Client should not be active after idle timeout when only pings/pongs flow"
8267        );
8268
8269        client.disconnect().await;
8270        server.abort();
8271    }
8272
8273    #[rstest]
8274    #[tokio::test]
8275    async fn test_idle_timeout_fires_when_only_pongs_received() {
8276        // Regression for the heartbeat-reply path. When the client heartbeat is
8277        // enabled, the peer auto-replies with pongs for every outgoing ping. If
8278        // those pongs refreshed last_data_time the idle timer would never fire on
8279        // a zombie connection (the motivating Polymarket scenario).
8280        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8281        let port = listener.local_addr().unwrap().port();
8282        let server = task::spawn(async move {
8283            let (stream, _) = listener.accept().await.unwrap();
8284            let mut ws = accept_async(stream).await.unwrap();
8285
8286            // Drain incoming frames so tungstenite's internal pong replies are
8287            // actually flushed to the client. Hold the connection open well past
8288            // the observation window.
8289            let deadline = tokio::time::Instant::now() + Duration::from_secs(6);
8290            while tokio::time::Instant::now() < deadline {
8291                if let Ok(Some(Err(_)) | None) =
8292                    tokio::time::timeout(Duration::from_millis(100), ws.next()).await
8293                {
8294                    break;
8295                }
8296            }
8297        });
8298
8299        let (handler, _rx) = channel_message_handler();
8300
8301        let config = WebSocketConfig {
8302            url: format!("ws://127.0.0.1:{port}"),
8303            headers: vec![],
8304            heartbeat_interval_secs: Some(1),
8305            heartbeat_payload: None,
8306            connect_timeout_ms: Some(2_000),
8307            reconnect_delay_initial_ms: Some(50),
8308            reconnect_delay_max_ms: Some(100),
8309            reconnect_backoff_factor: Some(1.0),
8310            reconnect_jitter_ms: Some(0),
8311            reconnect_max_attempts: Some(1),
8312            heartbeat_timeout_secs: None,
8313            idle_timeout_ms: Some(1_500),
8314            writer_capacity: None,
8315            backend: TransportBackend::Tungstenite,
8316            proxy_url: None,
8317            max_message_size_bytes: None,
8318            max_frame_size_bytes: None,
8319        };
8320
8321        let client = WebSocketClient::builder()
8322            .config(config)
8323            .message_handler(handler)
8324            .connect()
8325            .await
8326            .unwrap();
8327
8328        assert!(client.is_active());
8329
8330        // Heartbeat cadence is 1s; each ping draws a pong reply. Under the fix
8331        // the idle timer ignores those pongs and fires at ~1.5s. Under the bug
8332        // every pong reset the timer and the client would stay active.
8333        wait_until_async(
8334            || async { client.is_reconnecting() || client.is_disconnected() },
8335            Duration::from_millis(2_500),
8336        )
8337        .await;
8338
8339        assert!(
8340            !client.is_active(),
8341            "Client should not be active after idle timeout when only pongs flow"
8342        );
8343
8344        client.disconnect().await;
8345        server.abort();
8346    }
8347
8348    #[rstest]
8349    #[tokio::test]
8350    async fn test_disconnect_during_backoff_exits_promptly() {
8351        // Verify that disconnect interrupts backoff sleep (Finding 1).
8352        // Server accepts then drops, no second listener -> reconnect fails -> enters backoff.
8353        // We disconnect while backing off and assert the client shuts down quickly.
8354        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8355        let port = listener.local_addr().unwrap().port();
8356
8357        let server = task::spawn(async move {
8358            // Accept first connection, close immediately
8359            if let Ok((stream, _)) = listener.accept().await {
8360                let _ = accept_async(stream).await;
8361            }
8362
8363            // Don't accept again so reconnect fails and enters backoff
8364            sleep(Duration::from_mins(1)).await;
8365        });
8366
8367        let (handler, _rx) = channel_message_handler();
8368
8369        let config = WebSocketConfig {
8370            url: format!("ws://127.0.0.1:{port}"),
8371            headers: vec![],
8372            heartbeat_interval_secs: None,
8373            heartbeat_payload: None,
8374            connect_timeout_ms: Some(1_000),
8375            reconnect_delay_initial_ms: Some(10_000), // 10s backoff to ensure we're sleeping
8376            reconnect_delay_max_ms: Some(10_000),
8377            reconnect_backoff_factor: Some(1.0),
8378            reconnect_jitter_ms: Some(0),
8379            reconnect_max_attempts: None,
8380            heartbeat_timeout_secs: None,
8381            idle_timeout_ms: None,
8382            writer_capacity: None,
8383            backend: TransportBackend::Tungstenite,
8384            proxy_url: None,
8385            max_message_size_bytes: None,
8386            max_frame_size_bytes: None,
8387        };
8388
8389        let client = WebSocketClient::builder()
8390            .config(config)
8391            .message_handler(handler)
8392            .connect()
8393            .await
8394            .unwrap();
8395
8396        // Wait for client to enter reconnect
8397        wait_until_async(
8398            || async { client.is_reconnecting() },
8399            Duration::from_secs(3),
8400        )
8401        .await;
8402
8403        // Wait a bit more for the reconnect attempt to fail and enter backoff sleep
8404        sleep(Duration::from_millis(1_500)).await;
8405
8406        // Disconnect while backing off
8407        let start = std::time::Instant::now();
8408        client.disconnect().await;
8409        let elapsed = start.elapsed();
8410
8411        assert!(client.is_disconnected(), "Client should be disconnected");
8412        // Should exit well before the 10s backoff sleep completes
8413        assert!(
8414            elapsed < Duration::from_secs(2),
8415            "Disconnect should interrupt backoff sleep, took {elapsed:?}"
8416        );
8417
8418        server.abort();
8419    }
8420
8421    #[rstest]
8422    #[tokio::test]
8423    async fn test_rate_limit_cancelled_on_disconnect() {
8424        // Verify that a send blocked on rate limiting returns Closed when
8425        // the client disconnects (Finding 6).
8426        use std::{num::NonZeroU32, sync::Arc};
8427
8428        use crate::ratelimiter::quota::Quota;
8429
8430        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8431        let port = listener.local_addr().unwrap().port();
8432        let server = task::spawn(async move {
8433            if let Ok((stream, _)) = listener.accept().await {
8434                let mut ws = accept_async(stream).await.unwrap();
8435                // Keep alive and echo
8436                while let Some(Ok(msg)) = ws.next().await {
8437                    if ws.send(msg).await.is_err() {
8438                        break;
8439                    }
8440                }
8441            }
8442        });
8443
8444        let (handler, _rx) = channel_message_handler();
8445
8446        let config = WebSocketConfig {
8447            url: format!("ws://127.0.0.1:{port}"),
8448            headers: vec![],
8449            heartbeat_interval_secs: None,
8450            heartbeat_payload: None,
8451            connect_timeout_ms: Some(5_000),
8452            reconnect_delay_initial_ms: Some(100),
8453            reconnect_delay_max_ms: Some(500),
8454            reconnect_backoff_factor: Some(1.5),
8455            reconnect_jitter_ms: Some(0),
8456            reconnect_max_attempts: None,
8457            heartbeat_timeout_secs: None,
8458            idle_timeout_ms: None,
8459            writer_capacity: None,
8460            backend: TransportBackend::Tungstenite,
8461            proxy_url: None,
8462            max_message_size_bytes: None,
8463            max_frame_size_bytes: None,
8464        };
8465
8466        // Very restrictive: 1 req per 60 seconds
8467        let quota = Quota::with_period(Duration::from_mins(1))
8468            .unwrap()
8469            .allow_burst(NonZeroU32::new(1).unwrap());
8470
8471        let client = Arc::new(
8472            WebSocketClient::builder()
8473                .config(config)
8474                .message_handler(handler)
8475                .keyed_quotas(vec![("rate_key".to_string(), quota)])
8476                .connect()
8477                .await
8478                .unwrap(),
8479        );
8480
8481        let test_key: [Ustr; 1] = [Ustr::from("rate_key")];
8482
8483        // Exhaust the burst quota
8484        client
8485            .send_text("exhaust".to_string(), Some(test_key.as_slice()))
8486            .await
8487            .unwrap();
8488
8489        // Spawn a send that will block on rate limiter
8490        let client_clone = Arc::clone(&client);
8491        let send_handle = task::spawn(async move {
8492            client_clone
8493                .send_text("blocked".to_string(), Some(&[Ustr::from("rate_key")]))
8494                .await
8495        });
8496
8497        // Let the send block on rate limit
8498        sleep(Duration::from_millis(200)).await;
8499
8500        // Disconnect while send is blocked
8501        let start = std::time::Instant::now();
8502        client.disconnect().await;
8503        let elapsed_disconnect = start.elapsed();
8504
8505        // The blocked send should return Closed
8506        let result = tokio::time::timeout(Duration::from_secs(2), send_handle)
8507            .await
8508            .expect("Send task should complete quickly")
8509            .expect("Send task should not panic");
8510
8511        assert!(
8512            matches!(result, Err(crate::error::SendError::Closed)),
8513            "Blocked send should return Closed, was: {result:?}"
8514        );
8515
8516        // Disconnect should be fast, not waiting for the 60s rate limit
8517        assert!(
8518            elapsed_disconnect < Duration::from_secs(3),
8519            "Disconnect should not wait for rate limiter, took {elapsed_disconnect:?}"
8520        );
8521
8522        server.abort();
8523    }
8524
8525    #[rstest]
8526    #[tokio::test]
8527    async fn test_stream_mode_transitions_to_closed_on_dead_write_task() {
8528        // Verify that stream mode transitions to CLOSED (not RECONNECT) when
8529        // the write task dies (Finding 4). We force write failure by sending
8530        // after the server closes the connection.
8531        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
8532        let port = listener.local_addr().unwrap().port();
8533        let server = task::spawn(async move {
8534            if let Ok((stream, _)) = listener.accept().await
8535                && let Ok(ws) = accept_async(stream).await
8536            {
8537                // Close immediately to cause write errors
8538                drop(ws);
8539            }
8540        });
8541
8542        let config = WebSocketConfig {
8543            url: format!("ws://127.0.0.1:{port}"),
8544            headers: vec![],
8545            heartbeat_interval_secs: None,
8546            heartbeat_payload: None,
8547            connect_timeout_ms: Some(1_000),
8548            reconnect_delay_initial_ms: Some(50),
8549            reconnect_delay_max_ms: Some(100),
8550            reconnect_backoff_factor: Some(1.0),
8551            reconnect_jitter_ms: Some(0),
8552            reconnect_max_attempts: None,
8553            heartbeat_timeout_secs: None,
8554            idle_timeout_ms: None,
8555            writer_capacity: None,
8556            backend: TransportBackend::Tungstenite,
8557            proxy_url: None,
8558            max_message_size_bytes: None,
8559            max_frame_size_bytes: None,
8560        };
8561
8562        let (_reader, client) = WebSocketClient::stream_builder()
8563            .config(config)
8564            .connect()
8565            .await
8566            .unwrap();
8567
8568        assert!(client.is_active(), "Client should start active");
8569
8570        // Wait for server to close, then send to trigger write task failure
8571        sleep(Duration::from_millis(100)).await;
8572
8573        // Keep sending until the write task detects the broken connection
8574        for _ in 0..20 {
8575            let _ = client.send_text("ping".to_string(), None).await;
8576            sleep(Duration::from_millis(50)).await;
8577
8578            if !client.is_active() {
8579                break;
8580            }
8581        }
8582
8583        // Wait for controller to process the state change
8584        wait_until_async(|| async { !client.is_active() }, Duration::from_secs(5)).await;
8585
8586        // Stream mode should go to CLOSED, not RECONNECT
8587        assert!(
8588            client.is_closed() || client.is_disconnected(),
8589            "Stream mode should transition to CLOSED, not RECONNECT. \
8590             is_reconnecting={}, is_closed={}, is_disconnected={}",
8591            client.is_reconnecting(),
8592            client.is_closed(),
8593            client.is_disconnected(),
8594        );
8595        assert!(
8596            !client.is_reconnecting(),
8597            "Stream mode should never attempt reconnection"
8598        );
8599
8600        server.abort();
8601    }
8602
8603    #[derive(Default)]
8604    struct BlockingFailState {
8605        send_entered: AtomicBool,
8606        send_entered_notify: tokio::sync::Notify,
8607        released: AtomicBool,
8608        fail: AtomicBool,
8609        waker: parking_lot::Mutex<Option<std::task::Waker>>,
8610    }
8611
8612    impl BlockingFailState {
8613        fn trigger_failure(&self) {
8614            self.fail.store(true, Ordering::SeqCst);
8615            self.release_send();
8616        }
8617
8618        fn release_send(&self) {
8619            self.released.store(true, Ordering::SeqCst);
8620
8621            if let Some(waker) = self.waker.lock().take() {
8622                waker.wake();
8623            }
8624        }
8625    }
8626
8627    /// Transport whose sends block until released and can fail on demand.
8628    struct BlockingFailTransport {
8629        state: Arc<BlockingFailState>,
8630    }
8631
8632    impl futures_util::Stream for BlockingFailTransport {
8633        type Item = Result<Message, TransportError>;
8634
8635        fn poll_next(
8636            self: std::pin::Pin<&mut Self>,
8637            _cx: &mut std::task::Context<'_>,
8638        ) -> std::task::Poll<Option<Self::Item>> {
8639            std::task::Poll::Pending
8640        }
8641    }
8642
8643    impl futures_util::Sink<Message> for BlockingFailTransport {
8644        type Error = TransportError;
8645
8646        fn poll_ready(
8647            self: std::pin::Pin<&mut Self>,
8648            _cx: &mut std::task::Context<'_>,
8649        ) -> std::task::Poll<Result<(), Self::Error>> {
8650            std::task::Poll::Ready(Ok(()))
8651        }
8652
8653        fn start_send(self: std::pin::Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8654            Ok(())
8655        }
8656
8657        fn poll_flush(
8658            self: std::pin::Pin<&mut Self>,
8659            cx: &mut std::task::Context<'_>,
8660        ) -> std::task::Poll<Result<(), Self::Error>> {
8661            // Store the waker before checking the flag so trigger_failure
8662            // cannot slip between the check and the registration
8663            *self.state.waker.lock() = Some(cx.waker().clone());
8664            self.state.send_entered.store(true, Ordering::SeqCst);
8665            self.state.send_entered_notify.notify_one();
8666
8667            if !self.state.released.load(Ordering::SeqCst) {
8668                std::task::Poll::Pending
8669            } else if self.state.fail.load(Ordering::SeqCst) {
8670                std::task::Poll::Ready(Err(TransportError::ConnectionReset))
8671            } else {
8672                std::task::Poll::Ready(Ok(()))
8673            }
8674        }
8675
8676        fn poll_close(
8677            self: std::pin::Pin<&mut Self>,
8678            _cx: &mut std::task::Context<'_>,
8679        ) -> std::task::Poll<Result<(), Self::Error>> {
8680            std::task::Poll::Ready(Ok(()))
8681        }
8682    }
8683
8684    struct BlockingMessageState {
8685        polled_tx: Mutex<Option<std::sync::mpsc::Sender<()>>>,
8686        release: (Mutex<bool>, parking_lot::Condvar),
8687        message: Mutex<Option<Message>>,
8688    }
8689
8690    struct BlockingMessageTransport {
8691        state: Arc<BlockingMessageState>,
8692    }
8693
8694    impl futures_util::Stream for BlockingMessageTransport {
8695        type Item = Result<Message, TransportError>;
8696
8697        fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8698            if let Some(polled_tx) = self.state.polled_tx.lock().take() {
8699                polled_tx.send(()).unwrap();
8700            }
8701
8702            let (lock, condvar) = &self.state.release;
8703            let mut released = lock.lock();
8704            while !*released {
8705                condvar.wait(&mut released);
8706            }
8707
8708            Poll::Ready(self.state.message.lock().take().map(Ok))
8709        }
8710    }
8711
8712    impl futures_util::Sink<Message> for BlockingMessageTransport {
8713        type Error = TransportError;
8714
8715        fn poll_ready(
8716            self: Pin<&mut Self>,
8717            _cx: &mut Context<'_>,
8718        ) -> Poll<Result<(), Self::Error>> {
8719            Poll::Ready(Ok(()))
8720        }
8721
8722        fn start_send(self: Pin<&mut Self>, _item: Message) -> Result<(), Self::Error> {
8723            Ok(())
8724        }
8725
8726        fn poll_flush(
8727            self: Pin<&mut Self>,
8728            _cx: &mut Context<'_>,
8729        ) -> Poll<Result<(), Self::Error>> {
8730            Poll::Ready(Ok(()))
8731        }
8732
8733        fn poll_close(
8734            self: Pin<&mut Self>,
8735            _cx: &mut Context<'_>,
8736        ) -> Poll<Result<(), Self::Error>> {
8737            Poll::Ready(Ok(()))
8738        }
8739    }
8740
8741    struct RecordingState {
8742        messages: Arc<Mutex<Vec<Message>>>,
8743        recorded_notify: tokio::sync::Notify,
8744    }
8745
8746    struct RecordingTransport {
8747        state: Arc<RecordingState>,
8748    }
8749
8750    impl futures_util::Stream for RecordingTransport {
8751        type Item = Result<Message, TransportError>;
8752
8753        fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
8754            Poll::Pending
8755        }
8756    }
8757
8758    impl futures_util::Sink<Message> for RecordingTransport {
8759        type Error = TransportError;
8760
8761        fn poll_ready(
8762            self: Pin<&mut Self>,
8763            _cx: &mut Context<'_>,
8764        ) -> Poll<Result<(), Self::Error>> {
8765            Poll::Ready(Ok(()))
8766        }
8767
8768        fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> {
8769            self.state.messages.lock().push(item);
8770            self.state.recorded_notify.notify_one();
8771            Ok(())
8772        }
8773
8774        fn poll_flush(
8775            self: Pin<&mut Self>,
8776            _cx: &mut Context<'_>,
8777        ) -> Poll<Result<(), Self::Error>> {
8778            Poll::Ready(Ok(()))
8779        }
8780
8781        fn poll_close(
8782            self: Pin<&mut Self>,
8783            _cx: &mut Context<'_>,
8784        ) -> Poll<Result<(), Self::Error>> {
8785            Poll::Ready(Ok(()))
8786        }
8787    }
8788
8789    #[rstest]
8790    #[tokio::test(start_paused = true)]
8791    async fn test_pong_is_bound_to_connection_epoch() {
8792        let initial_state = Arc::new(RecordingState {
8793            messages: Arc::new(Mutex::new(Vec::new())),
8794            recorded_notify: tokio::sync::Notify::new(),
8795        });
8796
8797        let initial_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8798            state: initial_state,
8799        });
8800
8801        let (writer, _reader) = initial_transport.split();
8802        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8803        let state_notify = Arc::new(tokio::sync::Notify::new());
8804        let auth_tracker = Arc::new(OnceLock::new());
8805        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
8806        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
8807        let write_task = WebSocketClientInner::spawn_write_task(
8808            Arc::clone(&connection_state),
8809            Arc::clone(&state_notify),
8810            Arc::new(AtomicBool::new(true)),
8811            writer,
8812            writer_rx,
8813            Arc::new(AtomicU64::new(0)),
8814            auth_tracker,
8815            reconnect_buffer_waits_for_auth,
8816            None,
8817        );
8818
8819        let recorded = Arc::new(Mutex::new(Vec::new()));
8820
8821        let replacement_state = Arc::new(RecordingState {
8822            messages: Arc::clone(&recorded),
8823            recorded_notify: tokio::sync::Notify::new(),
8824        });
8825
8826        let replacement_transport: BoxedWsTransport = Box::pin(RecordingTransport {
8827            state: replacement_state,
8828        });
8829
8830        let (replacement_writer, _reader) = replacement_transport.split();
8831        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
8832        writer_tx
8833            .send(WriterCommand::Update(replacement_writer, update_tx))
8834            .unwrap();
8835        writer_tx
8836            .send_control(WriterCommand::SendPongOnConnection {
8837                data: b"stale-pong".to_vec(),
8838                connection_epoch: 0,
8839            })
8840            .unwrap();
8841
8842        tokio::time::advance(Duration::from_millis(100)).await;
8843        assert_eq!(update_rx.await.unwrap(), 1);
8844
8845        let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8846        writer_tx
8847            .send_control(WriterCommand::SendOnConnection {
8848                message: Message::text("sentinel-1"),
8849                connection_epoch: 1,
8850                response_tx: sentinel_tx,
8851            })
8852            .unwrap();
8853
8854        sentinel_rx.await.unwrap().unwrap();
8855        assert_eq!(recorded.lock().as_slice(), &[Message::text("sentinel-1")]);
8856
8857        writer_tx
8858            .send_control(WriterCommand::SendPongOnConnection {
8859                data: b"fresh-pong".to_vec(),
8860                connection_epoch: 1,
8861            })
8862            .unwrap();
8863
8864        let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
8865        writer_tx
8866            .send_control(WriterCommand::SendOnConnection {
8867                message: Message::text("sentinel-2"),
8868                connection_epoch: 1,
8869                response_tx: sentinel_tx,
8870            })
8871            .unwrap();
8872
8873        sentinel_rx.await.unwrap().unwrap();
8874        assert_eq!(
8875            recorded.lock().as_slice(),
8876            &[
8877                Message::text("sentinel-1"),
8878                Message::Pong(b"fresh-pong".to_vec().into()),
8879                Message::text("sentinel-2"),
8880            ]
8881        );
8882
8883        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
8884        state_notify.notify_waiters();
8885        drop(writer_tx);
8886        write_task.await.unwrap();
8887    }
8888
8889    #[rstest]
8890    #[case(Message::text("stale"))]
8891    #[case(Message::Binary(vec![1, 2, 3].into()))]
8892    #[case(Message::Ping(vec![1, 2, 3].into()))]
8893    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8894    async fn test_message_handler_drops_old_session_message(#[case] message: Message) {
8895        let (polled_tx, polled_rx) = std::sync::mpsc::channel();
8896
8897        let state = Arc::new(BlockingMessageState {
8898            polled_tx: Mutex::new(Some(polled_tx)),
8899            release: (Mutex::new(false), Condvar::new()),
8900            message: Mutex::new(Some(message)),
8901        });
8902
8903        let release_guard = CondvarReleaseGuard::new(&state.release);
8904
8905        let transport: BoxedWsTransport = Box::pin(BlockingMessageTransport {
8906            state: Arc::clone(&state),
8907        });
8908
8909        let (_writer, reader) = transport.split();
8910        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8911        let state_notify = Arc::new(tokio::sync::Notify::new());
8912        let read_fence = ReadSessionFence::new();
8913        let message_count = Arc::new(AtomicUsize::new(0));
8914        let ping_count = Arc::new(AtomicUsize::new(0));
8915        let message_count_clone = Arc::clone(&message_count);
8916        let ping_count_clone = Arc::clone(&ping_count);
8917        let message_handler: MessageHandler =
8918            Arc::new(move |_| _ = message_count_clone.fetch_add(1, Ordering::SeqCst));
8919        let message_handler = IncomingHandler::Message(message_handler);
8920        let ping_handler: PingHandler =
8921            Arc::new(move |_| _ = ping_count_clone.fetch_add(1, Ordering::SeqCst));
8922        let ping_handler = IncomingPingHandler::Ping(ping_handler);
8923
8924        let read_task = WebSocketClientInner::spawn_message_handler_task(
8925            Arc::clone(&connection_state),
8926            state_notify,
8927            read_fence.clone(),
8928            reader,
8929            0,
8930            Some(&message_handler),
8931            Some(&ping_handler),
8932            None,
8933            None,
8934            None,
8935            None,
8936            None,
8937        );
8938
8939        recv_rendezvous(polled_rx, "WebSocket reader poll entry").await;
8940        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8941        read_fence.invalidate();
8942        read_task.abort();
8943        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
8944        release_guard.release();
8945        await_task_termination(read_task, "old WebSocket read task").await;
8946
8947        assert_eq!(message_count.load(Ordering::SeqCst), 0);
8948        assert_eq!(ping_count.load(Ordering::SeqCst), 0);
8949    }
8950
8951    #[rstest]
8952    #[tokio::test]
8953    async fn test_reconnect_buffer_drain_stops_after_reconnect_request() {
8954        let state = Arc::new(BlockingFailState::default());
8955
8956        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
8957            state: Arc::clone(&state),
8958        });
8959
8960        let (mut writer, _reader) = transport.split();
8961        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
8962        let task_connection_state = Arc::clone(&connection_state);
8963
8964        let drain_task = tokio::spawn(async move {
8965            let auth_tracker = Arc::new(OnceLock::new());
8966            let reconnect_buffer_waits_for_auth = AtomicBool::new(false);
8967            let (sender, mut receiver) = crate::writer::channel(2);
8968            sender.send(Message::text("admitted")).unwrap();
8969            sender.send(Message::text("held-for-reconnect")).unwrap();
8970            let mut buffer = VecDeque::from([
8971                receiver.recv().await.unwrap(),
8972                receiver.recv().await.unwrap(),
8973            ]);
8974            let send_error = WebSocketClientInner::drain_reconnect_buffer(
8975                &mut buffer,
8976                &mut writer,
8977                &task_connection_state,
8978                &auth_tracker,
8979                &reconnect_buffer_waits_for_auth,
8980            )
8981            .await;
8982            (buffer, send_error)
8983        });
8984
8985        state.send_entered_notify.notified().await;
8986        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
8987        state.release_send();
8988
8989        let (buffer, send_error) = tokio::time::timeout(TEST_TIMEOUT, drain_task)
8990            .await
8991            .expect("buffer drain should stop after reconnect acceptance")
8992            .unwrap();
8993        assert!(!send_error);
8994        assert_eq!(
8995            buffer
8996                .into_iter()
8997                .map(|(message, _)| message)
8998                .collect::<Vec<_>>(),
8999            vec![Message::text("held-for-reconnect")]
9000        );
9001    }
9002
9003    #[rstest]
9004    #[case(0)]
9005    #[case(usize::MAX)]
9006    #[tokio::test]
9007    async fn test_stream_writer_rejects_invalid_capacity(#[case] capacity: usize) {
9008        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9009            state: Arc::new(BlockingFailState::default()),
9010        });
9011
9012        let (writer, _reader) = transport.split();
9013        let mut config = WebSocketConfig::builder()
9014            .url("ws://localhost".to_string())
9015            .build()
9016            .unwrap();
9017        config.writer_capacity = Some(capacity);
9018
9019        let result = WebSocketClientInner::new_with_writer(config, writer).await;
9020
9021        assert!(
9022            matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9023        );
9024    }
9025
9026    #[rstest]
9027    #[case(0)]
9028    #[case(usize::MAX)]
9029    #[tokio::test]
9030    async fn test_handlerless_connect_rejects_invalid_capacity(#[case] capacity: usize) {
9031        let mut config = WebSocketConfig::builder()
9032            .url("ws://127.0.0.1:1".to_string())
9033            .build()
9034            .unwrap();
9035        config.writer_capacity = Some(capacity);
9036
9037        let result = WebSocketClientInner::connect_url(config, None, None).await;
9038
9039        assert!(
9040            matches!(result, Err(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::InvalidInput && e.to_string().contains("writer_capacity"))
9041        );
9042    }
9043
9044    #[rstest]
9045    #[tokio::test(start_paused = true)]
9046    async fn test_stalled_websocket_send_reconnects_and_replays() {
9047        let state = Arc::new(BlockingFailState::default());
9048
9049        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9050            state: Arc::clone(&state),
9051        });
9052
9053        let (writer, _reader) = transport.split();
9054        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9055        let state_notify = Arc::new(tokio::sync::Notify::new());
9056        let auth_tracker = Arc::new(OnceLock::new());
9057        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9058        let (writer_tx, writer_rx) = crate::writer::channel(2);
9059        let states = Arc::new(Mutex::new(Vec::new()));
9060        let states_callback = Arc::clone(&states);
9061        let sink = SocketStateSink::new(move |state| {
9062            states_callback.lock().push(state);
9063        });
9064
9065        let write_task = WebSocketClientInner::spawn_write_task(
9066            Arc::clone(&connection_state),
9067            Arc::clone(&state_notify),
9068            Arc::new(AtomicBool::new(true)),
9069            writer,
9070            writer_rx,
9071            Arc::new(AtomicU64::new(0)),
9072            Arc::clone(&auth_tracker),
9073            reconnect_buffer_waits_for_auth,
9074            Some(sink),
9075        );
9076
9077        writer_tx
9078            .send(WriterCommand::Send(Message::text("complete-message")))
9079            .unwrap();
9080        state.send_entered_notify.notified().await;
9081        writer_tx
9082            .send(WriterCommand::Send(Message::text("second-message")))
9083            .unwrap();
9084        assert_eq!(
9085            ConnectionMode::from_atomic(&connection_state),
9086            ConnectionMode::Active
9087        );
9088
9089        for _ in 0..100 {
9090            assert!(matches!(
9091                writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9092                Err(SendError::BufferFull)
9093            ));
9094        }
9095
9096        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9097        tokio::task::yield_now().await;
9098        assert_eq!(
9099            ConnectionMode::from_atomic(&connection_state),
9100            ConnectionMode::Reconnect
9101        );
9102        assert!(matches!(
9103            writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
9104            Err(SendError::BufferFull)
9105        ));
9106
9107        let recorded = Arc::new(Mutex::new(Vec::new()));
9108
9109        let recording_state = Arc::new(RecordingState {
9110            messages: Arc::clone(&recorded),
9111            recorded_notify: tokio::sync::Notify::new(),
9112        });
9113
9114        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9115            state: Arc::clone(&recording_state),
9116        });
9117
9118        let (new_writer, _reader) = transport.split();
9119        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9120        writer_tx
9121            .send_update(WriterCommand::Update(new_writer, update_tx))
9122            .unwrap();
9123
9124        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9125        assert_eq!(
9126            tokio::time::timeout(Duration::from_secs(1), update_rx)
9127                .await
9128                .expect("writer update should not remain queued behind a stalled send")
9129                .unwrap(),
9130            1,
9131            "the replacement sink should install as connection epoch 1"
9132        );
9133        assert_eq!(
9134            ConnectionMode::from_atomic(&connection_state),
9135            ConnectionMode::Reconnect
9136        );
9137
9138        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9139        state_notify.notify_waiters();
9140        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9141        recording_state.recorded_notify.notified().await;
9142        assert_eq!(
9143            recorded.lock().as_slice(),
9144            &[
9145                Message::text("complete-message"),
9146                Message::text("second-message")
9147            ]
9148        );
9149
9150        writer_tx
9151            .send(WriterCommand::Send(Message::text("second-message")))
9152            .unwrap();
9153
9154        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9155        state_notify.notify_waiters();
9156        drop(writer_tx);
9157        write_task.await.unwrap();
9158
9159        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
9160    }
9161
9162    #[rstest]
9163    #[case(Message::Ping(vec![1, 2, 3].into()))]
9164    #[case(Message::Pong(vec![4, 5, 6].into()))]
9165    #[case(Message::Close(None))]
9166    #[tokio::test(start_paused = true)]
9167    async fn test_stalled_control_frame_is_not_replayed(#[case] control: Message) {
9168        // A control frame belongs to the connection it was issued on, so a failed write must
9169        // drop it instead of replaying it onto the replacement connection.
9170        let state = Arc::new(BlockingFailState::default());
9171
9172        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9173            state: Arc::clone(&state),
9174        });
9175
9176        let (writer, _reader) = transport.split();
9177        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9178        let state_notify = Arc::new(tokio::sync::Notify::new());
9179        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9180        let write_task = WebSocketClientInner::spawn_write_task(
9181            Arc::clone(&connection_state),
9182            Arc::clone(&state_notify),
9183            Arc::new(AtomicBool::new(true)),
9184            writer,
9185            writer_rx,
9186            Arc::new(AtomicU64::new(0)),
9187            Arc::new(OnceLock::new()),
9188            Arc::new(AtomicBool::new(false)),
9189            None,
9190        );
9191
9192        writer_tx.send(WriterCommand::Send(control)).unwrap();
9193        state.send_entered_notify.notified().await;
9194
9195        let recorded = Arc::new(Mutex::new(Vec::new()));
9196
9197        let recording_state = Arc::new(RecordingState {
9198            messages: Arc::clone(&recorded),
9199            recorded_notify: tokio::sync::Notify::new(),
9200        });
9201
9202        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9203            state: recording_state,
9204        });
9205
9206        let (new_writer, _reader) = transport.split();
9207        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9208        writer_tx
9209            .send(WriterCommand::Update(new_writer, update_tx))
9210            .unwrap();
9211
9212        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9213        assert_eq!(
9214            tokio::time::timeout(Duration::from_secs(1), update_rx)
9215                .await
9216                .expect("writer update should not remain queued behind a stalled send")
9217                .unwrap(),
9218            1,
9219            "the replacement sink should install as connection epoch 1"
9220        );
9221        assert_eq!(
9222            ConnectionMode::from_atomic(&connection_state),
9223            ConnectionMode::Reconnect,
9224            "a failed control-frame write should still trigger a reconnect"
9225        );
9226
9227        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9228        state_notify.notify_waiters();
9229
9230        // Each pass returns the writer to the loop top, where a buffered frame would drain
9231        for _ in 0..5 {
9232            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9233        }
9234
9235        let replayed = recorded.lock().clone();
9236        assert!(
9237            replayed.is_empty(),
9238            "a failed control frame must not reach the replacement connection, was {replayed:?}"
9239        );
9240
9241        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9242        state_notify.notify_waiters();
9243        drop(writer_tx);
9244        write_task.await.unwrap();
9245    }
9246
9247    #[rstest]
9248    #[case(Message::Ping(vec![1, 2, 3].into()))]
9249    #[case(Message::Pong(vec![4, 5, 6].into()))]
9250    #[case(Message::Close(None))]
9251    #[tokio::test(start_paused = true)]
9252    async fn test_control_frame_enqueued_during_reconnect_is_not_replayed(
9253        #[case] control: Message,
9254    ) {
9255        // `send_pong` and the heartbeat task check for an active connection before enqueueing,
9256        // so a mode flip can land their frame on the writer's reconnect-mode branch instead.
9257        let recorded = Arc::new(Mutex::new(Vec::new()));
9258
9259        let recording_state = Arc::new(RecordingState {
9260            messages: Arc::clone(&recorded),
9261            recorded_notify: tokio::sync::Notify::new(),
9262        });
9263
9264        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9265            state: recording_state,
9266        });
9267
9268        let (writer, _reader) = transport.split();
9269        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9270        let state_notify = Arc::new(tokio::sync::Notify::new());
9271        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9272        let write_task = WebSocketClientInner::spawn_write_task(
9273            Arc::clone(&connection_state),
9274            Arc::clone(&state_notify),
9275            Arc::new(AtomicBool::new(true)),
9276            writer,
9277            writer_rx,
9278            Arc::new(AtomicU64::new(0)),
9279            Arc::new(OnceLock::new()),
9280            Arc::new(AtomicBool::new(false)),
9281            None,
9282        );
9283
9284        writer_tx.send(WriterCommand::Send(control)).unwrap();
9285        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9286
9287        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9288        state_notify.notify_waiters();
9289
9290        // Each pass returns the writer to the loop top, where a buffered frame would drain
9291        for _ in 0..5 {
9292            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9293        }
9294
9295        let replayed = recorded.lock().clone();
9296        assert!(
9297            replayed.is_empty(),
9298            "a control frame enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9299        );
9300
9301        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9302        state_notify.notify_waiters();
9303        drop(writer_tx);
9304        write_task.await.unwrap();
9305    }
9306
9307    #[rstest]
9308    #[tokio::test(start_paused = true)]
9309    async fn test_stalled_text_heartbeat_is_not_replayed() {
9310        let state = Arc::new(BlockingFailState::default());
9311
9312        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9313            state: Arc::clone(&state),
9314        });
9315
9316        let (writer, _reader) = transport.split();
9317        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9318        let state_notify = Arc::new(tokio::sync::Notify::new());
9319        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9320        let write_task = WebSocketClientInner::spawn_write_task(
9321            Arc::clone(&connection_state),
9322            Arc::clone(&state_notify),
9323            Arc::new(AtomicBool::new(true)),
9324            writer,
9325            writer_rx,
9326            Arc::new(AtomicU64::new(0)),
9327            Arc::new(OnceLock::new()),
9328            Arc::new(AtomicBool::new(false)),
9329            None,
9330        );
9331
9332        writer_tx
9333            .send(WriterCommand::Heartbeat(Message::text(
9334                "{\"op\":\"heartbeat\"}",
9335            )))
9336            .unwrap();
9337        state.send_entered_notify.notified().await;
9338
9339        let recorded = Arc::new(Mutex::new(Vec::new()));
9340
9341        let recording_state = Arc::new(RecordingState {
9342            messages: Arc::clone(&recorded),
9343            recorded_notify: tokio::sync::Notify::new(),
9344        });
9345
9346        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9347            state: recording_state,
9348        });
9349
9350        let (new_writer, _reader) = transport.split();
9351        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9352        writer_tx
9353            .send(WriterCommand::Update(new_writer, update_tx))
9354            .unwrap();
9355
9356        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9357        assert_eq!(
9358            tokio::time::timeout(Duration::from_secs(1), update_rx)
9359                .await
9360                .expect("writer update should not remain queued behind a stalled send")
9361                .unwrap(),
9362            1,
9363            "the replacement sink should install as connection epoch 1"
9364        );
9365        assert_eq!(
9366            ConnectionMode::from_atomic(&connection_state),
9367            ConnectionMode::Reconnect,
9368            "a failed heartbeat write should still trigger a reconnect"
9369        );
9370
9371        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9372        state_notify.notify_waiters();
9373
9374        for _ in 0..5 {
9375            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9376        }
9377
9378        let replayed = recorded.lock().clone();
9379        assert!(
9380            replayed.is_empty(),
9381            "a failed text heartbeat must not reach the replacement connection, was {replayed:?}"
9382        );
9383
9384        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9385        state_notify.notify_waiters();
9386        drop(writer_tx);
9387        write_task.await.unwrap();
9388    }
9389
9390    #[rstest]
9391    #[tokio::test(start_paused = true)]
9392    async fn test_text_heartbeat_enqueued_during_reconnect_is_not_replayed() {
9393        let recorded = Arc::new(Mutex::new(Vec::new()));
9394
9395        let recording_state = Arc::new(RecordingState {
9396            messages: Arc::clone(&recorded),
9397            recorded_notify: tokio::sync::Notify::new(),
9398        });
9399
9400        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9401            state: recording_state,
9402        });
9403
9404        let (writer, _reader) = transport.split();
9405        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9406        let state_notify = Arc::new(tokio::sync::Notify::new());
9407        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9408        let write_task = WebSocketClientInner::spawn_write_task(
9409            Arc::clone(&connection_state),
9410            Arc::clone(&state_notify),
9411            Arc::new(AtomicBool::new(true)),
9412            writer,
9413            writer_rx,
9414            Arc::new(AtomicU64::new(0)),
9415            Arc::new(OnceLock::new()),
9416            Arc::new(AtomicBool::new(false)),
9417            None,
9418        );
9419
9420        writer_tx
9421            .send(WriterCommand::Heartbeat(Message::text(
9422                "{\"op\":\"heartbeat\"}",
9423            )))
9424            .unwrap();
9425        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9426
9427        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9428        state_notify.notify_waiters();
9429
9430        for _ in 0..5 {
9431            tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9432        }
9433
9434        let replayed = recorded.lock().clone();
9435        assert!(
9436            replayed.is_empty(),
9437            "a text heartbeat enqueued during reconnect must not reach the replacement connection, was {replayed:?}"
9438        );
9439
9440        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9441        state_notify.notify_waiters();
9442        drop(writer_tx);
9443        write_task.await.unwrap();
9444    }
9445
9446    #[rstest]
9447    #[case::text(
9448        Some("{\"op\":\"heartbeat\"}"),
9449        Message::text("{\"op\":\"heartbeat\"}")
9450    )]
9451    #[case::ping(None, Message::Ping(vec![].into()))]
9452    #[tokio::test(start_paused = true)]
9453    async fn test_heartbeat_task_enqueues_writer_heartbeat_command(
9454        #[case] payload: Option<&str>,
9455        #[case] expected: Message,
9456    ) {
9457        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9458        let (writer_tx, mut writer_rx) =
9459            crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9460        let task = WebSocketClientInner::spawn_heartbeat_task(
9461            Arc::clone(&connection_state),
9462            1,
9463            payload.map(ToString::to_string),
9464            writer_tx,
9465        );
9466
9467        tokio::time::advance(Duration::from_secs(1)).await;
9468        let cmd = writer_rx
9469            .recv()
9470            .await
9471            .expect("heartbeat task should enqueue");
9472
9473        match cmd.0 {
9474            WriterCommand::Heartbeat(msg) => assert_eq!(msg, expected),
9475            other => panic!("expected Heartbeat, was {other:?}"),
9476        }
9477
9478        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9479        tokio::time::advance(Duration::from_secs(1)).await;
9480        task.await.unwrap();
9481    }
9482
9483    #[cfg(target_os = "linux")]
9484    #[rstest]
9485    #[tokio::test]
9486    async fn test_heartbeat_task_warns_when_writer_channel_closed() {
9487        let capture = capture_logs_for(&["nautilus_network::websocket::client"]).await;
9488        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9489        let (writer_tx, mut writer_rx) =
9490            crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9491        let task = WebSocketClientInner::spawn_heartbeat_task(
9492            Arc::clone(&connection_state),
9493            1,
9494            None,
9495            writer_tx,
9496        );
9497
9498        let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
9499            .await
9500            .expect("timed out waiting for the first heartbeat")
9501            .expect("heartbeat channel closed before the first heartbeat");
9502        drop(writer_rx);
9503        tokio::time::sleep(Duration::from_millis(1_200)).await;
9504
9505        let messages = capture.messages();
9506        assert!(
9507            messages.iter().any(|(level, message)| {
9508                *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
9509            }),
9510            "expected WARN when the writer channel is closed, was {messages:?}"
9511        );
9512        assert!(
9513            !messages.iter().any(|(level, _)| *level == Level::Error),
9514            "a closed writer channel must not log ERROR, was {messages:?}"
9515        );
9516
9517        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9518        tokio::time::timeout(Duration::from_secs(2), task)
9519            .await
9520            .expect("heartbeat task should stop after close")
9521            .unwrap();
9522    }
9523
9524    #[rstest]
9525    #[tokio::test(start_paused = true)]
9526    async fn test_stalled_ownership_bound_send_times_out_without_replay() {
9527        let state = Arc::new(BlockingFailState::default());
9528
9529        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9530            state: Arc::clone(&state),
9531        });
9532
9533        let (writer, _reader) = transport.split();
9534        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
9535        let state_notify = Arc::new(tokio::sync::Notify::new());
9536        let auth_tracker = Arc::new(OnceLock::new());
9537        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9538        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9539        let write_task = WebSocketClientInner::spawn_write_task(
9540            Arc::clone(&connection_state),
9541            Arc::clone(&state_notify),
9542            Arc::new(AtomicBool::new(true)),
9543            writer,
9544            writer_rx,
9545            Arc::new(AtomicU64::new(0)),
9546            Arc::clone(&auth_tracker),
9547            reconnect_buffer_waits_for_auth,
9548            None,
9549        );
9550
9551        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
9552        writer_tx
9553            .send_control(WriterCommand::SendOnConnection {
9554                message: Message::text("ownership-bound"),
9555                connection_epoch: 0,
9556                response_tx,
9557            })
9558            .unwrap();
9559
9560        state.send_entered_notify.notified().await;
9561
9562        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
9563        let outcome = tokio::time::timeout(Duration::from_secs(1), response_rx)
9564            .await
9565            .expect("a stalled ownership-bound send must not wedge the writer task")
9566            .unwrap();
9567        assert!(
9568            matches!(outcome, Err(SendError::WriteTimeout)),
9569            "expected the write deadline to be reported, was {outcome:?}"
9570        );
9571        assert_eq!(
9572            ConnectionMode::from_atomic(&connection_state),
9573            ConnectionMode::Reconnect
9574        );
9575
9576        // The timed-out message is ownership-bound, so unlike an ordinary send it must
9577        // NOT be buffered for replay onto the replacement sink.
9578        let recorded = Arc::new(Mutex::new(Vec::new()));
9579
9580        let recording_state = Arc::new(RecordingState {
9581            messages: Arc::clone(&recorded),
9582            recorded_notify: tokio::sync::Notify::new(),
9583        });
9584
9585        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9586            state: Arc::clone(&recording_state),
9587        });
9588
9589        let (new_writer, _reader) = transport.split();
9590        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9591        writer_tx
9592            .send(WriterCommand::Update(new_writer, update_tx))
9593            .unwrap();
9594
9595        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9596        assert_eq!(
9597            tokio::time::timeout(Duration::from_secs(1), update_rx)
9598                .await
9599                .expect("writer update should not remain queued behind a stalled send")
9600                .unwrap(),
9601            1
9602        );
9603
9604        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9605        state_notify.notify_waiters();
9606        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9607
9608        // Barrier: two sentinels bound to the NEW epoch, the second sent only after the first
9609        // is acknowledged. The writer can accept the first from a `recv()` that was already
9610        // pending when the mode flipped, so only the second is necessarily processed after the
9611        // intervening loop-top drain - which is where a mistakenly buffered message would be
9612        // replayed. One sentinel alone would leave the ordering to chance.
9613        for name in ["sentinel-1", "sentinel-2"] {
9614            let (sentinel_tx, sentinel_rx) = tokio::sync::oneshot::channel();
9615            writer_tx
9616                .send_control(WriterCommand::SendOnConnection {
9617                    message: Message::text(name),
9618                    connection_epoch: 1,
9619                    response_tx: sentinel_tx,
9620                })
9621                .unwrap();
9622
9623            recording_state.recorded_notify.notified().await;
9624            sentinel_rx
9625                .await
9626                .unwrap()
9627                .expect("the sentinel should send on the replacement connection");
9628        }
9629
9630        assert_eq!(
9631            recorded.lock().as_slice(),
9632            &[Message::text("sentinel-1"), Message::text("sentinel-2")],
9633            "an ownership-bound message must never be replayed after its deadline expires"
9634        );
9635
9636        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9637        state_notify.notify_waiters();
9638        drop(writer_tx);
9639        write_task.await.unwrap();
9640    }
9641
9642    #[rstest]
9643    #[tokio::test(start_paused = true)]
9644    async fn test_stalled_websocket_replay_reconnects_and_retries_buffer() {
9645        let initial_messages = Arc::new(Mutex::new(Vec::new()));
9646
9647        let initial_recording_state = Arc::new(RecordingState {
9648            messages: initial_messages,
9649            recorded_notify: tokio::sync::Notify::new(),
9650        });
9651
9652        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9653            state: initial_recording_state,
9654        });
9655
9656        let (writer, _reader) = transport.split();
9657        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
9658        let state_notify = Arc::new(tokio::sync::Notify::new());
9659        let auth_tracker = Arc::new(OnceLock::new());
9660        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
9661        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
9662        let write_task = WebSocketClientInner::spawn_write_task(
9663            Arc::clone(&connection_state),
9664            Arc::clone(&state_notify),
9665            Arc::new(AtomicBool::new(true)),
9666            writer,
9667            writer_rx,
9668            Arc::new(AtomicU64::new(0)),
9669            Arc::clone(&auth_tracker),
9670            reconnect_buffer_waits_for_auth,
9671            None,
9672        );
9673
9674        writer_tx
9675            .send(WriterCommand::Send(Message::text("buffered-message")))
9676            .unwrap();
9677
9678        let blocking_state = Arc::new(BlockingFailState::default());
9679
9680        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9681            state: Arc::clone(&blocking_state),
9682        });
9683
9684        let (blocking_writer, _reader) = transport.split();
9685        let (blocking_tx, blocking_rx) = tokio::sync::oneshot::channel();
9686        writer_tx
9687            .send(WriterCommand::Update(blocking_writer, blocking_tx))
9688            .unwrap();
9689        assert_eq!(blocking_rx.await.unwrap(), 1);
9690
9691        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9692        state_notify.notify_waiters();
9693        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9694        blocking_state.send_entered_notify.notified().await;
9695
9696        let recorded = Arc::new(Mutex::new(Vec::new()));
9697
9698        let recording_state = Arc::new(RecordingState {
9699            messages: Arc::clone(&recorded),
9700            recorded_notify: tokio::sync::Notify::new(),
9701        });
9702
9703        let transport: BoxedWsTransport = Box::pin(RecordingTransport {
9704            state: Arc::clone(&recording_state),
9705        });
9706
9707        let (new_writer, _reader) = transport.split();
9708        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
9709        writer_tx
9710            .send(WriterCommand::Update(new_writer, update_tx))
9711            .unwrap();
9712
9713        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
9714        assert_eq!(
9715            tokio::time::timeout(Duration::from_secs(1), update_rx)
9716                .await
9717                .expect("writer update should not remain queued behind stalled replay")
9718                .unwrap(),
9719            2,
9720            "the second replacement sink should install as connection epoch 2"
9721        );
9722        assert_eq!(
9723            ConnectionMode::from_atomic(&connection_state),
9724            ConnectionMode::Reconnect
9725        );
9726
9727        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
9728        state_notify.notify_waiters();
9729        tokio::time::advance(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
9730        recording_state.recorded_notify.notified().await;
9731        assert_eq!(
9732            recorded.lock().as_slice(),
9733            &[Message::text("buffered-message")]
9734        );
9735
9736        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
9737        state_notify.notify_waiters();
9738        drop(writer_tx);
9739        write_task.await.unwrap();
9740    }
9741
9742    #[rstest]
9743    #[tokio::test]
9744    async fn test_new_with_writer_rejects_zero_heartbeat() {
9745        // Stream mode shares connect_url's validation: a zero heartbeat would
9746        // spawn a busy-loop ping flood
9747        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
9748            state: Arc::new(BlockingFailState::default()),
9749        });
9750
9751        let (writer, _reader) = transport.split();
9752
9753        let config = WebSocketConfig {
9754            url: "ws://127.0.0.1:1".to_string(),
9755            headers: vec![],
9756            heartbeat_interval_secs: Some(0),
9757            heartbeat_payload: None,
9758            connect_timeout_ms: None,
9759            reconnect_delay_initial_ms: None,
9760            reconnect_delay_max_ms: None,
9761            reconnect_backoff_factor: None,
9762            reconnect_jitter_ms: None,
9763            reconnect_max_attempts: None,
9764            heartbeat_timeout_secs: None,
9765            idle_timeout_ms: None,
9766            writer_capacity: None,
9767            backend: TransportBackend::Tungstenite,
9768            proxy_url: None,
9769            max_message_size_bytes: None,
9770            max_frame_size_bytes: None,
9771        };
9772
9773        let err = WebSocketClientInner::new_with_writer(config, writer)
9774            .await
9775            .expect_err("zero heartbeat should be rejected in stream mode");
9776        assert!(
9777            err.to_string()
9778                .contains("Heartbeat interval cannot be zero"),
9779            "error should mention zero heartbeat, was: {err}"
9780        );
9781    }
9782
9783    #[rstest]
9784    #[tokio::test]
9785    async fn test_connect_times_out_on_silent_server() {
9786        // A server that accepts TCP but never completes the WebSocket upgrade
9787        // must not hang connect() indefinitely
9788        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9789        let port = listener.local_addr().unwrap().port();
9790
9791        let server = task::spawn(async move {
9792            // Accept and hold the socket open without responding
9793            if let Ok((_stream, _)) = listener.accept().await {
9794                sleep(Duration::from_secs(30)).await;
9795            }
9796        });
9797
9798        let (handler, _rx) = channel_message_handler();
9799
9800        let config = WebSocketConfig {
9801            url: format!("ws://127.0.0.1:{port}"),
9802            headers: vec![],
9803            heartbeat_interval_secs: None,
9804            heartbeat_payload: None,
9805            connect_timeout_ms: Some(500),
9806            reconnect_delay_initial_ms: Some(50),
9807            reconnect_delay_max_ms: Some(100),
9808            reconnect_backoff_factor: Some(1.0),
9809            reconnect_jitter_ms: Some(0),
9810            reconnect_max_attempts: None,
9811            heartbeat_timeout_secs: None,
9812            idle_timeout_ms: None,
9813            writer_capacity: None,
9814            backend: TransportBackend::Tungstenite,
9815            proxy_url: None,
9816            max_message_size_bytes: None,
9817            max_frame_size_bytes: None,
9818        };
9819
9820        let result = tokio::time::timeout(
9821            Duration::from_secs(5),
9822            WebSocketClient::builder()
9823                .config(config)
9824                .message_handler(handler)
9825                .connect(),
9826        )
9827        .await
9828        .expect("connect should not hang on a silent server");
9829
9830        assert!(result.is_err(), "connect should fail with a timeout error");
9831        let err_msg = result.unwrap_err().to_string();
9832        assert!(
9833            err_msg.contains("timed out"),
9834            "error should mention the timeout, was: {err_msg}"
9835        );
9836
9837        server.abort();
9838    }
9839
9840    #[rstest]
9841    #[tokio::test]
9842    async fn test_reconnect_succeeds_with_timeout_shorter_than_swap_ceremony() {
9843        // Regression: the reconnect timeout used to cover the writer swap and
9844        // graceful-shutdown delays (~200ms minimum), so a short timeout caused
9845        // every otherwise-successful reconnect to be discarded mid-swap and the
9846        // already-swapped writer to be orphaned. The timeout now bounds only
9847        // connection establishment.
9848        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9849        let port = listener.local_addr().unwrap().port();
9850        let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel();
9851
9852        let server = task::spawn(async move {
9853            // Keep the first socket until connect() returns; an immediate drop
9854            // can fail the initial handshake
9855            if let Ok((stream, _)) = listener.accept().await
9856                && let Ok(ws) = accept_async(stream).await
9857            {
9858                let _ = release_first_rx.await;
9859                drop(ws);
9860            }
9861
9862            // A timed-out reconnect can consume an accept without completing the
9863            // handshake, so keep accepting and announce on every replacement
9864            loop {
9865                let Ok((stream, _)) = listener.accept().await else {
9866                    break;
9867                };
9868
9869                if let Ok(mut ws) = accept_async(stream).await {
9870                    loop {
9871                        if ws
9872                            .send(WsMessage::Text("reconnected-msg".to_string().into()))
9873                            .await
9874                            .is_err()
9875                        {
9876                            break;
9877                        }
9878
9879                        sleep(Duration::from_millis(50)).await;
9880                    }
9881                }
9882            }
9883        });
9884
9885        let (handler, mut rx) = channel_message_handler();
9886
9887        let config = WebSocketConfig {
9888            url: format!("ws://127.0.0.1:{port}"),
9889            headers: vec![],
9890            heartbeat_interval_secs: None,
9891            heartbeat_payload: None,
9892            connect_timeout_ms: Some(150), // Shorter than the ~200ms swap ceremony
9893            reconnect_delay_initial_ms: Some(25),
9894            reconnect_delay_max_ms: Some(50),
9895            reconnect_backoff_factor: Some(1.0),
9896            reconnect_jitter_ms: Some(0),
9897            reconnect_max_attempts: None,
9898            heartbeat_timeout_secs: None,
9899            idle_timeout_ms: None,
9900            writer_capacity: None,
9901            backend: TransportBackend::Tungstenite,
9902            proxy_url: None,
9903            max_message_size_bytes: None,
9904            max_frame_size_bytes: None,
9905        };
9906
9907        let client = WebSocketClient::builder()
9908            .config(config)
9909            .message_handler(handler)
9910            .connect()
9911            .await
9912            .expect("Initial WebSocket connection should succeed before triggering reconnect");
9913        wait_until_async(|| async { client.is_active() }, Duration::from_secs(2)).await;
9914        release_first_tx
9915            .send(())
9916            .expect("server should still be holding the first connection");
9917
9918        let received = tokio::time::timeout(Duration::from_secs(5), async {
9919            loop {
9920                match rx.recv().await {
9921                    Some(WsMessage::Text(text)) if text.as_str() == "reconnected-msg" => {
9922                        return true;
9923                    }
9924                    Some(_) => {}
9925                    None => return false,
9926                }
9927            }
9928        })
9929        .await
9930        .expect("Timed out waiting for a message from the replacement WebSocket connection");
9931
9932        assert!(
9933            received,
9934            "Replacement WebSocket message channel closed before delivering the reconnect message"
9935        );
9936        assert!(
9937            client.is_active(),
9938            "WebSocket client should be active after delivering the reconnect message"
9939        );
9940
9941        client.disconnect().await;
9942        server.abort();
9943    }
9944
9945    #[rstest]
9946    #[tokio::test]
9947    async fn test_idle_timeout_fires_under_ping_flood() {
9948        // Regression: the idle check used to run only when nothing arrived for a
9949        // full check interval (10ms), so pings flooding faster than that starved
9950        // it and a ping-only zombie connection never tripped the timeout
9951        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
9952        let port = listener.local_addr().unwrap().port();
9953        let server = task::spawn(async move {
9954            let (stream, _) = listener.accept().await.unwrap();
9955            let mut ws = accept_async(stream).await.unwrap();
9956
9957            for _ in 0..600 {
9958                sleep(Duration::from_millis(5)).await;
9959
9960                if ws.send(WsMessage::Ping(Vec::new().into())).await.is_err() {
9961                    break;
9962                }
9963            }
9964        });
9965
9966        let (handler, _rx) = channel_message_handler();
9967
9968        let config = WebSocketConfig {
9969            url: format!("ws://127.0.0.1:{port}"),
9970            headers: vec![],
9971            heartbeat_interval_secs: None,
9972            heartbeat_payload: None,
9973            connect_timeout_ms: Some(2_000),
9974            reconnect_delay_initial_ms: Some(50),
9975            reconnect_delay_max_ms: Some(100),
9976            reconnect_backoff_factor: Some(1.0),
9977            reconnect_jitter_ms: Some(0),
9978            reconnect_max_attempts: Some(1),
9979            heartbeat_timeout_secs: None,
9980            idle_timeout_ms: Some(500),
9981            writer_capacity: None,
9982            backend: TransportBackend::Tungstenite,
9983            proxy_url: None,
9984            max_message_size_bytes: None,
9985            max_frame_size_bytes: None,
9986        };
9987
9988        let client = WebSocketClient::builder()
9989            .config(config)
9990            .message_handler(handler)
9991            .connect()
9992            .await
9993            .unwrap();
9994
9995        assert!(client.is_active());
9996
9997        wait_until_async(
9998            || async { client.is_reconnecting() || client.is_disconnected() },
9999            Duration::from_millis(1_500),
10000        )
10001        .await;
10002
10003        assert!(
10004            !client.is_active(),
10005            "Client should not be active after idle timeout under a ping flood"
10006        );
10007
10008        client.disconnect().await;
10009        server.abort();
10010    }
10011
10012    #[rstest]
10013    #[tokio::test]
10014    async fn test_send_failure_does_not_overwrite_disconnect() {
10015        let state = Arc::new(BlockingFailState::default());
10016
10017        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10018            state: Arc::clone(&state),
10019        });
10020
10021        let (writer, _reader) = transport.split();
10022
10023        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10024        let state_notify = Arc::new(tokio::sync::Notify::new());
10025        let auth_tracker = Arc::new(OnceLock::new());
10026        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(false));
10027        let connection_epoch = Arc::new(AtomicU64::new(0));
10028
10029        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10030        let write_task = WebSocketClientInner::spawn_write_task(
10031            Arc::clone(&connection_state),
10032            Arc::clone(&state_notify),
10033            Arc::new(AtomicBool::new(true)),
10034            writer,
10035            writer_rx,
10036            connection_epoch,
10037            Arc::clone(&auth_tracker),
10038            Arc::clone(&reconnect_buffer_waits_for_auth),
10039            None,
10040        );
10041
10042        writer_tx
10043            .send(WriterCommand::Send(Message::text("doomed")))
10044            .unwrap();
10045
10046        // Wait until the writer task is blocked inside the transport send
10047        wait_until_async(
10048            || {
10049                let state = Arc::clone(&state);
10050                async move { state.send_entered.load(Ordering::SeqCst) }
10051            },
10052            Duration::from_secs(2),
10053        )
10054        .await;
10055
10056        // Disconnect lands while the send is in flight, then the send fails;
10057        // the writer error path must not overwrite DISCONNECT with RECONNECT
10058        connection_state.store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
10059        state.trigger_failure();
10060
10061        tokio::time::timeout(Duration::from_secs(2), write_task)
10062            .await
10063            .expect("write task should exit after disconnect")
10064            .unwrap();
10065
10066        assert_eq!(
10067            ConnectionMode::from_atomic(&connection_state),
10068            ConnectionMode::Disconnect,
10069            "Send failure must not resurrect a disconnecting client into RECONNECT"
10070        );
10071    }
10072
10073    #[tokio::test]
10074    async fn send_on_connection_write_failure_reports_broken_pipe_and_reconnects() {
10075        let state = Arc::new(BlockingFailState::default());
10076
10077        let transport: BoxedWsTransport = Box::pin(BlockingFailTransport {
10078            state: Arc::clone(&state),
10079        });
10080
10081        let (writer, _reader) = transport.split();
10082        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10083        let state_notify = Arc::new(tokio::sync::Notify::new());
10084        let connection_epoch = Arc::new(AtomicU64::new(0));
10085        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10086        let write_task = WebSocketClientInner::spawn_write_task(
10087            Arc::clone(&connection_state),
10088            Arc::clone(&state_notify),
10089            Arc::new(AtomicBool::new(true)),
10090            writer,
10091            writer_rx,
10092            Arc::clone(&connection_epoch),
10093            Arc::new(OnceLock::new()),
10094            Arc::new(AtomicBool::new(false)),
10095            None,
10096        );
10097
10098        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10099        writer_tx
10100            .send_control(WriterCommand::SendOnConnection {
10101                message: Message::text("doomed"),
10102                connection_epoch: 0,
10103                response_tx,
10104            })
10105            .unwrap();
10106
10107        wait_until_async(
10108            || {
10109                let state = Arc::clone(&state);
10110                async move { state.send_entered.load(Ordering::SeqCst) }
10111            },
10112            Duration::from_secs(2),
10113        )
10114        .await;
10115
10116        state.trigger_failure();
10117
10118        match response_rx.await.unwrap().unwrap_err() {
10119            SendError::BrokenPipe(message) => assert_eq!(message, "connection reset"),
10120            other => panic!("expected broken-pipe send error, was {other:?}"),
10121        }
10122
10123        wait_until_async(
10124            || async {
10125                ConnectionMode::from_atomic(&connection_state) == ConnectionMode::Reconnect
10126            },
10127            Duration::from_secs(2),
10128        )
10129        .await;
10130
10131        assert_eq!(
10132            ConnectionMode::from_atomic(&connection_state),
10133            ConnectionMode::Reconnect,
10134        );
10135        assert_eq!(connection_epoch.load(Ordering::Acquire), 0);
10136
10137        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10138        state_notify.notify_waiters();
10139        drop(writer_tx);
10140        write_task.await.unwrap();
10141    }
10142
10143    #[rstest]
10144    #[tokio::test]
10145    async fn test_send_bytes_on_connection_rejects_stale_epoch_without_replay() {
10146        let server = RecordingServer::setup().await;
10147        let (handler, _handler_rx) = channel_message_handler();
10148        let client = WebSocketClient::builder()
10149            .config(reconnect_test_config(server.port))
10150            .message_handler(handler)
10151            .connect()
10152            .await
10153            .unwrap();
10154        let stale_epoch = client.connection_epoch();
10155
10156        client
10157            .send_bytes_on_connection(vec![0x00, 0x7f, 0x80, 0xff], None, stale_epoch)
10158            .await
10159            .unwrap();
10160        assert!(client.request_reconnect());
10161        wait_until_async(
10162            || async { client.is_active() && client.connection_epoch() == stale_epoch + 1 },
10163            TEST_TIMEOUT,
10164        )
10165        .await;
10166
10167        let stale_result = client
10168            .send_bytes_on_connection(vec![0xde, 0xad], None, stale_epoch)
10169            .await;
10170        assert!(matches!(stale_result, Err(SendError::ConnectionChanged)));
10171
10172        client
10173            .send_bytes_on_connection(
10174                vec![0xfe, 0xed, 0xfa, 0xce],
10175                None,
10176                client.connection_epoch(),
10177            )
10178            .await
10179            .unwrap();
10180        wait_until_async(
10181            || {
10182                let binary_frames = Arc::clone(&server.binary_frames);
10183                async move { binary_frames.lock().await.len() == 2 }
10184            },
10185            TEST_TIMEOUT,
10186        )
10187        .await;
10188
10189        assert_eq!(
10190            server.binary_frames().await,
10191            vec![vec![0x00, 0x7f, 0x80, 0xff], vec![0xfe, 0xed, 0xfa, 0xce]],
10192        );
10193        assert!(server.messages().await.is_empty());
10194
10195        client.disconnect().await;
10196    }
10197
10198    #[tokio::test]
10199    async fn send_on_connection_rejects_stale_epoch_without_replay() {
10200        let server = RecordingServer::setup().await;
10201        let url = format!("ws://127.0.0.1:{}", server.port);
10202        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10203            &url,
10204            vec![],
10205            TransportBackend::Tungstenite,
10206            None,
10207            None,
10208            None,
10209        )
10210        .await
10211        .unwrap();
10212        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
10213        let state_notify = Arc::new(tokio::sync::Notify::new());
10214        let auth_tracker = Arc::new(OnceLock::new());
10215        let connection_epoch = Arc::new(AtomicU64::new(0));
10216        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10217        let write_task = WebSocketClientInner::spawn_write_task(
10218            Arc::clone(&connection_state),
10219            Arc::clone(&state_notify),
10220            Arc::new(AtomicBool::new(true)),
10221            writer,
10222            writer_rx,
10223            Arc::clone(&connection_epoch),
10224            auth_tracker,
10225            Arc::new(AtomicBool::new(false)),
10226            None,
10227        );
10228
10229        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10230        writer_tx
10231            .send_control(WriterCommand::SendOnConnection {
10232                message: Message::text("epoch-0"),
10233                connection_epoch: 0,
10234                response_tx,
10235            })
10236            .unwrap();
10237
10238        response_rx.await.unwrap().unwrap();
10239
10240        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10241        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10242        writer_tx
10243            .send_control(WriterCommand::SendOnConnection {
10244                message: Message::text("during-reconnect"),
10245                connection_epoch: 0,
10246                response_tx,
10247            })
10248            .unwrap();
10249
10250        assert!(matches!(
10251            response_rx.await.unwrap(),
10252            Err(SendError::ConnectionChanged),
10253        ));
10254
10255        let (replacement, _reader) = WebSocketClientInner::connect_with_server(
10256            &url,
10257            vec![],
10258            TransportBackend::Tungstenite,
10259            None,
10260            None,
10261            None,
10262        )
10263        .await
10264        .unwrap();
10265        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10266        writer_tx
10267            .send(WriterCommand::Update(replacement, update_tx))
10268            .unwrap();
10269        assert_eq!(update_rx.await.unwrap(), 1);
10270        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10271
10272        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10273        writer_tx
10274            .send_control(WriterCommand::SendOnConnection {
10275                message: Message::text("stale"),
10276                connection_epoch: 0,
10277                response_tx,
10278            })
10279            .unwrap();
10280
10281        assert!(matches!(
10282            response_rx.await.unwrap(),
10283            Err(SendError::ConnectionChanged),
10284        ));
10285
10286        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10287        writer_tx
10288            .send_control(WriterCommand::SendOnConnection {
10289                message: Message::text("epoch-1"),
10290                connection_epoch: 1,
10291                response_tx,
10292            })
10293            .unwrap();
10294
10295        response_rx.await.unwrap().unwrap();
10296
10297        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10298        let (second_replacement, _reader) = WebSocketClientInner::connect_with_server(
10299            &url,
10300            vec![],
10301            TransportBackend::Tungstenite,
10302            None,
10303            None,
10304            None,
10305        )
10306        .await
10307        .unwrap();
10308        let (update_tx, update_rx) = tokio::sync::oneshot::channel();
10309        writer_tx
10310            .send(WriterCommand::Update(second_replacement, update_tx))
10311            .unwrap();
10312        assert_eq!(update_rx.await.unwrap(), 2);
10313        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10314
10315        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10316        writer_tx
10317            .send_control(WriterCommand::SendOnConnection {
10318                message: Message::text("stale-after-second-reconnect"),
10319                connection_epoch: 1,
10320                response_tx,
10321            })
10322            .unwrap();
10323
10324        assert!(matches!(
10325            response_rx.await.unwrap(),
10326            Err(SendError::ConnectionChanged),
10327        ));
10328
10329        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10330        writer_tx
10331            .send_control(WriterCommand::SendOnConnection {
10332                message: Message::text("epoch-2"),
10333                connection_epoch: 2,
10334                response_tx,
10335            })
10336            .unwrap();
10337
10338        response_rx.await.unwrap().unwrap();
10339
10340        wait_until_async(
10341            || {
10342                let messages = Arc::clone(&server.messages);
10343                async move { messages.lock().await.len() == 3 }
10344            },
10345            Duration::from_secs(2),
10346        )
10347        .await;
10348
10349        assert_eq!(connection_epoch.load(Ordering::Acquire), 2);
10350        assert_eq!(
10351            server.messages().await,
10352            vec![
10353                "epoch-0".to_string(),
10354                "epoch-1".to_string(),
10355                "epoch-2".to_string(),
10356            ],
10357        );
10358
10359        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10360        state_notify.notify_waiters();
10361        drop(writer_tx);
10362        write_task.abort();
10363    }
10364
10365    #[rstest]
10366    fn test_reconnect_buffer_action_requires_active_mode() {
10367        let connection_state = AtomicU8::new(ConnectionMode::Active.as_u8());
10368        let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
10369        let auth_tracker = Arc::new(OnceLock::new());
10370        let tracker = AuthTracker::new();
10371        tracker.succeed();
10372        auth_tracker.set(tracker).unwrap();
10373
10374        assert_eq!(
10375            WebSocketClientInner::reconnect_buffer_action(
10376                &reconnect_buffer_waits_for_auth,
10377                &auth_tracker,
10378                &connection_state,
10379            ),
10380            ReconnectBufferAction::Drain,
10381        );
10382
10383        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
10384        assert_eq!(
10385            WebSocketClientInner::reconnect_buffer_action(
10386                &reconnect_buffer_waits_for_auth,
10387                &auth_tracker,
10388                &connection_state,
10389            ),
10390            ReconnectBufferAction::Wait,
10391        );
10392    }
10393
10394    #[tokio::test]
10395    async fn test_write_task_waits_for_auth_before_replaying_buffer() {
10396        use nautilus_common::testing::wait_until_async;
10397
10398        let server = RecordingServer::setup().await;
10399        let url = format!("ws://127.0.0.1:{}", server.port);
10400        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10401            &url,
10402            vec![],
10403            TransportBackend::Tungstenite,
10404            None,
10405            None,
10406            None,
10407        )
10408        .await
10409        .unwrap();
10410
10411        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10412        let state_notify = Arc::new(tokio::sync::Notify::new());
10413        let auth_tracker = Arc::new(OnceLock::new());
10414        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10415        let connection_epoch = Arc::new(AtomicU64::new(0));
10416        let tracker = AuthTracker::new();
10417        auth_tracker.set(tracker.clone()).unwrap();
10418
10419        let (writer_tx, writer_rx) = crate::writer::channel(1);
10420        let write_task = WebSocketClientInner::spawn_write_task(
10421            Arc::clone(&connection_state),
10422            Arc::clone(&state_notify),
10423            Arc::new(AtomicBool::new(true)),
10424            writer,
10425            writer_rx,
10426            Arc::clone(&connection_epoch),
10427            Arc::clone(&auth_tracker),
10428            Arc::clone(&reconnect_buffer_waits_for_auth),
10429            None,
10430        );
10431
10432        writer_tx
10433            .send(WriterCommand::Send(Message::Text("stale".into())))
10434            .unwrap();
10435
10436        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10437            &url,
10438            vec![],
10439            TransportBackend::Tungstenite,
10440            None,
10441            None,
10442            None,
10443        )
10444        .await
10445        .unwrap();
10446        let (tx, rx) = tokio::sync::oneshot::channel();
10447        writer_tx
10448            .send_update(WriterCommand::Update(new_writer, tx))
10449            .unwrap();
10450        assert_eq!(rx.await.unwrap(), 1);
10451
10452        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10453
10454        tokio::time::sleep(Duration::from_millis(300)).await;
10455        assert!(
10456            server.messages().await.is_empty(),
10457            "buffered messages should wait for re-authentication"
10458        );
10459
10460        assert!(matches!(
10461            writer_tx.send(WriterCommand::Send(Message::text("overflow"))),
10462            Err(SendError::BufferFull)
10463        ));
10464        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10465        writer_tx
10466            .send_control(WriterCommand::SendOnConnection {
10467                message: Message::text("authenticate"),
10468                connection_epoch: 1,
10469                response_tx,
10470            })
10471            .unwrap();
10472
10473        response_rx.await.unwrap().unwrap();
10474        tracker.succeed();
10475
10476        wait_until_async(
10477            || {
10478                let messages = Arc::clone(&server.messages);
10479                async move { messages.lock().await.len() == 2 }
10480            },
10481            Duration::from_secs(3),
10482        )
10483        .await;
10484
10485        assert_eq!(
10486            server.messages().await,
10487            vec!["authenticate".to_string(), "stale".to_string()]
10488        );
10489
10490        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10491        state_notify.notify_waiters();
10492        drop(writer_tx);
10493        write_task.abort();
10494    }
10495
10496    #[rstest]
10497    #[tokio::test]
10498    async fn test_write_task_buffers_send_behind_pending_replay() {
10499        use nautilus_common::testing::wait_until_async;
10500
10501        let server = RecordingServer::setup().await;
10502        let url = format!("ws://127.0.0.1:{}", server.port);
10503        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10504            &url,
10505            vec![],
10506            TransportBackend::Tungstenite,
10507            None,
10508            None,
10509            None,
10510        )
10511        .await
10512        .unwrap();
10513
10514        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10515        let state_notify = Arc::new(tokio::sync::Notify::new());
10516        let auth_tracker = Arc::new(OnceLock::new());
10517        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10518        let connection_epoch = Arc::new(AtomicU64::new(0));
10519        let tracker = AuthTracker::new();
10520        auth_tracker.set(tracker.clone()).unwrap();
10521
10522        // Two slots: the buffered message holds one while replay waits for auth, so a second
10523        // ordinary send is still admissible and reaches the writer loop.
10524        let (writer_tx, writer_rx) = crate::writer::channel(2);
10525        let write_task = WebSocketClientInner::spawn_write_task(
10526            Arc::clone(&connection_state),
10527            Arc::clone(&state_notify),
10528            Arc::new(AtomicBool::new(true)),
10529            writer,
10530            writer_rx,
10531            Arc::clone(&connection_epoch),
10532            Arc::clone(&auth_tracker),
10533            Arc::clone(&reconnect_buffer_waits_for_auth),
10534            None,
10535        );
10536
10537        writer_tx
10538            .send(WriterCommand::Send(Message::Text("stale".into())))
10539            .unwrap();
10540
10541        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10542            &url,
10543            vec![],
10544            TransportBackend::Tungstenite,
10545            None,
10546            None,
10547            None,
10548        )
10549        .await
10550        .unwrap();
10551        let (tx, rx) = tokio::sync::oneshot::channel();
10552        writer_tx
10553            .send_update(WriterCommand::Update(new_writer, tx))
10554            .unwrap();
10555        assert_eq!(rx.await.unwrap(), 1);
10556
10557        // Active with a non-empty replay buffer: the drain is gated on authentication.
10558        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10559        tokio::time::sleep(Duration::from_millis(300)).await;
10560        assert!(
10561            server.messages().await.is_empty(),
10562            "buffered message should wait for re-authentication"
10563        );
10564
10565        // A later ordinary send must not overtake the still-buffered message, nor bypass
10566        // the authentication gate by writing to the replacement connection.
10567        writer_tx
10568            .send(WriterCommand::Send(Message::Text("fresh".into())))
10569            .unwrap();
10570        tokio::time::sleep(Duration::from_millis(300)).await;
10571        assert!(
10572            server.messages().await.is_empty(),
10573            "a send arriving before the gate opens must not reach the connection, was {:?}",
10574            server.messages().await
10575        );
10576
10577        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10578        writer_tx
10579            .send_control(WriterCommand::SendOnConnection {
10580                message: Message::text("authenticate"),
10581                connection_epoch: 1,
10582                response_tx,
10583            })
10584            .unwrap();
10585
10586        response_rx.await.unwrap().unwrap();
10587        tracker.succeed();
10588
10589        wait_until_async(
10590            || {
10591                let messages = Arc::clone(&server.messages);
10592                async move { messages.lock().await.len() == 3 }
10593            },
10594            Duration::from_secs(3),
10595        )
10596        .await;
10597
10598        assert_eq!(
10599            server.messages().await,
10600            vec![
10601                "authenticate".to_string(),
10602                "stale".to_string(),
10603                "fresh".to_string(),
10604            ],
10605            "replay must preserve FIFO order behind the authentication handshake"
10606        );
10607
10608        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10609        state_notify.notify_waiters();
10610        drop(writer_tx);
10611        write_task.abort();
10612    }
10613
10614    #[rstest]
10615    #[tokio::test]
10616    async fn test_write_task_control_send_reaches_connection_behind_pending_replay() {
10617        use nautilus_common::testing::wait_until_async;
10618
10619        let server = RecordingServer::setup().await;
10620        let url = format!("ws://127.0.0.1:{}", server.port);
10621        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10622            &url,
10623            vec![],
10624            TransportBackend::Tungstenite,
10625            None,
10626            None,
10627            None,
10628        )
10629        .await
10630        .unwrap();
10631
10632        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10633        let state_notify = Arc::new(tokio::sync::Notify::new());
10634        let auth_tracker = Arc::new(OnceLock::new());
10635        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10636        let connection_epoch = Arc::new(AtomicU64::new(0));
10637        let tracker = AuthTracker::new();
10638        auth_tracker.set(tracker.clone()).unwrap();
10639
10640        let (writer_tx, writer_rx) = crate::writer::channel(2);
10641        let write_task = WebSocketClientInner::spawn_write_task(
10642            Arc::clone(&connection_state),
10643            Arc::clone(&state_notify),
10644            Arc::new(AtomicBool::new(true)),
10645            writer,
10646            writer_rx,
10647            Arc::clone(&connection_epoch),
10648            Arc::clone(&auth_tracker),
10649            Arc::clone(&reconnect_buffer_waits_for_auth),
10650            None,
10651        );
10652
10653        writer_tx
10654            .send(WriterCommand::Send(Message::Text("stale".into())))
10655            .unwrap();
10656
10657        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10658            &url,
10659            vec![],
10660            TransportBackend::Tungstenite,
10661            None,
10662            None,
10663            None,
10664        )
10665        .await
10666        .unwrap();
10667        let (tx, rx) = tokio::sync::oneshot::channel();
10668        writer_tx
10669            .send_update(WriterCommand::Update(new_writer, tx))
10670            .unwrap();
10671        assert_eq!(rx.await.unwrap(), 1);
10672
10673        // Active with a non-empty replay buffer: the drain is gated on authentication.
10674        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10675        tokio::time::sleep(Duration::from_millis(300)).await;
10676        assert!(
10677            server.messages().await.is_empty(),
10678            "buffered message should wait for re-authentication"
10679        );
10680
10681        // A control frame enqueued through the ordinary send path in the same window must not be
10682        // buffered with the replay: it has to reach the replacement connection while the gate is
10683        // still closed.
10684        writer_tx
10685            .send_control(WriterCommand::Send(Message::Pong(
10686                b"keepalive".to_vec().into(),
10687            )))
10688            .unwrap();
10689
10690        wait_until_async(
10691            || {
10692                let control_frames = Arc::clone(&server.control_frames);
10693                async move { !control_frames.lock().await.is_empty() }
10694            },
10695            Duration::from_secs(3),
10696        )
10697        .await;
10698
10699        assert_eq!(
10700            server.control_frames().await,
10701            vec!["pong:keepalive".to_string()],
10702            "a control frame sent during the wait must reach the new connection without replay"
10703        );
10704        assert!(
10705            server.messages().await.is_empty(),
10706            "ordinary sends must still wait for re-authentication, was {:?}",
10707            server.messages().await
10708        );
10709
10710        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
10711        writer_tx
10712            .send_control(WriterCommand::SendOnConnection {
10713                message: Message::text("authenticate"),
10714                connection_epoch: 1,
10715                response_tx,
10716            })
10717            .unwrap();
10718
10719        response_rx.await.unwrap().unwrap();
10720        tracker.succeed();
10721
10722        wait_until_async(
10723            || {
10724                let messages = Arc::clone(&server.messages);
10725                async move { messages.lock().await.len() == 2 }
10726            },
10727            Duration::from_secs(3),
10728        )
10729        .await;
10730
10731        assert_eq!(
10732            server.messages().await,
10733            vec!["authenticate".to_string(), "stale".to_string()],
10734            "the control frame must not disturb ordinary replay ordering"
10735        );
10736
10737        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10738        state_notify.notify_waiters();
10739        drop(writer_tx);
10740        write_task.abort();
10741    }
10742
10743    #[tokio::test]
10744    async fn test_write_task_discards_buffer_after_auth_failure() {
10745        let server = RecordingServer::setup().await;
10746        let url = format!("ws://127.0.0.1:{}", server.port);
10747        let (writer, _reader) = WebSocketClientInner::connect_with_server(
10748            &url,
10749            vec![],
10750            TransportBackend::Tungstenite,
10751            None,
10752            None,
10753            None,
10754        )
10755        .await
10756        .unwrap();
10757
10758        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
10759        let state_notify = Arc::new(tokio::sync::Notify::new());
10760        let auth_tracker = Arc::new(OnceLock::new());
10761        let reconnect_buffer_waits_for_auth = Arc::new(AtomicBool::new(true));
10762        let connection_epoch = Arc::new(AtomicU64::new(0));
10763        let tracker = AuthTracker::new();
10764        auth_tracker.set(tracker.clone()).unwrap();
10765
10766        let (writer_tx, writer_rx) = crate::writer::channel(crate::writer::DEFAULT_WRITER_CAPACITY);
10767        let write_task = WebSocketClientInner::spawn_write_task(
10768            Arc::clone(&connection_state),
10769            Arc::clone(&state_notify),
10770            Arc::new(AtomicBool::new(true)),
10771            writer,
10772            writer_rx,
10773            Arc::clone(&connection_epoch),
10774            Arc::clone(&auth_tracker),
10775            Arc::clone(&reconnect_buffer_waits_for_auth),
10776            None,
10777        );
10778
10779        writer_tx
10780            .send(WriterCommand::Send(Message::Text("stale".into())))
10781            .unwrap();
10782
10783        let (new_writer, _reader) = WebSocketClientInner::connect_with_server(
10784            &url,
10785            vec![],
10786            TransportBackend::Tungstenite,
10787            None,
10788            None,
10789            None,
10790        )
10791        .await
10792        .unwrap();
10793        let (tx, rx) = tokio::sync::oneshot::channel();
10794        writer_tx
10795            .send(WriterCommand::Update(new_writer, tx))
10796            .unwrap();
10797        assert_eq!(rx.await.unwrap(), 1);
10798
10799        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
10800        tracker.fail("rejected");
10801        tracker.invalidate();
10802        assert_eq!(tracker.auth_state(), AuthState::Failed);
10803        tokio::time::sleep(Duration::from_millis(300)).await;
10804        assert!(
10805            server.messages().await.is_empty(),
10806            "buffered messages should be discarded after authentication failure"
10807        );
10808
10809        let _auth_receiver = tracker.begin();
10810        tracker.succeed();
10811        tokio::time::sleep(Duration::from_millis(300)).await;
10812        assert!(
10813            server.messages().await.is_empty(),
10814            "discarded buffered messages should not replay on a later auth success"
10815        );
10816
10817        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
10818        state_notify.notify_waiters();
10819        drop(writer_tx);
10820        write_task.abort();
10821    }
10822
10823    #[rstest]
10824    #[tokio::test]
10825    async fn test_zero_idle_timeout_rejected() {
10826        let (handler, _rx) = channel_message_handler();
10827
10828        let config = WebSocketConfig {
10829            url: "ws://127.0.0.1:9999".to_string(),
10830            headers: vec![],
10831            heartbeat_interval_secs: None,
10832            heartbeat_payload: None,
10833            connect_timeout_ms: None,
10834            reconnect_delay_initial_ms: None,
10835            reconnect_delay_max_ms: None,
10836            reconnect_backoff_factor: None,
10837            reconnect_jitter_ms: None,
10838            reconnect_max_attempts: None,
10839            heartbeat_timeout_secs: None,
10840            idle_timeout_ms: Some(0),
10841            writer_capacity: None,
10842            backend: TransportBackend::Tungstenite,
10843            proxy_url: None,
10844            max_message_size_bytes: None,
10845            max_frame_size_bytes: None,
10846        };
10847
10848        let result = WebSocketClient::builder()
10849            .config(config)
10850            .message_handler(handler)
10851            .connect()
10852            .await;
10853
10854        assert!(result.is_err(), "Zero idle timeout should be rejected");
10855        let err_msg = result.unwrap_err().to_string();
10856        assert!(
10857            err_msg.contains("idle_timeout_ms"),
10858            "Error should name the offending field, was: {err_msg}"
10859        );
10860    }
10861
10862    #[rstest]
10863    #[tokio::test]
10864    async fn test_zero_heartbeat_timeout_rejected() {
10865        let (handler, _rx) = channel_message_handler();
10866
10867        let config = WebSocketConfig {
10868            url: "ws://127.0.0.1:9999".to_string(),
10869            headers: vec![],
10870            heartbeat_interval_secs: Some(30),
10871            heartbeat_payload: None,
10872            connect_timeout_ms: None,
10873            reconnect_delay_initial_ms: None,
10874            reconnect_delay_max_ms: None,
10875            reconnect_backoff_factor: None,
10876            reconnect_jitter_ms: None,
10877            reconnect_max_attempts: None,
10878            heartbeat_timeout_secs: Some(0),
10879            idle_timeout_ms: None,
10880            writer_capacity: None,
10881            backend: TransportBackend::Tungstenite,
10882            proxy_url: None,
10883            max_message_size_bytes: None,
10884            max_frame_size_bytes: None,
10885        };
10886
10887        let result = WebSocketClient::builder()
10888            .config(config)
10889            .message_handler(handler)
10890            .connect()
10891            .await;
10892
10893        assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
10894        let err_msg = result.unwrap_err().to_string();
10895        assert!(
10896            err_msg.contains("heartbeat_timeout_secs"),
10897            "Error should name the offending field, was: {err_msg}"
10898        );
10899    }
10900
10901    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
10902    #[rstest]
10903    #[tokio::test]
10904    async fn test_sockudo_backend_rejects_reserved_headers_before_connect() {
10905        let (handler, _rx) = channel_message_handler();
10906
10907        let config = WebSocketConfig {
10908            url: "ws://127.0.0.1:1".to_string(),
10909            headers: vec![("Host".to_string(), "example.com".to_string())],
10910            heartbeat_interval_secs: None,
10911            heartbeat_payload: None,
10912            connect_timeout_ms: None,
10913            reconnect_delay_initial_ms: None,
10914            reconnect_delay_max_ms: None,
10915            reconnect_backoff_factor: None,
10916            reconnect_jitter_ms: None,
10917            reconnect_max_attempts: None,
10918            heartbeat_timeout_secs: None,
10919            idle_timeout_ms: None,
10920            writer_capacity: None,
10921            backend: TransportBackend::Sockudo,
10922            proxy_url: None,
10923            max_message_size_bytes: None,
10924            max_frame_size_bytes: None,
10925        };
10926
10927        let err = WebSocketClient::builder()
10928            .config(config)
10929            .message_handler(handler)
10930            .connect()
10931            .await
10932            .expect_err("reserved header should fail before TCP connect");
10933
10934        assert!(
10935            err.to_string()
10936                .contains("reserved upgrade header not allowed in extra_headers"),
10937            "expected reserved-header failure, was: {err}"
10938        );
10939    }
10940
10941    // The log-capture harness is Linux-only for CI stability.
10942    #[cfg(all(feature = "transport-sockudo", target_os = "linux"))]
10943    #[rstest]
10944    #[tokio::test]
10945    async fn test_sockudo_reconnect_recovers_from_retryable_upgrade_rejection() {
10946        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
10947        let port = listener.local_addr().unwrap().port();
10948
10949        let server = task::spawn(async move {
10950            // First connection: accept the upgrade, then drop it to force a reconnect.
10951            let (stream, _) = listener.accept().await.unwrap();
10952            drop(accept_async(stream).await.unwrap());
10953
10954            // Second connection: reject the reconnect handshake with a retryable status.
10955            let (mut stream, _) = listener.accept().await.unwrap();
10956            let _request = read_http_request(&mut stream).await;
10957            stream
10958                .write_all(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
10959                .await
10960                .unwrap();
10961            stream.flush().await.unwrap();
10962            sleep(Duration::from_millis(50)).await;
10963            drop(stream);
10964
10965            // Third connection: accept the upgrade so the reconnect loop completes recovery.
10966            let (stream, _) = listener.accept().await.unwrap();
10967            let mut ws = accept_async(stream).await.unwrap();
10968            ws.send(WsMessage::Text("recovered".to_string().into()))
10969                .await
10970                .unwrap();
10971            sleep(Duration::from_secs(1)).await;
10972        });
10973
10974        let (handler, mut rx) = channel_message_handler();
10975
10976        let config = WebSocketConfig {
10977            url: format!(
10978                "ws://user-secret:password-secret@127.0.0.1:{port}/path-secret?token=query-secret"
10979            ),
10980            headers: vec![],
10981            heartbeat_interval_secs: None,
10982            heartbeat_payload: None,
10983            connect_timeout_ms: Some(2_000),
10984            reconnect_delay_initial_ms: Some(50),
10985            reconnect_delay_max_ms: Some(100),
10986            reconnect_backoff_factor: Some(1.0),
10987            reconnect_jitter_ms: Some(0),
10988            reconnect_max_attempts: None,
10989            heartbeat_timeout_secs: None,
10990            idle_timeout_ms: None,
10991            writer_capacity: None,
10992            backend: TransportBackend::Sockudo,
10993            proxy_url: None,
10994            max_message_size_bytes: None,
10995            max_frame_size_bytes: None,
10996        };
10997
10998        // Capture before connecting so the whole reconnect cycle is observed. The harness owns
10999        // the global logger, so the Nautilus logger that arms `shutdown_on_error` on an ERROR
11000        // record cannot run here; the absence of ERROR records is what rules the trigger out.
11001        let capture = capture_logs_for(RECONNECT_LOG_TARGETS).await;
11002
11003        let client = WebSocketClient::builder()
11004            .config(config)
11005            .message_handler(handler)
11006            .connect()
11007            .await
11008            .expect("sockudo connect should succeed");
11009
11010        let recovered = tokio::time::timeout(Duration::from_secs(10), async {
11011            loop {
11012                if let Ok(WsMessage::Text(text)) = rx.try_recv()
11013                    && text.as_str() == "recovered"
11014                {
11015                    return;
11016                }
11017
11018                tokio::time::sleep(Duration::from_millis(10)).await;
11019            }
11020        })
11021        .await;
11022
11023        let messages = capture.messages();
11024        client.disconnect().await;
11025        server.abort();
11026
11027        recovered.expect("client should recover after the retryable upgrade rejection");
11028
11029        let errors: Vec<_> = messages
11030            .iter()
11031            .filter(|(level, _)| *level == Level::Error)
11032            .collect();
11033        assert!(
11034            errors.is_empty(),
11035            "a retryable rejection must not log at ERROR, was: {errors:?}"
11036        );
11037
11038        // Asserted positively so the ERROR assertion above cannot pass by capturing nothing.
11039        assert!(
11040            messages.iter().any(|(level, message)| *level == Level::Warn
11041                && message.contains(&format!(
11042                    "Sockudo handshake rejected by 127.0.0.1:{port} with retryable status 502"
11043                ))),
11044            "expected a warning naming the retryable rejection, was: {messages:?}"
11045        );
11046
11047        for secret in [
11048            "user-secret",
11049            "password-secret",
11050            "path-secret",
11051            "query-secret",
11052        ] {
11053            assert!(
11054                messages
11055                    .iter()
11056                    .all(|(_, message)| !message.contains(secret)),
11057                "logs must not contain {secret}, was: {messages:?}"
11058            );
11059        }
11060    }
11061
11062    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11063    #[rstest]
11064    #[tokio::test]
11065    async fn test_sockudo_backend_replays_leftover_without_custom_headers() {
11066        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11067        let port = listener.local_addr().unwrap().port();
11068        let server = task::spawn(async move {
11069            if let Ok((mut stream, _)) = listener.accept().await {
11070                let request = read_http_request(&mut stream).await;
11071                let request = String::from_utf8(request).unwrap();
11072                let sec_websocket_key = extract_header(&request, "Sec-WebSocket-Key").unwrap();
11073                let accept = sockudo_handshake::generate_accept_key(sec_websocket_key);
11074                let mut response = format!(
11075                    concat!(
11076                        "HTTP/1.1 101 Switching Protocols\r\n",
11077                        "Upgrade: websocket\r\n",
11078                        "Connection: Upgrade\r\n",
11079                        "Sec-WebSocket-Accept: {}\r\n",
11080                        "\r\n",
11081                    ),
11082                    accept
11083                )
11084                .into_bytes();
11085                response.extend_from_slice(b"\x81\x05hello");
11086                stream.write_all(&response).await.unwrap();
11087            }
11088        });
11089
11090        let (handler, mut rx) = channel_message_handler();
11091
11092        let config = WebSocketConfig {
11093            url: format!("ws://127.0.0.1:{port}/ws"),
11094            headers: vec![],
11095            heartbeat_interval_secs: None,
11096            heartbeat_payload: None,
11097            connect_timeout_ms: Some(2_000),
11098            reconnect_delay_initial_ms: Some(50),
11099            reconnect_delay_max_ms: Some(100),
11100            reconnect_backoff_factor: Some(1.0),
11101            reconnect_jitter_ms: Some(0),
11102            reconnect_max_attempts: None,
11103            heartbeat_timeout_secs: None,
11104            idle_timeout_ms: None,
11105            writer_capacity: None,
11106            backend: TransportBackend::Sockudo,
11107            proxy_url: None,
11108            max_message_size_bytes: None,
11109            max_frame_size_bytes: None,
11110        };
11111
11112        let client = WebSocketClient::builder()
11113            .config(config)
11114            .message_handler(handler)
11115            .connect()
11116            .await
11117            .expect("sockudo connect without custom headers");
11118
11119        let received = tokio::time::timeout(Duration::from_secs(3), async {
11120            loop {
11121                if let Ok(msg) = rx.try_recv() {
11122                    return msg;
11123                }
11124
11125                tokio::time::sleep(Duration::from_millis(10)).await;
11126            }
11127        })
11128        .await
11129        .expect("did not receive leftover frame before timeout");
11130
11131        match received {
11132            WsMessage::Text(t) => assert_eq!(t.as_str(), "hello"),
11133            other => panic!("expected text, was {other:?}"),
11134        }
11135
11136        client.disconnect().await;
11137        tokio::time::timeout(Duration::from_secs(3), server)
11138            .await
11139            .expect("server did not close before timeout")
11140            .unwrap();
11141    }
11142
11143    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11144    #[rstest]
11145    #[tokio::test]
11146    async fn test_sockudo_backend_sends_custom_headers() {
11147        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11148        let port = listener.local_addr().unwrap().port();
11149        let server = task::spawn(async move {
11150            if let Ok((stream, _)) = listener.accept().await {
11151                let callback = HeaderAssertCallback {
11152                    key: "X-Test".to_string(),
11153                    value: HeaderValue::from_static("value"),
11154                };
11155
11156                if let Ok(mut ws) = accept_hdr_async(stream, callback).await {
11157                    while let Some(Ok(msg)) = ws.next().await {
11158                        if msg.is_text() || msg.is_binary() {
11159                            if ws.send(msg).await.is_err() {
11160                                break;
11161                            }
11162
11163                            continue;
11164                        }
11165
11166                        if msg.is_close() {
11167                            let _ = ws.close(None).await;
11168                            break;
11169                        }
11170                    }
11171                }
11172            }
11173        });
11174
11175        let (handler, mut rx) = channel_message_handler();
11176
11177        let config = WebSocketConfig {
11178            url: format!("ws://127.0.0.1:{port}"),
11179            headers: vec![("X-Test".to_string(), "value".to_string())],
11180            heartbeat_interval_secs: None,
11181            heartbeat_payload: None,
11182            connect_timeout_ms: Some(2_000),
11183            reconnect_delay_initial_ms: Some(50),
11184            reconnect_delay_max_ms: Some(100),
11185            reconnect_backoff_factor: Some(1.0),
11186            reconnect_jitter_ms: Some(0),
11187            reconnect_max_attempts: None,
11188            heartbeat_timeout_secs: None,
11189            idle_timeout_ms: None,
11190            writer_capacity: None,
11191            backend: TransportBackend::Sockudo,
11192            proxy_url: None,
11193            max_message_size_bytes: None,
11194            max_frame_size_bytes: None,
11195        };
11196
11197        let client = WebSocketClient::builder()
11198            .config(config)
11199            .message_handler(handler)
11200            .connect()
11201            .await
11202            .expect("sockudo connect with custom headers");
11203
11204        client.send_text("ping".to_string(), None).await.unwrap();
11205
11206        let received = tokio::time::timeout(Duration::from_secs(3), async {
11207            loop {
11208                if let Ok(msg) = rx.try_recv() {
11209                    return msg;
11210                }
11211
11212                tokio::time::sleep(Duration::from_millis(10)).await;
11213            }
11214        })
11215        .await
11216        .expect("did not receive echo before timeout");
11217
11218        match received {
11219            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11220            other => panic!("expected text, was {other:?}"),
11221        }
11222
11223        client.disconnect().await;
11224        tokio::time::timeout(Duration::from_secs(3), server)
11225            .await
11226            .expect("server did not close before timeout")
11227            .unwrap();
11228    }
11229
11230    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11231    #[rstest]
11232    #[tokio::test]
11233    async fn test_sockudo_backend_round_trip_text() {
11234        // tokio-tungstenite test peer paired with a sockudo client.
11235        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
11236        let port = listener.local_addr().unwrap().port();
11237        let server = task::spawn(async move {
11238            if let Ok((stream, _)) = listener.accept().await
11239                && let Ok(mut ws) = accept_async(stream).await
11240            {
11241                while let Some(Ok(msg)) = ws.next().await {
11242                    match msg {
11243                        WsMessage::Text(_) | WsMessage::Binary(_) => {
11244                            if ws.send(msg).await.is_err() {
11245                                break;
11246                            }
11247                        }
11248                        WsMessage::Close(_) => {
11249                            let _ = ws.close(None).await;
11250                            break;
11251                        }
11252                        _ => {}
11253                    }
11254                }
11255            }
11256        });
11257
11258        let (handler, mut rx) = channel_message_handler();
11259
11260        let config = WebSocketConfig {
11261            url: format!("ws://127.0.0.1:{port}"),
11262            headers: vec![],
11263            heartbeat_interval_secs: None,
11264            heartbeat_payload: None,
11265            connect_timeout_ms: Some(2_000),
11266            reconnect_delay_initial_ms: Some(50),
11267            reconnect_delay_max_ms: Some(100),
11268            reconnect_backoff_factor: Some(1.0),
11269            reconnect_jitter_ms: Some(0),
11270            reconnect_max_attempts: None,
11271            heartbeat_timeout_secs: None,
11272            idle_timeout_ms: None,
11273            writer_capacity: None,
11274            backend: TransportBackend::Sockudo,
11275            proxy_url: None,
11276            max_message_size_bytes: None,
11277            max_frame_size_bytes: None,
11278        };
11279
11280        let client = WebSocketClient::builder()
11281            .config(config)
11282            .message_handler(handler)
11283            .connect()
11284            .await
11285            .expect("sockudo connect");
11286
11287        client.send_text("ping".to_string(), None).await.unwrap();
11288
11289        let received = tokio::time::timeout(Duration::from_secs(3), async {
11290            loop {
11291                if let Ok(msg) = rx.try_recv() {
11292                    return msg;
11293                }
11294
11295                tokio::time::sleep(Duration::from_millis(10)).await;
11296            }
11297        })
11298        .await
11299        .expect("did not receive echo before timeout");
11300
11301        match received {
11302            WsMessage::Text(t) => assert_eq!(t.as_str(), "ping"),
11303            other => panic!("expected text, was {other:?}"),
11304        }
11305
11306        client.disconnect().await;
11307        server.abort();
11308    }
11309
11310    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11311    #[rstest]
11312    #[case::ws_default_port("ws://example.com/ws", "example.com", "example.com", 80, "/ws", false)]
11313    #[case::wss_default_port(
11314        "wss://example.com/ws",
11315        "example.com",
11316        "example.com",
11317        443,
11318        "/ws",
11319        true
11320    )]
11321    // url::Url normalizes explicit default ports (`:80` for ws, `:443` for wss)
11322    // away, so `parsed.port()` reports `None` here and Host stays unqualified.
11323    #[case::ws_explicit_default(
11324        "ws://example.com:80/ws",
11325        "example.com",
11326        "example.com",
11327        80,
11328        "/ws",
11329        false
11330    )]
11331    #[case::ws_non_default(
11332        "ws://example.com:8443/feed",
11333        "example.com",
11334        "example.com:8443",
11335        8443,
11336        "/feed",
11337        false
11338    )]
11339    #[case::wss_non_default(
11340        "wss://example.com:9443/feed",
11341        "example.com",
11342        "example.com:9443",
11343        9443,
11344        "/feed",
11345        true
11346    )]
11347    #[case::root_path(
11348        "ws://example.com:9000/",
11349        "example.com",
11350        "example.com:9000",
11351        9000,
11352        "/",
11353        false
11354    )]
11355    #[case::query_string(
11356        "ws://example.com/feed?token=abc&channel=trades",
11357        "example.com",
11358        "example.com",
11359        80,
11360        "/feed?token=abc&channel=trades",
11361        false
11362    )]
11363    // IPv6: bare host strips brackets for DNS/TCP/SNI; Host header keeps them.
11364    #[case::ipv6_default("ws://[::1]/feed", "::1", "[::1]", 80, "/feed", false)]
11365    #[case::ipv6_explicit_port("ws://[::1]:9000/feed", "::1", "[::1]:9000", 9000, "/feed", false)]
11366    #[case::ipv6_wss(
11367        "wss://[2001:db8::1]:8443/",
11368        "2001:db8::1",
11369        "[2001:db8::1]:8443",
11370        8443,
11371        "/",
11372        true
11373    )]
11374    fn sockudo_target_parses_url(
11375        #[case] url: &str,
11376        #[case] host: &str,
11377        #[case] host_header: &str,
11378        #[case] port: u16,
11379        #[case] path: &str,
11380        #[case] is_tls: bool,
11381    ) {
11382        let target = super::SockudoTarget::parse(url).expect("parse should succeed");
11383        assert_eq!(target.host, host);
11384        assert_eq!(target.host_header, host_header);
11385        assert_eq!(target.port, port);
11386        assert_eq!(target.path, path);
11387        assert_eq!(target.is_tls, is_tls);
11388    }
11389
11390    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11391    #[rstest]
11392    fn sockudo_target_rejects_unsupported_scheme() {
11393        let err = super::SockudoTarget::parse("http://example.com/feed").expect_err("not a ws URL");
11394        let msg = err.to_string();
11395        assert!(
11396            msg.contains("expected ws:// or wss://"),
11397            "unexpected error: {msg}"
11398        );
11399    }
11400
11401    #[cfg(all(feature = "transport-sockudo", not(feature = "turmoil")))]
11402    #[rstest]
11403    fn sockudo_target_rejects_malformed_url() {
11404        const SECRET: &str = "malformed-websocket-url-secret";
11405        let url = format!("not a url {SECRET}");
11406        let err = super::SockudoTarget::parse(&url).expect_err("malformed URL");
11407        let message = err.to_string();
11408
11409        assert!(
11410            matches!(err, super::TransportError::InvalidUrl(_)),
11411            "expected InvalidUrl, was: {err:?}"
11412        );
11413        assert!(!message.contains(SECRET));
11414        assert!(!message.contains(&url));
11415    }
11416}
11417
11418#[cfg(test)]
11419mod property_tests {
11420    use std::{
11421        collections::{HashSet, VecDeque},
11422        sync::{Arc, OnceLock, atomic::AtomicBool},
11423    };
11424
11425    use proptest::prelude::*;
11426    use rstest::rstest;
11427
11428    use super::{super::auth::AuthResultReceiver, *};
11429
11430    const AUTH_FAILED: &str = "model auth failed";
11431
11432    #[derive(Debug, Clone)]
11433    enum ReconnectBufferTraceOp {
11434        BeginAuth,
11435        AuthSucceeds,
11436        AuthFails,
11437        AuthInvalidates,
11438        ReconnectStarts,
11439        ReconnectCompletes,
11440        BufferedMessage(u8),
11441    }
11442
11443    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
11444    enum ModelConnectionMode {
11445        Active,
11446        Reconnect,
11447    }
11448
11449    #[derive(Debug, Clone, Copy)]
11450    enum ExpectedReconnectBufferAction {
11451        Drain,
11452        Wait,
11453        Discard,
11454    }
11455
11456    #[derive(Debug)]
11457    struct ReconnectBufferModel {
11458        mode: ModelConnectionMode,
11459        auth_state: AuthState,
11460        buffer: VecDeque<String>,
11461        released: Vec<String>,
11462        discarded: Vec<String>,
11463        live_sent: Vec<String>,
11464        handler_controls: Vec<&'static str>,
11465        next_message_index: usize,
11466    }
11467
11468    impl ReconnectBufferModel {
11469        fn new() -> Self {
11470            Self {
11471                mode: ModelConnectionMode::Active,
11472                auth_state: AuthState::Unauthenticated,
11473                buffer: VecDeque::new(),
11474                released: Vec::new(),
11475                discarded: Vec::new(),
11476                live_sent: Vec::new(),
11477                handler_controls: Vec::new(),
11478                next_message_index: 0,
11479            }
11480        }
11481
11482        fn next_payload(&mut self, raw: u8) -> String {
11483            let payload = format!("message-{}-{raw}", self.next_message_index);
11484            self.next_message_index += 1;
11485            payload
11486        }
11487
11488        fn expected_action(&self, waits_for_auth: bool) -> ExpectedReconnectBufferAction {
11489            if !waits_for_auth {
11490                return ExpectedReconnectBufferAction::Drain;
11491            }
11492
11493            match self.auth_state {
11494                AuthState::Authenticated => ExpectedReconnectBufferAction::Drain,
11495                AuthState::Failed => ExpectedReconnectBufferAction::Discard,
11496                AuthState::Unauthenticated => ExpectedReconnectBufferAction::Wait,
11497            }
11498        }
11499    }
11500
11501    fn reconnect_buffer_trace_op_strategy() -> impl Strategy<Value = ReconnectBufferTraceOp> {
11502        prop_oneof![
11503            Just(ReconnectBufferTraceOp::BeginAuth),
11504            Just(ReconnectBufferTraceOp::AuthSucceeds),
11505            Just(ReconnectBufferTraceOp::AuthFails),
11506            Just(ReconnectBufferTraceOp::AuthInvalidates),
11507            Just(ReconnectBufferTraceOp::ReconnectStarts),
11508            Just(ReconnectBufferTraceOp::ReconnectCompletes),
11509            any::<u8>().prop_map(ReconnectBufferTraceOp::BufferedMessage),
11510        ]
11511    }
11512
11513    fn reconnect_buffer_actions_match(
11514        actual: ReconnectBufferAction,
11515        expected: ExpectedReconnectBufferAction,
11516    ) -> bool {
11517        matches!(
11518            (actual, expected),
11519            (
11520                ReconnectBufferAction::Drain,
11521                ExpectedReconnectBufferAction::Drain
11522            ) | (
11523                ReconnectBufferAction::Wait,
11524                ExpectedReconnectBufferAction::Wait
11525            ) | (
11526                ReconnectBufferAction::Discard,
11527                ExpectedReconnectBufferAction::Discard
11528            )
11529        )
11530    }
11531
11532    fn apply_ready_reconnect_buffer_action(
11533        model: &mut ReconnectBufferModel,
11534        reconnect_buffer_waits_for_auth: &AtomicBool,
11535        auth_tracker: &Arc<OnceLock<AuthTracker>>,
11536        waits_for_auth: bool,
11537        step: usize,
11538        op: &ReconnectBufferTraceOp,
11539    ) -> Result<(), TestCaseError> {
11540        if model.mode != ModelConnectionMode::Active || model.buffer.is_empty() {
11541            return Ok(());
11542        }
11543
11544        let expected = model.expected_action(waits_for_auth);
11545        let actual = WebSocketClientInner::can_drain_reconnect_buffer(
11546            reconnect_buffer_waits_for_auth,
11547            auth_tracker,
11548        );
11549
11550        prop_assert!(
11551            reconnect_buffer_actions_match(actual, expected),
11552            "reconnect buffer action mismatch at step {}, op {:?}, waits_for_auth={}, auth_state={:?}",
11553            step,
11554            op,
11555            waits_for_auth,
11556            model.auth_state
11557        );
11558
11559        match expected {
11560            ExpectedReconnectBufferAction::Drain => {
11561                model.released.extend(model.buffer.drain(..));
11562            }
11563            ExpectedReconnectBufferAction::Wait => {}
11564            ExpectedReconnectBufferAction::Discard => {
11565                model.discarded.extend(model.buffer.drain(..));
11566            }
11567        }
11568
11569        Ok(())
11570    }
11571
11572    fn assert_reconnected_control_stays_separate(
11573        model: &ReconnectBufferModel,
11574        step: usize,
11575    ) -> Result<(), TestCaseError> {
11576        prop_assert!(
11577            model
11578                .handler_controls
11579                .iter()
11580                .all(|message| *message == RECONNECTED),
11581            "handler control stream contained a non-RECONNECTED message at step {}",
11582            step
11583        );
11584        prop_assert!(
11585            !model.buffer.iter().any(|message| message == RECONNECTED),
11586            "RECONNECTED control message entered reconnect buffer at step {}",
11587            step
11588        );
11589        prop_assert!(
11590            !model.released.iter().any(|message| message == RECONNECTED),
11591            "RECONNECTED control message entered replayed messages at step {}",
11592            step
11593        );
11594        prop_assert!(
11595            !model.discarded.iter().any(|message| message == RECONNECTED),
11596            "RECONNECTED control message entered discarded messages at step {}",
11597            step
11598        );
11599        prop_assert!(
11600            !model.live_sent.iter().any(|message| message == RECONNECTED),
11601            "RECONNECTED control message entered application sends at step {}",
11602            step
11603        );
11604
11605        Ok(())
11606    }
11607
11608    fn assert_messages_accounted_once(
11609        model: &ReconnectBufferModel,
11610        step: usize,
11611    ) -> Result<(), TestCaseError> {
11612        let mut seen = HashSet::new();
11613
11614        for message in model
11615            .released
11616            .iter()
11617            .chain(model.discarded.iter())
11618            .chain(model.buffer.iter())
11619            .chain(model.live_sent.iter())
11620        {
11621            prop_assert!(
11622                seen.insert(message.as_str()),
11623                "message {} appeared more than once at step {}",
11624                message,
11625                step
11626            );
11627        }
11628
11629        Ok(())
11630    }
11631
11632    fn apply_reconnect_buffer_trace_op(
11633        model: &mut ReconnectBufferModel,
11634        tracker: &AuthTracker,
11635        auth_receivers: &mut Vec<AuthResultReceiver>,
11636        op: &ReconnectBufferTraceOp,
11637    ) -> Result<(), TestCaseError> {
11638        match op {
11639            ReconnectBufferTraceOp::BeginAuth => {
11640                auth_receivers.push(tracker.begin());
11641                model.auth_state = AuthState::Unauthenticated;
11642            }
11643            ReconnectBufferTraceOp::AuthSucceeds => {
11644                tracker.succeed();
11645                model.auth_state = AuthState::Authenticated;
11646            }
11647            ReconnectBufferTraceOp::AuthFails => {
11648                tracker.fail(AUTH_FAILED);
11649                model.auth_state = AuthState::Failed;
11650            }
11651            ReconnectBufferTraceOp::AuthInvalidates => {
11652                tracker.invalidate();
11653
11654                if model.auth_state == AuthState::Authenticated {
11655                    model.auth_state = AuthState::Unauthenticated;
11656                }
11657            }
11658            ReconnectBufferTraceOp::ReconnectStarts => {
11659                tracker.invalidate();
11660
11661                if model.auth_state == AuthState::Authenticated {
11662                    model.auth_state = AuthState::Unauthenticated;
11663                }
11664
11665                model.mode = ModelConnectionMode::Reconnect;
11666            }
11667            ReconnectBufferTraceOp::ReconnectCompletes => {
11668                model.mode = ModelConnectionMode::Active;
11669                model.handler_controls.push(RECONNECTED);
11670            }
11671            ReconnectBufferTraceOp::BufferedMessage(raw) => {
11672                let payload = model.next_payload(*raw);
11673                prop_assert_ne!(payload.as_str(), RECONNECTED);
11674
11675                if model.mode == ModelConnectionMode::Reconnect {
11676                    model.buffer.push_back(payload);
11677                } else {
11678                    model.live_sent.push(payload);
11679                }
11680            }
11681        }
11682
11683        Ok(())
11684    }
11685
11686    proptest! {
11687        #![proptest_config(ProptestConfig::with_cases(256))]
11688
11689        /// Property: reconnect-buffer traces match the auth-gated release and
11690        /// discard model, and `RECONNECTED` remains a separate control signal.
11691        #[rstest]
11692        fn test_reconnect_buffer_trace_matches_auth_gate_model(
11693            waits_for_auth in any::<bool>(),
11694            ops in proptest::collection::vec(reconnect_buffer_trace_op_strategy(), 1..100)
11695        ) {
11696            let auth_tracker = Arc::new(OnceLock::new());
11697            let reconnect_buffer_waits_for_auth = AtomicBool::new(waits_for_auth);
11698            let tracker = AuthTracker::new();
11699            auth_tracker.set(tracker.clone()).unwrap();
11700            let mut auth_receivers = Vec::new();
11701            let mut model = ReconnectBufferModel::new();
11702
11703            for (step, op) in ops.iter().enumerate() {
11704                apply_reconnect_buffer_trace_op(
11705                    &mut model,
11706                    &tracker,
11707                    &mut auth_receivers,
11708                    op,
11709                )?;
11710
11711                prop_assert_eq!(
11712                    tracker.auth_state(),
11713                    model.auth_state,
11714                    "auth state mismatch at step {}, op {:?}",
11715                    step,
11716                    op
11717                );
11718
11719                apply_ready_reconnect_buffer_action(
11720                    &mut model,
11721                    &reconnect_buffer_waits_for_auth,
11722                    &auth_tracker,
11723                    waits_for_auth,
11724                    step,
11725                    op,
11726                )?;
11727                assert_reconnected_control_stays_separate(&model, step)?;
11728                prop_assert_eq!(
11729                    model.handler_controls.len(),
11730                    ops[..=step]
11731                        .iter()
11732                        .filter(|op| matches!(op, ReconnectBufferTraceOp::ReconnectCompletes))
11733                        .count(),
11734                    "handler control count mismatch at step {}",
11735                    step
11736                );
11737                assert_messages_accounted_once(&model, step)?;
11738            }
11739        }
11740
11741        /// Property: successful re-authentication releases buffered messages
11742        /// exactly once when replay is configured to wait for auth.
11743        #[rstest]
11744        fn test_reconnect_buffer_releases_after_auth_success_once(
11745            payloads in proptest::collection::vec(any::<u8>(), 1..32),
11746            extra_success_ticks in 0usize..16
11747        ) {
11748            let auth_tracker = Arc::new(OnceLock::new());
11749            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11750            let tracker = AuthTracker::new();
11751            auth_tracker.set(tracker.clone()).unwrap();
11752            let mut auth_receivers = Vec::new();
11753            let mut model = ReconnectBufferModel::new();
11754
11755            apply_reconnect_buffer_trace_op(
11756                &mut model,
11757                &tracker,
11758                &mut auth_receivers,
11759                &ReconnectBufferTraceOp::ReconnectStarts,
11760            )?;
11761            apply_reconnect_buffer_trace_op(
11762                &mut model,
11763                &tracker,
11764                &mut auth_receivers,
11765                &ReconnectBufferTraceOp::BeginAuth,
11766            )?;
11767
11768            for payload in payloads {
11769                apply_reconnect_buffer_trace_op(
11770                    &mut model,
11771                    &tracker,
11772                    &mut auth_receivers,
11773                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11774                )?;
11775            }
11776
11777            let buffered_len = model.buffer.len();
11778            apply_reconnect_buffer_trace_op(
11779                &mut model,
11780                &tracker,
11781                &mut auth_receivers,
11782                &ReconnectBufferTraceOp::ReconnectCompletes,
11783            )?;
11784            apply_ready_reconnect_buffer_action(
11785                &mut model,
11786                &reconnect_buffer_waits_for_auth,
11787                &auth_tracker,
11788                true,
11789                0,
11790                &ReconnectBufferTraceOp::ReconnectCompletes,
11791            )?;
11792
11793            prop_assert_eq!(model.released.len(), 0);
11794            prop_assert_eq!(model.buffer.len(), buffered_len);
11795
11796            apply_reconnect_buffer_trace_op(
11797                &mut model,
11798                &tracker,
11799                &mut auth_receivers,
11800                &ReconnectBufferTraceOp::AuthSucceeds,
11801            )?;
11802            apply_ready_reconnect_buffer_action(
11803                &mut model,
11804                &reconnect_buffer_waits_for_auth,
11805                &auth_tracker,
11806                true,
11807                1,
11808                &ReconnectBufferTraceOp::AuthSucceeds,
11809            )?;
11810
11811            prop_assert_eq!(model.released.len(), buffered_len);
11812            prop_assert!(model.buffer.is_empty());
11813            assert_messages_accounted_once(&model, 1)?;
11814
11815            for tick in 0..extra_success_ticks {
11816                apply_reconnect_buffer_trace_op(
11817                    &mut model,
11818                    &tracker,
11819                    &mut auth_receivers,
11820                    &ReconnectBufferTraceOp::AuthSucceeds,
11821                )?;
11822                apply_ready_reconnect_buffer_action(
11823                    &mut model,
11824                    &reconnect_buffer_waits_for_auth,
11825                    &auth_tracker,
11826                    true,
11827                    tick + 2,
11828                    &ReconnectBufferTraceOp::AuthSucceeds,
11829                )?;
11830                prop_assert_eq!(
11831                    model.released.len(),
11832                    buffered_len,
11833                    "buffered messages replayed more than once at tick {}",
11834                    tick
11835                );
11836            }
11837        }
11838
11839        /// Property: auth failure discards messages buffered before or after
11840        /// that failure, and later auth success does not replay discarded data.
11841        #[rstest]
11842        fn test_reconnect_buffer_discards_after_auth_failure(
11843            before_failure_payloads in proptest::collection::vec(any::<u8>(), 0..16),
11844            after_failure_payloads in proptest::collection::vec(any::<u8>(), 1..16),
11845            later_success_ticks in 0usize..16
11846        ) {
11847            let auth_tracker = Arc::new(OnceLock::new());
11848            let reconnect_buffer_waits_for_auth = AtomicBool::new(true);
11849            let tracker = AuthTracker::new();
11850            auth_tracker.set(tracker.clone()).unwrap();
11851            let mut auth_receivers = Vec::new();
11852            let mut model = ReconnectBufferModel::new();
11853
11854            apply_reconnect_buffer_trace_op(
11855                &mut model,
11856                &tracker,
11857                &mut auth_receivers,
11858                &ReconnectBufferTraceOp::ReconnectStarts,
11859            )?;
11860            apply_reconnect_buffer_trace_op(
11861                &mut model,
11862                &tracker,
11863                &mut auth_receivers,
11864                &ReconnectBufferTraceOp::BeginAuth,
11865            )?;
11866
11867            for payload in before_failure_payloads {
11868                apply_reconnect_buffer_trace_op(
11869                    &mut model,
11870                    &tracker,
11871                    &mut auth_receivers,
11872                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11873                )?;
11874            }
11875
11876            apply_reconnect_buffer_trace_op(
11877                &mut model,
11878                &tracker,
11879                &mut auth_receivers,
11880                &ReconnectBufferTraceOp::AuthFails,
11881            )?;
11882
11883            for payload in after_failure_payloads {
11884                apply_reconnect_buffer_trace_op(
11885                    &mut model,
11886                    &tracker,
11887                    &mut auth_receivers,
11888                    &ReconnectBufferTraceOp::BufferedMessage(payload),
11889                )?;
11890            }
11891
11892            let buffered_len = model.buffer.len();
11893            apply_reconnect_buffer_trace_op(
11894                &mut model,
11895                &tracker,
11896                &mut auth_receivers,
11897                &ReconnectBufferTraceOp::ReconnectCompletes,
11898            )?;
11899            apply_ready_reconnect_buffer_action(
11900                &mut model,
11901                &reconnect_buffer_waits_for_auth,
11902                &auth_tracker,
11903                true,
11904                0,
11905                &ReconnectBufferTraceOp::ReconnectCompletes,
11906            )?;
11907
11908            prop_assert_eq!(model.discarded.len(), buffered_len);
11909            prop_assert!(model.released.is_empty());
11910            prop_assert!(model.buffer.is_empty());
11911            assert_messages_accounted_once(&model, 0)?;
11912
11913            for tick in 0..later_success_ticks {
11914                apply_reconnect_buffer_trace_op(
11915                    &mut model,
11916                    &tracker,
11917                    &mut auth_receivers,
11918                    &ReconnectBufferTraceOp::BeginAuth,
11919                )?;
11920                apply_reconnect_buffer_trace_op(
11921                    &mut model,
11922                    &tracker,
11923                    &mut auth_receivers,
11924                    &ReconnectBufferTraceOp::AuthSucceeds,
11925                )?;
11926                apply_ready_reconnect_buffer_action(
11927                    &mut model,
11928                    &reconnect_buffer_waits_for_auth,
11929                    &auth_tracker,
11930                    true,
11931                    tick + 1,
11932                    &ReconnectBufferTraceOp::AuthSucceeds,
11933                )?;
11934                prop_assert!(
11935                    model.released.is_empty(),
11936                    "discarded messages replayed after later auth success at tick {}",
11937                    tick
11938                );
11939            }
11940        }
11941    }
11942}
11943
11944#[cfg(test)]
11945#[cfg(feature = "turmoil")]
11946mod turmoil_tests {
11947    use std::{sync::Arc, time::Duration};
11948
11949    use futures_util::{SinkExt, StreamExt};
11950    use nautilus_common::testing::wait_until_async;
11951    use rstest::rstest;
11952    use tokio_tungstenite::{accept_async, tungstenite::Message as WsMessage};
11953    use turmoil::{Builder, net};
11954
11955    use super::*;
11956    use crate::websocket::types::channel_message_handler;
11957
11958    const AUTH_BUFFER_WAIT_SEED: u64 = 0xA17B_0001;
11959    const AUTH_BUFFER_DISCARD_SEED: u64 = 0xA17B_0002;
11960
11961    fn seeded_turmoil_builder(seed: u64) -> Builder {
11962        let mut builder = Builder::new();
11963        builder.rng_seed(seed);
11964        builder
11965    }
11966
11967    #[rstest]
11968    fn test_turmoil_reconnect_buffer_waits_for_auth() {
11969        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_WAIT_SEED).build();
11970        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
11971        let server_messages = Arc::clone(&messages);
11972        sim.host("server", move || {
11973            let messages = Arc::clone(&server_messages);
11974            auth_buffer_server(messages)
11975        });
11976
11977        sim.client("client", async move {
11978            let tracker = AuthTracker::new();
11979            let (handler, _rx) = channel_message_handler();
11980            let client = WebSocketClient::builder()
11981                .config(turmoil_websocket_config())
11982                .message_handler(handler)
11983                .connect()
11984                .await
11985                .expect("Should connect");
11986
11987            client.set_auth_tracker(tracker.clone(), true);
11988            assert!(client.is_active(), "Client should start active");
11989
11990            wait_until_async(
11991                || async { client.is_reconnecting() },
11992                Duration::from_secs(3),
11993            )
11994            .await;
11995
11996            client
11997                .writer_tx
11998                .send(WriterCommand::Send(Message::Text("stale".into())))
11999                .unwrap();
12000
12001            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12002
12003            let _auth_receiver = tracker.begin();
12004
12005            tokio::time::sleep(Duration::from_millis(300)).await;
12006            assert!(
12007                messages.lock().await.is_empty(),
12008                "buffered messages should wait for auth after reconnect"
12009            );
12010
12011            tracker.succeed();
12012
12013            wait_until_async(
12014                || {
12015                    let messages = Arc::clone(&messages);
12016                    async move { messages.lock().await.as_slice() == ["stale"] }
12017                },
12018                Duration::from_secs(3),
12019            )
12020            .await;
12021
12022            assert_eq!(messages.lock().await.as_slice(), ["stale"]);
12023
12024            client.disconnect().await;
12025            assert!(client.is_disconnected());
12026
12027            Ok(())
12028        });
12029
12030        sim.run().unwrap();
12031    }
12032
12033    #[rstest]
12034    fn test_turmoil_reconnect_buffer_discards_after_auth_failure() {
12035        let mut sim = seeded_turmoil_builder(AUTH_BUFFER_DISCARD_SEED).build();
12036        let messages = Arc::new(tokio::sync::Mutex::new(Vec::new()));
12037        let server_messages = Arc::clone(&messages);
12038        sim.host("server", move || {
12039            let messages = Arc::clone(&server_messages);
12040            auth_buffer_server(messages)
12041        });
12042
12043        sim.client("client", async move {
12044            let tracker = AuthTracker::new();
12045            let (handler, _rx) = channel_message_handler();
12046            let client = WebSocketClient::builder()
12047                .config(turmoil_websocket_config())
12048                .message_handler(handler)
12049                .connect()
12050                .await
12051                .expect("Should connect");
12052
12053            client.set_auth_tracker(tracker.clone(), true);
12054            assert!(client.is_active(), "Client should start active");
12055
12056            wait_until_async(
12057                || async { client.is_reconnecting() },
12058                Duration::from_secs(3),
12059            )
12060            .await;
12061
12062            client
12063                .writer_tx
12064                .send(WriterCommand::Send(Message::Text("stale".into())))
12065                .unwrap();
12066
12067            wait_until_async(|| async { client.is_active() }, Duration::from_secs(3)).await;
12068
12069            let _auth_receiver = tracker.begin();
12070            tracker.fail("rejected");
12071
12072            tokio::time::sleep(Duration::from_millis(300)).await;
12073            assert!(
12074                messages.lock().await.is_empty(),
12075                "buffered messages should be discarded after auth failure"
12076            );
12077
12078            let _retry_auth_receiver = tracker.begin();
12079            tracker.succeed();
12080
12081            tokio::time::sleep(Duration::from_millis(300)).await;
12082            assert!(
12083                messages.lock().await.is_empty(),
12084                "discarded messages should not replay on a later auth success"
12085            );
12086
12087            client.disconnect().await;
12088            assert!(client.is_disconnected());
12089
12090            Ok(())
12091        });
12092
12093        sim.run().unwrap();
12094    }
12095
12096    fn turmoil_websocket_config() -> WebSocketConfig {
12097        WebSocketConfig {
12098            url: "ws://server:8080".to_string(),
12099            headers: vec![],
12100            heartbeat_interval_secs: None,
12101            heartbeat_payload: None,
12102            connect_timeout_ms: Some(5_000),
12103            reconnect_delay_initial_ms: Some(50),
12104            reconnect_delay_max_ms: Some(200),
12105            reconnect_backoff_factor: Some(1.0),
12106            reconnect_jitter_ms: Some(0),
12107            reconnect_max_attempts: None,
12108            heartbeat_timeout_secs: None,
12109            idle_timeout_ms: None,
12110            writer_capacity: None,
12111            backend: TransportBackend::Tungstenite,
12112            proxy_url: None,
12113            max_message_size_bytes: None,
12114            max_frame_size_bytes: None,
12115        }
12116    }
12117
12118    async fn auth_buffer_server(
12119        messages: Arc<tokio::sync::Mutex<Vec<String>>>,
12120    ) -> Result<(), Box<dyn std::error::Error>> {
12121        let listener = net::TcpListener::bind("0.0.0.0:8080").await?;
12122
12123        let (stream, _) = listener.accept().await?;
12124        let mut websocket = accept_async(stream).await?;
12125        let _ = websocket.send(WsMessage::Text("first".into())).await;
12126        drop(websocket);
12127
12128        tokio::time::sleep(Duration::from_millis(200)).await;
12129
12130        let (stream, _) = listener.accept().await?;
12131        let mut websocket = accept_async(stream).await?;
12132
12133        while let Some(msg) = websocket.next().await {
12134            match msg {
12135                Ok(WsMessage::Text(text)) => {
12136                    messages.lock().await.push(text.to_string());
12137                }
12138                Ok(WsMessage::Close(_)) => {
12139                    let _ = websocket.close(None).await;
12140                    break;
12141                }
12142                Ok(_) => {}
12143                Err(_) => break,
12144            }
12145        }
12146
12147        Ok(())
12148    }
12149}