Skip to main content

nautilus_network/socket/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Raw TCP client with optional TLS, suffix framing, heartbeats, and automatic reconnection.
17//!
18//! # State management
19//!
20//! The client tracks active, reconnecting, disconnecting, and closed states. State changes notify
21//! waiting tasks immediately instead of relying only on periodic checks.
22//!
23//! # Connection ownership
24//!
25//! A controller owns the connection lifecycle. A dedicated reader task passes complete messages to
26//! the configured callback, while a dedicated writer task serializes concurrent sends received over
27//! a channel.
28//!
29//! # Framing and heartbeats
30//!
31//! The configured suffix frames the byte stream in both directions. The writer appends it to sent
32//! messages and heartbeats, and the reader uses it to split incoming data into complete messages.
33//! Heartbeats are optional and run in a separate task.
34//!
35//! # Reconnection
36//!
37//! The writer buffers messages while reconnecting. A successful reconnect installs the replacement
38//! writer, drains that buffer, restarts the reader, and then invokes the configured
39//! post-reconnection callback.
40
41use std::{
42    collections::VecDeque,
43    fmt::Debug,
44    path::Path,
45    pin::pin,
46    sync::{
47        Arc,
48        atomic::{AtomicU8, Ordering},
49    },
50    time::Duration,
51};
52
53use bytes::Bytes;
54use nautilus_core::string::secret::REDACTED;
55use nautilus_cryptography::providers::install_cryptographic_provider;
56use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
57use tokio_tungstenite::tungstenite::{Error, client::IntoClientRequest, stream::Mode};
58
59use super::{SocketConfig, TcpMessageHandler, TcpReader, TcpWriter, WriterCommand};
60use crate::{
61    SocketStateSink,
62    backoff::{
63        ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
64    },
65    dst,
66    error::{SendError, is_connection_drop_io_error},
67    logging::{log_task_aborted, log_task_started, log_task_stopped},
68    mode::{
69        ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
70        ReconnectRequestOutcome,
71    },
72    net::TcpStream,
73    tls::{create_tls_config_from_certs_dir, tcp_tls},
74    writer::{self, DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
75};
76
77// Connection timing constants
78const CONNECTION_STATE_CHECK_INTERVAL_MS: u64 = 10;
79const GRACEFUL_SHUTDOWN_DELAY_MS: u64 = 100;
80const GRACEFUL_SHUTDOWN_TIMEOUT_SECS: u64 = 5;
81const WRITE_TIMEOUT_SECS: u64 = 5;
82
83// Maximum buffer size for read operations (10 MB)
84const MAX_READ_BUFFER_BYTES: usize = 10 * 1024 * 1024;
85
86struct BufferedWrite {
87    data: Bytes,
88    replay_key: Option<u64>,
89    _permit: tokio::sync::OwnedSemaphorePermit,
90}
91
92/// Produces protocol messages that must precede buffered application writes after reconnect.
93pub type SocketReconnectReplay = Arc<dyn Fn() -> Vec<Bytes> + Send + Sync>;
94
95struct SocketClientInner {
96    config: SocketConfig,
97    connector: Option<Arc<rustls::ClientConfig>>,
98    read_task: tokio::task::JoinHandle<()>,
99    read_fence: ReadSessionFence,
100    write_task: tokio::task::JoinHandle<()>,
101    writer_tx: WriterSender<WriterCommand>,
102    heartbeat_task: Option<tokio::task::JoinHandle<()>>,
103    connection_mode: Arc<AtomicU8>,
104    state_notify: Arc<tokio::sync::Notify>,
105    connect_timeout: Duration,
106    backoff: ExponentialBackoff,
107    reconnect_throttle: ReconnectThrottle,
108    reconnect_max_attempts: Option<u32>,
109    reconnect_attempt_count: u32,
110    state_sink: Option<SocketStateSink>,
111}
112
113impl SocketClientInner {
114    /// Connects to a URL with the specified configuration.
115    ///
116    /// # Errors
117    ///
118    /// Returns an error if connection fails or configuration is invalid.
119    async fn connect_url(
120        config: SocketConfig,
121        state_sink: Option<SocketStateSink>,
122    ) -> anyhow::Result<Self> {
123        install_cryptographic_provider();
124
125        // Validate suffix is non-empty to prevent panic in read loop (windows(0) panics)
126        if config.suffix.is_empty() {
127            anyhow::bail!("Socket suffix cannot be empty: suffix is required for message framing");
128        }
129
130        // Adapters build this config by struct literal, bypassing the builder, so this is the only
131        // place the field invariants are enforced for them.
132        config.validate()?;
133
134        let connect_timeout = Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000));
135
136        let reconnect_backoff = ExponentialBackoff::new(
137            Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
138            Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
139            config.reconnect_backoff_factor.unwrap_or(1.5),
140            config.reconnect_jitter_ms.unwrap_or(100),
141            true, // immediate-first
142        )?;
143
144        let connector = if let Some(dir) = &config.certs_dir {
145            let config = create_tls_config_from_certs_dir(Path::new(dir), false)?;
146            Some(Arc::new(config))
147        } else {
148            None
149        };
150
151        // Retry initial connection with exponential backoff to handle transient DNS/network issues
152        let max_retries = config.connection_max_retries.unwrap_or(5);
153
154        let mut backoff = ExponentialBackoff::new(
155            Duration::from_millis(500),
156            Duration::from_secs(5),
157            2.0,
158            250,
159            false,
160        )?;
161
162        let mut attempt = 0;
163        let (reader, writer) = loop {
164            attempt += 1;
165
166            let last_error = match dst::time::timeout(
167                connect_timeout,
168                Self::tls_connect_with_server(&config.url, config.mode, connector.clone()),
169            )
170            .await
171            {
172                Ok(Ok(result)) => {
173                    if attempt > 1 {
174                        log::info!("Socket connection established after {attempt} attempts");
175                    }
176
177                    break result;
178                }
179                Ok(Err(e)) => {
180                    let error = e.to_string();
181                    log::warn!(
182                        "Socket connection attempt {attempt}/{max_retries} to {REDACTED} failed: {error}"
183                    );
184                    error
185                }
186                Err(_) => {
187                    let error = format!(
188                        "Connection timeout after {:.1}s (possible DNS resolution failure)",
189                        connect_timeout.as_secs_f64()
190                    );
191                    log::warn!(
192                        "Socket connection attempt {attempt}/{max_retries} to {REDACTED} timed out"
193                    );
194                    error
195                }
196            };
197
198            if attempt >= max_retries {
199                anyhow::bail!(
200                    "Failed to connect to {REDACTED} after {max_retries} attempts: {last_error}. \
201                    If this is a DNS error, check your network configuration and DNS settings."
202                );
203            }
204
205            let delay = backoff.next_duration();
206            log::debug!(
207                "Retrying in {delay:?} (attempt {}/{})",
208                attempt + 1,
209                max_retries
210            );
211            dst::time::sleep(delay).await;
212        };
213
214        log::debug!("Connected");
215
216        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
217        let state_notify = Arc::new(tokio::sync::Notify::new());
218        let outcome =
219            ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
220        debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
221        let read_fence = ReadSessionFence::new();
222
223        let read_task = Self::spawn_read_task(
224            Arc::clone(&connection_mode),
225            read_fence.clone(),
226            reader,
227            config.message_handler.clone(),
228            config.suffix.clone(),
229            config.resolved_heartbeat_timeout(),
230        );
231
232        let (writer_tx, writer_rx) = writer::channel::<WriterCommand>(
233            config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
234        );
235
236        let write_task = Self::spawn_write_task(
237            Arc::clone(&connection_mode),
238            Arc::clone(&state_notify),
239            writer,
240            writer_rx,
241            config.suffix.clone(),
242            state_sink.clone(),
243        );
244
245        let heartbeat_task = config.heartbeat.as_ref().map(|heartbeat| {
246            Self::spawn_heartbeat_task(
247                Arc::clone(&connection_mode),
248                heartbeat.interval_secs,
249                heartbeat.payload.clone(),
250                writer_tx.clone(),
251            )
252        });
253
254        let reconnect_max_attempts = config.reconnect_max_attempts;
255
256        Ok(Self {
257            config,
258            connector,
259            read_task,
260            read_fence,
261            write_task,
262            writer_tx,
263            heartbeat_task,
264            connection_mode,
265            state_notify,
266            connect_timeout,
267            backoff: reconnect_backoff,
268            reconnect_throttle: ReconnectThrottle::default(),
269            reconnect_max_attempts,
270            reconnect_attempt_count: 0,
271            state_sink,
272        })
273    }
274
275    /// Parses a URL into its socket address and request URL.
276    ///
277    /// Accepts either:
278    /// - Raw socket address: "host:port" → returns ("host:port", "scheme://host:port")
279    /// - Full URL: "scheme://host:port/path" → returns ("host:port", original URL)
280    ///
281    /// # Errors
282    ///
283    /// Returns an error if the URL is invalid or missing required components.
284    fn parse_socket_url(url: &str, mode: Mode) -> Result<(String, String), Error> {
285        if url.contains("://") {
286            // URL with scheme (e.g., "wss://host:port/path")
287            let parsed = url.parse::<http::Uri>().map_err(|e| {
288                Error::Io(std::io::Error::new(
289                    std::io::ErrorKind::InvalidInput,
290                    format!("Invalid URL: {e}"),
291                ))
292            })?;
293
294            let host = parsed.host().ok_or_else(|| {
295                Error::Io(std::io::Error::new(
296                    std::io::ErrorKind::InvalidInput,
297                    "URL missing host",
298                ))
299            })?;
300
301            let port = parsed
302                .port_u16()
303                .unwrap_or_else(|| match parsed.scheme_str() {
304                    Some("wss" | "https") => 443,
305                    Some("ws" | "http") => 80,
306                    _ => match mode {
307                        Mode::Tls => 443,
308                        Mode::Plain => 80,
309                    },
310                });
311
312            Ok((format!("{host}:{port}"), url.to_string()))
313        } else {
314            // Raw socket address (e.g., "host:port")
315            // Construct a proper URL for the request based on mode
316            let scheme = match mode {
317                Mode::Tls => "wss",
318                Mode::Plain => "ws",
319            };
320
321            Ok((url.to_string(), format!("{scheme}://{url}")))
322        }
323    }
324
325    /// Establish a TLS or plain TCP connection with the server.
326    ///
327    /// Accepts either a raw socket address (e.g., "host:port") or a full URL with scheme
328    /// (e.g., "wss://host:port"). For FIX/raw socket connections, use the host:port format.
329    /// For WebSocket-style connections, include the scheme.
330    ///
331    /// # Errors
332    ///
333    /// Returns an error if the connection cannot be established.
334    pub(crate) async fn tls_connect_with_server(
335        url: &str,
336        mode: Mode,
337        connector: Option<Arc<rustls::ClientConfig>>,
338    ) -> Result<(TcpReader, TcpWriter), Error> {
339        log::debug!("Connecting to {REDACTED}");
340
341        let (socket_addr, request_url) = Self::parse_socket_url(url, mode)?;
342        let tcp_result = TcpStream::connect(&socket_addr).await;
343
344        match tcp_result {
345            Ok(stream) => {
346                log::debug!("TCP connection established to {socket_addr}, proceeding with TLS");
347
348                crate::net::apply_socket_options(&stream);
349
350                let request = request_url.into_client_request()?;
351                tcp_tls(&request, mode, stream, connector)
352                    .await
353                    .map(tokio::io::split)
354            }
355            Err(e) => {
356                log::warn!("TCP connection failed to {socket_addr}: {e:?}");
357                Err(Error::Io(e))
358            }
359        }
360    }
361
362    /// Reconnects to the server.
363    ///
364    /// Makes a new connection with server, uses the new read and write halves
365    /// to update the reader and writer.
366    ///
367    /// The reconnect timeout bounds only connection establishment. Once the
368    /// new writer is handed to the writer task the swap runs to completion,
369    /// so buffered messages can never drain into a connection that lost its
370    /// reader to a timeout; the writer task bounds both the old-writer
371    /// shutdown and the buffer drain with its graceful-shutdown timeout.
372    async fn reconnect(
373        &mut self,
374        reconnect_replay: Option<&SocketReconnectReplay>,
375    ) -> Result<ReconnectOutcome, Error> {
376        log::info!("Reconnecting");
377
378        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
379            log::debug!("Reconnect aborted due to disconnect state");
380            return Ok(ReconnectOutcome::Aborted);
381        }
382
383        // Bound only connection establishment; the swap below must run to completion
384        let (reader, new_writer) = dst::time::timeout(
385            self.connect_timeout,
386            Self::tls_connect_with_server(
387                &self.config.url,
388                self.config.mode,
389                self.connector.clone(),
390            ),
391        )
392        .await
393        .map_err(|_| {
394            Error::Io(std::io::Error::new(
395                std::io::ErrorKind::TimedOut,
396                format!(
397                    "reconnection timed out after {}s",
398                    self.connect_timeout.as_secs_f64()
399                ),
400            ))
401        })??;
402
403        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
404            log::debug!("Reconnect aborted mid-flight (after connect)");
405            return Ok(ReconnectOutcome::Aborted);
406        }
407
408        log::debug!("Connected");
409
410        // Use a oneshot channel to synchronize with the writer task.
411        // We must verify that the buffer was successfully drained before transitioning to ACTIVE
412        // to prevent silent message loss if the new connection drops immediately.
413        let (tx, rx) = tokio::sync::oneshot::channel();
414        let command = if let Some(reconnect_replay) = reconnect_replay {
415            WriterCommand::UpdateWithReplay(new_writer, reconnect_replay(), tx)
416        } else {
417            WriterCommand::Update(new_writer, tx)
418        };
419
420        if let Err(e) = self.writer_tx.send_update(command) {
421            log::error!("{e}");
422            return Err(Error::Io(std::io::Error::new(
423                std::io::ErrorKind::BrokenPipe,
424                format!("Failed to send update command: {e}"),
425            )));
426        }
427
428        // Wait for writer to confirm it has drained the buffer
429        match rx.await {
430            Ok(true) => log::debug!("Writer confirmed buffer drain success"),
431            Ok(false) => {
432                log::warn!("Writer failed to drain buffer, aborting reconnect");
433                // Return error to trigger retry logic in controller
434                return Err(Error::Io(std::io::Error::other(
435                    "Failed to drain reconnection buffer",
436                )));
437            }
438            Err(e) => {
439                log::error!("Writer dropped update channel: {e}");
440                return Err(Error::Io(std::io::Error::new(
441                    std::io::ErrorKind::BrokenPipe,
442                    "Writer task dropped response channel",
443                )));
444            }
445        }
446
447        // Delay before closing connection
448        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
449
450        if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
451            log::debug!("Reconnect aborted mid-flight (after delay)");
452            return Ok(ReconnectOutcome::Aborted);
453        }
454
455        self.read_fence.invalidate();
456
457        if !self.read_task.is_finished() {
458            self.read_task.abort();
459            log_task_aborted("read");
460        }
461
462        // Atomically transition from Reconnect to Active
463        // This prevents race condition where disconnect could be requested between check and store
464        if ConnectionMode::complete_reconnect_with_sink(
465            &self.connection_mode,
466            self.state_sink.as_ref(),
467        ) == ReconnectOutcome::Aborted
468        {
469            log::debug!("Reconnect aborted (state changed during reconnect)");
470            return Ok(ReconnectOutcome::Aborted);
471        }
472
473        // Spawn new read task
474        self.read_fence = ReadSessionFence::new();
475        self.read_task = Self::spawn_read_task(
476            Arc::clone(&self.connection_mode),
477            self.read_fence.clone(),
478            reader,
479            self.config.message_handler.clone(),
480            self.config.suffix.clone(),
481            self.config.resolved_heartbeat_timeout(),
482        );
483
484        log::info!("Reconnect succeeded");
485        Ok(ReconnectOutcome::Reconnected)
486    }
487
488    /// Returns whether the read and write tasks are still running.
489    ///
490    /// Returns `true` if both the read and write tasks are still running.
491    /// There may be some delay between the connection closing and the
492    /// client detecting it.
493    #[inline]
494    #[must_use]
495    pub(crate) fn is_alive(&self) -> bool {
496        !self.read_task.is_finished() && !self.write_task.is_finished()
497    }
498
499    #[must_use]
500    fn spawn_read_task<R>(
501        connection_state: Arc<AtomicU8>,
502        read_fence: ReadSessionFence,
503        reader: R,
504        handler: Option<TcpMessageHandler>,
505        suffix: Vec<u8>,
506        heartbeat_timeout_secs: Option<u64>,
507    ) -> tokio::task::JoinHandle<()>
508    where
509        R: AsyncRead + Unpin + Send + 'static,
510    {
511        log_task_started("read");
512
513        // Interval between checking the connection mode
514        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
515        let heartbeat_timeout = heartbeat_timeout_secs.map(Duration::from_secs);
516
517        tokio::task::spawn(Self::run_read_loop(
518            connection_state,
519            read_fence,
520            reader,
521            handler,
522            suffix,
523            heartbeat_timeout,
524            check_interval,
525        ))
526    }
527
528    async fn run_read_loop<R>(
529        connection_state: Arc<AtomicU8>,
530        read_fence: ReadSessionFence,
531        mut reader: R,
532        handler: Option<TcpMessageHandler>,
533        suffix: Vec<u8>,
534        heartbeat_timeout: Option<Duration>,
535        check_interval: Duration,
536    ) where
537        R: AsyncRead + Unpin,
538    {
539        let mut buf = Vec::new();
540        // Leading bytes of `buf` already searched, so a peer that withholds the suffix cannot
541        // force a rescan of the whole buffer on every read.
542        let mut scanned = 0;
543        let mut last_data_time = dst::time::Instant::now();
544
545        'read: loop {
546            if !ConnectionMode::from_atomic(&connection_state).is_active() || !read_fence.is_valid()
547            {
548                if !buf.is_empty() {
549                    log::debug!(
550                        "Dropping {} buffered bytes after socket session ended",
551                        buf.len()
552                    );
553                    buf.clear();
554                }
555
556                break;
557            }
558
559            match dst::time::timeout(check_interval, reader.read_buf(&mut buf)).await {
560                // Connection has been terminated or vector buffer is complete
561                Ok(Ok(0)) => {
562                    log::debug!("Connection closed by server");
563                    break;
564                }
565                Ok(Err(e)) => {
566                    log::debug!("Connection ended: {e}");
567                    break;
568                }
569                // Received bytes of data
570                Ok(Ok(bytes)) => {
571                    log::trace!("Received <binary> {bytes} bytes");
572                    last_data_time = dst::time::Instant::now();
573
574                    if !ConnectionMode::from_atomic(&connection_state).is_active()
575                        || !read_fence.is_valid()
576                    {
577                        log::debug!(
578                            "Dropping {} buffered bytes after socket session ended",
579                            buf.len()
580                        );
581                        buf.clear();
582                        break;
583                    }
584
585                    while let Some(i) = buf[scanned..]
586                        .windows(suffix.len())
587                        .position(|window| window == suffix)
588                    {
589                        let end = scanned + i + suffix.len();
590                        let mut data: Vec<u8> = buf.drain(0..end).collect();
591                        data.truncate(data.len() - suffix.len());
592                        scanned = 0;
593
594                        if let Some(ref handler) = handler {
595                            if !ConnectionMode::from_atomic(&connection_state).is_active()
596                                || !read_fence.is_valid()
597                            {
598                                log::debug!(
599                                    "Dropping {} buffered bytes after socket session ended",
600                                    data.len() + buf.len()
601                                );
602                                buf.clear();
603                                break 'read;
604                            }
605
606                            handler(&data);
607                        }
608                    }
609
610                    // A suffix can still complete across the last `suffix.len() - 1` bytes
611                    scanned = buf.len().saturating_sub(suffix.len() - 1);
612
613                    if buf.len() > MAX_READ_BUFFER_BYTES {
614                        log::error!(
615                            "Read buffer exceeded maximum size ({MAX_READ_BUFFER_BYTES} bytes), closing connection"
616                        );
617                        break;
618                    }
619                }
620                Err(_) => {
621                    if let Some(timeout) = heartbeat_timeout {
622                        let silent_for = last_data_time.elapsed();
623                        if silent_for >= timeout {
624                            log::warn!(
625                                "Heartbeat timeout: no bytes received for {:.1}s",
626                                silent_for.as_secs_f64()
627                            );
628                            break;
629                        }
630                    }
631                }
632            }
633        }
634
635        log_task_stopped("read");
636    }
637
638    /// Drains buffered messages after reconnection completes.
639    ///
640    /// Attempts to send all buffered messages that were queued during reconnection.
641    /// Uses a peek-and-pop pattern to preserve messages if sending fails midway through the buffer.
642    ///
643    /// # Returns
644    ///
645    /// Returns `true` if a send error occurred (buffer may still contain unsent messages),
646    /// `false` if all messages were sent successfully (buffer is empty).
647    async fn drain_reconnect_buffer<W>(
648        buffer: &mut VecDeque<BufferedWrite>,
649        writer: &mut W,
650        suffix: &[u8],
651        replay: &[Bytes],
652    ) -> bool
653    where
654        W: AsyncWrite + Unpin,
655    {
656        if buffer.is_empty() {
657            return false;
658        }
659
660        let initial_buffer_len = buffer.len();
661        log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
662
663        while let Some(buffered) = buffer.front() {
664            if buffered.replay_key.is_some()
665                && replay.iter().any(|replayed| replayed == &buffered.data)
666            {
667                buffer.pop_front();
668                continue;
669            }
670
671            let mut combined_msg = Vec::with_capacity(buffered.data.len() + suffix.len());
672            combined_msg.extend_from_slice(&buffered.data);
673            combined_msg.extend_from_slice(suffix);
674
675            if let Err(e) = writer.write_all(&combined_msg).await {
676                if is_connection_drop_io_error(&e) {
677                    log::warn!(
678                        "Failed to send buffered message with suffix after reconnection: {e}, {} messages remain in buffer",
679                        buffer.len()
680                    );
681                } else {
682                    log::error!(
683                        "Failed to send buffered message with suffix after reconnection: {e}, {} messages remain in buffer",
684                        buffer.len()
685                    );
686                }
687
688                return true;
689            }
690
691            buffer.pop_front();
692        }
693
694        log::info!("Successfully sent all {initial_buffer_len} buffered messages");
695
696        false
697    }
698
699    async fn replace_writer<W>(
700        active_writer: &mut W,
701        new_writer: W,
702        replay: Vec<Bytes>,
703        reconnect_buffer: &mut VecDeque<BufferedWrite>,
704        suffix: &[u8],
705    ) -> bool
706    where
707        W: AsyncWrite + Unpin,
708    {
709        log::debug!("Received new writer");
710        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
711
712        _ = dst::time::timeout(
713            Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
714            active_writer.shutdown(),
715        )
716        .await;
717
718        *active_writer = new_writer;
719        log::debug!("Updated writer");
720
721        let drain_result =
722            dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
723                for replay_msg in &replay {
724                    let mut framed = Vec::with_capacity(replay_msg.len() + suffix.len());
725                    framed.extend_from_slice(replay_msg);
726                    framed.extend_from_slice(suffix);
727                    if let Err(e) = active_writer.write_all(&framed).await {
728                        log::warn!("Failed to send reconnect replay: {e}");
729                        return true;
730                    }
731                }
732
733                Self::drain_reconnect_buffer(reconnect_buffer, active_writer, suffix, &replay).await
734            })
735            .await;
736
737        let send_error = drain_result.unwrap_or_else(|_| {
738            log::warn!(
739                "Timed out sending reconnect replay and buffered messages, {} buffered messages remain",
740                reconnect_buffer.len()
741            );
742            true
743        });
744
745        !send_error
746    }
747
748    fn spawn_write_task<W>(
749        connection_state: Arc<AtomicU8>,
750        state_notify: Arc<tokio::sync::Notify>,
751        writer: W,
752        mut writer_rx: WriterReceiver<WriterCommand<W>>,
753        suffix: Vec<u8>,
754        state_sink: Option<SocketStateSink>,
755    ) -> tokio::task::JoinHandle<()>
756    where
757        W: AsyncWrite + Unpin + Send + 'static,
758    {
759        log_task_started("write");
760
761        // Interval between checking the connection mode
762        let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
763
764        tokio::task::spawn(async move {
765            let mut active_writer = writer;
766            let mut reconnect_buffer: VecDeque<BufferedWrite> = VecDeque::new();
767            let mut write_buf: Vec<u8> = Vec::new();
768
769            loop {
770                let mode = ConnectionMode::from_atomic(&connection_state);
771
772                if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
773                    break;
774                }
775
776                if mode.is_active() && !reconnect_buffer.is_empty() {
777                    let drain_result = dst::time::timeout(
778                        Duration::from_secs(WRITE_TIMEOUT_SECS),
779                        Self::drain_reconnect_buffer(
780                            &mut reconnect_buffer,
781                            &mut active_writer,
782                            &suffix,
783                            &[],
784                        ),
785                    )
786                    .await;
787
788                    let send_error = drain_result.unwrap_or_else(|_| {
789                        log::warn!(
790                            "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
791                            reconnect_buffer.len()
792                        );
793                        true
794                    });
795
796                    if send_error
797                        && ConnectionMode::request_reconnect_with_sink(
798                            &connection_state,
799                            state_sink.as_ref(),
800                        )
801                    {
802                        log::warn!("Writer triggering reconnect");
803                        state_notify.notify_one();
804                    }
805
806                    continue;
807                }
808
809                match dst::time::timeout(check_interval, writer_rx.recv()).await {
810                    Ok(Some((msg, permit))) => {
811                        // Re-check connection mode after receiving a message
812                        let mode = ConnectionMode::from_atomic(&connection_state);
813
814                        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
815                            break;
816                        }
817
818                        // Writing past a non-empty buffer would reorder commands
819                        let must_buffer = mode.is_reconnect() || !reconnect_buffer.is_empty();
820
821                        match msg {
822                            WriterCommand::Update(new_writer, tx) => {
823                                let sent = Self::replace_writer(
824                                    &mut active_writer,
825                                    new_writer,
826                                    Vec::new(),
827                                    &mut reconnect_buffer,
828                                    &suffix,
829                                )
830                                .await;
831
832                                if let Err(e) = tx.send(sent) {
833                                    log::error!(
834                                        "Failed to report drain status to controller: {e:?}"
835                                    );
836                                }
837                            }
838                            WriterCommand::UpdateWithReplay(new_writer, replay, tx) => {
839                                let sent = Self::replace_writer(
840                                    &mut active_writer,
841                                    new_writer,
842                                    replay,
843                                    &mut reconnect_buffer,
844                                    &suffix,
845                                )
846                                .await;
847
848                                if let Err(e) = tx.send(sent) {
849                                    log::error!(
850                                        "Failed to report drain status to controller: {e:?}"
851                                    );
852                                }
853                            }
854                            WriterCommand::Send(data) if must_buffer => {
855                                log::debug!(
856                                    "Buffering message until reconnect drain completes ({} bytes)",
857                                    data.len()
858                                );
859                                Self::buffer_reconnect_write(
860                                    &mut reconnect_buffer,
861                                    data,
862                                    None,
863                                    permit,
864                                );
865                            }
866                            WriterCommand::SendOrReplay { key, data } if must_buffer => {
867                                log::debug!(
868                                    "Buffering replayable message until reconnect drain completes ({} bytes)",
869                                    data.len()
870                                );
871                                Self::buffer_reconnect_write(
872                                    &mut reconnect_buffer,
873                                    data,
874                                    Some(key),
875                                    permit,
876                                );
877                            }
878                            command @ (WriterCommand::Send(_)
879                            | WriterCommand::SendOrReplay { .. }) => {
880                                let (msg, replay_key) = match command {
881                                    WriterCommand::Send(data) => (data, None),
882                                    WriterCommand::SendOrReplay { key, data } => (data, Some(key)),
883                                    _ => unreachable!(),
884                                };
885
886                                write_buf.clear();
887                                write_buf.extend_from_slice(&msg);
888                                write_buf.extend_from_slice(&suffix);
889
890                                let write_result = dst::time::timeout(
891                                    Duration::from_secs(WRITE_TIMEOUT_SECS),
892                                    active_writer.write_all(&write_buf),
893                                )
894                                .await;
895
896                                let write_failed = match write_result {
897                                    Ok(Ok(())) => false,
898                                    Ok(Err(e)) => {
899                                        if is_connection_drop_io_error(&e) {
900                                            log::warn!("Failed to send message: {e}");
901                                        } else {
902                                            log::error!("Failed to send message: {e}");
903                                        }
904
905                                        true
906                                    }
907                                    Err(_) => {
908                                        log::warn!(
909                                            "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
910                                        );
911                                        true
912                                    }
913                                };
914
915                                if write_failed {
916                                    Self::buffer_reconnect_write(
917                                        &mut reconnect_buffer,
918                                        msg,
919                                        replay_key,
920                                        permit,
921                                    );
922
923                                    // CAS: a disconnect landing mid-write must not be overwritten
924                                    if ConnectionMode::request_reconnect_with_sink(
925                                        &connection_state,
926                                        state_sink.as_ref(),
927                                    ) {
928                                        log::warn!("Writer triggering reconnect");
929                                        state_notify.notify_one();
930                                    }
931                                }
932                            }
933                        }
934                    }
935                    Ok(None) => {
936                        // Channel closed - writer task should terminate
937                        log::debug!("Writer channel closed, terminating writer task");
938                        break;
939                    }
940                    Err(_) => {
941                        // Timeout - just continue the loop
942                    }
943                }
944            }
945
946            // Attempt to shutdown the writer gracefully before exiting,
947            // we ignore any error as the writer may already be closed.
948            _ = dst::time::timeout(
949                Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
950                active_writer.shutdown(),
951            )
952            .await;
953
954            log_task_stopped("write");
955        })
956    }
957
958    fn buffer_reconnect_write(
959        buffer: &mut VecDeque<BufferedWrite>,
960        data: Bytes,
961        replay_key: Option<u64>,
962        permit: tokio::sync::OwnedSemaphorePermit,
963    ) {
964        if let Some(key) = replay_key {
965            buffer.retain(|buffered| buffered.replay_key != Some(key));
966        }
967
968        buffer.push_back(BufferedWrite {
969            data,
970            replay_key,
971            _permit: permit,
972        });
973    }
974
975    fn spawn_heartbeat_task(
976        connection_state: Arc<AtomicU8>,
977        interval_secs: u64,
978        message: Vec<u8>,
979        writer_tx: WriterSender<WriterCommand>,
980    ) -> tokio::task::JoinHandle<()> {
981        log_task_started("heartbeat");
982
983        tokio::task::spawn(async move {
984            let interval = Duration::from_secs(interval_secs);
985            loop {
986                dst::time::sleep(interval).await;
987
988                match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
989                    ConnectionMode::Active => {
990                        let msg = WriterCommand::Send(message.clone().into());
991
992                        match writer_tx.send(msg) {
993                            Ok(()) => log::trace!("Sent heartbeat to writer task"),
994                            Err(e) => {
995                                log::warn!("Failed to send heartbeat to writer task: {e}");
996                            }
997                        }
998                    }
999                    ConnectionMode::Reconnect => {}
1000                    ConnectionMode::Disconnect | ConnectionMode::Closed => break,
1001                }
1002            }
1003
1004            log_task_stopped("heartbeat");
1005        })
1006    }
1007}
1008
1009impl Drop for SocketClientInner {
1010    fn drop(&mut self) {
1011        self.read_fence.invalidate();
1012
1013        if !self.read_task.is_finished() {
1014            self.read_task.abort();
1015            log_task_aborted("read");
1016        }
1017
1018        if !self.write_task.is_finished() {
1019            self.write_task.abort();
1020            log_task_aborted("write");
1021        }
1022
1023        if let Some(ref handle) = self.heartbeat_task.take()
1024            && !handle.is_finished()
1025        {
1026            handle.abort();
1027            log_task_aborted("heartbeat");
1028        }
1029    }
1030}
1031
1032/// A suffix-framed TCP client with optional TLS and automatic reconnection.
1033///
1034/// The internal writer task serializes concurrent calls to [`Self::send_bytes`]. The configured
1035/// suffix frames all sent and received messages, and an optional heartbeat task sends its payload
1036/// at the configured interval. See [`SocketConfig`] for framing and reconnect policy.
1037pub struct SocketClient {
1038    pub(crate) controller_task: tokio::task::JoinHandle<()>,
1039    pub(crate) connection_mode: Arc<AtomicU8>,
1040    pub(crate) state_notify: Arc<tokio::sync::Notify>,
1041    pub(crate) connect_timeout: Duration,
1042    pub writer_tx: WriterSender<WriterCommand>,
1043    state_sink: Option<SocketStateSink>,
1044    controller_lifecycle: Arc<ControllerLifecycle>,
1045    controller_notify: Arc<tokio::sync::Notify>,
1046}
1047
1048/// Cloneable controller handle for requesting one raw socket reconnect.
1049#[derive(Clone)]
1050pub struct SocketReconnectHandle {
1051    connection_mode: Arc<AtomicU8>,
1052    state_sink: Option<SocketStateSink>,
1053    controller_lifecycle: Arc<ControllerLifecycle>,
1054    controller_notify: Arc<tokio::sync::Notify>,
1055}
1056
1057impl Debug for SocketReconnectHandle {
1058    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1059        f.debug_struct(stringify!(SocketReconnectHandle))
1060            .field(
1061                "connection_mode",
1062                &ConnectionMode::from_atomic(&self.connection_mode),
1063            )
1064            .finish_non_exhaustive()
1065    }
1066}
1067
1068impl SocketReconnectHandle {
1069    /// Requests that the controller replace the active transport.
1070    ///
1071    /// An accepted request reports the transport unavailable and wakes the controller. Rejected
1072    /// requests leave the transport state unchanged.
1073    #[must_use]
1074    pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
1075        let Some(_request) = self.controller_lifecycle.enter_request() else {
1076            return ReconnectRequestOutcome::Closed;
1077        };
1078
1079        let outcome = ConnectionMode::request_reconnect_outcome_with_sink(
1080            &self.connection_mode,
1081            self.state_sink.as_ref(),
1082        );
1083
1084        if outcome == ReconnectRequestOutcome::Accepted {
1085            self.controller_notify.notify_one();
1086        }
1087
1088        outcome
1089    }
1090}
1091
1092impl Debug for SocketClient {
1093    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1094        f.debug_struct(stringify!(SocketClient)).finish()
1095    }
1096}
1097
1098#[bon::bon]
1099impl SocketClient {
1100    /// Returns a builder for a new [`SocketClient`] connection.
1101    ///
1102    /// After a successful reconnect, `post_reconnection` runs after the replacement writer is
1103    /// installed, buffered messages are drained, and the replacement reader is started. The
1104    /// callback does not run after the initial connection.
1105    ///
1106    /// `state_sink` receives connection state changes. `reconnect_replay` produces protocol setup
1107    /// messages that the client sends before buffered application messages after a reconnect.
1108    ///
1109    /// # Errors
1110    ///
1111    /// Returns an error if connecting to the server fails.
1112    #[builder(finish_fn = connect)]
1113    pub async fn builder(
1114        config: SocketConfig,
1115        post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1116        state_sink: Option<SocketStateSink>,
1117        reconnect_replay: Option<SocketReconnectReplay>,
1118    ) -> anyhow::Result<Self> {
1119        let inner = SocketClientInner::connect_url(config, state_sink).await?;
1120        let writer_tx = inner.writer_tx.clone();
1121        let connection_mode = Arc::clone(&inner.connection_mode);
1122        let state_notify = Arc::clone(&inner.state_notify);
1123        let connect_timeout = inner.connect_timeout;
1124        let state_sink = inner.state_sink.clone();
1125        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
1126        let controller_notify = Arc::new(tokio::sync::Notify::new());
1127        let controller_task = Self::spawn_controller_task(
1128            inner,
1129            Arc::clone(&connection_mode),
1130            Arc::clone(&state_notify),
1131            Arc::clone(&controller_lifecycle),
1132            Arc::clone(&controller_notify),
1133            post_reconnection,
1134            reconnect_replay,
1135        );
1136        let abort_handle = controller_task.abort_handle();
1137        controller_lifecycle.set_abort(move || abort_handle.abort());
1138
1139        Ok(Self {
1140            controller_task,
1141            connection_mode,
1142            state_notify,
1143            connect_timeout,
1144            writer_tx,
1145            state_sink,
1146            controller_lifecycle,
1147            controller_notify,
1148        })
1149    }
1150
1151    /// Returns a cloneable handle to this client's reconnect controller.
1152    #[must_use]
1153    pub fn reconnect_handle(&self) -> SocketReconnectHandle {
1154        SocketReconnectHandle {
1155            connection_mode: Arc::clone(&self.connection_mode),
1156            state_sink: self.state_sink.clone(),
1157            controller_lifecycle: Arc::clone(&self.controller_lifecycle),
1158            controller_notify: Arc::clone(&self.controller_notify),
1159        }
1160    }
1161
1162    /// Requests that the controller replace the active transport.
1163    ///
1164    /// Returns `true` only when this call transitions the client from active to reconnecting.
1165    /// Duplicate, disconnecting, and closed requests return `false`.
1166    #[must_use]
1167    pub fn request_reconnect(&self) -> bool {
1168        self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
1169    }
1170
1171    /// Returns the current connection mode.
1172    #[must_use]
1173    pub fn connection_mode(&self) -> ConnectionMode {
1174        ConnectionMode::from_atomic(&self.connection_mode)
1175    }
1176
1177    /// Returns whether the client connection is active.
1178    ///
1179    /// Returns `true` if the client is connected and has not been signaled to disconnect.
1180    /// The client will automatically retry connection based on its configuration.
1181    #[inline]
1182    #[must_use]
1183    pub fn is_active(&self) -> bool {
1184        self.connection_mode().is_active()
1185    }
1186
1187    /// Returns whether the client is reconnecting.
1188    ///
1189    /// Returns `true` if the client lost connection and is attempting to reestablish it.
1190    /// The client will automatically retry connection based on its configuration.
1191    #[inline]
1192    #[must_use]
1193    pub fn is_reconnecting(&self) -> bool {
1194        self.connection_mode().is_reconnect()
1195    }
1196
1197    /// Returns whether the client is disconnecting.
1198    ///
1199    /// Returns `true` if the client is in disconnect mode.
1200    #[inline]
1201    #[must_use]
1202    pub fn is_disconnecting(&self) -> bool {
1203        self.connection_mode().is_disconnect()
1204    }
1205
1206    /// Returns whether the client is closed.
1207    ///
1208    /// Returns `true` if the client has been explicitly disconnected or reached
1209    /// maximum reconnection attempts. In this state, the client cannot be reused
1210    /// and a new client must be created for further connections.
1211    #[inline]
1212    #[must_use]
1213    pub fn is_closed(&self) -> bool {
1214        self.connection_mode().is_closed()
1215    }
1216
1217    /// Close the client.
1218    ///
1219    /// Controller task will periodically check the disconnect mode
1220    /// and shutdown the client if it is not alive.
1221    pub async fn close(&self) {
1222        self.begin_shutdown();
1223        self.close_with_timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS))
1224            .await;
1225    }
1226
1227    /// Requests shutdown without waiting for the controller task.
1228    pub fn begin_shutdown(&self) {
1229        ConnectionMode::request_disconnect(&self.connection_mode);
1230        self.state_notify.notify_waiters();
1231    }
1232
1233    async fn close_with_timeout(&self, shutdown_timeout: Duration) {
1234        self.begin_shutdown();
1235
1236        if dst::time::timeout(shutdown_timeout, async {
1237            while !self.controller_task.is_finished() {
1238                dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
1239            }
1240        })
1241        .await
1242        .is_err()
1243        {
1244            log::warn!("Timeout waiting for controller task to finish");
1245        }
1246
1247        if !self.controller_task.is_finished() {
1248            self.controller_task.abort();
1249            log_task_aborted("controller");
1250        }
1251
1252        self.connection_mode
1253            .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1254        self.state_notify.notify_waiters();
1255    }
1256
1257    /// Checks whether the connection is in a terminal state (disconnecting or closed).
1258    ///
1259    /// Single atomic load to fail fast before waiting.
1260    #[inline]
1261    fn check_not_terminal(&self) -> Result<(), SendError> {
1262        match self.connection_mode() {
1263            ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
1264            _ => Ok(()),
1265        }
1266    }
1267
1268    /// Waits for the client to become active before sending.
1269    ///
1270    /// Uses `state_notify` for event-driven wakeup so sends resume immediately
1271    /// after reconnection completes. A fallback interval guards against missed
1272    /// notifications.
1273    async fn wait_for_active(&self) -> Result<(), SendError> {
1274        const FALLBACK_INTERVAL_MS: u64 = 100;
1275
1276        let mode = self.connection_mode();
1277        if mode.is_active() {
1278            return Ok(());
1279        }
1280
1281        if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1282            return Err(SendError::Closed);
1283        }
1284
1285        log::debug!("Waiting for client to become ACTIVE before sending...");
1286
1287        let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
1288
1289        dst::time::timeout(self.connect_timeout, async {
1290            loop {
1291                // Enable before the state check: an unpolled Notified is unregistered and misses notifies
1292                let mut notified = pin!(self.state_notify.notified());
1293                notified.as_mut().enable();
1294
1295                let mode = self.connection_mode();
1296                if mode.is_active() {
1297                    return Ok(());
1298                }
1299
1300                if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1301                    return Err(());
1302                }
1303
1304                tokio::select! {
1305                    biased;
1306                    () = notified => {}
1307                    () = dst::time::sleep(fallback_interval) => {}
1308                }
1309            }
1310        })
1311        .await
1312        .map_err(|_| SendError::Timeout)?
1313        .map_err(|()| SendError::Closed)
1314    }
1315
1316    /// Sends a message of the given `data`.
1317    ///
1318    /// Returns `Ok(())` when the message is enqueued to the writer channel. This does not
1319    /// guarantee delivery: if a disconnect occurs concurrently, the writer task may drop the
1320    /// message. During reconnection, messages are buffered and replayed on the new connection.
1321    ///
1322    /// # Errors
1323    ///
1324    /// Returns an error if the writer capacity is exhausted, or if the client cannot send.
1325    pub async fn send_bytes(&self, data: Vec<u8>) -> Result<(), SendError> {
1326        self.check_not_terminal()?;
1327        self.wait_for_active().await?;
1328
1329        let msg = WriterCommand::Send(data.into());
1330        self.writer_tx.send(msg)
1331    }
1332
1333    fn spawn_controller_task(
1334        mut inner: SocketClientInner,
1335        connection_mode: Arc<AtomicU8>,
1336        state_notify: Arc<tokio::sync::Notify>,
1337        controller_lifecycle: Arc<ControllerLifecycle>,
1338        controller_notify: Arc<tokio::sync::Notify>,
1339        post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1340        reconnect_replay: Option<SocketReconnectReplay>,
1341    ) -> tokio::task::JoinHandle<()> {
1342        const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
1343
1344        tokio::task::spawn(async move {
1345            let _activity = controller_lifecycle.activity();
1346            log_task_started("controller");
1347
1348            let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1349            let mut reconnected_at = None;
1350
1351            loop {
1352                tokio::select! {
1353                    biased;
1354                    () = controller_notify.notified() => {}
1355                    () = state_notify.notified() => {}
1356                    () = dst::time::sleep(fallback_interval) => {}
1357                }
1358
1359                let mut mode = ConnectionMode::from_atomic(&connection_mode);
1360
1361                if mode.is_disconnect() {
1362                    log::debug!("Disconnecting");
1363
1364                    let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
1365                    if dst::time::timeout(timeout, async {
1366                        // Delay awaiting graceful shutdown
1367                        dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1368
1369                        inner.read_fence.invalidate();
1370                        if !inner.read_task.is_finished() {
1371                            inner.read_task.abort();
1372                            log_task_aborted("read");
1373                        }
1374
1375                        if let Some(task) = &inner.heartbeat_task
1376                            && !task.is_finished()
1377                        {
1378                            task.abort();
1379                            log_task_aborted("heartbeat");
1380                        }
1381                    })
1382                    .await
1383                    .is_err()
1384                    {
1385                        log::warn!("Shutdown timed out after {}s", timeout.as_secs());
1386                    }
1387
1388                    log::debug!("Closed");
1389                    connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1390                    state_notify.notify_waiters();
1391                    break; // Controller finished
1392                }
1393
1394                if mode.is_closed() {
1395                    log::debug!("Connection closed");
1396
1397                    inner.read_fence.invalidate();
1398                    if !inner.read_task.is_finished() {
1399                        inner.read_task.abort();
1400                        log_task_aborted("read");
1401                    }
1402
1403                    if let Some(task) = &inner.heartbeat_task
1404                        && !task.is_finished()
1405                    {
1406                        task.abort();
1407                        log_task_aborted("heartbeat");
1408                    }
1409
1410                    state_notify.notify_waiters();
1411                    break;
1412                }
1413
1414                if mode.is_active() && !inner.is_alive() {
1415                    if ConnectionMode::request_reconnect_with_sink(
1416                        &connection_mode,
1417                        inner.state_sink.as_ref(),
1418                    ) {
1419                        log::info!("Detected dead read task, transitioning to RECONNECT");
1420                    }
1421
1422                    mode = ConnectionMode::from_atomic(&connection_mode);
1423                }
1424
1425                if mode.is_reconnect() {
1426                    let reconnect_uptime = reconnected_at
1427                        .take()
1428                        .map(|started: dst::time::Instant| started.elapsed());
1429                    let previous_reconnect_stable = reconnect_uptime
1430                        .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
1431
1432                    if previous_reconnect_stable {
1433                        inner.backoff.reset();
1434                        inner.reconnect_attempt_count = 0;
1435                        log::debug!(
1436                            "Socket remained active for at least {}s, resetting reconnect cycle",
1437                            RECONNECT_STABILITY_THRESHOLD.as_secs()
1438                        );
1439                    }
1440
1441                    // Check max reconnection attempts before attempting reconnect
1442                    if let Some(max_attempts) = inner.reconnect_max_attempts
1443                        && inner.reconnect_attempt_count >= max_attempts
1444                    {
1445                        log::error!(
1446                            "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
1447                        );
1448
1449                        if connection_mode
1450                            .compare_exchange(
1451                                ConnectionMode::Reconnect.as_u8(),
1452                                ConnectionMode::Closed.as_u8(),
1453                                Ordering::SeqCst,
1454                                Ordering::SeqCst,
1455                            )
1456                            .is_ok()
1457                        {
1458                            state_notify.notify_waiters();
1459                            break;
1460                        }
1461
1462                        continue;
1463                    }
1464
1465                    let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
1466                    {
1467                        inner.backoff.next_duration()
1468                    } else {
1469                        Duration::ZERO
1470                    };
1471
1472                    let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
1473                    if !duration.is_zero() {
1474                        log::warn!("Backing off for {}s...", duration.as_secs_f64());
1475
1476                        if !wait_reconnect_delay(
1477                            duration,
1478                            connection_mode.as_ref(),
1479                            state_notify.as_ref(),
1480                        )
1481                        .await
1482                        {
1483                            log::debug!("Backoff interrupted by terminal state");
1484                            continue;
1485                        }
1486                    }
1487
1488                    inner.reconnect_attempt_count += 1;
1489                    inner.reconnect_throttle.record_attempt();
1490
1491                    // Race reconnect against disconnect notification
1492                    let reconnect_result = tokio::select! {
1493                        biased;
1494                        result = inner.reconnect(reconnect_replay.as_ref()) => Some(result),
1495                        () = async {
1496                            loop {
1497                                // Enable before the check so a disconnect notify between iterations is not missed
1498                                let mut notified = pin!(state_notify.notified());
1499                                notified.as_mut().enable();
1500
1501                                if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
1502                                    break;
1503                                }
1504                                notified.await;
1505                            }
1506                        } => None,
1507                    };
1508
1509                    match reconnect_result {
1510                        None => {
1511                            log::debug!("Reconnect interrupted by disconnect");
1512                        }
1513                        Some(Ok(ReconnectOutcome::Reconnected)) => {
1514                            log::debug!("Reconnected successfully");
1515                            reconnected_at = Some(dst::time::Instant::now());
1516
1517                            state_notify.notify_waiters();
1518
1519                            // The outcome records a completed reconnection; emit recovery
1520                            // callbacks only while the replacement is still `Active`, not
1521                            // after a teardown or another drop.
1522                            if ConnectionMode::from_atomic(&connection_mode).is_active() {
1523                                if let Some(ref handler) = post_reconnection {
1524                                    handler();
1525                                    log::debug!("Called `post_reconnection` handler");
1526                                }
1527                            } else {
1528                                log::debug!(
1529                                    "Skipping post_reconnection handlers due to disconnect state"
1530                                );
1531                            }
1532                        }
1533                        Some(Ok(ReconnectOutcome::Aborted)) => {
1534                            log::debug!("Reconnect aborted");
1535                        }
1536                        Some(Err(e)) => {
1537                            let duration = inner.backoff.next_duration();
1538                            log::warn!(
1539                                "Reconnect attempt {} failed: {e}",
1540                                inner.reconnect_attempt_count
1541                            );
1542
1543                            if !duration.is_zero() {
1544                                log::warn!("Backing off for {}s...", duration.as_secs_f64());
1545
1546                                if !wait_reconnect_delay(
1547                                    duration,
1548                                    connection_mode.as_ref(),
1549                                    state_notify.as_ref(),
1550                                )
1551                                .await
1552                                {
1553                                    log::debug!("Backoff interrupted by terminal state");
1554                                }
1555                            }
1556                        }
1557                    }
1558                }
1559            }
1560
1561            log_task_stopped("controller");
1562        })
1563    }
1564}
1565
1566// Dropping cancels background work without reporting a terminal socket state transition.
1567impl Drop for SocketClient {
1568    fn drop(&mut self) {
1569        let controller_running = !self.controller_task.is_finished();
1570        self.controller_lifecycle.close_and_abort();
1571
1572        if controller_running {
1573            log_task_aborted("controller");
1574        }
1575    }
1576}
1577
1578#[cfg(test)]
1579#[cfg(not(feature = "turmoil"))]
1580#[cfg(not(all(feature = "simulation", madsim)))] // transport-layer I/O not simulated
1581#[cfg(target_os = "linux")] // Only run network tests on Linux (CI stability)
1582mod tests {
1583    use log::Level;
1584    use nautilus_common::testing::wait_until_async;
1585    use parking_lot::Mutex as BlockingMutex;
1586    use rstest::rstest;
1587    use tokio::{
1588        io::{AsyncReadExt, AsyncWriteExt},
1589        net::{TcpListener, TcpStream},
1590        sync::Mutex,
1591        task,
1592        time::{Duration, sleep},
1593    };
1594
1595    use super::*;
1596    use crate::{SocketState, logging::tests::capture_logs_for, socket::SocketHeartbeat};
1597
1598    async fn bind_test_server() -> (u16, TcpListener) {
1599        let listener = TcpListener::bind("127.0.0.1:0")
1600            .await
1601            .expect("Failed to bind ephemeral port");
1602        let port = listener.local_addr().unwrap().port();
1603        (port, listener)
1604    }
1605
1606    async fn run_echo_server(mut socket: TcpStream) {
1607        let mut buf = Vec::new();
1608        loop {
1609            match socket.read_buf(&mut buf).await {
1610                Ok(0) => {
1611                    break;
1612                }
1613                Ok(_n) => {
1614                    while let Some(idx) = buf.array_windows().position(|w| w == b"\r\n") {
1615                        let mut line = buf.drain(..idx + 2).collect::<Vec<u8>>();
1616                        // Remove trailing \r\n
1617                        line.truncate(line.len() - 2);
1618
1619                        if line == b"close" {
1620                            let _ = socket.shutdown().await;
1621                            return;
1622                        }
1623
1624                        let mut echo_data = line;
1625                        echo_data.extend_from_slice(b"\r\n");
1626                        if socket.write_all(&echo_data).await.is_err() {
1627                            break;
1628                        }
1629                    }
1630                }
1631                Err(e) => {
1632                    eprintln!("Server read error: {e}");
1633                    break;
1634                }
1635            }
1636        }
1637    }
1638
1639    #[rstest]
1640    #[tokio::test]
1641    async fn test_writer_capacity_from_config() {
1642        let (port, listener) = bind_test_server().await;
1643        let config = SocketConfig::builder()
1644            .url(format!("127.0.0.1:{port}"))
1645            .mode(Mode::Plain)
1646            .suffix(b"\r\n".to_vec())
1647            .writer_capacity(2)
1648            .build()
1649            .unwrap();
1650        let client = SocketClient::builder()
1651            .config(config)
1652            .connect()
1653            .await
1654            .unwrap();
1655        let (_peer, _) = listener.accept().await.unwrap();
1656
1657        // Ready sends do not yield on this current-thread runtime, keeping the queue occupied
1658        client.send_bytes(vec![1]).await.unwrap();
1659        client.send_bytes(vec![2]).await.unwrap();
1660        let overflow = client.send_bytes(vec![3]).await;
1661
1662        assert_eq!(client.connection_mode(), ConnectionMode::Active);
1663        assert!(matches!(overflow, Err(SendError::BufferFull)));
1664        client.close().await;
1665    }
1666
1667    #[rstest]
1668    #[case::drain_failed(
1669        Some(false),
1670        std::io::ErrorKind::Other,
1671        "Failed to drain reconnection buffer"
1672    )]
1673    #[case::reply_dropped(
1674        None,
1675        std::io::ErrorKind::BrokenPipe,
1676        "Writer task dropped response channel"
1677    )]
1678    #[tokio::test]
1679    async fn test_reconnect_writer_failure_preserves_reconnect_state(
1680        #[case] reply: Option<bool>,
1681        #[case] kind: std::io::ErrorKind,
1682        #[case] message: &str,
1683    ) {
1684        let (port, listener) = bind_test_server().await;
1685        let config = SocketConfig::builder()
1686            .url(format!("127.0.0.1:{port}"))
1687            .mode(Mode::Plain)
1688            .suffix(b"\r\n".to_vec())
1689            .build()
1690            .unwrap();
1691        let mut client = SocketClientInner::connect_url(config, None).await.unwrap();
1692        let (_peer, _) = listener.accept().await.unwrap();
1693        client.read_task.abort();
1694        client.write_task.abort();
1695        let _ = (&mut client.read_task).await;
1696        let _ = (&mut client.write_task).await;
1697        let (writer_tx, mut writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
1698        client.writer_tx = writer_tx;
1699        client
1700            .connection_mode
1701            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
1702
1703        let writer = tokio::spawn(async move {
1704            let WriterCommand::Update(_, sender) = writer_rx.recv().await.unwrap().0 else {
1705                panic!("expected writer update");
1706            };
1707
1708            if let Some(reply) = reply {
1709                sender.send(reply).unwrap();
1710            }
1711        });
1712
1713        let error = client.reconnect(None).await.unwrap_err();
1714        writer.await.unwrap();
1715
1716        let Error::Io(error) = error else {
1717            panic!("expected I/O error, was {error}");
1718        };
1719
1720        assert_eq!(error.kind(), kind);
1721        assert_eq!(error.to_string(), message);
1722        assert_eq!(
1723            ConnectionMode::from_atomic(&client.connection_mode),
1724            ConnectionMode::Reconnect
1725        );
1726    }
1727
1728    #[tokio::test]
1729    async fn test_basic_send_receive() {
1730        let (port, listener) = bind_test_server().await;
1731        let server_task = task::spawn(async move {
1732            let (socket, _) = listener.accept().await.unwrap();
1733            run_echo_server(socket).await;
1734        });
1735
1736        let config = SocketConfig {
1737            url: format!("127.0.0.1:{port}"),
1738            mode: Mode::Plain,
1739            suffix: b"\r\n".to_vec(),
1740            message_handler: None,
1741            heartbeat: None,
1742            connect_timeout_ms: None,
1743            reconnect_delay_initial_ms: None,
1744            reconnect_backoff_factor: None,
1745            reconnect_delay_max_ms: None,
1746            reconnect_jitter_ms: None,
1747            reconnect_max_attempts: None,
1748            connection_max_retries: None,
1749            heartbeat_timeout_secs: None,
1750            writer_capacity: None,
1751            certs_dir: None,
1752        };
1753
1754        let client = SocketClient::builder()
1755            .config(config)
1756            .connect()
1757            .await
1758            .expect("Client connect failed unexpectedly");
1759
1760        client.send_bytes(b"Hello".into()).await.unwrap();
1761        client.send_bytes(b"World".into()).await.unwrap();
1762
1763        // Wait a bit for the server to echo them back
1764        sleep(Duration::from_millis(100)).await;
1765
1766        client.send_bytes(b"close".into()).await.unwrap();
1767        server_task.await.unwrap();
1768        assert!(!client.is_closed());
1769    }
1770
1771    #[tokio::test]
1772    async fn test_reconnect_fail_exhausted() {
1773        let (port, listener) = bind_test_server().await;
1774        drop(listener); // We drop it immediately -> no server is listening
1775
1776        // Wait until port is truly unavailable (OS has released it)
1777        wait_until_async(
1778            || async {
1779                TcpStream::connect(format!("127.0.0.1:{port}"))
1780                    .await
1781                    .is_err()
1782            },
1783            Duration::from_secs(2),
1784        )
1785        .await;
1786
1787        let config = SocketConfig {
1788            url: format!("127.0.0.1:{port}"),
1789            mode: Mode::Plain,
1790            suffix: b"\r\n".to_vec(),
1791            message_handler: None,
1792            heartbeat: None,
1793            connect_timeout_ms: Some(100),
1794            reconnect_delay_initial_ms: Some(50),
1795            reconnect_backoff_factor: Some(1.0),
1796            reconnect_delay_max_ms: Some(50),
1797            reconnect_jitter_ms: Some(0),
1798            connection_max_retries: Some(1),
1799            reconnect_max_attempts: None,
1800            heartbeat_timeout_secs: None,
1801            writer_capacity: None,
1802            certs_dir: None,
1803        };
1804
1805        let client_res = SocketClient::builder().config(config).connect().await;
1806        assert!(
1807            client_res.is_err(),
1808            "Should fail quickly with no server listening"
1809        );
1810    }
1811
1812    #[rstest]
1813    #[tokio::test]
1814    async fn test_connect_failure_redacts_url_from_logs_and_error() {
1815        const ENDPOINT_PATH_SECRET: &str = "unique-endpoint-path-secret";
1816        const ENDPOINT_QUERY_SECRET: &str = "unique-endpoint-query-secret";
1817
1818        let (port, listener) = bind_test_server().await;
1819        drop(listener); // We drop it immediately -> no server is listening
1820
1821        // Wait until port is truly unavailable (OS has released it)
1822        wait_until_async(
1823            || async {
1824                TcpStream::connect(format!("127.0.0.1:{port}"))
1825                    .await
1826                    .is_err()
1827            },
1828            Duration::from_secs(2),
1829        )
1830        .await;
1831
1832        let capture = capture_logs_for(&["nautilus_network::socket::client"]).await;
1833
1834        let config = SocketConfig {
1835            url: format!(
1836                "ws://127.0.0.1:{port}/{ENDPOINT_PATH_SECRET}?api_key={ENDPOINT_QUERY_SECRET}"
1837            ),
1838            mode: Mode::Plain,
1839            suffix: b"\r\n".to_vec(),
1840            message_handler: None,
1841            heartbeat: None,
1842            connect_timeout_ms: Some(1_000),
1843            reconnect_delay_initial_ms: Some(50),
1844            reconnect_delay_max_ms: Some(50),
1845            reconnect_backoff_factor: Some(1.0),
1846            reconnect_jitter_ms: Some(0),
1847            connection_max_retries: Some(1),
1848            reconnect_max_attempts: None,
1849            heartbeat_timeout_secs: None,
1850            writer_capacity: None,
1851            certs_dir: None,
1852        };
1853
1854        let error = SocketClient::builder()
1855            .config(config)
1856            .connect()
1857            .await
1858            .expect_err("connection should fail with no server listening");
1859
1860        let error_message = error.to_string();
1861        assert!(
1862            error_message.starts_with(&format!("Failed to connect to {REDACTED}")),
1863            "connect error omitted the redaction placeholder: {error_message}"
1864        );
1865        assert!(
1866            !error_message.contains(ENDPOINT_PATH_SECRET)
1867                && !error_message.contains(ENDPOINT_QUERY_SECRET),
1868            "connect error exposed the secret marker: {error_message}"
1869        );
1870
1871        let messages = capture.messages();
1872
1873        // Asserted positively so the blanket secret assertion below cannot pass vacuously by
1874        // capturing no warning at all.
1875        let (_, attempt_warning) = messages
1876            .iter()
1877            .find(|(level, message)| {
1878                *level == Level::Warn && message.starts_with("Socket connection attempt 1/1 ")
1879            })
1880            .expect("connect-failure warning was not captured");
1881
1882        assert!(
1883            attempt_warning.contains(REDACTED),
1884            "connect-failure warning omitted the redaction placeholder: {attempt_warning}"
1885        );
1886        assert!(
1887            messages.iter().any(|(level, message)| {
1888                *level == Level::Debug && *message == format!("Connecting to {REDACTED}")
1889            }),
1890            "connect debug log should redact the endpoint, was {messages:?}"
1891        );
1892        assert!(
1893            messages.iter().all(|(_, message)| {
1894                !message.contains(ENDPOINT_PATH_SECRET) && !message.contains(ENDPOINT_QUERY_SECRET)
1895            }),
1896            "socket client logs exposed the secret marker: {messages:?}"
1897        );
1898    }
1899
1900    #[tokio::test]
1901    async fn test_user_disconnect() {
1902        let (port, listener) = bind_test_server().await;
1903        let server_task = task::spawn(async move {
1904            let (socket, _) = listener.accept().await.unwrap();
1905            let mut buf = [0u8; 1024];
1906            let _ = socket.try_read(&mut buf);
1907
1908            loop {
1909                sleep(Duration::from_secs(1)).await;
1910            }
1911        });
1912
1913        let config = SocketConfig {
1914            url: format!("127.0.0.1:{port}"),
1915            mode: Mode::Plain,
1916            suffix: b"\r\n".to_vec(),
1917            message_handler: None,
1918            heartbeat: None,
1919            connect_timeout_ms: None,
1920            reconnect_delay_initial_ms: None,
1921            reconnect_backoff_factor: None,
1922            reconnect_delay_max_ms: None,
1923            reconnect_jitter_ms: None,
1924            reconnect_max_attempts: None,
1925            connection_max_retries: None,
1926            heartbeat_timeout_secs: None,
1927            writer_capacity: None,
1928            certs_dir: None,
1929        };
1930
1931        let client = SocketClient::builder()
1932            .config(config)
1933            .connect()
1934            .await
1935            .unwrap();
1936
1937        client.close().await;
1938        assert!(client.is_closed());
1939        server_task.abort();
1940    }
1941
1942    #[tokio::test]
1943    async fn test_close_after_closed_returns_fast_and_preserves_state() {
1944        let (port, listener) = bind_test_server().await;
1945
1946        let server_task = task::spawn(async move {
1947            // Accept the first connection then drop it; never accept again so
1948            // the client exhausts its reconnect attempts and transitions to CLOSED
1949            let (socket, _) = listener.accept().await.unwrap();
1950            drop(socket);
1951            drop(listener);
1952            sleep(Duration::from_secs(5)).await;
1953        });
1954
1955        let config = SocketConfig {
1956            url: format!("127.0.0.1:{port}"),
1957            mode: Mode::Plain,
1958            suffix: b"\r\n".to_vec(),
1959            message_handler: None,
1960            heartbeat: None,
1961            connect_timeout_ms: Some(200),
1962            reconnect_delay_initial_ms: Some(50),
1963            reconnect_backoff_factor: Some(1.0),
1964            reconnect_delay_max_ms: Some(50),
1965            reconnect_jitter_ms: Some(0),
1966            connection_max_retries: None,
1967            reconnect_max_attempts: Some(1),
1968            heartbeat_timeout_secs: None,
1969            writer_capacity: None,
1970            certs_dir: None,
1971        };
1972
1973        let client = SocketClient::builder()
1974            .config(config)
1975            .connect()
1976            .await
1977            .unwrap();
1978
1979        wait_until_async(|| async { client.is_closed() }, Duration::from_secs(5)).await;
1980
1981        // Closing an already CLOSED client must return promptly (no 5s spin
1982        // waiting for a controller that has already exited) and must not
1983        // regress the terminal state to DISCONNECT
1984        let start = std::time::Instant::now();
1985        client.close().await;
1986        let elapsed = start.elapsed();
1987
1988        assert!(client.is_closed(), "Client should remain CLOSED");
1989        assert!(
1990            !client.is_disconnecting(),
1991            "Closed client should not report DISCONNECT after close()"
1992        );
1993        assert!(
1994            elapsed < Duration::from_secs(2),
1995            "close() on a closed client should return fast, took {elapsed:?}"
1996        );
1997
1998        server_task.abort();
1999    }
2000
2001    #[rstest]
2002    #[tokio::test]
2003    async fn test_heartbeat_task_warns_when_writer_channel_closed() {
2004        let capture = capture_logs_for(&["nautilus_network::socket::client"]).await;
2005        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2006        let (writer_tx, mut writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
2007        let task = SocketClientInner::spawn_heartbeat_task(
2008            Arc::clone(&connection_state),
2009            1,
2010            b"ping".to_vec(),
2011            writer_tx,
2012        );
2013
2014        let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
2015            .await
2016            .expect("timed out waiting for the first heartbeat")
2017            .expect("heartbeat channel closed before the first heartbeat");
2018        drop(writer_rx);
2019        tokio::time::sleep(Duration::from_millis(1_200)).await;
2020
2021        let messages = capture.messages();
2022        assert!(
2023            messages.iter().any(|(level, message)| {
2024                *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
2025            }),
2026            "expected WARN when the writer channel is closed, was {messages:?}"
2027        );
2028        assert!(
2029            !messages.iter().any(|(level, _)| *level == Level::Error),
2030            "a closed writer channel must not log ERROR, was {messages:?}"
2031        );
2032
2033        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
2034        tokio::time::timeout(Duration::from_secs(2), task)
2035            .await
2036            .expect("heartbeat task should stop after close")
2037            .unwrap();
2038    }
2039
2040    #[tokio::test]
2041    async fn test_heartbeat() {
2042        let (port, listener) = bind_test_server().await;
2043        let received = Arc::new(Mutex::new(Vec::new()));
2044        let received2 = Arc::clone(&received);
2045
2046        let server_task = task::spawn(async move {
2047            let (socket, _) = listener.accept().await.unwrap();
2048
2049            let mut buf = Vec::new();
2050            loop {
2051                match socket.try_read_buf(&mut buf) {
2052                    Ok(0) => break,
2053                    Ok(_) => {
2054                        while let Some(idx) = buf.array_windows().position(|w| w == b"\r\n") {
2055                            let mut line = buf.drain(..idx + 2).collect::<Vec<u8>>();
2056                            line.truncate(line.len() - 2);
2057                            received2.lock().await.push(line);
2058                        }
2059                    }
2060                    Err(_) => {
2061                        tokio::time::sleep(Duration::from_millis(10)).await;
2062                    }
2063                }
2064            }
2065        });
2066
2067        let config = SocketConfig {
2068            url: format!("127.0.0.1:{port}"),
2069            mode: Mode::Plain,
2070            suffix: b"\r\n".to_vec(),
2071            message_handler: None,
2072            heartbeat: Some(SocketHeartbeat {
2073                interval_secs: 1,
2074                payload: b"ping".to_vec(),
2075            }),
2076            connect_timeout_ms: None,
2077            reconnect_delay_initial_ms: None,
2078            reconnect_backoff_factor: None,
2079            reconnect_delay_max_ms: None,
2080            reconnect_jitter_ms: None,
2081            reconnect_max_attempts: None,
2082            connection_max_retries: None,
2083            heartbeat_timeout_secs: None,
2084            writer_capacity: None,
2085            certs_dir: None,
2086        };
2087
2088        let client = SocketClient::builder()
2089            .config(config)
2090            .connect()
2091            .await
2092            .unwrap();
2093
2094        // Wait ~3 seconds to collect some heartbeats
2095        sleep(Duration::from_secs(3)).await;
2096
2097        {
2098            let lock = received.lock().await;
2099            let pings = lock
2100                .iter()
2101                .filter(|line| line == &&b"ping".to_vec())
2102                .count();
2103            assert!(
2104                pings >= 2,
2105                "Expected at least 2 heartbeat pings; got {pings}"
2106            );
2107        }
2108
2109        client.close().await;
2110        server_task.abort();
2111    }
2112
2113    #[tokio::test]
2114    async fn test_reconnect_success() {
2115        let (port, listener) = bind_test_server().await;
2116
2117        // Spawn a server task that:
2118        // 1. Accepts the first connection and then drops it after a short delay (simulate disconnect)
2119        // 2. Waits a bit and then accepts a new connection and runs the echo server
2120        let server_task = task::spawn(async move {
2121            // Accept first connection
2122            let (mut socket, _) = listener.accept().await.expect("First accept failed");
2123
2124            // Wait briefly and then force-close the connection
2125            sleep(Duration::from_millis(500)).await;
2126            let _ = socket.shutdown().await;
2127
2128            // Wait for the client's reconnect attempt
2129            sleep(Duration::from_millis(500)).await;
2130
2131            // Run the echo server on the new connection
2132            let (socket, _) = listener.accept().await.expect("Second accept failed");
2133            run_echo_server(socket).await;
2134        });
2135
2136        let config = SocketConfig {
2137            url: format!("127.0.0.1:{port}"),
2138            mode: Mode::Plain,
2139            suffix: b"\r\n".to_vec(),
2140            message_handler: None,
2141            heartbeat: None,
2142            connect_timeout_ms: Some(5_000),
2143            reconnect_delay_initial_ms: Some(500),
2144            reconnect_delay_max_ms: Some(5_000),
2145            reconnect_backoff_factor: Some(2.0),
2146            reconnect_jitter_ms: Some(50),
2147            reconnect_max_attempts: None,
2148            connection_max_retries: None,
2149            heartbeat_timeout_secs: None,
2150            writer_capacity: None,
2151            certs_dir: None,
2152        };
2153
2154        let client = SocketClient::builder()
2155            .config(config)
2156            .connect()
2157            .await
2158            .expect("Client connect failed unexpectedly");
2159
2160        // Initially, the client should be active
2161        assert!(client.is_active(), "Client should start as active");
2162
2163        // Wait until the client loses connection (i.e. not active),
2164        // then wait until it reconnects (active again).
2165        wait_until_async(|| async { client.is_active() }, Duration::from_secs(10)).await;
2166
2167        client
2168            .send_bytes(b"TestReconnect".into())
2169            .await
2170            .expect("Send failed");
2171
2172        client.close().await;
2173        server_task.abort();
2174    }
2175
2176    #[rstest]
2177    #[tokio::test]
2178    async fn test_state_sink_reports_initial_loss_and_recovery() {
2179        let (port, listener) = bind_test_server().await;
2180        let server_task = task::spawn(async move {
2181            let (mut socket, _) = listener.accept().await.unwrap();
2182            sleep(Duration::from_millis(100)).await;
2183            socket.shutdown().await.unwrap();
2184
2185            let (socket, _) = listener.accept().await.unwrap();
2186            run_echo_server(socket).await;
2187        });
2188
2189        let config = SocketConfig {
2190            url: format!("127.0.0.1:{port}"),
2191            mode: Mode::Plain,
2192            suffix: b"\r\n".to_vec(),
2193            message_handler: None,
2194            heartbeat: None,
2195            connect_timeout_ms: Some(1_000),
2196            reconnect_delay_initial_ms: Some(10),
2197            reconnect_backoff_factor: Some(1.0),
2198            reconnect_delay_max_ms: Some(10),
2199            reconnect_jitter_ms: Some(0),
2200            reconnect_max_attempts: Some(3),
2201            connection_max_retries: Some(1),
2202            heartbeat_timeout_secs: None,
2203            writer_capacity: None,
2204            certs_dir: None,
2205        };
2206
2207        let states = Arc::new(BlockingMutex::new(Vec::new()));
2208        let states_callback = Arc::clone(&states);
2209        let sink = SocketStateSink::new(move |state| {
2210            states_callback.lock().push(state);
2211        });
2212
2213        let client = SocketClient::builder()
2214            .config(config)
2215            .state_sink(sink)
2216            .connect()
2217            .await
2218            .unwrap();
2219
2220        assert_eq!(*states.lock(), vec![SocketState::Connected]);
2221
2222        wait_until_async(
2223            || {
2224                let states = Arc::clone(&states);
2225                async move { states.lock().len() == 3 }
2226            },
2227            Duration::from_secs(5),
2228        )
2229        .await;
2230
2231        assert_eq!(
2232            *states.lock(),
2233            vec![
2234                SocketState::Connected,
2235                SocketState::Disconnected,
2236                SocketState::Connected,
2237            ]
2238        );
2239
2240        client.close().await;
2241        assert_eq!(states.lock().len(), 3);
2242        server_task.abort();
2243    }
2244
2245    #[rstest]
2246    #[tokio::test]
2247    async fn test_state_sink_ignores_initial_connection_failure() {
2248        let (port, listener) = bind_test_server().await;
2249        drop(listener);
2250
2251        let config = SocketConfig {
2252            url: format!("127.0.0.1:{port}"),
2253            mode: Mode::Plain,
2254            suffix: b"\r\n".to_vec(),
2255            message_handler: None,
2256            heartbeat: None,
2257            connect_timeout_ms: Some(100),
2258            reconnect_delay_initial_ms: Some(1),
2259            reconnect_backoff_factor: Some(1.0),
2260            reconnect_delay_max_ms: Some(1),
2261            reconnect_jitter_ms: Some(0),
2262            reconnect_max_attempts: Some(1),
2263            connection_max_retries: Some(1),
2264            heartbeat_timeout_secs: None,
2265            writer_capacity: None,
2266            certs_dir: None,
2267        };
2268
2269        let states = Arc::new(BlockingMutex::new(Vec::new()));
2270        let states_callback = Arc::clone(&states);
2271        let sink = SocketStateSink::new(move |state| {
2272            states_callback.lock().push(state);
2273        });
2274
2275        let result = SocketClient::builder()
2276            .config(config)
2277            .state_sink(sink)
2278            .connect()
2279            .await;
2280
2281        assert!(result.is_err());
2282        assert_eq!(*states.lock(), Vec::new());
2283    }
2284}
2285
2286#[cfg(test)]
2287#[cfg(not(feature = "turmoil"))]
2288#[cfg(not(all(feature = "simulation", madsim)))] // transport-layer I/O not simulated
2289mod rust_tests {
2290    use std::{
2291        pin::Pin,
2292        sync::{
2293            Arc,
2294            atomic::{AtomicBool, AtomicUsize},
2295        },
2296        task::{Context, Poll, Waker},
2297    };
2298
2299    use nautilus_common::testing::wait_until_async;
2300    use parking_lot::{Condvar, Mutex};
2301    use rstest::rstest;
2302    use tokio::{
2303        io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf},
2304        net::TcpListener,
2305        sync::oneshot,
2306        task::{self, yield_now},
2307        time::{Duration, sleep},
2308    };
2309
2310    use super::*;
2311    use crate::{SocketState, socket::SocketHeartbeat};
2312
2313    const TEST_TIMEOUT: Duration = Duration::from_secs(10);
2314
2315    struct CondvarReleaseGuard {
2316        release: Arc<(Mutex<bool>, Condvar)>,
2317    }
2318
2319    impl CondvarReleaseGuard {
2320        fn new(release: Arc<(Mutex<bool>, Condvar)>) -> Self {
2321            Self { release }
2322        }
2323
2324        fn release(&self) {
2325            let (lock, condvar) = self.release.as_ref();
2326            let mut released = lock.lock();
2327            *released = true;
2328            condvar.notify_all();
2329        }
2330    }
2331
2332    impl Drop for CondvarReleaseGuard {
2333        fn drop(&mut self) {
2334            self.release();
2335        }
2336    }
2337
2338    async fn recv_rendezvous<T: Send + 'static>(
2339        receiver: std::sync::mpsc::Receiver<T>,
2340        name: &'static str,
2341    ) -> T {
2342        let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
2343
2344        match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
2345            Ok(Ok(Ok(value))) => value,
2346            Ok(Ok(Err(e))) => {
2347                panic!("{name} did not arrive within the test timeout: {e}")
2348            }
2349            Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
2350            Err(e) => panic!("{name} receive task did not finish: {e}"),
2351        }
2352    }
2353
2354    async fn await_task_termination(
2355        task: tokio::task::JoinHandle<()>,
2356        name: &'static str,
2357        cancellation_ok: bool,
2358    ) {
2359        match tokio::time::timeout(TEST_TIMEOUT, task).await {
2360            Ok(Ok(())) => {}
2361            Ok(Err(e)) if cancellation_ok && e.is_cancelled() => {}
2362            Ok(Err(e)) => panic!("{name} failed: {e}"),
2363            Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
2364        }
2365    }
2366
2367    fn reconnect_test_config(port: u16) -> SocketConfig {
2368        SocketConfig {
2369            url: format!("127.0.0.1:{port}"),
2370            mode: Mode::Plain,
2371            suffix: b"\r\n".to_vec(),
2372            message_handler: None,
2373            heartbeat: None,
2374            connect_timeout_ms: Some(1_000),
2375            reconnect_delay_initial_ms: None,
2376            reconnect_backoff_factor: None,
2377            reconnect_delay_max_ms: None,
2378            reconnect_jitter_ms: None,
2379            connection_max_retries: Some(1),
2380            reconnect_max_attempts: None,
2381            heartbeat_timeout_secs: None,
2382            writer_capacity: None,
2383            certs_dir: None,
2384        }
2385    }
2386
2387    #[rstest]
2388    #[tokio::test]
2389    async fn test_reconnect_outcome_is_aborted_before_connect() {
2390        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2391        let port = listener.local_addr().unwrap().port();
2392        let server = tokio::spawn(async move {
2393            let (_socket, _) = listener.accept().await.unwrap();
2394            std::future::pending::<()>().await;
2395        });
2396
2397        let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2398            .await
2399            .unwrap();
2400        inner
2401            .connection_mode
2402            .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
2403
2404        let outcome = inner.reconnect(None).await.unwrap();
2405
2406        assert_eq!(outcome, ReconnectOutcome::Aborted);
2407        server.abort();
2408    }
2409
2410    #[rstest]
2411    #[tokio::test]
2412    async fn test_reconnect_outcome_is_reconnected() {
2413        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2414        let port = listener.local_addr().unwrap().port();
2415        let server = tokio::spawn(async move {
2416            let (_first, _) = listener.accept().await.unwrap();
2417            let (_second, _) = listener.accept().await.unwrap();
2418            std::future::pending::<()>().await;
2419        });
2420
2421        let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2422            .await
2423            .unwrap();
2424        inner
2425            .connection_mode
2426            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
2427
2428        let outcome = inner.reconnect(None).await.unwrap();
2429
2430        assert_eq!(outcome, ReconnectOutcome::Reconnected);
2431        assert_eq!(
2432            ConnectionMode::from_atomic(&inner.connection_mode),
2433            ConnectionMode::Active
2434        );
2435        server.abort();
2436    }
2437
2438    #[rstest]
2439    #[tokio::test]
2440    async fn test_reconnect_aborts_retired_reader() {
2441        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2442        let port = listener.local_addr().unwrap().port();
2443        let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2444            .await
2445            .unwrap();
2446        let (_peer, _) = listener.accept().await.unwrap();
2447        let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel::<()>();
2448        let pending_reader = tokio::spawn(async move {
2449            let _sender = dropped_tx;
2450            std::future::pending::<()>().await;
2451        });
2452
2453        let previous_reader = std::mem::replace(&mut inner.read_task, pending_reader);
2454        previous_reader.abort();
2455        let _ = previous_reader.await;
2456        let old_fence = inner.read_fence.clone();
2457        inner
2458            .connection_mode
2459            .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
2460
2461        let outcome = inner.reconnect(None).await.unwrap();
2462        let reader_result = tokio::time::timeout(Duration::from_secs(1), dropped_rx)
2463            .await
2464            .expect("retired reader task must be canceled");
2465
2466        assert_eq!(outcome, ReconnectOutcome::Reconnected);
2467        assert!(
2468            reader_result.is_err(),
2469            "retired reader must drop its sender without sending"
2470        );
2471        assert!(!old_fence.is_valid());
2472        assert!(inner.read_fence.is_valid());
2473        assert_eq!(
2474            ConnectionMode::from_atomic(&inner.connection_mode),
2475            ConnectionMode::Active
2476        );
2477    }
2478
2479    #[rstest]
2480    #[tokio::test]
2481    async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
2482        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2483        let port = listener.local_addr().unwrap().port();
2484        let server = tokio::spawn(async move {
2485            let (_socket, _) = listener.accept().await.unwrap();
2486            std::future::pending::<()>().await;
2487        });
2488
2489        let mut config = reconnect_test_config(port);
2490        config.heartbeat = Some(SocketHeartbeat {
2491            interval_secs: 60,
2492            payload: b"ping\r\n".to_vec(),
2493        });
2494
2495        let inner = SocketClientInner::connect_url(config, None).await.unwrap();
2496        let read_fence = inner.read_fence.clone();
2497        let read_abort = inner.read_task.abort_handle();
2498        let write_abort = inner.write_task.abort_handle();
2499        let heartbeat_abort = inner
2500            .heartbeat_task
2501            .as_ref()
2502            .expect("heartbeat task should be spawned for a configured heartbeat")
2503            .abort_handle();
2504
2505        assert!(read_fence.is_valid(), "read fence should start valid");
2506        assert!(
2507            !read_abort.is_finished(),
2508            "read task should be running before drop"
2509        );
2510        assert!(
2511            !write_abort.is_finished(),
2512            "write task should be running before drop"
2513        );
2514        assert!(
2515            !heartbeat_abort.is_finished(),
2516            "heartbeat task should be running before drop"
2517        );
2518
2519        drop(inner);
2520        wait_until_async(
2521            || async {
2522                read_abort.is_finished()
2523                    && write_abort.is_finished()
2524                    && heartbeat_abort.is_finished()
2525            },
2526            TEST_TIMEOUT,
2527        )
2528        .await;
2529
2530        assert!(!read_fence.is_valid(), "read fence was not invalidated");
2531        assert!(read_abort.is_finished(), "read task was not aborted");
2532        assert!(write_abort.is_finished(), "write task was not aborted");
2533        assert!(
2534            heartbeat_abort.is_finished(),
2535            "heartbeat task was not aborted"
2536        );
2537        server.abort();
2538    }
2539
2540    struct ScriptedReader {
2541        first: Option<Vec<u8>>,
2542        remainder: Arc<Mutex<Option<Vec<u8>>>>,
2543        pending_tx: Option<oneshot::Sender<()>>,
2544        waker: Arc<Mutex<Option<Waker>>>,
2545    }
2546
2547    impl AsyncRead for ScriptedReader {
2548        fn poll_read(
2549            mut self: Pin<&mut Self>,
2550            cx: &mut Context<'_>,
2551            buf: &mut ReadBuf<'_>,
2552        ) -> Poll<std::io::Result<()>> {
2553            if let Some(first) = self.first.take() {
2554                buf.put_slice(&first);
2555                return Poll::Ready(Ok(()));
2556            }
2557
2558            if let Some(remainder) = self.remainder.lock().take() {
2559                buf.put_slice(&remainder);
2560                return Poll::Ready(Ok(()));
2561            }
2562
2563            if let Some(tx) = self.pending_tx.take() {
2564                let _ = tx.send(());
2565            }
2566
2567            *self.waker.lock() = Some(cx.waker().clone());
2568            Poll::Pending
2569        }
2570    }
2571
2572    /// Yields one scripted chunk per read, then end of stream.
2573    struct ChunkReader {
2574        chunks: VecDeque<&'static [u8]>,
2575    }
2576
2577    impl AsyncRead for ChunkReader {
2578        fn poll_read(
2579            mut self: Pin<&mut Self>,
2580            _cx: &mut Context<'_>,
2581            buf: &mut ReadBuf<'_>,
2582        ) -> Poll<std::io::Result<()>> {
2583            if let Some(chunk) = self.chunks.pop_front() {
2584                buf.put_slice(chunk);
2585            }
2586
2587            Poll::Ready(Ok(()))
2588        }
2589    }
2590
2591    struct LimitProbeReader {
2592        remaining: usize,
2593        read_after_limit: Arc<AtomicBool>,
2594    }
2595
2596    impl AsyncRead for LimitProbeReader {
2597        fn poll_read(
2598            mut self: Pin<&mut Self>,
2599            _cx: &mut Context<'_>,
2600            buf: &mut ReadBuf<'_>,
2601        ) -> Poll<std::io::Result<()>> {
2602            if self.remaining == 0 {
2603                self.read_after_limit.store(true, Ordering::SeqCst);
2604                return Poll::Ready(Ok(()));
2605            }
2606
2607            let read_len = self.remaining.min(buf.remaining());
2608            buf.initialize_unfilled_to(read_len).fill(b'x');
2609            buf.advance(read_len);
2610            self.remaining -= read_len;
2611            Poll::Ready(Ok(()))
2612        }
2613    }
2614
2615    struct BackpressuredWriter {
2616        stream: DuplexStream,
2617        pending_tx: Option<oneshot::Sender<()>>,
2618    }
2619
2620    impl AsyncWrite for BackpressuredWriter {
2621        fn poll_write(
2622            mut self: Pin<&mut Self>,
2623            cx: &mut Context<'_>,
2624            buf: &[u8],
2625        ) -> Poll<std::io::Result<usize>> {
2626            let result = Pin::new(&mut self.stream).poll_write(cx, buf);
2627            if result.is_pending()
2628                && let Some(tx) = self.pending_tx.take()
2629            {
2630                let _ = tx.send(());
2631            }
2632
2633            result
2634        }
2635
2636        fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2637            Pin::new(&mut self.stream).poll_flush(cx)
2638        }
2639
2640        fn poll_shutdown(
2641            mut self: Pin<&mut Self>,
2642            cx: &mut Context<'_>,
2643        ) -> Poll<std::io::Result<()>> {
2644            Pin::new(&mut self.stream).poll_shutdown(cx)
2645        }
2646    }
2647
2648    struct FailingWriter;
2649
2650    impl AsyncWrite for FailingWriter {
2651        fn poll_write(
2652            self: Pin<&mut Self>,
2653            _cx: &mut Context<'_>,
2654            _buf: &[u8],
2655        ) -> Poll<std::io::Result<usize>> {
2656            Poll::Ready(Err(std::io::Error::new(
2657                std::io::ErrorKind::BrokenPipe,
2658                "test writer failure",
2659            )))
2660        }
2661
2662        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2663            Poll::Ready(Ok(()))
2664        }
2665
2666        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2667            Poll::Ready(Ok(()))
2668        }
2669    }
2670
2671    struct RecordingWriter {
2672        bytes: Arc<Mutex<Vec<u8>>>,
2673    }
2674
2675    impl AsyncWrite for RecordingWriter {
2676        fn poll_write(
2677            self: Pin<&mut Self>,
2678            _cx: &mut Context<'_>,
2679            buf: &[u8],
2680        ) -> Poll<std::io::Result<usize>> {
2681            self.bytes.lock().extend_from_slice(buf);
2682            Poll::Ready(Ok(buf.len()))
2683        }
2684
2685        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2686            Poll::Ready(Ok(()))
2687        }
2688
2689        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2690            Poll::Ready(Ok(()))
2691        }
2692    }
2693
2694    fn test_socket_client(
2695        connection_state: Arc<AtomicU8>,
2696        state_notify: Arc<tokio::sync::Notify>,
2697        controller_task: tokio::task::JoinHandle<()>,
2698    ) -> SocketClient {
2699        let (writer_tx, _writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
2700        let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2701        let abort_handle = controller_task.abort_handle();
2702        controller_lifecycle.set_abort(move || abort_handle.abort());
2703
2704        SocketClient {
2705            controller_task,
2706            connection_mode: connection_state,
2707            state_notify,
2708            connect_timeout: Duration::from_secs(1),
2709            writer_tx,
2710            controller_lifecycle,
2711            controller_notify: Arc::new(tokio::sync::Notify::new()),
2712            state_sink: None,
2713        }
2714    }
2715
2716    #[rstest]
2717    #[tokio::test]
2718    async fn test_send_reports_writer_buffer_full() {
2719        let state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2720        let notify = Arc::new(tokio::sync::Notify::new());
2721
2722        let task = tokio::spawn(std::future::pending::<()>());
2723        let mut client = test_socket_client(state, notify, task);
2724        let (writer_tx, mut writer_rx) = writer::channel(1);
2725        client.writer_tx = writer_tx;
2726
2727        client.send_bytes(vec![1]).await.unwrap();
2728        assert!(matches!(
2729            client.send_bytes(vec![2]).await,
2730            Err(SendError::BufferFull)
2731        ));
2732        let (command, permit) = writer_rx.recv().await.unwrap();
2733        assert!(matches!(command, WriterCommand::Send(data) if data == vec![1]));
2734        assert!(matches!(
2735            client.send_bytes(vec![2]).await,
2736            Err(SendError::BufferFull)
2737        ));
2738        drop(permit);
2739        client.send_bytes(vec![3]).await.unwrap();
2740        let (command, _) = writer_rx.recv().await.unwrap();
2741        assert!(matches!(command, WriterCommand::Send(data) if data == vec![3]));
2742    }
2743
2744    #[rstest]
2745    #[tokio::test]
2746    async fn test_reconnect_handle_is_closed_after_client_drop() {
2747        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2748        let state_notify = Arc::new(tokio::sync::Notify::new());
2749
2750        let controller_task = tokio::spawn(std::future::pending::<()>());
2751        let client = test_socket_client(connection_state, state_notify, controller_task);
2752        let handle = client.reconnect_handle();
2753
2754        drop(client);
2755
2756        assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
2757    }
2758
2759    #[rstest]
2760    #[tokio::test]
2761    async fn test_concurrent_drop_defers_controller_abort_until_request_completes() {
2762        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2763        let state_notify = Arc::new(tokio::sync::Notify::new());
2764
2765        let controller_task = tokio::spawn(std::future::pending::<()>());
2766        let controller_abort = controller_task.abort_handle();
2767        let mut client = test_socket_client(connection_state, state_notify, controller_task);
2768        let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
2769        let callback_release_guard = CondvarReleaseGuard::new(Arc::clone(&callback_release));
2770        let callback_release_clone = Arc::clone(&callback_release);
2771        let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
2772        let states = Arc::new(Mutex::new(Vec::new()));
2773        let states_clone = Arc::clone(&states);
2774        client.state_sink = Some(SocketStateSink::new(move |state| {
2775            states_clone.lock().push(state);
2776            callback_entered_tx.send(()).unwrap();
2777            let (lock, condvar) = callback_release_clone.as_ref();
2778            let mut released = lock.lock();
2779            while !*released {
2780                condvar.wait(&mut released);
2781            }
2782        }));
2783
2784        let handle = client.reconnect_handle();
2785        let surviving_handle = handle.clone();
2786        let controller_notify = Arc::clone(&handle.controller_notify);
2787        let request_thread = std::thread::spawn(move || handle.request_reconnect());
2788
2789        recv_rendezvous(callback_entered_rx, "socket state callback entry").await;
2790        let (drop_finished_tx, drop_finished_rx) = std::sync::mpsc::channel();
2791
2792        let drop_thread = std::thread::spawn(move || {
2793            drop(client);
2794            drop_finished_tx.send(()).unwrap();
2795        });
2796
2797        recv_rendezvous(drop_finished_rx, "client drop").await;
2798        drop_thread.join().unwrap();
2799        assert!(!controller_abort.is_finished());
2800
2801        callback_release_guard.release();
2802        assert_eq!(
2803            request_thread.join().unwrap(),
2804            ReconnectRequestOutcome::Accepted
2805        );
2806        tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2807            .await
2808            .expect("accepted request should notify before deferred controller abort");
2809        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
2810
2811        assert_eq!(
2812            surviving_handle.request_reconnect(),
2813            ReconnectRequestOutcome::Closed
2814        );
2815        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2816        assert!(
2817            tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2818                .await
2819                .is_err(),
2820            "closed request should not notify controller",
2821        );
2822    }
2823
2824    #[rstest]
2825    #[tokio::test]
2826    async fn test_reconnect_state_callback_can_drop_client() {
2827        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2828        let state_notify = Arc::new(tokio::sync::Notify::new());
2829
2830        let controller_task = tokio::spawn(std::future::pending::<()>());
2831        let controller_abort = controller_task.abort_handle();
2832        let mut client = test_socket_client(connection_state, state_notify, controller_task);
2833        let client_slot = Arc::new(Mutex::new(None));
2834        let client_slot_callback = Arc::clone(&client_slot);
2835        let states = Arc::new(Mutex::new(Vec::new()));
2836        let states_callback = Arc::clone(&states);
2837        client.state_sink = Some(SocketStateSink::new(move |state| {
2838            states_callback.lock().push(state);
2839            drop(client_slot_callback.lock().take());
2840        }));
2841
2842        let handle = client.reconnect_handle();
2843        let controller_notify = Arc::clone(&handle.controller_notify);
2844        *client_slot.lock() = Some(client);
2845        let (result_tx, result_rx) = std::sync::mpsc::channel();
2846        std::thread::spawn(move || result_tx.send(handle.request_reconnect()).unwrap());
2847
2848        assert_eq!(
2849            recv_rendezvous(result_rx, "reconnect request").await,
2850            ReconnectRequestOutcome::Accepted
2851        );
2852        tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2853            .await
2854            .expect("accepted request should notify before deferred controller abort");
2855        wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
2856
2857        assert!(client_slot.lock().is_none());
2858        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2859    }
2860
2861    #[rstest]
2862    #[tokio::test]
2863    async fn test_manual_reconnect_uses_controller_path() {
2864        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2865        let port = listener.local_addr().unwrap().port();
2866        let (first_accepted_tx, first_accepted_rx) = oneshot::channel();
2867        let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
2868        let (payload_tx, payload_rx) = oneshot::channel();
2869        let (release_tx, release_rx) = oneshot::channel();
2870
2871        let server = task::spawn(async move {
2872            let (_first, _) = listener.accept().await.unwrap();
2873            first_accepted_tx.send(()).unwrap();
2874
2875            let (mut second, _) = listener.accept().await.unwrap();
2876            second_accepted_tx.send(()).unwrap();
2877            let mut payload = [0_u8; 8];
2878            second.read_exact(&mut payload).await.unwrap();
2879            payload_tx.send(payload).unwrap();
2880            let _ = release_rx.await;
2881        });
2882
2883        let states = Arc::new(Mutex::new(Vec::new()));
2884        let states_callback = Arc::clone(&states);
2885        let sink = SocketStateSink::new(move |state| {
2886            states_callback.lock().push(state);
2887        });
2888
2889        let callback_count = Arc::new(AtomicUsize::new(0));
2890        let callback_count_clone = Arc::clone(&callback_count);
2891        let post_reconnection = Arc::new(move || {
2892            callback_count_clone.fetch_add(1, Ordering::SeqCst);
2893        });
2894
2895        let client = SocketClient::builder()
2896            .config(reconnect_test_config(port))
2897            .post_reconnection(post_reconnection)
2898            .state_sink(sink)
2899            .connect()
2900            .await
2901            .unwrap();
2902        first_accepted_rx.await.unwrap();
2903        let handle = client.reconnect_handle();
2904
2905        assert!(client.request_reconnect());
2906        assert_eq!(
2907            handle.request_reconnect(),
2908            ReconnectRequestOutcome::AlreadyReconnecting
2909        );
2910        assert!(!client.request_reconnect());
2911        assert_eq!(
2912            *states.lock(),
2913            vec![SocketState::Connected, SocketState::Disconnected]
2914        );
2915
2916        tokio::time::timeout(TEST_TIMEOUT, second_accepted_rx)
2917            .await
2918            .expect("controller should establish a replacement connection")
2919            .unwrap();
2920        wait_until_async(
2921            || async { client.is_active() && callback_count.load(Ordering::SeqCst) == 1 },
2922            TEST_TIMEOUT,
2923        )
2924        .await;
2925        client.send_bytes(b"manual".to_vec()).await.unwrap();
2926
2927        assert_eq!(
2928            tokio::time::timeout(TEST_TIMEOUT, payload_rx)
2929                .await
2930                .expect("replacement connection should receive the framed payload")
2931                .unwrap(),
2932            *b"manual\r\n"
2933        );
2934        assert_eq!(callback_count.load(Ordering::SeqCst), 1);
2935        assert_eq!(
2936            *states.lock(),
2937            vec![
2938                SocketState::Connected,
2939                SocketState::Disconnected,
2940                SocketState::Connected,
2941            ]
2942        );
2943
2944        client.close().await;
2945        release_tx.send(()).unwrap();
2946        server.await.unwrap();
2947        assert_eq!(callback_count.load(Ordering::SeqCst), 1);
2948        assert_eq!(states.lock().len(), 3);
2949    }
2950
2951    #[rstest]
2952    #[case(ConnectionMode::Disconnect)]
2953    #[case(ConnectionMode::Closed)]
2954    #[tokio::test]
2955    async fn test_send_bytes_rejects_terminal_state(#[case] mode: ConnectionMode) {
2956        let connection_state = Arc::new(AtomicU8::new(mode.as_u8()));
2957        let state_notify = Arc::new(tokio::sync::Notify::new());
2958
2959        let controller_task = tokio::spawn(std::future::pending::<()>());
2960        let client = test_socket_client(connection_state, state_notify, controller_task);
2961
2962        let result = client.send_bytes(b"terminal".to_vec()).await;
2963
2964        assert!(matches!(result, Err(SendError::Closed)));
2965    }
2966
2967    #[rstest]
2968    #[tokio::test(start_paused = true)]
2969    async fn test_close_sets_closed_after_controller_was_aborted() {
2970        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2971        let state_notify = Arc::new(tokio::sync::Notify::new());
2972
2973        let controller_task = tokio::spawn(std::future::pending::<()>());
2974        controller_task.abort();
2975        let client = test_socket_client(connection_state, state_notify, controller_task);
2976
2977        client.close_with_timeout(Duration::from_millis(1)).await;
2978
2979        assert!(client.is_closed());
2980    }
2981
2982    #[rstest]
2983    #[tokio::test]
2984    async fn test_reconnect_exhaustion_closes_client() {
2985        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2986        let port = listener.local_addr().unwrap().port();
2987        let (accepted_tx, accepted_rx) = oneshot::channel();
2988        let (release_tx, release_rx) = oneshot::channel();
2989
2990        let server = task::spawn(async move {
2991            let (socket, _) = listener.accept().await.unwrap();
2992            let _ = accepted_tx.send(());
2993            let _ = release_rx.await;
2994            drop(socket);
2995            drop(listener);
2996        });
2997
2998        let config = SocketConfig {
2999            url: format!("127.0.0.1:{port}"),
3000            mode: Mode::Plain,
3001            suffix: b"\r\n".to_vec(),
3002            message_handler: None,
3003            heartbeat: None,
3004            connect_timeout_ms: Some(100),
3005            reconnect_delay_initial_ms: Some(1),
3006            reconnect_delay_max_ms: Some(1),
3007            reconnect_backoff_factor: Some(1.0),
3008            reconnect_jitter_ms: Some(0),
3009            connection_max_retries: Some(1),
3010            reconnect_max_attempts: Some(1),
3011            heartbeat_timeout_secs: None,
3012            writer_capacity: None,
3013            certs_dir: None,
3014        };
3015
3016        let states = Arc::new(Mutex::new(Vec::new()));
3017        let states_callback = Arc::clone(&states);
3018        let sink = SocketStateSink::new(move |state| {
3019            states_callback.lock().push(state);
3020        });
3021
3022        let client = SocketClient::builder()
3023            .config(config)
3024            .state_sink(sink)
3025            .connect()
3026            .await
3027            .unwrap();
3028        accepted_rx.await.unwrap();
3029        release_tx.send(()).unwrap();
3030
3031        wait_until_async(|| async { client.is_closed() }, Duration::from_secs(5)).await;
3032        assert!(client.is_closed());
3033        assert_eq!(
3034            *states.lock(),
3035            vec![SocketState::Connected, SocketState::Disconnected]
3036        );
3037
3038        client.close().await;
3039        assert!(client.is_closed());
3040        assert_eq!(states.lock().len(), 2);
3041        server.await.unwrap();
3042    }
3043
3044    #[rstest]
3045    #[tokio::test]
3046    async fn test_drop_suppresses_socket_state_event() {
3047        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3048        let port = listener.local_addr().unwrap().port();
3049        let (accepted_tx, accepted_rx) = oneshot::channel();
3050
3051        let server = task::spawn(async move {
3052            let (_socket, _) = listener.accept().await.unwrap();
3053            let _ = accepted_tx.send(());
3054            std::future::pending::<()>().await;
3055        });
3056
3057        let config = SocketConfig {
3058            url: format!("127.0.0.1:{port}"),
3059            mode: Mode::Plain,
3060            suffix: b"\r\n".to_vec(),
3061            message_handler: None,
3062            heartbeat: None,
3063            connect_timeout_ms: Some(100),
3064            reconnect_delay_initial_ms: Some(1),
3065            reconnect_delay_max_ms: Some(1),
3066            reconnect_backoff_factor: Some(1.0),
3067            reconnect_jitter_ms: Some(0),
3068            connection_max_retries: Some(1),
3069            reconnect_max_attempts: Some(1),
3070            heartbeat_timeout_secs: None,
3071            writer_capacity: None,
3072            certs_dir: None,
3073        };
3074
3075        let states = Arc::new(Mutex::new(Vec::new()));
3076        let states_callback = Arc::clone(&states);
3077        let sink = SocketStateSink::new(move |state| {
3078            states_callback.lock().push(state);
3079        });
3080
3081        let client = SocketClient::builder()
3082            .config(config)
3083            .state_sink(sink)
3084            .connect()
3085            .await
3086            .unwrap();
3087        accepted_rx.await.unwrap();
3088
3089        drop(client);
3090        sleep(Duration::from_millis(25)).await;
3091
3092        assert_eq!(*states.lock(), vec![SocketState::Connected]);
3093        server.abort();
3094    }
3095
3096    #[rstest]
3097    #[tokio::test]
3098    async fn test_graceful_close_is_idempotent() {
3099        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3100        let port = listener.local_addr().unwrap().port();
3101        let (accepted_tx, accepted_rx) = oneshot::channel();
3102
3103        let server = task::spawn(async move {
3104            let (_socket, _) = listener.accept().await.unwrap();
3105            let _ = accepted_tx.send(());
3106            std::future::pending::<()>().await;
3107        });
3108
3109        let config = SocketConfig {
3110            url: format!("127.0.0.1:{port}"),
3111            mode: Mode::Plain,
3112            suffix: b"\r\n".to_vec(),
3113            message_handler: None,
3114            heartbeat: None,
3115            connect_timeout_ms: None,
3116            reconnect_delay_initial_ms: None,
3117            reconnect_delay_max_ms: None,
3118            reconnect_backoff_factor: None,
3119            reconnect_jitter_ms: None,
3120            connection_max_retries: None,
3121            reconnect_max_attempts: None,
3122            heartbeat_timeout_secs: None,
3123            writer_capacity: None,
3124            certs_dir: None,
3125        };
3126
3127        let client = SocketClient::builder()
3128            .config(config)
3129            .connect()
3130            .await
3131            .unwrap();
3132        accepted_rx.await.unwrap();
3133
3134        client.close().await;
3135        client.close().await;
3136
3137        assert!(client.is_closed());
3138        server.abort();
3139    }
3140
3141    #[rstest]
3142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3143    async fn test_read_loop_drops_remaining_frames_after_session_replaced() {
3144        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3145        let read_fence = ReadSessionFence::new();
3146        let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3147        let received_clone = Arc::clone(&received);
3148        let handler_release = Arc::new((Mutex::new(false), Condvar::new()));
3149        let handler_release_guard = CondvarReleaseGuard::new(Arc::clone(&handler_release));
3150        let handler_release_clone = Arc::clone(&handler_release);
3151        let (handler_entered_tx, handler_entered_rx) = std::sync::mpsc::channel();
3152
3153        let handler: TcpMessageHandler = Arc::new(move |data| {
3154            received_clone.lock().push(data.to_vec());
3155            handler_entered_tx.send(()).unwrap();
3156            let (lock, condvar) = handler_release_clone.as_ref();
3157            let mut released = lock.lock();
3158            while !*released {
3159                condvar.wait(&mut released);
3160            }
3161        });
3162
3163        let reader = ScriptedReader {
3164            first: Some(b"first\r\nsecond\r\n".to_vec()),
3165            remainder: Arc::new(Mutex::new(None)),
3166            pending_tx: None,
3167            waker: Arc::new(Mutex::new(None)),
3168        };
3169
3170        let read_task = SocketClientInner::spawn_read_task(
3171            Arc::clone(&connection_state),
3172            read_fence.clone(),
3173            reader,
3174            Some(handler),
3175            b"\r\n".to_vec(),
3176            None,
3177        );
3178
3179        recv_rendezvous(handler_entered_rx, "socket first-handler entry").await;
3180        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3181        read_fence.invalidate();
3182        read_task.abort();
3183        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3184
3185        handler_release_guard.release();
3186        await_task_termination(read_task, "old socket read task", true).await;
3187
3188        assert_eq!(received.lock().as_slice(), &[b"first".to_vec()]);
3189    }
3190
3191    #[rstest]
3192    #[tokio::test(start_paused = true)]
3193    async fn test_read_loop_drops_partial_old_session_frame() {
3194        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3195        let old_read_fence = ReadSessionFence::new();
3196        let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3197        let received_clone = Arc::clone(&received);
3198        let handler: TcpMessageHandler =
3199            Arc::new(move |data| received_clone.lock().push(data.to_vec()));
3200        let (pending_tx, pending_rx) = oneshot::channel();
3201        let remainder = Arc::new(Mutex::new(None));
3202        let waker = Arc::new(Mutex::new(None));
3203
3204        let reader = ScriptedReader {
3205            first: Some(b"old".to_vec()),
3206            remainder: Arc::clone(&remainder),
3207            pending_tx: Some(pending_tx),
3208            waker: Arc::clone(&waker),
3209        };
3210
3211        let read_task = SocketClientInner::spawn_read_task(
3212            Arc::clone(&connection_state),
3213            old_read_fence.clone(),
3214            reader,
3215            Some(handler),
3216            b"\r\n".to_vec(),
3217            None,
3218        );
3219
3220        pending_rx.await.unwrap();
3221        connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3222        old_read_fence.invalidate();
3223        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3224        *remainder.lock() = Some(b"\r\nnew\r\n".to_vec());
3225        waker.lock().take().unwrap().wake();
3226        read_task.await.unwrap();
3227
3228        let fresh_reader = ScriptedReader {
3229            first: Some(b"new\r\n".to_vec()),
3230            remainder: Arc::new(Mutex::new(None)),
3231            pending_tx: None,
3232            waker: Arc::new(Mutex::new(None)),
3233        };
3234
3235        let fresh_connection_state = Arc::clone(&connection_state);
3236        let fresh_received = Arc::clone(&received);
3237        let fresh_handler: TcpMessageHandler = Arc::new(move |data| {
3238            fresh_received.lock().push(data.to_vec());
3239            fresh_connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3240        });
3241
3242        SocketClientInner::spawn_read_task(
3243            Arc::clone(&connection_state),
3244            ReadSessionFence::new(),
3245            fresh_reader,
3246            Some(fresh_handler),
3247            b"\r\n".to_vec(),
3248            None,
3249        )
3250        .await
3251        .unwrap();
3252
3253        assert_eq!(received.lock().as_slice(), &[b"new".to_vec()]);
3254    }
3255
3256    #[rstest]
3257    #[tokio::test(start_paused = true)]
3258    async fn test_read_loop_stops_when_first_handler_ends_session() {
3259        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3260        let handler_state = Arc::clone(&connection_state);
3261        let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3262        let received_clone = Arc::clone(&received);
3263        let handler: TcpMessageHandler = Arc::new(move |data| {
3264            received_clone.lock().push(data.to_vec());
3265            handler_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3266        });
3267
3268        let (pending_tx, _pending_rx) = oneshot::channel();
3269
3270        let reader = ScriptedReader {
3271            first: Some(b"first\r\nsecond\r\n".to_vec()),
3272            remainder: Arc::new(Mutex::new(None)),
3273            pending_tx: Some(pending_tx),
3274            waker: Arc::new(Mutex::new(None)),
3275        };
3276
3277        SocketClientInner::spawn_read_task(
3278            connection_state,
3279            ReadSessionFence::new(),
3280            reader,
3281            Some(handler),
3282            b"\r\n".to_vec(),
3283            None,
3284        )
3285        .await
3286        .unwrap();
3287
3288        assert_eq!(received.lock().as_slice(), &[b"first".to_vec()]);
3289    }
3290
3291    #[tokio::test]
3292    async fn test_read_loop_frames_suffix_split_across_reads() {
3293        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3294        let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3295        let received_clone = Arc::clone(&received);
3296        let handler: TcpMessageHandler =
3297            Arc::new(move |data| received_clone.lock().push(data.to_vec()));
3298
3299        // The suffix straddles two reads twice: after `first` and for the final empty frame
3300        let reader = ChunkReader {
3301            chunks: VecDeque::from([&b"fir"[..], b"st\r", b"\nsecond\r\n\r", b"\n"]),
3302        };
3303
3304        SocketClientInner::run_read_loop(
3305            connection_state,
3306            ReadSessionFence::new(),
3307            reader,
3308            Some(handler),
3309            b"\r\n".to_vec(),
3310            None,
3311            Duration::from_secs(1),
3312        )
3313        .await;
3314
3315        assert_eq!(
3316            received.lock().as_slice(),
3317            &[b"first".to_vec(), b"second".to_vec(), Vec::new()]
3318        );
3319    }
3320
3321    #[tokio::test]
3322    async fn test_read_loop_closes_when_unframed_buffer_exceeds_limit() {
3323        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3324        let read_after_limit = Arc::new(AtomicBool::new(false));
3325
3326        let reader = LimitProbeReader {
3327            remaining: MAX_READ_BUFFER_BYTES + 1,
3328            read_after_limit: Arc::clone(&read_after_limit),
3329        };
3330
3331        SocketClientInner::run_read_loop(
3332            connection_state,
3333            ReadSessionFence::new(),
3334            reader,
3335            None,
3336            b"\r\n".to_vec(),
3337            None,
3338            Duration::from_secs(1),
3339        )
3340        .await;
3341
3342        assert!(!read_after_limit.load(Ordering::SeqCst));
3343    }
3344
3345    #[rstest]
3346    #[tokio::test(start_paused = true)]
3347    async fn test_stalled_socket_write_sends_reconnect_replay_before_buffer() {
3348        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3349
3350        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3351        let state_notify = Arc::new(tokio::sync::Notify::new());
3352        let (stream, _non_reading_peer) = tokio::io::duplex(1);
3353        let (pending_tx, pending_rx) = oneshot::channel();
3354
3355        let writer: TestWriter = Box::pin(BackpressuredWriter {
3356            stream,
3357            pending_tx: Some(pending_tx),
3358        });
3359
3360        let (writer_tx, writer_rx) = writer::channel::<WriterCommand<TestWriter>>(2);
3361        let states = Arc::new(Mutex::new(Vec::new()));
3362        let states_callback = Arc::clone(&states);
3363        let sink = SocketStateSink::new(move |state| {
3364            states_callback.lock().push(state);
3365        });
3366
3367        let write_task = SocketClientInner::spawn_write_task(
3368            Arc::clone(&connection_state),
3369            Arc::clone(&state_notify),
3370            writer,
3371            writer_rx,
3372            b"\r\n".to_vec(),
3373            Some(sink),
3374        );
3375
3376        writer_tx
3377            .send(WriterCommand::Send(Bytes::from_static(b"complete-message")))
3378            .unwrap();
3379        pending_rx.await.unwrap();
3380        writer_tx
3381            .send(WriterCommand::Send(Bytes::from_static(b"second-message")))
3382            .unwrap();
3383        assert_eq!(
3384            ConnectionMode::from_atomic(&connection_state),
3385            ConnectionMode::Active
3386        );
3387
3388        for _ in 0..100 {
3389            assert!(matches!(
3390                writer_tx.send(WriterCommand::Send(Bytes::from_static(b"overflow"))),
3391                Err(SendError::BufferFull)
3392            ));
3393        }
3394
3395        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
3396        tokio::task::yield_now().await;
3397        assert_eq!(
3398            ConnectionMode::from_atomic(&connection_state),
3399            ConnectionMode::Reconnect
3400        );
3401        assert!(matches!(
3402            writer_tx.send(WriterCommand::Send(Bytes::from_static(b"overflow"))),
3403            Err(SendError::BufferFull)
3404        ));
3405
3406        let recorded = Arc::new(Mutex::new(Vec::new()));
3407
3408        let new_writer: TestWriter = Box::pin(RecordingWriter {
3409            bytes: Arc::clone(&recorded),
3410        });
3411
3412        let (update_tx, update_rx) = oneshot::channel();
3413        writer_tx
3414            .send_update(WriterCommand::UpdateWithReplay(
3415                new_writer,
3416                vec![Bytes::from_static(b"authentication")],
3417                update_tx,
3418            ))
3419            .unwrap();
3420
3421        tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
3422        yield_now().await;
3423
3424        assert!(
3425            tokio::time::timeout(Duration::from_secs(10), update_rx)
3426                .await
3427                .expect("writer update should not remain queued behind a stalled write")
3428                .unwrap(),
3429            "writer should report successful replay"
3430        );
3431        assert_eq!(
3432            ConnectionMode::from_atomic(&connection_state),
3433            ConnectionMode::Reconnect
3434        );
3435        assert_eq!(
3436            recorded.lock().as_slice(),
3437            b"authentication\r\ncomplete-message\r\nsecond-message\r\n"
3438        );
3439
3440        writer_tx
3441            .send(WriterCommand::Send(Bytes::from_static(b"second-message")))
3442            .unwrap();
3443
3444        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3445        state_notify.notify_waiters();
3446        drop(writer_tx);
3447        write_task.await.unwrap();
3448
3449        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
3450    }
3451
3452    #[rstest]
3453    #[tokio::test(start_paused = true)]
3454    async fn test_send_after_writer_update_drains_when_reconnect_completes() {
3455        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3456
3457        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3458        let state_notify = Arc::new(tokio::sync::Notify::new());
3459
3460        let initial_writer: TestWriter = Box::pin(RecordingWriter {
3461            bytes: Arc::new(Mutex::new(Vec::new())),
3462        });
3463
3464        let (writer_tx, writer_rx) =
3465            writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3466        let write_task = SocketClientInner::spawn_write_task(
3467            Arc::clone(&connection_state),
3468            Arc::clone(&state_notify),
3469            initial_writer,
3470            writer_rx,
3471            b"\r\n".to_vec(),
3472            None,
3473        );
3474
3475        let recorded = Arc::new(Mutex::new(Vec::new()));
3476
3477        let new_writer: TestWriter = Box::pin(RecordingWriter {
3478            bytes: Arc::clone(&recorded),
3479        });
3480
3481        let (update_tx, update_rx) = oneshot::channel();
3482        writer_tx
3483            .send(WriterCommand::Update(new_writer, update_tx))
3484            .unwrap();
3485        assert!(update_rx.await.unwrap());
3486
3487        writer_tx
3488            .send(WriterCommand::Send(Bytes::from_static(b"late")))
3489            .unwrap();
3490        yield_now().await;
3491
3492        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3493        state_notify.notify_waiters();
3494        writer_tx
3495            .send(WriterCommand::Send(Bytes::from_static(b"new")))
3496            .unwrap();
3497        tokio::time::advance(Duration::from_millis(
3498            CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3499        ))
3500        .await;
3501        yield_now().await;
3502
3503        let actual = recorded.lock().clone();
3504        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3505        state_notify.notify_waiters();
3506        drop(writer_tx);
3507        write_task.await.unwrap();
3508
3509        assert_eq!(actual, b"late\r\nnew\r\n");
3510    }
3511
3512    #[rstest]
3513    #[case::latest_wins(
3514        &["subscription-a", "subscription-b"],
3515        &["subscription-b"],
3516        b"subscription-b\r\n",
3517    )]
3518    #[case::different_replay_drains(
3519        &["subscription-b"],
3520        &["subscription-a"],
3521        b"subscription-a\r\nsubscription-b\r\n",
3522    )]
3523    #[tokio::test]
3524    async fn test_send_or_replay_buffering(
3525        #[case] buffered: &[&str],
3526        #[case] replay: &[&str],
3527        #[case] expected: &[u8],
3528    ) {
3529        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3530
3531        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3532        let state_notify = Arc::new(tokio::sync::Notify::new());
3533
3534        let initial_writer: TestWriter = Box::pin(RecordingWriter {
3535            bytes: Arc::new(Mutex::new(Vec::new())),
3536        });
3537
3538        let (writer_tx, writer_rx) =
3539            writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3540        let write_task = SocketClientInner::spawn_write_task(
3541            Arc::clone(&connection_state),
3542            Arc::clone(&state_notify),
3543            initial_writer,
3544            writer_rx,
3545            b"\r\n".to_vec(),
3546            None,
3547        );
3548
3549        for data in buffered {
3550            writer_tx
3551                .send(WriterCommand::SendOrReplay {
3552                    key: 7,
3553                    data: Bytes::copy_from_slice(data.as_bytes()),
3554                })
3555                .unwrap();
3556        }
3557
3558        let recorded = Arc::new(Mutex::new(Vec::new()));
3559
3560        let new_writer: TestWriter = Box::pin(RecordingWriter {
3561            bytes: Arc::clone(&recorded),
3562        });
3563
3564        let (update_tx, update_rx) = oneshot::channel();
3565        writer_tx
3566            .send(WriterCommand::UpdateWithReplay(
3567                new_writer,
3568                replay
3569                    .iter()
3570                    .map(|data| Bytes::copy_from_slice(data.as_bytes()))
3571                    .collect(),
3572                update_tx,
3573            ))
3574            .unwrap();
3575
3576        assert!(update_rx.await.unwrap());
3577        assert_eq!(recorded.lock().as_slice(), expected);
3578        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3579        state_notify.notify_waiters();
3580        drop(writer_tx);
3581        write_task.await.unwrap();
3582    }
3583
3584    #[rstest]
3585    #[tokio::test(start_paused = true)]
3586    async fn test_send_or_replay_after_reconnect_supersedes_buffered_command() {
3587        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3588
3589        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3590        let state_notify = Arc::new(tokio::sync::Notify::new());
3591        let recorded = Arc::new(Mutex::new(Vec::new()));
3592
3593        let writer: TestWriter = Box::pin(RecordingWriter {
3594            bytes: Arc::clone(&recorded),
3595        });
3596
3597        let (writer_tx, writer_rx) =
3598            writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3599        let write_task = SocketClientInner::spawn_write_task(
3600            Arc::clone(&connection_state),
3601            Arc::clone(&state_notify),
3602            writer,
3603            writer_rx,
3604            b"\r\n".to_vec(),
3605            None,
3606        );
3607
3608        writer_tx
3609            .send(WriterCommand::SendOrReplay {
3610                key: 1,
3611                data: Bytes::from_static(b"subscription-old"),
3612            })
3613            .unwrap();
3614
3615        yield_now().await;
3616
3617        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3618        writer_tx
3619            .send(WriterCommand::SendOrReplay {
3620                key: 1,
3621                data: Bytes::from_static(b"subscription-new"),
3622            })
3623            .unwrap();
3624
3625        tokio::time::advance(Duration::from_millis(
3626            CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3627        ))
3628        .await;
3629        yield_now().await;
3630
3631        let actual = recorded.lock().clone();
3632        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3633        state_notify.notify_waiters();
3634        drop(writer_tx);
3635        write_task.await.unwrap();
3636
3637        assert_eq!(actual, b"subscription-new\r\n");
3638    }
3639
3640    #[rstest]
3641    #[tokio::test(start_paused = true)]
3642    async fn test_active_reconnect_buffer_write_failure_reconnects_and_retries() {
3643        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3644
3645        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3646        let state_notify = Arc::new(tokio::sync::Notify::new());
3647
3648        let initial_writer: TestWriter = Box::pin(RecordingWriter {
3649            bytes: Arc::new(Mutex::new(Vec::new())),
3650        });
3651
3652        let (writer_tx, writer_rx) =
3653            writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3654        let states = Arc::new(Mutex::new(Vec::new()));
3655        let states_callback = Arc::clone(&states);
3656        let sink = SocketStateSink::new(move |state| {
3657            states_callback.lock().push(state);
3658        });
3659
3660        let write_task = SocketClientInner::spawn_write_task(
3661            Arc::clone(&connection_state),
3662            Arc::clone(&state_notify),
3663            initial_writer,
3664            writer_rx,
3665            b"\r\n".to_vec(),
3666            Some(sink),
3667        );
3668
3669        let (update_tx, update_rx) = oneshot::channel();
3670        let failing_writer: TestWriter = Box::pin(FailingWriter);
3671        writer_tx
3672            .send(WriterCommand::Update(failing_writer, update_tx))
3673            .unwrap();
3674        assert!(update_rx.await.unwrap());
3675
3676        writer_tx
3677            .send(WriterCommand::Send(Bytes::from_static(b"late")))
3678            .unwrap();
3679        yield_now().await;
3680        connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3681        state_notify.notify_waiters();
3682        tokio::time::advance(Duration::from_millis(
3683            CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3684        ))
3685        .await;
3686        yield_now().await;
3687
3688        assert_eq!(
3689            ConnectionMode::from_atomic(&connection_state),
3690            ConnectionMode::Reconnect
3691        );
3692        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
3693
3694        let recorded = Arc::new(Mutex::new(Vec::new()));
3695
3696        let replacement: TestWriter = Box::pin(RecordingWriter {
3697            bytes: Arc::clone(&recorded),
3698        });
3699
3700        let (retry_tx, retry_rx) = oneshot::channel();
3701        writer_tx
3702            .send(WriterCommand::Update(replacement, retry_tx))
3703            .unwrap();
3704        assert!(retry_rx.await.unwrap());
3705
3706        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3707        state_notify.notify_waiters();
3708        drop(writer_tx);
3709        write_task.await.unwrap();
3710
3711        assert_eq!(recorded.lock().as_slice(), b"late\r\n");
3712    }
3713
3714    #[rstest]
3715    #[tokio::test(start_paused = true)]
3716    async fn test_failed_send_or_replay_is_not_duplicated_after_replay() {
3717        type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3718
3719        let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3720        let state_notify = Arc::new(tokio::sync::Notify::new());
3721        let (stream, _non_reading_peer) = tokio::io::duplex(1);
3722        let (pending_tx, pending_rx) = oneshot::channel();
3723
3724        let writer: TestWriter = Box::pin(BackpressuredWriter {
3725            stream,
3726            pending_tx: Some(pending_tx),
3727        });
3728
3729        let (writer_tx, writer_rx) =
3730            writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3731        let write_task = SocketClientInner::spawn_write_task(
3732            Arc::clone(&connection_state),
3733            Arc::clone(&state_notify),
3734            writer,
3735            writer_rx,
3736            b"\r\n".to_vec(),
3737            None,
3738        );
3739        let subscription = Bytes::from_static(b"subscription");
3740
3741        writer_tx
3742            .send(WriterCommand::SendOrReplay {
3743                key: 7,
3744                data: subscription.clone(),
3745            })
3746            .unwrap();
3747
3748        pending_rx.await.unwrap();
3749
3750        let recorded = Arc::new(Mutex::new(Vec::new()));
3751
3752        let new_writer: TestWriter = Box::pin(RecordingWriter {
3753            bytes: Arc::clone(&recorded),
3754        });
3755
3756        let (update_tx, update_rx) = oneshot::channel();
3757        writer_tx
3758            .send(WriterCommand::UpdateWithReplay(
3759                new_writer,
3760                vec![subscription],
3761                update_tx,
3762            ))
3763            .unwrap();
3764
3765        tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
3766        yield_now().await;
3767        assert!(update_rx.await.unwrap());
3768        assert_eq!(recorded.lock().as_slice(), b"subscription\r\n");
3769
3770        connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3771        state_notify.notify_waiters();
3772        drop(writer_tx);
3773        write_task.await.unwrap();
3774    }
3775
3776    #[rstest]
3777    #[tokio::test]
3778    async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
3779        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3780        listener.set_nonblocking(true).unwrap();
3781        let port = listener.local_addr().unwrap().port();
3782
3783        let config = SocketConfig {
3784            url: format!("127.0.0.1:{port}"),
3785            mode: Mode::Plain,
3786            suffix: b"\r\n".to_vec(),
3787            message_handler: None,
3788            heartbeat: None,
3789            connect_timeout_ms: Some(1_000),
3790            reconnect_delay_initial_ms: Some(50),
3791            reconnect_delay_max_ms: Some(100),
3792            reconnect_backoff_factor: Some(100.1),
3793            reconnect_jitter_ms: Some(0),
3794            connection_max_retries: Some(1),
3795            reconnect_max_attempts: None,
3796            heartbeat_timeout_secs: None,
3797            writer_capacity: None,
3798            certs_dir: None,
3799        };
3800
3801        let error = match SocketClientInner::connect_url(config, None).await {
3802            Ok(_) => panic!("invalid reconnect backoff should be rejected"),
3803            Err(e) => e,
3804        };
3805
3806        assert!(
3807            error.to_string().contains("factor"),
3808            "error should mention the invalid factor, was: {error}"
3809        );
3810        assert_eq!(
3811            listener.accept().unwrap_err().kind(),
3812            std::io::ErrorKind::WouldBlock,
3813            "invalid reconnect backoff must be rejected before connecting"
3814        );
3815    }
3816
3817    #[rstest]
3818    #[tokio::test]
3819    async fn test_reconnect_then_close() {
3820        // Bind an ephemeral port
3821        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3822        let port = listener.local_addr().unwrap().port();
3823
3824        // Server task: accept one connection and then drop it
3825        let server = task::spawn(async move {
3826            if let Ok((mut sock, _)) = listener.accept().await {
3827                drop(sock.shutdown());
3828            }
3829
3830            // Keep listener alive briefly to avoid premature exit
3831            sleep(Duration::from_secs(1)).await;
3832        });
3833
3834        // Configure client with a short reconnect backoff
3835        let config = SocketConfig {
3836            url: format!("127.0.0.1:{port}"),
3837            mode: Mode::Plain,
3838            suffix: b"\r\n".to_vec(),
3839            message_handler: None,
3840            heartbeat: None,
3841            connect_timeout_ms: Some(1_000),
3842            reconnect_delay_initial_ms: Some(50),
3843            reconnect_delay_max_ms: Some(100),
3844            reconnect_backoff_factor: Some(1.0),
3845            reconnect_jitter_ms: Some(0),
3846            connection_max_retries: Some(1),
3847            reconnect_max_attempts: None,
3848            heartbeat_timeout_secs: None,
3849            writer_capacity: None,
3850            certs_dir: None,
3851        };
3852
3853        // Connect client (handler=None)
3854        let client = SocketClient::builder()
3855            .config(config.clone())
3856            .connect()
3857            .await
3858            .unwrap();
3859
3860        // Wait for client to detect dropped connection and enter reconnect state
3861        wait_until_async(
3862            || async { client.is_reconnecting() },
3863            Duration::from_secs(2),
3864        )
3865        .await;
3866
3867        // Now close the client
3868        client.close().await;
3869        assert!(client.is_closed());
3870        server.abort();
3871    }
3872
3873    #[rstest]
3874    #[tokio::test]
3875    async fn test_reconnect_state_flips_when_reader_stops() {
3876        // Bind an ephemeral port and accept a single connection which we immediately close.
3877        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3878        let port = listener.local_addr().unwrap().port();
3879        let server = task::spawn(async move {
3880            if let Ok((sock, _)) = listener.accept().await {
3881                drop(sock);
3882            }
3883
3884            // Give the client a moment to observe the closed connection.
3885            sleep(Duration::from_millis(50)).await;
3886        });
3887
3888        let config = SocketConfig {
3889            url: format!("127.0.0.1:{port}"),
3890            mode: Mode::Plain,
3891            suffix: b"\r\n".to_vec(),
3892            message_handler: None,
3893            heartbeat: None,
3894            connect_timeout_ms: Some(1_000),
3895            reconnect_delay_initial_ms: Some(50),
3896            reconnect_delay_max_ms: Some(100),
3897            reconnect_backoff_factor: Some(1.0),
3898            reconnect_jitter_ms: Some(0),
3899            connection_max_retries: Some(1),
3900            reconnect_max_attempts: None,
3901            heartbeat_timeout_secs: None,
3902            writer_capacity: None,
3903            certs_dir: None,
3904        };
3905
3906        let client = SocketClient::builder()
3907            .config(config)
3908            .connect()
3909            .await
3910            .unwrap();
3911
3912        wait_until_async(
3913            || async { client.is_reconnecting() },
3914            Duration::from_secs(2),
3915        )
3916        .await;
3917
3918        client.close().await;
3919        server.abort();
3920    }
3921
3922    #[rstest]
3923    fn test_parse_socket_url_raw_address() {
3924        // Raw socket address with TLS mode
3925        let (socket_addr, request_url) =
3926            SocketClientInner::parse_socket_url("example.com:6130", Mode::Tls).unwrap();
3927        assert_eq!(socket_addr, "example.com:6130");
3928        assert_eq!(request_url, "wss://example.com:6130");
3929
3930        // Raw socket address with Plain mode
3931        let (socket_addr, request_url) =
3932            SocketClientInner::parse_socket_url("localhost:8080", Mode::Plain).unwrap();
3933        assert_eq!(socket_addr, "localhost:8080");
3934        assert_eq!(request_url, "ws://localhost:8080");
3935    }
3936
3937    #[rstest]
3938    fn test_parse_socket_url_with_scheme() {
3939        // Full URL with wss scheme
3940        let (socket_addr, request_url) =
3941            SocketClientInner::parse_socket_url("wss://example.com:443/path", Mode::Tls).unwrap();
3942        assert_eq!(socket_addr, "example.com:443");
3943        assert_eq!(request_url, "wss://example.com:443/path");
3944
3945        // Full URL with ws scheme
3946        let (socket_addr, request_url) =
3947            SocketClientInner::parse_socket_url("ws://localhost:8080", Mode::Plain).unwrap();
3948        assert_eq!(socket_addr, "localhost:8080");
3949        assert_eq!(request_url, "ws://localhost:8080");
3950    }
3951
3952    #[rstest]
3953    fn test_parse_socket_url_default_ports() {
3954        // wss without explicit port defaults to 443
3955        let (socket_addr, _) =
3956            SocketClientInner::parse_socket_url("wss://example.com", Mode::Tls).unwrap();
3957        assert_eq!(socket_addr, "example.com:443");
3958
3959        // ws without explicit port defaults to 80
3960        let (socket_addr, _) =
3961            SocketClientInner::parse_socket_url("ws://example.com", Mode::Plain).unwrap();
3962        assert_eq!(socket_addr, "example.com:80");
3963
3964        // https defaults to 443
3965        let (socket_addr, _) =
3966            SocketClientInner::parse_socket_url("https://example.com", Mode::Tls).unwrap();
3967        assert_eq!(socket_addr, "example.com:443");
3968
3969        // http defaults to 80
3970        let (socket_addr, _) =
3971            SocketClientInner::parse_socket_url("http://example.com", Mode::Plain).unwrap();
3972        assert_eq!(socket_addr, "example.com:80");
3973    }
3974
3975    #[rstest]
3976    fn test_parse_socket_url_unknown_scheme_uses_mode() {
3977        // Unknown scheme defaults to mode-based port
3978        let (socket_addr, _) =
3979            SocketClientInner::parse_socket_url("custom://example.com", Mode::Tls).unwrap();
3980        assert_eq!(socket_addr, "example.com:443");
3981
3982        let (socket_addr, _) =
3983            SocketClientInner::parse_socket_url("custom://example.com", Mode::Plain).unwrap();
3984        assert_eq!(socket_addr, "example.com:80");
3985    }
3986
3987    #[rstest]
3988    fn test_parse_socket_url_ipv6() {
3989        // IPv6 address with port
3990        let (socket_addr, request_url) =
3991            SocketClientInner::parse_socket_url("[::1]:8080", Mode::Plain).unwrap();
3992        assert_eq!(socket_addr, "[::1]:8080");
3993        assert_eq!(request_url, "ws://[::1]:8080");
3994
3995        // IPv6 in URL
3996        let (socket_addr, _) =
3997            SocketClientInner::parse_socket_url("ws://[::1]:8080", Mode::Plain).unwrap();
3998        assert_eq!(socket_addr, "[::1]:8080");
3999    }
4000
4001    #[rstest]
4002    #[tokio::test]
4003    async fn test_url_parsing_raw_socket_address() {
4004        // Test that raw socket addresses (host:port) work correctly
4005        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4006        let port = listener.local_addr().unwrap().port();
4007        let server = task::spawn(async move {
4008            if let Ok((sock, _)) = listener.accept().await {
4009                drop(sock);
4010            }
4011
4012            sleep(Duration::from_millis(50)).await;
4013        });
4014
4015        let config = SocketConfig {
4016            url: format!("127.0.0.1:{port}"), // Raw socket address format
4017            mode: Mode::Plain,
4018            suffix: b"\r\n".to_vec(),
4019            message_handler: None,
4020            heartbeat: None,
4021            connect_timeout_ms: Some(1_000),
4022            reconnect_delay_initial_ms: Some(50),
4023            reconnect_delay_max_ms: Some(100),
4024            reconnect_backoff_factor: Some(1.0),
4025            reconnect_jitter_ms: Some(0),
4026            connection_max_retries: Some(1),
4027            reconnect_max_attempts: None,
4028            heartbeat_timeout_secs: None,
4029            writer_capacity: None,
4030            certs_dir: None,
4031        };
4032
4033        // Should successfully connect with raw socket address
4034        let client = SocketClient::builder().config(config).connect().await;
4035        assert!(
4036            client.is_ok(),
4037            "Client should connect with raw socket address format"
4038        );
4039
4040        if let Ok(client) = client {
4041            client.close().await;
4042        }
4043
4044        server.abort();
4045    }
4046
4047    #[rstest]
4048    #[tokio::test]
4049    async fn test_url_parsing_with_scheme() {
4050        // Test that URLs with schemes also work
4051        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4052        let port = listener.local_addr().unwrap().port();
4053        let server = task::spawn(async move {
4054            if let Ok((sock, _)) = listener.accept().await {
4055                drop(sock);
4056            }
4057
4058            sleep(Duration::from_millis(50)).await;
4059        });
4060
4061        let config = SocketConfig {
4062            url: format!("ws://127.0.0.1:{port}"), // URL with scheme
4063            mode: Mode::Plain,
4064            suffix: b"\r\n".to_vec(),
4065            message_handler: None,
4066            heartbeat: None,
4067            connect_timeout_ms: Some(1_000),
4068            reconnect_delay_initial_ms: Some(50),
4069            reconnect_delay_max_ms: Some(100),
4070            reconnect_backoff_factor: Some(1.0),
4071            reconnect_jitter_ms: Some(0),
4072            connection_max_retries: Some(1),
4073            reconnect_max_attempts: None,
4074            heartbeat_timeout_secs: None,
4075            writer_capacity: None,
4076            certs_dir: None,
4077        };
4078
4079        // Should successfully connect with URL format
4080        let client = SocketClient::builder().config(config).connect().await;
4081        assert!(
4082            client.is_ok(),
4083            "Client should connect with URL scheme format"
4084        );
4085
4086        if let Ok(client) = client {
4087            client.close().await;
4088        }
4089
4090        server.abort();
4091    }
4092
4093    #[rstest]
4094    fn test_parse_socket_url_ipv6_with_zone() {
4095        // IPv6 with zone ID (link-local address)
4096        let (socket_addr, request_url) =
4097            SocketClientInner::parse_socket_url("[fe80::1%eth0]:8080", Mode::Plain).unwrap();
4098        assert_eq!(socket_addr, "[fe80::1%eth0]:8080");
4099        assert_eq!(request_url, "ws://[fe80::1%eth0]:8080");
4100
4101        // Verify zone is preserved in URL format too
4102        let (socket_addr, request_url) =
4103            SocketClientInner::parse_socket_url("ws://[fe80::1%lo]:9090", Mode::Plain).unwrap();
4104        assert_eq!(socket_addr, "[fe80::1%lo]:9090");
4105        assert_eq!(request_url, "ws://[fe80::1%lo]:9090");
4106    }
4107
4108    #[rstest]
4109    #[tokio::test]
4110    async fn test_ipv6_loopback_connection() {
4111        // Test IPv6 loopback address connection
4112        // Skip if IPv6 is not available on the system
4113        if TcpListener::bind("[::1]:0").await.is_err() {
4114            return;
4115        }
4116
4117        let listener = TcpListener::bind("[::1]:0").await.unwrap();
4118        let port = listener.local_addr().unwrap().port();
4119        let server = task::spawn(async move {
4120            if let Ok((mut sock, _)) = listener.accept().await {
4121                let mut buf = vec![0u8; 1024];
4122                if let Ok(n) = sock.read(&mut buf).await {
4123                    // Echo back
4124                    let _ = sock.write_all(&buf[..n]).await;
4125                }
4126            }
4127
4128            sleep(Duration::from_millis(50)).await;
4129        });
4130
4131        let config = SocketConfig {
4132            url: format!("[::1]:{port}"), // IPv6 loopback
4133            mode: Mode::Plain,
4134            suffix: b"\r\n".to_vec(),
4135            message_handler: None,
4136            heartbeat: None,
4137            connect_timeout_ms: Some(1_000),
4138            reconnect_delay_initial_ms: Some(50),
4139            reconnect_delay_max_ms: Some(100),
4140            reconnect_backoff_factor: Some(1.0),
4141            reconnect_jitter_ms: Some(0),
4142            connection_max_retries: Some(1),
4143            reconnect_max_attempts: None,
4144            heartbeat_timeout_secs: None,
4145            writer_capacity: None,
4146            certs_dir: None,
4147        };
4148
4149        let client = SocketClient::builder().config(config).connect().await;
4150        assert!(
4151            client.is_ok(),
4152            "Client should connect to IPv6 loopback address"
4153        );
4154
4155        if let Ok(client) = client {
4156            client.close().await;
4157        }
4158
4159        server.abort();
4160    }
4161
4162    #[rstest]
4163    #[tokio::test]
4164    async fn test_send_waits_during_reconnection() {
4165        // Test that send operations wait for reconnection to complete (up to configured timeout)
4166        use nautilus_common::testing::wait_until_async;
4167
4168        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4169        let port = listener.local_addr().unwrap().port();
4170
4171        let server = task::spawn(async move {
4172            // First connection - accept and immediately close
4173            if let Ok((sock, _)) = listener.accept().await {
4174                drop(sock);
4175            }
4176
4177            // Wait before accepting second connection
4178            sleep(Duration::from_millis(500)).await;
4179
4180            // Second connection - accept and keep alive
4181            if let Ok((mut sock, _)) = listener.accept().await {
4182                // Echo messages
4183                let mut buf = vec![0u8; 1024];
4184                while let Ok(n) = sock.read(&mut buf).await {
4185                    if n == 0 {
4186                        break;
4187                    }
4188
4189                    if sock.write_all(&buf[..n]).await.is_err() {
4190                        break;
4191                    }
4192                }
4193            }
4194        });
4195
4196        let config = SocketConfig {
4197            url: format!("127.0.0.1:{port}"),
4198            mode: Mode::Plain,
4199            suffix: b"\r\n".to_vec(),
4200            message_handler: None,
4201            heartbeat: None,
4202            connect_timeout_ms: Some(5_000), // 5s timeout - enough for reconnect
4203            reconnect_delay_initial_ms: Some(100),
4204            reconnect_delay_max_ms: Some(200),
4205            reconnect_backoff_factor: Some(1.0),
4206            reconnect_jitter_ms: Some(0),
4207            connection_max_retries: Some(1),
4208            reconnect_max_attempts: None,
4209            heartbeat_timeout_secs: None,
4210            writer_capacity: None,
4211            certs_dir: None,
4212        };
4213
4214        let client = SocketClient::builder()
4215            .config(config)
4216            .connect()
4217            .await
4218            .unwrap();
4219
4220        // Wait for reconnection to trigger
4221        wait_until_async(
4222            || async { client.is_reconnecting() },
4223            Duration::from_secs(2),
4224        )
4225        .await;
4226
4227        // Try to send while reconnecting - should wait and succeed after reconnect
4228        let send_result = tokio::time::timeout(
4229            Duration::from_secs(3),
4230            client.send_bytes(b"test_message".to_vec()),
4231        )
4232        .await;
4233
4234        assert!(
4235            send_result.is_ok() && send_result.unwrap().is_ok(),
4236            "Send should succeed after waiting for reconnection"
4237        );
4238
4239        client.close().await;
4240        server.abort();
4241    }
4242
4243    #[rstest]
4244    #[tokio::test]
4245    async fn test_send_bytes_timeout_uses_configured_connect_timeout() {
4246        // Test that send_bytes operations respect the configured connect_timeout.
4247        // When a client is stuck in RECONNECT longer than the timeout, sends should fail with Timeout.
4248        use nautilus_common::testing::wait_until_async;
4249
4250        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4251        let port = listener.local_addr().unwrap().port();
4252
4253        let server = task::spawn(async move {
4254            // Accept first connection and immediately close it
4255            if let Ok((sock, _)) = listener.accept().await {
4256                drop(sock);
4257            }
4258
4259            // Drop listener entirely so reconnection fails completely
4260            drop(listener);
4261            sleep(Duration::from_mins(1)).await;
4262        });
4263
4264        let config = SocketConfig {
4265            url: format!("127.0.0.1:{port}"),
4266            mode: Mode::Plain,
4267            suffix: b"\r\n".to_vec(),
4268            message_handler: None,
4269            heartbeat: None,
4270            connect_timeout_ms: Some(1_000), // 1s timeout for faster test
4271            reconnect_delay_initial_ms: Some(200), // Short backoff (but > timeout) to keep client in RECONNECT
4272            reconnect_delay_max_ms: Some(200),
4273            reconnect_backoff_factor: Some(1.0),
4274            reconnect_jitter_ms: Some(0),
4275            connection_max_retries: Some(1),
4276            reconnect_max_attempts: None,
4277            heartbeat_timeout_secs: None,
4278            writer_capacity: None,
4279            certs_dir: None,
4280        };
4281
4282        let client = SocketClient::builder()
4283            .config(config)
4284            .connect()
4285            .await
4286            .unwrap();
4287
4288        // Wait for client to enter RECONNECT state
4289        wait_until_async(
4290            || async { client.is_reconnecting() },
4291            Duration::from_secs(3),
4292        )
4293        .await;
4294
4295        // Attempt send while stuck in RECONNECT - should timeout after 1s (configured timeout)
4296        // The client will try to reconnect for 1s, fail, then wait 5s backoff before next attempt
4297        let start = std::time::Instant::now();
4298        let send_result = client.send_bytes(b"test".to_vec()).await;
4299        let elapsed = start.elapsed();
4300
4301        assert!(
4302            send_result.is_err(),
4303            "Send should fail when client stuck in RECONNECT, was: {send_result:?}"
4304        );
4305        assert!(
4306            matches!(send_result, Err(crate::error::SendError::Timeout)),
4307            "Send should return Timeout error, was: {send_result:?}"
4308        );
4309        // Verify timeout respects configured value (1s), but don't check upper bound
4310        // as CI scheduler jitter can cause legitimate delays beyond the timeout
4311        assert!(
4312            elapsed >= Duration::from_millis(900),
4313            "Send should timeout after at least 1s (configured timeout), took {elapsed:?}"
4314        );
4315
4316        client.close().await;
4317        server.abort();
4318    }
4319
4320    #[rstest]
4321    #[tokio::test]
4322    async fn test_heartbeat_timeout_triggers_reconnect() {
4323        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4324        let port = listener.local_addr().unwrap().port();
4325
4326        // Server accepts connection but sends nothing (simulates silent death)
4327        let server = task::spawn(async move {
4328            let (_sock1, _) = listener.accept().await.unwrap();
4329            // Hold connection open but send nothing, wait for reconnect attempt
4330            sleep(Duration::from_secs(5)).await;
4331        });
4332
4333        let config = SocketConfig {
4334            url: format!("127.0.0.1:{port}"),
4335            mode: Mode::Plain,
4336            suffix: b"\r\n".to_vec(),
4337            message_handler: None,
4338            heartbeat: None,
4339            connect_timeout_ms: Some(2_000),
4340            reconnect_delay_initial_ms: Some(50),
4341            reconnect_delay_max_ms: Some(100),
4342            reconnect_backoff_factor: Some(1.0),
4343            reconnect_jitter_ms: Some(0),
4344            connection_max_retries: Some(1),
4345            reconnect_max_attempts: Some(1),
4346            heartbeat_timeout_secs: Some(1),
4347            writer_capacity: None,
4348            certs_dir: None,
4349        };
4350
4351        let client = SocketClient::builder()
4352            .config(config)
4353            .connect()
4354            .await
4355            .unwrap();
4356
4357        assert!(client.is_active());
4358
4359        // Wait for the dead-peer timeout to fire and the client to enter reconnect
4360        wait_until_async(
4361            || async { client.is_reconnecting() || client.is_closed() },
4362            Duration::from_secs(4),
4363        )
4364        .await;
4365
4366        assert!(
4367            !client.is_active(),
4368            "Client should not be active after the dead-peer timeout"
4369        );
4370
4371        client.close().await;
4372        server.abort();
4373    }
4374
4375    #[rstest]
4376    #[tokio::test]
4377    async fn test_heartbeat_timeout_resets_on_inbound_bytes() {
4378        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4379        let port = listener.local_addr().unwrap().port();
4380
4381        // Server sends data every 200ms (well within the 1s idle timeout)
4382        let server = task::spawn(async move {
4383            let (mut sock, _) = listener.accept().await.unwrap();
4384
4385            for _ in 0..10 {
4386                sleep(Duration::from_millis(200)).await;
4387
4388                if sock.write_all(b"ping\r\n").await.is_err() {
4389                    break;
4390                }
4391            }
4392        });
4393
4394        let config = SocketConfig {
4395            url: format!("127.0.0.1:{port}"),
4396            mode: Mode::Plain,
4397            suffix: b"\r\n".to_vec(),
4398            message_handler: None,
4399            heartbeat: None,
4400            connect_timeout_ms: Some(2_000),
4401            reconnect_delay_initial_ms: Some(50),
4402            reconnect_delay_max_ms: Some(100),
4403            reconnect_backoff_factor: Some(1.0),
4404            reconnect_jitter_ms: Some(0),
4405            connection_max_retries: Some(1),
4406            reconnect_max_attempts: Some(1),
4407            heartbeat_timeout_secs: Some(1),
4408            writer_capacity: None,
4409            certs_dir: None,
4410        };
4411
4412        let client = SocketClient::builder()
4413            .config(config)
4414            .connect()
4415            .await
4416            .unwrap();
4417
4418        assert!(client.is_active());
4419
4420        // Wait 1.5s - data arrives every 200ms so idle timeout (1s) should NOT fire
4421        sleep(Duration::from_millis(1_500)).await;
4422
4423        assert!(
4424            client.is_active(),
4425            "Client should remain active when data is flowing"
4426        );
4427
4428        client.close().await;
4429        server.abort();
4430    }
4431
4432    #[rstest]
4433    #[tokio::test]
4434    async fn test_close_during_backoff_exits_promptly() {
4435        // Verify that close() interrupts backoff sleep (Finding 1).
4436        // Server accepts then drops, no second listener -> reconnect fails -> enters backoff.
4437        // We close while backing off and assert the client shuts down quickly.
4438        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4439        let port = listener.local_addr().unwrap().port();
4440
4441        let server = task::spawn(async move {
4442            // Accept first connection, close immediately
4443            if let Ok((mut sock, _)) = listener.accept().await {
4444                drop(sock.shutdown());
4445            }
4446
4447            // Don't accept again so reconnect fails and enters backoff
4448            sleep(Duration::from_mins(1)).await;
4449        });
4450
4451        let config = SocketConfig {
4452            url: format!("127.0.0.1:{port}"),
4453            mode: Mode::Plain,
4454            suffix: b"\r\n".to_vec(),
4455            message_handler: None,
4456            heartbeat: None,
4457            connect_timeout_ms: Some(1_000),
4458            reconnect_delay_initial_ms: Some(10_000), // 10s backoff to ensure we're sleeping
4459            reconnect_delay_max_ms: Some(10_000),
4460            reconnect_backoff_factor: Some(1.0),
4461            reconnect_jitter_ms: Some(0),
4462            connection_max_retries: None,
4463            reconnect_max_attempts: None,
4464            heartbeat_timeout_secs: None,
4465            writer_capacity: None,
4466            certs_dir: None,
4467        };
4468
4469        let client = SocketClient::builder()
4470            .config(config)
4471            .connect()
4472            .await
4473            .unwrap();
4474
4475        // Wait for client to enter reconnect
4476        wait_until_async(
4477            || async { client.is_reconnecting() },
4478            Duration::from_secs(3),
4479        )
4480        .await;
4481
4482        // Wait for the reconnect attempt to fail and enter backoff sleep
4483        sleep(Duration::from_millis(1_500)).await;
4484
4485        // Close while backing off
4486        let start = std::time::Instant::now();
4487        client.close().await;
4488        let elapsed = start.elapsed();
4489
4490        assert!(client.is_closed(), "Client should be closed");
4491        // Should exit well before the 10s backoff sleep completes
4492        assert!(
4493            elapsed < Duration::from_secs(2),
4494            "Close should interrupt backoff sleep, took {elapsed:?}"
4495        );
4496
4497        server.abort();
4498    }
4499
4500    #[rstest]
4501    #[tokio::test]
4502    async fn test_zero_heartbeat_timeout_rejected() {
4503        let config = SocketConfig {
4504            url: "127.0.0.1:9999".to_string(),
4505            mode: Mode::Plain,
4506            suffix: b"\r\n".to_vec(),
4507            message_handler: None,
4508            heartbeat: None,
4509            connect_timeout_ms: None,
4510            reconnect_delay_initial_ms: None,
4511            reconnect_delay_max_ms: None,
4512            reconnect_backoff_factor: None,
4513            reconnect_jitter_ms: None,
4514            reconnect_max_attempts: None,
4515            connection_max_retries: Some(1),
4516            heartbeat_timeout_secs: Some(0),
4517            writer_capacity: None,
4518            certs_dir: None,
4519        };
4520
4521        let result = SocketClient::builder().config(config).connect().await;
4522
4523        assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
4524        let err_msg = result.unwrap_err().to_string();
4525        assert!(
4526            err_msg.contains("heartbeat_timeout_secs"),
4527            "Error should name the offending field, was: {err_msg}"
4528        );
4529    }
4530
4531    #[rstest]
4532    #[tokio::test]
4533    async fn test_empty_suffix_rejected() {
4534        let config = SocketConfig {
4535            url: "127.0.0.1:9999".to_string(),
4536            mode: Mode::Plain,
4537            suffix: vec![],
4538            message_handler: None,
4539            heartbeat: None,
4540            connect_timeout_ms: None,
4541            reconnect_delay_initial_ms: None,
4542            reconnect_delay_max_ms: None,
4543            reconnect_backoff_factor: None,
4544            reconnect_jitter_ms: None,
4545            reconnect_max_attempts: None,
4546            connection_max_retries: Some(1),
4547            heartbeat_timeout_secs: None,
4548            writer_capacity: None,
4549            certs_dir: None,
4550        };
4551
4552        let result = SocketClient::builder().config(config).connect().await;
4553
4554        assert!(
4555            result.is_err(),
4556            "Empty suffix should cause connection to fail"
4557        );
4558        let err_msg = result.unwrap_err().to_string();
4559        assert!(
4560            err_msg.contains("suffix cannot be empty"),
4561            "Error should mention empty suffix, was: {err_msg}"
4562        );
4563    }
4564}
4565
4566#[cfg(test)]
4567mod reconnect_request_tests {
4568    use std::sync::{Arc, atomic::AtomicU8};
4569
4570    use parking_lot::Mutex;
4571    use rstest::rstest;
4572
4573    use super::*;
4574    use crate::SocketState;
4575
4576    fn handle(
4577        mode: ConnectionMode,
4578    ) -> (
4579        SocketReconnectHandle,
4580        Arc<tokio::sync::Notify>,
4581        Arc<Mutex<Vec<SocketState>>>,
4582    ) {
4583        let controller_notify = Arc::new(tokio::sync::Notify::new());
4584        let states = Arc::new(Mutex::new(Vec::new()));
4585        let states_callback = Arc::clone(&states);
4586        let state_sink = SocketStateSink::new(move |state| {
4587            states_callback.lock().push(state);
4588        });
4589
4590        let handle = SocketReconnectHandle {
4591            connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
4592            state_sink: Some(state_sink),
4593            controller_lifecycle: Arc::new(ControllerLifecycle::new()),
4594            controller_notify: Arc::clone(&controller_notify),
4595        };
4596
4597        (handle, controller_notify, states)
4598    }
4599
4600    #[rstest]
4601    #[tokio::test]
4602    async fn accepted_request_reports_loss_and_wakes_controller_once() {
4603        let (handle, controller_notify, states) = handle(ConnectionMode::Active);
4604
4605        assert_eq!(
4606            handle.request_reconnect(),
4607            ReconnectRequestOutcome::Accepted
4608        );
4609        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
4610        tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4611            .await
4612            .expect("accepted request should notify controller");
4613
4614        assert_eq!(
4615            handle.request_reconnect(),
4616            ReconnectRequestOutcome::AlreadyReconnecting
4617        );
4618        assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
4619        assert!(
4620            tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4621                .await
4622                .is_err(),
4623            "duplicate request should not notify controller",
4624        );
4625    }
4626
4627    #[rstest]
4628    #[case(
4629        ConnectionMode::Reconnect,
4630        ReconnectRequestOutcome::AlreadyReconnecting
4631    )]
4632    #[case(ConnectionMode::Disconnect, ReconnectRequestOutcome::Disconnected)]
4633    #[case(ConnectionMode::Closed, ReconnectRequestOutcome::Closed)]
4634    #[tokio::test]
4635    async fn rejected_request_preserves_state_and_does_not_wake_controller(
4636        #[case] mode: ConnectionMode,
4637        #[case] expected: ReconnectRequestOutcome,
4638    ) {
4639        let (handle, controller_notify, states) = handle(mode);
4640
4641        assert_eq!(handle.request_reconnect(), expected);
4642        assert!(states.lock().is_empty());
4643        assert!(
4644            tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4645                .await
4646                .is_err(),
4647            "rejected request should not notify controller",
4648        );
4649    }
4650}