1use std::{
42 collections::VecDeque,
43 fmt::Debug,
44 path::Path,
45 pin::pin,
46 sync::{
47 Arc,
48 atomic::{AtomicU8, Ordering},
49 },
50 time::Duration,
51};
52
53use bytes::Bytes;
54use nautilus_core::string::secret::REDACTED;
55use nautilus_cryptography::providers::install_cryptographic_provider;
56use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
57use tokio_tungstenite::tungstenite::{Error, client::IntoClientRequest, stream::Mode};
58
59use super::{SocketConfig, TcpMessageHandler, TcpReader, TcpWriter, WriterCommand};
60use crate::{
61 SocketStateSink,
62 backoff::{
63 ExponentialBackoff, RECONNECT_STABILITY_THRESHOLD, ReconnectThrottle, wait_reconnect_delay,
64 },
65 dst,
66 error::{SendError, is_connection_drop_io_error},
67 logging::{log_task_aborted, log_task_started, log_task_stopped},
68 mode::{
69 ConnectionMode, ControllerLifecycle, ReadSessionFence, ReconnectOutcome,
70 ReconnectRequestOutcome,
71 },
72 net::TcpStream,
73 tls::{create_tls_config_from_certs_dir, tcp_tls},
74 writer::{self, DEFAULT_WRITER_CAPACITY, WriterReceiver, WriterSender},
75};
76
77const CONNECTION_STATE_CHECK_INTERVAL_MS: u64 = 10;
79const GRACEFUL_SHUTDOWN_DELAY_MS: u64 = 100;
80const GRACEFUL_SHUTDOWN_TIMEOUT_SECS: u64 = 5;
81const WRITE_TIMEOUT_SECS: u64 = 5;
82
83const MAX_READ_BUFFER_BYTES: usize = 10 * 1024 * 1024;
85
86struct BufferedWrite {
87 data: Bytes,
88 replay_key: Option<u64>,
89 _permit: tokio::sync::OwnedSemaphorePermit,
90}
91
92pub type SocketReconnectReplay = Arc<dyn Fn() -> Vec<Bytes> + Send + Sync>;
94
95struct SocketClientInner {
96 config: SocketConfig,
97 connector: Option<Arc<rustls::ClientConfig>>,
98 read_task: tokio::task::JoinHandle<()>,
99 read_fence: ReadSessionFence,
100 write_task: tokio::task::JoinHandle<()>,
101 writer_tx: WriterSender<WriterCommand>,
102 heartbeat_task: Option<tokio::task::JoinHandle<()>>,
103 connection_mode: Arc<AtomicU8>,
104 state_notify: Arc<tokio::sync::Notify>,
105 connect_timeout: Duration,
106 backoff: ExponentialBackoff,
107 reconnect_throttle: ReconnectThrottle,
108 reconnect_max_attempts: Option<u32>,
109 reconnect_attempt_count: u32,
110 state_sink: Option<SocketStateSink>,
111}
112
113impl SocketClientInner {
114 async fn connect_url(
120 config: SocketConfig,
121 state_sink: Option<SocketStateSink>,
122 ) -> anyhow::Result<Self> {
123 install_cryptographic_provider();
124
125 if config.suffix.is_empty() {
127 anyhow::bail!("Socket suffix cannot be empty: suffix is required for message framing");
128 }
129
130 config.validate()?;
133
134 let connect_timeout = Duration::from_millis(config.connect_timeout_ms.unwrap_or(10_000));
135
136 let reconnect_backoff = ExponentialBackoff::new(
137 Duration::from_millis(config.reconnect_delay_initial_ms.unwrap_or(2_000)),
138 Duration::from_millis(config.reconnect_delay_max_ms.unwrap_or(30_000)),
139 config.reconnect_backoff_factor.unwrap_or(1.5),
140 config.reconnect_jitter_ms.unwrap_or(100),
141 true, )?;
143
144 let connector = if let Some(dir) = &config.certs_dir {
145 let config = create_tls_config_from_certs_dir(Path::new(dir), false)?;
146 Some(Arc::new(config))
147 } else {
148 None
149 };
150
151 let max_retries = config.connection_max_retries.unwrap_or(5);
153
154 let mut backoff = ExponentialBackoff::new(
155 Duration::from_millis(500),
156 Duration::from_secs(5),
157 2.0,
158 250,
159 false,
160 )?;
161
162 let mut attempt = 0;
163 let (reader, writer) = loop {
164 attempt += 1;
165
166 let last_error = match dst::time::timeout(
167 connect_timeout,
168 Self::tls_connect_with_server(&config.url, config.mode, connector.clone()),
169 )
170 .await
171 {
172 Ok(Ok(result)) => {
173 if attempt > 1 {
174 log::info!("Socket connection established after {attempt} attempts");
175 }
176
177 break result;
178 }
179 Ok(Err(e)) => {
180 let error = e.to_string();
181 log::warn!(
182 "Socket connection attempt {attempt}/{max_retries} to {REDACTED} failed: {error}"
183 );
184 error
185 }
186 Err(_) => {
187 let error = format!(
188 "Connection timeout after {:.1}s (possible DNS resolution failure)",
189 connect_timeout.as_secs_f64()
190 );
191 log::warn!(
192 "Socket connection attempt {attempt}/{max_retries} to {REDACTED} timed out"
193 );
194 error
195 }
196 };
197
198 if attempt >= max_retries {
199 anyhow::bail!(
200 "Failed to connect to {REDACTED} after {max_retries} attempts: {last_error}. \
201 If this is a DNS error, check your network configuration and DNS settings."
202 );
203 }
204
205 let delay = backoff.next_duration();
206 log::debug!(
207 "Retrying in {delay:?} (attempt {}/{})",
208 attempt + 1,
209 max_retries
210 );
211 dst::time::sleep(delay).await;
212 };
213
214 log::debug!("Connected");
215
216 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
217 let state_notify = Arc::new(tokio::sync::Notify::new());
218 let outcome =
219 ConnectionMode::complete_reconnect_with_sink(&connection_mode, state_sink.as_ref());
220 debug_assert_eq!(outcome, ReconnectOutcome::Reconnected);
221 let read_fence = ReadSessionFence::new();
222
223 let read_task = Self::spawn_read_task(
224 Arc::clone(&connection_mode),
225 read_fence.clone(),
226 reader,
227 config.message_handler.clone(),
228 config.suffix.clone(),
229 config.resolved_heartbeat_timeout(),
230 );
231
232 let (writer_tx, writer_rx) = writer::channel::<WriterCommand>(
233 config.writer_capacity.unwrap_or(DEFAULT_WRITER_CAPACITY),
234 );
235
236 let write_task = Self::spawn_write_task(
237 Arc::clone(&connection_mode),
238 Arc::clone(&state_notify),
239 writer,
240 writer_rx,
241 config.suffix.clone(),
242 state_sink.clone(),
243 );
244
245 let heartbeat_task = config.heartbeat.as_ref().map(|heartbeat| {
246 Self::spawn_heartbeat_task(
247 Arc::clone(&connection_mode),
248 heartbeat.interval_secs,
249 heartbeat.payload.clone(),
250 writer_tx.clone(),
251 )
252 });
253
254 let reconnect_max_attempts = config.reconnect_max_attempts;
255
256 Ok(Self {
257 config,
258 connector,
259 read_task,
260 read_fence,
261 write_task,
262 writer_tx,
263 heartbeat_task,
264 connection_mode,
265 state_notify,
266 connect_timeout,
267 backoff: reconnect_backoff,
268 reconnect_throttle: ReconnectThrottle::default(),
269 reconnect_max_attempts,
270 reconnect_attempt_count: 0,
271 state_sink,
272 })
273 }
274
275 fn parse_socket_url(url: &str, mode: Mode) -> Result<(String, String), Error> {
285 if url.contains("://") {
286 let parsed = url.parse::<http::Uri>().map_err(|e| {
288 Error::Io(std::io::Error::new(
289 std::io::ErrorKind::InvalidInput,
290 format!("Invalid URL: {e}"),
291 ))
292 })?;
293
294 let host = parsed.host().ok_or_else(|| {
295 Error::Io(std::io::Error::new(
296 std::io::ErrorKind::InvalidInput,
297 "URL missing host",
298 ))
299 })?;
300
301 let port = parsed
302 .port_u16()
303 .unwrap_or_else(|| match parsed.scheme_str() {
304 Some("wss" | "https") => 443,
305 Some("ws" | "http") => 80,
306 _ => match mode {
307 Mode::Tls => 443,
308 Mode::Plain => 80,
309 },
310 });
311
312 Ok((format!("{host}:{port}"), url.to_string()))
313 } else {
314 let scheme = match mode {
317 Mode::Tls => "wss",
318 Mode::Plain => "ws",
319 };
320
321 Ok((url.to_string(), format!("{scheme}://{url}")))
322 }
323 }
324
325 pub(crate) async fn tls_connect_with_server(
335 url: &str,
336 mode: Mode,
337 connector: Option<Arc<rustls::ClientConfig>>,
338 ) -> Result<(TcpReader, TcpWriter), Error> {
339 log::debug!("Connecting to {REDACTED}");
340
341 let (socket_addr, request_url) = Self::parse_socket_url(url, mode)?;
342 let tcp_result = TcpStream::connect(&socket_addr).await;
343
344 match tcp_result {
345 Ok(stream) => {
346 log::debug!("TCP connection established to {socket_addr}, proceeding with TLS");
347
348 crate::net::apply_socket_options(&stream);
349
350 let request = request_url.into_client_request()?;
351 tcp_tls(&request, mode, stream, connector)
352 .await
353 .map(tokio::io::split)
354 }
355 Err(e) => {
356 log::warn!("TCP connection failed to {socket_addr}: {e:?}");
357 Err(Error::Io(e))
358 }
359 }
360 }
361
362 async fn reconnect(
373 &mut self,
374 reconnect_replay: Option<&SocketReconnectReplay>,
375 ) -> Result<ReconnectOutcome, Error> {
376 log::info!("Reconnecting");
377
378 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
379 log::debug!("Reconnect aborted due to disconnect state");
380 return Ok(ReconnectOutcome::Aborted);
381 }
382
383 let (reader, new_writer) = dst::time::timeout(
385 self.connect_timeout,
386 Self::tls_connect_with_server(
387 &self.config.url,
388 self.config.mode,
389 self.connector.clone(),
390 ),
391 )
392 .await
393 .map_err(|_| {
394 Error::Io(std::io::Error::new(
395 std::io::ErrorKind::TimedOut,
396 format!(
397 "reconnection timed out after {}s",
398 self.connect_timeout.as_secs_f64()
399 ),
400 ))
401 })??;
402
403 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
404 log::debug!("Reconnect aborted mid-flight (after connect)");
405 return Ok(ReconnectOutcome::Aborted);
406 }
407
408 log::debug!("Connected");
409
410 let (tx, rx) = tokio::sync::oneshot::channel();
414 let command = if let Some(reconnect_replay) = reconnect_replay {
415 WriterCommand::UpdateWithReplay(new_writer, reconnect_replay(), tx)
416 } else {
417 WriterCommand::Update(new_writer, tx)
418 };
419
420 if let Err(e) = self.writer_tx.send_update(command) {
421 log::error!("{e}");
422 return Err(Error::Io(std::io::Error::new(
423 std::io::ErrorKind::BrokenPipe,
424 format!("Failed to send update command: {e}"),
425 )));
426 }
427
428 match rx.await {
430 Ok(true) => log::debug!("Writer confirmed buffer drain success"),
431 Ok(false) => {
432 log::warn!("Writer failed to drain buffer, aborting reconnect");
433 return Err(Error::Io(std::io::Error::other(
435 "Failed to drain reconnection buffer",
436 )));
437 }
438 Err(e) => {
439 log::error!("Writer dropped update channel: {e}");
440 return Err(Error::Io(std::io::Error::new(
441 std::io::ErrorKind::BrokenPipe,
442 "Writer task dropped response channel",
443 )));
444 }
445 }
446
447 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
449
450 if ConnectionMode::from_atomic(&self.connection_mode).is_disconnect() {
451 log::debug!("Reconnect aborted mid-flight (after delay)");
452 return Ok(ReconnectOutcome::Aborted);
453 }
454
455 self.read_fence.invalidate();
456
457 if !self.read_task.is_finished() {
458 self.read_task.abort();
459 log_task_aborted("read");
460 }
461
462 if ConnectionMode::complete_reconnect_with_sink(
465 &self.connection_mode,
466 self.state_sink.as_ref(),
467 ) == ReconnectOutcome::Aborted
468 {
469 log::debug!("Reconnect aborted (state changed during reconnect)");
470 return Ok(ReconnectOutcome::Aborted);
471 }
472
473 self.read_fence = ReadSessionFence::new();
475 self.read_task = Self::spawn_read_task(
476 Arc::clone(&self.connection_mode),
477 self.read_fence.clone(),
478 reader,
479 self.config.message_handler.clone(),
480 self.config.suffix.clone(),
481 self.config.resolved_heartbeat_timeout(),
482 );
483
484 log::info!("Reconnect succeeded");
485 Ok(ReconnectOutcome::Reconnected)
486 }
487
488 #[inline]
494 #[must_use]
495 pub(crate) fn is_alive(&self) -> bool {
496 !self.read_task.is_finished() && !self.write_task.is_finished()
497 }
498
499 #[must_use]
500 fn spawn_read_task<R>(
501 connection_state: Arc<AtomicU8>,
502 read_fence: ReadSessionFence,
503 reader: R,
504 handler: Option<TcpMessageHandler>,
505 suffix: Vec<u8>,
506 heartbeat_timeout_secs: Option<u64>,
507 ) -> tokio::task::JoinHandle<()>
508 where
509 R: AsyncRead + Unpin + Send + 'static,
510 {
511 log_task_started("read");
512
513 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
515 let heartbeat_timeout = heartbeat_timeout_secs.map(Duration::from_secs);
516
517 tokio::task::spawn(Self::run_read_loop(
518 connection_state,
519 read_fence,
520 reader,
521 handler,
522 suffix,
523 heartbeat_timeout,
524 check_interval,
525 ))
526 }
527
528 async fn run_read_loop<R>(
529 connection_state: Arc<AtomicU8>,
530 read_fence: ReadSessionFence,
531 mut reader: R,
532 handler: Option<TcpMessageHandler>,
533 suffix: Vec<u8>,
534 heartbeat_timeout: Option<Duration>,
535 check_interval: Duration,
536 ) where
537 R: AsyncRead + Unpin,
538 {
539 let mut buf = Vec::new();
540 let mut scanned = 0;
543 let mut last_data_time = dst::time::Instant::now();
544
545 'read: loop {
546 if !ConnectionMode::from_atomic(&connection_state).is_active() || !read_fence.is_valid()
547 {
548 if !buf.is_empty() {
549 log::debug!(
550 "Dropping {} buffered bytes after socket session ended",
551 buf.len()
552 );
553 buf.clear();
554 }
555
556 break;
557 }
558
559 match dst::time::timeout(check_interval, reader.read_buf(&mut buf)).await {
560 Ok(Ok(0)) => {
562 log::debug!("Connection closed by server");
563 break;
564 }
565 Ok(Err(e)) => {
566 log::debug!("Connection ended: {e}");
567 break;
568 }
569 Ok(Ok(bytes)) => {
571 log::trace!("Received <binary> {bytes} bytes");
572 last_data_time = dst::time::Instant::now();
573
574 if !ConnectionMode::from_atomic(&connection_state).is_active()
575 || !read_fence.is_valid()
576 {
577 log::debug!(
578 "Dropping {} buffered bytes after socket session ended",
579 buf.len()
580 );
581 buf.clear();
582 break;
583 }
584
585 while let Some(i) = buf[scanned..]
586 .windows(suffix.len())
587 .position(|window| window == suffix)
588 {
589 let end = scanned + i + suffix.len();
590 let mut data: Vec<u8> = buf.drain(0..end).collect();
591 data.truncate(data.len() - suffix.len());
592 scanned = 0;
593
594 if let Some(ref handler) = handler {
595 if !ConnectionMode::from_atomic(&connection_state).is_active()
596 || !read_fence.is_valid()
597 {
598 log::debug!(
599 "Dropping {} buffered bytes after socket session ended",
600 data.len() + buf.len()
601 );
602 buf.clear();
603 break 'read;
604 }
605
606 handler(&data);
607 }
608 }
609
610 scanned = buf.len().saturating_sub(suffix.len() - 1);
612
613 if buf.len() > MAX_READ_BUFFER_BYTES {
614 log::error!(
615 "Read buffer exceeded maximum size ({MAX_READ_BUFFER_BYTES} bytes), closing connection"
616 );
617 break;
618 }
619 }
620 Err(_) => {
621 if let Some(timeout) = heartbeat_timeout {
622 let silent_for = last_data_time.elapsed();
623 if silent_for >= timeout {
624 log::warn!(
625 "Heartbeat timeout: no bytes received for {:.1}s",
626 silent_for.as_secs_f64()
627 );
628 break;
629 }
630 }
631 }
632 }
633 }
634
635 log_task_stopped("read");
636 }
637
638 async fn drain_reconnect_buffer<W>(
648 buffer: &mut VecDeque<BufferedWrite>,
649 writer: &mut W,
650 suffix: &[u8],
651 replay: &[Bytes],
652 ) -> bool
653 where
654 W: AsyncWrite + Unpin,
655 {
656 if buffer.is_empty() {
657 return false;
658 }
659
660 let initial_buffer_len = buffer.len();
661 log::info!("Sending {initial_buffer_len} buffered messages after reconnection");
662
663 while let Some(buffered) = buffer.front() {
664 if buffered.replay_key.is_some()
665 && replay.iter().any(|replayed| replayed == &buffered.data)
666 {
667 buffer.pop_front();
668 continue;
669 }
670
671 let mut combined_msg = Vec::with_capacity(buffered.data.len() + suffix.len());
672 combined_msg.extend_from_slice(&buffered.data);
673 combined_msg.extend_from_slice(suffix);
674
675 if let Err(e) = writer.write_all(&combined_msg).await {
676 if is_connection_drop_io_error(&e) {
677 log::warn!(
678 "Failed to send buffered message with suffix after reconnection: {e}, {} messages remain in buffer",
679 buffer.len()
680 );
681 } else {
682 log::error!(
683 "Failed to send buffered message with suffix after reconnection: {e}, {} messages remain in buffer",
684 buffer.len()
685 );
686 }
687
688 return true;
689 }
690
691 buffer.pop_front();
692 }
693
694 log::info!("Successfully sent all {initial_buffer_len} buffered messages");
695
696 false
697 }
698
699 async fn replace_writer<W>(
700 active_writer: &mut W,
701 new_writer: W,
702 replay: Vec<Bytes>,
703 reconnect_buffer: &mut VecDeque<BufferedWrite>,
704 suffix: &[u8],
705 ) -> bool
706 where
707 W: AsyncWrite + Unpin,
708 {
709 log::debug!("Received new writer");
710 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
711
712 _ = dst::time::timeout(
713 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
714 active_writer.shutdown(),
715 )
716 .await;
717
718 *active_writer = new_writer;
719 log::debug!("Updated writer");
720
721 let drain_result =
722 dst::time::timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS), async {
723 for replay_msg in &replay {
724 let mut framed = Vec::with_capacity(replay_msg.len() + suffix.len());
725 framed.extend_from_slice(replay_msg);
726 framed.extend_from_slice(suffix);
727 if let Err(e) = active_writer.write_all(&framed).await {
728 log::warn!("Failed to send reconnect replay: {e}");
729 return true;
730 }
731 }
732
733 Self::drain_reconnect_buffer(reconnect_buffer, active_writer, suffix, &replay).await
734 })
735 .await;
736
737 let send_error = drain_result.unwrap_or_else(|_| {
738 log::warn!(
739 "Timed out sending reconnect replay and buffered messages, {} buffered messages remain",
740 reconnect_buffer.len()
741 );
742 true
743 });
744
745 !send_error
746 }
747
748 fn spawn_write_task<W>(
749 connection_state: Arc<AtomicU8>,
750 state_notify: Arc<tokio::sync::Notify>,
751 writer: W,
752 mut writer_rx: WriterReceiver<WriterCommand<W>>,
753 suffix: Vec<u8>,
754 state_sink: Option<SocketStateSink>,
755 ) -> tokio::task::JoinHandle<()>
756 where
757 W: AsyncWrite + Unpin + Send + 'static,
758 {
759 log_task_started("write");
760
761 let check_interval = Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS);
763
764 tokio::task::spawn(async move {
765 let mut active_writer = writer;
766 let mut reconnect_buffer: VecDeque<BufferedWrite> = VecDeque::new();
767 let mut write_buf: Vec<u8> = Vec::new();
768
769 loop {
770 let mode = ConnectionMode::from_atomic(&connection_state);
771
772 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
773 break;
774 }
775
776 if mode.is_active() && !reconnect_buffer.is_empty() {
777 let drain_result = dst::time::timeout(
778 Duration::from_secs(WRITE_TIMEOUT_SECS),
779 Self::drain_reconnect_buffer(
780 &mut reconnect_buffer,
781 &mut active_writer,
782 &suffix,
783 &[],
784 ),
785 )
786 .await;
787
788 let send_error = drain_result.unwrap_or_else(|_| {
789 log::warn!(
790 "Timed out draining reconnect buffer after {WRITE_TIMEOUT_SECS}s, {} messages remain",
791 reconnect_buffer.len()
792 );
793 true
794 });
795
796 if send_error
797 && ConnectionMode::request_reconnect_with_sink(
798 &connection_state,
799 state_sink.as_ref(),
800 )
801 {
802 log::warn!("Writer triggering reconnect");
803 state_notify.notify_one();
804 }
805
806 continue;
807 }
808
809 match dst::time::timeout(check_interval, writer_rx.recv()).await {
810 Ok(Some((msg, permit))) => {
811 let mode = ConnectionMode::from_atomic(&connection_state);
813
814 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
815 break;
816 }
817
818 let must_buffer = mode.is_reconnect() || !reconnect_buffer.is_empty();
820
821 match msg {
822 WriterCommand::Update(new_writer, tx) => {
823 let sent = Self::replace_writer(
824 &mut active_writer,
825 new_writer,
826 Vec::new(),
827 &mut reconnect_buffer,
828 &suffix,
829 )
830 .await;
831
832 if let Err(e) = tx.send(sent) {
833 log::error!(
834 "Failed to report drain status to controller: {e:?}"
835 );
836 }
837 }
838 WriterCommand::UpdateWithReplay(new_writer, replay, tx) => {
839 let sent = Self::replace_writer(
840 &mut active_writer,
841 new_writer,
842 replay,
843 &mut reconnect_buffer,
844 &suffix,
845 )
846 .await;
847
848 if let Err(e) = tx.send(sent) {
849 log::error!(
850 "Failed to report drain status to controller: {e:?}"
851 );
852 }
853 }
854 WriterCommand::Send(data) if must_buffer => {
855 log::debug!(
856 "Buffering message until reconnect drain completes ({} bytes)",
857 data.len()
858 );
859 Self::buffer_reconnect_write(
860 &mut reconnect_buffer,
861 data,
862 None,
863 permit,
864 );
865 }
866 WriterCommand::SendOrReplay { key, data } if must_buffer => {
867 log::debug!(
868 "Buffering replayable message until reconnect drain completes ({} bytes)",
869 data.len()
870 );
871 Self::buffer_reconnect_write(
872 &mut reconnect_buffer,
873 data,
874 Some(key),
875 permit,
876 );
877 }
878 command @ (WriterCommand::Send(_)
879 | WriterCommand::SendOrReplay { .. }) => {
880 let (msg, replay_key) = match command {
881 WriterCommand::Send(data) => (data, None),
882 WriterCommand::SendOrReplay { key, data } => (data, Some(key)),
883 _ => unreachable!(),
884 };
885
886 write_buf.clear();
887 write_buf.extend_from_slice(&msg);
888 write_buf.extend_from_slice(&suffix);
889
890 let write_result = dst::time::timeout(
891 Duration::from_secs(WRITE_TIMEOUT_SECS),
892 active_writer.write_all(&write_buf),
893 )
894 .await;
895
896 let write_failed = match write_result {
897 Ok(Ok(())) => false,
898 Ok(Err(e)) => {
899 if is_connection_drop_io_error(&e) {
900 log::warn!("Failed to send message: {e}");
901 } else {
902 log::error!("Failed to send message: {e}");
903 }
904
905 true
906 }
907 Err(_) => {
908 log::warn!(
909 "Timed out sending message after {WRITE_TIMEOUT_SECS}s"
910 );
911 true
912 }
913 };
914
915 if write_failed {
916 Self::buffer_reconnect_write(
917 &mut reconnect_buffer,
918 msg,
919 replay_key,
920 permit,
921 );
922
923 if ConnectionMode::request_reconnect_with_sink(
925 &connection_state,
926 state_sink.as_ref(),
927 ) {
928 log::warn!("Writer triggering reconnect");
929 state_notify.notify_one();
930 }
931 }
932 }
933 }
934 }
935 Ok(None) => {
936 log::debug!("Writer channel closed, terminating writer task");
938 break;
939 }
940 Err(_) => {
941 }
943 }
944 }
945
946 _ = dst::time::timeout(
949 Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS),
950 active_writer.shutdown(),
951 )
952 .await;
953
954 log_task_stopped("write");
955 })
956 }
957
958 fn buffer_reconnect_write(
959 buffer: &mut VecDeque<BufferedWrite>,
960 data: Bytes,
961 replay_key: Option<u64>,
962 permit: tokio::sync::OwnedSemaphorePermit,
963 ) {
964 if let Some(key) = replay_key {
965 buffer.retain(|buffered| buffered.replay_key != Some(key));
966 }
967
968 buffer.push_back(BufferedWrite {
969 data,
970 replay_key,
971 _permit: permit,
972 });
973 }
974
975 fn spawn_heartbeat_task(
976 connection_state: Arc<AtomicU8>,
977 interval_secs: u64,
978 message: Vec<u8>,
979 writer_tx: WriterSender<WriterCommand>,
980 ) -> tokio::task::JoinHandle<()> {
981 log_task_started("heartbeat");
982
983 tokio::task::spawn(async move {
984 let interval = Duration::from_secs(interval_secs);
985 loop {
986 dst::time::sleep(interval).await;
987
988 match ConnectionMode::from_u8(connection_state.load(Ordering::SeqCst)) {
989 ConnectionMode::Active => {
990 let msg = WriterCommand::Send(message.clone().into());
991
992 match writer_tx.send(msg) {
993 Ok(()) => log::trace!("Sent heartbeat to writer task"),
994 Err(e) => {
995 log::warn!("Failed to send heartbeat to writer task: {e}");
996 }
997 }
998 }
999 ConnectionMode::Reconnect => {}
1000 ConnectionMode::Disconnect | ConnectionMode::Closed => break,
1001 }
1002 }
1003
1004 log_task_stopped("heartbeat");
1005 })
1006 }
1007}
1008
1009impl Drop for SocketClientInner {
1010 fn drop(&mut self) {
1011 self.read_fence.invalidate();
1012
1013 if !self.read_task.is_finished() {
1014 self.read_task.abort();
1015 log_task_aborted("read");
1016 }
1017
1018 if !self.write_task.is_finished() {
1019 self.write_task.abort();
1020 log_task_aborted("write");
1021 }
1022
1023 if let Some(ref handle) = self.heartbeat_task.take()
1024 && !handle.is_finished()
1025 {
1026 handle.abort();
1027 log_task_aborted("heartbeat");
1028 }
1029 }
1030}
1031
1032pub struct SocketClient {
1038 pub(crate) controller_task: tokio::task::JoinHandle<()>,
1039 pub(crate) connection_mode: Arc<AtomicU8>,
1040 pub(crate) state_notify: Arc<tokio::sync::Notify>,
1041 pub(crate) connect_timeout: Duration,
1042 pub writer_tx: WriterSender<WriterCommand>,
1043 state_sink: Option<SocketStateSink>,
1044 controller_lifecycle: Arc<ControllerLifecycle>,
1045 controller_notify: Arc<tokio::sync::Notify>,
1046}
1047
1048#[derive(Clone)]
1050pub struct SocketReconnectHandle {
1051 connection_mode: Arc<AtomicU8>,
1052 state_sink: Option<SocketStateSink>,
1053 controller_lifecycle: Arc<ControllerLifecycle>,
1054 controller_notify: Arc<tokio::sync::Notify>,
1055}
1056
1057impl Debug for SocketReconnectHandle {
1058 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1059 f.debug_struct(stringify!(SocketReconnectHandle))
1060 .field(
1061 "connection_mode",
1062 &ConnectionMode::from_atomic(&self.connection_mode),
1063 )
1064 .finish_non_exhaustive()
1065 }
1066}
1067
1068impl SocketReconnectHandle {
1069 #[must_use]
1074 pub fn request_reconnect(&self) -> ReconnectRequestOutcome {
1075 let Some(_request) = self.controller_lifecycle.enter_request() else {
1076 return ReconnectRequestOutcome::Closed;
1077 };
1078
1079 let outcome = ConnectionMode::request_reconnect_outcome_with_sink(
1080 &self.connection_mode,
1081 self.state_sink.as_ref(),
1082 );
1083
1084 if outcome == ReconnectRequestOutcome::Accepted {
1085 self.controller_notify.notify_one();
1086 }
1087
1088 outcome
1089 }
1090}
1091
1092impl Debug for SocketClient {
1093 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1094 f.debug_struct(stringify!(SocketClient)).finish()
1095 }
1096}
1097
1098#[bon::bon]
1099impl SocketClient {
1100 #[builder(finish_fn = connect)]
1113 pub async fn builder(
1114 config: SocketConfig,
1115 post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1116 state_sink: Option<SocketStateSink>,
1117 reconnect_replay: Option<SocketReconnectReplay>,
1118 ) -> anyhow::Result<Self> {
1119 let inner = SocketClientInner::connect_url(config, state_sink).await?;
1120 let writer_tx = inner.writer_tx.clone();
1121 let connection_mode = Arc::clone(&inner.connection_mode);
1122 let state_notify = Arc::clone(&inner.state_notify);
1123 let connect_timeout = inner.connect_timeout;
1124 let state_sink = inner.state_sink.clone();
1125 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
1126 let controller_notify = Arc::new(tokio::sync::Notify::new());
1127 let controller_task = Self::spawn_controller_task(
1128 inner,
1129 Arc::clone(&connection_mode),
1130 Arc::clone(&state_notify),
1131 Arc::clone(&controller_lifecycle),
1132 Arc::clone(&controller_notify),
1133 post_reconnection,
1134 reconnect_replay,
1135 );
1136 let abort_handle = controller_task.abort_handle();
1137 controller_lifecycle.set_abort(move || abort_handle.abort());
1138
1139 Ok(Self {
1140 controller_task,
1141 connection_mode,
1142 state_notify,
1143 connect_timeout,
1144 writer_tx,
1145 state_sink,
1146 controller_lifecycle,
1147 controller_notify,
1148 })
1149 }
1150
1151 #[must_use]
1153 pub fn reconnect_handle(&self) -> SocketReconnectHandle {
1154 SocketReconnectHandle {
1155 connection_mode: Arc::clone(&self.connection_mode),
1156 state_sink: self.state_sink.clone(),
1157 controller_lifecycle: Arc::clone(&self.controller_lifecycle),
1158 controller_notify: Arc::clone(&self.controller_notify),
1159 }
1160 }
1161
1162 #[must_use]
1167 pub fn request_reconnect(&self) -> bool {
1168 self.reconnect_handle().request_reconnect() == ReconnectRequestOutcome::Accepted
1169 }
1170
1171 #[must_use]
1173 pub fn connection_mode(&self) -> ConnectionMode {
1174 ConnectionMode::from_atomic(&self.connection_mode)
1175 }
1176
1177 #[inline]
1182 #[must_use]
1183 pub fn is_active(&self) -> bool {
1184 self.connection_mode().is_active()
1185 }
1186
1187 #[inline]
1192 #[must_use]
1193 pub fn is_reconnecting(&self) -> bool {
1194 self.connection_mode().is_reconnect()
1195 }
1196
1197 #[inline]
1201 #[must_use]
1202 pub fn is_disconnecting(&self) -> bool {
1203 self.connection_mode().is_disconnect()
1204 }
1205
1206 #[inline]
1212 #[must_use]
1213 pub fn is_closed(&self) -> bool {
1214 self.connection_mode().is_closed()
1215 }
1216
1217 pub async fn close(&self) {
1222 self.begin_shutdown();
1223 self.close_with_timeout(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS))
1224 .await;
1225 }
1226
1227 pub fn begin_shutdown(&self) {
1229 ConnectionMode::request_disconnect(&self.connection_mode);
1230 self.state_notify.notify_waiters();
1231 }
1232
1233 async fn close_with_timeout(&self, shutdown_timeout: Duration) {
1234 self.begin_shutdown();
1235
1236 if dst::time::timeout(shutdown_timeout, async {
1237 while !self.controller_task.is_finished() {
1238 dst::time::sleep(Duration::from_millis(CONNECTION_STATE_CHECK_INTERVAL_MS)).await;
1239 }
1240 })
1241 .await
1242 .is_err()
1243 {
1244 log::warn!("Timeout waiting for controller task to finish");
1245 }
1246
1247 if !self.controller_task.is_finished() {
1248 self.controller_task.abort();
1249 log_task_aborted("controller");
1250 }
1251
1252 self.connection_mode
1253 .store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1254 self.state_notify.notify_waiters();
1255 }
1256
1257 #[inline]
1261 fn check_not_terminal(&self) -> Result<(), SendError> {
1262 match self.connection_mode() {
1263 ConnectionMode::Disconnect | ConnectionMode::Closed => Err(SendError::Closed),
1264 _ => Ok(()),
1265 }
1266 }
1267
1268 async fn wait_for_active(&self) -> Result<(), SendError> {
1274 const FALLBACK_INTERVAL_MS: u64 = 100;
1275
1276 let mode = self.connection_mode();
1277 if mode.is_active() {
1278 return Ok(());
1279 }
1280
1281 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1282 return Err(SendError::Closed);
1283 }
1284
1285 log::debug!("Waiting for client to become ACTIVE before sending...");
1286
1287 let fallback_interval = Duration::from_millis(FALLBACK_INTERVAL_MS);
1288
1289 dst::time::timeout(self.connect_timeout, async {
1290 loop {
1291 let mut notified = pin!(self.state_notify.notified());
1293 notified.as_mut().enable();
1294
1295 let mode = self.connection_mode();
1296 if mode.is_active() {
1297 return Ok(());
1298 }
1299
1300 if matches!(mode, ConnectionMode::Disconnect | ConnectionMode::Closed) {
1301 return Err(());
1302 }
1303
1304 tokio::select! {
1305 biased;
1306 () = notified => {}
1307 () = dst::time::sleep(fallback_interval) => {}
1308 }
1309 }
1310 })
1311 .await
1312 .map_err(|_| SendError::Timeout)?
1313 .map_err(|()| SendError::Closed)
1314 }
1315
1316 pub async fn send_bytes(&self, data: Vec<u8>) -> Result<(), SendError> {
1326 self.check_not_terminal()?;
1327 self.wait_for_active().await?;
1328
1329 let msg = WriterCommand::Send(data.into());
1330 self.writer_tx.send(msg)
1331 }
1332
1333 fn spawn_controller_task(
1334 mut inner: SocketClientInner,
1335 connection_mode: Arc<AtomicU8>,
1336 state_notify: Arc<tokio::sync::Notify>,
1337 controller_lifecycle: Arc<ControllerLifecycle>,
1338 controller_notify: Arc<tokio::sync::Notify>,
1339 post_reconnection: Option<Arc<dyn Fn() + Send + Sync>>,
1340 reconnect_replay: Option<SocketReconnectReplay>,
1341 ) -> tokio::task::JoinHandle<()> {
1342 const CONTROLLER_FALLBACK_INTERVAL_MS: u64 = 100;
1343
1344 tokio::task::spawn(async move {
1345 let _activity = controller_lifecycle.activity();
1346 log_task_started("controller");
1347
1348 let fallback_interval = Duration::from_millis(CONTROLLER_FALLBACK_INTERVAL_MS);
1349 let mut reconnected_at = None;
1350
1351 loop {
1352 tokio::select! {
1353 biased;
1354 () = controller_notify.notified() => {}
1355 () = state_notify.notified() => {}
1356 () = dst::time::sleep(fallback_interval) => {}
1357 }
1358
1359 let mut mode = ConnectionMode::from_atomic(&connection_mode);
1360
1361 if mode.is_disconnect() {
1362 log::debug!("Disconnecting");
1363
1364 let timeout = Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS);
1365 if dst::time::timeout(timeout, async {
1366 dst::time::sleep(Duration::from_millis(GRACEFUL_SHUTDOWN_DELAY_MS)).await;
1368
1369 inner.read_fence.invalidate();
1370 if !inner.read_task.is_finished() {
1371 inner.read_task.abort();
1372 log_task_aborted("read");
1373 }
1374
1375 if let Some(task) = &inner.heartbeat_task
1376 && !task.is_finished()
1377 {
1378 task.abort();
1379 log_task_aborted("heartbeat");
1380 }
1381 })
1382 .await
1383 .is_err()
1384 {
1385 log::warn!("Shutdown timed out after {}s", timeout.as_secs());
1386 }
1387
1388 log::debug!("Closed");
1389 connection_mode.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
1390 state_notify.notify_waiters();
1391 break; }
1393
1394 if mode.is_closed() {
1395 log::debug!("Connection closed");
1396
1397 inner.read_fence.invalidate();
1398 if !inner.read_task.is_finished() {
1399 inner.read_task.abort();
1400 log_task_aborted("read");
1401 }
1402
1403 if let Some(task) = &inner.heartbeat_task
1404 && !task.is_finished()
1405 {
1406 task.abort();
1407 log_task_aborted("heartbeat");
1408 }
1409
1410 state_notify.notify_waiters();
1411 break;
1412 }
1413
1414 if mode.is_active() && !inner.is_alive() {
1415 if ConnectionMode::request_reconnect_with_sink(
1416 &connection_mode,
1417 inner.state_sink.as_ref(),
1418 ) {
1419 log::info!("Detected dead read task, transitioning to RECONNECT");
1420 }
1421
1422 mode = ConnectionMode::from_atomic(&connection_mode);
1423 }
1424
1425 if mode.is_reconnect() {
1426 let reconnect_uptime = reconnected_at
1427 .take()
1428 .map(|started: dst::time::Instant| started.elapsed());
1429 let previous_reconnect_stable = reconnect_uptime
1430 .is_some_and(|uptime| uptime >= RECONNECT_STABILITY_THRESHOLD);
1431
1432 if previous_reconnect_stable {
1433 inner.backoff.reset();
1434 inner.reconnect_attempt_count = 0;
1435 log::debug!(
1436 "Socket remained active for at least {}s, resetting reconnect cycle",
1437 RECONNECT_STABILITY_THRESHOLD.as_secs()
1438 );
1439 }
1440
1441 if let Some(max_attempts) = inner.reconnect_max_attempts
1443 && inner.reconnect_attempt_count >= max_attempts
1444 {
1445 log::error!(
1446 "Max reconnection attempts ({max_attempts}) exceeded, transitioning to CLOSED"
1447 );
1448
1449 if connection_mode
1450 .compare_exchange(
1451 ConnectionMode::Reconnect.as_u8(),
1452 ConnectionMode::Closed.as_u8(),
1453 Ordering::SeqCst,
1454 Ordering::SeqCst,
1455 )
1456 .is_ok()
1457 {
1458 state_notify.notify_waiters();
1459 break;
1460 }
1461
1462 continue;
1463 }
1464
1465 let backoff_delay = if reconnect_uptime.is_some() && !previous_reconnect_stable
1466 {
1467 inner.backoff.next_duration()
1468 } else {
1469 Duration::ZERO
1470 };
1471
1472 let duration = inner.reconnect_throttle.gated_delay(backoff_delay);
1473 if !duration.is_zero() {
1474 log::warn!("Backing off for {}s...", duration.as_secs_f64());
1475
1476 if !wait_reconnect_delay(
1477 duration,
1478 connection_mode.as_ref(),
1479 state_notify.as_ref(),
1480 )
1481 .await
1482 {
1483 log::debug!("Backoff interrupted by terminal state");
1484 continue;
1485 }
1486 }
1487
1488 inner.reconnect_attempt_count += 1;
1489 inner.reconnect_throttle.record_attempt();
1490
1491 let reconnect_result = tokio::select! {
1493 biased;
1494 result = inner.reconnect(reconnect_replay.as_ref()) => Some(result),
1495 () = async {
1496 loop {
1497 let mut notified = pin!(state_notify.notified());
1499 notified.as_mut().enable();
1500
1501 if ConnectionMode::from_atomic(&connection_mode).is_disconnect() {
1502 break;
1503 }
1504 notified.await;
1505 }
1506 } => None,
1507 };
1508
1509 match reconnect_result {
1510 None => {
1511 log::debug!("Reconnect interrupted by disconnect");
1512 }
1513 Some(Ok(ReconnectOutcome::Reconnected)) => {
1514 log::debug!("Reconnected successfully");
1515 reconnected_at = Some(dst::time::Instant::now());
1516
1517 state_notify.notify_waiters();
1518
1519 if ConnectionMode::from_atomic(&connection_mode).is_active() {
1523 if let Some(ref handler) = post_reconnection {
1524 handler();
1525 log::debug!("Called `post_reconnection` handler");
1526 }
1527 } else {
1528 log::debug!(
1529 "Skipping post_reconnection handlers due to disconnect state"
1530 );
1531 }
1532 }
1533 Some(Ok(ReconnectOutcome::Aborted)) => {
1534 log::debug!("Reconnect aborted");
1535 }
1536 Some(Err(e)) => {
1537 let duration = inner.backoff.next_duration();
1538 log::warn!(
1539 "Reconnect attempt {} failed: {e}",
1540 inner.reconnect_attempt_count
1541 );
1542
1543 if !duration.is_zero() {
1544 log::warn!("Backing off for {}s...", duration.as_secs_f64());
1545
1546 if !wait_reconnect_delay(
1547 duration,
1548 connection_mode.as_ref(),
1549 state_notify.as_ref(),
1550 )
1551 .await
1552 {
1553 log::debug!("Backoff interrupted by terminal state");
1554 }
1555 }
1556 }
1557 }
1558 }
1559 }
1560
1561 log_task_stopped("controller");
1562 })
1563 }
1564}
1565
1566impl Drop for SocketClient {
1568 fn drop(&mut self) {
1569 let controller_running = !self.controller_task.is_finished();
1570 self.controller_lifecycle.close_and_abort();
1571
1572 if controller_running {
1573 log_task_aborted("controller");
1574 }
1575 }
1576}
1577
1578#[cfg(test)]
1579#[cfg(not(feature = "turmoil"))]
1580#[cfg(not(all(feature = "simulation", madsim)))] #[cfg(target_os = "linux")] mod tests {
1583 use log::Level;
1584 use nautilus_common::testing::wait_until_async;
1585 use parking_lot::Mutex as BlockingMutex;
1586 use rstest::rstest;
1587 use tokio::{
1588 io::{AsyncReadExt, AsyncWriteExt},
1589 net::{TcpListener, TcpStream},
1590 sync::Mutex,
1591 task,
1592 time::{Duration, sleep},
1593 };
1594
1595 use super::*;
1596 use crate::{SocketState, logging::tests::capture_logs_for, socket::SocketHeartbeat};
1597
1598 async fn bind_test_server() -> (u16, TcpListener) {
1599 let listener = TcpListener::bind("127.0.0.1:0")
1600 .await
1601 .expect("Failed to bind ephemeral port");
1602 let port = listener.local_addr().unwrap().port();
1603 (port, listener)
1604 }
1605
1606 async fn run_echo_server(mut socket: TcpStream) {
1607 let mut buf = Vec::new();
1608 loop {
1609 match socket.read_buf(&mut buf).await {
1610 Ok(0) => {
1611 break;
1612 }
1613 Ok(_n) => {
1614 while let Some(idx) = buf.array_windows().position(|w| w == b"\r\n") {
1615 let mut line = buf.drain(..idx + 2).collect::<Vec<u8>>();
1616 line.truncate(line.len() - 2);
1618
1619 if line == b"close" {
1620 let _ = socket.shutdown().await;
1621 return;
1622 }
1623
1624 let mut echo_data = line;
1625 echo_data.extend_from_slice(b"\r\n");
1626 if socket.write_all(&echo_data).await.is_err() {
1627 break;
1628 }
1629 }
1630 }
1631 Err(e) => {
1632 eprintln!("Server read error: {e}");
1633 break;
1634 }
1635 }
1636 }
1637 }
1638
1639 #[rstest]
1640 #[tokio::test]
1641 async fn test_writer_capacity_from_config() {
1642 let (port, listener) = bind_test_server().await;
1643 let config = SocketConfig::builder()
1644 .url(format!("127.0.0.1:{port}"))
1645 .mode(Mode::Plain)
1646 .suffix(b"\r\n".to_vec())
1647 .writer_capacity(2)
1648 .build()
1649 .unwrap();
1650 let client = SocketClient::builder()
1651 .config(config)
1652 .connect()
1653 .await
1654 .unwrap();
1655 let (_peer, _) = listener.accept().await.unwrap();
1656
1657 client.send_bytes(vec![1]).await.unwrap();
1659 client.send_bytes(vec![2]).await.unwrap();
1660 let overflow = client.send_bytes(vec![3]).await;
1661
1662 assert_eq!(client.connection_mode(), ConnectionMode::Active);
1663 assert!(matches!(overflow, Err(SendError::BufferFull)));
1664 client.close().await;
1665 }
1666
1667 #[rstest]
1668 #[case::drain_failed(
1669 Some(false),
1670 std::io::ErrorKind::Other,
1671 "Failed to drain reconnection buffer"
1672 )]
1673 #[case::reply_dropped(
1674 None,
1675 std::io::ErrorKind::BrokenPipe,
1676 "Writer task dropped response channel"
1677 )]
1678 #[tokio::test]
1679 async fn test_reconnect_writer_failure_preserves_reconnect_state(
1680 #[case] reply: Option<bool>,
1681 #[case] kind: std::io::ErrorKind,
1682 #[case] message: &str,
1683 ) {
1684 let (port, listener) = bind_test_server().await;
1685 let config = SocketConfig::builder()
1686 .url(format!("127.0.0.1:{port}"))
1687 .mode(Mode::Plain)
1688 .suffix(b"\r\n".to_vec())
1689 .build()
1690 .unwrap();
1691 let mut client = SocketClientInner::connect_url(config, None).await.unwrap();
1692 let (_peer, _) = listener.accept().await.unwrap();
1693 client.read_task.abort();
1694 client.write_task.abort();
1695 let _ = (&mut client.read_task).await;
1696 let _ = (&mut client.write_task).await;
1697 let (writer_tx, mut writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
1698 client.writer_tx = writer_tx;
1699 client
1700 .connection_mode
1701 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
1702
1703 let writer = tokio::spawn(async move {
1704 let WriterCommand::Update(_, sender) = writer_rx.recv().await.unwrap().0 else {
1705 panic!("expected writer update");
1706 };
1707
1708 if let Some(reply) = reply {
1709 sender.send(reply).unwrap();
1710 }
1711 });
1712
1713 let error = client.reconnect(None).await.unwrap_err();
1714 writer.await.unwrap();
1715
1716 let Error::Io(error) = error else {
1717 panic!("expected I/O error, was {error}");
1718 };
1719
1720 assert_eq!(error.kind(), kind);
1721 assert_eq!(error.to_string(), message);
1722 assert_eq!(
1723 ConnectionMode::from_atomic(&client.connection_mode),
1724 ConnectionMode::Reconnect
1725 );
1726 }
1727
1728 #[tokio::test]
1729 async fn test_basic_send_receive() {
1730 let (port, listener) = bind_test_server().await;
1731 let server_task = task::spawn(async move {
1732 let (socket, _) = listener.accept().await.unwrap();
1733 run_echo_server(socket).await;
1734 });
1735
1736 let config = SocketConfig {
1737 url: format!("127.0.0.1:{port}"),
1738 mode: Mode::Plain,
1739 suffix: b"\r\n".to_vec(),
1740 message_handler: None,
1741 heartbeat: None,
1742 connect_timeout_ms: None,
1743 reconnect_delay_initial_ms: None,
1744 reconnect_backoff_factor: None,
1745 reconnect_delay_max_ms: None,
1746 reconnect_jitter_ms: None,
1747 reconnect_max_attempts: None,
1748 connection_max_retries: None,
1749 heartbeat_timeout_secs: None,
1750 writer_capacity: None,
1751 certs_dir: None,
1752 };
1753
1754 let client = SocketClient::builder()
1755 .config(config)
1756 .connect()
1757 .await
1758 .expect("Client connect failed unexpectedly");
1759
1760 client.send_bytes(b"Hello".into()).await.unwrap();
1761 client.send_bytes(b"World".into()).await.unwrap();
1762
1763 sleep(Duration::from_millis(100)).await;
1765
1766 client.send_bytes(b"close".into()).await.unwrap();
1767 server_task.await.unwrap();
1768 assert!(!client.is_closed());
1769 }
1770
1771 #[tokio::test]
1772 async fn test_reconnect_fail_exhausted() {
1773 let (port, listener) = bind_test_server().await;
1774 drop(listener); wait_until_async(
1778 || async {
1779 TcpStream::connect(format!("127.0.0.1:{port}"))
1780 .await
1781 .is_err()
1782 },
1783 Duration::from_secs(2),
1784 )
1785 .await;
1786
1787 let config = SocketConfig {
1788 url: format!("127.0.0.1:{port}"),
1789 mode: Mode::Plain,
1790 suffix: b"\r\n".to_vec(),
1791 message_handler: None,
1792 heartbeat: None,
1793 connect_timeout_ms: Some(100),
1794 reconnect_delay_initial_ms: Some(50),
1795 reconnect_backoff_factor: Some(1.0),
1796 reconnect_delay_max_ms: Some(50),
1797 reconnect_jitter_ms: Some(0),
1798 connection_max_retries: Some(1),
1799 reconnect_max_attempts: None,
1800 heartbeat_timeout_secs: None,
1801 writer_capacity: None,
1802 certs_dir: None,
1803 };
1804
1805 let client_res = SocketClient::builder().config(config).connect().await;
1806 assert!(
1807 client_res.is_err(),
1808 "Should fail quickly with no server listening"
1809 );
1810 }
1811
1812 #[rstest]
1813 #[tokio::test]
1814 async fn test_connect_failure_redacts_url_from_logs_and_error() {
1815 const ENDPOINT_PATH_SECRET: &str = "unique-endpoint-path-secret";
1816 const ENDPOINT_QUERY_SECRET: &str = "unique-endpoint-query-secret";
1817
1818 let (port, listener) = bind_test_server().await;
1819 drop(listener); wait_until_async(
1823 || async {
1824 TcpStream::connect(format!("127.0.0.1:{port}"))
1825 .await
1826 .is_err()
1827 },
1828 Duration::from_secs(2),
1829 )
1830 .await;
1831
1832 let capture = capture_logs_for(&["nautilus_network::socket::client"]).await;
1833
1834 let config = SocketConfig {
1835 url: format!(
1836 "ws://127.0.0.1:{port}/{ENDPOINT_PATH_SECRET}?api_key={ENDPOINT_QUERY_SECRET}"
1837 ),
1838 mode: Mode::Plain,
1839 suffix: b"\r\n".to_vec(),
1840 message_handler: None,
1841 heartbeat: None,
1842 connect_timeout_ms: Some(1_000),
1843 reconnect_delay_initial_ms: Some(50),
1844 reconnect_delay_max_ms: Some(50),
1845 reconnect_backoff_factor: Some(1.0),
1846 reconnect_jitter_ms: Some(0),
1847 connection_max_retries: Some(1),
1848 reconnect_max_attempts: None,
1849 heartbeat_timeout_secs: None,
1850 writer_capacity: None,
1851 certs_dir: None,
1852 };
1853
1854 let error = SocketClient::builder()
1855 .config(config)
1856 .connect()
1857 .await
1858 .expect_err("connection should fail with no server listening");
1859
1860 let error_message = error.to_string();
1861 assert!(
1862 error_message.starts_with(&format!("Failed to connect to {REDACTED}")),
1863 "connect error omitted the redaction placeholder: {error_message}"
1864 );
1865 assert!(
1866 !error_message.contains(ENDPOINT_PATH_SECRET)
1867 && !error_message.contains(ENDPOINT_QUERY_SECRET),
1868 "connect error exposed the secret marker: {error_message}"
1869 );
1870
1871 let messages = capture.messages();
1872
1873 let (_, attempt_warning) = messages
1876 .iter()
1877 .find(|(level, message)| {
1878 *level == Level::Warn && message.starts_with("Socket connection attempt 1/1 ")
1879 })
1880 .expect("connect-failure warning was not captured");
1881
1882 assert!(
1883 attempt_warning.contains(REDACTED),
1884 "connect-failure warning omitted the redaction placeholder: {attempt_warning}"
1885 );
1886 assert!(
1887 messages.iter().any(|(level, message)| {
1888 *level == Level::Debug && *message == format!("Connecting to {REDACTED}")
1889 }),
1890 "connect debug log should redact the endpoint, was {messages:?}"
1891 );
1892 assert!(
1893 messages.iter().all(|(_, message)| {
1894 !message.contains(ENDPOINT_PATH_SECRET) && !message.contains(ENDPOINT_QUERY_SECRET)
1895 }),
1896 "socket client logs exposed the secret marker: {messages:?}"
1897 );
1898 }
1899
1900 #[tokio::test]
1901 async fn test_user_disconnect() {
1902 let (port, listener) = bind_test_server().await;
1903 let server_task = task::spawn(async move {
1904 let (socket, _) = listener.accept().await.unwrap();
1905 let mut buf = [0u8; 1024];
1906 let _ = socket.try_read(&mut buf);
1907
1908 loop {
1909 sleep(Duration::from_secs(1)).await;
1910 }
1911 });
1912
1913 let config = SocketConfig {
1914 url: format!("127.0.0.1:{port}"),
1915 mode: Mode::Plain,
1916 suffix: b"\r\n".to_vec(),
1917 message_handler: None,
1918 heartbeat: None,
1919 connect_timeout_ms: None,
1920 reconnect_delay_initial_ms: None,
1921 reconnect_backoff_factor: None,
1922 reconnect_delay_max_ms: None,
1923 reconnect_jitter_ms: None,
1924 reconnect_max_attempts: None,
1925 connection_max_retries: None,
1926 heartbeat_timeout_secs: None,
1927 writer_capacity: None,
1928 certs_dir: None,
1929 };
1930
1931 let client = SocketClient::builder()
1932 .config(config)
1933 .connect()
1934 .await
1935 .unwrap();
1936
1937 client.close().await;
1938 assert!(client.is_closed());
1939 server_task.abort();
1940 }
1941
1942 #[tokio::test]
1943 async fn test_close_after_closed_returns_fast_and_preserves_state() {
1944 let (port, listener) = bind_test_server().await;
1945
1946 let server_task = task::spawn(async move {
1947 let (socket, _) = listener.accept().await.unwrap();
1950 drop(socket);
1951 drop(listener);
1952 sleep(Duration::from_secs(5)).await;
1953 });
1954
1955 let config = SocketConfig {
1956 url: format!("127.0.0.1:{port}"),
1957 mode: Mode::Plain,
1958 suffix: b"\r\n".to_vec(),
1959 message_handler: None,
1960 heartbeat: None,
1961 connect_timeout_ms: Some(200),
1962 reconnect_delay_initial_ms: Some(50),
1963 reconnect_backoff_factor: Some(1.0),
1964 reconnect_delay_max_ms: Some(50),
1965 reconnect_jitter_ms: Some(0),
1966 connection_max_retries: None,
1967 reconnect_max_attempts: Some(1),
1968 heartbeat_timeout_secs: None,
1969 writer_capacity: None,
1970 certs_dir: None,
1971 };
1972
1973 let client = SocketClient::builder()
1974 .config(config)
1975 .connect()
1976 .await
1977 .unwrap();
1978
1979 wait_until_async(|| async { client.is_closed() }, Duration::from_secs(5)).await;
1980
1981 let start = std::time::Instant::now();
1985 client.close().await;
1986 let elapsed = start.elapsed();
1987
1988 assert!(client.is_closed(), "Client should remain CLOSED");
1989 assert!(
1990 !client.is_disconnecting(),
1991 "Closed client should not report DISCONNECT after close()"
1992 );
1993 assert!(
1994 elapsed < Duration::from_secs(2),
1995 "close() on a closed client should return fast, took {elapsed:?}"
1996 );
1997
1998 server_task.abort();
1999 }
2000
2001 #[rstest]
2002 #[tokio::test]
2003 async fn test_heartbeat_task_warns_when_writer_channel_closed() {
2004 let capture = capture_logs_for(&["nautilus_network::socket::client"]).await;
2005 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2006 let (writer_tx, mut writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
2007 let task = SocketClientInner::spawn_heartbeat_task(
2008 Arc::clone(&connection_state),
2009 1,
2010 b"ping".to_vec(),
2011 writer_tx,
2012 );
2013
2014 let _ = tokio::time::timeout(Duration::from_secs(2), writer_rx.recv())
2015 .await
2016 .expect("timed out waiting for the first heartbeat")
2017 .expect("heartbeat channel closed before the first heartbeat");
2018 drop(writer_rx);
2019 tokio::time::sleep(Duration::from_millis(1_200)).await;
2020
2021 let messages = capture.messages();
2022 assert!(
2023 messages.iter().any(|(level, message)| {
2024 *level == Level::Warn && message.contains("Failed to send heartbeat to writer task")
2025 }),
2026 "expected WARN when the writer channel is closed, was {messages:?}"
2027 );
2028 assert!(
2029 !messages.iter().any(|(level, _)| *level == Level::Error),
2030 "a closed writer channel must not log ERROR, was {messages:?}"
2031 );
2032
2033 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
2034 tokio::time::timeout(Duration::from_secs(2), task)
2035 .await
2036 .expect("heartbeat task should stop after close")
2037 .unwrap();
2038 }
2039
2040 #[tokio::test]
2041 async fn test_heartbeat() {
2042 let (port, listener) = bind_test_server().await;
2043 let received = Arc::new(Mutex::new(Vec::new()));
2044 let received2 = Arc::clone(&received);
2045
2046 let server_task = task::spawn(async move {
2047 let (socket, _) = listener.accept().await.unwrap();
2048
2049 let mut buf = Vec::new();
2050 loop {
2051 match socket.try_read_buf(&mut buf) {
2052 Ok(0) => break,
2053 Ok(_) => {
2054 while let Some(idx) = buf.array_windows().position(|w| w == b"\r\n") {
2055 let mut line = buf.drain(..idx + 2).collect::<Vec<u8>>();
2056 line.truncate(line.len() - 2);
2057 received2.lock().await.push(line);
2058 }
2059 }
2060 Err(_) => {
2061 tokio::time::sleep(Duration::from_millis(10)).await;
2062 }
2063 }
2064 }
2065 });
2066
2067 let config = SocketConfig {
2068 url: format!("127.0.0.1:{port}"),
2069 mode: Mode::Plain,
2070 suffix: b"\r\n".to_vec(),
2071 message_handler: None,
2072 heartbeat: Some(SocketHeartbeat {
2073 interval_secs: 1,
2074 payload: b"ping".to_vec(),
2075 }),
2076 connect_timeout_ms: None,
2077 reconnect_delay_initial_ms: None,
2078 reconnect_backoff_factor: None,
2079 reconnect_delay_max_ms: None,
2080 reconnect_jitter_ms: None,
2081 reconnect_max_attempts: None,
2082 connection_max_retries: None,
2083 heartbeat_timeout_secs: None,
2084 writer_capacity: None,
2085 certs_dir: None,
2086 };
2087
2088 let client = SocketClient::builder()
2089 .config(config)
2090 .connect()
2091 .await
2092 .unwrap();
2093
2094 sleep(Duration::from_secs(3)).await;
2096
2097 {
2098 let lock = received.lock().await;
2099 let pings = lock
2100 .iter()
2101 .filter(|line| line == &&b"ping".to_vec())
2102 .count();
2103 assert!(
2104 pings >= 2,
2105 "Expected at least 2 heartbeat pings; got {pings}"
2106 );
2107 }
2108
2109 client.close().await;
2110 server_task.abort();
2111 }
2112
2113 #[tokio::test]
2114 async fn test_reconnect_success() {
2115 let (port, listener) = bind_test_server().await;
2116
2117 let server_task = task::spawn(async move {
2121 let (mut socket, _) = listener.accept().await.expect("First accept failed");
2123
2124 sleep(Duration::from_millis(500)).await;
2126 let _ = socket.shutdown().await;
2127
2128 sleep(Duration::from_millis(500)).await;
2130
2131 let (socket, _) = listener.accept().await.expect("Second accept failed");
2133 run_echo_server(socket).await;
2134 });
2135
2136 let config = SocketConfig {
2137 url: format!("127.0.0.1:{port}"),
2138 mode: Mode::Plain,
2139 suffix: b"\r\n".to_vec(),
2140 message_handler: None,
2141 heartbeat: None,
2142 connect_timeout_ms: Some(5_000),
2143 reconnect_delay_initial_ms: Some(500),
2144 reconnect_delay_max_ms: Some(5_000),
2145 reconnect_backoff_factor: Some(2.0),
2146 reconnect_jitter_ms: Some(50),
2147 reconnect_max_attempts: None,
2148 connection_max_retries: None,
2149 heartbeat_timeout_secs: None,
2150 writer_capacity: None,
2151 certs_dir: None,
2152 };
2153
2154 let client = SocketClient::builder()
2155 .config(config)
2156 .connect()
2157 .await
2158 .expect("Client connect failed unexpectedly");
2159
2160 assert!(client.is_active(), "Client should start as active");
2162
2163 wait_until_async(|| async { client.is_active() }, Duration::from_secs(10)).await;
2166
2167 client
2168 .send_bytes(b"TestReconnect".into())
2169 .await
2170 .expect("Send failed");
2171
2172 client.close().await;
2173 server_task.abort();
2174 }
2175
2176 #[rstest]
2177 #[tokio::test]
2178 async fn test_state_sink_reports_initial_loss_and_recovery() {
2179 let (port, listener) = bind_test_server().await;
2180 let server_task = task::spawn(async move {
2181 let (mut socket, _) = listener.accept().await.unwrap();
2182 sleep(Duration::from_millis(100)).await;
2183 socket.shutdown().await.unwrap();
2184
2185 let (socket, _) = listener.accept().await.unwrap();
2186 run_echo_server(socket).await;
2187 });
2188
2189 let config = SocketConfig {
2190 url: format!("127.0.0.1:{port}"),
2191 mode: Mode::Plain,
2192 suffix: b"\r\n".to_vec(),
2193 message_handler: None,
2194 heartbeat: None,
2195 connect_timeout_ms: Some(1_000),
2196 reconnect_delay_initial_ms: Some(10),
2197 reconnect_backoff_factor: Some(1.0),
2198 reconnect_delay_max_ms: Some(10),
2199 reconnect_jitter_ms: Some(0),
2200 reconnect_max_attempts: Some(3),
2201 connection_max_retries: Some(1),
2202 heartbeat_timeout_secs: None,
2203 writer_capacity: None,
2204 certs_dir: None,
2205 };
2206
2207 let states = Arc::new(BlockingMutex::new(Vec::new()));
2208 let states_callback = Arc::clone(&states);
2209 let sink = SocketStateSink::new(move |state| {
2210 states_callback.lock().push(state);
2211 });
2212
2213 let client = SocketClient::builder()
2214 .config(config)
2215 .state_sink(sink)
2216 .connect()
2217 .await
2218 .unwrap();
2219
2220 assert_eq!(*states.lock(), vec![SocketState::Connected]);
2221
2222 wait_until_async(
2223 || {
2224 let states = Arc::clone(&states);
2225 async move { states.lock().len() == 3 }
2226 },
2227 Duration::from_secs(5),
2228 )
2229 .await;
2230
2231 assert_eq!(
2232 *states.lock(),
2233 vec![
2234 SocketState::Connected,
2235 SocketState::Disconnected,
2236 SocketState::Connected,
2237 ]
2238 );
2239
2240 client.close().await;
2241 assert_eq!(states.lock().len(), 3);
2242 server_task.abort();
2243 }
2244
2245 #[rstest]
2246 #[tokio::test]
2247 async fn test_state_sink_ignores_initial_connection_failure() {
2248 let (port, listener) = bind_test_server().await;
2249 drop(listener);
2250
2251 let config = SocketConfig {
2252 url: format!("127.0.0.1:{port}"),
2253 mode: Mode::Plain,
2254 suffix: b"\r\n".to_vec(),
2255 message_handler: None,
2256 heartbeat: None,
2257 connect_timeout_ms: Some(100),
2258 reconnect_delay_initial_ms: Some(1),
2259 reconnect_backoff_factor: Some(1.0),
2260 reconnect_delay_max_ms: Some(1),
2261 reconnect_jitter_ms: Some(0),
2262 reconnect_max_attempts: Some(1),
2263 connection_max_retries: Some(1),
2264 heartbeat_timeout_secs: None,
2265 writer_capacity: None,
2266 certs_dir: None,
2267 };
2268
2269 let states = Arc::new(BlockingMutex::new(Vec::new()));
2270 let states_callback = Arc::clone(&states);
2271 let sink = SocketStateSink::new(move |state| {
2272 states_callback.lock().push(state);
2273 });
2274
2275 let result = SocketClient::builder()
2276 .config(config)
2277 .state_sink(sink)
2278 .connect()
2279 .await;
2280
2281 assert!(result.is_err());
2282 assert_eq!(*states.lock(), Vec::new());
2283 }
2284}
2285
2286#[cfg(test)]
2287#[cfg(not(feature = "turmoil"))]
2288#[cfg(not(all(feature = "simulation", madsim)))] mod rust_tests {
2290 use std::{
2291 pin::Pin,
2292 sync::{
2293 Arc,
2294 atomic::{AtomicBool, AtomicUsize},
2295 },
2296 task::{Context, Poll, Waker},
2297 };
2298
2299 use nautilus_common::testing::wait_until_async;
2300 use parking_lot::{Condvar, Mutex};
2301 use rstest::rstest;
2302 use tokio::{
2303 io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf},
2304 net::TcpListener,
2305 sync::oneshot,
2306 task::{self, yield_now},
2307 time::{Duration, sleep},
2308 };
2309
2310 use super::*;
2311 use crate::{SocketState, socket::SocketHeartbeat};
2312
2313 const TEST_TIMEOUT: Duration = Duration::from_secs(10);
2314
2315 struct CondvarReleaseGuard {
2316 release: Arc<(Mutex<bool>, Condvar)>,
2317 }
2318
2319 impl CondvarReleaseGuard {
2320 fn new(release: Arc<(Mutex<bool>, Condvar)>) -> Self {
2321 Self { release }
2322 }
2323
2324 fn release(&self) {
2325 let (lock, condvar) = self.release.as_ref();
2326 let mut released = lock.lock();
2327 *released = true;
2328 condvar.notify_all();
2329 }
2330 }
2331
2332 impl Drop for CondvarReleaseGuard {
2333 fn drop(&mut self) {
2334 self.release();
2335 }
2336 }
2337
2338 async fn recv_rendezvous<T: Send + 'static>(
2339 receiver: std::sync::mpsc::Receiver<T>,
2340 name: &'static str,
2341 ) -> T {
2342 let receive_task = tokio::task::spawn_blocking(move || receiver.recv_timeout(TEST_TIMEOUT));
2343
2344 match tokio::time::timeout(TEST_TIMEOUT * 2, receive_task).await {
2345 Ok(Ok(Ok(value))) => value,
2346 Ok(Ok(Err(e))) => {
2347 panic!("{name} did not arrive within the test timeout: {e}")
2348 }
2349 Ok(Err(e)) => panic!("{name} receive task failed: {e}"),
2350 Err(e) => panic!("{name} receive task did not finish: {e}"),
2351 }
2352 }
2353
2354 async fn await_task_termination(
2355 task: tokio::task::JoinHandle<()>,
2356 name: &'static str,
2357 cancellation_ok: bool,
2358 ) {
2359 match tokio::time::timeout(TEST_TIMEOUT, task).await {
2360 Ok(Ok(())) => {}
2361 Ok(Err(e)) if cancellation_ok && e.is_cancelled() => {}
2362 Ok(Err(e)) => panic!("{name} failed: {e}"),
2363 Err(e) => panic!("{name} did not terminate within the test timeout: {e}"),
2364 }
2365 }
2366
2367 fn reconnect_test_config(port: u16) -> SocketConfig {
2368 SocketConfig {
2369 url: format!("127.0.0.1:{port}"),
2370 mode: Mode::Plain,
2371 suffix: b"\r\n".to_vec(),
2372 message_handler: None,
2373 heartbeat: None,
2374 connect_timeout_ms: Some(1_000),
2375 reconnect_delay_initial_ms: None,
2376 reconnect_backoff_factor: None,
2377 reconnect_delay_max_ms: None,
2378 reconnect_jitter_ms: None,
2379 connection_max_retries: Some(1),
2380 reconnect_max_attempts: None,
2381 heartbeat_timeout_secs: None,
2382 writer_capacity: None,
2383 certs_dir: None,
2384 }
2385 }
2386
2387 #[rstest]
2388 #[tokio::test]
2389 async fn test_reconnect_outcome_is_aborted_before_connect() {
2390 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2391 let port = listener.local_addr().unwrap().port();
2392 let server = tokio::spawn(async move {
2393 let (_socket, _) = listener.accept().await.unwrap();
2394 std::future::pending::<()>().await;
2395 });
2396
2397 let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2398 .await
2399 .unwrap();
2400 inner
2401 .connection_mode
2402 .store(ConnectionMode::Disconnect.as_u8(), Ordering::SeqCst);
2403
2404 let outcome = inner.reconnect(None).await.unwrap();
2405
2406 assert_eq!(outcome, ReconnectOutcome::Aborted);
2407 server.abort();
2408 }
2409
2410 #[rstest]
2411 #[tokio::test]
2412 async fn test_reconnect_outcome_is_reconnected() {
2413 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2414 let port = listener.local_addr().unwrap().port();
2415 let server = tokio::spawn(async move {
2416 let (_first, _) = listener.accept().await.unwrap();
2417 let (_second, _) = listener.accept().await.unwrap();
2418 std::future::pending::<()>().await;
2419 });
2420
2421 let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2422 .await
2423 .unwrap();
2424 inner
2425 .connection_mode
2426 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
2427
2428 let outcome = inner.reconnect(None).await.unwrap();
2429
2430 assert_eq!(outcome, ReconnectOutcome::Reconnected);
2431 assert_eq!(
2432 ConnectionMode::from_atomic(&inner.connection_mode),
2433 ConnectionMode::Active
2434 );
2435 server.abort();
2436 }
2437
2438 #[rstest]
2439 #[tokio::test]
2440 async fn test_reconnect_aborts_retired_reader() {
2441 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2442 let port = listener.local_addr().unwrap().port();
2443 let mut inner = SocketClientInner::connect_url(reconnect_test_config(port), None)
2444 .await
2445 .unwrap();
2446 let (_peer, _) = listener.accept().await.unwrap();
2447 let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel::<()>();
2448 let pending_reader = tokio::spawn(async move {
2449 let _sender = dropped_tx;
2450 std::future::pending::<()>().await;
2451 });
2452
2453 let previous_reader = std::mem::replace(&mut inner.read_task, pending_reader);
2454 previous_reader.abort();
2455 let _ = previous_reader.await;
2456 let old_fence = inner.read_fence.clone();
2457 inner
2458 .connection_mode
2459 .store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
2460
2461 let outcome = inner.reconnect(None).await.unwrap();
2462 let reader_result = tokio::time::timeout(Duration::from_secs(1), dropped_rx)
2463 .await
2464 .expect("retired reader task must be canceled");
2465
2466 assert_eq!(outcome, ReconnectOutcome::Reconnected);
2467 assert!(
2468 reader_result.is_err(),
2469 "retired reader must drop its sender without sending"
2470 );
2471 assert!(!old_fence.is_valid());
2472 assert!(inner.read_fence.is_valid());
2473 assert_eq!(
2474 ConnectionMode::from_atomic(&inner.connection_mode),
2475 ConnectionMode::Active
2476 );
2477 }
2478
2479 #[rstest]
2480 #[tokio::test]
2481 async fn test_inner_drop_invalidates_read_fence_and_aborts_tasks() {
2482 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2483 let port = listener.local_addr().unwrap().port();
2484 let server = tokio::spawn(async move {
2485 let (_socket, _) = listener.accept().await.unwrap();
2486 std::future::pending::<()>().await;
2487 });
2488
2489 let mut config = reconnect_test_config(port);
2490 config.heartbeat = Some(SocketHeartbeat {
2491 interval_secs: 60,
2492 payload: b"ping\r\n".to_vec(),
2493 });
2494
2495 let inner = SocketClientInner::connect_url(config, None).await.unwrap();
2496 let read_fence = inner.read_fence.clone();
2497 let read_abort = inner.read_task.abort_handle();
2498 let write_abort = inner.write_task.abort_handle();
2499 let heartbeat_abort = inner
2500 .heartbeat_task
2501 .as_ref()
2502 .expect("heartbeat task should be spawned for a configured heartbeat")
2503 .abort_handle();
2504
2505 assert!(read_fence.is_valid(), "read fence should start valid");
2506 assert!(
2507 !read_abort.is_finished(),
2508 "read task should be running before drop"
2509 );
2510 assert!(
2511 !write_abort.is_finished(),
2512 "write task should be running before drop"
2513 );
2514 assert!(
2515 !heartbeat_abort.is_finished(),
2516 "heartbeat task should be running before drop"
2517 );
2518
2519 drop(inner);
2520 wait_until_async(
2521 || async {
2522 read_abort.is_finished()
2523 && write_abort.is_finished()
2524 && heartbeat_abort.is_finished()
2525 },
2526 TEST_TIMEOUT,
2527 )
2528 .await;
2529
2530 assert!(!read_fence.is_valid(), "read fence was not invalidated");
2531 assert!(read_abort.is_finished(), "read task was not aborted");
2532 assert!(write_abort.is_finished(), "write task was not aborted");
2533 assert!(
2534 heartbeat_abort.is_finished(),
2535 "heartbeat task was not aborted"
2536 );
2537 server.abort();
2538 }
2539
2540 struct ScriptedReader {
2541 first: Option<Vec<u8>>,
2542 remainder: Arc<Mutex<Option<Vec<u8>>>>,
2543 pending_tx: Option<oneshot::Sender<()>>,
2544 waker: Arc<Mutex<Option<Waker>>>,
2545 }
2546
2547 impl AsyncRead for ScriptedReader {
2548 fn poll_read(
2549 mut self: Pin<&mut Self>,
2550 cx: &mut Context<'_>,
2551 buf: &mut ReadBuf<'_>,
2552 ) -> Poll<std::io::Result<()>> {
2553 if let Some(first) = self.first.take() {
2554 buf.put_slice(&first);
2555 return Poll::Ready(Ok(()));
2556 }
2557
2558 if let Some(remainder) = self.remainder.lock().take() {
2559 buf.put_slice(&remainder);
2560 return Poll::Ready(Ok(()));
2561 }
2562
2563 if let Some(tx) = self.pending_tx.take() {
2564 let _ = tx.send(());
2565 }
2566
2567 *self.waker.lock() = Some(cx.waker().clone());
2568 Poll::Pending
2569 }
2570 }
2571
2572 struct ChunkReader {
2574 chunks: VecDeque<&'static [u8]>,
2575 }
2576
2577 impl AsyncRead for ChunkReader {
2578 fn poll_read(
2579 mut self: Pin<&mut Self>,
2580 _cx: &mut Context<'_>,
2581 buf: &mut ReadBuf<'_>,
2582 ) -> Poll<std::io::Result<()>> {
2583 if let Some(chunk) = self.chunks.pop_front() {
2584 buf.put_slice(chunk);
2585 }
2586
2587 Poll::Ready(Ok(()))
2588 }
2589 }
2590
2591 struct LimitProbeReader {
2592 remaining: usize,
2593 read_after_limit: Arc<AtomicBool>,
2594 }
2595
2596 impl AsyncRead for LimitProbeReader {
2597 fn poll_read(
2598 mut self: Pin<&mut Self>,
2599 _cx: &mut Context<'_>,
2600 buf: &mut ReadBuf<'_>,
2601 ) -> Poll<std::io::Result<()>> {
2602 if self.remaining == 0 {
2603 self.read_after_limit.store(true, Ordering::SeqCst);
2604 return Poll::Ready(Ok(()));
2605 }
2606
2607 let read_len = self.remaining.min(buf.remaining());
2608 buf.initialize_unfilled_to(read_len).fill(b'x');
2609 buf.advance(read_len);
2610 self.remaining -= read_len;
2611 Poll::Ready(Ok(()))
2612 }
2613 }
2614
2615 struct BackpressuredWriter {
2616 stream: DuplexStream,
2617 pending_tx: Option<oneshot::Sender<()>>,
2618 }
2619
2620 impl AsyncWrite for BackpressuredWriter {
2621 fn poll_write(
2622 mut self: Pin<&mut Self>,
2623 cx: &mut Context<'_>,
2624 buf: &[u8],
2625 ) -> Poll<std::io::Result<usize>> {
2626 let result = Pin::new(&mut self.stream).poll_write(cx, buf);
2627 if result.is_pending()
2628 && let Some(tx) = self.pending_tx.take()
2629 {
2630 let _ = tx.send(());
2631 }
2632
2633 result
2634 }
2635
2636 fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2637 Pin::new(&mut self.stream).poll_flush(cx)
2638 }
2639
2640 fn poll_shutdown(
2641 mut self: Pin<&mut Self>,
2642 cx: &mut Context<'_>,
2643 ) -> Poll<std::io::Result<()>> {
2644 Pin::new(&mut self.stream).poll_shutdown(cx)
2645 }
2646 }
2647
2648 struct FailingWriter;
2649
2650 impl AsyncWrite for FailingWriter {
2651 fn poll_write(
2652 self: Pin<&mut Self>,
2653 _cx: &mut Context<'_>,
2654 _buf: &[u8],
2655 ) -> Poll<std::io::Result<usize>> {
2656 Poll::Ready(Err(std::io::Error::new(
2657 std::io::ErrorKind::BrokenPipe,
2658 "test writer failure",
2659 )))
2660 }
2661
2662 fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2663 Poll::Ready(Ok(()))
2664 }
2665
2666 fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2667 Poll::Ready(Ok(()))
2668 }
2669 }
2670
2671 struct RecordingWriter {
2672 bytes: Arc<Mutex<Vec<u8>>>,
2673 }
2674
2675 impl AsyncWrite for RecordingWriter {
2676 fn poll_write(
2677 self: Pin<&mut Self>,
2678 _cx: &mut Context<'_>,
2679 buf: &[u8],
2680 ) -> Poll<std::io::Result<usize>> {
2681 self.bytes.lock().extend_from_slice(buf);
2682 Poll::Ready(Ok(buf.len()))
2683 }
2684
2685 fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2686 Poll::Ready(Ok(()))
2687 }
2688
2689 fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
2690 Poll::Ready(Ok(()))
2691 }
2692 }
2693
2694 fn test_socket_client(
2695 connection_state: Arc<AtomicU8>,
2696 state_notify: Arc<tokio::sync::Notify>,
2697 controller_task: tokio::task::JoinHandle<()>,
2698 ) -> SocketClient {
2699 let (writer_tx, _writer_rx) = writer::channel(DEFAULT_WRITER_CAPACITY);
2700 let controller_lifecycle = Arc::new(ControllerLifecycle::new());
2701 let abort_handle = controller_task.abort_handle();
2702 controller_lifecycle.set_abort(move || abort_handle.abort());
2703
2704 SocketClient {
2705 controller_task,
2706 connection_mode: connection_state,
2707 state_notify,
2708 connect_timeout: Duration::from_secs(1),
2709 writer_tx,
2710 controller_lifecycle,
2711 controller_notify: Arc::new(tokio::sync::Notify::new()),
2712 state_sink: None,
2713 }
2714 }
2715
2716 #[rstest]
2717 #[tokio::test]
2718 async fn test_send_reports_writer_buffer_full() {
2719 let state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2720 let notify = Arc::new(tokio::sync::Notify::new());
2721
2722 let task = tokio::spawn(std::future::pending::<()>());
2723 let mut client = test_socket_client(state, notify, task);
2724 let (writer_tx, mut writer_rx) = writer::channel(1);
2725 client.writer_tx = writer_tx;
2726
2727 client.send_bytes(vec![1]).await.unwrap();
2728 assert!(matches!(
2729 client.send_bytes(vec![2]).await,
2730 Err(SendError::BufferFull)
2731 ));
2732 let (command, permit) = writer_rx.recv().await.unwrap();
2733 assert!(matches!(command, WriterCommand::Send(data) if data == vec![1]));
2734 assert!(matches!(
2735 client.send_bytes(vec![2]).await,
2736 Err(SendError::BufferFull)
2737 ));
2738 drop(permit);
2739 client.send_bytes(vec![3]).await.unwrap();
2740 let (command, _) = writer_rx.recv().await.unwrap();
2741 assert!(matches!(command, WriterCommand::Send(data) if data == vec![3]));
2742 }
2743
2744 #[rstest]
2745 #[tokio::test]
2746 async fn test_reconnect_handle_is_closed_after_client_drop() {
2747 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2748 let state_notify = Arc::new(tokio::sync::Notify::new());
2749
2750 let controller_task = tokio::spawn(std::future::pending::<()>());
2751 let client = test_socket_client(connection_state, state_notify, controller_task);
2752 let handle = client.reconnect_handle();
2753
2754 drop(client);
2755
2756 assert_eq!(handle.request_reconnect(), ReconnectRequestOutcome::Closed);
2757 }
2758
2759 #[rstest]
2760 #[tokio::test]
2761 async fn test_concurrent_drop_defers_controller_abort_until_request_completes() {
2762 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2763 let state_notify = Arc::new(tokio::sync::Notify::new());
2764
2765 let controller_task = tokio::spawn(std::future::pending::<()>());
2766 let controller_abort = controller_task.abort_handle();
2767 let mut client = test_socket_client(connection_state, state_notify, controller_task);
2768 let callback_release = Arc::new((Mutex::new(false), Condvar::new()));
2769 let callback_release_guard = CondvarReleaseGuard::new(Arc::clone(&callback_release));
2770 let callback_release_clone = Arc::clone(&callback_release);
2771 let (callback_entered_tx, callback_entered_rx) = std::sync::mpsc::channel();
2772 let states = Arc::new(Mutex::new(Vec::new()));
2773 let states_clone = Arc::clone(&states);
2774 client.state_sink = Some(SocketStateSink::new(move |state| {
2775 states_clone.lock().push(state);
2776 callback_entered_tx.send(()).unwrap();
2777 let (lock, condvar) = callback_release_clone.as_ref();
2778 let mut released = lock.lock();
2779 while !*released {
2780 condvar.wait(&mut released);
2781 }
2782 }));
2783
2784 let handle = client.reconnect_handle();
2785 let surviving_handle = handle.clone();
2786 let controller_notify = Arc::clone(&handle.controller_notify);
2787 let request_thread = std::thread::spawn(move || handle.request_reconnect());
2788
2789 recv_rendezvous(callback_entered_rx, "socket state callback entry").await;
2790 let (drop_finished_tx, drop_finished_rx) = std::sync::mpsc::channel();
2791
2792 let drop_thread = std::thread::spawn(move || {
2793 drop(client);
2794 drop_finished_tx.send(()).unwrap();
2795 });
2796
2797 recv_rendezvous(drop_finished_rx, "client drop").await;
2798 drop_thread.join().unwrap();
2799 assert!(!controller_abort.is_finished());
2800
2801 callback_release_guard.release();
2802 assert_eq!(
2803 request_thread.join().unwrap(),
2804 ReconnectRequestOutcome::Accepted
2805 );
2806 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2807 .await
2808 .expect("accepted request should notify before deferred controller abort");
2809 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
2810
2811 assert_eq!(
2812 surviving_handle.request_reconnect(),
2813 ReconnectRequestOutcome::Closed
2814 );
2815 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2816 assert!(
2817 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2818 .await
2819 .is_err(),
2820 "closed request should not notify controller",
2821 );
2822 }
2823
2824 #[rstest]
2825 #[tokio::test]
2826 async fn test_reconnect_state_callback_can_drop_client() {
2827 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2828 let state_notify = Arc::new(tokio::sync::Notify::new());
2829
2830 let controller_task = tokio::spawn(std::future::pending::<()>());
2831 let controller_abort = controller_task.abort_handle();
2832 let mut client = test_socket_client(connection_state, state_notify, controller_task);
2833 let client_slot = Arc::new(Mutex::new(None));
2834 let client_slot_callback = Arc::clone(&client_slot);
2835 let states = Arc::new(Mutex::new(Vec::new()));
2836 let states_callback = Arc::clone(&states);
2837 client.state_sink = Some(SocketStateSink::new(move |state| {
2838 states_callback.lock().push(state);
2839 drop(client_slot_callback.lock().take());
2840 }));
2841
2842 let handle = client.reconnect_handle();
2843 let controller_notify = Arc::clone(&handle.controller_notify);
2844 *client_slot.lock() = Some(client);
2845 let (result_tx, result_rx) = std::sync::mpsc::channel();
2846 std::thread::spawn(move || result_tx.send(handle.request_reconnect()).unwrap());
2847
2848 assert_eq!(
2849 recv_rendezvous(result_rx, "reconnect request").await,
2850 ReconnectRequestOutcome::Accepted
2851 );
2852 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
2853 .await
2854 .expect("accepted request should notify before deferred controller abort");
2855 wait_until_async(|| async { controller_abort.is_finished() }, TEST_TIMEOUT).await;
2856
2857 assert!(client_slot.lock().is_none());
2858 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
2859 }
2860
2861 #[rstest]
2862 #[tokio::test]
2863 async fn test_manual_reconnect_uses_controller_path() {
2864 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2865 let port = listener.local_addr().unwrap().port();
2866 let (first_accepted_tx, first_accepted_rx) = oneshot::channel();
2867 let (second_accepted_tx, second_accepted_rx) = oneshot::channel();
2868 let (payload_tx, payload_rx) = oneshot::channel();
2869 let (release_tx, release_rx) = oneshot::channel();
2870
2871 let server = task::spawn(async move {
2872 let (_first, _) = listener.accept().await.unwrap();
2873 first_accepted_tx.send(()).unwrap();
2874
2875 let (mut second, _) = listener.accept().await.unwrap();
2876 second_accepted_tx.send(()).unwrap();
2877 let mut payload = [0_u8; 8];
2878 second.read_exact(&mut payload).await.unwrap();
2879 payload_tx.send(payload).unwrap();
2880 let _ = release_rx.await;
2881 });
2882
2883 let states = Arc::new(Mutex::new(Vec::new()));
2884 let states_callback = Arc::clone(&states);
2885 let sink = SocketStateSink::new(move |state| {
2886 states_callback.lock().push(state);
2887 });
2888
2889 let callback_count = Arc::new(AtomicUsize::new(0));
2890 let callback_count_clone = Arc::clone(&callback_count);
2891 let post_reconnection = Arc::new(move || {
2892 callback_count_clone.fetch_add(1, Ordering::SeqCst);
2893 });
2894
2895 let client = SocketClient::builder()
2896 .config(reconnect_test_config(port))
2897 .post_reconnection(post_reconnection)
2898 .state_sink(sink)
2899 .connect()
2900 .await
2901 .unwrap();
2902 first_accepted_rx.await.unwrap();
2903 let handle = client.reconnect_handle();
2904
2905 assert!(client.request_reconnect());
2906 assert_eq!(
2907 handle.request_reconnect(),
2908 ReconnectRequestOutcome::AlreadyReconnecting
2909 );
2910 assert!(!client.request_reconnect());
2911 assert_eq!(
2912 *states.lock(),
2913 vec![SocketState::Connected, SocketState::Disconnected]
2914 );
2915
2916 tokio::time::timeout(TEST_TIMEOUT, second_accepted_rx)
2917 .await
2918 .expect("controller should establish a replacement connection")
2919 .unwrap();
2920 wait_until_async(
2921 || async { client.is_active() && callback_count.load(Ordering::SeqCst) == 1 },
2922 TEST_TIMEOUT,
2923 )
2924 .await;
2925 client.send_bytes(b"manual".to_vec()).await.unwrap();
2926
2927 assert_eq!(
2928 tokio::time::timeout(TEST_TIMEOUT, payload_rx)
2929 .await
2930 .expect("replacement connection should receive the framed payload")
2931 .unwrap(),
2932 *b"manual\r\n"
2933 );
2934 assert_eq!(callback_count.load(Ordering::SeqCst), 1);
2935 assert_eq!(
2936 *states.lock(),
2937 vec![
2938 SocketState::Connected,
2939 SocketState::Disconnected,
2940 SocketState::Connected,
2941 ]
2942 );
2943
2944 client.close().await;
2945 release_tx.send(()).unwrap();
2946 server.await.unwrap();
2947 assert_eq!(callback_count.load(Ordering::SeqCst), 1);
2948 assert_eq!(states.lock().len(), 3);
2949 }
2950
2951 #[rstest]
2952 #[case(ConnectionMode::Disconnect)]
2953 #[case(ConnectionMode::Closed)]
2954 #[tokio::test]
2955 async fn test_send_bytes_rejects_terminal_state(#[case] mode: ConnectionMode) {
2956 let connection_state = Arc::new(AtomicU8::new(mode.as_u8()));
2957 let state_notify = Arc::new(tokio::sync::Notify::new());
2958
2959 let controller_task = tokio::spawn(std::future::pending::<()>());
2960 let client = test_socket_client(connection_state, state_notify, controller_task);
2961
2962 let result = client.send_bytes(b"terminal".to_vec()).await;
2963
2964 assert!(matches!(result, Err(SendError::Closed)));
2965 }
2966
2967 #[rstest]
2968 #[tokio::test(start_paused = true)]
2969 async fn test_close_sets_closed_after_controller_was_aborted() {
2970 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
2971 let state_notify = Arc::new(tokio::sync::Notify::new());
2972
2973 let controller_task = tokio::spawn(std::future::pending::<()>());
2974 controller_task.abort();
2975 let client = test_socket_client(connection_state, state_notify, controller_task);
2976
2977 client.close_with_timeout(Duration::from_millis(1)).await;
2978
2979 assert!(client.is_closed());
2980 }
2981
2982 #[rstest]
2983 #[tokio::test]
2984 async fn test_reconnect_exhaustion_closes_client() {
2985 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2986 let port = listener.local_addr().unwrap().port();
2987 let (accepted_tx, accepted_rx) = oneshot::channel();
2988 let (release_tx, release_rx) = oneshot::channel();
2989
2990 let server = task::spawn(async move {
2991 let (socket, _) = listener.accept().await.unwrap();
2992 let _ = accepted_tx.send(());
2993 let _ = release_rx.await;
2994 drop(socket);
2995 drop(listener);
2996 });
2997
2998 let config = SocketConfig {
2999 url: format!("127.0.0.1:{port}"),
3000 mode: Mode::Plain,
3001 suffix: b"\r\n".to_vec(),
3002 message_handler: None,
3003 heartbeat: None,
3004 connect_timeout_ms: Some(100),
3005 reconnect_delay_initial_ms: Some(1),
3006 reconnect_delay_max_ms: Some(1),
3007 reconnect_backoff_factor: Some(1.0),
3008 reconnect_jitter_ms: Some(0),
3009 connection_max_retries: Some(1),
3010 reconnect_max_attempts: Some(1),
3011 heartbeat_timeout_secs: None,
3012 writer_capacity: None,
3013 certs_dir: None,
3014 };
3015
3016 let states = Arc::new(Mutex::new(Vec::new()));
3017 let states_callback = Arc::clone(&states);
3018 let sink = SocketStateSink::new(move |state| {
3019 states_callback.lock().push(state);
3020 });
3021
3022 let client = SocketClient::builder()
3023 .config(config)
3024 .state_sink(sink)
3025 .connect()
3026 .await
3027 .unwrap();
3028 accepted_rx.await.unwrap();
3029 release_tx.send(()).unwrap();
3030
3031 wait_until_async(|| async { client.is_closed() }, Duration::from_secs(5)).await;
3032 assert!(client.is_closed());
3033 assert_eq!(
3034 *states.lock(),
3035 vec![SocketState::Connected, SocketState::Disconnected]
3036 );
3037
3038 client.close().await;
3039 assert!(client.is_closed());
3040 assert_eq!(states.lock().len(), 2);
3041 server.await.unwrap();
3042 }
3043
3044 #[rstest]
3045 #[tokio::test]
3046 async fn test_drop_suppresses_socket_state_event() {
3047 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3048 let port = listener.local_addr().unwrap().port();
3049 let (accepted_tx, accepted_rx) = oneshot::channel();
3050
3051 let server = task::spawn(async move {
3052 let (_socket, _) = listener.accept().await.unwrap();
3053 let _ = accepted_tx.send(());
3054 std::future::pending::<()>().await;
3055 });
3056
3057 let config = SocketConfig {
3058 url: format!("127.0.0.1:{port}"),
3059 mode: Mode::Plain,
3060 suffix: b"\r\n".to_vec(),
3061 message_handler: None,
3062 heartbeat: None,
3063 connect_timeout_ms: Some(100),
3064 reconnect_delay_initial_ms: Some(1),
3065 reconnect_delay_max_ms: Some(1),
3066 reconnect_backoff_factor: Some(1.0),
3067 reconnect_jitter_ms: Some(0),
3068 connection_max_retries: Some(1),
3069 reconnect_max_attempts: Some(1),
3070 heartbeat_timeout_secs: None,
3071 writer_capacity: None,
3072 certs_dir: None,
3073 };
3074
3075 let states = Arc::new(Mutex::new(Vec::new()));
3076 let states_callback = Arc::clone(&states);
3077 let sink = SocketStateSink::new(move |state| {
3078 states_callback.lock().push(state);
3079 });
3080
3081 let client = SocketClient::builder()
3082 .config(config)
3083 .state_sink(sink)
3084 .connect()
3085 .await
3086 .unwrap();
3087 accepted_rx.await.unwrap();
3088
3089 drop(client);
3090 sleep(Duration::from_millis(25)).await;
3091
3092 assert_eq!(*states.lock(), vec![SocketState::Connected]);
3093 server.abort();
3094 }
3095
3096 #[rstest]
3097 #[tokio::test]
3098 async fn test_graceful_close_is_idempotent() {
3099 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3100 let port = listener.local_addr().unwrap().port();
3101 let (accepted_tx, accepted_rx) = oneshot::channel();
3102
3103 let server = task::spawn(async move {
3104 let (_socket, _) = listener.accept().await.unwrap();
3105 let _ = accepted_tx.send(());
3106 std::future::pending::<()>().await;
3107 });
3108
3109 let config = SocketConfig {
3110 url: format!("127.0.0.1:{port}"),
3111 mode: Mode::Plain,
3112 suffix: b"\r\n".to_vec(),
3113 message_handler: None,
3114 heartbeat: None,
3115 connect_timeout_ms: None,
3116 reconnect_delay_initial_ms: None,
3117 reconnect_delay_max_ms: None,
3118 reconnect_backoff_factor: None,
3119 reconnect_jitter_ms: None,
3120 connection_max_retries: None,
3121 reconnect_max_attempts: None,
3122 heartbeat_timeout_secs: None,
3123 writer_capacity: None,
3124 certs_dir: None,
3125 };
3126
3127 let client = SocketClient::builder()
3128 .config(config)
3129 .connect()
3130 .await
3131 .unwrap();
3132 accepted_rx.await.unwrap();
3133
3134 client.close().await;
3135 client.close().await;
3136
3137 assert!(client.is_closed());
3138 server.abort();
3139 }
3140
3141 #[rstest]
3142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3143 async fn test_read_loop_drops_remaining_frames_after_session_replaced() {
3144 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3145 let read_fence = ReadSessionFence::new();
3146 let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3147 let received_clone = Arc::clone(&received);
3148 let handler_release = Arc::new((Mutex::new(false), Condvar::new()));
3149 let handler_release_guard = CondvarReleaseGuard::new(Arc::clone(&handler_release));
3150 let handler_release_clone = Arc::clone(&handler_release);
3151 let (handler_entered_tx, handler_entered_rx) = std::sync::mpsc::channel();
3152
3153 let handler: TcpMessageHandler = Arc::new(move |data| {
3154 received_clone.lock().push(data.to_vec());
3155 handler_entered_tx.send(()).unwrap();
3156 let (lock, condvar) = handler_release_clone.as_ref();
3157 let mut released = lock.lock();
3158 while !*released {
3159 condvar.wait(&mut released);
3160 }
3161 });
3162
3163 let reader = ScriptedReader {
3164 first: Some(b"first\r\nsecond\r\n".to_vec()),
3165 remainder: Arc::new(Mutex::new(None)),
3166 pending_tx: None,
3167 waker: Arc::new(Mutex::new(None)),
3168 };
3169
3170 let read_task = SocketClientInner::spawn_read_task(
3171 Arc::clone(&connection_state),
3172 read_fence.clone(),
3173 reader,
3174 Some(handler),
3175 b"\r\n".to_vec(),
3176 None,
3177 );
3178
3179 recv_rendezvous(handler_entered_rx, "socket first-handler entry").await;
3180 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3181 read_fence.invalidate();
3182 read_task.abort();
3183 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3184
3185 handler_release_guard.release();
3186 await_task_termination(read_task, "old socket read task", true).await;
3187
3188 assert_eq!(received.lock().as_slice(), &[b"first".to_vec()]);
3189 }
3190
3191 #[rstest]
3192 #[tokio::test(start_paused = true)]
3193 async fn test_read_loop_drops_partial_old_session_frame() {
3194 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3195 let old_read_fence = ReadSessionFence::new();
3196 let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3197 let received_clone = Arc::clone(&received);
3198 let handler: TcpMessageHandler =
3199 Arc::new(move |data| received_clone.lock().push(data.to_vec()));
3200 let (pending_tx, pending_rx) = oneshot::channel();
3201 let remainder = Arc::new(Mutex::new(None));
3202 let waker = Arc::new(Mutex::new(None));
3203
3204 let reader = ScriptedReader {
3205 first: Some(b"old".to_vec()),
3206 remainder: Arc::clone(&remainder),
3207 pending_tx: Some(pending_tx),
3208 waker: Arc::clone(&waker),
3209 };
3210
3211 let read_task = SocketClientInner::spawn_read_task(
3212 Arc::clone(&connection_state),
3213 old_read_fence.clone(),
3214 reader,
3215 Some(handler),
3216 b"\r\n".to_vec(),
3217 None,
3218 );
3219
3220 pending_rx.await.unwrap();
3221 connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3222 old_read_fence.invalidate();
3223 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3224 *remainder.lock() = Some(b"\r\nnew\r\n".to_vec());
3225 waker.lock().take().unwrap().wake();
3226 read_task.await.unwrap();
3227
3228 let fresh_reader = ScriptedReader {
3229 first: Some(b"new\r\n".to_vec()),
3230 remainder: Arc::new(Mutex::new(None)),
3231 pending_tx: None,
3232 waker: Arc::new(Mutex::new(None)),
3233 };
3234
3235 let fresh_connection_state = Arc::clone(&connection_state);
3236 let fresh_received = Arc::clone(&received);
3237 let fresh_handler: TcpMessageHandler = Arc::new(move |data| {
3238 fresh_received.lock().push(data.to_vec());
3239 fresh_connection_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3240 });
3241
3242 SocketClientInner::spawn_read_task(
3243 Arc::clone(&connection_state),
3244 ReadSessionFence::new(),
3245 fresh_reader,
3246 Some(fresh_handler),
3247 b"\r\n".to_vec(),
3248 None,
3249 )
3250 .await
3251 .unwrap();
3252
3253 assert_eq!(received.lock().as_slice(), &[b"new".to_vec()]);
3254 }
3255
3256 #[rstest]
3257 #[tokio::test(start_paused = true)]
3258 async fn test_read_loop_stops_when_first_handler_ends_session() {
3259 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3260 let handler_state = Arc::clone(&connection_state);
3261 let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3262 let received_clone = Arc::clone(&received);
3263 let handler: TcpMessageHandler = Arc::new(move |data| {
3264 received_clone.lock().push(data.to_vec());
3265 handler_state.store(ConnectionMode::Reconnect.as_u8(), Ordering::SeqCst);
3266 });
3267
3268 let (pending_tx, _pending_rx) = oneshot::channel();
3269
3270 let reader = ScriptedReader {
3271 first: Some(b"first\r\nsecond\r\n".to_vec()),
3272 remainder: Arc::new(Mutex::new(None)),
3273 pending_tx: Some(pending_tx),
3274 waker: Arc::new(Mutex::new(None)),
3275 };
3276
3277 SocketClientInner::spawn_read_task(
3278 connection_state,
3279 ReadSessionFence::new(),
3280 reader,
3281 Some(handler),
3282 b"\r\n".to_vec(),
3283 None,
3284 )
3285 .await
3286 .unwrap();
3287
3288 assert_eq!(received.lock().as_slice(), &[b"first".to_vec()]);
3289 }
3290
3291 #[tokio::test]
3292 async fn test_read_loop_frames_suffix_split_across_reads() {
3293 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3294 let received = Arc::new(Mutex::new(Vec::<Vec<u8>>::new()));
3295 let received_clone = Arc::clone(&received);
3296 let handler: TcpMessageHandler =
3297 Arc::new(move |data| received_clone.lock().push(data.to_vec()));
3298
3299 let reader = ChunkReader {
3301 chunks: VecDeque::from([&b"fir"[..], b"st\r", b"\nsecond\r\n\r", b"\n"]),
3302 };
3303
3304 SocketClientInner::run_read_loop(
3305 connection_state,
3306 ReadSessionFence::new(),
3307 reader,
3308 Some(handler),
3309 b"\r\n".to_vec(),
3310 None,
3311 Duration::from_secs(1),
3312 )
3313 .await;
3314
3315 assert_eq!(
3316 received.lock().as_slice(),
3317 &[b"first".to_vec(), b"second".to_vec(), Vec::new()]
3318 );
3319 }
3320
3321 #[tokio::test]
3322 async fn test_read_loop_closes_when_unframed_buffer_exceeds_limit() {
3323 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3324 let read_after_limit = Arc::new(AtomicBool::new(false));
3325
3326 let reader = LimitProbeReader {
3327 remaining: MAX_READ_BUFFER_BYTES + 1,
3328 read_after_limit: Arc::clone(&read_after_limit),
3329 };
3330
3331 SocketClientInner::run_read_loop(
3332 connection_state,
3333 ReadSessionFence::new(),
3334 reader,
3335 None,
3336 b"\r\n".to_vec(),
3337 None,
3338 Duration::from_secs(1),
3339 )
3340 .await;
3341
3342 assert!(!read_after_limit.load(Ordering::SeqCst));
3343 }
3344
3345 #[rstest]
3346 #[tokio::test(start_paused = true)]
3347 async fn test_stalled_socket_write_sends_reconnect_replay_before_buffer() {
3348 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3349
3350 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3351 let state_notify = Arc::new(tokio::sync::Notify::new());
3352 let (stream, _non_reading_peer) = tokio::io::duplex(1);
3353 let (pending_tx, pending_rx) = oneshot::channel();
3354
3355 let writer: TestWriter = Box::pin(BackpressuredWriter {
3356 stream,
3357 pending_tx: Some(pending_tx),
3358 });
3359
3360 let (writer_tx, writer_rx) = writer::channel::<WriterCommand<TestWriter>>(2);
3361 let states = Arc::new(Mutex::new(Vec::new()));
3362 let states_callback = Arc::clone(&states);
3363 let sink = SocketStateSink::new(move |state| {
3364 states_callback.lock().push(state);
3365 });
3366
3367 let write_task = SocketClientInner::spawn_write_task(
3368 Arc::clone(&connection_state),
3369 Arc::clone(&state_notify),
3370 writer,
3371 writer_rx,
3372 b"\r\n".to_vec(),
3373 Some(sink),
3374 );
3375
3376 writer_tx
3377 .send(WriterCommand::Send(Bytes::from_static(b"complete-message")))
3378 .unwrap();
3379 pending_rx.await.unwrap();
3380 writer_tx
3381 .send(WriterCommand::Send(Bytes::from_static(b"second-message")))
3382 .unwrap();
3383 assert_eq!(
3384 ConnectionMode::from_atomic(&connection_state),
3385 ConnectionMode::Active
3386 );
3387
3388 for _ in 0..100 {
3389 assert!(matches!(
3390 writer_tx.send(WriterCommand::Send(Bytes::from_static(b"overflow"))),
3391 Err(SendError::BufferFull)
3392 ));
3393 }
3394
3395 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
3396 tokio::task::yield_now().await;
3397 assert_eq!(
3398 ConnectionMode::from_atomic(&connection_state),
3399 ConnectionMode::Reconnect
3400 );
3401 assert!(matches!(
3402 writer_tx.send(WriterCommand::Send(Bytes::from_static(b"overflow"))),
3403 Err(SendError::BufferFull)
3404 ));
3405
3406 let recorded = Arc::new(Mutex::new(Vec::new()));
3407
3408 let new_writer: TestWriter = Box::pin(RecordingWriter {
3409 bytes: Arc::clone(&recorded),
3410 });
3411
3412 let (update_tx, update_rx) = oneshot::channel();
3413 writer_tx
3414 .send_update(WriterCommand::UpdateWithReplay(
3415 new_writer,
3416 vec![Bytes::from_static(b"authentication")],
3417 update_tx,
3418 ))
3419 .unwrap();
3420
3421 tokio::time::advance(Duration::from_secs(GRACEFUL_SHUTDOWN_TIMEOUT_SECS)).await;
3422 yield_now().await;
3423
3424 assert!(
3425 tokio::time::timeout(Duration::from_secs(10), update_rx)
3426 .await
3427 .expect("writer update should not remain queued behind a stalled write")
3428 .unwrap(),
3429 "writer should report successful replay"
3430 );
3431 assert_eq!(
3432 ConnectionMode::from_atomic(&connection_state),
3433 ConnectionMode::Reconnect
3434 );
3435 assert_eq!(
3436 recorded.lock().as_slice(),
3437 b"authentication\r\ncomplete-message\r\nsecond-message\r\n"
3438 );
3439
3440 writer_tx
3441 .send(WriterCommand::Send(Bytes::from_static(b"second-message")))
3442 .unwrap();
3443
3444 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3445 state_notify.notify_waiters();
3446 drop(writer_tx);
3447 write_task.await.unwrap();
3448
3449 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
3450 }
3451
3452 #[rstest]
3453 #[tokio::test(start_paused = true)]
3454 async fn test_send_after_writer_update_drains_when_reconnect_completes() {
3455 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3456
3457 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3458 let state_notify = Arc::new(tokio::sync::Notify::new());
3459
3460 let initial_writer: TestWriter = Box::pin(RecordingWriter {
3461 bytes: Arc::new(Mutex::new(Vec::new())),
3462 });
3463
3464 let (writer_tx, writer_rx) =
3465 writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3466 let write_task = SocketClientInner::spawn_write_task(
3467 Arc::clone(&connection_state),
3468 Arc::clone(&state_notify),
3469 initial_writer,
3470 writer_rx,
3471 b"\r\n".to_vec(),
3472 None,
3473 );
3474
3475 let recorded = Arc::new(Mutex::new(Vec::new()));
3476
3477 let new_writer: TestWriter = Box::pin(RecordingWriter {
3478 bytes: Arc::clone(&recorded),
3479 });
3480
3481 let (update_tx, update_rx) = oneshot::channel();
3482 writer_tx
3483 .send(WriterCommand::Update(new_writer, update_tx))
3484 .unwrap();
3485 assert!(update_rx.await.unwrap());
3486
3487 writer_tx
3488 .send(WriterCommand::Send(Bytes::from_static(b"late")))
3489 .unwrap();
3490 yield_now().await;
3491
3492 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3493 state_notify.notify_waiters();
3494 writer_tx
3495 .send(WriterCommand::Send(Bytes::from_static(b"new")))
3496 .unwrap();
3497 tokio::time::advance(Duration::from_millis(
3498 CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3499 ))
3500 .await;
3501 yield_now().await;
3502
3503 let actual = recorded.lock().clone();
3504 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3505 state_notify.notify_waiters();
3506 drop(writer_tx);
3507 write_task.await.unwrap();
3508
3509 assert_eq!(actual, b"late\r\nnew\r\n");
3510 }
3511
3512 #[rstest]
3513 #[case::latest_wins(
3514 &["subscription-a", "subscription-b"],
3515 &["subscription-b"],
3516 b"subscription-b\r\n",
3517 )]
3518 #[case::different_replay_drains(
3519 &["subscription-b"],
3520 &["subscription-a"],
3521 b"subscription-a\r\nsubscription-b\r\n",
3522 )]
3523 #[tokio::test]
3524 async fn test_send_or_replay_buffering(
3525 #[case] buffered: &[&str],
3526 #[case] replay: &[&str],
3527 #[case] expected: &[u8],
3528 ) {
3529 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3530
3531 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3532 let state_notify = Arc::new(tokio::sync::Notify::new());
3533
3534 let initial_writer: TestWriter = Box::pin(RecordingWriter {
3535 bytes: Arc::new(Mutex::new(Vec::new())),
3536 });
3537
3538 let (writer_tx, writer_rx) =
3539 writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3540 let write_task = SocketClientInner::spawn_write_task(
3541 Arc::clone(&connection_state),
3542 Arc::clone(&state_notify),
3543 initial_writer,
3544 writer_rx,
3545 b"\r\n".to_vec(),
3546 None,
3547 );
3548
3549 for data in buffered {
3550 writer_tx
3551 .send(WriterCommand::SendOrReplay {
3552 key: 7,
3553 data: Bytes::copy_from_slice(data.as_bytes()),
3554 })
3555 .unwrap();
3556 }
3557
3558 let recorded = Arc::new(Mutex::new(Vec::new()));
3559
3560 let new_writer: TestWriter = Box::pin(RecordingWriter {
3561 bytes: Arc::clone(&recorded),
3562 });
3563
3564 let (update_tx, update_rx) = oneshot::channel();
3565 writer_tx
3566 .send(WriterCommand::UpdateWithReplay(
3567 new_writer,
3568 replay
3569 .iter()
3570 .map(|data| Bytes::copy_from_slice(data.as_bytes()))
3571 .collect(),
3572 update_tx,
3573 ))
3574 .unwrap();
3575
3576 assert!(update_rx.await.unwrap());
3577 assert_eq!(recorded.lock().as_slice(), expected);
3578 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3579 state_notify.notify_waiters();
3580 drop(writer_tx);
3581 write_task.await.unwrap();
3582 }
3583
3584 #[rstest]
3585 #[tokio::test(start_paused = true)]
3586 async fn test_send_or_replay_after_reconnect_supersedes_buffered_command() {
3587 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3588
3589 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3590 let state_notify = Arc::new(tokio::sync::Notify::new());
3591 let recorded = Arc::new(Mutex::new(Vec::new()));
3592
3593 let writer: TestWriter = Box::pin(RecordingWriter {
3594 bytes: Arc::clone(&recorded),
3595 });
3596
3597 let (writer_tx, writer_rx) =
3598 writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3599 let write_task = SocketClientInner::spawn_write_task(
3600 Arc::clone(&connection_state),
3601 Arc::clone(&state_notify),
3602 writer,
3603 writer_rx,
3604 b"\r\n".to_vec(),
3605 None,
3606 );
3607
3608 writer_tx
3609 .send(WriterCommand::SendOrReplay {
3610 key: 1,
3611 data: Bytes::from_static(b"subscription-old"),
3612 })
3613 .unwrap();
3614
3615 yield_now().await;
3616
3617 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3618 writer_tx
3619 .send(WriterCommand::SendOrReplay {
3620 key: 1,
3621 data: Bytes::from_static(b"subscription-new"),
3622 })
3623 .unwrap();
3624
3625 tokio::time::advance(Duration::from_millis(
3626 CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3627 ))
3628 .await;
3629 yield_now().await;
3630
3631 let actual = recorded.lock().clone();
3632 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3633 state_notify.notify_waiters();
3634 drop(writer_tx);
3635 write_task.await.unwrap();
3636
3637 assert_eq!(actual, b"subscription-new\r\n");
3638 }
3639
3640 #[rstest]
3641 #[tokio::test(start_paused = true)]
3642 async fn test_active_reconnect_buffer_write_failure_reconnects_and_retries() {
3643 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3644
3645 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Reconnect.as_u8()));
3646 let state_notify = Arc::new(tokio::sync::Notify::new());
3647
3648 let initial_writer: TestWriter = Box::pin(RecordingWriter {
3649 bytes: Arc::new(Mutex::new(Vec::new())),
3650 });
3651
3652 let (writer_tx, writer_rx) =
3653 writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3654 let states = Arc::new(Mutex::new(Vec::new()));
3655 let states_callback = Arc::clone(&states);
3656 let sink = SocketStateSink::new(move |state| {
3657 states_callback.lock().push(state);
3658 });
3659
3660 let write_task = SocketClientInner::spawn_write_task(
3661 Arc::clone(&connection_state),
3662 Arc::clone(&state_notify),
3663 initial_writer,
3664 writer_rx,
3665 b"\r\n".to_vec(),
3666 Some(sink),
3667 );
3668
3669 let (update_tx, update_rx) = oneshot::channel();
3670 let failing_writer: TestWriter = Box::pin(FailingWriter);
3671 writer_tx
3672 .send(WriterCommand::Update(failing_writer, update_tx))
3673 .unwrap();
3674 assert!(update_rx.await.unwrap());
3675
3676 writer_tx
3677 .send(WriterCommand::Send(Bytes::from_static(b"late")))
3678 .unwrap();
3679 yield_now().await;
3680 connection_state.store(ConnectionMode::Active.as_u8(), Ordering::SeqCst);
3681 state_notify.notify_waiters();
3682 tokio::time::advance(Duration::from_millis(
3683 CONNECTION_STATE_CHECK_INTERVAL_MS * 2,
3684 ))
3685 .await;
3686 yield_now().await;
3687
3688 assert_eq!(
3689 ConnectionMode::from_atomic(&connection_state),
3690 ConnectionMode::Reconnect
3691 );
3692 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
3693
3694 let recorded = Arc::new(Mutex::new(Vec::new()));
3695
3696 let replacement: TestWriter = Box::pin(RecordingWriter {
3697 bytes: Arc::clone(&recorded),
3698 });
3699
3700 let (retry_tx, retry_rx) = oneshot::channel();
3701 writer_tx
3702 .send(WriterCommand::Update(replacement, retry_tx))
3703 .unwrap();
3704 assert!(retry_rx.await.unwrap());
3705
3706 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3707 state_notify.notify_waiters();
3708 drop(writer_tx);
3709 write_task.await.unwrap();
3710
3711 assert_eq!(recorded.lock().as_slice(), b"late\r\n");
3712 }
3713
3714 #[rstest]
3715 #[tokio::test(start_paused = true)]
3716 async fn test_failed_send_or_replay_is_not_duplicated_after_replay() {
3717 type TestWriter = Pin<Box<dyn AsyncWrite + Send>>;
3718
3719 let connection_state = Arc::new(AtomicU8::new(ConnectionMode::Active.as_u8()));
3720 let state_notify = Arc::new(tokio::sync::Notify::new());
3721 let (stream, _non_reading_peer) = tokio::io::duplex(1);
3722 let (pending_tx, pending_rx) = oneshot::channel();
3723
3724 let writer: TestWriter = Box::pin(BackpressuredWriter {
3725 stream,
3726 pending_tx: Some(pending_tx),
3727 });
3728
3729 let (writer_tx, writer_rx) =
3730 writer::channel::<WriterCommand<TestWriter>>(DEFAULT_WRITER_CAPACITY);
3731 let write_task = SocketClientInner::spawn_write_task(
3732 Arc::clone(&connection_state),
3733 Arc::clone(&state_notify),
3734 writer,
3735 writer_rx,
3736 b"\r\n".to_vec(),
3737 None,
3738 );
3739 let subscription = Bytes::from_static(b"subscription");
3740
3741 writer_tx
3742 .send(WriterCommand::SendOrReplay {
3743 key: 7,
3744 data: subscription.clone(),
3745 })
3746 .unwrap();
3747
3748 pending_rx.await.unwrap();
3749
3750 let recorded = Arc::new(Mutex::new(Vec::new()));
3751
3752 let new_writer: TestWriter = Box::pin(RecordingWriter {
3753 bytes: Arc::clone(&recorded),
3754 });
3755
3756 let (update_tx, update_rx) = oneshot::channel();
3757 writer_tx
3758 .send(WriterCommand::UpdateWithReplay(
3759 new_writer,
3760 vec![subscription],
3761 update_tx,
3762 ))
3763 .unwrap();
3764
3765 tokio::time::advance(Duration::from_secs(WRITE_TIMEOUT_SECS)).await;
3766 yield_now().await;
3767 assert!(update_rx.await.unwrap());
3768 assert_eq!(recorded.lock().as_slice(), b"subscription\r\n");
3769
3770 connection_state.store(ConnectionMode::Closed.as_u8(), Ordering::SeqCst);
3771 state_notify.notify_waiters();
3772 drop(writer_tx);
3773 write_task.await.unwrap();
3774 }
3775
3776 #[rstest]
3777 #[tokio::test]
3778 async fn test_connect_url_rejects_invalid_reconnect_backoff_before_connect() {
3779 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3780 listener.set_nonblocking(true).unwrap();
3781 let port = listener.local_addr().unwrap().port();
3782
3783 let config = SocketConfig {
3784 url: format!("127.0.0.1:{port}"),
3785 mode: Mode::Plain,
3786 suffix: b"\r\n".to_vec(),
3787 message_handler: None,
3788 heartbeat: None,
3789 connect_timeout_ms: Some(1_000),
3790 reconnect_delay_initial_ms: Some(50),
3791 reconnect_delay_max_ms: Some(100),
3792 reconnect_backoff_factor: Some(100.1),
3793 reconnect_jitter_ms: Some(0),
3794 connection_max_retries: Some(1),
3795 reconnect_max_attempts: None,
3796 heartbeat_timeout_secs: None,
3797 writer_capacity: None,
3798 certs_dir: None,
3799 };
3800
3801 let error = match SocketClientInner::connect_url(config, None).await {
3802 Ok(_) => panic!("invalid reconnect backoff should be rejected"),
3803 Err(e) => e,
3804 };
3805
3806 assert!(
3807 error.to_string().contains("factor"),
3808 "error should mention the invalid factor, was: {error}"
3809 );
3810 assert_eq!(
3811 listener.accept().unwrap_err().kind(),
3812 std::io::ErrorKind::WouldBlock,
3813 "invalid reconnect backoff must be rejected before connecting"
3814 );
3815 }
3816
3817 #[rstest]
3818 #[tokio::test]
3819 async fn test_reconnect_then_close() {
3820 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3822 let port = listener.local_addr().unwrap().port();
3823
3824 let server = task::spawn(async move {
3826 if let Ok((mut sock, _)) = listener.accept().await {
3827 drop(sock.shutdown());
3828 }
3829
3830 sleep(Duration::from_secs(1)).await;
3832 });
3833
3834 let config = SocketConfig {
3836 url: format!("127.0.0.1:{port}"),
3837 mode: Mode::Plain,
3838 suffix: b"\r\n".to_vec(),
3839 message_handler: None,
3840 heartbeat: None,
3841 connect_timeout_ms: Some(1_000),
3842 reconnect_delay_initial_ms: Some(50),
3843 reconnect_delay_max_ms: Some(100),
3844 reconnect_backoff_factor: Some(1.0),
3845 reconnect_jitter_ms: Some(0),
3846 connection_max_retries: Some(1),
3847 reconnect_max_attempts: None,
3848 heartbeat_timeout_secs: None,
3849 writer_capacity: None,
3850 certs_dir: None,
3851 };
3852
3853 let client = SocketClient::builder()
3855 .config(config.clone())
3856 .connect()
3857 .await
3858 .unwrap();
3859
3860 wait_until_async(
3862 || async { client.is_reconnecting() },
3863 Duration::from_secs(2),
3864 )
3865 .await;
3866
3867 client.close().await;
3869 assert!(client.is_closed());
3870 server.abort();
3871 }
3872
3873 #[rstest]
3874 #[tokio::test]
3875 async fn test_reconnect_state_flips_when_reader_stops() {
3876 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3878 let port = listener.local_addr().unwrap().port();
3879 let server = task::spawn(async move {
3880 if let Ok((sock, _)) = listener.accept().await {
3881 drop(sock);
3882 }
3883
3884 sleep(Duration::from_millis(50)).await;
3886 });
3887
3888 let config = SocketConfig {
3889 url: format!("127.0.0.1:{port}"),
3890 mode: Mode::Plain,
3891 suffix: b"\r\n".to_vec(),
3892 message_handler: None,
3893 heartbeat: None,
3894 connect_timeout_ms: Some(1_000),
3895 reconnect_delay_initial_ms: Some(50),
3896 reconnect_delay_max_ms: Some(100),
3897 reconnect_backoff_factor: Some(1.0),
3898 reconnect_jitter_ms: Some(0),
3899 connection_max_retries: Some(1),
3900 reconnect_max_attempts: None,
3901 heartbeat_timeout_secs: None,
3902 writer_capacity: None,
3903 certs_dir: None,
3904 };
3905
3906 let client = SocketClient::builder()
3907 .config(config)
3908 .connect()
3909 .await
3910 .unwrap();
3911
3912 wait_until_async(
3913 || async { client.is_reconnecting() },
3914 Duration::from_secs(2),
3915 )
3916 .await;
3917
3918 client.close().await;
3919 server.abort();
3920 }
3921
3922 #[rstest]
3923 fn test_parse_socket_url_raw_address() {
3924 let (socket_addr, request_url) =
3926 SocketClientInner::parse_socket_url("example.com:6130", Mode::Tls).unwrap();
3927 assert_eq!(socket_addr, "example.com:6130");
3928 assert_eq!(request_url, "wss://example.com:6130");
3929
3930 let (socket_addr, request_url) =
3932 SocketClientInner::parse_socket_url("localhost:8080", Mode::Plain).unwrap();
3933 assert_eq!(socket_addr, "localhost:8080");
3934 assert_eq!(request_url, "ws://localhost:8080");
3935 }
3936
3937 #[rstest]
3938 fn test_parse_socket_url_with_scheme() {
3939 let (socket_addr, request_url) =
3941 SocketClientInner::parse_socket_url("wss://example.com:443/path", Mode::Tls).unwrap();
3942 assert_eq!(socket_addr, "example.com:443");
3943 assert_eq!(request_url, "wss://example.com:443/path");
3944
3945 let (socket_addr, request_url) =
3947 SocketClientInner::parse_socket_url("ws://localhost:8080", Mode::Plain).unwrap();
3948 assert_eq!(socket_addr, "localhost:8080");
3949 assert_eq!(request_url, "ws://localhost:8080");
3950 }
3951
3952 #[rstest]
3953 fn test_parse_socket_url_default_ports() {
3954 let (socket_addr, _) =
3956 SocketClientInner::parse_socket_url("wss://example.com", Mode::Tls).unwrap();
3957 assert_eq!(socket_addr, "example.com:443");
3958
3959 let (socket_addr, _) =
3961 SocketClientInner::parse_socket_url("ws://example.com", Mode::Plain).unwrap();
3962 assert_eq!(socket_addr, "example.com:80");
3963
3964 let (socket_addr, _) =
3966 SocketClientInner::parse_socket_url("https://example.com", Mode::Tls).unwrap();
3967 assert_eq!(socket_addr, "example.com:443");
3968
3969 let (socket_addr, _) =
3971 SocketClientInner::parse_socket_url("http://example.com", Mode::Plain).unwrap();
3972 assert_eq!(socket_addr, "example.com:80");
3973 }
3974
3975 #[rstest]
3976 fn test_parse_socket_url_unknown_scheme_uses_mode() {
3977 let (socket_addr, _) =
3979 SocketClientInner::parse_socket_url("custom://example.com", Mode::Tls).unwrap();
3980 assert_eq!(socket_addr, "example.com:443");
3981
3982 let (socket_addr, _) =
3983 SocketClientInner::parse_socket_url("custom://example.com", Mode::Plain).unwrap();
3984 assert_eq!(socket_addr, "example.com:80");
3985 }
3986
3987 #[rstest]
3988 fn test_parse_socket_url_ipv6() {
3989 let (socket_addr, request_url) =
3991 SocketClientInner::parse_socket_url("[::1]:8080", Mode::Plain).unwrap();
3992 assert_eq!(socket_addr, "[::1]:8080");
3993 assert_eq!(request_url, "ws://[::1]:8080");
3994
3995 let (socket_addr, _) =
3997 SocketClientInner::parse_socket_url("ws://[::1]:8080", Mode::Plain).unwrap();
3998 assert_eq!(socket_addr, "[::1]:8080");
3999 }
4000
4001 #[rstest]
4002 #[tokio::test]
4003 async fn test_url_parsing_raw_socket_address() {
4004 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4006 let port = listener.local_addr().unwrap().port();
4007 let server = task::spawn(async move {
4008 if let Ok((sock, _)) = listener.accept().await {
4009 drop(sock);
4010 }
4011
4012 sleep(Duration::from_millis(50)).await;
4013 });
4014
4015 let config = SocketConfig {
4016 url: format!("127.0.0.1:{port}"), mode: Mode::Plain,
4018 suffix: b"\r\n".to_vec(),
4019 message_handler: None,
4020 heartbeat: None,
4021 connect_timeout_ms: Some(1_000),
4022 reconnect_delay_initial_ms: Some(50),
4023 reconnect_delay_max_ms: Some(100),
4024 reconnect_backoff_factor: Some(1.0),
4025 reconnect_jitter_ms: Some(0),
4026 connection_max_retries: Some(1),
4027 reconnect_max_attempts: None,
4028 heartbeat_timeout_secs: None,
4029 writer_capacity: None,
4030 certs_dir: None,
4031 };
4032
4033 let client = SocketClient::builder().config(config).connect().await;
4035 assert!(
4036 client.is_ok(),
4037 "Client should connect with raw socket address format"
4038 );
4039
4040 if let Ok(client) = client {
4041 client.close().await;
4042 }
4043
4044 server.abort();
4045 }
4046
4047 #[rstest]
4048 #[tokio::test]
4049 async fn test_url_parsing_with_scheme() {
4050 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4052 let port = listener.local_addr().unwrap().port();
4053 let server = task::spawn(async move {
4054 if let Ok((sock, _)) = listener.accept().await {
4055 drop(sock);
4056 }
4057
4058 sleep(Duration::from_millis(50)).await;
4059 });
4060
4061 let config = SocketConfig {
4062 url: format!("ws://127.0.0.1:{port}"), mode: Mode::Plain,
4064 suffix: b"\r\n".to_vec(),
4065 message_handler: None,
4066 heartbeat: None,
4067 connect_timeout_ms: Some(1_000),
4068 reconnect_delay_initial_ms: Some(50),
4069 reconnect_delay_max_ms: Some(100),
4070 reconnect_backoff_factor: Some(1.0),
4071 reconnect_jitter_ms: Some(0),
4072 connection_max_retries: Some(1),
4073 reconnect_max_attempts: None,
4074 heartbeat_timeout_secs: None,
4075 writer_capacity: None,
4076 certs_dir: None,
4077 };
4078
4079 let client = SocketClient::builder().config(config).connect().await;
4081 assert!(
4082 client.is_ok(),
4083 "Client should connect with URL scheme format"
4084 );
4085
4086 if let Ok(client) = client {
4087 client.close().await;
4088 }
4089
4090 server.abort();
4091 }
4092
4093 #[rstest]
4094 fn test_parse_socket_url_ipv6_with_zone() {
4095 let (socket_addr, request_url) =
4097 SocketClientInner::parse_socket_url("[fe80::1%eth0]:8080", Mode::Plain).unwrap();
4098 assert_eq!(socket_addr, "[fe80::1%eth0]:8080");
4099 assert_eq!(request_url, "ws://[fe80::1%eth0]:8080");
4100
4101 let (socket_addr, request_url) =
4103 SocketClientInner::parse_socket_url("ws://[fe80::1%lo]:9090", Mode::Plain).unwrap();
4104 assert_eq!(socket_addr, "[fe80::1%lo]:9090");
4105 assert_eq!(request_url, "ws://[fe80::1%lo]:9090");
4106 }
4107
4108 #[rstest]
4109 #[tokio::test]
4110 async fn test_ipv6_loopback_connection() {
4111 if TcpListener::bind("[::1]:0").await.is_err() {
4114 return;
4115 }
4116
4117 let listener = TcpListener::bind("[::1]:0").await.unwrap();
4118 let port = listener.local_addr().unwrap().port();
4119 let server = task::spawn(async move {
4120 if let Ok((mut sock, _)) = listener.accept().await {
4121 let mut buf = vec![0u8; 1024];
4122 if let Ok(n) = sock.read(&mut buf).await {
4123 let _ = sock.write_all(&buf[..n]).await;
4125 }
4126 }
4127
4128 sleep(Duration::from_millis(50)).await;
4129 });
4130
4131 let config = SocketConfig {
4132 url: format!("[::1]:{port}"), mode: Mode::Plain,
4134 suffix: b"\r\n".to_vec(),
4135 message_handler: None,
4136 heartbeat: None,
4137 connect_timeout_ms: Some(1_000),
4138 reconnect_delay_initial_ms: Some(50),
4139 reconnect_delay_max_ms: Some(100),
4140 reconnect_backoff_factor: Some(1.0),
4141 reconnect_jitter_ms: Some(0),
4142 connection_max_retries: Some(1),
4143 reconnect_max_attempts: None,
4144 heartbeat_timeout_secs: None,
4145 writer_capacity: None,
4146 certs_dir: None,
4147 };
4148
4149 let client = SocketClient::builder().config(config).connect().await;
4150 assert!(
4151 client.is_ok(),
4152 "Client should connect to IPv6 loopback address"
4153 );
4154
4155 if let Ok(client) = client {
4156 client.close().await;
4157 }
4158
4159 server.abort();
4160 }
4161
4162 #[rstest]
4163 #[tokio::test]
4164 async fn test_send_waits_during_reconnection() {
4165 use nautilus_common::testing::wait_until_async;
4167
4168 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4169 let port = listener.local_addr().unwrap().port();
4170
4171 let server = task::spawn(async move {
4172 if let Ok((sock, _)) = listener.accept().await {
4174 drop(sock);
4175 }
4176
4177 sleep(Duration::from_millis(500)).await;
4179
4180 if let Ok((mut sock, _)) = listener.accept().await {
4182 let mut buf = vec![0u8; 1024];
4184 while let Ok(n) = sock.read(&mut buf).await {
4185 if n == 0 {
4186 break;
4187 }
4188
4189 if sock.write_all(&buf[..n]).await.is_err() {
4190 break;
4191 }
4192 }
4193 }
4194 });
4195
4196 let config = SocketConfig {
4197 url: format!("127.0.0.1:{port}"),
4198 mode: Mode::Plain,
4199 suffix: b"\r\n".to_vec(),
4200 message_handler: None,
4201 heartbeat: None,
4202 connect_timeout_ms: Some(5_000), reconnect_delay_initial_ms: Some(100),
4204 reconnect_delay_max_ms: Some(200),
4205 reconnect_backoff_factor: Some(1.0),
4206 reconnect_jitter_ms: Some(0),
4207 connection_max_retries: Some(1),
4208 reconnect_max_attempts: None,
4209 heartbeat_timeout_secs: None,
4210 writer_capacity: None,
4211 certs_dir: None,
4212 };
4213
4214 let client = SocketClient::builder()
4215 .config(config)
4216 .connect()
4217 .await
4218 .unwrap();
4219
4220 wait_until_async(
4222 || async { client.is_reconnecting() },
4223 Duration::from_secs(2),
4224 )
4225 .await;
4226
4227 let send_result = tokio::time::timeout(
4229 Duration::from_secs(3),
4230 client.send_bytes(b"test_message".to_vec()),
4231 )
4232 .await;
4233
4234 assert!(
4235 send_result.is_ok() && send_result.unwrap().is_ok(),
4236 "Send should succeed after waiting for reconnection"
4237 );
4238
4239 client.close().await;
4240 server.abort();
4241 }
4242
4243 #[rstest]
4244 #[tokio::test]
4245 async fn test_send_bytes_timeout_uses_configured_connect_timeout() {
4246 use nautilus_common::testing::wait_until_async;
4249
4250 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4251 let port = listener.local_addr().unwrap().port();
4252
4253 let server = task::spawn(async move {
4254 if let Ok((sock, _)) = listener.accept().await {
4256 drop(sock);
4257 }
4258
4259 drop(listener);
4261 sleep(Duration::from_mins(1)).await;
4262 });
4263
4264 let config = SocketConfig {
4265 url: format!("127.0.0.1:{port}"),
4266 mode: Mode::Plain,
4267 suffix: b"\r\n".to_vec(),
4268 message_handler: None,
4269 heartbeat: None,
4270 connect_timeout_ms: Some(1_000), reconnect_delay_initial_ms: Some(200), reconnect_delay_max_ms: Some(200),
4273 reconnect_backoff_factor: Some(1.0),
4274 reconnect_jitter_ms: Some(0),
4275 connection_max_retries: Some(1),
4276 reconnect_max_attempts: None,
4277 heartbeat_timeout_secs: None,
4278 writer_capacity: None,
4279 certs_dir: None,
4280 };
4281
4282 let client = SocketClient::builder()
4283 .config(config)
4284 .connect()
4285 .await
4286 .unwrap();
4287
4288 wait_until_async(
4290 || async { client.is_reconnecting() },
4291 Duration::from_secs(3),
4292 )
4293 .await;
4294
4295 let start = std::time::Instant::now();
4298 let send_result = client.send_bytes(b"test".to_vec()).await;
4299 let elapsed = start.elapsed();
4300
4301 assert!(
4302 send_result.is_err(),
4303 "Send should fail when client stuck in RECONNECT, was: {send_result:?}"
4304 );
4305 assert!(
4306 matches!(send_result, Err(crate::error::SendError::Timeout)),
4307 "Send should return Timeout error, was: {send_result:?}"
4308 );
4309 assert!(
4312 elapsed >= Duration::from_millis(900),
4313 "Send should timeout after at least 1s (configured timeout), took {elapsed:?}"
4314 );
4315
4316 client.close().await;
4317 server.abort();
4318 }
4319
4320 #[rstest]
4321 #[tokio::test]
4322 async fn test_heartbeat_timeout_triggers_reconnect() {
4323 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4324 let port = listener.local_addr().unwrap().port();
4325
4326 let server = task::spawn(async move {
4328 let (_sock1, _) = listener.accept().await.unwrap();
4329 sleep(Duration::from_secs(5)).await;
4331 });
4332
4333 let config = SocketConfig {
4334 url: format!("127.0.0.1:{port}"),
4335 mode: Mode::Plain,
4336 suffix: b"\r\n".to_vec(),
4337 message_handler: None,
4338 heartbeat: None,
4339 connect_timeout_ms: Some(2_000),
4340 reconnect_delay_initial_ms: Some(50),
4341 reconnect_delay_max_ms: Some(100),
4342 reconnect_backoff_factor: Some(1.0),
4343 reconnect_jitter_ms: Some(0),
4344 connection_max_retries: Some(1),
4345 reconnect_max_attempts: Some(1),
4346 heartbeat_timeout_secs: Some(1),
4347 writer_capacity: None,
4348 certs_dir: None,
4349 };
4350
4351 let client = SocketClient::builder()
4352 .config(config)
4353 .connect()
4354 .await
4355 .unwrap();
4356
4357 assert!(client.is_active());
4358
4359 wait_until_async(
4361 || async { client.is_reconnecting() || client.is_closed() },
4362 Duration::from_secs(4),
4363 )
4364 .await;
4365
4366 assert!(
4367 !client.is_active(),
4368 "Client should not be active after the dead-peer timeout"
4369 );
4370
4371 client.close().await;
4372 server.abort();
4373 }
4374
4375 #[rstest]
4376 #[tokio::test]
4377 async fn test_heartbeat_timeout_resets_on_inbound_bytes() {
4378 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4379 let port = listener.local_addr().unwrap().port();
4380
4381 let server = task::spawn(async move {
4383 let (mut sock, _) = listener.accept().await.unwrap();
4384
4385 for _ in 0..10 {
4386 sleep(Duration::from_millis(200)).await;
4387
4388 if sock.write_all(b"ping\r\n").await.is_err() {
4389 break;
4390 }
4391 }
4392 });
4393
4394 let config = SocketConfig {
4395 url: format!("127.0.0.1:{port}"),
4396 mode: Mode::Plain,
4397 suffix: b"\r\n".to_vec(),
4398 message_handler: None,
4399 heartbeat: None,
4400 connect_timeout_ms: Some(2_000),
4401 reconnect_delay_initial_ms: Some(50),
4402 reconnect_delay_max_ms: Some(100),
4403 reconnect_backoff_factor: Some(1.0),
4404 reconnect_jitter_ms: Some(0),
4405 connection_max_retries: Some(1),
4406 reconnect_max_attempts: Some(1),
4407 heartbeat_timeout_secs: Some(1),
4408 writer_capacity: None,
4409 certs_dir: None,
4410 };
4411
4412 let client = SocketClient::builder()
4413 .config(config)
4414 .connect()
4415 .await
4416 .unwrap();
4417
4418 assert!(client.is_active());
4419
4420 sleep(Duration::from_millis(1_500)).await;
4422
4423 assert!(
4424 client.is_active(),
4425 "Client should remain active when data is flowing"
4426 );
4427
4428 client.close().await;
4429 server.abort();
4430 }
4431
4432 #[rstest]
4433 #[tokio::test]
4434 async fn test_close_during_backoff_exits_promptly() {
4435 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
4439 let port = listener.local_addr().unwrap().port();
4440
4441 let server = task::spawn(async move {
4442 if let Ok((mut sock, _)) = listener.accept().await {
4444 drop(sock.shutdown());
4445 }
4446
4447 sleep(Duration::from_mins(1)).await;
4449 });
4450
4451 let config = SocketConfig {
4452 url: format!("127.0.0.1:{port}"),
4453 mode: Mode::Plain,
4454 suffix: b"\r\n".to_vec(),
4455 message_handler: None,
4456 heartbeat: None,
4457 connect_timeout_ms: Some(1_000),
4458 reconnect_delay_initial_ms: Some(10_000), reconnect_delay_max_ms: Some(10_000),
4460 reconnect_backoff_factor: Some(1.0),
4461 reconnect_jitter_ms: Some(0),
4462 connection_max_retries: None,
4463 reconnect_max_attempts: None,
4464 heartbeat_timeout_secs: None,
4465 writer_capacity: None,
4466 certs_dir: None,
4467 };
4468
4469 let client = SocketClient::builder()
4470 .config(config)
4471 .connect()
4472 .await
4473 .unwrap();
4474
4475 wait_until_async(
4477 || async { client.is_reconnecting() },
4478 Duration::from_secs(3),
4479 )
4480 .await;
4481
4482 sleep(Duration::from_millis(1_500)).await;
4484
4485 let start = std::time::Instant::now();
4487 client.close().await;
4488 let elapsed = start.elapsed();
4489
4490 assert!(client.is_closed(), "Client should be closed");
4491 assert!(
4493 elapsed < Duration::from_secs(2),
4494 "Close should interrupt backoff sleep, took {elapsed:?}"
4495 );
4496
4497 server.abort();
4498 }
4499
4500 #[rstest]
4501 #[tokio::test]
4502 async fn test_zero_heartbeat_timeout_rejected() {
4503 let config = SocketConfig {
4504 url: "127.0.0.1:9999".to_string(),
4505 mode: Mode::Plain,
4506 suffix: b"\r\n".to_vec(),
4507 message_handler: None,
4508 heartbeat: None,
4509 connect_timeout_ms: None,
4510 reconnect_delay_initial_ms: None,
4511 reconnect_delay_max_ms: None,
4512 reconnect_backoff_factor: None,
4513 reconnect_jitter_ms: None,
4514 reconnect_max_attempts: None,
4515 connection_max_retries: Some(1),
4516 heartbeat_timeout_secs: Some(0),
4517 writer_capacity: None,
4518 certs_dir: None,
4519 };
4520
4521 let result = SocketClient::builder().config(config).connect().await;
4522
4523 assert!(result.is_err(), "Zero heartbeat timeout should be rejected");
4524 let err_msg = result.unwrap_err().to_string();
4525 assert!(
4526 err_msg.contains("heartbeat_timeout_secs"),
4527 "Error should name the offending field, was: {err_msg}"
4528 );
4529 }
4530
4531 #[rstest]
4532 #[tokio::test]
4533 async fn test_empty_suffix_rejected() {
4534 let config = SocketConfig {
4535 url: "127.0.0.1:9999".to_string(),
4536 mode: Mode::Plain,
4537 suffix: vec![],
4538 message_handler: None,
4539 heartbeat: None,
4540 connect_timeout_ms: None,
4541 reconnect_delay_initial_ms: None,
4542 reconnect_delay_max_ms: None,
4543 reconnect_backoff_factor: None,
4544 reconnect_jitter_ms: None,
4545 reconnect_max_attempts: None,
4546 connection_max_retries: Some(1),
4547 heartbeat_timeout_secs: None,
4548 writer_capacity: None,
4549 certs_dir: None,
4550 };
4551
4552 let result = SocketClient::builder().config(config).connect().await;
4553
4554 assert!(
4555 result.is_err(),
4556 "Empty suffix should cause connection to fail"
4557 );
4558 let err_msg = result.unwrap_err().to_string();
4559 assert!(
4560 err_msg.contains("suffix cannot be empty"),
4561 "Error should mention empty suffix, was: {err_msg}"
4562 );
4563 }
4564}
4565
4566#[cfg(test)]
4567mod reconnect_request_tests {
4568 use std::sync::{Arc, atomic::AtomicU8};
4569
4570 use parking_lot::Mutex;
4571 use rstest::rstest;
4572
4573 use super::*;
4574 use crate::SocketState;
4575
4576 fn handle(
4577 mode: ConnectionMode,
4578 ) -> (
4579 SocketReconnectHandle,
4580 Arc<tokio::sync::Notify>,
4581 Arc<Mutex<Vec<SocketState>>>,
4582 ) {
4583 let controller_notify = Arc::new(tokio::sync::Notify::new());
4584 let states = Arc::new(Mutex::new(Vec::new()));
4585 let states_callback = Arc::clone(&states);
4586 let state_sink = SocketStateSink::new(move |state| {
4587 states_callback.lock().push(state);
4588 });
4589
4590 let handle = SocketReconnectHandle {
4591 connection_mode: Arc::new(AtomicU8::new(mode.as_u8())),
4592 state_sink: Some(state_sink),
4593 controller_lifecycle: Arc::new(ControllerLifecycle::new()),
4594 controller_notify: Arc::clone(&controller_notify),
4595 };
4596
4597 (handle, controller_notify, states)
4598 }
4599
4600 #[rstest]
4601 #[tokio::test]
4602 async fn accepted_request_reports_loss_and_wakes_controller_once() {
4603 let (handle, controller_notify, states) = handle(ConnectionMode::Active);
4604
4605 assert_eq!(
4606 handle.request_reconnect(),
4607 ReconnectRequestOutcome::Accepted
4608 );
4609 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
4610 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4611 .await
4612 .expect("accepted request should notify controller");
4613
4614 assert_eq!(
4615 handle.request_reconnect(),
4616 ReconnectRequestOutcome::AlreadyReconnecting
4617 );
4618 assert_eq!(*states.lock(), vec![SocketState::Disconnected]);
4619 assert!(
4620 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4621 .await
4622 .is_err(),
4623 "duplicate request should not notify controller",
4624 );
4625 }
4626
4627 #[rstest]
4628 #[case(
4629 ConnectionMode::Reconnect,
4630 ReconnectRequestOutcome::AlreadyReconnecting
4631 )]
4632 #[case(ConnectionMode::Disconnect, ReconnectRequestOutcome::Disconnected)]
4633 #[case(ConnectionMode::Closed, ReconnectRequestOutcome::Closed)]
4634 #[tokio::test]
4635 async fn rejected_request_preserves_state_and_does_not_wake_controller(
4636 #[case] mode: ConnectionMode,
4637 #[case] expected: ReconnectRequestOutcome,
4638 ) {
4639 let (handle, controller_notify, states) = handle(mode);
4640
4641 assert_eq!(handle.request_reconnect(), expected);
4642 assert!(states.lock().is_empty());
4643 assert!(
4644 tokio::time::timeout(Duration::from_millis(10), controller_notify.notified())
4645 .await
4646 .is_err(),
4647 "rejected request should not notify controller",
4648 );
4649 }
4650}