1use std::{
32 collections::{HashMap, VecDeque},
33 fmt::Debug,
34 sync::{
35 Arc,
36 atomic::{AtomicBool, Ordering},
37 },
38 time::Duration,
39};
40
41use bytes::Bytes;
42use futures::stream::Stream;
43use jiff::{Timestamp, tz::Offset};
44use nautilus_common::{
45 enums::SerializationEncoding,
46 live::get_runtime,
47 logging::{log_task_error, log_task_started, log_task_stopped},
48 msgbus::{
49 BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory, MessageBusConfig,
50 switchboard::CLOSE_TOPIC,
51 },
52};
53use nautilus_core::{
54 UUID4,
55 time::{duration_since_unix_epoch, get_atomic_clock_realtime},
56};
57use nautilus_cryptography::providers::install_cryptographic_provider;
58use nautilus_model::identifiers::TraderId;
59use redis::{AsyncCommands, RetryMethod, aio::ConnectionManager, streams};
60use serde::{Deserialize, Serialize};
61use streams::StreamReadOptions;
62use ustr::Ustr;
63
64use super::{
65 REDIS_MINID, REDIS_XTRIM, await_handle,
66 stream_fields::{PAYLOAD_KIND_FIELD, PAYLOAD_KIND_TYPED, bus_message_fields},
67};
68use crate::redis::{RedisConnectionConfig, create_redis_connection, get_stream_key};
69
70const MSGBUS_PUBLISH: &str = "msgbus-publish";
71const MSGBUS_STREAM: &str = "msgbus-stream";
72const MSGBUS_HEARTBEAT: &str = "msgbus-heartbeat";
73const HEARTBEAT_TOPIC: &str = "health:heartbeat";
74const TRIM_BUFFER_SECS: u64 = 60;
75const PUBLISH_BATCH_SIZE: usize = 1_000;
76
77type RedisStreamBulk = Vec<HashMap<String, Vec<HashMap<String, redis::Value>>>>;
78
79#[cfg_attr(
83 feature = "python",
84 expect(
85 clippy::unsafe_derive_deserialize,
86 reason = "config deserializes plain fields; unsafe methods come from generated PyO3 integration"
87 )
88)]
89#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(default, deny_unknown_fields)]
91#[cfg_attr(
92 feature = "python",
93 pyo3::pyclass(module = "nautilus_trader.infrastructure", from_py_object)
94)]
95#[cfg_attr(
96 feature = "python",
97 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.infrastructure")
98)]
99pub struct RedisMessageBusConfig {
100 pub host: Option<String>,
102 pub port: Option<u16>,
104 pub username: Option<String>,
106 pub password: Option<String>,
108 pub ssl: bool,
110 pub connection_timeout: u16,
112 pub response_timeout: u16,
114 pub number_of_retries: usize,
116 pub exponent_base: u64,
118 pub max_delay: u64,
120 pub factor: u64,
122}
123
124impl Debug for RedisMessageBusConfig {
125 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126 let redacted = self.password.as_ref().map(|_| "***");
127 f.debug_struct(stringify!(RedisMessageBusConfig))
128 .field("host", &self.host)
129 .field("port", &self.port)
130 .field("username", &self.username)
131 .field("password", &redacted)
132 .field("ssl", &self.ssl)
133 .field("connection_timeout", &self.connection_timeout)
134 .field("response_timeout", &self.response_timeout)
135 .field("number_of_retries", &self.number_of_retries)
136 .field("exponent_base", &self.exponent_base)
137 .field("max_delay", &self.max_delay)
138 .field("factor", &self.factor)
139 .finish()
140 }
141}
142
143impl Default for RedisMessageBusConfig {
144 fn default() -> Self {
145 Self {
146 host: None,
147 port: None,
148 username: None,
149 password: None,
150 ssl: false,
151 connection_timeout: 20,
152 response_timeout: 20,
153 number_of_retries: 100,
154 exponent_base: 2,
155 max_delay: 1000,
156 factor: 2,
157 }
158 }
159}
160
161impl RedisConnectionConfig for RedisMessageBusConfig {
162 fn host(&self) -> Option<&str> {
163 self.host.as_deref()
164 }
165
166 fn port(&self) -> Option<u16> {
167 self.port
168 }
169
170 fn username(&self) -> Option<&str> {
171 self.username.as_deref()
172 }
173
174 fn password(&self) -> Option<&str> {
175 self.password.as_deref()
176 }
177
178 fn ssl(&self) -> bool {
179 self.ssl
180 }
181
182 fn connection_timeout(&self) -> u16 {
183 self.connection_timeout
184 }
185
186 fn response_timeout(&self) -> u16 {
187 self.response_timeout
188 }
189
190 fn number_of_retries(&self) -> usize {
191 self.number_of_retries
192 }
193
194 fn exponent_base(&self) -> u64 {
195 self.exponent_base
196 }
197
198 fn max_delay(&self) -> u64 {
199 self.max_delay
200 }
201
202 fn factor(&self) -> u64 {
203 self.factor
204 }
205}
206
207impl MessageBusBackingFactory for RedisMessageBusConfig {
208 fn create(
209 &self,
210 trader_id: TraderId,
211 instance_id: UUID4,
212 config: MessageBusConfig,
213 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
214 Ok(Box::new(RedisMessageBusBacking::new(
215 trader_id,
216 instance_id,
217 config,
218 self.clone(),
219 )?))
220 }
221}
222
223#[derive(Debug, Clone)]
225pub struct RedisMessageBusFactory {
226 config: RedisMessageBusConfig,
227}
228
229impl RedisMessageBusFactory {
230 #[must_use]
232 pub const fn new(config: RedisMessageBusConfig) -> Self {
233 Self { config }
234 }
235}
236
237impl MessageBusBackingFactory for RedisMessageBusFactory {
238 fn create(
239 &self,
240 trader_id: TraderId,
241 instance_id: UUID4,
242 config: MessageBusConfig,
243 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
244 self.config.create(trader_id, instance_id, config)
245 }
246}
247
248pub struct RedisMessageBusBacking {
249 pub trader_id: TraderId,
251 pub instance_id: UUID4,
253 pub_tx: tokio::sync::mpsc::UnboundedSender<BusMessage>,
254 pub_handle: Option<tokio::task::JoinHandle<()>>,
255 stream_rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
256 stream_handle: Option<tokio::task::JoinHandle<()>>,
257 stream_signal: Arc<AtomicBool>,
258 heartbeat_handle: Option<tokio::task::JoinHandle<()>>,
259 heartbeat_signal: Arc<AtomicBool>,
260}
261
262impl Debug for RedisMessageBusBacking {
263 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
264 f.debug_struct(stringify!(RedisMessageBusBacking))
265 .field("trader_id", &self.trader_id)
266 .field("instance_id", &self.instance_id)
267 .finish_non_exhaustive()
268 }
269}
270
271impl RedisMessageBusBacking {
272 pub fn new(
278 trader_id: TraderId,
279 instance_id: UUID4,
280 config: MessageBusConfig,
281 backing: RedisMessageBusConfig,
282 ) -> anyhow::Result<Self> {
283 install_cryptographic_provider();
284
285 if config.heartbeat_interval_secs == Some(0) {
286 anyhow::bail!("heartbeat_interval_secs must be greater than 0");
287 }
288
289 let external_streams = config.external_streams.clone().unwrap_or_default();
290 let heartbeat_interval_secs = config.heartbeat_interval_secs;
291 let publish = backing.clone();
292
293 let (pub_tx, pub_rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
294
295 let pub_handle = Some(get_runtime().spawn(async move {
297 if let Err(e) = publish_messages(pub_rx, trader_id, instance_id, config, publish).await
298 {
299 log_task_error(MSGBUS_PUBLISH, &e);
300 }
301 }));
302
303 let stream_signal = Arc::new(AtomicBool::new(false));
305 let (stream_rx, stream_handle) = if external_streams.is_empty() {
306 (None, None)
307 } else {
308 let stream_signal_clone = Arc::clone(&stream_signal);
309 let (stream_tx, stream_rx) = tokio::sync::mpsc::channel::<BusMessage>(100_000);
310 (
311 Some(stream_rx),
312 Some(get_runtime().spawn(async move {
313 if let Err(e) = stream_messages(
314 stream_tx,
315 backing.clone(),
316 external_streams,
317 stream_signal_clone,
318 )
319 .await
320 {
321 log_task_error(MSGBUS_STREAM, &e);
322 }
323 })),
324 )
325 };
326
327 let heartbeat_signal = Arc::new(AtomicBool::new(false));
329 let heartbeat_handle = if let Some(heartbeat_interval_secs) = heartbeat_interval_secs {
330 let signal = Arc::clone(&heartbeat_signal);
331 let pub_tx_clone = pub_tx.clone();
332
333 Some(get_runtime().spawn(async move {
334 run_heartbeat(heartbeat_interval_secs, signal, pub_tx_clone).await;
335 }))
336 } else {
337 None
338 };
339
340 Ok(Self {
341 trader_id,
342 instance_id,
343 pub_tx,
344 pub_handle,
345 stream_rx,
346 stream_handle,
347 stream_signal,
348 heartbeat_handle,
349 heartbeat_signal,
350 })
351 }
352}
353
354impl MessageBusBacking for RedisMessageBusBacking {
355 fn is_closed(&self) -> bool {
357 self.pub_tx.is_closed()
358 }
359
360 fn publish(&self, message: BusMessage) {
362 if let Err(e) = self.pub_tx.send(message) {
363 log::error!("Failed to send message: {e}");
364 }
365 }
366
367 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
368 self.get_stream_receiver()
369 }
370
371 fn close(&mut self) {
373 log::debug!("Closing");
374
375 self.stream_signal.store(true, Ordering::Relaxed);
376 self.heartbeat_signal.store(true, Ordering::Relaxed);
377
378 if !self.pub_tx.is_closed() {
379 let msg = BusMessage::new_close();
380
381 if let Err(e) = self.pub_tx.send(msg) {
382 log::warn!("Failed to send close message: {e:?}");
383 }
384 }
385
386 tokio::task::block_in_place(|| {
388 get_runtime().block_on(async {
389 self.close_async().await;
390 });
391 });
392
393 log::debug!("Closed");
394 }
395}
396
397impl RedisMessageBusBacking {
398 pub fn get_stream_receiver(
404 &mut self,
405 ) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
406 self.stream_rx
407 .take()
408 .ok_or_else(|| anyhow::anyhow!("Stream receiver already taken"))
409 }
410
411 pub fn stream(
413 mut stream_rx: tokio::sync::mpsc::Receiver<BusMessage>,
414 ) -> impl Stream<Item = BusMessage> + 'static {
415 async_stream::stream! {
416 while let Some(msg) = stream_rx.recv().await {
417 yield msg;
418 }
419 }
420 }
421
422 pub async fn close_async(&mut self) {
423 await_handle(self.pub_handle.take(), MSGBUS_PUBLISH).await;
424 await_handle(self.stream_handle.take(), MSGBUS_STREAM).await;
425 await_handle(self.heartbeat_handle.take(), MSGBUS_HEARTBEAT).await;
426 }
427}
428
429pub async fn publish_messages(
444 mut rx: tokio::sync::mpsc::UnboundedReceiver<BusMessage>,
445 trader_id: TraderId,
446 instance_id: UUID4,
447 config: MessageBusConfig,
448 backing: RedisMessageBusConfig,
449) -> anyhow::Result<()> {
450 log_task_started(MSGBUS_PUBLISH);
451
452 let con = create_redis_connection(MSGBUS_PUBLISH, &backing).await?;
453 let stream_key = get_stream_key(trader_id, instance_id, &config);
454
455 let autotrim_duration = config
457 .autotrim_mins
458 .filter(|&mins| mins > 0)
459 .map(|mins| Duration::from_secs(u64::from(mins) * 60));
460 let autotrim_maxlen = config
461 .autotrim_maxlen
462 .filter(|&maxlen| maxlen > 0)
463 .map(usize::try_from)
464 .transpose()?;
465
466 let mut writer = StreamWriter {
467 con: Some(con),
468 reconnect_config: RedisMessageBusConfig {
471 number_of_retries: 0,
472 ..backing
473 },
474 stream_key,
475 stream_per_topic: config.stream_per_topic,
476 autotrim_duration,
477 autotrim_maxlen,
478 last_trim_index: HashMap::new(),
479 buffer: VecDeque::new(),
480 write_error_count: 0,
481 };
482
483 let buffer_interval = Duration::from_millis(u64::from(config.buffer_interval_ms.unwrap_or(0)));
485
486 let flush_timer = tokio::time::sleep(buffer_interval);
490 tokio::pin!(flush_timer);
491
492 let retry_timer = tokio::time::sleep(Duration::ZERO);
495 tokio::pin!(retry_timer);
496
497 loop {
498 let retry_delay = tokio::select! {
499 maybe_msg = rx.recv() => {
500 let Some(msg) = maybe_msg else {
501 log::debug!("Channel hung up");
502 break;
503 };
504
505 if msg.topic == CLOSE_TOPIC {
506 log::debug!("Received close message");
507 break;
508 }
509
510 writer.buffer.push_back(msg);
511
512 if buffer_interval.is_zero() && !writer.is_retrying() {
514 writer.flush().await?
515 } else {
516 None
517 }
518 }
519 () = &mut flush_timer, if !buffer_interval.is_zero() => {
522 let retry_delay = if writer.is_retrying() {
523 None
524 } else {
525 writer.flush().await?
526 };
527
528 flush_timer.as_mut().reset(tokio::time::Instant::now() + buffer_interval);
530 retry_delay
531 }
532 () = &mut retry_timer, if writer.is_retrying() => writer.flush().await?,
533 };
534
535 if let Some(retry_delay) = retry_delay {
536 retry_timer
537 .as_mut()
538 .reset(tokio::time::Instant::now() + retry_delay);
539 }
540 }
541
542 if writer.flush().await?.is_some() {
544 anyhow::bail!(
545 "Failed to publish buffered messages before closing ({} unpublished)",
546 writer.buffer.len()
547 );
548 }
549
550 log_task_stopped(MSGBUS_PUBLISH);
551 Ok(())
552}
553
554struct StreamWriter {
559 con: Option<ConnectionManager>,
560 reconnect_config: RedisMessageBusConfig,
561 stream_key: String,
562 stream_per_topic: bool,
563 autotrim_duration: Option<Duration>,
564 autotrim_maxlen: Option<usize>,
565 last_trim_index: HashMap<String, usize>,
566 buffer: VecDeque<BusMessage>,
567 write_error_count: usize,
568}
569
570impl StreamWriter {
571 const fn is_retrying(&self) -> bool {
573 self.write_error_count > 0
574 }
575
576 async fn flush(&mut self) -> anyhow::Result<Option<Duration>> {
585 while !self.buffer.is_empty() {
586 let mut con = match self.con.take() {
587 Some(con) => con,
588 None => {
589 match create_redis_connection(MSGBUS_PUBLISH, &self.reconnect_config).await {
590 Ok(con) => con,
591 Err(e) => {
592 log::error!("Error reconnecting to publish: {e:?}");
593 return Ok(Some(self.next_retry_delay()));
594 }
595 }
596 }
597 };
598
599 let count = self.buffer.len().min(PUBLISH_BATCH_SIZE);
600 let result = write_buffer(
601 &mut con,
602 &self.stream_key,
603 self.stream_per_topic,
604 self.autotrim_duration,
605 self.autotrim_maxlen,
606 &mut self.last_trim_index,
607 self.buffer.range(..count),
608 )
609 .await;
610
611 match result {
612 Ok(()) => {
613 self.buffer.drain(..count);
614 self.con = Some(con);
615 self.write_error_count = 0;
616 }
617 Err(e) if is_retryable_write_error(&e) => {
618 log::error!("Error publishing to stream: {e:?}");
619 return Ok(Some(self.next_retry_delay()));
620 }
621 Err(e) => anyhow::bail!("Error publishing to stream: {e:?}"),
622 }
623 }
624
625 Ok(None)
626 }
627
628 fn next_retry_delay(&mut self) -> Duration {
629 let retry_delay = stream_retry_delay(&self.reconnect_config, self.write_error_count);
630 self.write_error_count = self.write_error_count.saturating_add(1);
631 retry_delay
632 }
633}
634
635async fn write_buffer(
636 conn: &mut redis::aio::ConnectionManager,
637 stream_key: &str,
638 stream_per_topic: bool,
639 autotrim_duration: Option<Duration>,
640 autotrim_maxlen: Option<usize>,
641 last_trim_index: &mut HashMap<String, usize>,
642 messages: impl Iterator<Item = &BusMessage>,
643) -> redis::RedisResult<()> {
644 let mut pipe = redis::pipe();
645 pipe.atomic();
646
647 for msg in messages {
648 let encoding = msg.encoding.to_string();
649 let items = bus_message_fields(msg, &encoding);
650 let stream_key = if stream_per_topic {
651 format!("{stream_key}:{}", msg.topic)
652 } else {
653 stream_key.to_string()
654 };
655
656 if let Some(maxlen) = autotrim_maxlen {
657 pipe.xadd_maxlen(
658 &stream_key,
659 streams::StreamMaxlen::Approx(maxlen),
660 "*",
661 items.as_slice(),
662 );
663 } else {
664 pipe.xadd(&stream_key, "*", items.as_slice());
665 }
666
667 if autotrim_duration.is_none() {
668 continue; }
670
671 let last_trim_ms = last_trim_index.entry(stream_key.clone()).or_insert(0); let unix_duration_now = duration_since_unix_epoch();
674 let trim_buffer = Duration::from_secs(TRIM_BUFFER_SECS);
675
676 if *last_trim_ms < unix_duration_now.saturating_sub(trim_buffer).as_millis() as usize {
678 let min_timestamp_ms = unix_duration_now
679 .saturating_sub(autotrim_duration.unwrap())
680 .as_millis() as usize;
681 let result: Result<(), redis::RedisError> = redis::cmd(REDIS_XTRIM)
682 .arg(stream_key.clone())
683 .arg(REDIS_MINID)
684 .arg(min_timestamp_ms)
685 .query_async(conn)
686 .await;
687
688 if let Err(e) = result {
689 if is_retryable_write_error(&e) {
692 return Err(e);
693 }
694 log::error!("Error trimming stream '{stream_key}': {e}");
695 } else {
696 last_trim_index.insert(stream_key.clone(), unix_duration_now.as_millis() as usize);
697 }
698 }
699 }
700
701 pipe.query_async(conn).await
702}
703
704pub async fn stream_messages(
712 tx: tokio::sync::mpsc::Sender<BusMessage>,
713 config: RedisMessageBusConfig,
714 stream_keys: Vec<String>,
715 stream_signal: Arc<AtomicBool>,
716) -> anyhow::Result<()> {
717 log_task_started(MSGBUS_STREAM);
718
719 let Some(mut con) = connect_stream_connection(&config, &stream_signal).await? else {
720 log_task_stopped(MSGBUS_STREAM);
721 return Ok(());
722 };
723
724 let mut read_error_count = 0;
725
726 let stream_keys = &stream_keys
727 .iter()
728 .map(String::as_str)
729 .collect::<Vec<&str>>();
730
731 log::debug!("Listening to streams: [{}]", stream_keys.join(", "));
732
733 let clock = get_atomic_clock_realtime();
735 let timestamp_ms = clock.get_time_ms();
736 let initial_id = timestamp_ms.to_string();
737
738 let mut last_ids: HashMap<String, String> = stream_keys
739 .iter()
740 .map(|&key| (key.to_string(), initial_id.clone()))
741 .collect();
742
743 let opts = StreamReadOptions::default().block(100);
744
745 'outer: loop {
746 if stream_signal.load(Ordering::Relaxed) {
747 log::debug!("Received streaming terminate signal");
748 break;
749 }
750
751 let ids: Vec<String> = stream_keys
752 .iter()
753 .map(|&key| last_ids[key].clone())
754 .collect();
755 let id_refs: Vec<&str> = ids.iter().map(String::as_str).collect();
756
757 let result: Result<RedisStreamBulk, _> =
758 con.xread_options(&[&stream_keys], &[&id_refs], &opts).await;
759
760 match result {
761 Ok(stream_bulk) => {
762 read_error_count = 0;
763
764 if stream_bulk.is_empty() {
765 continue;
767 }
768
769 for entry in &stream_bulk {
770 for (stream_key, stream_msgs) in entry {
771 for stream_msg in stream_msgs {
772 for (id, array) in stream_msg {
773 last_ids.insert(stream_key.clone(), id.clone());
774
775 match decode_bus_message(array) {
776 Ok(msg) => {
777 if let Err(e) = tx.send(msg).await {
778 log::debug!("Channel closed: {e:?}");
779 break 'outer; }
781 }
782 Err(e) => {
783 log::error!("{e:?}");
784 }
785 }
786 }
787 }
788 }
789 }
790 }
791 Err(e) => {
792 if !is_retryable_stream_error(&e) {
793 anyhow::bail!("Error reading from stream: {e:?}");
794 }
795
796 log::error!("Error reading from stream: {e:?}");
797
798 let Some(reconnected) =
799 reconnect_stream_connection(&config, &stream_signal, &mut read_error_count)
800 .await?
801 else {
802 break;
803 };
804 con = reconnected;
805 }
806 }
807 }
808
809 log_task_stopped(MSGBUS_STREAM);
810 Ok(())
811}
812
813async fn connect_stream_connection(
814 config: &RedisMessageBusConfig,
815 stream_signal: &Arc<AtomicBool>,
816) -> anyhow::Result<Option<ConnectionManager>> {
817 let connect = create_redis_connection(MSGBUS_STREAM, config);
818 let terminate = wait_for_stream_signal(stream_signal);
819
820 tokio::pin!(connect);
821 tokio::pin!(terminate);
822
823 tokio::select! {
824 result = &mut connect => result.map(Some),
825 () = &mut terminate => Ok(None),
826 }
827}
828
829async fn reconnect_stream_connection(
830 config: &RedisMessageBusConfig,
831 stream_signal: &Arc<AtomicBool>,
832 read_error_count: &mut usize,
833) -> anyhow::Result<Option<ConnectionManager>> {
834 loop {
835 let retry_delay = stream_retry_delay(config, *read_error_count);
836 *read_error_count = (*read_error_count).saturating_add(1);
837
838 if !wait_for_retry_delay(retry_delay, stream_signal).await {
839 return Ok(None);
840 }
841
842 match connect_stream_connection(config, stream_signal).await {
843 Ok(Some(con)) => return Ok(Some(con)),
844 Ok(None) => return Ok(None),
845 Err(e) => {
846 log::error!("Error reconnecting to stream: {e:?}");
847 }
848 }
849 }
850}
851
852fn stream_retry_delay(config: &RedisMessageBusConfig, attempt: usize) -> Duration {
853 let exponent = u32::try_from(attempt.min(32)).unwrap_or(32);
854 let delay_ms = config
855 .factor
856 .saturating_mul(config.exponent_base.saturating_pow(exponent));
857 let max_delay = Duration::from_secs(config.max_delay);
858
859 Duration::from_millis(delay_ms)
860 .min(max_delay)
861 .max(Duration::from_millis(1))
862}
863
864fn is_retryable_stream_error(error: &redis::RedisError) -> bool {
865 matches!(
866 error.retry_method(),
867 RetryMethod::Reconnect
868 | RetryMethod::ReconnectFromInitialConnections
869 | RetryMethod::RetryImmediately
870 | RetryMethod::WaitAndRetry
871 )
872}
873
874fn is_retryable_write_error(error: &redis::RedisError) -> bool {
877 if is_retryable_stream_error(error) {
878 return true;
879 }
880
881 error.clone().into_server_errors().is_some_and(|errors| {
882 !errors.is_empty()
883 && errors.iter().all(|(_, server_error)| {
884 is_retryable_stream_error(&redis::RedisError::from(server_error.clone()))
885 })
886 })
887}
888
889async fn wait_for_retry_delay(retry_delay: Duration, stream_signal: &Arc<AtomicBool>) -> bool {
890 let retry_timer = tokio::time::sleep(retry_delay);
891 let terminate = wait_for_stream_signal(stream_signal);
892
893 tokio::pin!(retry_timer);
894 tokio::pin!(terminate);
895
896 tokio::select! {
897 () = &mut retry_timer => true,
898 () = &mut terminate => false,
899 }
900}
901
902async fn wait_for_stream_signal(stream_signal: &Arc<AtomicBool>) {
903 let check_timer = tokio::time::interval(Duration::from_millis(100));
904
905 tokio::pin!(check_timer);
906
907 while !stream_signal.load(Ordering::Relaxed) {
908 check_timer.tick().await;
909 }
910}
911
912fn decode_bus_message(stream_msg: &redis::Value) -> anyhow::Result<BusMessage> {
914 let redis::Value::Array(fields) = stream_msg else {
915 anyhow::bail!("Invalid stream message format: {stream_msg:?}");
916 };
917
918 if fields.len() < 4 || fields.len() % 2 != 0 {
919 anyhow::bail!("Invalid stream message format: {stream_msg:?}");
920 }
921
922 let mut topic: Option<String> = None;
923 let mut type_name: Option<String> = None;
924 let mut typed_payload = false;
925 let mut encoding = SerializationEncoding::default();
926 let mut payload: Option<Bytes> = None;
927
928 for pair in fields.as_chunks::<2>().0 {
929 let redis::Value::BulkString(key) = &pair[0] else {
930 anyhow::bail!("Invalid stream field key: {stream_msg:?}");
931 };
932
933 match key.as_slice() {
934 b"topic" => {
935 let redis::Value::BulkString(bytes) = &pair[1] else {
936 anyhow::bail!("Invalid topic format: {stream_msg:?}");
937 };
938 topic = Some(
939 String::from_utf8(bytes.clone())
940 .map_err(|e| anyhow::anyhow!("Error parsing topic: {e}"))?,
941 );
942 }
943 b"type" => {
944 let redis::Value::BulkString(bytes) = &pair[1] else {
945 anyhow::bail!("Invalid type format: {stream_msg:?}");
946 };
947 type_name = Some(
948 String::from_utf8(bytes.clone())
949 .map_err(|e| anyhow::anyhow!("Error parsing type: {e}"))?,
950 );
951 }
952 key if key == PAYLOAD_KIND_FIELD.as_bytes() => {
953 let redis::Value::BulkString(bytes) = &pair[1] else {
954 anyhow::bail!("Invalid payload kind format: {stream_msg:?}");
955 };
956 let value = std::str::from_utf8(bytes)
957 .map_err(|e| anyhow::anyhow!("Error parsing payload kind: {e}"))?;
958 anyhow::ensure!(
959 value == PAYLOAD_KIND_TYPED,
960 "Unknown payload kind '{value}'"
961 );
962 typed_payload = true;
963 }
964 b"encoding" => {
965 let redis::Value::BulkString(bytes) = &pair[1] else {
966 anyhow::bail!("Invalid encoding format: {stream_msg:?}");
967 };
968 let value = std::str::from_utf8(bytes)
969 .map_err(|e| anyhow::anyhow!("Error parsing encoding: {e}"))?;
970 encoding = value
971 .parse()
972 .map_err(|e| anyhow::anyhow!("Error parsing encoding: {e}"))?;
973 }
974 b"payload" => {
975 let redis::Value::BulkString(bytes) = &pair[1] else {
976 anyhow::bail!("Invalid payload format: {stream_msg:?}");
977 };
978 payload = Some(Bytes::copy_from_slice(bytes));
979 }
980 _ => {}
981 }
982 }
983
984 let Some(topic) = topic else {
985 anyhow::bail!("Stream message missing topic: {stream_msg:?}");
986 };
987 let Some(payload) = payload else {
988 anyhow::bail!("Stream message missing payload: {stream_msg:?}");
989 };
990 let payload_type = match type_name {
991 Some(type_name) if typed_payload => BusPayloadType::from_typed_name(&type_name)
992 .ok_or_else(|| anyhow::anyhow!("Unknown typed payload '{type_name}'"))?,
993 Some(type_name) => BusPayloadType::from_name(&type_name),
994 None if typed_payload => {
995 anyhow::bail!("Typed stream message missing type: {stream_msg:?}")
996 }
997 None => BusPayloadType::Custom(Ustr::default()),
998 };
999
1000 Ok(BusMessage::with_str_topic(
1001 topic,
1002 payload_type,
1003 payload,
1004 encoding,
1005 ))
1006}
1007
1008async fn run_heartbeat(
1009 heartbeat_interval_secs: u16,
1010 signal: Arc<AtomicBool>,
1011 pub_tx: tokio::sync::mpsc::UnboundedSender<BusMessage>,
1012) {
1013 log_task_started("heartbeat");
1014 log::debug!("Heartbeat at {heartbeat_interval_secs} second intervals");
1015
1016 let heartbeat_interval = Duration::from_secs(u64::from(heartbeat_interval_secs));
1017 let heartbeat_timer = tokio::time::interval(heartbeat_interval);
1018
1019 let check_interval = Duration::from_millis(100);
1020 let check_timer = tokio::time::interval(check_interval);
1021
1022 tokio::pin!(heartbeat_timer);
1023 tokio::pin!(check_timer);
1024
1025 loop {
1026 if signal.load(Ordering::Relaxed) {
1027 log::debug!("Received heartbeat terminate signal");
1028 break;
1029 }
1030
1031 tokio::select! {
1032 _ = heartbeat_timer.tick() => {
1033 let heartbeat = create_heartbeat_msg();
1034 if let Err(e) = pub_tx.send(heartbeat) {
1035 log::debug!("Error sending heartbeat: {e}");
1037 }
1038 },
1039 _ = check_timer.tick() => {}
1040 }
1041 }
1042
1043 log_task_stopped("heartbeat");
1044}
1045
1046fn create_heartbeat_msg() -> BusMessage {
1047 let payload = Bytes::from(
1048 Timestamp::now()
1049 .display_with_offset(Offset::UTC)
1050 .to_string()
1051 .into_bytes(),
1052 );
1053 BusMessage::with_str_topic(
1054 HEARTBEAT_TOPIC,
1055 BusPayloadType::Custom(Ustr::default()),
1056 payload,
1057 SerializationEncoding::default(),
1058 )
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063 use std::sync::atomic::AtomicUsize;
1064
1065 use nautilus_common::{msgbus::external_io_from_backing, testing::wait_until_async};
1066 use redis::Value;
1067 use rstest::*;
1068 use serde_json::json;
1069 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
1070
1071 use super::*;
1072
1073 #[rstest]
1074 fn test_default_redis_message_bus_config() {
1075 let config = RedisMessageBusConfig::default();
1076
1077 assert_eq!(config.host, None);
1078 assert_eq!(config.port, None);
1079 assert_eq!(config.username, None);
1080 assert_eq!(config.password, None);
1081 assert!(!config.ssl);
1082 assert_eq!(config.connection_timeout, 20);
1083 assert_eq!(config.response_timeout, 20);
1084 assert_eq!(config.number_of_retries, 100);
1085 assert_eq!(config.exponent_base, 2);
1086 assert_eq!(config.max_delay, 1000);
1087 assert_eq!(config.factor, 2);
1088 }
1089
1090 #[rstest]
1091 fn test_deserialize_redis_message_bus_config() {
1092 let config_json = json!({
1093 "host": "localhost",
1094 "port": 6379,
1095 "username": "user",
1096 "password": "pass",
1097 "ssl": true,
1098 "connection_timeout": 30,
1099 "response_timeout": 10,
1100 "number_of_retries": 3,
1101 "exponent_base": 2,
1102 "max_delay": 10,
1103 "factor": 2
1104 });
1105
1106 let config: RedisMessageBusConfig = serde_json::from_value(config_json).unwrap();
1107
1108 assert_eq!(config.host, Some("localhost".to_string()));
1109 assert_eq!(config.port, Some(6379));
1110 assert_eq!(config.username, Some("user".to_string()));
1111 assert_eq!(config.password, Some("pass".to_string()));
1112 assert!(config.ssl);
1113 assert_eq!(config.connection_timeout, 30);
1114 assert_eq!(config.response_timeout, 10);
1115 assert_eq!(config.number_of_retries, 3);
1116 assert_eq!(config.exponent_base, 2);
1117 assert_eq!(config.max_delay, 10);
1118 assert_eq!(config.factor, 2);
1119 }
1120
1121 #[rstest]
1122 fn test_deserialize_redis_message_bus_config_rejects_type_selector() {
1123 let config_json = json!({
1124 "type": "redis",
1125 });
1126
1127 let error = serde_json::from_value::<RedisMessageBusConfig>(config_json).unwrap_err();
1128
1129 assert!(error.to_string().contains("unknown field `type`"));
1130 }
1131
1132 #[rstest]
1133 fn test_bus_message_fields_distinguish_custom_and_typed_names() {
1134 let custom = BusMessage::with_str_topic(
1135 "external.test",
1136 BusPayloadType::Custom(Ustr::from("TradingCommand")),
1137 Bytes::from_static(b"custom"),
1138 SerializationEncoding::Json,
1139 );
1140 let typed = BusMessage::with_str_topic(
1141 "external.test",
1142 BusPayloadType::TradingCommand,
1143 Bytes::from_static(b"typed"),
1144 SerializationEncoding::Json,
1145 );
1146
1147 let custom_fields = bus_message_fields(&custom, "json");
1148 let typed_fields = bus_message_fields(&typed, "json");
1149 let custom_fields = custom_fields.as_slice();
1150 let typed_fields = typed_fields.as_slice();
1151
1152 assert_eq!(custom_fields.len(), 4);
1153 assert_eq!(typed_fields.len(), 5);
1154 assert_eq!(
1155 custom_fields
1156 .iter()
1157 .find(|(key, _)| *key == "type")
1158 .expect("custom fields must include type")
1159 .1,
1160 b"TradingCommand"
1161 );
1162 assert_eq!(
1163 typed_fields
1164 .iter()
1165 .find(|(key, _)| *key == "type")
1166 .expect("typed fields must include type")
1167 .1,
1168 b"TradingCommand"
1169 );
1170 assert!(
1171 custom_fields
1172 .iter()
1173 .all(|(key, _)| *key != PAYLOAD_KIND_FIELD)
1174 );
1175 assert_eq!(
1176 typed_fields
1177 .iter()
1178 .find(|(key, _)| *key == PAYLOAD_KIND_FIELD)
1179 .expect("typed fields must include payload kind")
1180 .1,
1181 PAYLOAD_KIND_TYPED.as_bytes()
1182 );
1183 }
1184
1185 #[rstest]
1186 fn test_decode_bus_message_valid() {
1187 let stream_msg = Value::Array(vec![
1188 Value::BulkString(b"topic".to_vec()),
1189 Value::BulkString(b"topic1".to_vec()),
1190 Value::BulkString(b"type".to_vec()),
1191 Value::BulkString(b"QuoteTick".to_vec()),
1192 Value::BulkString(b"payload".to_vec()),
1193 Value::BulkString(b"data1".to_vec()),
1194 Value::BulkString(b"encoding".to_vec()),
1195 Value::BulkString(b"msgpack".to_vec()),
1196 ]);
1197
1198 let result = decode_bus_message(&stream_msg);
1199 assert!(result.is_ok());
1200 let msg = result.unwrap();
1201 assert_eq!(msg.topic, "topic1");
1202 assert_eq!(msg.payload_type, BusPayloadType::QuoteTick);
1203 assert_eq!(msg.encoding, SerializationEncoding::MsgPack);
1204 assert_eq!(msg.payload, Bytes::from("data1"));
1205 }
1206
1207 #[rstest]
1208 fn test_decode_bus_message_defaults_legacy_headers() {
1209 let stream_msg = Value::Array(vec![
1210 Value::BulkString(b"topic".to_vec()),
1211 Value::BulkString(b"topic1".to_vec()),
1212 Value::BulkString(b"payload".to_vec()),
1213 Value::BulkString(b"data1".to_vec()),
1214 ]);
1215
1216 let result = decode_bus_message(&stream_msg);
1217 assert!(result.is_ok());
1218 let msg = result.unwrap();
1219 assert_eq!(msg.topic, "topic1");
1220 assert_eq!(msg.payload_type, BusPayloadType::Custom(Ustr::default()));
1221 assert_eq!(msg.encoding, SerializationEncoding::Json);
1222 assert_eq!(msg.payload, Bytes::from("data1"));
1223 }
1224
1225 #[rstest]
1226 fn test_decode_bus_message_reads_chrono_heartbeat() {
1227 let heartbeat =
1228 include_str!("../../test_data/redis_msgbus_heartbeat_chrono.txt").trim_end();
1229 let stream_msg = Value::Array(vec![
1230 Value::BulkString(b"topic".to_vec()),
1231 Value::BulkString(HEARTBEAT_TOPIC.as_bytes().to_vec()),
1232 Value::BulkString(b"payload".to_vec()),
1233 Value::BulkString(heartbeat.as_bytes().to_vec()),
1234 ]);
1235
1236 let msg = decode_bus_message(&stream_msg).unwrap();
1237 let timestamp = std::str::from_utf8(&msg.payload)
1238 .unwrap()
1239 .parse::<Timestamp>()
1240 .unwrap();
1241
1242 assert_eq!(msg.topic, HEARTBEAT_TOPIC);
1243 assert_eq!(msg.payload_type, BusPayloadType::Custom(Ustr::default()));
1244 assert_eq!(msg.encoding, SerializationEncoding::Json);
1245 assert_eq!(msg.payload.as_ref(), heartbeat.as_bytes());
1246 assert_eq!(timestamp.as_nanosecond(), 1_785_805_323_456_789_000);
1247 }
1248
1249 #[rstest]
1250 fn test_decode_bus_message_typed_name_without_kind_is_custom() {
1251 let stream_msg = Value::Array(vec![
1252 Value::BulkString(b"topic".to_vec()),
1253 Value::BulkString(b"topic1".to_vec()),
1254 Value::BulkString(b"type".to_vec()),
1255 Value::BulkString(b"TradingCommand".to_vec()),
1256 Value::BulkString(b"payload".to_vec()),
1257 Value::BulkString(b"data1".to_vec()),
1258 ]);
1259
1260 let result = decode_bus_message(&stream_msg);
1261 assert!(result.is_ok());
1262 let msg = result.unwrap();
1263 assert_eq!(
1264 msg.payload_type,
1265 BusPayloadType::Custom(Ustr::from("TradingCommand"))
1266 );
1267 assert_eq!(msg.encoding, SerializationEncoding::Json);
1268 }
1269
1270 #[rstest]
1271 fn test_decode_bus_message_typed_kind_resolves_fixed_type() {
1272 let stream_msg = Value::Array(vec![
1273 Value::BulkString(b"topic".to_vec()),
1274 Value::BulkString(b"external.test".to_vec()),
1275 Value::BulkString(b"type".to_vec()),
1276 Value::BulkString(b"TradingCommand".to_vec()),
1277 Value::BulkString(PAYLOAD_KIND_FIELD.as_bytes().to_vec()),
1278 Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1279 Value::BulkString(b"payload".to_vec()),
1280 Value::BulkString(b"data1".to_vec()),
1281 ]);
1282
1283 let msg = decode_bus_message(&stream_msg).expect("typed message must decode");
1284
1285 assert_eq!(msg.topic, "external.test");
1286 assert_eq!(msg.payload_type, BusPayloadType::TradingCommand);
1287 assert_eq!(msg.encoding, SerializationEncoding::Json);
1288 assert_eq!(msg.payload, Bytes::from_static(b"data1"));
1289 }
1290
1291 #[rstest]
1292 fn test_decode_bus_message_rejects_invalid_typed_metadata() {
1293 let unknown_kind = stream_message_with_typed_metadata(
1294 Some("TradingCommand"),
1295 Value::BulkString(b"unknown".to_vec()),
1296 );
1297 let missing_type = stream_message_with_typed_metadata(
1298 None,
1299 Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1300 );
1301 let unknown_type = stream_message_with_typed_metadata(
1302 Some("UnknownPayload"),
1303 Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1304 );
1305 let invalid_kind =
1306 stream_message_with_typed_metadata(Some("TradingCommand"), Value::Int(42));
1307 let cases = [
1308 (&unknown_kind, "Unknown payload kind 'unknown'".to_string()),
1309 (
1310 &missing_type,
1311 format!("Typed stream message missing type: {missing_type:?}"),
1312 ),
1313 (
1314 &unknown_type,
1315 "Unknown typed payload 'UnknownPayload'".to_string(),
1316 ),
1317 (
1318 &invalid_kind,
1319 format!("Invalid payload kind format: {invalid_kind:?}"),
1320 ),
1321 ];
1322
1323 for (stream_msg, expected) in cases {
1324 let error =
1325 decode_bus_message(stream_msg).expect_err("invalid typed metadata must not decode");
1326
1327 assert_eq!(error.to_string(), expected);
1328 }
1329 }
1330
1331 fn stream_message_with_typed_metadata(type_name: Option<&str>, payload_kind: Value) -> Value {
1332 let mut fields = vec![
1333 Value::BulkString(b"topic".to_vec()),
1334 Value::BulkString(b"external.test".to_vec()),
1335 ];
1336
1337 if let Some(type_name) = type_name {
1338 fields.extend([
1339 Value::BulkString(b"type".to_vec()),
1340 Value::BulkString(type_name.as_bytes().to_vec()),
1341 ]);
1342 }
1343 fields.extend([
1344 Value::BulkString(PAYLOAD_KIND_FIELD.as_bytes().to_vec()),
1345 payload_kind,
1346 Value::BulkString(b"payload".to_vec()),
1347 Value::BulkString(b"data1".to_vec()),
1348 ]);
1349 Value::Array(fields)
1350 }
1351
1352 #[rstest]
1353 fn test_decode_bus_message_accepts_unordered_metadata_fields() {
1354 let stream_msg = Value::Array(vec![
1355 Value::BulkString(b"payload".to_vec()),
1356 Value::BulkString(b"data1".to_vec()),
1357 Value::BulkString(b"encoding".to_vec()),
1358 Value::BulkString(b"msgpack".to_vec()),
1359 Value::BulkString(b"type".to_vec()),
1360 Value::BulkString(b"TradeTick".to_vec()),
1361 Value::BulkString(b"topic".to_vec()),
1362 Value::BulkString(b"topic1".to_vec()),
1363 ]);
1364
1365 let msg = decode_bus_message(&stream_msg).unwrap();
1366
1367 assert_eq!(msg.topic, "topic1");
1368 assert_eq!(msg.payload_type, BusPayloadType::TradeTick);
1369 assert_eq!(msg.encoding, SerializationEncoding::MsgPack);
1370 assert_eq!(msg.payload, Bytes::from("data1"));
1371 }
1372
1373 #[rstest]
1374 fn test_decode_bus_message_rejects_invalid_encoding_header() {
1375 let stream_msg = Value::Array(vec![
1376 Value::BulkString(b"topic".to_vec()),
1377 Value::BulkString(b"topic1".to_vec()),
1378 Value::BulkString(b"encoding".to_vec()),
1379 Value::BulkString(b"invalid".to_vec()),
1380 Value::BulkString(b"payload".to_vec()),
1381 Value::BulkString(b"data1".to_vec()),
1382 ]);
1383
1384 let error = decode_bus_message(&stream_msg).unwrap_err();
1385
1386 assert!(
1387 error.to_string().contains("Error parsing encoding"),
1388 "{error:?}"
1389 );
1390 }
1391
1392 #[rstest]
1393 fn test_decode_bus_message_missing_fields() {
1394 let stream_msg = Value::Array(vec![
1395 Value::BulkString(b"0".to_vec()),
1396 Value::BulkString(b"topic1".to_vec()),
1397 ]);
1398
1399 let result = decode_bus_message(&stream_msg);
1400 assert!(result.is_err());
1401 assert_eq!(
1402 format!("{}", result.unwrap_err()),
1403 "Invalid stream message format: array([bulk-string('\"0\"'), bulk-string('\"topic1\"')])"
1404 );
1405 }
1406
1407 #[rstest]
1408 fn test_decode_bus_message_invalid_topic_format() {
1409 let stream_msg = Value::Array(vec![
1410 Value::BulkString(b"topic".to_vec()),
1411 Value::Int(42),
1412 Value::BulkString(b"payload".to_vec()),
1413 Value::BulkString(b"data1".to_vec()),
1414 ]);
1415
1416 let result = decode_bus_message(&stream_msg);
1417 assert!(result.is_err());
1418 assert_eq!(
1419 format!("{}", result.unwrap_err()),
1420 "Invalid topic format: array([bulk-string('\"topic\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1421 );
1422 }
1423
1424 #[rstest]
1425 fn test_decode_bus_message_invalid_type_format() {
1426 let stream_msg = Value::Array(vec![
1427 Value::BulkString(b"topic".to_vec()),
1428 Value::BulkString(b"topic1".to_vec()),
1429 Value::BulkString(b"type".to_vec()),
1430 Value::Int(42),
1431 Value::BulkString(b"payload".to_vec()),
1432 Value::BulkString(b"data1".to_vec()),
1433 ]);
1434
1435 let result = decode_bus_message(&stream_msg);
1436 assert!(result.is_err());
1437 assert_eq!(
1438 format!("{}", result.unwrap_err()),
1439 "Invalid type format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"type\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1440 );
1441 }
1442
1443 #[rstest]
1444 fn test_decode_bus_message_invalid_encoding_format() {
1445 let stream_msg = Value::Array(vec![
1446 Value::BulkString(b"topic".to_vec()),
1447 Value::BulkString(b"topic1".to_vec()),
1448 Value::BulkString(b"encoding".to_vec()),
1449 Value::Int(42),
1450 Value::BulkString(b"payload".to_vec()),
1451 Value::BulkString(b"data1".to_vec()),
1452 ]);
1453
1454 let result = decode_bus_message(&stream_msg);
1455 assert!(result.is_err());
1456 assert_eq!(
1457 format!("{}", result.unwrap_err()),
1458 "Invalid encoding format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"encoding\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1459 );
1460 }
1461
1462 #[rstest]
1463 fn test_decode_bus_message_invalid_payload_format() {
1464 let stream_msg = Value::Array(vec![
1465 Value::BulkString(b"topic".to_vec()),
1466 Value::BulkString(b"topic1".to_vec()),
1467 Value::BulkString(b"payload".to_vec()),
1468 Value::Int(42),
1469 ]);
1470
1471 let result = decode_bus_message(&stream_msg);
1472 assert!(result.is_err());
1473 assert_eq!(
1474 format!("{}", result.unwrap_err()),
1475 "Invalid payload format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"payload\"'), int(42)])"
1476 );
1477 }
1478
1479 #[rstest]
1480 fn test_decode_bus_message_invalid_stream_msg_format() {
1481 let stream_msg = Value::BulkString(b"not an array".to_vec());
1482
1483 let result = decode_bus_message(&stream_msg);
1484 assert!(result.is_err());
1485 assert_eq!(
1486 format!("{}", result.unwrap_err()),
1487 "Invalid stream message format: bulk-string('\"not an array\"')"
1488 );
1489 }
1490
1491 #[rstest]
1492 fn test_new_rejects_zero_heartbeat_interval() {
1493 let trader_id = TraderId::from("tester-001");
1494 let instance_id = UUID4::new();
1495 let config = MessageBusConfig {
1496 heartbeat_interval_secs: Some(0),
1497 ..Default::default()
1498 };
1499
1500 let result = RedisMessageBusBacking::new(
1501 trader_id,
1502 instance_id,
1503 config,
1504 RedisMessageBusConfig::default(),
1505 );
1506
1507 assert!(result.is_err());
1508 assert_eq!(
1509 result.unwrap_err().to_string(),
1510 "heartbeat_interval_secs must be greater than 0"
1511 );
1512 }
1513
1514 #[rstest]
1515 fn test_stream_retry_delay_uses_config_bounds() {
1516 let config = RedisMessageBusConfig {
1517 factor: 10,
1518 exponent_base: 2,
1519 max_delay: 1,
1520 ..Default::default()
1521 };
1522
1523 assert_eq!(stream_retry_delay(&config, 0), Duration::from_millis(10));
1524 assert_eq!(stream_retry_delay(&config, 1), Duration::from_millis(20));
1525 assert_eq!(stream_retry_delay(&config, 10), Duration::from_secs(1));
1526 }
1527
1528 #[rstest]
1529 fn test_stream_error_retry_classification() {
1530 let dropped =
1531 redis::RedisError::from(std::io::Error::from(std::io::ErrorKind::ConnectionReset));
1532 let client: redis::RedisError = (redis::ErrorKind::Client, "client error").into();
1533
1534 assert!(is_retryable_stream_error(&dropped));
1535 assert!(!is_retryable_stream_error(&client));
1536 }
1537
1538 #[tokio::test]
1539 async fn test_wait_for_retry_delay_returns_false_when_signaled() {
1540 let stream_signal = Arc::new(AtomicBool::new(true));
1541 let signal = Arc::clone(&stream_signal);
1542 let fut = async move { wait_for_retry_delay(Duration::from_secs(30), &signal).await };
1543
1544 let handle = tokio::spawn(fut);
1545
1546 wait_until_async(|| async { handle.is_finished() }, Duration::from_secs(1)).await;
1547
1548 assert!(!handle.await.unwrap());
1549 }
1550
1551 #[rstest]
1552 #[case::loading("LOADING Redis is loading the dataset in memory")]
1553 #[case::master_down(
1554 "MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'."
1555 )]
1556 #[tokio::test(flavor = "multi_thread")]
1557 async fn test_publish_messages_retries_transaction_aborted_by_retryable_error(
1558 #[case] queued_error: &'static str,
1559 ) {
1560 let fake = FakeRedis::start(queued_error).await;
1561 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1562 let backing = fake.config();
1563
1564 let handle = tokio::spawn(async move {
1565 publish_messages(
1566 rx,
1567 TraderId::from("tester-001"),
1568 UUID4::new(),
1569 MessageBusConfig::default(),
1570 backing,
1571 )
1572 .await
1573 });
1574
1575 tx.send(fake_bus_message()).unwrap();
1576 wait_until_async(
1577 || {
1578 let aborted = Arc::clone(&fake.aborted);
1579 async move { aborted.load(Ordering::SeqCst) > 0 }
1580 },
1581 Duration::from_secs(5),
1582 )
1583 .await;
1584 assert!(!handle.is_finished());
1585
1586 fake.accept_writes.store(true, Ordering::SeqCst);
1587 wait_until_async(
1588 || {
1589 let committed = Arc::clone(&fake.committed);
1590 async move { committed.load(Ordering::SeqCst) > 0 }
1591 },
1592 Duration::from_secs(5),
1593 )
1594 .await;
1595
1596 tx.send(BusMessage::new_close()).unwrap();
1597 let result = handle.await.unwrap();
1598
1599 assert!(result.is_ok());
1600 }
1601
1602 #[rstest]
1603 #[tokio::test(flavor = "multi_thread")]
1604 async fn test_publish_messages_returns_transaction_aborted_by_unrecoverable_error() {
1605 let fake = FakeRedis::start("ERR syntax error").await;
1606 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1607 let backing = fake.config();
1608
1609 let handle = tokio::spawn(async move {
1610 publish_messages(
1611 rx,
1612 TraderId::from("tester-001"),
1613 UUID4::new(),
1614 MessageBusConfig::default(),
1615 backing,
1616 )
1617 .await
1618 });
1619
1620 tx.send(fake_bus_message()).unwrap();
1621
1622 let result = tokio::time::timeout(Duration::from_secs(5), handle)
1623 .await
1624 .expect("an unrecoverable aborted transaction should end the task")
1625 .unwrap();
1626
1627 assert_eq!(fake.aborted.load(Ordering::SeqCst), 1);
1628 assert!(
1629 result
1630 .unwrap_err()
1631 .to_string()
1632 .contains("Error publishing to stream")
1633 );
1634 }
1635
1636 #[rstest]
1637 fn test_external_io_from_backing_takes_stream_receiver() {
1638 let (stream_tx, stream_rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
1639 let backing = backing_with_stream_receiver(stream_rx);
1640 let message = BusMessage::with_str_topic(
1641 "events/data",
1642 BusPayloadType::QuoteTick,
1643 Bytes::from_static(b"payload"),
1644 SerializationEncoding::Json,
1645 );
1646
1647 let (_egress, mut ingress) = external_io_from_backing(Box::new(backing));
1648 stream_tx.try_send(message.clone()).unwrap();
1649 let mut receiver = ingress.take_receiver().unwrap();
1650 let received = receiver.try_recv().unwrap();
1651
1652 assert_eq!(received.topic, message.topic);
1653 assert_eq!(received.payload, message.payload);
1654 assert!(ingress.take_receiver().is_err());
1655 }
1656
1657 fn backing_with_stream_receiver(
1658 stream_rx: tokio::sync::mpsc::Receiver<BusMessage>,
1659 ) -> RedisMessageBusBacking {
1660 let (pub_tx, _pub_rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1661 RedisMessageBusBacking {
1662 trader_id: TraderId::from("tester-001"),
1663 instance_id: UUID4::new(),
1664 pub_tx,
1665 pub_handle: None,
1666 stream_rx: Some(stream_rx),
1667 stream_handle: None,
1668 stream_signal: Arc::new(AtomicBool::new(false)),
1669 heartbeat_handle: None,
1670 heartbeat_signal: Arc::new(AtomicBool::new(false)),
1671 }
1672 }
1673
1674 fn fake_bus_message() -> BusMessage {
1675 BusMessage::with_str_topic(
1676 "test_topic",
1677 BusPayloadType::QuoteTick,
1678 Bytes::from("test_payload"),
1679 SerializationEncoding::Json,
1680 )
1681 }
1682
1683 struct FakeRedis {
1686 port: u16,
1687 accept_writes: Arc<AtomicBool>,
1688 aborted: Arc<AtomicUsize>,
1689 committed: Arc<AtomicUsize>,
1690 }
1691
1692 impl FakeRedis {
1693 async fn start(queued_error: &'static str) -> Self {
1694 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1695 let port = listener.local_addr().unwrap().port();
1696 let accept_writes = Arc::new(AtomicBool::new(false));
1697 let aborted = Arc::new(AtomicUsize::new(0));
1698 let committed = Arc::new(AtomicUsize::new(0));
1699
1700 let accept_writes_clone = Arc::clone(&accept_writes);
1701 let aborted_clone = Arc::clone(&aborted);
1702 let committed_clone = Arc::clone(&committed);
1703
1704 tokio::spawn(async move {
1705 while let Ok((socket, _)) = listener.accept().await {
1706 let accept_writes = Arc::clone(&accept_writes_clone);
1707 let aborted = Arc::clone(&aborted_clone);
1708 let committed = Arc::clone(&committed_clone);
1709
1710 tokio::spawn(async move {
1711 let (read, mut write) = socket.into_split();
1712 let mut reader = tokio::io::BufReader::new(read);
1713 let mut queued = 0;
1714 let mut queue_failed = false;
1715
1716 while let Some(command) = read_resp_command(&mut reader).await {
1717 let reply = match command[0].to_ascii_uppercase().as_slice() {
1718 b"INFO" => {
1719 let info = "redis_version:8.0.0\r\n";
1720 format!("${}\r\n{info}\r\n", info.len())
1721 }
1722 b"MULTI" => {
1723 queued = 0;
1724 queue_failed = false;
1725 "+OK\r\n".to_string()
1726 }
1727 b"XADD" if accept_writes.load(Ordering::SeqCst) => {
1728 queued += 1;
1729 "+QUEUED\r\n".to_string()
1730 }
1731 b"XADD" => {
1732 queue_failed = true;
1733 format!("-{queued_error}\r\n")
1734 }
1735 b"EXEC" if queue_failed => {
1736 aborted.fetch_add(1, Ordering::SeqCst);
1737 "-EXECABORT Transaction discarded because of previous errors.\r\n"
1738 .to_string()
1739 }
1740 b"EXEC" => {
1741 committed.fetch_add(1, Ordering::SeqCst);
1742 format!("*{queued}\r\n{}", "$3\r\n0-1\r\n".repeat(queued))
1743 }
1744 _ => "+OK\r\n".to_string(),
1745 };
1746
1747 if write.write_all(reply.as_bytes()).await.is_err() {
1748 break;
1749 }
1750 }
1751 });
1752 }
1753 });
1754
1755 Self {
1756 port,
1757 accept_writes,
1758 aborted,
1759 committed,
1760 }
1761 }
1762
1763 fn config(&self) -> RedisMessageBusConfig {
1764 RedisMessageBusConfig {
1765 port: Some(self.port),
1766 ..Default::default()
1767 }
1768 }
1769 }
1770
1771 async fn read_resp_command(
1772 reader: &mut tokio::io::BufReader<tokio::net::tcp::OwnedReadHalf>,
1773 ) -> Option<Vec<Vec<u8>>> {
1774 let mut line = String::new();
1775 if reader.read_line(&mut line).await.ok()? == 0 {
1776 return None;
1777 }
1778 let count: usize = line.trim_end().strip_prefix('*')?.parse().ok()?;
1779 let mut command = Vec::with_capacity(count);
1780
1781 for _ in 0..count {
1782 line.clear();
1783 reader.read_line(&mut line).await.ok()?;
1784 let len: usize = line.trim_end().strip_prefix('$')?.parse().ok()?;
1785 let mut arg = vec![0; len + 2];
1786 reader.read_exact(&mut arg).await.ok()?;
1787 arg.truncate(len);
1788 command.push(arg);
1789 }
1790
1791 Some(command)
1792 }
1793}
1794
1795#[cfg(target_os = "linux")] #[cfg(test)]
1797mod serial_tests {
1798 use std::{
1799 sync::{atomic::AtomicUsize, mpsc},
1800 thread,
1801 };
1802
1803 #[cfg(feature = "python")]
1804 use nautilus_common::python::msgbus::get_global_msgbus_factory_registry;
1805 use nautilus_common::{
1806 enums::Environment,
1807 msgbus::{self, TypedHandler},
1808 testing::wait_until_async,
1809 };
1810 use nautilus_live::{
1811 builder::LiveNodeBuilder,
1812 config::{LiveExecutionEngineConfig, LiveNodeConfig},
1813 };
1814 use nautilus_model::data::{QuoteTick, TradeTick};
1815 #[cfg(feature = "python")]
1816 use pyo3::{Py, Python, types::PyModule};
1817 use redis::aio::ConnectionManager;
1818 use rstest::*;
1819
1820 use super::*;
1821
1822 #[fixture]
1823 async fn redis_connection() -> ConnectionManager {
1824 let config = RedisMessageBusConfig {
1825 connection_timeout: 1,
1826 number_of_retries: 0,
1827 ..Default::default()
1828 };
1829 create_redis_connection(MSGBUS_STREAM, &config)
1830 .await
1831 .expect("A running Redis service is required for this test")
1832 }
1833
1834 fn redis_msgbus_factory(config: RedisMessageBusConfig) -> Box<dyn MessageBusBackingFactory> {
1835 #[cfg(feature = "python")]
1836 {
1837 Python::initialize();
1838 Python::attach(|py| {
1839 let module = PyModule::new(py, "infrastructure").unwrap();
1840 crate::python::infrastructure(py, &module).unwrap();
1841 let factory = Py::new(py, config).unwrap().into_any();
1842
1843 get_global_msgbus_factory_registry()
1844 .extract(py, factory)
1845 .unwrap()
1846 })
1847 }
1848
1849 #[cfg(not(feature = "python"))]
1850 Box::new(config)
1851 }
1852
1853 #[rstest]
1854 #[tokio::test(flavor = "multi_thread")]
1855 async fn test_stream_messages_terminate_signal(#[future] redis_connection: ConnectionManager) {
1856 let _con = redis_connection.await;
1857 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1858
1859 let trader_id = TraderId::from("tester-001");
1860 let instance_id = UUID4::new();
1861 let config = MessageBusConfig {
1862 use_instance_id: true,
1863 ..Default::default()
1864 };
1865
1866 let stream_key = get_stream_key(trader_id, instance_id, &config);
1867 let external_streams = vec![stream_key.clone()];
1868 let stream_signal = Arc::new(AtomicBool::new(false));
1869 let stream_signal_clone = Arc::clone(&stream_signal);
1870
1871 let handle = tokio::spawn(async move {
1873 stream_messages(
1874 tx,
1875 RedisMessageBusConfig::default(),
1876 external_streams,
1877 stream_signal_clone,
1878 )
1879 .await
1880 .unwrap();
1881 });
1882
1883 stream_signal.store(true, Ordering::Relaxed);
1884 let _ = rx.recv().await; rx.close();
1888 handle.await.unwrap();
1889 }
1890
1891 #[rstest]
1892 #[tokio::test(flavor = "multi_thread")]
1893 async fn test_stream_messages_when_receiver_closed(
1894 #[future] redis_connection: ConnectionManager,
1895 ) {
1896 let mut con = redis_connection.await;
1897 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1898
1899 let trader_id = TraderId::from("tester-001");
1900 let instance_id = UUID4::new();
1901 let config = MessageBusConfig {
1902 use_instance_id: true,
1903 ..Default::default()
1904 };
1905
1906 let stream_key = get_stream_key(trader_id, instance_id, &config);
1907 let external_streams = vec![stream_key.clone()];
1908 let stream_signal = Arc::new(AtomicBool::new(false));
1909 let stream_signal_clone = Arc::clone(&stream_signal);
1910
1911 let clock = get_atomic_clock_realtime();
1914 let future_id = (clock.get_time_ms() + 1_000_000).to_string();
1915
1916 let _: () = con
1918 .xadd(
1919 stream_key,
1920 future_id,
1921 &[("topic", "topic1"), ("payload", "data1")],
1922 )
1923 .await
1924 .unwrap();
1925
1926 rx.close();
1928
1929 let handle = tokio::spawn(async move {
1931 stream_messages(
1932 tx,
1933 RedisMessageBusConfig::default(),
1934 external_streams,
1935 stream_signal_clone,
1936 )
1937 .await
1938 .unwrap();
1939 });
1940
1941 handle.await.unwrap();
1943 }
1944
1945 #[rstest]
1946 #[tokio::test(flavor = "multi_thread")]
1947 async fn test_stream_messages(#[future] redis_connection: ConnectionManager) {
1948 let mut con = redis_connection.await;
1949 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1950
1951 let trader_id = TraderId::from("tester-001");
1952 let instance_id = UUID4::new();
1953 let config = MessageBusConfig {
1954 use_instance_id: true,
1955 ..Default::default()
1956 };
1957
1958 let stream_key = get_stream_key(trader_id, instance_id, &config);
1959 let external_streams = vec![stream_key.clone()];
1960 let stream_signal = Arc::new(AtomicBool::new(false));
1961 let stream_signal_clone = Arc::clone(&stream_signal);
1962
1963 let clock = get_atomic_clock_realtime();
1966 let future_id = (clock.get_time_ms() + 1_000_000).to_string();
1967
1968 let _: () = con
1970 .xadd(
1971 stream_key,
1972 future_id,
1973 &[("topic", "topic1"), ("payload", "data1")],
1974 )
1975 .await
1976 .unwrap();
1977
1978 let handle = tokio::spawn(async move {
1980 stream_messages(
1981 tx,
1982 RedisMessageBusConfig::default(),
1983 external_streams,
1984 stream_signal_clone,
1985 )
1986 .await
1987 .unwrap();
1988 });
1989
1990 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
1992 assert_eq!(msg.topic, "topic1");
1993 assert_eq!(msg.payload, Bytes::from("data1"));
1994
1995 rx.close();
1997 stream_signal.store(true, Ordering::Relaxed);
1998 handle.await.unwrap();
1999 }
2000
2001 #[rstest]
2002 #[tokio::test(flavor = "multi_thread")]
2003 async fn test_stream_messages_skips_preexisting_entries(
2004 #[future] redis_connection: ConnectionManager,
2005 ) {
2006 let mut con = redis_connection.await;
2007 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2008
2009 let suffix = UUID4::new();
2010 let stream_key = format!("test:stream:no-backlog:{suffix}");
2011 let external_streams = vec![stream_key.clone()];
2012 let stream_signal = Arc::new(AtomicBool::new(false));
2013 let stream_signal_clone = Arc::clone(&stream_signal);
2014
2015 let _: () = con
2016 .xadd(
2017 &stream_key,
2018 "1-0",
2019 &[("topic", "preexisting"), ("payload", "old")],
2020 )
2021 .await
2022 .unwrap();
2023
2024 let handle = tokio::spawn(async move {
2025 stream_messages(
2026 tx,
2027 RedisMessageBusConfig::default(),
2028 external_streams,
2029 stream_signal_clone,
2030 )
2031 .await
2032 .unwrap();
2033 });
2034
2035 let future_id = (get_atomic_clock_realtime().get_time_ms() + 1_000_000).to_string();
2036 let _: () = con
2037 .xadd(
2038 &stream_key,
2039 future_id,
2040 &[("topic", "live"), ("payload", "new")],
2041 )
2042 .await
2043 .unwrap();
2044
2045 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2046
2047 rx.close();
2048 stream_signal.store(true, Ordering::Relaxed);
2049 handle.await.unwrap();
2050 let _: usize = con.del(stream_key).await.unwrap();
2051
2052 assert_eq!(msg.topic, "live");
2053 assert_eq!(msg.payload, Bytes::from("new"));
2054 }
2055
2056 #[rstest]
2057 #[tokio::test(flavor = "multi_thread")]
2058 async fn test_stream_messages_skips_malformed_entry(
2059 #[future] redis_connection: ConnectionManager,
2060 ) {
2061 let mut con = redis_connection.await;
2062 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2063
2064 let suffix = UUID4::new();
2065 let stream_key = format!("test:stream:malformed:{suffix}");
2066 let external_streams = vec![stream_key.clone()];
2067 let stream_signal = Arc::new(AtomicBool::new(false));
2068 let stream_signal_clone = Arc::clone(&stream_signal);
2069
2070 let clock = get_atomic_clock_realtime();
2071 let base_id = clock.get_time_ms() + 1_000_000;
2072
2073 let _: () = con
2074 .xadd(
2075 &stream_key,
2076 format!("{}", base_id + 1),
2077 &[("topic", "missing-payload")],
2078 )
2079 .await
2080 .unwrap();
2081 let _: () = con
2082 .xadd(
2083 &stream_key,
2084 format!("{}", base_id + 2),
2085 &[("topic", "valid"), ("payload", "data")],
2086 )
2087 .await
2088 .unwrap();
2089
2090 let handle = tokio::spawn(async move {
2091 stream_messages(
2092 tx,
2093 RedisMessageBusConfig::default(),
2094 external_streams,
2095 stream_signal_clone,
2096 )
2097 .await
2098 .unwrap();
2099 });
2100
2101 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2102
2103 rx.close();
2104 stream_signal.store(true, Ordering::Relaxed);
2105 handle.await.unwrap();
2106
2107 assert_eq!(msg.topic, "valid");
2108 assert_eq!(msg.payload, Bytes::from("data"));
2109 }
2110
2111 #[rstest]
2112 #[tokio::test(flavor = "multi_thread")]
2113 async fn test_stream_messages_returns_unrecoverable_read_error(
2114 #[future] redis_connection: ConnectionManager,
2115 ) {
2116 let mut con = redis_connection.await;
2117 let (tx, _rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2118
2119 let suffix = UUID4::new();
2120 let stream_key = format!("test:stream:wrong-type:{suffix}");
2121 let external_streams = vec![stream_key.clone()];
2122 let stream_signal = Arc::new(AtomicBool::new(false));
2123
2124 let _: () = con.set(&stream_key, "not-a-stream").await.unwrap();
2125
2126 let result = stream_messages(
2127 tx,
2128 RedisMessageBusConfig::default(),
2129 external_streams,
2130 stream_signal,
2131 )
2132 .await;
2133
2134 let _: () = con.del(&stream_key).await.unwrap();
2135
2136 assert!(result.is_err());
2137 assert!(
2138 result
2139 .unwrap_err()
2140 .to_string()
2141 .contains("Error reading from stream")
2142 );
2143 }
2144
2145 #[rstest]
2146 #[tokio::test(flavor = "multi_thread")]
2147 async fn test_stream_connection_returns_none_when_signaled() {
2148 let config = RedisMessageBusConfig {
2149 port: Some(1),
2150 connection_timeout: 20,
2151 ..Default::default()
2152 };
2153 let stream_signal = Arc::new(AtomicBool::new(true));
2154 let signal = Arc::clone(&stream_signal);
2155 let handle = tokio::spawn(async move { connect_stream_connection(&config, &signal).await });
2156
2157 wait_until_async(|| async { handle.is_finished() }, Duration::from_secs(1)).await;
2158
2159 assert!(handle.await.unwrap().unwrap().is_none());
2160 }
2161
2162 #[rstest]
2163 #[case::market_data(BusPayloadType::QuoteTick)]
2164 #[case::typed(BusPayloadType::TradingCommand)]
2165 #[tokio::test(flavor = "multi_thread")]
2166 async fn test_publish_messages(
2167 #[future] redis_connection: ConnectionManager,
2168 #[case] payload_type: BusPayloadType,
2169 ) {
2170 let mut con = redis_connection.await;
2171 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2172
2173 let trader_id = TraderId::from("tester-001");
2174 let instance_id = UUID4::new();
2175 let config = MessageBusConfig {
2176 use_instance_id: true,
2177 stream_per_topic: false,
2178 ..Default::default()
2179 };
2180 let stream_key = get_stream_key(trader_id, instance_id, &config);
2181
2182 let handle = tokio::spawn(async move {
2184 publish_messages(
2185 rx,
2186 trader_id,
2187 instance_id,
2188 config,
2189 RedisMessageBusConfig::default(),
2190 )
2191 .await
2192 .unwrap();
2193 });
2194
2195 let msg = BusMessage::with_str_topic(
2197 "test_topic",
2198 payload_type,
2199 Bytes::from("test_payload"),
2200 SerializationEncoding::Json,
2201 );
2202 tx.send(msg).unwrap();
2203
2204 wait_until_async(
2206 || {
2207 let mut con = con.clone();
2208 let stream_key = stream_key.clone();
2209 async move {
2210 let messages: RedisStreamBulk =
2211 con.xread(&[&stream_key], &["0"]).await.unwrap();
2212 !messages.is_empty()
2213 }
2214 },
2215 Duration::from_secs(3),
2216 )
2217 .await;
2218
2219 let messages: RedisStreamBulk = con.xread(&[&stream_key], &["0"]).await.unwrap();
2221 assert_eq!(messages.len(), 1);
2222 let stream_msgs = messages[0].get(&stream_key).unwrap();
2223 let stream_msg_array = &stream_msgs[0].values().next().unwrap();
2224 let decoded_message = decode_bus_message(stream_msg_array).unwrap();
2225 assert_eq!(decoded_message.topic, "test_topic");
2226 assert_eq!(decoded_message.payload_type, payload_type);
2227 assert_eq!(decoded_message.encoding, SerializationEncoding::Json);
2228 assert_eq!(decoded_message.payload, Bytes::from("test_payload"));
2229
2230 let msg = BusMessage::new_close();
2232 tx.send(msg).unwrap();
2233
2234 handle.await.unwrap();
2236 }
2237
2238 #[rstest]
2239 #[tokio::test(flavor = "multi_thread")]
2240 async fn test_publish_messages_applies_autotrim_maxlen(
2241 #[future] redis_connection: ConnectionManager,
2242 ) {
2243 let mut con = redis_connection.await;
2244 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2245 let trader_id = TraderId::from("tester-001");
2246 let instance_id = UUID4::new();
2247 let config = MessageBusConfig {
2248 buffer_interval_ms: Some(10),
2249 autotrim_mins: Some(60),
2250 autotrim_maxlen: Some(10),
2251 use_instance_id: true,
2252 stream_per_topic: false,
2253 ..Default::default()
2254 };
2255 let stream_key = get_stream_key(trader_id, instance_id, &config);
2256
2257 let handle = tokio::spawn(async move {
2258 publish_messages(
2259 rx,
2260 trader_id,
2261 instance_id,
2262 config,
2263 RedisMessageBusConfig::default(),
2264 )
2265 .await
2266 .unwrap();
2267 });
2268 let msg = BusMessage::with_str_topic(
2269 "test_topic",
2270 BusPayloadType::QuoteTick,
2271 Bytes::from_static(b"test_payload"),
2272 SerializationEncoding::Json,
2273 );
2274
2275 let messages_sent = 250;
2278
2279 for _ in 0..messages_sent {
2280 tx.send(msg.clone()).unwrap();
2281 }
2282 tx.send(BusMessage::new_close()).unwrap();
2283 handle.await.unwrap();
2284
2285 let stream_len: usize = con.xlen(&stream_key).await.unwrap();
2286 let _: usize = con.del(stream_key).await.unwrap();
2287
2288 assert!(stream_len >= 10);
2289 assert!(stream_len < messages_sent);
2290 }
2291
2292 #[rstest]
2293 #[case::immediate(None)]
2294 #[case::buffered(Some(10))]
2295 #[tokio::test(flavor = "multi_thread")]
2296 async fn test_publish_messages_retries_buffer_after_connection_loss(
2297 #[future] redis_connection: ConnectionManager,
2298 #[case] buffer_interval_ms: Option<u32>,
2299 ) {
2300 let mut con = redis_connection.await;
2301 let proxy = RedisProxy::start().await;
2302 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2303 let trader_id = TraderId::from("tester-001");
2304 let instance_id = UUID4::new();
2305 let config = MessageBusConfig {
2306 use_instance_id: true,
2307 stream_per_topic: false,
2308 buffer_interval_ms,
2309 ..Default::default()
2310 };
2311 let stream_key = get_stream_key(trader_id, instance_id, &config);
2312 let backing = proxy.config();
2313
2314 let handle = tokio::spawn(async move {
2315 publish_messages(rx, trader_id, instance_id, config, backing).await
2316 });
2317
2318 tx.send(test_bus_message("first")).unwrap();
2319 wait_for_stream_len(&con, &stream_key, 1).await;
2320
2321 proxy.interrupt().await;
2324 let backlog: Vec<String> = (0..2_500).map(|i| format!("backlog-{i}")).collect();
2325 for topic in &backlog {
2326 tx.send(test_bus_message(topic)).unwrap();
2327 }
2328 tokio::time::sleep(Duration::from_millis(200)).await;
2329
2330 assert_eq!(read_stream_topics(&mut con, &stream_key).await, ["first"]);
2331 assert!(!handle.is_finished());
2332
2333 proxy.resume();
2334 wait_for_last_topic(&con, &stream_key, backlog.last().unwrap()).await;
2335 tx.send(test_bus_message("last")).unwrap();
2336 wait_for_last_topic(&con, &stream_key, "last").await;
2337
2338 let topics = read_stream_topics(&mut con, &stream_key).await;
2339 tx.send(BusMessage::new_close()).unwrap();
2340 let result = handle.await.unwrap();
2341 let _: usize = con.del(&stream_key).await.unwrap();
2342
2343 let mut expected = vec!["first".to_string()];
2346 if topics.get(1).is_some_and(|topic| topic == "first") {
2347 expected.push("first".to_string());
2348 }
2349 expected.extend(backlog);
2350 expected.push("last".to_string());
2351 assert_eq!(topics, expected);
2352 assert!(result.is_ok());
2353 }
2354
2355 #[rstest]
2356 #[case::no_autotrim(None)]
2357 #[case::autotrim(Some(60))]
2358 #[tokio::test(flavor = "multi_thread")]
2359 async fn test_publish_messages_close_returns_during_outage(
2360 #[future] redis_connection: ConnectionManager,
2361 #[case] autotrim_mins: Option<u32>,
2362 ) {
2363 let mut con = redis_connection.await;
2364 let proxy = RedisProxy::start().await;
2365 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2366 let trader_id = TraderId::from("tester-001");
2367 let instance_id = UUID4::new();
2368
2369 let config = MessageBusConfig {
2371 use_instance_id: true,
2372 stream_per_topic: true,
2373 autotrim_mins,
2374 ..Default::default()
2375 };
2376 let stream_key = get_stream_key(trader_id, instance_id, &config);
2377 let first_key = format!("{stream_key}:first");
2378 let second_key = format!("{stream_key}:second");
2379 let backing = proxy.config();
2380
2381 let handle = tokio::spawn(async move {
2382 publish_messages(rx, trader_id, instance_id, config, backing).await
2383 });
2384
2385 tx.send(test_bus_message("first")).unwrap();
2386 wait_for_stream_len(&con, &first_key, 1).await;
2387
2388 proxy.interrupt().await;
2389 tx.send(test_bus_message("second")).unwrap();
2390 tx.send(BusMessage::new_close()).unwrap();
2391
2392 let result = tokio::time::timeout(Duration::from_secs(5), handle)
2393 .await
2394 .expect("close should return during an outage")
2395 .unwrap();
2396 let second_len: usize = con.xlen(&second_key).await.unwrap();
2397 let _: usize = con.del(&[&first_key, &second_key]).await.unwrap();
2398
2399 let error = result.unwrap_err().to_string();
2402 let unpublished = ["1", "2"].map(|count| {
2403 format!("Failed to publish buffered messages before closing ({count} unpublished)")
2404 });
2405 assert_eq!(second_len, 0);
2406 assert!(unpublished.contains(&error), "{error}");
2407 }
2408
2409 #[rstest]
2410 #[tokio::test(flavor = "multi_thread")]
2411 async fn test_publish_messages_returns_unrecoverable_write_error(
2412 #[future] redis_connection: ConnectionManager,
2413 ) {
2414 let mut con = redis_connection.await;
2415 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2416 let trader_id = TraderId::from("tester-001");
2417 let instance_id = UUID4::new();
2418 let config = MessageBusConfig {
2419 use_instance_id: true,
2420 stream_per_topic: false,
2421 ..Default::default()
2422 };
2423 let stream_key = get_stream_key(trader_id, instance_id, &config);
2424
2425 let _: () = con.set(&stream_key, "not-a-stream").await.unwrap();
2426
2427 let handle = tokio::spawn(async move {
2428 publish_messages(
2429 rx,
2430 trader_id,
2431 instance_id,
2432 config,
2433 RedisMessageBusConfig::default(),
2434 )
2435 .await
2436 });
2437
2438 tx.send(test_bus_message("first")).unwrap();
2439
2440 let result = tokio::time::timeout(Duration::from_secs(5), handle)
2441 .await
2442 .expect("a non-retryable write error should end the task")
2443 .unwrap();
2444 let _: usize = con.del(&stream_key).await.unwrap();
2445
2446 assert!(
2447 result
2448 .unwrap_err()
2449 .to_string()
2450 .contains("Error publishing to stream")
2451 );
2452 }
2453
2454 #[rstest]
2455 #[tokio::test(flavor = "multi_thread")]
2456 async fn test_two_live_nodes_publish_and_ingest_external_redis_stream(
2457 #[future] redis_connection: ConnectionManager,
2458 ) {
2459 let _con = redis_connection.await;
2460 let redis_config = RedisMessageBusConfig::default();
2461 let trader_a = TraderId::from("NODEA-001");
2462 let instance_a = UUID4::new();
2463 let node_a_msgbus = MessageBusConfig {
2464 use_instance_id: true,
2465 stream_per_topic: false,
2466 ..Default::default()
2467 };
2468 let stream_key = get_stream_key(trader_a, instance_a, &node_a_msgbus);
2469 let node_b_msgbus = MessageBusConfig {
2470 external_streams: Some(vec![stream_key]),
2471 stream_per_topic: false,
2472 ..Default::default()
2473 };
2474 let quote = QuoteTick::default();
2475 let trade = TradeTick::default();
2476 let ready_topic = "test.quotes.ready";
2477 let (built_tx, built_rx) = mpsc::channel::<()>();
2478 let (ready_tx, ready_rx) = mpsc::channel::<()>();
2479 let (quote_tx, quote_rx) = mpsc::channel::<QuoteTick>();
2480 let (trade_tx, trade_rx) = mpsc::channel::<TradeTick>();
2481
2482 let node_b = thread::spawn({
2483 let redis_config = redis_config.clone();
2484 move || -> anyhow::Result<()> {
2485 let runtime = tokio::runtime::Builder::new_multi_thread()
2486 .worker_threads(2)
2487 .enable_all()
2488 .build()?;
2489
2490 runtime.block_on(async move {
2491 let config = LiveNodeConfig {
2492 environment: Environment::Sandbox,
2493 trader_id: TraderId::from("NODEB-001"),
2494 msgbus: Some(node_b_msgbus),
2495 exec_engine: LiveExecutionEngineConfig {
2496 reconciliation: false,
2497 ..Default::default()
2498 },
2499 delay_post_stop: Duration::ZERO,
2500 timeout_connection: Duration::from_millis(500),
2501 timeout_disconnection: Duration::from_millis(500),
2502 ..Default::default()
2503 };
2504 let mut node = LiveNodeBuilder::from_config(config)?
2505 .with_external_msgbus_factory(redis_msgbus_factory(redis_config))
2506 .build()?;
2507 let handle = node.handle();
2508 let quote_handler = TypedHandler::from({
2509 let quote_tx = quote_tx.clone();
2510 let handle = handle.clone();
2511 move |quote: &QuoteTick| {
2512 let _ = quote_tx.send(*quote);
2513 handle.stop();
2514 }
2515 });
2516 let trade_handler = TypedHandler::from(move |trade: &TradeTick| {
2517 let _ = trade_tx.send(*trade);
2518 });
2519
2520 let ready_handler = TypedHandler::from(move |_: &QuoteTick| {
2521 let _ = ready_tx.send(());
2522 });
2523
2524 msgbus::subscribe_quotes(ready_topic.into(), ready_handler, None);
2525 msgbus::subscribe_quotes("data.quotes.*".into(), quote_handler, None);
2526 msgbus::subscribe_trades("data.trades.*".into(), trade_handler, None);
2527 msgbus::get_message_bus()
2528 .borrow_mut()
2529 .add_streaming_type(BusPayloadType::QuoteTick);
2530 built_tx.send(())?;
2531 let result = tokio::time::timeout(Duration::from_secs(10), node.run()).await;
2532 msgbus::get_message_bus().borrow_mut().dispose();
2533
2534 match result {
2535 Ok(Ok(())) => Ok(()),
2536 Ok(Err(e)) => Err(e),
2537 Err(e) => anyhow::bail!("node B timed out: {e}"),
2538 }
2539 })
2540 }
2541 });
2542
2543 built_rx
2544 .recv_timeout(Duration::from_secs(5))
2545 .expect("node B should build and register subscriptions");
2546
2547 let node_a = thread::spawn(move || -> anyhow::Result<()> {
2548 let runtime = tokio::runtime::Builder::new_multi_thread()
2549 .worker_threads(2)
2550 .enable_all()
2551 .build()?;
2552
2553 runtime.block_on(async move {
2554 let config = LiveNodeConfig {
2555 environment: Environment::Sandbox,
2556 trader_id: trader_a,
2557 instance_id: Some(instance_a),
2558 msgbus: Some(node_a_msgbus),
2559 exec_engine: LiveExecutionEngineConfig {
2560 reconciliation: false,
2561 ..Default::default()
2562 },
2563 delay_post_stop: Duration::ZERO,
2564 timeout_connection: Duration::from_millis(500),
2565 timeout_disconnection: Duration::from_millis(500),
2566 ..Default::default()
2567 };
2568 let _node = LiveNodeBuilder::from_config(config)?
2569 .with_external_msgbus_factory(Box::new(redis_config))
2570 .build()?;
2571
2572 wait_until_async(
2574 || {
2575 let ready = ready_rx.try_recv().is_ok();
2576
2577 if !ready {
2578 msgbus::publish_quote(ready_topic.into(), "e);
2579 }
2580
2581 async move { ready }
2582 },
2583 Duration::from_secs(5),
2584 )
2585 .await;
2586
2587 msgbus::publish_trade("data.trades.TEST".into(), &trade);
2588 msgbus::publish_quote("data.quotes.TEST".into(), "e);
2589 msgbus::get_message_bus().borrow_mut().dispose();
2590
2591 Ok(())
2592 })
2593 });
2594
2595 node_a
2596 .join()
2597 .expect("node A thread should not panic")
2598 .expect("node A should publish externally");
2599 let received_quote = quote_rx
2600 .recv_timeout(Duration::from_secs(10))
2601 .expect("node B should republish the registered quote type");
2602 node_b
2603 .join()
2604 .expect("node B thread should not panic")
2605 .expect("node B should ingest and stop cleanly");
2606
2607 assert_eq!(received_quote, quote);
2608 assert!(
2609 trade_rx.try_recv().is_err(),
2610 "unregistered trade type should not republish internally"
2611 );
2612 }
2613
2614 #[rstest]
2615 #[tokio::test(flavor = "multi_thread")]
2616 async fn test_stream_messages_multiple_streams(#[future] redis_connection: ConnectionManager) {
2617 let mut con = redis_connection.await;
2618 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2619
2620 let suffix = UUID4::new();
2622 let stream_key1 = format!("test:stream:{suffix}:1");
2623 let stream_key2 = format!("test:stream:{suffix}:2");
2624 let external_streams = vec![stream_key1.clone(), stream_key2.clone()];
2625 let stream_signal = Arc::new(AtomicBool::new(false));
2626 let stream_signal_clone = Arc::clone(&stream_signal);
2627
2628 let clock = get_atomic_clock_realtime();
2629 let base_id = clock.get_time_ms() + 1_000_000;
2630
2631 let handle = tokio::spawn(async move {
2633 stream_messages(
2634 tx,
2635 RedisMessageBusConfig::default(),
2636 external_streams,
2637 stream_signal_clone,
2638 )
2639 .await
2640 .unwrap();
2641 });
2642
2643 let _: () = con
2645 .xadd(
2646 &stream_key1,
2647 format!("{}", base_id + 100),
2648 &[("topic", "stream1-first"), ("payload", "data")],
2649 )
2650 .await
2651 .unwrap();
2652
2653 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2654 assert_eq!(msg.topic, "stream1-first");
2655
2656 let _: () = con
2658 .xadd(
2659 &stream_key2,
2660 format!("{}", base_id + 50),
2661 &[("topic", "stream2-second"), ("payload", "data")],
2662 )
2663 .await
2664 .unwrap();
2665
2666 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2667 assert_eq!(msg.topic, "stream2-second");
2668
2669 rx.close();
2671 stream_signal.store(true, Ordering::Relaxed);
2672 handle.await.unwrap();
2673 }
2674
2675 #[rstest]
2676 #[tokio::test(flavor = "multi_thread")]
2677 async fn test_stream_messages_interleaved_at_different_rates(
2678 #[future] redis_connection: ConnectionManager,
2679 ) {
2680 let mut con = redis_connection.await;
2681 let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2682
2683 let suffix = UUID4::new();
2685 let stream_key1 = format!("test:stream:interleaved:{suffix}:1");
2686 let stream_key2 = format!("test:stream:interleaved:{suffix}:2");
2687 let stream_key3 = format!("test:stream:interleaved:{suffix}:3");
2688 let external_streams = vec![
2689 stream_key1.clone(),
2690 stream_key2.clone(),
2691 stream_key3.clone(),
2692 ];
2693 let stream_signal = Arc::new(AtomicBool::new(false));
2694 let stream_signal_clone = Arc::clone(&stream_signal);
2695
2696 let clock = get_atomic_clock_realtime();
2697 let base_id = clock.get_time_ms() + 1_000_000;
2698
2699 let handle = tokio::spawn(async move {
2700 stream_messages(
2701 tx,
2702 RedisMessageBusConfig::default(),
2703 external_streams,
2704 stream_signal_clone,
2705 )
2706 .await
2707 .unwrap();
2708 });
2709
2710 let _: () = con
2712 .xadd(
2713 &stream_key1,
2714 format!("{}", base_id + 100),
2715 &[("topic", "s1m1"), ("payload", "data")],
2716 )
2717 .await
2718 .unwrap();
2719 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2720 assert_eq!(msg.topic, "s1m1");
2721
2722 let _: () = con
2724 .xadd(
2725 &stream_key2,
2726 format!("{}", base_id + 50),
2727 &[("topic", "s2m1"), ("payload", "data")],
2728 )
2729 .await
2730 .unwrap();
2731 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2732 assert_eq!(msg.topic, "s2m1");
2733
2734 let _: () = con
2736 .xadd(
2737 &stream_key3,
2738 format!("{}", base_id + 25),
2739 &[("topic", "s3m1"), ("payload", "data")],
2740 )
2741 .await
2742 .unwrap();
2743 let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2744 assert_eq!(msg.topic, "s3m1");
2745
2746 rx.close();
2748 stream_signal.store(true, Ordering::Relaxed);
2749 handle.await.unwrap();
2750 }
2751
2752 #[rstest]
2753 #[tokio::test(flavor = "multi_thread")]
2754 async fn test_close() {
2755 let trader_id = TraderId::from("tester-001");
2756 let instance_id = UUID4::new();
2757 let config = MessageBusConfig {
2758 use_instance_id: true,
2759 ..Default::default()
2760 };
2761
2762 let mut db = RedisMessageBusBacking::new(
2763 trader_id,
2764 instance_id,
2765 config,
2766 RedisMessageBusConfig::default(),
2767 )
2768 .unwrap();
2769
2770 MessageBusBacking::close(&mut db);
2772 }
2773
2774 #[rstest]
2775 #[tokio::test(flavor = "multi_thread")]
2776 async fn test_heartbeat_task() {
2777 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2778 let signal = Arc::new(AtomicBool::new(false));
2779
2780 let handle = tokio::spawn(run_heartbeat(1, Arc::clone(&signal), tx));
2782
2783 let heartbeat = receive_unbounded_bus_message(&mut rx, Duration::from_secs(2)).await;
2784
2785 signal.store(true, Ordering::Relaxed);
2787 handle.await.unwrap();
2788
2789 assert_eq!(heartbeat.topic, HEARTBEAT_TOPIC);
2791 }
2792
2793 async fn receive_bus_message(
2794 rx: &mut tokio::sync::mpsc::Receiver<BusMessage>,
2795 timeout: Duration,
2796 ) -> BusMessage {
2797 let mut received = None;
2798
2799 wait_until_async(
2800 || {
2801 if received.is_none() {
2802 received = rx.try_recv().ok();
2803 }
2804
2805 let has_received = received.is_some();
2806 async move { has_received }
2807 },
2808 timeout,
2809 )
2810 .await;
2811
2812 received.unwrap()
2813 }
2814
2815 async fn receive_unbounded_bus_message(
2816 rx: &mut tokio::sync::mpsc::UnboundedReceiver<BusMessage>,
2817 timeout: Duration,
2818 ) -> BusMessage {
2819 let mut received = None;
2820
2821 wait_until_async(
2822 || {
2823 if received.is_none() {
2824 received = rx.try_recv().ok();
2825 }
2826
2827 let has_received = received.is_some();
2828 async move { has_received }
2829 },
2830 timeout,
2831 )
2832 .await;
2833
2834 received.unwrap()
2835 }
2836
2837 fn test_bus_message(topic: &str) -> BusMessage {
2838 BusMessage::with_str_topic(
2839 topic,
2840 BusPayloadType::QuoteTick,
2841 Bytes::from("test_payload"),
2842 SerializationEncoding::Json,
2843 )
2844 }
2845
2846 async fn wait_for_stream_len(con: &ConnectionManager, stream_key: &str, len: usize) {
2847 wait_until_async(
2848 || {
2849 let mut con = con.clone();
2850 let stream_key = stream_key.to_string();
2851 async move {
2852 let stream_len: usize = con.xlen(&stream_key).await.unwrap();
2853 stream_len >= len
2854 }
2855 },
2856 Duration::from_secs(5),
2857 )
2858 .await;
2859 }
2860
2861 async fn wait_for_last_topic(con: &ConnectionManager, stream_key: &str, topic: &str) {
2862 wait_until_async(
2863 || {
2864 let mut con = con.clone();
2865 let stream_key = stream_key.to_string();
2866 let topic = topic.to_string();
2867 async move {
2868 read_stream_topics(&mut con, &stream_key).await.last() == Some(&topic)
2869 }
2870 },
2871 Duration::from_secs(5),
2872 )
2873 .await;
2874 }
2875
2876 async fn read_stream_topics(con: &mut ConnectionManager, stream_key: &str) -> Vec<String> {
2877 let messages: RedisStreamBulk = con.xread(&[stream_key], &["0"]).await.unwrap();
2878 messages
2879 .iter()
2880 .flat_map(HashMap::values)
2881 .flatten()
2882 .flat_map(HashMap::values)
2883 .map(|fields| decode_bus_message(fields).unwrap().topic.to_string())
2884 .collect()
2885 }
2886
2887 struct RedisProxy {
2889 port: u16,
2890 accepting: Arc<AtomicBool>,
2891 open_connections: Arc<AtomicUsize>,
2892 disconnect_tx: Arc<tokio::sync::watch::Sender<u64>>,
2893 }
2894
2895 impl RedisProxy {
2896 async fn start() -> Self {
2897 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2898 let port = listener.local_addr().unwrap().port();
2899 let accepting = Arc::new(AtomicBool::new(true));
2900 let open_connections = Arc::new(AtomicUsize::new(0));
2901 let disconnect_tx = Arc::new(tokio::sync::watch::channel(0_u64).0);
2902
2903 let accepting_clone = Arc::clone(&accepting);
2904 let open_connections_clone = Arc::clone(&open_connections);
2905 let disconnect_tx_clone = Arc::clone(&disconnect_tx);
2906
2907 tokio::spawn(async move {
2908 while let Ok((mut inbound, _)) = listener.accept().await {
2909 let mut disconnect_rx = disconnect_tx_clone.subscribe();
2911
2912 if !accepting_clone.load(Ordering::SeqCst) {
2913 continue; }
2915
2916 let open_connections = Arc::clone(&open_connections_clone);
2917 open_connections.fetch_add(1, Ordering::SeqCst);
2918
2919 tokio::spawn(async move {
2920 if let Ok(mut outbound) =
2921 tokio::net::TcpStream::connect("127.0.0.1:6379").await
2922 {
2923 tokio::select! {
2924 _ = tokio::io::copy_bidirectional(&mut inbound, &mut outbound) => {}
2925 _ = disconnect_rx.changed() => {}
2926 }
2927 }
2928 open_connections.fetch_sub(1, Ordering::SeqCst);
2929 });
2930 }
2931 });
2932
2933 Self {
2934 port,
2935 accepting,
2936 open_connections,
2937 disconnect_tx,
2938 }
2939 }
2940
2941 fn config(&self) -> RedisMessageBusConfig {
2942 RedisMessageBusConfig {
2943 port: Some(self.port),
2944 ..Default::default()
2945 }
2946 }
2947
2948 async fn interrupt(&self) {
2950 self.accepting.store(false, Ordering::SeqCst);
2951 self.disconnect_tx.send_modify(|count| *count += 1);
2952
2953 wait_until_async(
2954 || {
2955 let open_connections = Arc::clone(&self.open_connections);
2956 async move { open_connections.load(Ordering::SeqCst) == 0 }
2957 },
2958 Duration::from_secs(1),
2959 )
2960 .await;
2961 }
2962
2963 fn resume(&self) {
2964 self.accepting.store(true, Ordering::SeqCst);
2965 }
2966 }
2967}