Skip to main content

nautilus_infrastructure/redis/
msgbus.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Redis-backed message bus backing for the system.
17//!
18//! # Architecture
19//!
20//! Runs background tasks on `get_runtime()` for publishing, stream reading,
21//! and heartbeats. Messages are sent via an unbounded `tokio::sync::mpsc`
22//! channel to the publish task, which buffers and writes them to Redis
23//! streams. Each background task owns its own Redis connection created on
24//! the Nautilus runtime.
25//!
26//! Handles are stored as `Option<JoinHandle>` for idempotent shutdown via
27//! `close_async()`. The synchronous `close()` uses `block_in_place` to
28//! bridge into the async shutdown path and must be called from outside any
29//! `current_thread` Tokio runtime.
30
31use std::{
32    collections::{HashMap, VecDeque},
33    fmt::Debug,
34    sync::{
35        Arc,
36        atomic::{AtomicBool, Ordering},
37    },
38    time::Duration,
39};
40
41use bytes::Bytes;
42use futures::stream::Stream;
43use jiff::{Timestamp, tz::Offset};
44use nautilus_common::{
45    enums::SerializationEncoding,
46    live::get_runtime,
47    logging::{log_task_error, log_task_started, log_task_stopped},
48    msgbus::{
49        BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory, MessageBusConfig,
50        switchboard::CLOSE_TOPIC,
51    },
52};
53use nautilus_core::{
54    UUID4,
55    time::{duration_since_unix_epoch, get_atomic_clock_realtime},
56};
57use nautilus_cryptography::providers::install_cryptographic_provider;
58use nautilus_model::identifiers::TraderId;
59use redis::{AsyncCommands, RetryMethod, aio::ConnectionManager, streams};
60use serde::{Deserialize, Serialize};
61use streams::StreamReadOptions;
62use ustr::Ustr;
63
64use super::{
65    REDIS_MINID, REDIS_XTRIM, await_handle,
66    stream_fields::{PAYLOAD_KIND_FIELD, PAYLOAD_KIND_TYPED, bus_message_fields},
67};
68use crate::redis::{RedisConnectionConfig, create_redis_connection, get_stream_key};
69
70const MSGBUS_PUBLISH: &str = "msgbus-publish";
71const MSGBUS_STREAM: &str = "msgbus-stream";
72const MSGBUS_HEARTBEAT: &str = "msgbus-heartbeat";
73const HEARTBEAT_TOPIC: &str = "health:heartbeat";
74const TRIM_BUFFER_SECS: u64 = 60;
75const PUBLISH_BATCH_SIZE: usize = 1_000;
76
77type RedisStreamBulk = Vec<HashMap<String, Vec<HashMap<String, redis::Value>>>>;
78
79/// Configuration for a Redis-backed message bus backing.
80///
81/// Redis 6.2 or higher is required for correct operation.
82#[cfg_attr(
83    feature = "python",
84    expect(
85        clippy::unsafe_derive_deserialize,
86        reason = "config deserializes plain fields; unsafe methods come from generated PyO3 integration"
87    )
88)]
89#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(default, deny_unknown_fields)]
91#[cfg_attr(
92    feature = "python",
93    pyo3::pyclass(module = "nautilus_trader.infrastructure", from_py_object)
94)]
95#[cfg_attr(
96    feature = "python",
97    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.infrastructure")
98)]
99pub struct RedisMessageBusConfig {
100    /// The Redis host address. If `None`, `127.0.0.1` is used.
101    pub host: Option<String>,
102    /// The Redis port. If `None`, `6379` is used.
103    pub port: Option<u16>,
104    /// The Redis account username.
105    pub username: Option<String>,
106    /// The Redis account password.
107    pub password: Option<String>,
108    /// If Redis should use an SSL-enabled connection.
109    pub ssl: bool,
110    /// The timeout (in seconds) to wait for a new connection.
111    pub connection_timeout: u16,
112    /// The timeout (in seconds) to wait for a response.
113    pub response_timeout: u16,
114    /// The number of retry attempts with exponential backoff for connection attempts.
115    pub number_of_retries: usize,
116    /// The base value for exponential backoff calculation.
117    pub exponent_base: u64,
118    /// The maximum delay between retry attempts (in seconds).
119    pub max_delay: u64,
120    /// The multiplication factor for retry delay calculation.
121    pub factor: u64,
122}
123
124impl Debug for RedisMessageBusConfig {
125    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126        let redacted = self.password.as_ref().map(|_| "***");
127        f.debug_struct(stringify!(RedisMessageBusConfig))
128            .field("host", &self.host)
129            .field("port", &self.port)
130            .field("username", &self.username)
131            .field("password", &redacted)
132            .field("ssl", &self.ssl)
133            .field("connection_timeout", &self.connection_timeout)
134            .field("response_timeout", &self.response_timeout)
135            .field("number_of_retries", &self.number_of_retries)
136            .field("exponent_base", &self.exponent_base)
137            .field("max_delay", &self.max_delay)
138            .field("factor", &self.factor)
139            .finish()
140    }
141}
142
143impl Default for RedisMessageBusConfig {
144    fn default() -> Self {
145        Self {
146            host: None,
147            port: None,
148            username: None,
149            password: None,
150            ssl: false,
151            connection_timeout: 20,
152            response_timeout: 20,
153            number_of_retries: 100,
154            exponent_base: 2,
155            max_delay: 1000,
156            factor: 2,
157        }
158    }
159}
160
161impl RedisConnectionConfig for RedisMessageBusConfig {
162    fn host(&self) -> Option<&str> {
163        self.host.as_deref()
164    }
165
166    fn port(&self) -> Option<u16> {
167        self.port
168    }
169
170    fn username(&self) -> Option<&str> {
171        self.username.as_deref()
172    }
173
174    fn password(&self) -> Option<&str> {
175        self.password.as_deref()
176    }
177
178    fn ssl(&self) -> bool {
179        self.ssl
180    }
181
182    fn connection_timeout(&self) -> u16 {
183        self.connection_timeout
184    }
185
186    fn response_timeout(&self) -> u16 {
187        self.response_timeout
188    }
189
190    fn number_of_retries(&self) -> usize {
191        self.number_of_retries
192    }
193
194    fn exponent_base(&self) -> u64 {
195        self.exponent_base
196    }
197
198    fn max_delay(&self) -> u64 {
199        self.max_delay
200    }
201
202    fn factor(&self) -> u64 {
203        self.factor
204    }
205}
206
207impl MessageBusBackingFactory for RedisMessageBusConfig {
208    fn create(
209        &self,
210        trader_id: TraderId,
211        instance_id: UUID4,
212        config: MessageBusConfig,
213    ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
214        Ok(Box::new(RedisMessageBusBacking::new(
215            trader_id,
216            instance_id,
217            config,
218            self.clone(),
219        )?))
220    }
221}
222
223/// Factory for constructing Redis message bus backings.
224#[derive(Debug, Clone)]
225pub struct RedisMessageBusFactory {
226    config: RedisMessageBusConfig,
227}
228
229impl RedisMessageBusFactory {
230    /// Creates a new [`RedisMessageBusFactory`] from the given Redis configuration.
231    #[must_use]
232    pub const fn new(config: RedisMessageBusConfig) -> Self {
233        Self { config }
234    }
235}
236
237impl MessageBusBackingFactory for RedisMessageBusFactory {
238    fn create(
239        &self,
240        trader_id: TraderId,
241        instance_id: UUID4,
242        config: MessageBusConfig,
243    ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
244        self.config.create(trader_id, instance_id, config)
245    }
246}
247
248pub struct RedisMessageBusBacking {
249    /// The trader ID for this message bus backing.
250    pub trader_id: TraderId,
251    /// The instance ID for this message bus backing.
252    pub instance_id: UUID4,
253    pub_tx: tokio::sync::mpsc::UnboundedSender<BusMessage>,
254    pub_handle: Option<tokio::task::JoinHandle<()>>,
255    stream_rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
256    stream_handle: Option<tokio::task::JoinHandle<()>>,
257    stream_signal: Arc<AtomicBool>,
258    heartbeat_handle: Option<tokio::task::JoinHandle<()>>,
259    heartbeat_signal: Arc<AtomicBool>,
260}
261
262impl Debug for RedisMessageBusBacking {
263    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
264        f.debug_struct(stringify!(RedisMessageBusBacking))
265            .field("trader_id", &self.trader_id)
266            .field("instance_id", &self.instance_id)
267            .finish_non_exhaustive()
268    }
269}
270
271impl RedisMessageBusBacking {
272    /// Creates a new [`RedisMessageBusBacking`] instance for the given `trader_id`, `instance_id`, and `config`.
273    ///
274    /// # Errors
275    ///
276    /// Returns an error if the heartbeat interval is configured as zero seconds.
277    pub fn new(
278        trader_id: TraderId,
279        instance_id: UUID4,
280        config: MessageBusConfig,
281        backing: RedisMessageBusConfig,
282    ) -> anyhow::Result<Self> {
283        install_cryptographic_provider();
284
285        if config.heartbeat_interval_secs == Some(0) {
286            anyhow::bail!("heartbeat_interval_secs must be greater than 0");
287        }
288
289        let external_streams = config.external_streams.clone().unwrap_or_default();
290        let heartbeat_interval_secs = config.heartbeat_interval_secs;
291        let publish = backing.clone();
292
293        let (pub_tx, pub_rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
294
295        // Create publish task (start the runtime here for now)
296        let pub_handle = Some(get_runtime().spawn(async move {
297            if let Err(e) = publish_messages(pub_rx, trader_id, instance_id, config, publish).await
298            {
299                log_task_error(MSGBUS_PUBLISH, &e);
300            }
301        }));
302
303        // Conditionally create stream task and channel if external streams configured
304        let stream_signal = Arc::new(AtomicBool::new(false));
305        let (stream_rx, stream_handle) = if external_streams.is_empty() {
306            (None, None)
307        } else {
308            let stream_signal_clone = Arc::clone(&stream_signal);
309            let (stream_tx, stream_rx) = tokio::sync::mpsc::channel::<BusMessage>(100_000);
310            (
311                Some(stream_rx),
312                Some(get_runtime().spawn(async move {
313                    if let Err(e) = stream_messages(
314                        stream_tx,
315                        backing.clone(),
316                        external_streams,
317                        stream_signal_clone,
318                    )
319                    .await
320                    {
321                        log_task_error(MSGBUS_STREAM, &e);
322                    }
323                })),
324            )
325        };
326
327        // Create heartbeat task
328        let heartbeat_signal = Arc::new(AtomicBool::new(false));
329        let heartbeat_handle = if let Some(heartbeat_interval_secs) = heartbeat_interval_secs {
330            let signal = Arc::clone(&heartbeat_signal);
331            let pub_tx_clone = pub_tx.clone();
332
333            Some(get_runtime().spawn(async move {
334                run_heartbeat(heartbeat_interval_secs, signal, pub_tx_clone).await;
335            }))
336        } else {
337            None
338        };
339
340        Ok(Self {
341            trader_id,
342            instance_id,
343            pub_tx,
344            pub_handle,
345            stream_rx,
346            stream_handle,
347            stream_signal,
348            heartbeat_handle,
349            heartbeat_signal,
350        })
351    }
352}
353
354impl MessageBusBacking for RedisMessageBusBacking {
355    /// Returns whether the message bus backing publishing channel is closed.
356    fn is_closed(&self) -> bool {
357        self.pub_tx.is_closed()
358    }
359
360    /// Queues a serialized bus message for external publication.
361    fn publish(&self, message: BusMessage) {
362        if let Err(e) = self.pub_tx.send(message) {
363            log::error!("Failed to send message: {e}");
364        }
365    }
366
367    fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
368        self.get_stream_receiver()
369    }
370
371    /// Closes the message bus backing.
372    fn close(&mut self) {
373        log::debug!("Closing");
374
375        self.stream_signal.store(true, Ordering::Relaxed);
376        self.heartbeat_signal.store(true, Ordering::Relaxed);
377
378        if !self.pub_tx.is_closed() {
379            let msg = BusMessage::new_close();
380
381            if let Err(e) = self.pub_tx.send(msg) {
382                log::warn!("Failed to send close message: {e:?}");
383            }
384        }
385
386        // Keep close sync for now to avoid async trait method
387        tokio::task::block_in_place(|| {
388            get_runtime().block_on(async {
389                self.close_async().await;
390            });
391        });
392
393        log::debug!("Closed");
394    }
395}
396
397impl RedisMessageBusBacking {
398    /// Retrieves the Redis stream receiver for this message bus instance.
399    ///
400    /// # Errors
401    ///
402    /// Returns an error if the stream receiver has already been taken.
403    pub fn get_stream_receiver(
404        &mut self,
405    ) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
406        self.stream_rx
407            .take()
408            .ok_or_else(|| anyhow::anyhow!("Stream receiver already taken"))
409    }
410
411    /// Streams messages arriving on the stream receiver channel.
412    pub fn stream(
413        mut stream_rx: tokio::sync::mpsc::Receiver<BusMessage>,
414    ) -> impl Stream<Item = BusMessage> + 'static {
415        async_stream::stream! {
416            while let Some(msg) = stream_rx.recv().await {
417                yield msg;
418            }
419        }
420    }
421
422    pub async fn close_async(&mut self) {
423        await_handle(self.pub_handle.take(), MSGBUS_PUBLISH).await;
424        await_handle(self.stream_handle.take(), MSGBUS_STREAM).await;
425        await_handle(self.heartbeat_handle.take(), MSGBUS_HEARTBEAT).await;
426    }
427}
428
429/// Publishes messages received on `rx` to Redis streams for the given `trader_id` and `instance_id`, using `config`.
430///
431/// Buffered messages are written in batches of at most 1,000 per `MULTI`/`EXEC`, and each batch
432/// is only removed once its `EXEC` is acknowledged. On a retryable write error the task keeps
433/// receiving, waits out the stream reader's backoff, and retries the same messages on a fresh
434/// connection, so a batch whose reply was lost can be written twice.
435///
436/// # Errors
437///
438/// Returns an error if:
439/// - The backing configuration is missing in `config`.
440/// - Establishing the initial Redis connection fails.
441/// - A Redis write fails with a non-retryable error.
442/// - The buffered messages cannot be written when the task closes.
443pub async fn publish_messages(
444    mut rx: tokio::sync::mpsc::UnboundedReceiver<BusMessage>,
445    trader_id: TraderId,
446    instance_id: UUID4,
447    config: MessageBusConfig,
448    backing: RedisMessageBusConfig,
449) -> anyhow::Result<()> {
450    log_task_started(MSGBUS_PUBLISH);
451
452    let con = create_redis_connection(MSGBUS_PUBLISH, &backing).await?;
453    let stream_key = get_stream_key(trader_id, instance_id, &config);
454
455    // Auto-trimming
456    let autotrim_duration = config
457        .autotrim_mins
458        .filter(|&mins| mins > 0)
459        .map(|mins| Duration::from_secs(u64::from(mins) * 60));
460    let autotrim_maxlen = config
461        .autotrim_maxlen
462        .filter(|&maxlen| maxlen > 0)
463        .map(usize::try_from)
464        .transpose()?;
465
466    let mut writer = StreamWriter {
467        con: Some(con),
468        // Fresh connections fail fast, so this task's backoff governs rather than the
469        // connection manager's retry schedule, and a close is not held up behind it.
470        reconnect_config: RedisMessageBusConfig {
471            number_of_retries: 0,
472            ..backing
473        },
474        stream_key,
475        stream_per_topic: config.stream_per_topic,
476        autotrim_duration,
477        autotrim_maxlen,
478        last_trim_index: HashMap::new(),
479        buffer: VecDeque::new(),
480        write_error_count: 0,
481    };
482
483    // Buffering
484    let buffer_interval = Duration::from_millis(u64::from(config.buffer_interval_ms.unwrap_or(0)));
485
486    // A sleep used to trigger periodic flushing of the buffer.
487    // When `buffer_interval` is zero we skip using the timer and flush immediately
488    // after every message.
489    let flush_timer = tokio::time::sleep(buffer_interval);
490    tokio::pin!(flush_timer);
491
492    // A sleep used to retry the buffer after a retryable write error,
493    // only polled while a retry is pending.
494    let retry_timer = tokio::time::sleep(Duration::ZERO);
495    tokio::pin!(retry_timer);
496
497    loop {
498        let retry_delay = tokio::select! {
499            maybe_msg = rx.recv() => {
500                let Some(msg) = maybe_msg else {
501                    log::debug!("Channel hung up");
502                    break;
503                };
504
505                if msg.topic == CLOSE_TOPIC {
506                    log::debug!("Received close message");
507                    break;
508                }
509
510                writer.buffer.push_back(msg);
511
512                // Immediate flush mode, unless a retry is pending
513                if buffer_interval.is_zero() && !writer.is_retrying() {
514                    writer.flush().await?
515                } else {
516                    None
517                }
518            }
519            // Only poll the timer when the interval is non-zero. This avoids
520            // unnecessarily waking the task when immediate flushing is enabled.
521            () = &mut flush_timer, if !buffer_interval.is_zero() => {
522                let retry_delay = if writer.is_retrying() {
523                    None
524                } else {
525                    writer.flush().await?
526                };
527
528                // Schedule the next tick
529                flush_timer.as_mut().reset(tokio::time::Instant::now() + buffer_interval);
530                retry_delay
531            }
532            () = &mut retry_timer, if writer.is_retrying() => writer.flush().await?,
533        };
534
535        if let Some(retry_delay) = retry_delay {
536            retry_timer
537                .as_mut()
538                .reset(tokio::time::Instant::now() + retry_delay);
539        }
540    }
541
542    // Drain any remaining messages, with a single attempt
543    if writer.flush().await?.is_some() {
544        anyhow::bail!(
545            "Failed to publish buffered messages before closing ({} unpublished)",
546            writer.buffer.len()
547        );
548    }
549
550    log_task_stopped(MSGBUS_PUBLISH);
551    Ok(())
552}
553
554/// Holds the publish task's buffer and connection.
555///
556/// Messages leave the buffer only once their `EXEC` is acknowledged. A retryable write error
557/// drops the connection, so the next attempt opens a fresh one, as the stream reader does.
558struct StreamWriter {
559    con: Option<ConnectionManager>,
560    reconnect_config: RedisMessageBusConfig,
561    stream_key: String,
562    stream_per_topic: bool,
563    autotrim_duration: Option<Duration>,
564    autotrim_maxlen: Option<usize>,
565    last_trim_index: HashMap<String, usize>,
566    buffer: VecDeque<BusMessage>,
567    write_error_count: usize,
568}
569
570impl StreamWriter {
571    /// Returns whether a failed write is waiting to be retried.
572    const fn is_retrying(&self) -> bool {
573        self.write_error_count > 0
574    }
575
576    /// Writes the buffer in atomic pipelines of at most [`PUBLISH_BATCH_SIZE`] messages, removing
577    /// each batch once it is acknowledged.
578    ///
579    /// Returns the delay before the next attempt when a retryable error left messages buffered.
580    ///
581    /// # Errors
582    ///
583    /// Returns an error if a write fails with a non-retryable error.
584    async fn flush(&mut self) -> anyhow::Result<Option<Duration>> {
585        while !self.buffer.is_empty() {
586            let mut con = match self.con.take() {
587                Some(con) => con,
588                None => {
589                    match create_redis_connection(MSGBUS_PUBLISH, &self.reconnect_config).await {
590                        Ok(con) => con,
591                        Err(e) => {
592                            log::error!("Error reconnecting to publish: {e:?}");
593                            return Ok(Some(self.next_retry_delay()));
594                        }
595                    }
596                }
597            };
598
599            let count = self.buffer.len().min(PUBLISH_BATCH_SIZE);
600            let result = write_buffer(
601                &mut con,
602                &self.stream_key,
603                self.stream_per_topic,
604                self.autotrim_duration,
605                self.autotrim_maxlen,
606                &mut self.last_trim_index,
607                self.buffer.range(..count),
608            )
609            .await;
610
611            match result {
612                Ok(()) => {
613                    self.buffer.drain(..count);
614                    self.con = Some(con);
615                    self.write_error_count = 0;
616                }
617                Err(e) if is_retryable_write_error(&e) => {
618                    log::error!("Error publishing to stream: {e:?}");
619                    return Ok(Some(self.next_retry_delay()));
620                }
621                Err(e) => anyhow::bail!("Error publishing to stream: {e:?}"),
622            }
623        }
624
625        Ok(None)
626    }
627
628    fn next_retry_delay(&mut self) -> Duration {
629        let retry_delay = stream_retry_delay(&self.reconnect_config, self.write_error_count);
630        self.write_error_count = self.write_error_count.saturating_add(1);
631        retry_delay
632    }
633}
634
635async fn write_buffer(
636    conn: &mut redis::aio::ConnectionManager,
637    stream_key: &str,
638    stream_per_topic: bool,
639    autotrim_duration: Option<Duration>,
640    autotrim_maxlen: Option<usize>,
641    last_trim_index: &mut HashMap<String, usize>,
642    messages: impl Iterator<Item = &BusMessage>,
643) -> redis::RedisResult<()> {
644    let mut pipe = redis::pipe();
645    pipe.atomic();
646
647    for msg in messages {
648        let encoding = msg.encoding.to_string();
649        let items = bus_message_fields(msg, &encoding);
650        let stream_key = if stream_per_topic {
651            format!("{stream_key}:{}", msg.topic)
652        } else {
653            stream_key.to_string()
654        };
655
656        if let Some(maxlen) = autotrim_maxlen {
657            pipe.xadd_maxlen(
658                &stream_key,
659                streams::StreamMaxlen::Approx(maxlen),
660                "*",
661                items.as_slice(),
662            );
663        } else {
664            pipe.xadd(&stream_key, "*", items.as_slice());
665        }
666
667        if autotrim_duration.is_none() {
668            continue; // Nothing else to do
669        }
670
671        // Autotrim stream
672        let last_trim_ms = last_trim_index.entry(stream_key.clone()).or_insert(0); // Remove clone
673        let unix_duration_now = duration_since_unix_epoch();
674        let trim_buffer = Duration::from_secs(TRIM_BUFFER_SECS);
675
676        // Improve efficiency of this by batching
677        if *last_trim_ms < unix_duration_now.saturating_sub(trim_buffer).as_millis() as usize {
678            let min_timestamp_ms = unix_duration_now
679                .saturating_sub(autotrim_duration.unwrap())
680                .as_millis() as usize;
681            let result: Result<(), redis::RedisError> = redis::cmd(REDIS_XTRIM)
682                .arg(stream_key.clone())
683                .arg(REDIS_MINID)
684                .arg(min_timestamp_ms)
685                .query_async(conn)
686                .await;
687
688            if let Err(e) = result {
689                // A dropped connection fails the batch here, before the connection manager's
690                // own reconnect can hold up the pipeline below.
691                if is_retryable_write_error(&e) {
692                    return Err(e);
693                }
694                log::error!("Error trimming stream '{stream_key}': {e}");
695            } else {
696                last_trim_index.insert(stream_key.clone(), unix_duration_now.as_millis() as usize);
697            }
698        }
699    }
700
701    pipe.query_async(conn).await
702}
703
704/// Streams messages from Redis streams and sends them over the provided `tx` channel.
705///
706/// # Errors
707///
708/// Returns an error if:
709/// - Establishing the Redis connection fails before the terminate signal is received.
710/// - A Redis read operation returns a non-retryable error.
711pub async fn stream_messages(
712    tx: tokio::sync::mpsc::Sender<BusMessage>,
713    config: RedisMessageBusConfig,
714    stream_keys: Vec<String>,
715    stream_signal: Arc<AtomicBool>,
716) -> anyhow::Result<()> {
717    log_task_started(MSGBUS_STREAM);
718
719    let Some(mut con) = connect_stream_connection(&config, &stream_signal).await? else {
720        log_task_stopped(MSGBUS_STREAM);
721        return Ok(());
722    };
723
724    let mut read_error_count = 0;
725
726    let stream_keys = &stream_keys
727        .iter()
728        .map(String::as_str)
729        .collect::<Vec<&str>>();
730
731    log::debug!("Listening to streams: [{}]", stream_keys.join(", "));
732
733    // Start streaming from current timestamp
734    let clock = get_atomic_clock_realtime();
735    let timestamp_ms = clock.get_time_ms();
736    let initial_id = timestamp_ms.to_string();
737
738    let mut last_ids: HashMap<String, String> = stream_keys
739        .iter()
740        .map(|&key| (key.to_string(), initial_id.clone()))
741        .collect();
742
743    let opts = StreamReadOptions::default().block(100);
744
745    'outer: loop {
746        if stream_signal.load(Ordering::Relaxed) {
747            log::debug!("Received streaming terminate signal");
748            break;
749        }
750
751        let ids: Vec<String> = stream_keys
752            .iter()
753            .map(|&key| last_ids[key].clone())
754            .collect();
755        let id_refs: Vec<&str> = ids.iter().map(String::as_str).collect();
756
757        let result: Result<RedisStreamBulk, _> =
758            con.xread_options(&[&stream_keys], &[&id_refs], &opts).await;
759
760        match result {
761            Ok(stream_bulk) => {
762                read_error_count = 0;
763
764                if stream_bulk.is_empty() {
765                    // Timeout occurred: no messages received
766                    continue;
767                }
768
769                for entry in &stream_bulk {
770                    for (stream_key, stream_msgs) in entry {
771                        for stream_msg in stream_msgs {
772                            for (id, array) in stream_msg {
773                                last_ids.insert(stream_key.clone(), id.clone());
774
775                                match decode_bus_message(array) {
776                                    Ok(msg) => {
777                                        if let Err(e) = tx.send(msg).await {
778                                            log::debug!("Channel closed: {e:?}");
779                                            break 'outer; // End streaming
780                                        }
781                                    }
782                                    Err(e) => {
783                                        log::error!("{e:?}");
784                                    }
785                                }
786                            }
787                        }
788                    }
789                }
790            }
791            Err(e) => {
792                if !is_retryable_stream_error(&e) {
793                    anyhow::bail!("Error reading from stream: {e:?}");
794                }
795
796                log::error!("Error reading from stream: {e:?}");
797
798                let Some(reconnected) =
799                    reconnect_stream_connection(&config, &stream_signal, &mut read_error_count)
800                        .await?
801                else {
802                    break;
803                };
804                con = reconnected;
805            }
806        }
807    }
808
809    log_task_stopped(MSGBUS_STREAM);
810    Ok(())
811}
812
813async fn connect_stream_connection(
814    config: &RedisMessageBusConfig,
815    stream_signal: &Arc<AtomicBool>,
816) -> anyhow::Result<Option<ConnectionManager>> {
817    let connect = create_redis_connection(MSGBUS_STREAM, config);
818    let terminate = wait_for_stream_signal(stream_signal);
819
820    tokio::pin!(connect);
821    tokio::pin!(terminate);
822
823    tokio::select! {
824        result = &mut connect => result.map(Some),
825        () = &mut terminate => Ok(None),
826    }
827}
828
829async fn reconnect_stream_connection(
830    config: &RedisMessageBusConfig,
831    stream_signal: &Arc<AtomicBool>,
832    read_error_count: &mut usize,
833) -> anyhow::Result<Option<ConnectionManager>> {
834    loop {
835        let retry_delay = stream_retry_delay(config, *read_error_count);
836        *read_error_count = (*read_error_count).saturating_add(1);
837
838        if !wait_for_retry_delay(retry_delay, stream_signal).await {
839            return Ok(None);
840        }
841
842        match connect_stream_connection(config, stream_signal).await {
843            Ok(Some(con)) => return Ok(Some(con)),
844            Ok(None) => return Ok(None),
845            Err(e) => {
846                log::error!("Error reconnecting to stream: {e:?}");
847            }
848        }
849    }
850}
851
852fn stream_retry_delay(config: &RedisMessageBusConfig, attempt: usize) -> Duration {
853    let exponent = u32::try_from(attempt.min(32)).unwrap_or(32);
854    let delay_ms = config
855        .factor
856        .saturating_mul(config.exponent_base.saturating_pow(exponent));
857    let max_delay = Duration::from_secs(config.max_delay);
858
859    Duration::from_millis(delay_ms)
860        .min(max_delay)
861        .max(Duration::from_millis(1))
862}
863
864fn is_retryable_stream_error(error: &redis::RedisError) -> bool {
865    matches!(
866        error.retry_method(),
867        RetryMethod::Reconnect
868            | RetryMethod::ReconnectFromInitialConnections
869            | RetryMethod::RetryImmediately
870            | RetryMethod::WaitAndRetry
871    )
872}
873
874// A transaction aborted with `EXECABORT` carries the errors its commands were queued with,
875// so it is retryable when all of them are, such as `LOADING` while Redis restarts.
876fn is_retryable_write_error(error: &redis::RedisError) -> bool {
877    if is_retryable_stream_error(error) {
878        return true;
879    }
880
881    error.clone().into_server_errors().is_some_and(|errors| {
882        !errors.is_empty()
883            && errors.iter().all(|(_, server_error)| {
884                is_retryable_stream_error(&redis::RedisError::from(server_error.clone()))
885            })
886    })
887}
888
889async fn wait_for_retry_delay(retry_delay: Duration, stream_signal: &Arc<AtomicBool>) -> bool {
890    let retry_timer = tokio::time::sleep(retry_delay);
891    let terminate = wait_for_stream_signal(stream_signal);
892
893    tokio::pin!(retry_timer);
894    tokio::pin!(terminate);
895
896    tokio::select! {
897        () = &mut retry_timer => true,
898        () = &mut terminate => false,
899    }
900}
901
902async fn wait_for_stream_signal(stream_signal: &Arc<AtomicBool>) {
903    let check_timer = tokio::time::interval(Duration::from_millis(100));
904
905    tokio::pin!(check_timer);
906
907    while !stream_signal.load(Ordering::Relaxed) {
908        check_timer.tick().await;
909    }
910}
911
912// Redis fields are unordered, and older streams may omit type or encoding headers
913fn decode_bus_message(stream_msg: &redis::Value) -> anyhow::Result<BusMessage> {
914    let redis::Value::Array(fields) = stream_msg else {
915        anyhow::bail!("Invalid stream message format: {stream_msg:?}");
916    };
917
918    if fields.len() < 4 || fields.len() % 2 != 0 {
919        anyhow::bail!("Invalid stream message format: {stream_msg:?}");
920    }
921
922    let mut topic: Option<String> = None;
923    let mut type_name: Option<String> = None;
924    let mut typed_payload = false;
925    let mut encoding = SerializationEncoding::default();
926    let mut payload: Option<Bytes> = None;
927
928    for pair in fields.as_chunks::<2>().0 {
929        let redis::Value::BulkString(key) = &pair[0] else {
930            anyhow::bail!("Invalid stream field key: {stream_msg:?}");
931        };
932
933        match key.as_slice() {
934            b"topic" => {
935                let redis::Value::BulkString(bytes) = &pair[1] else {
936                    anyhow::bail!("Invalid topic format: {stream_msg:?}");
937                };
938                topic = Some(
939                    String::from_utf8(bytes.clone())
940                        .map_err(|e| anyhow::anyhow!("Error parsing topic: {e}"))?,
941                );
942            }
943            b"type" => {
944                let redis::Value::BulkString(bytes) = &pair[1] else {
945                    anyhow::bail!("Invalid type format: {stream_msg:?}");
946                };
947                type_name = Some(
948                    String::from_utf8(bytes.clone())
949                        .map_err(|e| anyhow::anyhow!("Error parsing type: {e}"))?,
950                );
951            }
952            key if key == PAYLOAD_KIND_FIELD.as_bytes() => {
953                let redis::Value::BulkString(bytes) = &pair[1] else {
954                    anyhow::bail!("Invalid payload kind format: {stream_msg:?}");
955                };
956                let value = std::str::from_utf8(bytes)
957                    .map_err(|e| anyhow::anyhow!("Error parsing payload kind: {e}"))?;
958                anyhow::ensure!(
959                    value == PAYLOAD_KIND_TYPED,
960                    "Unknown payload kind '{value}'"
961                );
962                typed_payload = true;
963            }
964            b"encoding" => {
965                let redis::Value::BulkString(bytes) = &pair[1] else {
966                    anyhow::bail!("Invalid encoding format: {stream_msg:?}");
967                };
968                let value = std::str::from_utf8(bytes)
969                    .map_err(|e| anyhow::anyhow!("Error parsing encoding: {e}"))?;
970                encoding = value
971                    .parse()
972                    .map_err(|e| anyhow::anyhow!("Error parsing encoding: {e}"))?;
973            }
974            b"payload" => {
975                let redis::Value::BulkString(bytes) = &pair[1] else {
976                    anyhow::bail!("Invalid payload format: {stream_msg:?}");
977                };
978                payload = Some(Bytes::copy_from_slice(bytes));
979            }
980            _ => {}
981        }
982    }
983
984    let Some(topic) = topic else {
985        anyhow::bail!("Stream message missing topic: {stream_msg:?}");
986    };
987    let Some(payload) = payload else {
988        anyhow::bail!("Stream message missing payload: {stream_msg:?}");
989    };
990    let payload_type = match type_name {
991        Some(type_name) if typed_payload => BusPayloadType::from_typed_name(&type_name)
992            .ok_or_else(|| anyhow::anyhow!("Unknown typed payload '{type_name}'"))?,
993        Some(type_name) => BusPayloadType::from_name(&type_name),
994        None if typed_payload => {
995            anyhow::bail!("Typed stream message missing type: {stream_msg:?}")
996        }
997        None => BusPayloadType::Custom(Ustr::default()),
998    };
999
1000    Ok(BusMessage::with_str_topic(
1001        topic,
1002        payload_type,
1003        payload,
1004        encoding,
1005    ))
1006}
1007
1008async fn run_heartbeat(
1009    heartbeat_interval_secs: u16,
1010    signal: Arc<AtomicBool>,
1011    pub_tx: tokio::sync::mpsc::UnboundedSender<BusMessage>,
1012) {
1013    log_task_started("heartbeat");
1014    log::debug!("Heartbeat at {heartbeat_interval_secs} second intervals");
1015
1016    let heartbeat_interval = Duration::from_secs(u64::from(heartbeat_interval_secs));
1017    let heartbeat_timer = tokio::time::interval(heartbeat_interval);
1018
1019    let check_interval = Duration::from_millis(100);
1020    let check_timer = tokio::time::interval(check_interval);
1021
1022    tokio::pin!(heartbeat_timer);
1023    tokio::pin!(check_timer);
1024
1025    loop {
1026        if signal.load(Ordering::Relaxed) {
1027            log::debug!("Received heartbeat terminate signal");
1028            break;
1029        }
1030
1031        tokio::select! {
1032            _ = heartbeat_timer.tick() => {
1033                let heartbeat = create_heartbeat_msg();
1034                if let Err(e) = pub_tx.send(heartbeat) {
1035                    // We expect an error if the channel is closed during shutdown
1036                    log::debug!("Error sending heartbeat: {e}");
1037                }
1038            },
1039            _ = check_timer.tick() => {}
1040        }
1041    }
1042
1043    log_task_stopped("heartbeat");
1044}
1045
1046fn create_heartbeat_msg() -> BusMessage {
1047    let payload = Bytes::from(
1048        Timestamp::now()
1049            .display_with_offset(Offset::UTC)
1050            .to_string()
1051            .into_bytes(),
1052    );
1053    BusMessage::with_str_topic(
1054        HEARTBEAT_TOPIC,
1055        BusPayloadType::Custom(Ustr::default()),
1056        payload,
1057        SerializationEncoding::default(),
1058    )
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063    use std::sync::atomic::AtomicUsize;
1064
1065    use nautilus_common::{msgbus::external_io_from_backing, testing::wait_until_async};
1066    use redis::Value;
1067    use rstest::*;
1068    use serde_json::json;
1069    use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
1070
1071    use super::*;
1072
1073    #[rstest]
1074    fn test_default_redis_message_bus_config() {
1075        let config = RedisMessageBusConfig::default();
1076
1077        assert_eq!(config.host, None);
1078        assert_eq!(config.port, None);
1079        assert_eq!(config.username, None);
1080        assert_eq!(config.password, None);
1081        assert!(!config.ssl);
1082        assert_eq!(config.connection_timeout, 20);
1083        assert_eq!(config.response_timeout, 20);
1084        assert_eq!(config.number_of_retries, 100);
1085        assert_eq!(config.exponent_base, 2);
1086        assert_eq!(config.max_delay, 1000);
1087        assert_eq!(config.factor, 2);
1088    }
1089
1090    #[rstest]
1091    fn test_deserialize_redis_message_bus_config() {
1092        let config_json = json!({
1093            "host": "localhost",
1094            "port": 6379,
1095            "username": "user",
1096            "password": "pass",
1097            "ssl": true,
1098            "connection_timeout": 30,
1099            "response_timeout": 10,
1100            "number_of_retries": 3,
1101            "exponent_base": 2,
1102            "max_delay": 10,
1103            "factor": 2
1104        });
1105
1106        let config: RedisMessageBusConfig = serde_json::from_value(config_json).unwrap();
1107
1108        assert_eq!(config.host, Some("localhost".to_string()));
1109        assert_eq!(config.port, Some(6379));
1110        assert_eq!(config.username, Some("user".to_string()));
1111        assert_eq!(config.password, Some("pass".to_string()));
1112        assert!(config.ssl);
1113        assert_eq!(config.connection_timeout, 30);
1114        assert_eq!(config.response_timeout, 10);
1115        assert_eq!(config.number_of_retries, 3);
1116        assert_eq!(config.exponent_base, 2);
1117        assert_eq!(config.max_delay, 10);
1118        assert_eq!(config.factor, 2);
1119    }
1120
1121    #[rstest]
1122    fn test_deserialize_redis_message_bus_config_rejects_type_selector() {
1123        let config_json = json!({
1124            "type": "redis",
1125        });
1126
1127        let error = serde_json::from_value::<RedisMessageBusConfig>(config_json).unwrap_err();
1128
1129        assert!(error.to_string().contains("unknown field `type`"));
1130    }
1131
1132    #[rstest]
1133    fn test_bus_message_fields_distinguish_custom_and_typed_names() {
1134        let custom = BusMessage::with_str_topic(
1135            "external.test",
1136            BusPayloadType::Custom(Ustr::from("TradingCommand")),
1137            Bytes::from_static(b"custom"),
1138            SerializationEncoding::Json,
1139        );
1140        let typed = BusMessage::with_str_topic(
1141            "external.test",
1142            BusPayloadType::TradingCommand,
1143            Bytes::from_static(b"typed"),
1144            SerializationEncoding::Json,
1145        );
1146
1147        let custom_fields = bus_message_fields(&custom, "json");
1148        let typed_fields = bus_message_fields(&typed, "json");
1149        let custom_fields = custom_fields.as_slice();
1150        let typed_fields = typed_fields.as_slice();
1151
1152        assert_eq!(custom_fields.len(), 4);
1153        assert_eq!(typed_fields.len(), 5);
1154        assert_eq!(
1155            custom_fields
1156                .iter()
1157                .find(|(key, _)| *key == "type")
1158                .expect("custom fields must include type")
1159                .1,
1160            b"TradingCommand"
1161        );
1162        assert_eq!(
1163            typed_fields
1164                .iter()
1165                .find(|(key, _)| *key == "type")
1166                .expect("typed fields must include type")
1167                .1,
1168            b"TradingCommand"
1169        );
1170        assert!(
1171            custom_fields
1172                .iter()
1173                .all(|(key, _)| *key != PAYLOAD_KIND_FIELD)
1174        );
1175        assert_eq!(
1176            typed_fields
1177                .iter()
1178                .find(|(key, _)| *key == PAYLOAD_KIND_FIELD)
1179                .expect("typed fields must include payload kind")
1180                .1,
1181            PAYLOAD_KIND_TYPED.as_bytes()
1182        );
1183    }
1184
1185    #[rstest]
1186    fn test_decode_bus_message_valid() {
1187        let stream_msg = Value::Array(vec![
1188            Value::BulkString(b"topic".to_vec()),
1189            Value::BulkString(b"topic1".to_vec()),
1190            Value::BulkString(b"type".to_vec()),
1191            Value::BulkString(b"QuoteTick".to_vec()),
1192            Value::BulkString(b"payload".to_vec()),
1193            Value::BulkString(b"data1".to_vec()),
1194            Value::BulkString(b"encoding".to_vec()),
1195            Value::BulkString(b"msgpack".to_vec()),
1196        ]);
1197
1198        let result = decode_bus_message(&stream_msg);
1199        assert!(result.is_ok());
1200        let msg = result.unwrap();
1201        assert_eq!(msg.topic, "topic1");
1202        assert_eq!(msg.payload_type, BusPayloadType::QuoteTick);
1203        assert_eq!(msg.encoding, SerializationEncoding::MsgPack);
1204        assert_eq!(msg.payload, Bytes::from("data1"));
1205    }
1206
1207    #[rstest]
1208    fn test_decode_bus_message_defaults_legacy_headers() {
1209        let stream_msg = Value::Array(vec![
1210            Value::BulkString(b"topic".to_vec()),
1211            Value::BulkString(b"topic1".to_vec()),
1212            Value::BulkString(b"payload".to_vec()),
1213            Value::BulkString(b"data1".to_vec()),
1214        ]);
1215
1216        let result = decode_bus_message(&stream_msg);
1217        assert!(result.is_ok());
1218        let msg = result.unwrap();
1219        assert_eq!(msg.topic, "topic1");
1220        assert_eq!(msg.payload_type, BusPayloadType::Custom(Ustr::default()));
1221        assert_eq!(msg.encoding, SerializationEncoding::Json);
1222        assert_eq!(msg.payload, Bytes::from("data1"));
1223    }
1224
1225    #[rstest]
1226    fn test_decode_bus_message_reads_chrono_heartbeat() {
1227        let heartbeat =
1228            include_str!("../../test_data/redis_msgbus_heartbeat_chrono.txt").trim_end();
1229        let stream_msg = Value::Array(vec![
1230            Value::BulkString(b"topic".to_vec()),
1231            Value::BulkString(HEARTBEAT_TOPIC.as_bytes().to_vec()),
1232            Value::BulkString(b"payload".to_vec()),
1233            Value::BulkString(heartbeat.as_bytes().to_vec()),
1234        ]);
1235
1236        let msg = decode_bus_message(&stream_msg).unwrap();
1237        let timestamp = std::str::from_utf8(&msg.payload)
1238            .unwrap()
1239            .parse::<Timestamp>()
1240            .unwrap();
1241
1242        assert_eq!(msg.topic, HEARTBEAT_TOPIC);
1243        assert_eq!(msg.payload_type, BusPayloadType::Custom(Ustr::default()));
1244        assert_eq!(msg.encoding, SerializationEncoding::Json);
1245        assert_eq!(msg.payload.as_ref(), heartbeat.as_bytes());
1246        assert_eq!(timestamp.as_nanosecond(), 1_785_805_323_456_789_000);
1247    }
1248
1249    #[rstest]
1250    fn test_decode_bus_message_typed_name_without_kind_is_custom() {
1251        let stream_msg = Value::Array(vec![
1252            Value::BulkString(b"topic".to_vec()),
1253            Value::BulkString(b"topic1".to_vec()),
1254            Value::BulkString(b"type".to_vec()),
1255            Value::BulkString(b"TradingCommand".to_vec()),
1256            Value::BulkString(b"payload".to_vec()),
1257            Value::BulkString(b"data1".to_vec()),
1258        ]);
1259
1260        let result = decode_bus_message(&stream_msg);
1261        assert!(result.is_ok());
1262        let msg = result.unwrap();
1263        assert_eq!(
1264            msg.payload_type,
1265            BusPayloadType::Custom(Ustr::from("TradingCommand"))
1266        );
1267        assert_eq!(msg.encoding, SerializationEncoding::Json);
1268    }
1269
1270    #[rstest]
1271    fn test_decode_bus_message_typed_kind_resolves_fixed_type() {
1272        let stream_msg = Value::Array(vec![
1273            Value::BulkString(b"topic".to_vec()),
1274            Value::BulkString(b"external.test".to_vec()),
1275            Value::BulkString(b"type".to_vec()),
1276            Value::BulkString(b"TradingCommand".to_vec()),
1277            Value::BulkString(PAYLOAD_KIND_FIELD.as_bytes().to_vec()),
1278            Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1279            Value::BulkString(b"payload".to_vec()),
1280            Value::BulkString(b"data1".to_vec()),
1281        ]);
1282
1283        let msg = decode_bus_message(&stream_msg).expect("typed message must decode");
1284
1285        assert_eq!(msg.topic, "external.test");
1286        assert_eq!(msg.payload_type, BusPayloadType::TradingCommand);
1287        assert_eq!(msg.encoding, SerializationEncoding::Json);
1288        assert_eq!(msg.payload, Bytes::from_static(b"data1"));
1289    }
1290
1291    #[rstest]
1292    fn test_decode_bus_message_rejects_invalid_typed_metadata() {
1293        let unknown_kind = stream_message_with_typed_metadata(
1294            Some("TradingCommand"),
1295            Value::BulkString(b"unknown".to_vec()),
1296        );
1297        let missing_type = stream_message_with_typed_metadata(
1298            None,
1299            Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1300        );
1301        let unknown_type = stream_message_with_typed_metadata(
1302            Some("UnknownPayload"),
1303            Value::BulkString(PAYLOAD_KIND_TYPED.as_bytes().to_vec()),
1304        );
1305        let invalid_kind =
1306            stream_message_with_typed_metadata(Some("TradingCommand"), Value::Int(42));
1307        let cases = [
1308            (&unknown_kind, "Unknown payload kind 'unknown'".to_string()),
1309            (
1310                &missing_type,
1311                format!("Typed stream message missing type: {missing_type:?}"),
1312            ),
1313            (
1314                &unknown_type,
1315                "Unknown typed payload 'UnknownPayload'".to_string(),
1316            ),
1317            (
1318                &invalid_kind,
1319                format!("Invalid payload kind format: {invalid_kind:?}"),
1320            ),
1321        ];
1322
1323        for (stream_msg, expected) in cases {
1324            let error =
1325                decode_bus_message(stream_msg).expect_err("invalid typed metadata must not decode");
1326
1327            assert_eq!(error.to_string(), expected);
1328        }
1329    }
1330
1331    fn stream_message_with_typed_metadata(type_name: Option<&str>, payload_kind: Value) -> Value {
1332        let mut fields = vec![
1333            Value::BulkString(b"topic".to_vec()),
1334            Value::BulkString(b"external.test".to_vec()),
1335        ];
1336
1337        if let Some(type_name) = type_name {
1338            fields.extend([
1339                Value::BulkString(b"type".to_vec()),
1340                Value::BulkString(type_name.as_bytes().to_vec()),
1341            ]);
1342        }
1343        fields.extend([
1344            Value::BulkString(PAYLOAD_KIND_FIELD.as_bytes().to_vec()),
1345            payload_kind,
1346            Value::BulkString(b"payload".to_vec()),
1347            Value::BulkString(b"data1".to_vec()),
1348        ]);
1349        Value::Array(fields)
1350    }
1351
1352    #[rstest]
1353    fn test_decode_bus_message_accepts_unordered_metadata_fields() {
1354        let stream_msg = Value::Array(vec![
1355            Value::BulkString(b"payload".to_vec()),
1356            Value::BulkString(b"data1".to_vec()),
1357            Value::BulkString(b"encoding".to_vec()),
1358            Value::BulkString(b"msgpack".to_vec()),
1359            Value::BulkString(b"type".to_vec()),
1360            Value::BulkString(b"TradeTick".to_vec()),
1361            Value::BulkString(b"topic".to_vec()),
1362            Value::BulkString(b"topic1".to_vec()),
1363        ]);
1364
1365        let msg = decode_bus_message(&stream_msg).unwrap();
1366
1367        assert_eq!(msg.topic, "topic1");
1368        assert_eq!(msg.payload_type, BusPayloadType::TradeTick);
1369        assert_eq!(msg.encoding, SerializationEncoding::MsgPack);
1370        assert_eq!(msg.payload, Bytes::from("data1"));
1371    }
1372
1373    #[rstest]
1374    fn test_decode_bus_message_rejects_invalid_encoding_header() {
1375        let stream_msg = Value::Array(vec![
1376            Value::BulkString(b"topic".to_vec()),
1377            Value::BulkString(b"topic1".to_vec()),
1378            Value::BulkString(b"encoding".to_vec()),
1379            Value::BulkString(b"invalid".to_vec()),
1380            Value::BulkString(b"payload".to_vec()),
1381            Value::BulkString(b"data1".to_vec()),
1382        ]);
1383
1384        let error = decode_bus_message(&stream_msg).unwrap_err();
1385
1386        assert!(
1387            error.to_string().contains("Error parsing encoding"),
1388            "{error:?}"
1389        );
1390    }
1391
1392    #[rstest]
1393    fn test_decode_bus_message_missing_fields() {
1394        let stream_msg = Value::Array(vec![
1395            Value::BulkString(b"0".to_vec()),
1396            Value::BulkString(b"topic1".to_vec()),
1397        ]);
1398
1399        let result = decode_bus_message(&stream_msg);
1400        assert!(result.is_err());
1401        assert_eq!(
1402            format!("{}", result.unwrap_err()),
1403            "Invalid stream message format: array([bulk-string('\"0\"'), bulk-string('\"topic1\"')])"
1404        );
1405    }
1406
1407    #[rstest]
1408    fn test_decode_bus_message_invalid_topic_format() {
1409        let stream_msg = Value::Array(vec![
1410            Value::BulkString(b"topic".to_vec()),
1411            Value::Int(42),
1412            Value::BulkString(b"payload".to_vec()),
1413            Value::BulkString(b"data1".to_vec()),
1414        ]);
1415
1416        let result = decode_bus_message(&stream_msg);
1417        assert!(result.is_err());
1418        assert_eq!(
1419            format!("{}", result.unwrap_err()),
1420            "Invalid topic format: array([bulk-string('\"topic\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1421        );
1422    }
1423
1424    #[rstest]
1425    fn test_decode_bus_message_invalid_type_format() {
1426        let stream_msg = Value::Array(vec![
1427            Value::BulkString(b"topic".to_vec()),
1428            Value::BulkString(b"topic1".to_vec()),
1429            Value::BulkString(b"type".to_vec()),
1430            Value::Int(42),
1431            Value::BulkString(b"payload".to_vec()),
1432            Value::BulkString(b"data1".to_vec()),
1433        ]);
1434
1435        let result = decode_bus_message(&stream_msg);
1436        assert!(result.is_err());
1437        assert_eq!(
1438            format!("{}", result.unwrap_err()),
1439            "Invalid type format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"type\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1440        );
1441    }
1442
1443    #[rstest]
1444    fn test_decode_bus_message_invalid_encoding_format() {
1445        let stream_msg = Value::Array(vec![
1446            Value::BulkString(b"topic".to_vec()),
1447            Value::BulkString(b"topic1".to_vec()),
1448            Value::BulkString(b"encoding".to_vec()),
1449            Value::Int(42),
1450            Value::BulkString(b"payload".to_vec()),
1451            Value::BulkString(b"data1".to_vec()),
1452        ]);
1453
1454        let result = decode_bus_message(&stream_msg);
1455        assert!(result.is_err());
1456        assert_eq!(
1457            format!("{}", result.unwrap_err()),
1458            "Invalid encoding format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"encoding\"'), int(42), bulk-string('\"payload\"'), bulk-string('\"data1\"')])"
1459        );
1460    }
1461
1462    #[rstest]
1463    fn test_decode_bus_message_invalid_payload_format() {
1464        let stream_msg = Value::Array(vec![
1465            Value::BulkString(b"topic".to_vec()),
1466            Value::BulkString(b"topic1".to_vec()),
1467            Value::BulkString(b"payload".to_vec()),
1468            Value::Int(42),
1469        ]);
1470
1471        let result = decode_bus_message(&stream_msg);
1472        assert!(result.is_err());
1473        assert_eq!(
1474            format!("{}", result.unwrap_err()),
1475            "Invalid payload format: array([bulk-string('\"topic\"'), bulk-string('\"topic1\"'), bulk-string('\"payload\"'), int(42)])"
1476        );
1477    }
1478
1479    #[rstest]
1480    fn test_decode_bus_message_invalid_stream_msg_format() {
1481        let stream_msg = Value::BulkString(b"not an array".to_vec());
1482
1483        let result = decode_bus_message(&stream_msg);
1484        assert!(result.is_err());
1485        assert_eq!(
1486            format!("{}", result.unwrap_err()),
1487            "Invalid stream message format: bulk-string('\"not an array\"')"
1488        );
1489    }
1490
1491    #[rstest]
1492    fn test_new_rejects_zero_heartbeat_interval() {
1493        let trader_id = TraderId::from("tester-001");
1494        let instance_id = UUID4::new();
1495        let config = MessageBusConfig {
1496            heartbeat_interval_secs: Some(0),
1497            ..Default::default()
1498        };
1499
1500        let result = RedisMessageBusBacking::new(
1501            trader_id,
1502            instance_id,
1503            config,
1504            RedisMessageBusConfig::default(),
1505        );
1506
1507        assert!(result.is_err());
1508        assert_eq!(
1509            result.unwrap_err().to_string(),
1510            "heartbeat_interval_secs must be greater than 0"
1511        );
1512    }
1513
1514    #[rstest]
1515    fn test_stream_retry_delay_uses_config_bounds() {
1516        let config = RedisMessageBusConfig {
1517            factor: 10,
1518            exponent_base: 2,
1519            max_delay: 1,
1520            ..Default::default()
1521        };
1522
1523        assert_eq!(stream_retry_delay(&config, 0), Duration::from_millis(10));
1524        assert_eq!(stream_retry_delay(&config, 1), Duration::from_millis(20));
1525        assert_eq!(stream_retry_delay(&config, 10), Duration::from_secs(1));
1526    }
1527
1528    #[rstest]
1529    fn test_stream_error_retry_classification() {
1530        let dropped =
1531            redis::RedisError::from(std::io::Error::from(std::io::ErrorKind::ConnectionReset));
1532        let client: redis::RedisError = (redis::ErrorKind::Client, "client error").into();
1533
1534        assert!(is_retryable_stream_error(&dropped));
1535        assert!(!is_retryable_stream_error(&client));
1536    }
1537
1538    #[tokio::test]
1539    async fn test_wait_for_retry_delay_returns_false_when_signaled() {
1540        let stream_signal = Arc::new(AtomicBool::new(true));
1541        let signal = Arc::clone(&stream_signal);
1542        let fut = async move { wait_for_retry_delay(Duration::from_secs(30), &signal).await };
1543
1544        let handle = tokio::spawn(fut);
1545
1546        wait_until_async(|| async { handle.is_finished() }, Duration::from_secs(1)).await;
1547
1548        assert!(!handle.await.unwrap());
1549    }
1550
1551    #[rstest]
1552    #[case::loading("LOADING Redis is loading the dataset in memory")]
1553    #[case::master_down(
1554        "MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'."
1555    )]
1556    #[tokio::test(flavor = "multi_thread")]
1557    async fn test_publish_messages_retries_transaction_aborted_by_retryable_error(
1558        #[case] queued_error: &'static str,
1559    ) {
1560        let fake = FakeRedis::start(queued_error).await;
1561        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1562        let backing = fake.config();
1563
1564        let handle = tokio::spawn(async move {
1565            publish_messages(
1566                rx,
1567                TraderId::from("tester-001"),
1568                UUID4::new(),
1569                MessageBusConfig::default(),
1570                backing,
1571            )
1572            .await
1573        });
1574
1575        tx.send(fake_bus_message()).unwrap();
1576        wait_until_async(
1577            || {
1578                let aborted = Arc::clone(&fake.aborted);
1579                async move { aborted.load(Ordering::SeqCst) > 0 }
1580            },
1581            Duration::from_secs(5),
1582        )
1583        .await;
1584        assert!(!handle.is_finished());
1585
1586        fake.accept_writes.store(true, Ordering::SeqCst);
1587        wait_until_async(
1588            || {
1589                let committed = Arc::clone(&fake.committed);
1590                async move { committed.load(Ordering::SeqCst) > 0 }
1591            },
1592            Duration::from_secs(5),
1593        )
1594        .await;
1595
1596        tx.send(BusMessage::new_close()).unwrap();
1597        let result = handle.await.unwrap();
1598
1599        assert!(result.is_ok());
1600    }
1601
1602    #[rstest]
1603    #[tokio::test(flavor = "multi_thread")]
1604    async fn test_publish_messages_returns_transaction_aborted_by_unrecoverable_error() {
1605        let fake = FakeRedis::start("ERR syntax error").await;
1606        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1607        let backing = fake.config();
1608
1609        let handle = tokio::spawn(async move {
1610            publish_messages(
1611                rx,
1612                TraderId::from("tester-001"),
1613                UUID4::new(),
1614                MessageBusConfig::default(),
1615                backing,
1616            )
1617            .await
1618        });
1619
1620        tx.send(fake_bus_message()).unwrap();
1621
1622        let result = tokio::time::timeout(Duration::from_secs(5), handle)
1623            .await
1624            .expect("an unrecoverable aborted transaction should end the task")
1625            .unwrap();
1626
1627        assert_eq!(fake.aborted.load(Ordering::SeqCst), 1);
1628        assert!(
1629            result
1630                .unwrap_err()
1631                .to_string()
1632                .contains("Error publishing to stream")
1633        );
1634    }
1635
1636    #[rstest]
1637    fn test_external_io_from_backing_takes_stream_receiver() {
1638        let (stream_tx, stream_rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
1639        let backing = backing_with_stream_receiver(stream_rx);
1640        let message = BusMessage::with_str_topic(
1641            "events/data",
1642            BusPayloadType::QuoteTick,
1643            Bytes::from_static(b"payload"),
1644            SerializationEncoding::Json,
1645        );
1646
1647        let (_egress, mut ingress) = external_io_from_backing(Box::new(backing));
1648        stream_tx.try_send(message.clone()).unwrap();
1649        let mut receiver = ingress.take_receiver().unwrap();
1650        let received = receiver.try_recv().unwrap();
1651
1652        assert_eq!(received.topic, message.topic);
1653        assert_eq!(received.payload, message.payload);
1654        assert!(ingress.take_receiver().is_err());
1655    }
1656
1657    fn backing_with_stream_receiver(
1658        stream_rx: tokio::sync::mpsc::Receiver<BusMessage>,
1659    ) -> RedisMessageBusBacking {
1660        let (pub_tx, _pub_rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
1661        RedisMessageBusBacking {
1662            trader_id: TraderId::from("tester-001"),
1663            instance_id: UUID4::new(),
1664            pub_tx,
1665            pub_handle: None,
1666            stream_rx: Some(stream_rx),
1667            stream_handle: None,
1668            stream_signal: Arc::new(AtomicBool::new(false)),
1669            heartbeat_handle: None,
1670            heartbeat_signal: Arc::new(AtomicBool::new(false)),
1671        }
1672    }
1673
1674    fn fake_bus_message() -> BusMessage {
1675        BusMessage::with_str_topic(
1676            "test_topic",
1677            BusPayloadType::QuoteTick,
1678            Bytes::from("test_payload"),
1679            SerializationEncoding::Json,
1680        )
1681    }
1682
1683    /// A loopback RESP server standing in for Redis, which answers every queued `XADD` with
1684    /// `queued_error` until `accept_writes` is set, so each `EXEC` aborts with `EXECABORT`.
1685    struct FakeRedis {
1686        port: u16,
1687        accept_writes: Arc<AtomicBool>,
1688        aborted: Arc<AtomicUsize>,
1689        committed: Arc<AtomicUsize>,
1690    }
1691
1692    impl FakeRedis {
1693        async fn start(queued_error: &'static str) -> Self {
1694            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1695            let port = listener.local_addr().unwrap().port();
1696            let accept_writes = Arc::new(AtomicBool::new(false));
1697            let aborted = Arc::new(AtomicUsize::new(0));
1698            let committed = Arc::new(AtomicUsize::new(0));
1699
1700            let accept_writes_clone = Arc::clone(&accept_writes);
1701            let aborted_clone = Arc::clone(&aborted);
1702            let committed_clone = Arc::clone(&committed);
1703
1704            tokio::spawn(async move {
1705                while let Ok((socket, _)) = listener.accept().await {
1706                    let accept_writes = Arc::clone(&accept_writes_clone);
1707                    let aborted = Arc::clone(&aborted_clone);
1708                    let committed = Arc::clone(&committed_clone);
1709
1710                    tokio::spawn(async move {
1711                        let (read, mut write) = socket.into_split();
1712                        let mut reader = tokio::io::BufReader::new(read);
1713                        let mut queued = 0;
1714                        let mut queue_failed = false;
1715
1716                        while let Some(command) = read_resp_command(&mut reader).await {
1717                            let reply = match command[0].to_ascii_uppercase().as_slice() {
1718                                b"INFO" => {
1719                                    let info = "redis_version:8.0.0\r\n";
1720                                    format!("${}\r\n{info}\r\n", info.len())
1721                                }
1722                                b"MULTI" => {
1723                                    queued = 0;
1724                                    queue_failed = false;
1725                                    "+OK\r\n".to_string()
1726                                }
1727                                b"XADD" if accept_writes.load(Ordering::SeqCst) => {
1728                                    queued += 1;
1729                                    "+QUEUED\r\n".to_string()
1730                                }
1731                                b"XADD" => {
1732                                    queue_failed = true;
1733                                    format!("-{queued_error}\r\n")
1734                                }
1735                                b"EXEC" if queue_failed => {
1736                                    aborted.fetch_add(1, Ordering::SeqCst);
1737                                    "-EXECABORT Transaction discarded because of previous errors.\r\n"
1738                                        .to_string()
1739                                }
1740                                b"EXEC" => {
1741                                    committed.fetch_add(1, Ordering::SeqCst);
1742                                    format!("*{queued}\r\n{}", "$3\r\n0-1\r\n".repeat(queued))
1743                                }
1744                                _ => "+OK\r\n".to_string(),
1745                            };
1746
1747                            if write.write_all(reply.as_bytes()).await.is_err() {
1748                                break;
1749                            }
1750                        }
1751                    });
1752                }
1753            });
1754
1755            Self {
1756                port,
1757                accept_writes,
1758                aborted,
1759                committed,
1760            }
1761        }
1762
1763        fn config(&self) -> RedisMessageBusConfig {
1764            RedisMessageBusConfig {
1765                port: Some(self.port),
1766                ..Default::default()
1767            }
1768        }
1769    }
1770
1771    async fn read_resp_command(
1772        reader: &mut tokio::io::BufReader<tokio::net::tcp::OwnedReadHalf>,
1773    ) -> Option<Vec<Vec<u8>>> {
1774        let mut line = String::new();
1775        if reader.read_line(&mut line).await.ok()? == 0 {
1776            return None;
1777        }
1778        let count: usize = line.trim_end().strip_prefix('*')?.parse().ok()?;
1779        let mut command = Vec::with_capacity(count);
1780
1781        for _ in 0..count {
1782            line.clear();
1783            reader.read_line(&mut line).await.ok()?;
1784            let len: usize = line.trim_end().strip_prefix('$')?.parse().ok()?;
1785            let mut arg = vec![0; len + 2];
1786            reader.read_exact(&mut arg).await.ok()?;
1787            arg.truncate(len);
1788            command.push(arg);
1789        }
1790
1791        Some(command)
1792    }
1793}
1794
1795#[cfg(target_os = "linux")] // Run Redis tests on Linux platforms only
1796#[cfg(test)]
1797mod serial_tests {
1798    use std::{
1799        sync::{atomic::AtomicUsize, mpsc},
1800        thread,
1801    };
1802
1803    #[cfg(feature = "python")]
1804    use nautilus_common::python::msgbus::get_global_msgbus_factory_registry;
1805    use nautilus_common::{
1806        enums::Environment,
1807        msgbus::{self, TypedHandler},
1808        testing::wait_until_async,
1809    };
1810    use nautilus_live::{
1811        builder::LiveNodeBuilder,
1812        config::{LiveExecutionEngineConfig, LiveNodeConfig},
1813    };
1814    use nautilus_model::data::{QuoteTick, TradeTick};
1815    #[cfg(feature = "python")]
1816    use pyo3::{Py, Python, types::PyModule};
1817    use redis::aio::ConnectionManager;
1818    use rstest::*;
1819
1820    use super::*;
1821
1822    #[fixture]
1823    async fn redis_connection() -> ConnectionManager {
1824        let config = RedisMessageBusConfig {
1825            connection_timeout: 1,
1826            number_of_retries: 0,
1827            ..Default::default()
1828        };
1829        create_redis_connection(MSGBUS_STREAM, &config)
1830            .await
1831            .expect("A running Redis service is required for this test")
1832    }
1833
1834    fn redis_msgbus_factory(config: RedisMessageBusConfig) -> Box<dyn MessageBusBackingFactory> {
1835        #[cfg(feature = "python")]
1836        {
1837            Python::initialize();
1838            Python::attach(|py| {
1839                let module = PyModule::new(py, "infrastructure").unwrap();
1840                crate::python::infrastructure(py, &module).unwrap();
1841                let factory = Py::new(py, config).unwrap().into_any();
1842
1843                get_global_msgbus_factory_registry()
1844                    .extract(py, factory)
1845                    .unwrap()
1846            })
1847        }
1848
1849        #[cfg(not(feature = "python"))]
1850        Box::new(config)
1851    }
1852
1853    #[rstest]
1854    #[tokio::test(flavor = "multi_thread")]
1855    async fn test_stream_messages_terminate_signal(#[future] redis_connection: ConnectionManager) {
1856        let _con = redis_connection.await;
1857        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1858
1859        let trader_id = TraderId::from("tester-001");
1860        let instance_id = UUID4::new();
1861        let config = MessageBusConfig {
1862            use_instance_id: true,
1863            ..Default::default()
1864        };
1865
1866        let stream_key = get_stream_key(trader_id, instance_id, &config);
1867        let external_streams = vec![stream_key.clone()];
1868        let stream_signal = Arc::new(AtomicBool::new(false));
1869        let stream_signal_clone = Arc::clone(&stream_signal);
1870
1871        // Start the message streaming task
1872        let handle = tokio::spawn(async move {
1873            stream_messages(
1874                tx,
1875                RedisMessageBusConfig::default(),
1876                external_streams,
1877                stream_signal_clone,
1878            )
1879            .await
1880            .unwrap();
1881        });
1882
1883        stream_signal.store(true, Ordering::Relaxed);
1884        let _ = rx.recv().await; // Wait for the tx to close
1885
1886        // Shutdown and cleanup
1887        rx.close();
1888        handle.await.unwrap();
1889    }
1890
1891    #[rstest]
1892    #[tokio::test(flavor = "multi_thread")]
1893    async fn test_stream_messages_when_receiver_closed(
1894        #[future] redis_connection: ConnectionManager,
1895    ) {
1896        let mut con = redis_connection.await;
1897        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1898
1899        let trader_id = TraderId::from("tester-001");
1900        let instance_id = UUID4::new();
1901        let config = MessageBusConfig {
1902            use_instance_id: true,
1903            ..Default::default()
1904        };
1905
1906        let stream_key = get_stream_key(trader_id, instance_id, &config);
1907        let external_streams = vec![stream_key.clone()];
1908        let stream_signal = Arc::new(AtomicBool::new(false));
1909        let stream_signal_clone = Arc::clone(&stream_signal);
1910
1911        // Use a message ID in the future, as streaming begins
1912        // around the timestamp the task is spawned.
1913        let clock = get_atomic_clock_realtime();
1914        let future_id = (clock.get_time_ms() + 1_000_000).to_string();
1915
1916        // Publish test message
1917        let _: () = con
1918            .xadd(
1919                stream_key,
1920                future_id,
1921                &[("topic", "topic1"), ("payload", "data1")],
1922            )
1923            .await
1924            .unwrap();
1925
1926        // Immediately close channel
1927        rx.close();
1928
1929        // Start the message streaming task
1930        let handle = tokio::spawn(async move {
1931            stream_messages(
1932                tx,
1933                RedisMessageBusConfig::default(),
1934                external_streams,
1935                stream_signal_clone,
1936            )
1937            .await
1938            .unwrap();
1939        });
1940
1941        // Shutdown and cleanup
1942        handle.await.unwrap();
1943    }
1944
1945    #[rstest]
1946    #[tokio::test(flavor = "multi_thread")]
1947    async fn test_stream_messages(#[future] redis_connection: ConnectionManager) {
1948        let mut con = redis_connection.await;
1949        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
1950
1951        let trader_id = TraderId::from("tester-001");
1952        let instance_id = UUID4::new();
1953        let config = MessageBusConfig {
1954            use_instance_id: true,
1955            ..Default::default()
1956        };
1957
1958        let stream_key = get_stream_key(trader_id, instance_id, &config);
1959        let external_streams = vec![stream_key.clone()];
1960        let stream_signal = Arc::new(AtomicBool::new(false));
1961        let stream_signal_clone = Arc::clone(&stream_signal);
1962
1963        // Use a message ID in the future, as streaming begins
1964        // around the timestamp the task is spawned.
1965        let clock = get_atomic_clock_realtime();
1966        let future_id = (clock.get_time_ms() + 1_000_000).to_string();
1967
1968        // Publish test message
1969        let _: () = con
1970            .xadd(
1971                stream_key,
1972                future_id,
1973                &[("topic", "topic1"), ("payload", "data1")],
1974            )
1975            .await
1976            .unwrap();
1977
1978        // Start the message streaming task
1979        let handle = tokio::spawn(async move {
1980            stream_messages(
1981                tx,
1982                RedisMessageBusConfig::default(),
1983                external_streams,
1984                stream_signal_clone,
1985            )
1986            .await
1987            .unwrap();
1988        });
1989
1990        // Receive and verify the message
1991        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
1992        assert_eq!(msg.topic, "topic1");
1993        assert_eq!(msg.payload, Bytes::from("data1"));
1994
1995        // Shutdown and cleanup
1996        rx.close();
1997        stream_signal.store(true, Ordering::Relaxed);
1998        handle.await.unwrap();
1999    }
2000
2001    #[rstest]
2002    #[tokio::test(flavor = "multi_thread")]
2003    async fn test_stream_messages_skips_preexisting_entries(
2004        #[future] redis_connection: ConnectionManager,
2005    ) {
2006        let mut con = redis_connection.await;
2007        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2008
2009        let suffix = UUID4::new();
2010        let stream_key = format!("test:stream:no-backlog:{suffix}");
2011        let external_streams = vec![stream_key.clone()];
2012        let stream_signal = Arc::new(AtomicBool::new(false));
2013        let stream_signal_clone = Arc::clone(&stream_signal);
2014
2015        let _: () = con
2016            .xadd(
2017                &stream_key,
2018                "1-0",
2019                &[("topic", "preexisting"), ("payload", "old")],
2020            )
2021            .await
2022            .unwrap();
2023
2024        let handle = tokio::spawn(async move {
2025            stream_messages(
2026                tx,
2027                RedisMessageBusConfig::default(),
2028                external_streams,
2029                stream_signal_clone,
2030            )
2031            .await
2032            .unwrap();
2033        });
2034
2035        let future_id = (get_atomic_clock_realtime().get_time_ms() + 1_000_000).to_string();
2036        let _: () = con
2037            .xadd(
2038                &stream_key,
2039                future_id,
2040                &[("topic", "live"), ("payload", "new")],
2041            )
2042            .await
2043            .unwrap();
2044
2045        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2046
2047        rx.close();
2048        stream_signal.store(true, Ordering::Relaxed);
2049        handle.await.unwrap();
2050        let _: usize = con.del(stream_key).await.unwrap();
2051
2052        assert_eq!(msg.topic, "live");
2053        assert_eq!(msg.payload, Bytes::from("new"));
2054    }
2055
2056    #[rstest]
2057    #[tokio::test(flavor = "multi_thread")]
2058    async fn test_stream_messages_skips_malformed_entry(
2059        #[future] redis_connection: ConnectionManager,
2060    ) {
2061        let mut con = redis_connection.await;
2062        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2063
2064        let suffix = UUID4::new();
2065        let stream_key = format!("test:stream:malformed:{suffix}");
2066        let external_streams = vec![stream_key.clone()];
2067        let stream_signal = Arc::new(AtomicBool::new(false));
2068        let stream_signal_clone = Arc::clone(&stream_signal);
2069
2070        let clock = get_atomic_clock_realtime();
2071        let base_id = clock.get_time_ms() + 1_000_000;
2072
2073        let _: () = con
2074            .xadd(
2075                &stream_key,
2076                format!("{}", base_id + 1),
2077                &[("topic", "missing-payload")],
2078            )
2079            .await
2080            .unwrap();
2081        let _: () = con
2082            .xadd(
2083                &stream_key,
2084                format!("{}", base_id + 2),
2085                &[("topic", "valid"), ("payload", "data")],
2086            )
2087            .await
2088            .unwrap();
2089
2090        let handle = tokio::spawn(async move {
2091            stream_messages(
2092                tx,
2093                RedisMessageBusConfig::default(),
2094                external_streams,
2095                stream_signal_clone,
2096            )
2097            .await
2098            .unwrap();
2099        });
2100
2101        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2102
2103        rx.close();
2104        stream_signal.store(true, Ordering::Relaxed);
2105        handle.await.unwrap();
2106
2107        assert_eq!(msg.topic, "valid");
2108        assert_eq!(msg.payload, Bytes::from("data"));
2109    }
2110
2111    #[rstest]
2112    #[tokio::test(flavor = "multi_thread")]
2113    async fn test_stream_messages_returns_unrecoverable_read_error(
2114        #[future] redis_connection: ConnectionManager,
2115    ) {
2116        let mut con = redis_connection.await;
2117        let (tx, _rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2118
2119        let suffix = UUID4::new();
2120        let stream_key = format!("test:stream:wrong-type:{suffix}");
2121        let external_streams = vec![stream_key.clone()];
2122        let stream_signal = Arc::new(AtomicBool::new(false));
2123
2124        let _: () = con.set(&stream_key, "not-a-stream").await.unwrap();
2125
2126        let result = stream_messages(
2127            tx,
2128            RedisMessageBusConfig::default(),
2129            external_streams,
2130            stream_signal,
2131        )
2132        .await;
2133
2134        let _: () = con.del(&stream_key).await.unwrap();
2135
2136        assert!(result.is_err());
2137        assert!(
2138            result
2139                .unwrap_err()
2140                .to_string()
2141                .contains("Error reading from stream")
2142        );
2143    }
2144
2145    #[rstest]
2146    #[tokio::test(flavor = "multi_thread")]
2147    async fn test_stream_connection_returns_none_when_signaled() {
2148        let config = RedisMessageBusConfig {
2149            port: Some(1),
2150            connection_timeout: 20,
2151            ..Default::default()
2152        };
2153        let stream_signal = Arc::new(AtomicBool::new(true));
2154        let signal = Arc::clone(&stream_signal);
2155        let handle = tokio::spawn(async move { connect_stream_connection(&config, &signal).await });
2156
2157        wait_until_async(|| async { handle.is_finished() }, Duration::from_secs(1)).await;
2158
2159        assert!(handle.await.unwrap().unwrap().is_none());
2160    }
2161
2162    #[rstest]
2163    #[case::market_data(BusPayloadType::QuoteTick)]
2164    #[case::typed(BusPayloadType::TradingCommand)]
2165    #[tokio::test(flavor = "multi_thread")]
2166    async fn test_publish_messages(
2167        #[future] redis_connection: ConnectionManager,
2168        #[case] payload_type: BusPayloadType,
2169    ) {
2170        let mut con = redis_connection.await;
2171        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2172
2173        let trader_id = TraderId::from("tester-001");
2174        let instance_id = UUID4::new();
2175        let config = MessageBusConfig {
2176            use_instance_id: true,
2177            stream_per_topic: false,
2178            ..Default::default()
2179        };
2180        let stream_key = get_stream_key(trader_id, instance_id, &config);
2181
2182        // Start the publish_messages task
2183        let handle = tokio::spawn(async move {
2184            publish_messages(
2185                rx,
2186                trader_id,
2187                instance_id,
2188                config,
2189                RedisMessageBusConfig::default(),
2190            )
2191            .await
2192            .unwrap();
2193        });
2194
2195        // Send a test message
2196        let msg = BusMessage::with_str_topic(
2197            "test_topic",
2198            payload_type,
2199            Bytes::from("test_payload"),
2200            SerializationEncoding::Json,
2201        );
2202        tx.send(msg).unwrap();
2203
2204        // Wait until the message is published to Redis
2205        wait_until_async(
2206            || {
2207                let mut con = con.clone();
2208                let stream_key = stream_key.clone();
2209                async move {
2210                    let messages: RedisStreamBulk =
2211                        con.xread(&[&stream_key], &["0"]).await.unwrap();
2212                    !messages.is_empty()
2213                }
2214            },
2215            Duration::from_secs(3),
2216        )
2217        .await;
2218
2219        // Verify the message was published to Redis
2220        let messages: RedisStreamBulk = con.xread(&[&stream_key], &["0"]).await.unwrap();
2221        assert_eq!(messages.len(), 1);
2222        let stream_msgs = messages[0].get(&stream_key).unwrap();
2223        let stream_msg_array = &stream_msgs[0].values().next().unwrap();
2224        let decoded_message = decode_bus_message(stream_msg_array).unwrap();
2225        assert_eq!(decoded_message.topic, "test_topic");
2226        assert_eq!(decoded_message.payload_type, payload_type);
2227        assert_eq!(decoded_message.encoding, SerializationEncoding::Json);
2228        assert_eq!(decoded_message.payload, Bytes::from("test_payload"));
2229
2230        // Stop publishing task
2231        let msg = BusMessage::new_close();
2232        tx.send(msg).unwrap();
2233
2234        // Shutdown and cleanup
2235        handle.await.unwrap();
2236    }
2237
2238    #[rstest]
2239    #[tokio::test(flavor = "multi_thread")]
2240    async fn test_publish_messages_applies_autotrim_maxlen(
2241        #[future] redis_connection: ConnectionManager,
2242    ) {
2243        let mut con = redis_connection.await;
2244        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2245        let trader_id = TraderId::from("tester-001");
2246        let instance_id = UUID4::new();
2247        let config = MessageBusConfig {
2248            buffer_interval_ms: Some(10),
2249            autotrim_mins: Some(60),
2250            autotrim_maxlen: Some(10),
2251            use_instance_id: true,
2252            stream_per_topic: false,
2253            ..Default::default()
2254        };
2255        let stream_key = get_stream_key(trader_id, instance_id, &config);
2256
2257        let handle = tokio::spawn(async move {
2258            publish_messages(
2259                rx,
2260                trader_id,
2261                instance_id,
2262                config,
2263                RedisMessageBusConfig::default(),
2264            )
2265            .await
2266            .unwrap();
2267        });
2268        let msg = BusMessage::with_str_topic(
2269            "test_topic",
2270            BusPayloadType::QuoteTick,
2271            Bytes::from_static(b"test_payload"),
2272            SerializationEncoding::Json,
2273        );
2274
2275        // The integration service uses Redis's default stream node limits,
2276        // so these writes span multiple macro nodes.
2277        let messages_sent = 250;
2278
2279        for _ in 0..messages_sent {
2280            tx.send(msg.clone()).unwrap();
2281        }
2282        tx.send(BusMessage::new_close()).unwrap();
2283        handle.await.unwrap();
2284
2285        let stream_len: usize = con.xlen(&stream_key).await.unwrap();
2286        let _: usize = con.del(stream_key).await.unwrap();
2287
2288        assert!(stream_len >= 10);
2289        assert!(stream_len < messages_sent);
2290    }
2291
2292    #[rstest]
2293    #[case::immediate(None)]
2294    #[case::buffered(Some(10))]
2295    #[tokio::test(flavor = "multi_thread")]
2296    async fn test_publish_messages_retries_buffer_after_connection_loss(
2297        #[future] redis_connection: ConnectionManager,
2298        #[case] buffer_interval_ms: Option<u32>,
2299    ) {
2300        let mut con = redis_connection.await;
2301        let proxy = RedisProxy::start().await;
2302        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2303        let trader_id = TraderId::from("tester-001");
2304        let instance_id = UUID4::new();
2305        let config = MessageBusConfig {
2306            use_instance_id: true,
2307            stream_per_topic: false,
2308            buffer_interval_ms,
2309            ..Default::default()
2310        };
2311        let stream_key = get_stream_key(trader_id, instance_id, &config);
2312        let backing = proxy.config();
2313
2314        let handle = tokio::spawn(async move {
2315            publish_messages(rx, trader_id, instance_id, config, backing).await
2316        });
2317
2318        tx.send(test_bus_message("first")).unwrap();
2319        wait_for_stream_len(&con, &stream_key, 1).await;
2320
2321        // The next write fails on the dropped connection and new connections are closed at once,
2322        // so the backlog builds up and is written in several batches once the proxy accepts again
2323        proxy.interrupt().await;
2324        let backlog: Vec<String> = (0..2_500).map(|i| format!("backlog-{i}")).collect();
2325        for topic in &backlog {
2326            tx.send(test_bus_message(topic)).unwrap();
2327        }
2328        tokio::time::sleep(Duration::from_millis(200)).await;
2329
2330        assert_eq!(read_stream_topics(&mut con, &stream_key).await, ["first"]);
2331        assert!(!handle.is_finished());
2332
2333        proxy.resume();
2334        wait_for_last_topic(&con, &stream_key, backlog.last().unwrap()).await;
2335        tx.send(test_bus_message("last")).unwrap();
2336        wait_for_last_topic(&con, &stream_key, "last").await;
2337
2338        let topics = read_stream_topics(&mut con, &stream_key).await;
2339        tx.send(BusMessage::new_close()).unwrap();
2340        let result = handle.await.unwrap();
2341        let _: usize = con.del(&stream_key).await.unwrap();
2342
2343        // Seeing `first` on the stream does not show that its acknowledgement reached the
2344        // publisher before the interruption, and if it did not, `first` is written again.
2345        let mut expected = vec!["first".to_string()];
2346        if topics.get(1).is_some_and(|topic| topic == "first") {
2347            expected.push("first".to_string());
2348        }
2349        expected.extend(backlog);
2350        expected.push("last".to_string());
2351        assert_eq!(topics, expected);
2352        assert!(result.is_ok());
2353    }
2354
2355    #[rstest]
2356    #[case::no_autotrim(None)]
2357    #[case::autotrim(Some(60))]
2358    #[tokio::test(flavor = "multi_thread")]
2359    async fn test_publish_messages_close_returns_during_outage(
2360        #[future] redis_connection: ConnectionManager,
2361        #[case] autotrim_mins: Option<u32>,
2362    ) {
2363        let mut con = redis_connection.await;
2364        let proxy = RedisProxy::start().await;
2365        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2366        let trader_id = TraderId::from("tester-001");
2367        let instance_id = UUID4::new();
2368
2369        // A stream per topic, so the write after the interruption is due a trim
2370        let config = MessageBusConfig {
2371            use_instance_id: true,
2372            stream_per_topic: true,
2373            autotrim_mins,
2374            ..Default::default()
2375        };
2376        let stream_key = get_stream_key(trader_id, instance_id, &config);
2377        let first_key = format!("{stream_key}:first");
2378        let second_key = format!("{stream_key}:second");
2379        let backing = proxy.config();
2380
2381        let handle = tokio::spawn(async move {
2382            publish_messages(rx, trader_id, instance_id, config, backing).await
2383        });
2384
2385        tx.send(test_bus_message("first")).unwrap();
2386        wait_for_stream_len(&con, &first_key, 1).await;
2387
2388        proxy.interrupt().await;
2389        tx.send(test_bus_message("second")).unwrap();
2390        tx.send(BusMessage::new_close()).unwrap();
2391
2392        let result = tokio::time::timeout(Duration::from_secs(5), handle)
2393            .await
2394            .expect("close should return during an outage")
2395            .unwrap();
2396        let second_len: usize = con.xlen(&second_key).await.unwrap();
2397        let _: usize = con.del(&[&first_key, &second_key]).await.unwrap();
2398
2399        // Seeing `first` on the stream does not show that its acknowledgement reached the
2400        // publisher before the interruption, and if it did not, `first` is still buffered.
2401        let error = result.unwrap_err().to_string();
2402        let unpublished = ["1", "2"].map(|count| {
2403            format!("Failed to publish buffered messages before closing ({count} unpublished)")
2404        });
2405        assert_eq!(second_len, 0);
2406        assert!(unpublished.contains(&error), "{error}");
2407    }
2408
2409    #[rstest]
2410    #[tokio::test(flavor = "multi_thread")]
2411    async fn test_publish_messages_returns_unrecoverable_write_error(
2412        #[future] redis_connection: ConnectionManager,
2413    ) {
2414        let mut con = redis_connection.await;
2415        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2416        let trader_id = TraderId::from("tester-001");
2417        let instance_id = UUID4::new();
2418        let config = MessageBusConfig {
2419            use_instance_id: true,
2420            stream_per_topic: false,
2421            ..Default::default()
2422        };
2423        let stream_key = get_stream_key(trader_id, instance_id, &config);
2424
2425        let _: () = con.set(&stream_key, "not-a-stream").await.unwrap();
2426
2427        let handle = tokio::spawn(async move {
2428            publish_messages(
2429                rx,
2430                trader_id,
2431                instance_id,
2432                config,
2433                RedisMessageBusConfig::default(),
2434            )
2435            .await
2436        });
2437
2438        tx.send(test_bus_message("first")).unwrap();
2439
2440        let result = tokio::time::timeout(Duration::from_secs(5), handle)
2441            .await
2442            .expect("a non-retryable write error should end the task")
2443            .unwrap();
2444        let _: usize = con.del(&stream_key).await.unwrap();
2445
2446        assert!(
2447            result
2448                .unwrap_err()
2449                .to_string()
2450                .contains("Error publishing to stream")
2451        );
2452    }
2453
2454    #[rstest]
2455    #[tokio::test(flavor = "multi_thread")]
2456    async fn test_two_live_nodes_publish_and_ingest_external_redis_stream(
2457        #[future] redis_connection: ConnectionManager,
2458    ) {
2459        let _con = redis_connection.await;
2460        let redis_config = RedisMessageBusConfig::default();
2461        let trader_a = TraderId::from("NODEA-001");
2462        let instance_a = UUID4::new();
2463        let node_a_msgbus = MessageBusConfig {
2464            use_instance_id: true,
2465            stream_per_topic: false,
2466            ..Default::default()
2467        };
2468        let stream_key = get_stream_key(trader_a, instance_a, &node_a_msgbus);
2469        let node_b_msgbus = MessageBusConfig {
2470            external_streams: Some(vec![stream_key]),
2471            stream_per_topic: false,
2472            ..Default::default()
2473        };
2474        let quote = QuoteTick::default();
2475        let trade = TradeTick::default();
2476        let ready_topic = "test.quotes.ready";
2477        let (built_tx, built_rx) = mpsc::channel::<()>();
2478        let (ready_tx, ready_rx) = mpsc::channel::<()>();
2479        let (quote_tx, quote_rx) = mpsc::channel::<QuoteTick>();
2480        let (trade_tx, trade_rx) = mpsc::channel::<TradeTick>();
2481
2482        let node_b = thread::spawn({
2483            let redis_config = redis_config.clone();
2484            move || -> anyhow::Result<()> {
2485                let runtime = tokio::runtime::Builder::new_multi_thread()
2486                    .worker_threads(2)
2487                    .enable_all()
2488                    .build()?;
2489
2490                runtime.block_on(async move {
2491                    let config = LiveNodeConfig {
2492                        environment: Environment::Sandbox,
2493                        trader_id: TraderId::from("NODEB-001"),
2494                        msgbus: Some(node_b_msgbus),
2495                        exec_engine: LiveExecutionEngineConfig {
2496                            reconciliation: false,
2497                            ..Default::default()
2498                        },
2499                        delay_post_stop: Duration::ZERO,
2500                        timeout_connection: Duration::from_millis(500),
2501                        timeout_disconnection: Duration::from_millis(500),
2502                        ..Default::default()
2503                    };
2504                    let mut node = LiveNodeBuilder::from_config(config)?
2505                        .with_external_msgbus_factory(redis_msgbus_factory(redis_config))
2506                        .build()?;
2507                    let handle = node.handle();
2508                    let quote_handler = TypedHandler::from({
2509                        let quote_tx = quote_tx.clone();
2510                        let handle = handle.clone();
2511                        move |quote: &QuoteTick| {
2512                            let _ = quote_tx.send(*quote);
2513                            handle.stop();
2514                        }
2515                    });
2516                    let trade_handler = TypedHandler::from(move |trade: &TradeTick| {
2517                        let _ = trade_tx.send(*trade);
2518                    });
2519
2520                    let ready_handler = TypedHandler::from(move |_: &QuoteTick| {
2521                        let _ = ready_tx.send(());
2522                    });
2523
2524                    msgbus::subscribe_quotes(ready_topic.into(), ready_handler, None);
2525                    msgbus::subscribe_quotes("data.quotes.*".into(), quote_handler, None);
2526                    msgbus::subscribe_trades("data.trades.*".into(), trade_handler, None);
2527                    msgbus::get_message_bus()
2528                        .borrow_mut()
2529                        .add_streaming_type(BusPayloadType::QuoteTick);
2530                    built_tx.send(())?;
2531                    let result = tokio::time::timeout(Duration::from_secs(10), node.run()).await;
2532                    msgbus::get_message_bus().borrow_mut().dispose();
2533
2534                    match result {
2535                        Ok(Ok(())) => Ok(()),
2536                        Ok(Err(e)) => Err(e),
2537                        Err(e) => anyhow::bail!("node B timed out: {e}"),
2538                    }
2539                })
2540            }
2541        });
2542
2543        built_rx
2544            .recv_timeout(Duration::from_secs(5))
2545            .expect("node B should build and register subscriptions");
2546
2547        let node_a = thread::spawn(move || -> anyhow::Result<()> {
2548            let runtime = tokio::runtime::Builder::new_multi_thread()
2549                .worker_threads(2)
2550                .enable_all()
2551                .build()?;
2552
2553            runtime.block_on(async move {
2554                let config = LiveNodeConfig {
2555                    environment: Environment::Sandbox,
2556                    trader_id: trader_a,
2557                    instance_id: Some(instance_a),
2558                    msgbus: Some(node_a_msgbus),
2559                    exec_engine: LiveExecutionEngineConfig {
2560                        reconciliation: false,
2561                        ..Default::default()
2562                    },
2563                    delay_post_stop: Duration::ZERO,
2564                    timeout_connection: Duration::from_millis(500),
2565                    timeout_disconnection: Duration::from_millis(500),
2566                    ..Default::default()
2567                };
2568                let _node = LiveNodeBuilder::from_config(config)?
2569                    .with_external_msgbus_factory(Box::new(redis_config))
2570                    .build()?;
2571
2572                // A running node does not guarantee its Redis reader has set its initial cursor
2573                wait_until_async(
2574                    || {
2575                        let ready = ready_rx.try_recv().is_ok();
2576
2577                        if !ready {
2578                            msgbus::publish_quote(ready_topic.into(), &quote);
2579                        }
2580
2581                        async move { ready }
2582                    },
2583                    Duration::from_secs(5),
2584                )
2585                .await;
2586
2587                msgbus::publish_trade("data.trades.TEST".into(), &trade);
2588                msgbus::publish_quote("data.quotes.TEST".into(), &quote);
2589                msgbus::get_message_bus().borrow_mut().dispose();
2590
2591                Ok(())
2592            })
2593        });
2594
2595        node_a
2596            .join()
2597            .expect("node A thread should not panic")
2598            .expect("node A should publish externally");
2599        let received_quote = quote_rx
2600            .recv_timeout(Duration::from_secs(10))
2601            .expect("node B should republish the registered quote type");
2602        node_b
2603            .join()
2604            .expect("node B thread should not panic")
2605            .expect("node B should ingest and stop cleanly");
2606
2607        assert_eq!(received_quote, quote);
2608        assert!(
2609            trade_rx.try_recv().is_err(),
2610            "unregistered trade type should not republish internally"
2611        );
2612    }
2613
2614    #[rstest]
2615    #[tokio::test(flavor = "multi_thread")]
2616    async fn test_stream_messages_multiple_streams(#[future] redis_connection: ConnectionManager) {
2617        let mut con = redis_connection.await;
2618        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2619
2620        // Setup multiple stream keys
2621        let suffix = UUID4::new();
2622        let stream_key1 = format!("test:stream:{suffix}:1");
2623        let stream_key2 = format!("test:stream:{suffix}:2");
2624        let external_streams = vec![stream_key1.clone(), stream_key2.clone()];
2625        let stream_signal = Arc::new(AtomicBool::new(false));
2626        let stream_signal_clone = Arc::clone(&stream_signal);
2627
2628        let clock = get_atomic_clock_realtime();
2629        let base_id = clock.get_time_ms() + 1_000_000;
2630
2631        // Start streaming task
2632        let handle = tokio::spawn(async move {
2633            stream_messages(
2634                tx,
2635                RedisMessageBusConfig::default(),
2636                external_streams,
2637                stream_signal_clone,
2638            )
2639            .await
2640            .unwrap();
2641        });
2642
2643        // Publish to stream 1 at higher ID
2644        let _: () = con
2645            .xadd(
2646                &stream_key1,
2647                format!("{}", base_id + 100),
2648                &[("topic", "stream1-first"), ("payload", "data")],
2649            )
2650            .await
2651            .unwrap();
2652
2653        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2654        assert_eq!(msg.topic, "stream1-first");
2655
2656        // Publish to stream 2 at lower ID (tests independent cursor tracking)
2657        let _: () = con
2658            .xadd(
2659                &stream_key2,
2660                format!("{}", base_id + 50),
2661                &[("topic", "stream2-second"), ("payload", "data")],
2662            )
2663            .await
2664            .unwrap();
2665
2666        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2667        assert_eq!(msg.topic, "stream2-second");
2668
2669        // Shutdown and cleanup
2670        rx.close();
2671        stream_signal.store(true, Ordering::Relaxed);
2672        handle.await.unwrap();
2673    }
2674
2675    #[rstest]
2676    #[tokio::test(flavor = "multi_thread")]
2677    async fn test_stream_messages_interleaved_at_different_rates(
2678        #[future] redis_connection: ConnectionManager,
2679    ) {
2680        let mut con = redis_connection.await;
2681        let (tx, mut rx) = tokio::sync::mpsc::channel::<BusMessage>(100);
2682
2683        // Setup multiple stream keys
2684        let suffix = UUID4::new();
2685        let stream_key1 = format!("test:stream:interleaved:{suffix}:1");
2686        let stream_key2 = format!("test:stream:interleaved:{suffix}:2");
2687        let stream_key3 = format!("test:stream:interleaved:{suffix}:3");
2688        let external_streams = vec![
2689            stream_key1.clone(),
2690            stream_key2.clone(),
2691            stream_key3.clone(),
2692        ];
2693        let stream_signal = Arc::new(AtomicBool::new(false));
2694        let stream_signal_clone = Arc::clone(&stream_signal);
2695
2696        let clock = get_atomic_clock_realtime();
2697        let base_id = clock.get_time_ms() + 1_000_000;
2698
2699        let handle = tokio::spawn(async move {
2700            stream_messages(
2701                tx,
2702                RedisMessageBusConfig::default(),
2703                external_streams,
2704                stream_signal_clone,
2705            )
2706            .await
2707            .unwrap();
2708        });
2709
2710        // Stream 1 advances with high ID
2711        let _: () = con
2712            .xadd(
2713                &stream_key1,
2714                format!("{}", base_id + 100),
2715                &[("topic", "s1m1"), ("payload", "data")],
2716            )
2717            .await
2718            .unwrap();
2719        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2720        assert_eq!(msg.topic, "s1m1");
2721
2722        // Stream 2 gets message at lower ID - would be skipped with global cursor
2723        let _: () = con
2724            .xadd(
2725                &stream_key2,
2726                format!("{}", base_id + 50),
2727                &[("topic", "s2m1"), ("payload", "data")],
2728            )
2729            .await
2730            .unwrap();
2731        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2732        assert_eq!(msg.topic, "s2m1");
2733
2734        // Stream 3 gets message at even lower ID
2735        let _: () = con
2736            .xadd(
2737                &stream_key3,
2738                format!("{}", base_id + 25),
2739                &[("topic", "s3m1"), ("payload", "data")],
2740            )
2741            .await
2742            .unwrap();
2743        let msg = receive_bus_message(&mut rx, Duration::from_secs(2)).await;
2744        assert_eq!(msg.topic, "s3m1");
2745
2746        // Shutdown and cleanup
2747        rx.close();
2748        stream_signal.store(true, Ordering::Relaxed);
2749        handle.await.unwrap();
2750    }
2751
2752    #[rstest]
2753    #[tokio::test(flavor = "multi_thread")]
2754    async fn test_close() {
2755        let trader_id = TraderId::from("tester-001");
2756        let instance_id = UUID4::new();
2757        let config = MessageBusConfig {
2758            use_instance_id: true,
2759            ..Default::default()
2760        };
2761
2762        let mut db = RedisMessageBusBacking::new(
2763            trader_id,
2764            instance_id,
2765            config,
2766            RedisMessageBusConfig::default(),
2767        )
2768        .unwrap();
2769
2770        // Close the message bus backing (test should not hang)
2771        MessageBusBacking::close(&mut db);
2772    }
2773
2774    #[rstest]
2775    #[tokio::test(flavor = "multi_thread")]
2776    async fn test_heartbeat_task() {
2777        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<BusMessage>();
2778        let signal = Arc::new(AtomicBool::new(false));
2779
2780        // Start the heartbeat task with a short interval
2781        let handle = tokio::spawn(run_heartbeat(1, Arc::clone(&signal), tx));
2782
2783        let heartbeat = receive_unbounded_bus_message(&mut rx, Duration::from_secs(2)).await;
2784
2785        // Stop the heartbeat task
2786        signal.store(true, Ordering::Relaxed);
2787        handle.await.unwrap();
2788
2789        // Ensure heartbeats were sent
2790        assert_eq!(heartbeat.topic, HEARTBEAT_TOPIC);
2791    }
2792
2793    async fn receive_bus_message(
2794        rx: &mut tokio::sync::mpsc::Receiver<BusMessage>,
2795        timeout: Duration,
2796    ) -> BusMessage {
2797        let mut received = None;
2798
2799        wait_until_async(
2800            || {
2801                if received.is_none() {
2802                    received = rx.try_recv().ok();
2803                }
2804
2805                let has_received = received.is_some();
2806                async move { has_received }
2807            },
2808            timeout,
2809        )
2810        .await;
2811
2812        received.unwrap()
2813    }
2814
2815    async fn receive_unbounded_bus_message(
2816        rx: &mut tokio::sync::mpsc::UnboundedReceiver<BusMessage>,
2817        timeout: Duration,
2818    ) -> BusMessage {
2819        let mut received = None;
2820
2821        wait_until_async(
2822            || {
2823                if received.is_none() {
2824                    received = rx.try_recv().ok();
2825                }
2826
2827                let has_received = received.is_some();
2828                async move { has_received }
2829            },
2830            timeout,
2831        )
2832        .await;
2833
2834        received.unwrap()
2835    }
2836
2837    fn test_bus_message(topic: &str) -> BusMessage {
2838        BusMessage::with_str_topic(
2839            topic,
2840            BusPayloadType::QuoteTick,
2841            Bytes::from("test_payload"),
2842            SerializationEncoding::Json,
2843        )
2844    }
2845
2846    async fn wait_for_stream_len(con: &ConnectionManager, stream_key: &str, len: usize) {
2847        wait_until_async(
2848            || {
2849                let mut con = con.clone();
2850                let stream_key = stream_key.to_string();
2851                async move {
2852                    let stream_len: usize = con.xlen(&stream_key).await.unwrap();
2853                    stream_len >= len
2854                }
2855            },
2856            Duration::from_secs(5),
2857        )
2858        .await;
2859    }
2860
2861    async fn wait_for_last_topic(con: &ConnectionManager, stream_key: &str, topic: &str) {
2862        wait_until_async(
2863            || {
2864                let mut con = con.clone();
2865                let stream_key = stream_key.to_string();
2866                let topic = topic.to_string();
2867                async move {
2868                    read_stream_topics(&mut con, &stream_key).await.last() == Some(&topic)
2869                }
2870            },
2871            Duration::from_secs(5),
2872        )
2873        .await;
2874    }
2875
2876    async fn read_stream_topics(con: &mut ConnectionManager, stream_key: &str) -> Vec<String> {
2877        let messages: RedisStreamBulk = con.xread(&[stream_key], &["0"]).await.unwrap();
2878        messages
2879            .iter()
2880            .flat_map(HashMap::values)
2881            .flatten()
2882            .flat_map(HashMap::values)
2883            .map(|fields| decode_bus_message(fields).unwrap().topic.to_string())
2884            .collect()
2885    }
2886
2887    /// A loopback TCP proxy to the local Redis that can drop its connections and close new ones.
2888    struct RedisProxy {
2889        port: u16,
2890        accepting: Arc<AtomicBool>,
2891        open_connections: Arc<AtomicUsize>,
2892        disconnect_tx: Arc<tokio::sync::watch::Sender<u64>>,
2893    }
2894
2895    impl RedisProxy {
2896        async fn start() -> Self {
2897            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2898            let port = listener.local_addr().unwrap().port();
2899            let accepting = Arc::new(AtomicBool::new(true));
2900            let open_connections = Arc::new(AtomicUsize::new(0));
2901            let disconnect_tx = Arc::new(tokio::sync::watch::channel(0_u64).0);
2902
2903            let accepting_clone = Arc::clone(&accepting);
2904            let open_connections_clone = Arc::clone(&open_connections);
2905            let disconnect_tx_clone = Arc::clone(&disconnect_tx);
2906
2907            tokio::spawn(async move {
2908                while let Ok((mut inbound, _)) = listener.accept().await {
2909                    // Subscribe before checking, so an interrupt cannot miss this connection
2910                    let mut disconnect_rx = disconnect_tx_clone.subscribe();
2911
2912                    if !accepting_clone.load(Ordering::SeqCst) {
2913                        continue; // Closing the socket at once fails the client's first command
2914                    }
2915
2916                    let open_connections = Arc::clone(&open_connections_clone);
2917                    open_connections.fetch_add(1, Ordering::SeqCst);
2918
2919                    tokio::spawn(async move {
2920                        if let Ok(mut outbound) =
2921                            tokio::net::TcpStream::connect("127.0.0.1:6379").await
2922                        {
2923                            tokio::select! {
2924                                _ = tokio::io::copy_bidirectional(&mut inbound, &mut outbound) => {}
2925                                _ = disconnect_rx.changed() => {}
2926                            }
2927                        }
2928                        open_connections.fetch_sub(1, Ordering::SeqCst);
2929                    });
2930                }
2931            });
2932
2933            Self {
2934                port,
2935                accepting,
2936                open_connections,
2937                disconnect_tx,
2938            }
2939        }
2940
2941        fn config(&self) -> RedisMessageBusConfig {
2942            RedisMessageBusConfig {
2943                port: Some(self.port),
2944                ..Default::default()
2945            }
2946        }
2947
2948        /// Drops every open connection and closes new ones at once until [`Self::resume`].
2949        async fn interrupt(&self) {
2950            self.accepting.store(false, Ordering::SeqCst);
2951            self.disconnect_tx.send_modify(|count| *count += 1);
2952
2953            wait_until_async(
2954                || {
2955                    let open_connections = Arc::clone(&self.open_connections);
2956                    async move { open_connections.load(Ordering::SeqCst) == 0 }
2957                },
2958                Duration::from_secs(1),
2959            )
2960            .await;
2961        }
2962
2963        fn resume(&self) {
2964            self.accepting.store(true, Ordering::SeqCst);
2965        }
2966    }
2967}