Skip to main content

nautilus_live/
runner.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Async event loop runner for live and sandbox trading nodes.
17//!
18//! `AsyncRunner` owns seven tokio mpsc channel pairs plus a shutdown
19//! signal channel. Construction creates the channels without side
20//! effects. The sender halves are placed into thread-local storage
21//! via [`AsyncRunner::bind_senders`] so that adapters and engine
22//! components can resolve them through the `get_*_sender()` accessors
23//! in `nautilus_common::runner` and `nautilus_common::live::runner`.
24//!
25//! Channel pairs:
26//!
27//! - **Time events**: timer callbacks dispatched by the clock.
28//! - **System events**: system notifications handled by the live node.
29//! - **System commands**: control requests handled by the live node.
30//! - **Execution events**: fills, order updates, and account state from
31//!   execution clients to the execution engine.
32//! - **Trading commands**: deferred order actions routed to their direct endpoint.
33//! - **Data events**: market data from adapters to the data engine.
34//! - **Data commands**: subscribe/unsubscribe requests to data clients.
35//!
36//! Both `AsyncRunner::run` and `LiveNode::run` use a `biased;` select with
37//! system and execution branches polled ahead of data branches. Within each
38//! system or execution channel pair, events precede commands. Timer and data channels rotate.
39//!
40//! The runner can drive the event loop in two ways:
41//!
42//! - **Standalone**: call [`AsyncRunner::run`], which binds senders and
43//!   enters a `tokio::select!` loop internally.
44//! - **Integrated**: call [`AsyncRunner::take_channels`] to extract the
45//!   receivers and run the `select!` loop directly inside `LiveNode::run`,
46//!   where it is interleaved with startup, reconciliation, and shutdown
47//!   phases.
48//!
49//! # Invariants
50//!
51//! - `bind_senders` must be called before any code that reads from TLS.
52//!   This includes adapter constructors, clock initialization, and
53//!   execution client start methods. Every path from construction to
54//!   the event loop must bind before the first TLS read.
55//! - The event loop and all TLS consumers must execute on the same
56//!   thread. Senders are cloneable and `Send`, but the `RefCell`-backed
57//!   TLS slots are not accessible from other threads.
58//! - Only one runner at a time should own the TLS slots on a given
59//!   thread. `bind_senders` overwrites any existing TLS contents on the
60//!   thread, so the last caller wins.
61
62use std::{
63    fmt::Debug,
64    sync::Arc,
65    thread::{self, ThreadId},
66};
67
68use nautilus_common::{
69    live::{
70        dispatch::DispatchMessage,
71        runner::{
72            replace_data_event_sender, replace_exec_event_sender, replace_system_command_sender,
73            replace_system_event_sender,
74        },
75        sender::{DispatchSender, EventSender},
76    },
77    messages::{
78        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent, data::DataCommand,
79        execution::TradingCommand,
80    },
81    msgbus::{self, MessagingSwitchboard},
82    runner::{
83        DataCommandSender, TimeEventMessage, TimeEventSender, TradingCommandMessage,
84        TradingCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
85        replace_time_event_sender,
86    },
87};
88use nautilus_model::events::OrderEventAny;
89
90use crate::dispatch::drain_callbacks;
91#[cfg(feature = "node")]
92use crate::node::{LiveNodeHandle, NodeState};
93
94/// Asynchronous implementation of `DataCommandSender` for live environments.
95#[derive(Debug)]
96pub struct AsyncDataCommandSender {
97    cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
98    owner: ThreadId,
99    #[cfg(feature = "node")]
100    node_handle: Option<LiveNodeHandle>,
101}
102
103impl AsyncDataCommandSender {
104    /// Creates a sender owned by the calling thread.
105    ///
106    /// Construct it on the runtime thread so command sends capture callback roots.
107    #[must_use]
108    pub fn new(cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>) -> Self {
109        Self {
110            cmd_tx,
111            owner: thread::current().id(),
112            #[cfg(feature = "node")]
113            node_handle: None,
114        }
115    }
116}
117
118impl DataCommandSender for AsyncDataCommandSender {
119    fn execute(&self, command: DataCommand) {
120        if let Err(e) = self.cmd_tx.send(DispatchMessage::new(command, self.owner)) {
121            // Disposal releases retained subscriptions after the node drops its receivers
122            #[cfg(feature = "node")]
123            if node_stopped(self.node_handle.as_ref()) {
124                return;
125            }
126
127            log::error!("Failed to send data command: {e}");
128        }
129    }
130}
131
132/// Asynchronous implementation of `TimeEventSender` for live environments.
133#[derive(Debug, Clone)]
134pub struct AsyncTimeEventSender {
135    time_tx: DispatchSender<TimeEventMessage>,
136    #[cfg(feature = "node")]
137    node_handle: Option<LiveNodeHandle>,
138}
139
140impl AsyncTimeEventSender {
141    /// Creates a sender owned by the calling runtime thread.
142    #[must_use]
143    pub fn new(
144        time_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
145    ) -> Self {
146        Self {
147            time_tx: DispatchSender::new(time_tx),
148            #[cfg(feature = "node")]
149            node_handle: None,
150        }
151    }
152}
153
154impl TimeEventSender for AsyncTimeEventSender {
155    fn send(&self, message: TimeEventMessage) {
156        if let Err(e) = self.time_tx.send(message) {
157            #[cfg(feature = "node")]
158            if node_stopped(self.node_handle.as_ref()) {
159                return;
160            }
161
162            log::error!("Failed to send time event message: {e}");
163        }
164    }
165}
166
167/// Asynchronous implementation of `TradingCommandSender` for live environments.
168#[derive(Debug)]
169pub struct AsyncTradingCommandSender {
170    cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
171    owner: ThreadId,
172}
173
174impl AsyncTradingCommandSender {
175    /// Creates a sender owned by the calling thread.
176    ///
177    /// Construct it on the runtime thread so command sends capture callback roots.
178    #[must_use]
179    pub fn new(
180        cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
181    ) -> Self {
182        Self {
183            cmd_tx,
184            owner: thread::current().id(),
185        }
186    }
187}
188
189impl TradingCommandSender for AsyncTradingCommandSender {
190    fn execute(&self, message: TradingCommandMessage) {
191        if let Err(e) = self.cmd_tx.send(DispatchMessage::new(message, self.owner)) {
192            log::error!("Failed to send trading command: {e}");
193        }
194    }
195}
196
197pub trait Runner {
198    fn run(&mut self);
199}
200
201/// Channel receivers for the async event loop.
202///
203/// These can be extracted from `AsyncRunner` via `take_channels()` to drive
204/// the event loop directly on the same thread as the msgbus endpoints.
205#[derive(Debug)]
206pub struct AsyncRunnerChannels {
207    pub time_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
208    pub system_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
209    pub system_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
210    pub exec_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
211    pub exec_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
212    pub data_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
213    pub data_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
214}
215
216#[cfg(feature = "node")]
217#[allow(
218    clippy::large_enum_variant,
219    reason = "runner events are consumed immediately; boxing would add routing allocations"
220)]
221pub(crate) enum PendingRunnerEvent {
222    TimeEvent(DispatchMessage<TimeEventMessage>),
223    SystemEvent(DispatchMessage<SystemEvent>),
224    SystemCommand(DispatchMessage<SystemCommand>),
225    ExecEvent(DispatchMessage<ExecutionEvent>),
226    ExecCommand(DispatchMessage<TradingCommandMessage>),
227    DataEvent(DispatchMessage<DataEvent>),
228    DataCommand(DispatchMessage<DataCommand>),
229}
230
231pub struct AsyncRunner {
232    channels: AsyncRunnerChannels,
233    skip_time: bool,
234    skip_data: bool,
235    time_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
236    system_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemEvent>>,
237    system_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemCommand>>,
238    signal_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
239    signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
240    exec_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<ExecutionEvent>>,
241    exec_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
242    data_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataEvent>>,
243    data_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
244}
245
246/// Handle for stopping the `AsyncRunner` from another context.
247#[derive(Clone, Debug)]
248pub struct AsyncRunnerHandle {
249    signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
250}
251
252impl AsyncRunnerHandle {
253    /// Signals the runner to stop.
254    pub fn stop(&self) {
255        if let Err(e) = self.signal_tx.send(()) {
256            log::error!("Failed to send shutdown signal: {e}");
257        }
258    }
259}
260
261impl Default for AsyncRunner {
262    fn default() -> Self {
263        Self::new()
264    }
265}
266
267impl Debug for AsyncRunner {
268    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
269        f.debug_struct(stringify!(AsyncRunner)).finish()
270    }
271}
272
273impl AsyncRunner {
274    /// Creates a new [`AsyncRunner`] instance.
275    ///
276    /// Creates channels but does not bind senders to thread-local storage.
277    /// Call [`bind_senders`](Self::bind_senders) before creating clients that
278    /// read from TLS, and again before entering the event loop.
279    #[must_use]
280    #[rustfmt::skip]
281    pub fn new() -> Self {
282        use tokio::sync::mpsc::unbounded_channel; // tokio-import-ok
283
284        let (time_evt_tx, time_evt_rx) = unbounded_channel::<DispatchMessage<TimeEventMessage>>();
285        let (system_evt_tx, system_evt_rx) = unbounded_channel::<DispatchMessage<SystemEvent>>();
286        let (system_cmd_tx, system_cmd_rx) = unbounded_channel::<DispatchMessage<SystemCommand>>();
287        let (signal_tx, signal_rx) = unbounded_channel::<()>();
288        let (exec_evt_tx, exec_evt_rx) = unbounded_channel::<DispatchMessage<ExecutionEvent>>();
289        let (exec_cmd_tx, exec_cmd_rx) = unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
290        let (data_evt_tx, data_evt_rx) = unbounded_channel::<DispatchMessage<DataEvent>>();
291        let (data_cmd_tx, data_cmd_rx) = unbounded_channel::<DispatchMessage<DataCommand>>();
292
293        Self {
294            channels: AsyncRunnerChannels {
295                time_evt_rx,
296                system_evt_rx,
297                system_cmd_rx,
298                exec_evt_rx,
299                exec_cmd_rx,
300                data_evt_rx,
301                data_cmd_rx,
302            },
303            skip_time: false,
304            skip_data: false,
305            time_evt_tx,
306            system_evt_tx,
307            system_cmd_tx,
308            signal_rx,
309            signal_tx,
310            exec_evt_tx,
311            exec_cmd_tx,
312            data_evt_tx,
313            data_cmd_tx,
314        }
315    }
316
317    /// Binds this runner's channel senders to thread-local storage.
318    ///
319    /// Call before creating clients that read from TLS (e.g., in the builder),
320    /// and again before entering the event loop to reclaim ownership if another
321    /// runner was constructed on this thread in the interim.
322    pub fn bind_senders(&self) {
323        self.bind_senders_with_data_sender(AsyncDataCommandSender::new(self.data_cmd_tx.clone()));
324    }
325
326    #[cfg(feature = "node")]
327    pub(crate) fn bind_senders_for_node(&self, handle: LiveNodeHandle) {
328        self.bind_senders_with_data_sender(AsyncDataCommandSender {
329            cmd_tx: self.data_cmd_tx.clone(),
330            owner: thread::current().id(),
331            node_handle: Some(handle),
332        });
333    }
334
335    #[rustfmt::skip]
336    fn bind_senders_with_data_sender(&self, sender: AsyncDataCommandSender) {
337        replace_time_event_sender(Arc::new(AsyncTimeEventSender {
338            time_tx: DispatchSender::new(self.time_evt_tx.clone()),
339            #[cfg(feature = "node")]
340            node_handle: sender.node_handle.clone(),
341        }));
342        replace_system_event_sender(EventSender::new(self.system_evt_tx.clone()));
343        replace_system_command_sender(DispatchSender::new(self.system_cmd_tx.clone()));
344        replace_exec_event_sender(EventSender::new(self.exec_evt_tx.clone()));
345        replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(self.exec_cmd_tx.clone())));
346        replace_data_event_sender(EventSender::new(self.data_evt_tx.clone()));
347        replace_data_cmd_sender(Arc::new(sender));
348    }
349
350    /// Stops the runner with an internal shutdown signal.
351    pub fn stop(&self) {
352        if let Err(e) = self.signal_tx.send(()) {
353            log::error!("Failed to send shutdown signal: {e}");
354        }
355    }
356
357    /// Returns a handle that can be used to stop the runner from another context.
358    #[must_use]
359    pub fn handle(&self) -> AsyncRunnerHandle {
360        AsyncRunnerHandle {
361            signal_tx: self.signal_tx.clone(),
362        }
363    }
364
365    /// Consumes the runner and returns the channel receivers for direct event loop driving.
366    ///
367    /// This is used when the event loop needs to run on the same thread as the msgbus
368    /// endpoints (which use thread-local storage).
369    #[must_use]
370    pub fn take_channels(self) -> AsyncRunnerChannels {
371        self.channels
372    }
373
374    /// Flushes all pending data events and commands from the channels.
375    ///
376    /// Loops until both data channels are empty, processing each item
377    /// into the cache immediately. Used in `start()` where channels are
378    /// not extracted.
379    pub fn flush_pending_data(&mut self) {
380        let mut total = 0;
381
382        loop {
383            let mut progressed = false;
384
385            // Events drain before commands here even though the runtime select
386            // prefers the opposite for everything-else: `LiveNode::start()`
387            // calls this after `connect_data_clients()` to push queued
388            // instrument definitions into the cache. A pending
389            // subscription command (e.g. `SubscribeBars`) processed before the
390            // matching instrument lands would be rejected by the data engine.
391            while let Ok(evt) = self.channels.data_evt_rx.try_recv() {
392                Self::dispatch_data_event(evt);
393                progressed = true;
394                total += 1;
395            }
396
397            while let Ok(cmd) = self.channels.data_cmd_rx.try_recv() {
398                Self::handle_data_command(cmd);
399                progressed = true;
400                total += 1;
401            }
402
403            if !progressed {
404                break;
405            }
406        }
407
408        if total > 0 {
409            log::debug!("Flushed {total} pending data events/commands");
410        }
411    }
412
413    #[cfg(feature = "node")]
414    pub(crate) fn drain_pending_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
415        let mut events = Vec::new();
416
417        while let Ok(event) = self.channels.system_evt_rx.try_recv() {
418            events.push(event);
419        }
420
421        events
422    }
423
424    #[cfg(feature = "node")]
425    pub(crate) fn drain_pending_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
426        let mut commands = Vec::new();
427
428        while let Ok(command) = self.channels.system_cmd_rx.try_recv() {
429            commands.push(command);
430        }
431
432        commands
433    }
434
435    /// Runs the async runner event loop.
436    ///
437    /// This method processes time, system, execution, and data events in an async loop.
438    /// It will run until a signal is received or the event streams are closed.
439    ///
440    /// # Errors
441    ///
442    /// Returns an error if callback dispatch fails. Pending channel messages and the failure latch
443    /// are retained.
444    pub async fn run(&mut self) -> anyhow::Result<()> {
445        self.bind_senders();
446
447        log::info!("AsyncRunner starting");
448
449        loop {
450            let callbacks_pending = drain_callbacks().await?;
451            let commands_empty = self.channels.data_cmd_rx.is_empty();
452            let poll_time =
453                !self.skip_time || (self.channels.data_evt_rx.is_empty() && commands_empty);
454            let poll_data = !self.skip_data || commands_empty;
455
456            tokio::select! {
457                biased;
458
459                Some(()) = self.signal_rx.recv() => {
460                    log::info!("AsyncRunner received signal, shutting down");
461                    return Ok(());
462                },
463                () = std::future::ready(()), if callbacks_pending => {},
464                Some(handler) = self.channels.time_evt_rx.recv(), if poll_time => {
465                    self.skip_time = true;
466                    let _ = Self::handle_time_event(handler);
467                },
468                Some(event) = self.channels.system_evt_rx.recv() => {
469                    self.skip_time = false;
470                    log::error!("System event {event} requires the LiveNode runner");
471                },
472                Some(command) = self.channels.system_cmd_rx.recv() => {
473                    self.skip_time = false;
474                    log::error!("System command {command} requires the LiveNode runner");
475                },
476                Some(evt) = self.channels.exec_evt_rx.recv() => {
477                    self.skip_time = false;
478                    Self::dispatch_exec_event(evt);
479                },
480                Some(cmd) = self.channels.exec_cmd_rx.recv() => {
481                    self.skip_time = false;
482                    Self::handle_trading_command(cmd);
483                },
484                Some(evt) = self.channels.data_evt_rx.recv(), if poll_data => {
485                    self.skip_time = false;
486                    self.skip_data = true;
487                    Self::dispatch_data_event(evt);
488                },
489                Some(cmd) = self.channels.data_cmd_rx.recv() => {
490                    self.skip_time = false;
491                    self.skip_data = false;
492                    Self::handle_data_command(cmd);
493                },
494                else => {
495                    log::debug!("AsyncRunner all channels closed, exiting");
496                    return Ok(());
497                }
498            };
499        }
500    }
501
502    /// Handles a time event under its originating callback root.
503    ///
504    /// # Panics
505    ///
506    /// Panics if a rooted message is processed outside its owner thread.
507    #[inline]
508    #[must_use]
509    pub fn handle_time_event(message: DispatchMessage<TimeEventMessage>) -> bool {
510        message.dispatch(TimeEventMessage::dispatch)
511    }
512
513    /// Handles a data command under its captured callback root by sending to the `DataEngine`.
514    ///
515    /// # Panics
516    ///
517    /// Panics if a rooted command is processed outside its owner thread.
518    #[inline]
519    pub fn handle_data_command(cmd: DispatchMessage<DataCommand>) {
520        cmd.dispatch(|cmd| {
521            msgbus::send_data_command(MessagingSwitchboard::data_engine_execute(), cmd);
522        });
523    }
524
525    /// Dispatches a received data event under its originating callback root.
526    ///
527    /// # Panics
528    ///
529    /// Panics if a rooted message is processed outside its owner thread.
530    pub fn dispatch_data_event(event: DispatchMessage<DataEvent>) {
531        event.dispatch(Self::handle_data_event);
532    }
533
534    /// Dispatches a received execution event under its originating callback root.
535    ///
536    /// # Panics
537    ///
538    /// Panics if a rooted message is processed outside its owner thread.
539    pub fn dispatch_exec_event(event: DispatchMessage<ExecutionEvent>) {
540        event.dispatch(Self::handle_exec_event);
541    }
542
543    /// Handles a data event by sending to the appropriate `DataEngine` endpoint.
544    #[inline]
545    pub fn handle_data_event(event: DataEvent) {
546        match event {
547            DataEvent::Data(data) => {
548                msgbus::send_data(MessagingSwitchboard::data_engine_process_data(), data);
549            }
550            DataEvent::ResponseEnvelope(response) => {
551                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &response);
552            }
553            DataEvent::RequestFailed(failure) => {
554                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &failure);
555            }
556            DataEvent::Response(resp) => {
557                msgbus::send_data_response(MessagingSwitchboard::data_engine_response(), resp);
558            }
559            #[cfg(feature = "defi")]
560            DataEvent::DeFi(data) => {
561                msgbus::send_defi_data(MessagingSwitchboard::data_engine_process_defi_data(), data);
562            }
563            #[cfg(feature = "defi")]
564            DataEvent::PoolSnapshotResponse(response) => {
565                msgbus::send_any(MessagingSwitchboard::data_engine_process(), &response);
566            }
567            #[cfg(not(feature = "defi"))]
568            #[allow(
569                unreachable_patterns,
570                reason = "DeFi variants can exist without this crate's defi feature"
571            )]
572            other => log::error!(
573                "Cannot handle {other} data event, nautilus-live built without its `defi` feature"
574            ),
575        }
576    }
577
578    /// Dispatches an internal execution command directly to the execution engine.
579    #[inline]
580    pub fn handle_exec_command(cmd: TradingCommand) {
581        msgbus::send_trading_command(MessagingSwitchboard::exec_engine_execute(), cmd);
582    }
583
584    /// Dispatches a trading command and its deferred children under their captured callback roots.
585    ///
586    /// # Panics
587    ///
588    /// Panics if a rooted command is processed outside its owner thread.
589    #[inline]
590    pub fn handle_trading_command(message: DispatchMessage<TradingCommandMessage>) {
591        message.dispatch_trading(|_| {});
592    }
593
594    /// Handles an execution event by sending to the appropriate engine endpoint.
595    #[inline]
596    pub fn handle_exec_event(event: ExecutionEvent) {
597        match event {
598            ExecutionEvent::Order(order_event) => {
599                msgbus::send_order_event(MessagingSwitchboard::exec_engine_process(), order_event);
600            }
601            ExecutionEvent::OrderSubmittedBatch(batch) => {
602                for submitted in batch {
603                    msgbus::send_order_event(
604                        MessagingSwitchboard::exec_engine_process(),
605                        OrderEventAny::Submitted(submitted),
606                    );
607                }
608            }
609            ExecutionEvent::OrderAcceptedBatch(batch) => {
610                for accepted in batch {
611                    msgbus::send_order_event(
612                        MessagingSwitchboard::exec_engine_process(),
613                        OrderEventAny::Accepted(accepted),
614                    );
615                }
616            }
617            ExecutionEvent::OrderCanceledBatch(batch) => {
618                for canceled in batch {
619                    msgbus::send_order_event(
620                        MessagingSwitchboard::exec_engine_process(),
621                        OrderEventAny::Canceled(canceled),
622                    );
623                }
624            }
625            ExecutionEvent::Report(report) => {
626                Self::handle_exec_report(report);
627            }
628            ExecutionEvent::Account(ref account) => {
629                msgbus::send_account_state(
630                    MessagingSwitchboard::portfolio_update_account(),
631                    account,
632                );
633            }
634            ExecutionEvent::Data(ref data) => {
635                let topic = msgbus::switchboard::get_custom_topic(&data.data_type);
636                msgbus::publish_any(topic, data);
637            }
638        }
639    }
640
641    #[inline]
642    pub fn handle_exec_report(report: ExecutionReport) {
643        let endpoint = MessagingSwitchboard::exec_engine_reconcile_execution_report();
644        msgbus::send_execution_report(endpoint, report);
645    }
646}
647
648#[cfg(feature = "node")]
649impl AsyncRunner {
650    pub(crate) fn poll_pending(&mut self, mut process: impl FnMut(PendingRunnerEvent)) -> usize {
651        self.bind_senders();
652
653        let pending = (
654            self.channels.time_evt_rx.len(),
655            self.channels.system_evt_rx.len(),
656            self.channels.system_cmd_rx.len(),
657            self.channels.exec_evt_rx.len(),
658            self.channels.exec_cmd_rx.len(),
659            self.channels.data_evt_rx.len(),
660            self.channels.data_cmd_rx.len(),
661        );
662
663        let total =
664            pending.0 + pending.1 + pending.2 + pending.3 + pending.4 + pending.5 + pending.6;
665        log::debug!("Draining shutdown queues ({total} events initially pending)");
666        let mut progress = 0usize;
667        let mut process = |event| {
668            process(event);
669            progress += 1;
670            if progress.is_multiple_of(10_000) {
671                log::debug!("Processed {progress} events during shutdown drain");
672            }
673        };
674
675        let mut processed = 0;
676        processed += poll_channel(
677            &mut self.channels.time_evt_rx,
678            pending.0,
679            PendingRunnerEvent::TimeEvent,
680            &mut process,
681        );
682        processed += poll_channel(
683            &mut self.channels.system_evt_rx,
684            pending.1,
685            PendingRunnerEvent::SystemEvent,
686            &mut process,
687        );
688        processed += poll_channel(
689            &mut self.channels.system_cmd_rx,
690            pending.2,
691            PendingRunnerEvent::SystemCommand,
692            &mut process,
693        );
694        processed += poll_channel(
695            &mut self.channels.exec_evt_rx,
696            pending.3,
697            PendingRunnerEvent::ExecEvent,
698            &mut process,
699        );
700        processed += poll_channel(
701            &mut self.channels.exec_cmd_rx,
702            pending.4,
703            PendingRunnerEvent::ExecCommand,
704            &mut process,
705        );
706        processed += poll_channel(
707            &mut self.channels.data_evt_rx,
708            pending.5,
709            PendingRunnerEvent::DataEvent,
710            &mut process,
711        );
712        processed += poll_channel(
713            &mut self.channels.data_cmd_rx,
714            pending.6,
715            PendingRunnerEvent::DataCommand,
716            &mut process,
717        );
718        log::debug!("Drained {processed} remaining events during shutdown");
719        processed
720    }
721
722    pub(crate) async fn recv(&mut self) -> Option<PendingRunnerEvent> {
723        let commands_empty = self.channels.data_cmd_rx.is_empty();
724        let poll_time = !self.skip_time || (self.channels.data_evt_rx.is_empty() && commands_empty);
725        let poll_data = !self.skip_data || commands_empty;
726        tokio::select! {
727            biased;
728
729            Some(message) = self.channels.time_evt_rx.recv(), if poll_time => {
730                self.skip_time = true;
731                Some(PendingRunnerEvent::TimeEvent(message))
732            }
733            Some(event) = self.channels.system_evt_rx.recv() => {
734                self.skip_time = false;
735                Some(PendingRunnerEvent::SystemEvent(event))
736            }
737            Some(command) = self.channels.system_cmd_rx.recv() => {
738                self.skip_time = false;
739                Some(PendingRunnerEvent::SystemCommand(command))
740            }
741            Some(event) = self.channels.exec_evt_rx.recv() => {
742                self.skip_time = false;
743                Some(PendingRunnerEvent::ExecEvent(event))
744            }
745            Some(command) = self.channels.exec_cmd_rx.recv() => {
746                self.skip_time = false;
747                Some(PendingRunnerEvent::ExecCommand(command))
748            }
749            Some(event) = self.channels.data_evt_rx.recv(), if poll_data => {
750                self.skip_time = false;
751                self.skip_data = true;
752                Some(PendingRunnerEvent::DataEvent(event))
753            }
754            Some(command) = self.channels.data_cmd_rx.recv() => {
755                self.skip_time = false;
756                self.skip_data = false;
757                Some(PendingRunnerEvent::DataCommand(command))
758            }
759            else => None,
760        }
761    }
762}
763
764#[cfg(feature = "node")]
765fn node_stopped(handle: Option<&LiveNodeHandle>) -> bool {
766    handle.is_some_and(|handle| handle.state() == NodeState::Stopped)
767}
768
769#[cfg(feature = "node")]
770fn poll_channel<T>(
771    receiver: &mut tokio::sync::mpsc::UnboundedReceiver<T>,
772    pending: usize,
773    event: impl Fn(T) -> PendingRunnerEvent,
774    process: &mut impl FnMut(PendingRunnerEvent),
775) -> usize {
776    let mut processed = 0;
777
778    for _ in 0..pending {
779        let Ok(message) = receiver.try_recv() else {
780            break;
781        };
782
783        process(event(message));
784        processed += 1;
785    }
786
787    processed
788}
789
790#[cfg(test)]
791mod tests {
792    #[cfg(not(all(feature = "simulation", madsim)))]
793    use std::num::NonZeroU64;
794    use std::{cell::RefCell, rc::Rc, sync::Arc, time::Duration};
795
796    #[cfg(not(all(feature = "simulation", madsim)))]
797    use nautilus_common::live::LiveTimer;
798    use nautilus_common::{
799        actor,
800        cache::Cache,
801        clock::VirtualClock,
802        live::{
803            dispatch::DispatchMessage,
804            runner::{
805                get_data_event_sender, get_exec_event_sender, get_system_command_sender,
806                get_system_event_sender, try_get_system_command_sender,
807                try_get_system_event_sender,
808            },
809        },
810        messages::{
811            ExecutionEvent,
812            data::{SubscribeCommand, SubscribeCustomData},
813            execution::{CancelAllOrders, QueryAccount, TradingCommand},
814            system::{ReconnectSocket, SocketState, SocketStateChange},
815        },
816        msgbus::{Endpoint, MStr, TypedIntoHandler, stubs::get_typed_into_message_saving_handler},
817        runner::{
818            SyncTradingCommandSender, TimeEventMessage, drain_trading_cmd_queue,
819            get_data_cmd_sender, get_time_event_sender, get_trading_cmd_sender,
820            replace_exec_cmd_sender, try_get_time_event_sender, try_get_trading_cmd_sender,
821        },
822        timer::{TimeEvent, TimeEventCallback},
823    };
824    #[cfg(not(all(feature = "simulation", madsim)))]
825    use nautilus_core::time::get_atomic_clock_realtime;
826    use nautilus_core::{UUID4, UnixNanos};
827    use nautilus_execution::engine::ExecutionEngine;
828    #[cfg(feature = "node")]
829    use nautilus_model::types::Money;
830    use nautilus_model::{
831        data::{
832            Data, DataType, FundingRateUpdate, InstrumentStatus, OptionGreeks, quote::QuoteTick,
833            stubs::stub_instrument_status,
834        },
835        enums::{AccountType, OrderSide},
836        events::{OrderEventAny, account::state::AccountState, order::spec::OrderSubmittedSpec},
837        identifiers::{
838            AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TraderId, Venue,
839        },
840        instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt},
841        types::{Price, Quantity},
842    };
843    use rstest::rstest;
844    use ustr::Ustr;
845
846    use super::*;
847
848    #[rstest]
849    fn test_runner_dispatches_typed_data_once_in_order() {
850        let mut runner = AsyncRunner::new();
851        let expected = vec![
852            Data::from(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt())),
853            Data::FundingRate(FundingRateUpdate::new(
854                InstrumentId::from("ETHUSDT.BINANCE"),
855                "0.00012".parse().unwrap(),
856                Some(480),
857                Some(UnixNanos::from(300)),
858                UnixNanos::from(100),
859                UnixNanos::from(200),
860            )),
861            Data::InstrumentStatus(InstrumentStatus {
862                ts_event: UnixNanos::from(400),
863                ts_init: UnixNanos::from(500),
864                ..stub_instrument_status()
865            }),
866            Data::OptionGreeks(OptionGreeks {
867                instrument_id: InstrumentId::from("BTC-20270101-50000-C.DERIBIT"),
868                mark_iv: Some(0.55),
869                bid_iv: Some(0.54),
870                ask_iv: Some(0.56),
871                underlying_price: Some(50001.0),
872                open_interest: Some(123.0),
873                ts_event: UnixNanos::from(600),
874                ts_init: UnixNanos::from(700),
875                ..OptionGreeks::default()
876            }),
877        ];
878        let received = Rc::new(RefCell::new(Vec::<Data>::new()));
879        let observed = received.clone();
880        msgbus::register_data_endpoint(
881            MessagingSwitchboard::data_engine_process_data(),
882            TypedIntoHandler::from(move |data| observed.borrow_mut().push(data)),
883        );
884
885        for data in &expected {
886            runner
887                .data_evt_tx
888                .send(DataEvent::Data(data.clone()).into())
889                .unwrap();
890        }
891
892        runner.flush_pending_data();
893
894        assert_eq!(
895            serde_json::to_value(&*received.borrow()).unwrap(),
896            serde_json::to_value(&expected).unwrap(),
897        );
898    }
899
900    #[rstest]
901    #[case(false)]
902    #[case(true)]
903    #[tokio::test]
904    async fn test_runner_callback_failure_stops_before_next_command(#[case] before_run: bool) {
905        actor::clear_callbacks().unwrap();
906        let mut runner = AsyncRunner::new();
907        let received = Rc::new(RefCell::new(Vec::new()));
908        let observed = received.clone();
909        msgbus::register_trading_command_endpoint(
910            MessagingSwitchboard::risk_engine_execute(),
911            TypedIntoHandler::from(move |command: TradingCommand| {
912                observed.borrow_mut().push(command.ts_init());
913                crate::dispatch::tests::latch_callback_failure();
914            }),
915        );
916
917        let sender = AsyncTradingCommandSender::new(runner.exec_cmd_tx.clone());
918        let seed_endpoint = ustr::Ustr::from("test.callback-root-seed");
919        msgbus::register_trading_command_endpoint(
920            seed_endpoint.into(),
921            TypedIntoHandler::from(move |command: TradingCommand| {
922                sender.execute(TradingCommandMessage::new(
923                    MessagingSwitchboard::risk_engine_execute(),
924                    command,
925                ));
926            }),
927        );
928
929        for timestamp in [17, 23] {
930            SyncTradingCommandSender.execute(TradingCommandMessage::new(
931                seed_endpoint.into(),
932                TradingCommand::QueryAccount(QueryAccount::new(
933                    "TRADER-001".into(),
934                    None,
935                    "SIM-001".into(),
936                    UUID4::new(),
937                    timestamp.into(),
938                    None,
939                    None,
940                )),
941            ));
942        }
943
944        drain_trading_cmd_queue();
945        let first = runner.channels.exec_cmd_rx.try_recv().unwrap();
946        let second = runner.channels.exec_cmd_rx.try_recv().unwrap();
947        assert!(first.is_rooted());
948        assert!(second.is_rooted());
949        runner.exec_cmd_tx.send(first).unwrap();
950        runner.exec_cmd_tx.send(second).unwrap();
951
952        if before_run {
953            crate::dispatch::tests::latch_callback_failure();
954        }
955
956        let error = runner.run().await.unwrap_err();
957
958        assert_eq!(
959            error.downcast_ref::<actor::CallbackDispatchError>(),
960            Some(&actor::CallbackDispatchError::DeliveryUnwound)
961        );
962        assert_eq!(
963            *received.borrow(),
964            if before_run {
965                vec![]
966            } else {
967                vec![UnixNanos::from(17)]
968            }
969        );
970        assert_eq!(
971            runner.channels.exec_cmd_rx.len(),
972            if before_run { 2 } else { 1 }
973        );
974        assert!(!runner.exec_cmd_tx.is_closed());
975        assert_eq!(
976            actor::callback_failure(),
977            Some(actor::CallbackDispatchError::DeliveryUnwound)
978        );
979        drop(runner);
980        assert_eq!(actor::clear_callbacks(), Ok(()));
981    }
982
983    #[tokio::test]
984    async fn test_runner_stop_preserves_messages_and_channel_only_scheduling() {
985        let mut runner = AsyncRunner::new();
986        let received = Rc::new(RefCell::new(Vec::new()));
987        let stop = runner.signal_tx.clone();
988
989        for timestamp in 1..=65 {
990            let received = received.clone();
991            let stop = stop.clone();
992            runner
993                .time_evt_tx
994                .send(
995                    TimeEventMessage::new(
996                        TimeEvent::new(
997                            "runner-resume".into(),
998                            UUID4::new(),
999                            timestamp.into(),
1000                            timestamp.into(),
1001                        ),
1002                        TimeEventCallback::RustLocal(Rc::new(move |_| {
1003                            received.borrow_mut().push(timestamp);
1004                            if timestamp == 65 {
1005                                stop.send(()).unwrap();
1006                            }
1007                        })),
1008                    )
1009                    .into(),
1010                )
1011                .unwrap();
1012        }
1013
1014        runner.stop();
1015        runner.run().await.unwrap();
1016
1017        assert!(received.borrow().is_empty());
1018        assert_eq!(runner.channels.time_evt_rx.len(), 65);
1019        assert!(!runner.time_evt_tx.is_closed());
1020
1021        let (result, at_yield) = tokio::join!(biased;
1022            runner.run(),
1023            async { received.borrow().clone() },
1024        );
1025        result.unwrap();
1026
1027        let expected: Vec<u64> = (1..=65).collect();
1028        assert_eq!(at_yield, expected);
1029        assert_eq!(*received.borrow(), expected);
1030        assert_eq!(runner.channels.time_evt_rx.len(), 0);
1031        assert!(!runner.time_evt_tx.is_closed());
1032    }
1033
1034    fn test_quote() -> QuoteTick {
1035        QuoteTick {
1036            instrument_id: InstrumentId::from("EUR/USD.SIM"),
1037            bid_price: Price::from("1.10000"),
1038            ask_price: Price::from("1.10001"),
1039            bid_size: Quantity::from(1_000_000),
1040            ask_size: Quantity::from(1_000_000),
1041            ts_event: UnixNanos::default(),
1042            ts_init: UnixNanos::default(),
1043        }
1044    }
1045
1046    fn test_system_event() -> SystemEvent {
1047        SystemEvent::SocketState(SocketStateChange::new(
1048            ClientId::from("BINANCE"),
1049            Some(Venue::from("BINANCE")),
1050            Ustr::from("binance-futures-market-streams"),
1051            SocketState::Connected,
1052        ))
1053    }
1054
1055    fn test_system_command() -> SystemCommand {
1056        SystemCommand::ReconnectSocket(ReconnectSocket::new(
1057            TraderId::from("TRADER-001"),
1058            ClientId::from("POLYMARKET"),
1059            Ustr::from("polymarket-market-streams"),
1060            UnixNanos::from(3),
1061        ))
1062    }
1063
1064    #[cfg(feature = "node")]
1065    #[rstest]
1066    fn test_poll_pending_processes_entry_snapshot_across_channels() {
1067        let mut runner = AsyncRunner::new();
1068        let time_evt_tx = runner.time_evt_tx.clone();
1069        let data_evt_tx = runner.data_evt_tx.clone();
1070        let data_cmd_tx = runner.data_cmd_tx.clone();
1071        let exec_evt_tx = runner.exec_evt_tx.clone();
1072        let exec_cmd_tx = runner.exec_cmd_tx.clone();
1073
1074        let time_event = TimeEvent::new(
1075            Ustr::from("test"),
1076            UUID4::new(),
1077            UnixNanos::from(1),
1078            UnixNanos::from(2),
1079        );
1080        time_evt_tx
1081            .send(
1082                (TimeEventMessage::new(time_event, TimeEventCallback::from(|_: TimeEvent| {})))
1083                    .into(),
1084            )
1085            .unwrap();
1086        exec_evt_tx
1087            .send(
1088                (ExecutionEvent::Order(OrderEventAny::Submitted(
1089                    OrderSubmittedSpec::builder()
1090                        .client_order_id(ClientOrderId::from("O-POLL-001"))
1091                        .build(),
1092                )))
1093                .into(),
1094            )
1095            .unwrap();
1096        exec_cmd_tx
1097            .send(
1098                TradingCommandMessage::new(
1099                    MessagingSwitchboard::exec_engine_execute(),
1100                    TradingCommand::CancelAllOrders(CancelAllOrders::new(
1101                        TraderId::from("TRADER-001"),
1102                        None,
1103                        StrategyId::from("S-POLL-001"),
1104                        InstrumentId::from("EUR/USD.SIM"),
1105                        Some(OrderSide::Buy),
1106                        UUID4::new(),
1107                        UnixNanos::from(3),
1108                        None,
1109                        None,
1110                    )),
1111                )
1112                .into(),
1113            )
1114            .unwrap();
1115        data_evt_tx
1116            .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1117            .unwrap();
1118        data_cmd_tx
1119            .send(
1120                DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1121                    client_id: Some(ClientId::from("POLL")),
1122                    venue: None,
1123                    data_type: DataType::new("QuoteTick", None, None),
1124                    command_id: UUID4::new(),
1125                    ts_init: UnixNanos::from(4),
1126                    correlation_id: None,
1127                    params: None,
1128                }))
1129                .into(),
1130            )
1131            .unwrap();
1132
1133        runner.bind_senders();
1134        get_system_command_sender()
1135            .send(test_system_command())
1136            .unwrap();
1137        get_system_event_sender().send(test_system_event()).unwrap();
1138        get_system_event_sender().send(test_system_event()).unwrap();
1139        let mut processed_by_channel = [0; 7];
1140        let mut processed_order = Vec::new();
1141
1142        let first = runner.poll_pending(|event| match event {
1143            PendingRunnerEvent::TimeEvent(_) => {
1144                processed_by_channel[0] += 1;
1145                processed_order.push("time");
1146            }
1147            PendingRunnerEvent::SystemEvent(_) => {
1148                processed_by_channel[1] += 1;
1149                processed_order.push("system_event");
1150            }
1151            PendingRunnerEvent::SystemCommand(_) => {
1152                processed_by_channel[2] += 1;
1153                processed_order.push("system_command");
1154            }
1155            PendingRunnerEvent::ExecEvent(_) => {
1156                processed_by_channel[3] += 1;
1157                processed_order.push("exec_event");
1158            }
1159            PendingRunnerEvent::ExecCommand(_) => {
1160                processed_by_channel[4] += 1;
1161                processed_order.push("exec_command");
1162            }
1163            PendingRunnerEvent::DataEvent(_) => {
1164                processed_by_channel[5] += 1;
1165                processed_order.push("data_event");
1166                data_evt_tx
1167                    .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1168                    .unwrap();
1169            }
1170            PendingRunnerEvent::DataCommand(_) => {
1171                processed_by_channel[6] += 1;
1172                processed_order.push("data_command");
1173            }
1174        });
1175
1176        let second = runner.poll_pending(|event| match event {
1177            PendingRunnerEvent::DataEvent(_) => {
1178                processed_by_channel[5] += 1;
1179                processed_order.push("data_event");
1180            }
1181            _ => panic!("Unexpected runner event"),
1182        });
1183
1184        assert_eq!(first, 8);
1185        assert_eq!(second, 1);
1186        assert_eq!(processed_by_channel, [1, 2, 1, 1, 1, 2, 1]);
1187        assert_eq!(
1188            processed_order,
1189            [
1190                "time",
1191                "system_event",
1192                "system_event",
1193                "system_command",
1194                "exec_event",
1195                "exec_command",
1196                "data_event",
1197                "data_command",
1198                "data_event",
1199            ]
1200        );
1201    }
1202
1203    #[cfg(feature = "node")]
1204    #[tokio::test]
1205    async fn test_recv_processes_system_event_before_command() {
1206        let mut runner = AsyncRunner::new();
1207
1208        runner
1209            .system_cmd_tx
1210            .send((test_system_command()).into())
1211            .unwrap();
1212        runner
1213            .system_evt_tx
1214            .send((test_system_event()).into())
1215            .unwrap();
1216
1217        assert!(matches!(
1218            runner.recv().await,
1219            Some(PendingRunnerEvent::SystemEvent(_))
1220        ));
1221        assert!(matches!(
1222            runner.recv().await,
1223            Some(PendingRunnerEvent::SystemCommand(_))
1224        ));
1225    }
1226
1227    #[cfg(feature = "node")]
1228    #[rstest]
1229    #[case::timers_and_data(true, true, false)]
1230    #[case::timers_only(true, false, false)]
1231    #[case::data_only(false, true, false)]
1232    #[case::execution_backlog(true, true, true)]
1233    #[tokio::test]
1234    async fn test_runner_dispatch_progress_under_load(
1235        #[case] timers: bool,
1236        #[case] data: bool,
1237        #[case] execution: bool,
1238        #[values(false, true)] receive: bool,
1239    ) {
1240        let mut runner = AsyncRunner::new();
1241        let stop = runner.signal_tx.clone();
1242        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1243            if event.ts_init == UnixNanos::from(8) {
1244                stop.send(()).unwrap();
1245            }
1246        }));
1247        let received = Rc::new(std::cell::Cell::new(0_u64));
1248        let observed = received.clone();
1249        let stop = runner.signal_tx.clone();
1250        let exec_event = || {
1251            ExecutionEvent::Order(OrderEventAny::Submitted(
1252                OrderSubmittedSpec::builder().build(),
1253            ))
1254        };
1255        let exec_tx = runner.exec_evt_tx.clone();
1256        msgbus::register_data_endpoint(
1257            MessagingSwitchboard::data_engine_process_data(),
1258            TypedIntoHandler::from(move |event| {
1259                let Data::Quote(quote) = event else {
1260                    panic!("Expected quote")
1261                };
1262                observed.set(observed.get() + 1);
1263                assert_eq!(quote.ts_init.as_u64(), observed.get());
1264                // Execution between data events must not restart the data rotation
1265                exec_tx.send(exec_event().into()).unwrap();
1266
1267                if !timers && quote.ts_init == UnixNanos::from(8) {
1268                    stop.send(()).unwrap();
1269                }
1270            }),
1271        );
1272        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1273            client_id: Some(ClientId::from("TEST")),
1274            venue: None,
1275            data_type: DataType::new("QuoteTick", None, None),
1276            command_id: UUID4::new(),
1277            ts_init: UnixNanos::default(),
1278            correlation_id: None,
1279            params: None,
1280        }));
1281
1282        // Stop halfway through a ready backlog, before starvation can disappear by draining it
1283        for ts in (1..=16_u64).map(UnixNanos::from) {
1284            if timers {
1285                let event = TimeEvent::new("fairness".into(), UUID4::new(), ts, ts);
1286                runner
1287                    .time_evt_tx
1288                    .send(TimeEventMessage::new(event, callback.clone()).into())
1289                    .unwrap();
1290            }
1291
1292            if data {
1293                let mut quote = test_quote();
1294                quote.ts_init = ts;
1295                runner
1296                    .data_evt_tx
1297                    .send(DataEvent::Data(Data::Quote(quote)).into())
1298                    .unwrap();
1299            }
1300
1301            if execution {
1302                runner.exec_evt_tx.send(exec_event().into()).unwrap();
1303            }
1304            runner.data_cmd_tx.send(command.clone().into()).unwrap();
1305        }
1306
1307        if receive {
1308            while runner.signal_rx.try_recv().is_err() {
1309                match runner.recv().await.unwrap() {
1310                    PendingRunnerEvent::TimeEvent(event) => {
1311                        let _ = AsyncRunner::handle_time_event(event);
1312                    }
1313                    PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
1314                    PendingRunnerEvent::DataCommand(cmd) => AsyncRunner::handle_data_command(cmd),
1315                    PendingRunnerEvent::ExecEvent(event) => AsyncRunner::dispatch_exec_event(event),
1316                    _ => panic!("Unexpected runner event"),
1317                }
1318            }
1319        } else {
1320            runner.run().await.unwrap();
1321        }
1322
1323        if execution {
1324            assert!((1..16).contains(&runner.channels.exec_evt_rx.len()));
1325            assert_eq!(received.get(), 0);
1326        } else {
1327            assert!((1..16).contains(&runner.channels.data_cmd_rx.len()));
1328
1329            if data {
1330                assert!(received.get() > 0);
1331                assert!(!runner.channels.data_evt_rx.is_empty());
1332            }
1333        }
1334
1335        if timers {
1336            assert_eq!(runner.channels.time_evt_rx.len(), 8);
1337        }
1338    }
1339
1340    #[cfg(feature = "node")]
1341    #[rstest]
1342    fn test_runner_senders_shutdown_logging() {
1343        struct ErrorCapture(std::sync::Mutex<Vec<String>>);
1344
1345        impl log::Log for ErrorCapture {
1346            fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1347                metadata.level() == log::Level::Error
1348                    && metadata.target() == "nautilus_live::runner"
1349            }
1350
1351            fn log(&self, record: &log::Record<'_>) {
1352                if self.enabled(record.metadata()) {
1353                    self.0.lock().unwrap().push(record.args().to_string());
1354                }
1355            }
1356
1357            fn flush(&self) {}
1358        }
1359
1360        static ERRORS: ErrorCapture = ErrorCapture(std::sync::Mutex::new(Vec::new()));
1361        log::set_logger(&ERRORS).unwrap();
1362        log::set_max_level(log::LevelFilter::Error);
1363
1364        let runner = AsyncRunner::new();
1365        let handle = LiveNodeHandle::new();
1366        handle.set_starting();
1367        runner.bind_senders_for_node(handle.clone());
1368        let sender = get_data_cmd_sender();
1369        let time_sender = get_time_event_sender();
1370        let mut channels = runner.take_channels();
1371
1372        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1373            client_id: Some(ClientId::from("TEST")),
1374            venue: None,
1375            data_type: DataType::new("QuoteTick", None, None),
1376            command_id: UUID4::new(),
1377            ts_init: UnixNanos::default(),
1378            correlation_id: None,
1379            params: None,
1380        }));
1381
1382        let event = TimeEvent::new("shutdown-timer".into(), UUID4::new(), 17.into(), 23.into());
1383        let expected_events = vec![event.clone(); 2];
1384        let received = Rc::new(RefCell::new(Vec::new()));
1385        let observed = received.clone();
1386        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1387            observed.borrow_mut().push(event);
1388        }));
1389
1390        // Final draining still delivers commands and timer callbacks while receivers are alive
1391        for stopped in [false, true] {
1392            if stopped {
1393                handle.set_stopped();
1394            } else {
1395                handle.set_shutting_down();
1396            }
1397
1398            sender.execute(command.clone());
1399            assert_eq!(
1400                channels
1401                    .data_cmd_rx
1402                    .try_recv()
1403                    .unwrap()
1404                    .dispatch(|command| command),
1405                command
1406            );
1407
1408            time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1409            assert!(AsyncRunner::handle_time_event(
1410                channels.time_evt_rx.try_recv().unwrap()
1411            ));
1412        }
1413
1414        assert_eq!(*received.borrow(), expected_events);
1415
1416        drop(channels);
1417        sender.execute(command.clone());
1418        time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1419        assert_eq!(*ERRORS.0.lock().unwrap(), Vec::<String>::new());
1420
1421        // A stopped previous node must not hide an unexpected closure in its replacement
1422        let runner = AsyncRunner::new();
1423        let handle = LiveNodeHandle::new();
1424        runner.bind_senders_for_node(handle.clone());
1425        let sender = get_data_cmd_sender();
1426        let time_sender = get_time_event_sender();
1427        drop(runner);
1428
1429        for shutting_down in [false, true] {
1430            if shutting_down {
1431                handle.set_shutting_down();
1432            } else {
1433                handle.set_starting();
1434            }
1435
1436            sender.execute(command.clone());
1437            time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1438        }
1439
1440        let expected_errors = vec![
1441            "Failed to send data command: channel closed",
1442            "Failed to send time event message: channel closed",
1443        ];
1444        assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors.repeat(2));
1445
1446        ERRORS.0.lock().unwrap().clear();
1447        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1448        drop(rx);
1449        AsyncDataCommandSender::new(tx).execute(command);
1450        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1451        drop(rx);
1452        AsyncTimeEventSender::new(tx).send(TimeEventMessage::new(event, callback));
1453        assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors);
1454        assert_eq!(*received.borrow(), expected_events);
1455    }
1456
1457    #[tokio::test]
1458    async fn test_async_data_command_sender_execute() {
1459        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1460        let sender = AsyncDataCommandSender::new(tx);
1461
1462        let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1463            client_id: Some(ClientId::from("TEST")),
1464            venue: None,
1465            data_type: DataType::new("QuoteTick", None, None),
1466            command_id: UUID4::new(),
1467            ts_init: UnixNanos::default(),
1468            correlation_id: None,
1469            params: None,
1470        }));
1471
1472        sender.execute(command.clone());
1473
1474        let received = rx.recv().await.unwrap().dispatch(|command| command);
1475        match (received, command) {
1476            (
1477                DataCommand::Subscribe(SubscribeCommand::Data(r)),
1478                DataCommand::Subscribe(SubscribeCommand::Data(c)),
1479            ) => {
1480                assert_eq!(r.client_id, c.client_id);
1481                assert_eq!(r.data_type, c.data_type);
1482            }
1483            _ => panic!("Command mismatch"),
1484        }
1485    }
1486
1487    #[rstest]
1488    #[case(false)]
1489    #[case(true)]
1490    fn test_async_command_senders_capture_owner_root(#[case] rooted: bool) {
1491        let runner = AsyncRunner::new();
1492        runner.bind_senders();
1493        let mut channels = runner.take_channels();
1494
1495        let data = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1496            client_id: Some(ClientId::from("ROOT")),
1497            venue: None,
1498            data_type: DataType::new("QuoteTick", None, None),
1499            command_id: UUID4::new(),
1500            ts_init: UnixNanos::from(7),
1501            correlation_id: None,
1502            params: None,
1503        }));
1504
1505        let trading = TradingCommand::CancelAllOrders(CancelAllOrders::new(
1506            TraderId::from("TRADER-001"),
1507            None,
1508            StrategyId::from("ROOT-001"),
1509            InstrumentId::from("EUR/USD.SIM"),
1510            Some(OrderSide::Sell),
1511            UUID4::new(),
1512            UnixNanos::from(11),
1513            None,
1514            None,
1515        ));
1516        let expected_data = data.clone();
1517        let expected_trading = trading.clone();
1518        let trigger = trading.clone();
1519
1520        let send = move || {
1521            get_data_cmd_sender().execute(data.clone());
1522            get_trading_cmd_sender().execute(TradingCommandMessage::new(
1523                MessagingSwitchboard::exec_engine_execute(),
1524                trading.clone(),
1525            ));
1526        };
1527
1528        if rooted {
1529            msgbus::register_trading_command_endpoint(
1530                MessagingSwitchboard::risk_engine_execute(),
1531                TypedIntoHandler::from(move |_| send()),
1532            );
1533            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1534                MessagingSwitchboard::risk_engine_execute(),
1535                trigger,
1536            ));
1537            drain_trading_cmd_queue();
1538        } else {
1539            send();
1540        }
1541
1542        let data = channels.data_cmd_rx.try_recv().unwrap();
1543        let trading = channels.exec_cmd_rx.try_recv().unwrap();
1544
1545        let results = thread::spawn(move || {
1546            let data = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1547                data.dispatch(|command| command)
1548            }));
1549
1550            let trading = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1551                trading.dispatch(|message| (message.endpoint(), message.command().clone()))
1552            }));
1553
1554            (data, trading)
1555        })
1556        .join()
1557        .unwrap();
1558
1559        if rooted {
1560            for error in [results.0.unwrap_err(), results.1.unwrap_err()] {
1561                let message = error.downcast_ref::<String>().unwrap();
1562                assert!(message.contains("command context dispatched outside its owner thread"));
1563            }
1564        } else {
1565            assert_eq!(results.0.unwrap(), expected_data);
1566            assert_eq!(
1567                results.1.unwrap(),
1568                (
1569                    MessagingSwitchboard::exec_engine_execute(),
1570                    expected_trading
1571                ),
1572            );
1573        }
1574
1575        assert!(channels.data_cmd_rx.is_empty());
1576        assert!(channels.exec_cmd_rx.is_empty());
1577    }
1578
1579    #[rstest]
1580    #[case(false)]
1581    #[case(true)]
1582    fn test_event_senders_capture_owner_root(
1583        #[case] rooted: bool,
1584        #[values(false, true)] dispatch: bool,
1585    ) {
1586        let runner = AsyncRunner::new();
1587        runner.bind_senders();
1588        let mut channels = runner.take_channels();
1589        let expected_quote = test_quote();
1590        let expected_order = OrderSubmittedSpec::builder()
1591            .client_order_id(ClientOrderId::from("EVENT-017"))
1592            .build();
1593        let order = expected_order.clone();
1594
1595        let send = move || {
1596            get_data_event_sender()
1597                .send(DataEvent::Data(Data::Quote(expected_quote)))
1598                .unwrap();
1599            get_exec_event_sender()
1600                .send(ExecutionEvent::Order(OrderEventAny::Submitted(
1601                    order.clone(),
1602                )))
1603                .unwrap();
1604        };
1605
1606        if rooted {
1607            msgbus::register_trading_command_endpoint(
1608                MessagingSwitchboard::risk_engine_execute(),
1609                TypedIntoHandler::from(move |_| send()),
1610            );
1611            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1612                MessagingSwitchboard::risk_engine_execute(),
1613                TradingCommand::QueryAccount(QueryAccount::new(
1614                    "TRADER-001".into(),
1615                    None,
1616                    "SIM-001".into(),
1617                    UUID4::new(),
1618                    19.into(),
1619                    None,
1620                    None,
1621                )),
1622            ));
1623            drain_trading_cmd_queue();
1624        } else {
1625            send();
1626        }
1627
1628        if dispatch {
1629            msgbus::register_data_endpoint(
1630                MessagingSwitchboard::data_engine_process_data(),
1631                TypedIntoHandler::from(move |data| {
1632                    assert_eq!(data, Data::Quote(expected_quote));
1633                    get_data_event_sender().send(DataEvent::Data(data)).unwrap();
1634                }),
1635            );
1636
1637            let order = expected_order.clone();
1638            msgbus::register_order_event_endpoint(
1639                MessagingSwitchboard::exec_engine_process(),
1640                TypedIntoHandler::from(move |event| {
1641                    assert_eq!(event, OrderEventAny::Submitted(order.clone()));
1642                    get_exec_event_sender()
1643                        .send(ExecutionEvent::Order(event))
1644                        .unwrap();
1645                }),
1646            );
1647
1648            AsyncRunner::dispatch_data_event(channels.data_evt_rx.try_recv().unwrap());
1649            AsyncRunner::dispatch_exec_event(channels.exec_evt_rx.try_recv().unwrap());
1650        }
1651
1652        let data = channels.data_evt_rx.try_recv().unwrap();
1653        let exec = channels.exec_evt_rx.try_recv().unwrap();
1654
1655        let (data, exec) = thread::spawn(move || {
1656            (
1657                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1658                    data.dispatch(|event| event)
1659                })),
1660                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1661                    exec.dispatch(|event| event)
1662                })),
1663            )
1664        })
1665        .join()
1666        .unwrap();
1667
1668        if rooted || dispatch {
1669            for error in [data.unwrap_err(), exec.unwrap_err()] {
1670                assert!(
1671                    error
1672                        .downcast_ref::<String>()
1673                        .unwrap()
1674                        .contains("command context dispatched outside its owner thread")
1675                );
1676            }
1677        } else {
1678            let DataEvent::Data(Data::Quote(quote)) = data.unwrap() else {
1679                panic!("expected quote")
1680            };
1681
1682            let ExecutionEvent::Order(OrderEventAny::Submitted(order)) = exec.unwrap() else {
1683                panic!("expected submitted order")
1684            };
1685
1686            assert_eq!(quote, expected_quote);
1687            assert_eq!(order, expected_order);
1688        }
1689
1690        assert!(channels.data_evt_rx.is_empty());
1691        assert!(channels.exec_evt_rx.is_empty());
1692    }
1693
1694    #[rstest]
1695    fn test_system_and_time_senders_capture_owner_root(#[values(false, true)] rooted: bool) {
1696        let runner = AsyncRunner::new();
1697        runner.bind_senders();
1698        let mut channels = runner.take_channels();
1699        let event = TimeEvent::new("system-time".into(), UUID4::new(), 17.into(), 23.into());
1700        let observed = Rc::new(RefCell::new(Vec::new()));
1701        let received = observed.clone();
1702        let expected_event = event.clone();
1703
1704        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1705            received.borrow_mut().push(event);
1706            get_system_command_sender()
1707                .send(test_system_command())
1708                .unwrap();
1709        }));
1710
1711        let send = move || {
1712            get_system_event_sender().send(test_system_event()).unwrap();
1713            get_system_command_sender()
1714                .send(test_system_command())
1715                .unwrap();
1716            get_time_event_sender().send(TimeEventMessage::new(event.clone(), callback.clone()));
1717        };
1718
1719        if rooted {
1720            msgbus::register_trading_command_endpoint(
1721                MessagingSwitchboard::risk_engine_execute(),
1722                TypedIntoHandler::from(move |_| send()),
1723            );
1724            SyncTradingCommandSender.execute(TradingCommandMessage::new(
1725                MessagingSwitchboard::risk_engine_execute(),
1726                TradingCommand::QueryAccount(QueryAccount::new(
1727                    "TRADER-001".into(),
1728                    None,
1729                    "SIM-001".into(),
1730                    UUID4::new(),
1731                    31.into(),
1732                    None,
1733                    None,
1734                )),
1735            ));
1736            drain_trading_cmd_queue();
1737        } else {
1738            send();
1739        }
1740
1741        let system_event = channels.system_evt_rx.try_recv().unwrap();
1742        let system_command = channels.system_cmd_rx.try_recv().unwrap();
1743        let time_event = channels.time_evt_rx.try_recv().unwrap();
1744        assert_eq!(system_event.is_rooted(), rooted);
1745        assert_eq!(system_command.is_rooted(), rooted);
1746        assert_eq!(time_event.is_rooted(), rooted);
1747        assert!(AsyncRunner::handle_time_event(time_event));
1748        let child = channels.system_cmd_rx.try_recv().unwrap();
1749
1750        assert_eq!(system_event.dispatch(|event| event), test_system_event());
1751        assert_eq!(
1752            system_command.dispatch(|command| command),
1753            test_system_command()
1754        );
1755        assert_eq!(*observed.borrow(), vec![expected_event]);
1756        assert!(child.is_rooted());
1757        assert_eq!(child.dispatch(|command| command), test_system_command());
1758        assert!(channels.time_evt_rx.is_empty());
1759        assert!(channels.system_evt_rx.is_empty());
1760        assert!(channels.system_cmd_rx.is_empty());
1761    }
1762
1763    #[cfg(not(all(feature = "simulation", madsim)))]
1764    #[tokio::test]
1765    async fn test_live_timer_started_under_root_emits_independent_event() {
1766        let runner = AsyncRunner::new();
1767        runner.bind_senders();
1768        let mut channels = runner.take_channels();
1769        let now = get_atomic_clock_realtime().get_time_ns();
1770        let name = Ustr::from("ROOTED_TIMER");
1771        let owner = thread::current().id();
1772        let observed = Rc::new(RefCell::new(Vec::new()));
1773        let received = observed.clone();
1774
1775        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1776            received
1777                .borrow_mut()
1778                .push((event.name, event.ts_event, thread::current().id()));
1779        }));
1780
1781        let timer = Rc::new(RefCell::new(None));
1782        let started = timer.clone();
1783        msgbus::register_trading_command_endpoint(
1784            MessagingSwitchboard::risk_engine_execute(),
1785            TypedIntoHandler::from(move |_| {
1786                get_system_command_sender()
1787                    .send(test_system_command())
1788                    .unwrap();
1789
1790                let mut timer = LiveTimer::new(
1791                    name,
1792                    NonZeroU64::new(1_000_000).unwrap(),
1793                    now,
1794                    Some(now),
1795                    callback.clone(),
1796                    true,
1797                    Some(get_time_event_sender()),
1798                );
1799                timer.start();
1800                *started.borrow_mut() = Some(timer);
1801            }),
1802        );
1803
1804        SyncTradingCommandSender.execute(TradingCommandMessage::new(
1805            MessagingSwitchboard::risk_engine_execute(),
1806            TradingCommand::QueryAccount(QueryAccount::new(
1807                "TRADER-001".into(),
1808                None,
1809                "SIM-001".into(),
1810                UUID4::new(),
1811                31.into(),
1812                None,
1813                None,
1814            )),
1815        ));
1816        drain_trading_cmd_queue();
1817        let witness = channels.system_cmd_rx.try_recv().unwrap();
1818        let message = tokio::time::timeout(Duration::from_secs(2), channels.time_evt_rx.recv())
1819            .await
1820            .unwrap()
1821            .unwrap();
1822        timer.borrow_mut().take().unwrap().cancel();
1823
1824        assert!(witness.is_rooted());
1825        assert!(!message.is_rooted());
1826        assert!(observed.borrow().is_empty());
1827        assert!(AsyncRunner::handle_time_event(message));
1828        assert_eq!(*observed.borrow(), vec![(name, now, owner)]);
1829        assert!(channels.time_evt_rx.is_empty());
1830    }
1831
1832    #[tokio::test]
1833    async fn test_async_time_event_sender_send() {
1834        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1835        let sender = AsyncTimeEventSender::new(tx);
1836
1837        let event = TimeEvent::new(
1838            Ustr::from("test"),
1839            UUID4::new(),
1840            UnixNanos::from(1),
1841            UnixNanos::from(2),
1842        );
1843        let callback = TimeEventCallback::from(|_: TimeEvent| {});
1844        let message = TimeEventMessage::new(event, callback);
1845
1846        sender.send(message);
1847
1848        assert!(rx.recv().await.is_some());
1849    }
1850
1851    #[rstest]
1852    fn test_async_trading_command_sender_preserves_target_endpoints() {
1853        std::thread::spawn(|| {
1854            msgbus::get_message_bus().borrow_mut().dispose();
1855            let (risk_handler, risk_saving_handler) =
1856                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1857                    "RiskEngine.execute",
1858                )));
1859            msgbus::register_trading_command_endpoint(
1860                MessagingSwitchboard::risk_engine_execute(),
1861                risk_handler,
1862            );
1863            let (exec_handler, exec_saving_handler) =
1864                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1865                    "ExecEngine.execute",
1866                )));
1867            msgbus::register_trading_command_endpoint(
1868                MessagingSwitchboard::exec_engine_execute(),
1869                exec_handler,
1870            );
1871
1872            let (tx, mut rx) =
1873                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1874            let sender = AsyncTradingCommandSender::new(tx);
1875            sender.execute(TradingCommandMessage::new(
1876                MessagingSwitchboard::risk_engine_execute(),
1877                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1878                    TraderId::from("TRADER-001"),
1879                    None,
1880                    StrategyId::from("RISK-001"),
1881                    InstrumentId::from("EUR/USD.SIM"),
1882                    Some(OrderSide::Buy),
1883                    UUID4::new(),
1884                    UnixNanos::default(),
1885                    None,
1886                    None,
1887                )),
1888            ));
1889            sender.execute(TradingCommandMessage::new(
1890                MessagingSwitchboard::exec_engine_execute(),
1891                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1892                    TraderId::from("TRADER-001"),
1893                    None,
1894                    StrategyId::from("EXEC-001"),
1895                    InstrumentId::from("EUR/USD.SIM"),
1896                    Some(OrderSide::Sell),
1897                    UUID4::new(),
1898                    UnixNanos::default(),
1899                    None,
1900                    None,
1901                )),
1902            ));
1903
1904            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1905            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1906
1907            let risk_commands = risk_saving_handler.get_messages();
1908            let exec_commands = exec_saving_handler.get_messages();
1909            assert!(rx.try_recv().is_err());
1910            assert_eq!(risk_commands.len(), 1);
1911            assert_eq!(
1912                risk_commands[0].strategy_id(),
1913                Some(StrategyId::from("RISK-001"))
1914            );
1915            assert_eq!(exec_commands.len(), 1);
1916            assert_eq!(
1917                exec_commands[0].strategy_id(),
1918                Some(StrategyId::from("EXEC-001"))
1919            );
1920        })
1921        .join()
1922        .unwrap();
1923    }
1924
1925    #[rstest]
1926    fn test_async_runner_preserves_deferred_follow_up_order() {
1927        std::thread::spawn(|| {
1928            msgbus::get_message_bus().borrow_mut().dispose();
1929            let clock = Rc::new(RefCell::new(VirtualClock::new()));
1930            let cache = Rc::new(RefCell::new(Cache::default()));
1931            let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
1932            ExecutionEngine::register_msgbus_handlers(&exec_engine);
1933            msgbus::register_trading_command_endpoint(
1934                MessagingSwitchboard::risk_engine_execute(),
1935                TypedIntoHandler::from(|command: TradingCommand| {
1936                    msgbus::send_trading_command(
1937                        MessagingSwitchboard::exec_engine_queue_execute(),
1938                        command,
1939                    );
1940                }),
1941            );
1942
1943            let (exec_handler, exec_saving_handler) =
1944                get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1945                    "ExecEngine.execute",
1946                )));
1947            msgbus::register_trading_command_endpoint(
1948                MessagingSwitchboard::exec_engine_execute(),
1949                exec_handler,
1950            );
1951
1952            let (tx, mut rx) =
1953                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1954            let sender = Arc::new(AsyncTradingCommandSender::new(tx));
1955            replace_exec_cmd_sender(sender.clone());
1956            sender.execute(TradingCommandMessage::new(
1957                MessagingSwitchboard::risk_engine_execute(),
1958                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1959                    TraderId::from("TRADER-001"),
1960                    None,
1961                    StrategyId::from("FIRST-001"),
1962                    InstrumentId::from("EUR/USD.SIM"),
1963                    Some(OrderSide::Buy),
1964                    UUID4::new(),
1965                    UnixNanos::default(),
1966                    None,
1967                    None,
1968                )),
1969            ));
1970            sender.execute(TradingCommandMessage::new(
1971                MessagingSwitchboard::exec_engine_execute(),
1972                TradingCommand::CancelAllOrders(CancelAllOrders::new(
1973                    TraderId::from("TRADER-001"),
1974                    None,
1975                    StrategyId::from("SECOND-001"),
1976                    InstrumentId::from("EUR/USD.SIM"),
1977                    Some(OrderSide::Sell),
1978                    UUID4::new(),
1979                    UnixNanos::default(),
1980                    None,
1981                    None,
1982                )),
1983            ));
1984
1985            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1986            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1987
1988            let commands = exec_saving_handler.get_messages();
1989            let strategy_ids = commands
1990                .iter()
1991                .map(TradingCommand::strategy_id)
1992                .collect::<Vec<_>>();
1993            assert!(rx.try_recv().is_err());
1994            assert_eq!(commands.len(), 2);
1995            assert_eq!(
1996                strategy_ids,
1997                vec![
1998                    Some(StrategyId::from("FIRST-001")),
1999                    Some(StrategyId::from("SECOND-001"))
2000                ]
2001            );
2002        })
2003        .join()
2004        .unwrap();
2005    }
2006
2007    #[rstest]
2008    fn test_async_runner_dispatches_deferred_exec_command_once() {
2009        std::thread::spawn(|| {
2010            msgbus::get_message_bus().borrow_mut().dispose();
2011            let clock = Rc::new(RefCell::new(VirtualClock::new()));
2012            let cache = Rc::new(RefCell::new(Cache::default()));
2013            let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
2014            ExecutionEngine::register_msgbus_handlers(&exec_engine);
2015
2016            let (tx, mut rx) =
2017                tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2018            replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(tx)));
2019            let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2020                TraderId::from("TRADER-001"),
2021                None,
2022                StrategyId::from("EXEC-001"),
2023                InstrumentId::from("EUR/USD.SIM"),
2024                Some(OrderSide::Buy),
2025                UUID4::new(),
2026                UnixNanos::default(),
2027                None,
2028                None,
2029            ));
2030
2031            msgbus::send_trading_command(
2032                MessagingSwitchboard::exec_engine_queue_execute(),
2033                command,
2034            );
2035            assert_eq!(exec_engine.borrow().command_count(), 0);
2036
2037            AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
2038
2039            assert!(rx.try_recv().is_err());
2040            assert_eq!(exec_engine.borrow().command_count(), 1);
2041        })
2042        .join()
2043        .unwrap();
2044    }
2045
2046    #[cfg(feature = "node")]
2047    #[rstest]
2048    #[case::cash(AccountType::Cash)]
2049    #[case::margin(AccountType::Margin)]
2050    fn test_account_event_publishes_after_portfolio_application(#[case] account_type: AccountType) {
2051        use nautilus_common::msgbus::{MessageBus, TypedHandler};
2052        use nautilus_model::{
2053            accounts::Account,
2054            types::{AccountBalance, Currency},
2055        };
2056        use nautilus_portfolio::Portfolio;
2057
2058        // Native logging tests use nextest's per-test process isolation.
2059        struct AccountLogCapture(std::sync::Mutex<Vec<String>>);
2060
2061        impl log::Log for AccountLogCapture {
2062            fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
2063                metadata.level() == log::Level::Info
2064                    && metadata.target() == "nautilus_portfolio::portfolio"
2065            }
2066
2067            fn log(&self, record: &log::Record<'_>) {
2068                if self.enabled(record.metadata()) {
2069                    self.0.lock().unwrap().push(record.args().to_string());
2070                }
2071            }
2072
2073            fn flush(&self) {}
2074        }
2075
2076        static LOGS: AccountLogCapture = AccountLogCapture(std::sync::Mutex::new(Vec::new()));
2077        log::set_logger(&LOGS).unwrap();
2078        log::set_max_level(log::LevelFilter::Info);
2079
2080        msgbus::set_message_bus(Rc::new(RefCell::new(MessageBus::new(
2081            TraderId::from("TESTER-001"),
2082            UUID4::new(),
2083            None,
2084            None,
2085        ))));
2086        let cache = Rc::new(RefCell::new(Cache::default()));
2087        let clock = Rc::new(RefCell::new(VirtualClock::new()));
2088        let _portfolio = Portfolio::new(clock, cache.clone(), None);
2089        let observed = Rc::new(RefCell::new(Vec::new()));
2090        let received = observed.clone();
2091        let source = cache.clone();
2092        let handler = TypedHandler::from(move |event: &AccountState| {
2093            let cache = source.borrow();
2094            let account = cache.account(&event.account_id).unwrap();
2095            assert_eq!(account.last_event().unwrap().event_id, event.event_id);
2096            assert_eq!(account.account_type(), event.account_type);
2097            assert_eq!(
2098                account.balances().into_values().collect::<Vec<_>>(),
2099                event.balances
2100            );
2101            received
2102                .borrow_mut()
2103                .push((event.event_id, account.events().len()));
2104        });
2105
2106        // Subscribers must see applied state regardless of their topic priority
2107        msgbus::subscribe_account_state("events.account.*".into(), handler.clone(), Some(20));
2108        let scoped = Rc::new(RefCell::new(Vec::new()));
2109        let scoped_received = scoped.clone();
2110        let scoped_handler = TypedHandler::from(move |event: &AccountState| {
2111            scoped_received.borrow_mut().push(event.event_id);
2112        });
2113        msgbus::subscribe_account_state(
2114            "events.account.SIM-001".into(),
2115            scoped_handler.clone(),
2116            None,
2117        );
2118        let mut expected = Vec::new();
2119        let mut expected_scoped = Vec::new();
2120        for (index, (account_id, total, locked, free, history_len)) in [
2121            ("SIM-001", "100 USD", "20 USD", "80 USD", 1),
2122            ("SIM-002", "200 USD", "50 USD", "150 USD", 1),
2123            ("SIM-001", "90 USD", "10 USD", "80 USD", 2),
2124            ("SIM-001", "0 USD", "0 USD", "0 USD", 3),
2125        ]
2126        .into_iter()
2127        .enumerate()
2128        {
2129            let event = AccountState::new(
2130                AccountId::from(account_id),
2131                account_type,
2132                vec![AccountBalance::new(
2133                    Money::from(total),
2134                    Money::from(locked),
2135                    Money::from(free),
2136                )],
2137                vec![],
2138                true,
2139                UUID4::new(),
2140                UnixNanos::from(index as u64 + 1),
2141                UnixNanos::from(index as u64 + 1),
2142                Some(Currency::USD()),
2143            );
2144            expected.push((event.event_id, history_len));
2145            if account_id == "SIM-001" {
2146                expected_scoped.push(event.event_id);
2147            }
2148            let expected_log = format!("Updated {event}");
2149            AsyncRunner::handle_exec_event(ExecutionEvent::Account(event));
2150            assert_eq!(
2151                LOGS.0
2152                    .lock()
2153                    .unwrap()
2154                    .iter()
2155                    .filter(|line| **line == expected_log)
2156                    .count(),
2157                1
2158            );
2159        }
2160        assert_eq!(*observed.borrow(), expected);
2161        assert_eq!(*scoped.borrow(), expected_scoped);
2162        let cache = cache.borrow();
2163        assert_eq!(
2164            cache
2165                .account(&AccountId::from("SIM-001"))
2166                .unwrap()
2167                .events()
2168                .len(),
2169            3
2170        );
2171        assert_eq!(
2172            cache
2173                .account(&AccountId::from("SIM-002"))
2174                .unwrap()
2175                .events()
2176                .len(),
2177            1
2178        );
2179        msgbus::unsubscribe_account_state("events.account.*".into(), &handler);
2180        msgbus::unsubscribe_account_state("events.account.SIM-001".into(), &scoped_handler);
2181    }
2182
2183    #[cfg(feature = "node")]
2184    #[rstest]
2185    #[case::cash_update(AccountType::Cash, true)]
2186    #[case::wallet_initial(AccountType::Wallet, false)]
2187    #[case::wallet_update(AccountType::Wallet, true)]
2188    fn test_rejected_account_event_is_not_published(
2189        #[case] account_type: AccountType,
2190        #[case] existing_account: bool,
2191    ) {
2192        use nautilus_common::msgbus::{MessageBus, TypedHandler};
2193        use nautilus_model::types::{AccountBalance, Currency};
2194        use nautilus_portfolio::Portfolio;
2195
2196        msgbus::set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
2197        let cache = Rc::new(RefCell::new(Cache::default()));
2198        let mut portfolio = Portfolio::new(
2199            Rc::new(RefCell::new(VirtualClock::new())),
2200            cache.clone(),
2201            None,
2202        );
2203        let state = |amount: &str| {
2204            AccountState::new(
2205                AccountId::from("SIM-001"),
2206                account_type,
2207                vec![AccountBalance::new(
2208                    Money::from(amount),
2209                    Money::zero(Currency::USD()),
2210                    Money::from(amount),
2211                )],
2212                vec![],
2213                true,
2214                UUID4::new(),
2215                UnixNanos::from(1),
2216                UnixNanos::from(1),
2217                (account_type != AccountType::Wallet).then_some(Currency::USD()),
2218            )
2219        };
2220        let initial = state("100.00 USD");
2221
2222        if existing_account {
2223            portfolio.update_account(&initial);
2224        }
2225        let received = Rc::new(RefCell::new(Vec::new()));
2226        let handler = TypedHandler::from({
2227            let received = received.clone();
2228            move |event: &AccountState| received.borrow_mut().push(event.clone())
2229        });
2230        msgbus::subscribe_account_state("events.account.*".into(), handler.clone(), None);
2231        AsyncRunner::handle_exec_event(ExecutionEvent::Account(state("-1.00 USD")));
2232        assert!(received.borrow().is_empty());
2233        let cache = cache.borrow();
2234
2235        if existing_account {
2236            assert_eq!(
2237                cache.account(&initial.account_id).unwrap().last_event(),
2238                Some(initial)
2239            );
2240        } else {
2241            assert!(cache.account(&initial.account_id).is_none());
2242        }
2243        msgbus::unsubscribe_account_state("events.account.*".into(), &handler);
2244    }
2245
2246    #[rstest]
2247    #[case::risk(MessagingSwitchboard::risk_engine_execute())]
2248    #[case::execution(MessagingSwitchboard::exec_engine_execute())]
2249    #[tokio::test(flavor = "current_thread")]
2250    async fn test_runner_dispatches_commands_and_data_before_handle_stop(
2251        #[case] target: MStr<Endpoint>,
2252    ) {
2253        let count = 10_u64;
2254        let mut runner = AsyncRunner::new();
2255        let handle = runner.handle();
2256        let commands = Rc::new(RefCell::new(Vec::new()));
2257        let data = Rc::new(RefCell::new(Vec::new()));
2258        let (observed_tx, mut observed_rx) = tokio::sync::mpsc::unbounded_channel();
2259
2260        for endpoint in [
2261            MessagingSwitchboard::risk_engine_execute(),
2262            MessagingSwitchboard::exec_engine_execute(),
2263        ] {
2264            let observed_commands = commands.clone();
2265            let commands_tx = observed_tx.clone();
2266            msgbus::register_trading_command_endpoint(
2267                endpoint,
2268                TypedIntoHandler::from(move |command| {
2269                    observed_commands.borrow_mut().push((endpoint, command));
2270                    commands_tx.send(()).unwrap();
2271                }),
2272            );
2273        }
2274
2275        let observed_data = data.clone();
2276        msgbus::register_data_endpoint(
2277            MessagingSwitchboard::data_engine_process_data(),
2278            TypedIntoHandler::from(move |event| {
2279                observed_data.borrow_mut().push(event);
2280                observed_tx.send(()).unwrap();
2281            }),
2282        );
2283
2284        let mut expected_commands = Vec::new();
2285        let mut expected_data = Vec::new();
2286
2287        for index in 1..=count {
2288            let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2289                TraderId::from("TRADER-001"),
2290                Some(ClientId::from("SIM")),
2291                StrategyId::from(format!("S-{index:03}")),
2292                InstrumentId::from("EUR/USD.SIM"),
2293                Some(OrderSide::Sell),
2294                UUID4::new(),
2295                UnixNanos::from(index),
2296                None,
2297                Some(UUID4::new()),
2298            ));
2299            runner
2300                .exec_cmd_tx
2301                .send(TradingCommandMessage::new(target, command.clone()).into())
2302                .unwrap();
2303            expected_commands.push((target, command));
2304            let mut quote = test_quote();
2305            quote.ts_init = UnixNanos::from(index);
2306            let event = Data::Quote(quote);
2307            runner
2308                .data_evt_tx
2309                .send(DataEvent::Data(event.clone()).into())
2310                .unwrap();
2311            expected_data.push(event);
2312        }
2313
2314        let driver = async {
2315            for _ in 0..count * 2 {
2316                observed_rx.recv().await.unwrap();
2317            }
2318
2319            handle.stop();
2320        };
2321
2322        let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
2323            tokio::join!(runner.run(), driver)
2324        })
2325        .await
2326        .expect("runner dispatches every message and stops");
2327
2328        result.unwrap();
2329
2330        assert_eq!(&*commands.borrow(), &expected_commands);
2331        assert_eq!(
2332            serde_json::to_value(&*data.borrow()).unwrap(),
2333            serde_json::to_value(expected_data).unwrap()
2334        );
2335        assert!(runner.channels.exec_cmd_rx.is_empty());
2336        assert!(runner.channels.data_evt_rx.is_empty());
2337        assert!(observed_rx.try_recv().is_err());
2338    }
2339
2340    #[tokio::test(flavor = "current_thread")]
2341    async fn test_runner_dispatches_every_concurrent_producer_in_order() {
2342        let mut runner = AsyncRunner::new();
2343        let handle = runner.handle();
2344        let (observed_tx, mut observed_rx) = tokio::sync::mpsc::unbounded_channel();
2345        msgbus::register_data_endpoint(
2346            MessagingSwitchboard::data_engine_process_data(),
2347            TypedIntoHandler::from(move |event| observed_tx.send(event).unwrap()),
2348        );
2349        let mut producers = Vec::new();
2350
2351        for producer in 0..5_u64 {
2352            let sender = runner.data_evt_tx.clone();
2353
2354            producers.push(tokio::spawn(async move {
2355                for index in 0..20 {
2356                    let mut quote = test_quote();
2357                    quote.ts_init = UnixNanos::from(producer * 20 + index);
2358                    sender
2359                        .send(DataEvent::Data(Data::Quote(quote)).into())
2360                        .unwrap();
2361                    tokio::task::yield_now().await;
2362                }
2363            }));
2364        }
2365
2366        let driver = async {
2367            for producer in producers {
2368                producer.await.unwrap();
2369            }
2370
2371            let mut observed = vec![Vec::new(); 5];
2372
2373            for _ in 0..100 {
2374                let Data::Quote(quote) = observed_rx.recv().await.unwrap() else {
2375                    panic!("expected a quote");
2376                };
2377
2378                let timestamp = quote.ts_init.as_u64();
2379                let mut expected = test_quote();
2380                expected.ts_init = quote.ts_init;
2381                assert_eq!(quote, expected);
2382                observed[usize::try_from(timestamp / 20).unwrap()].push(timestamp % 20);
2383            }
2384
2385            handle.stop();
2386            observed
2387        };
2388
2389        let (result, observed) = tokio::time::timeout(Duration::from_secs(5), async {
2390            tokio::join!(runner.run(), driver)
2391        })
2392        .await
2393        .expect("runner receives every producer's messages and stops");
2394
2395        result.unwrap();
2396
2397        assert_eq!(observed, vec![(0..20_u64).collect::<Vec<_>>(); 5]);
2398        assert!(runner.channels.data_evt_rx.is_empty());
2399        assert!(observed_rx.try_recv().is_err());
2400    }
2401
2402    #[rstest]
2403    fn test_new_does_not_bind_tls() {
2404        std::thread::spawn(|| {
2405            let _runner = AsyncRunner::new();
2406            assert!(try_get_time_event_sender().is_none());
2407            assert!(try_get_system_command_sender().is_none());
2408            assert!(try_get_system_event_sender().is_none());
2409            assert!(try_get_trading_cmd_sender().is_none());
2410        })
2411        .join()
2412        .unwrap();
2413    }
2414
2415    #[rstest]
2416    fn test_bind_senders_routes_to_runner_channels() {
2417        std::thread::spawn(|| {
2418            let mut runner = AsyncRunner::new();
2419            runner.bind_senders();
2420
2421            get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2422                SubscribeCustomData {
2423                    client_id: Some(ClientId::from("TEST")),
2424                    venue: None,
2425                    data_type: DataType::new("test", None, None),
2426                    command_id: UUID4::new(),
2427                    ts_init: UnixNanos::default(),
2428                    correlation_id: None,
2429                    params: None,
2430                },
2431            )));
2432
2433            assert!(runner.channels.data_cmd_rx.try_recv().is_ok());
2434
2435            get_trading_cmd_sender().execute(TradingCommandMessage::new(
2436                MessagingSwitchboard::exec_engine_execute(),
2437                TradingCommand::CancelAllOrders(CancelAllOrders::new(
2438                    TraderId::from("TRADER-001"),
2439                    None,
2440                    StrategyId::from("S-001"),
2441                    InstrumentId::from("EUR/USD.SIM"),
2442                    Some(OrderSide::Buy),
2443                    UUID4::new(),
2444                    UnixNanos::default(),
2445                    None,
2446                    None, // correlation_id
2447                )),
2448            ));
2449            assert!(runner.channels.exec_cmd_rx.try_recv().is_ok());
2450
2451            let event = TimeEvent::new(
2452                Ustr::from("test"),
2453                UUID4::new(),
2454                UnixNanos::from(1),
2455                UnixNanos::from(2),
2456            );
2457            let callback = TimeEventCallback::from(|_: TimeEvent| {});
2458            get_time_event_sender().send(TimeEventMessage::new(event, callback));
2459            assert!(runner.channels.time_evt_rx.try_recv().is_ok());
2460
2461            get_system_event_sender().send(test_system_event()).unwrap();
2462            assert_eq!(
2463                runner
2464                    .channels
2465                    .system_evt_rx
2466                    .try_recv()
2467                    .unwrap()
2468                    .dispatch(|value| value),
2469                test_system_event()
2470            );
2471
2472            get_system_command_sender()
2473                .send(test_system_command())
2474                .unwrap();
2475            assert_eq!(
2476                runner
2477                    .channels
2478                    .system_cmd_rx
2479                    .try_recv()
2480                    .unwrap()
2481                    .dispatch(|value| value),
2482                test_system_command()
2483            );
2484
2485            get_data_event_sender()
2486                .send(DataEvent::Data(Data::Quote(test_quote())))
2487                .unwrap();
2488            assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2489
2490            let account = AccountState::new(
2491                AccountId::from("SIM-001"),
2492                AccountType::Cash,
2493                vec![],
2494                vec![],
2495                true,
2496                UUID4::new(),
2497                UnixNanos::from(1),
2498                UnixNanos::from(2),
2499                None,
2500            );
2501            get_exec_event_sender()
2502                .send(ExecutionEvent::Account(account))
2503                .unwrap();
2504            assert!(runner.channels.exec_evt_rx.try_recv().is_ok());
2505        })
2506        .join()
2507        .unwrap();
2508    }
2509
2510    #[cfg(feature = "node")]
2511    #[rstest]
2512    fn test_drain_pending_system_events_keeps_data_events_separate() {
2513        std::thread::spawn(|| {
2514            let mut runner = AsyncRunner::new();
2515            runner.bind_senders();
2516            let system_event = test_system_event();
2517
2518            get_system_event_sender().send(system_event).unwrap();
2519            get_data_event_sender()
2520                .send(DataEvent::Data(Data::Quote(test_quote())))
2521                .unwrap();
2522
2523            let system_events = runner
2524                .drain_pending_system_events()
2525                .into_iter()
2526                .map(|message| message.dispatch(|value| value))
2527                .collect::<Vec<_>>();
2528
2529            assert_eq!(system_events, vec![system_event]);
2530            assert!(runner.channels.system_evt_rx.try_recv().is_err());
2531            assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2532        })
2533        .join()
2534        .unwrap();
2535    }
2536
2537    #[cfg(feature = "node")]
2538    #[rstest]
2539    fn test_drain_pending_system_commands_keeps_events_separate() {
2540        std::thread::spawn(|| {
2541            let mut runner = AsyncRunner::new();
2542            runner.bind_senders();
2543            let system_command = test_system_command();
2544
2545            get_system_command_sender().send(system_command).unwrap();
2546            get_system_event_sender().send(test_system_event()).unwrap();
2547
2548            let system_commands = runner
2549                .drain_pending_system_commands()
2550                .into_iter()
2551                .map(|message| message.dispatch(|value| value))
2552                .collect::<Vec<_>>();
2553
2554            assert_eq!(system_commands, vec![system_command]);
2555            assert!(runner.channels.system_cmd_rx.try_recv().is_err());
2556            assert!(runner.channels.system_evt_rx.try_recv().is_ok());
2557        })
2558        .join()
2559        .unwrap();
2560    }
2561
2562    #[rstest]
2563    fn test_bind_senders_reclaims_tls_from_previous_runner() {
2564        std::thread::spawn(|| {
2565            let mut runner1 = AsyncRunner::new();
2566            runner1.bind_senders();
2567
2568            let mut runner2 = AsyncRunner::new();
2569            runner2.bind_senders();
2570
2571            get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2572                SubscribeCustomData {
2573                    client_id: Some(ClientId::from("TEST")),
2574                    venue: None,
2575                    data_type: DataType::new("test", None, None),
2576                    command_id: UUID4::new(),
2577                    ts_init: UnixNanos::default(),
2578                    correlation_id: None,
2579                    params: None,
2580                },
2581            )));
2582
2583            assert!(runner2.channels.data_cmd_rx.try_recv().is_ok());
2584            assert!(runner1.channels.data_cmd_rx.try_recv().is_err());
2585        })
2586        .join()
2587        .unwrap();
2588    }
2589
2590    #[cfg(feature = "defi")]
2591    #[rstest]
2592    fn test_handle_data_event_routes_pool_snapshot_response_to_data_engine() {
2593        use nautilus_common::{
2594            messages::defi::PoolSnapshotResponse, msgbus::ShareableMessageHandler,
2595        };
2596        use nautilus_model::defi::{
2597            data::block::BlockPosition,
2598            pool_analysis::snapshot::{PoolAnalytics, PoolSnapshot, PoolState},
2599        };
2600
2601        let received = Rc::new(RefCell::new(Vec::new()));
2602        let sink = received.clone();
2603        msgbus::register_any(
2604            MessagingSwitchboard::data_engine_process(),
2605            ShareableMessageHandler::from_typed(move |response: &PoolSnapshotResponse| {
2606                sink.borrow_mut().push(response.correlation_id);
2607            }),
2608        );
2609
2610        let correlation_id = UUID4::new();
2611
2612        let snapshot = PoolSnapshot::new(
2613            InstrumentId::from("0x11b815efB8f581194ae79006d24E0d814B7697F6.Arbitrum:UniswapV3"),
2614            PoolState::default(),
2615            Vec::new(),
2616            Vec::new(),
2617            PoolAnalytics::default(),
2618            BlockPosition::new(1000, "0x0".to_string(), 0, 0),
2619            UnixNanos::default(),
2620            UnixNanos::default(),
2621        );
2622
2623        AsyncRunner::handle_data_event(DataEvent::PoolSnapshotResponse(PoolSnapshotResponse::new(
2624            correlation_id,
2625            snapshot,
2626        )));
2627
2628        assert_eq!(*received.borrow(), vec![correlation_id]);
2629    }
2630}