1use std::{
63 fmt::Debug,
64 sync::Arc,
65 thread::{self, ThreadId},
66};
67
68use nautilus_common::{
69 live::{
70 dispatch::DispatchMessage,
71 runner::{
72 replace_data_event_sender, replace_exec_event_sender, replace_system_command_sender,
73 replace_system_event_sender,
74 },
75 sender::{DispatchSender, EventSender},
76 },
77 messages::{
78 DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent, data::DataCommand,
79 execution::TradingCommand,
80 },
81 msgbus::{self, MessagingSwitchboard},
82 runner::{
83 DataCommandSender, TimeEventMessage, TimeEventSender, TradingCommandMessage,
84 TradingCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
85 replace_time_event_sender,
86 },
87};
88use nautilus_model::events::OrderEventAny;
89
90use crate::dispatch::drain_callbacks;
91#[cfg(feature = "node")]
92use crate::node::{LiveNodeHandle, NodeState};
93
94#[derive(Debug)]
96pub struct AsyncDataCommandSender {
97 cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
98 owner: ThreadId,
99 #[cfg(feature = "node")]
100 node_handle: Option<LiveNodeHandle>,
101}
102
103impl AsyncDataCommandSender {
104 #[must_use]
108 pub fn new(cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>) -> Self {
109 Self {
110 cmd_tx,
111 owner: thread::current().id(),
112 #[cfg(feature = "node")]
113 node_handle: None,
114 }
115 }
116}
117
118impl DataCommandSender for AsyncDataCommandSender {
119 fn execute(&self, command: DataCommand) {
120 if let Err(e) = self.cmd_tx.send(DispatchMessage::new(command, self.owner)) {
121 #[cfg(feature = "node")]
123 if node_stopped(self.node_handle.as_ref()) {
124 return;
125 }
126
127 log::error!("Failed to send data command: {e}");
128 }
129 }
130}
131
132#[derive(Debug, Clone)]
134pub struct AsyncTimeEventSender {
135 time_tx: DispatchSender<TimeEventMessage>,
136 #[cfg(feature = "node")]
137 node_handle: Option<LiveNodeHandle>,
138}
139
140impl AsyncTimeEventSender {
141 #[must_use]
143 pub fn new(
144 time_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
145 ) -> Self {
146 Self {
147 time_tx: DispatchSender::new(time_tx),
148 #[cfg(feature = "node")]
149 node_handle: None,
150 }
151 }
152}
153
154impl TimeEventSender for AsyncTimeEventSender {
155 fn send(&self, message: TimeEventMessage) {
156 if let Err(e) = self.time_tx.send(message) {
157 #[cfg(feature = "node")]
158 if node_stopped(self.node_handle.as_ref()) {
159 return;
160 }
161
162 log::error!("Failed to send time event message: {e}");
163 }
164 }
165}
166
167#[derive(Debug)]
169pub struct AsyncTradingCommandSender {
170 cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
171 owner: ThreadId,
172}
173
174impl AsyncTradingCommandSender {
175 #[must_use]
179 pub fn new(
180 cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
181 ) -> Self {
182 Self {
183 cmd_tx,
184 owner: thread::current().id(),
185 }
186 }
187}
188
189impl TradingCommandSender for AsyncTradingCommandSender {
190 fn execute(&self, message: TradingCommandMessage) {
191 if let Err(e) = self.cmd_tx.send(DispatchMessage::new(message, self.owner)) {
192 log::error!("Failed to send trading command: {e}");
193 }
194 }
195}
196
197pub trait Runner {
198 fn run(&mut self);
199}
200
201#[derive(Debug)]
206pub struct AsyncRunnerChannels {
207 pub time_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
208 pub system_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
209 pub system_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
210 pub exec_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
211 pub exec_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
212 pub data_evt_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
213 pub data_cmd_rx: tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
214}
215
216#[cfg(feature = "node")]
217#[allow(
218 clippy::large_enum_variant,
219 reason = "runner events are consumed immediately; boxing would add routing allocations"
220)]
221pub(crate) enum PendingRunnerEvent {
222 TimeEvent(DispatchMessage<TimeEventMessage>),
223 SystemEvent(DispatchMessage<SystemEvent>),
224 SystemCommand(DispatchMessage<SystemCommand>),
225 ExecEvent(DispatchMessage<ExecutionEvent>),
226 ExecCommand(DispatchMessage<TradingCommandMessage>),
227 DataEvent(DispatchMessage<DataEvent>),
228 DataCommand(DispatchMessage<DataCommand>),
229}
230
231pub struct AsyncRunner {
232 channels: AsyncRunnerChannels,
233 skip_time: bool,
234 skip_data: bool,
235 time_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TimeEventMessage>>,
236 system_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemEvent>>,
237 system_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<SystemCommand>>,
238 signal_rx: tokio::sync::mpsc::UnboundedReceiver<()>,
239 signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
240 exec_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<ExecutionEvent>>,
241 exec_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<TradingCommandMessage>>,
242 data_evt_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataEvent>>,
243 data_cmd_tx: tokio::sync::mpsc::UnboundedSender<DispatchMessage<DataCommand>>,
244}
245
246#[derive(Clone, Debug)]
248pub struct AsyncRunnerHandle {
249 signal_tx: tokio::sync::mpsc::UnboundedSender<()>,
250}
251
252impl AsyncRunnerHandle {
253 pub fn stop(&self) {
255 if let Err(e) = self.signal_tx.send(()) {
256 log::error!("Failed to send shutdown signal: {e}");
257 }
258 }
259}
260
261impl Default for AsyncRunner {
262 fn default() -> Self {
263 Self::new()
264 }
265}
266
267impl Debug for AsyncRunner {
268 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
269 f.debug_struct(stringify!(AsyncRunner)).finish()
270 }
271}
272
273impl AsyncRunner {
274 #[must_use]
280 #[rustfmt::skip]
281 pub fn new() -> Self {
282 use tokio::sync::mpsc::unbounded_channel; let (time_evt_tx, time_evt_rx) = unbounded_channel::<DispatchMessage<TimeEventMessage>>();
285 let (system_evt_tx, system_evt_rx) = unbounded_channel::<DispatchMessage<SystemEvent>>();
286 let (system_cmd_tx, system_cmd_rx) = unbounded_channel::<DispatchMessage<SystemCommand>>();
287 let (signal_tx, signal_rx) = unbounded_channel::<()>();
288 let (exec_evt_tx, exec_evt_rx) = unbounded_channel::<DispatchMessage<ExecutionEvent>>();
289 let (exec_cmd_tx, exec_cmd_rx) = unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
290 let (data_evt_tx, data_evt_rx) = unbounded_channel::<DispatchMessage<DataEvent>>();
291 let (data_cmd_tx, data_cmd_rx) = unbounded_channel::<DispatchMessage<DataCommand>>();
292
293 Self {
294 channels: AsyncRunnerChannels {
295 time_evt_rx,
296 system_evt_rx,
297 system_cmd_rx,
298 exec_evt_rx,
299 exec_cmd_rx,
300 data_evt_rx,
301 data_cmd_rx,
302 },
303 skip_time: false,
304 skip_data: false,
305 time_evt_tx,
306 system_evt_tx,
307 system_cmd_tx,
308 signal_rx,
309 signal_tx,
310 exec_evt_tx,
311 exec_cmd_tx,
312 data_evt_tx,
313 data_cmd_tx,
314 }
315 }
316
317 pub fn bind_senders(&self) {
323 self.bind_senders_with_data_sender(AsyncDataCommandSender::new(self.data_cmd_tx.clone()));
324 }
325
326 #[cfg(feature = "node")]
327 pub(crate) fn bind_senders_for_node(&self, handle: LiveNodeHandle) {
328 self.bind_senders_with_data_sender(AsyncDataCommandSender {
329 cmd_tx: self.data_cmd_tx.clone(),
330 owner: thread::current().id(),
331 node_handle: Some(handle),
332 });
333 }
334
335 #[rustfmt::skip]
336 fn bind_senders_with_data_sender(&self, sender: AsyncDataCommandSender) {
337 replace_time_event_sender(Arc::new(AsyncTimeEventSender {
338 time_tx: DispatchSender::new(self.time_evt_tx.clone()),
339 #[cfg(feature = "node")]
340 node_handle: sender.node_handle.clone(),
341 }));
342 replace_system_event_sender(EventSender::new(self.system_evt_tx.clone()));
343 replace_system_command_sender(DispatchSender::new(self.system_cmd_tx.clone()));
344 replace_exec_event_sender(EventSender::new(self.exec_evt_tx.clone()));
345 replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(self.exec_cmd_tx.clone())));
346 replace_data_event_sender(EventSender::new(self.data_evt_tx.clone()));
347 replace_data_cmd_sender(Arc::new(sender));
348 }
349
350 pub fn stop(&self) {
352 if let Err(e) = self.signal_tx.send(()) {
353 log::error!("Failed to send shutdown signal: {e}");
354 }
355 }
356
357 #[must_use]
359 pub fn handle(&self) -> AsyncRunnerHandle {
360 AsyncRunnerHandle {
361 signal_tx: self.signal_tx.clone(),
362 }
363 }
364
365 #[must_use]
370 pub fn take_channels(self) -> AsyncRunnerChannels {
371 self.channels
372 }
373
374 pub fn flush_pending_data(&mut self) {
380 let mut total = 0;
381
382 loop {
383 let mut progressed = false;
384
385 while let Ok(evt) = self.channels.data_evt_rx.try_recv() {
392 Self::dispatch_data_event(evt);
393 progressed = true;
394 total += 1;
395 }
396
397 while let Ok(cmd) = self.channels.data_cmd_rx.try_recv() {
398 Self::handle_data_command(cmd);
399 progressed = true;
400 total += 1;
401 }
402
403 if !progressed {
404 break;
405 }
406 }
407
408 if total > 0 {
409 log::debug!("Flushed {total} pending data events/commands");
410 }
411 }
412
413 #[cfg(feature = "node")]
414 pub(crate) fn drain_pending_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
415 let mut events = Vec::new();
416
417 while let Ok(event) = self.channels.system_evt_rx.try_recv() {
418 events.push(event);
419 }
420
421 events
422 }
423
424 #[cfg(feature = "node")]
425 pub(crate) fn drain_pending_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
426 let mut commands = Vec::new();
427
428 while let Ok(command) = self.channels.system_cmd_rx.try_recv() {
429 commands.push(command);
430 }
431
432 commands
433 }
434
435 pub async fn run(&mut self) -> anyhow::Result<()> {
445 self.bind_senders();
446
447 log::info!("AsyncRunner starting");
448
449 loop {
450 let callbacks_pending = drain_callbacks().await?;
451 let commands_empty = self.channels.data_cmd_rx.is_empty();
452 let poll_time =
453 !self.skip_time || (self.channels.data_evt_rx.is_empty() && commands_empty);
454 let poll_data = !self.skip_data || commands_empty;
455
456 tokio::select! {
457 biased;
458
459 Some(()) = self.signal_rx.recv() => {
460 log::info!("AsyncRunner received signal, shutting down");
461 return Ok(());
462 },
463 () = std::future::ready(()), if callbacks_pending => {},
464 Some(handler) = self.channels.time_evt_rx.recv(), if poll_time => {
465 self.skip_time = true;
466 let _ = Self::handle_time_event(handler);
467 },
468 Some(event) = self.channels.system_evt_rx.recv() => {
469 self.skip_time = false;
470 log::error!("System event {event} requires the LiveNode runner");
471 },
472 Some(command) = self.channels.system_cmd_rx.recv() => {
473 self.skip_time = false;
474 log::error!("System command {command} requires the LiveNode runner");
475 },
476 Some(evt) = self.channels.exec_evt_rx.recv() => {
477 self.skip_time = false;
478 Self::dispatch_exec_event(evt);
479 },
480 Some(cmd) = self.channels.exec_cmd_rx.recv() => {
481 self.skip_time = false;
482 Self::handle_trading_command(cmd);
483 },
484 Some(evt) = self.channels.data_evt_rx.recv(), if poll_data => {
485 self.skip_time = false;
486 self.skip_data = true;
487 Self::dispatch_data_event(evt);
488 },
489 Some(cmd) = self.channels.data_cmd_rx.recv() => {
490 self.skip_time = false;
491 self.skip_data = false;
492 Self::handle_data_command(cmd);
493 },
494 else => {
495 log::debug!("AsyncRunner all channels closed, exiting");
496 return Ok(());
497 }
498 };
499 }
500 }
501
502 #[inline]
508 #[must_use]
509 pub fn handle_time_event(message: DispatchMessage<TimeEventMessage>) -> bool {
510 message.dispatch(TimeEventMessage::dispatch)
511 }
512
513 #[inline]
519 pub fn handle_data_command(cmd: DispatchMessage<DataCommand>) {
520 cmd.dispatch(|cmd| {
521 msgbus::send_data_command(MessagingSwitchboard::data_engine_execute(), cmd);
522 });
523 }
524
525 pub fn dispatch_data_event(event: DispatchMessage<DataEvent>) {
531 event.dispatch(Self::handle_data_event);
532 }
533
534 pub fn dispatch_exec_event(event: DispatchMessage<ExecutionEvent>) {
540 event.dispatch(Self::handle_exec_event);
541 }
542
543 #[inline]
545 pub fn handle_data_event(event: DataEvent) {
546 match event {
547 DataEvent::Data(data) => {
548 msgbus::send_data(MessagingSwitchboard::data_engine_process_data(), data);
549 }
550 DataEvent::ResponseEnvelope(response) => {
551 msgbus::send_any(MessagingSwitchboard::data_engine_process(), &response);
552 }
553 DataEvent::RequestFailed(failure) => {
554 msgbus::send_any(MessagingSwitchboard::data_engine_process(), &failure);
555 }
556 DataEvent::Response(resp) => {
557 msgbus::send_data_response(MessagingSwitchboard::data_engine_response(), resp);
558 }
559 #[cfg(feature = "defi")]
560 DataEvent::DeFi(data) => {
561 msgbus::send_defi_data(MessagingSwitchboard::data_engine_process_defi_data(), data);
562 }
563 #[cfg(feature = "defi")]
564 DataEvent::PoolSnapshotResponse(response) => {
565 msgbus::send_any(MessagingSwitchboard::data_engine_process(), &response);
566 }
567 #[cfg(not(feature = "defi"))]
568 #[allow(
569 unreachable_patterns,
570 reason = "DeFi variants can exist without this crate's defi feature"
571 )]
572 other => log::error!(
573 "Cannot handle {other} data event, nautilus-live built without its `defi` feature"
574 ),
575 }
576 }
577
578 #[inline]
580 pub fn handle_exec_command(cmd: TradingCommand) {
581 msgbus::send_trading_command(MessagingSwitchboard::exec_engine_execute(), cmd);
582 }
583
584 #[inline]
590 pub fn handle_trading_command(message: DispatchMessage<TradingCommandMessage>) {
591 message.dispatch_trading(|_| {});
592 }
593
594 #[inline]
596 pub fn handle_exec_event(event: ExecutionEvent) {
597 match event {
598 ExecutionEvent::Order(order_event) => {
599 msgbus::send_order_event(MessagingSwitchboard::exec_engine_process(), order_event);
600 }
601 ExecutionEvent::OrderSubmittedBatch(batch) => {
602 for submitted in batch {
603 msgbus::send_order_event(
604 MessagingSwitchboard::exec_engine_process(),
605 OrderEventAny::Submitted(submitted),
606 );
607 }
608 }
609 ExecutionEvent::OrderAcceptedBatch(batch) => {
610 for accepted in batch {
611 msgbus::send_order_event(
612 MessagingSwitchboard::exec_engine_process(),
613 OrderEventAny::Accepted(accepted),
614 );
615 }
616 }
617 ExecutionEvent::OrderCanceledBatch(batch) => {
618 for canceled in batch {
619 msgbus::send_order_event(
620 MessagingSwitchboard::exec_engine_process(),
621 OrderEventAny::Canceled(canceled),
622 );
623 }
624 }
625 ExecutionEvent::Report(report) => {
626 Self::handle_exec_report(report);
627 }
628 ExecutionEvent::Account(ref account) => {
629 msgbus::send_account_state(
630 MessagingSwitchboard::portfolio_update_account(),
631 account,
632 );
633 }
634 ExecutionEvent::Data(ref data) => {
635 let topic = msgbus::switchboard::get_custom_topic(&data.data_type);
636 msgbus::publish_any(topic, data);
637 }
638 }
639 }
640
641 #[inline]
642 pub fn handle_exec_report(report: ExecutionReport) {
643 let endpoint = MessagingSwitchboard::exec_engine_reconcile_execution_report();
644 msgbus::send_execution_report(endpoint, report);
645 }
646}
647
648#[cfg(feature = "node")]
649impl AsyncRunner {
650 pub(crate) fn poll_pending(&mut self, mut process: impl FnMut(PendingRunnerEvent)) -> usize {
651 self.bind_senders();
652
653 let pending = (
654 self.channels.time_evt_rx.len(),
655 self.channels.system_evt_rx.len(),
656 self.channels.system_cmd_rx.len(),
657 self.channels.exec_evt_rx.len(),
658 self.channels.exec_cmd_rx.len(),
659 self.channels.data_evt_rx.len(),
660 self.channels.data_cmd_rx.len(),
661 );
662
663 let total =
664 pending.0 + pending.1 + pending.2 + pending.3 + pending.4 + pending.5 + pending.6;
665 log::debug!("Draining shutdown queues ({total} events initially pending)");
666 let mut progress = 0usize;
667 let mut process = |event| {
668 process(event);
669 progress += 1;
670 if progress.is_multiple_of(10_000) {
671 log::debug!("Processed {progress} events during shutdown drain");
672 }
673 };
674
675 let mut processed = 0;
676 processed += poll_channel(
677 &mut self.channels.time_evt_rx,
678 pending.0,
679 PendingRunnerEvent::TimeEvent,
680 &mut process,
681 );
682 processed += poll_channel(
683 &mut self.channels.system_evt_rx,
684 pending.1,
685 PendingRunnerEvent::SystemEvent,
686 &mut process,
687 );
688 processed += poll_channel(
689 &mut self.channels.system_cmd_rx,
690 pending.2,
691 PendingRunnerEvent::SystemCommand,
692 &mut process,
693 );
694 processed += poll_channel(
695 &mut self.channels.exec_evt_rx,
696 pending.3,
697 PendingRunnerEvent::ExecEvent,
698 &mut process,
699 );
700 processed += poll_channel(
701 &mut self.channels.exec_cmd_rx,
702 pending.4,
703 PendingRunnerEvent::ExecCommand,
704 &mut process,
705 );
706 processed += poll_channel(
707 &mut self.channels.data_evt_rx,
708 pending.5,
709 PendingRunnerEvent::DataEvent,
710 &mut process,
711 );
712 processed += poll_channel(
713 &mut self.channels.data_cmd_rx,
714 pending.6,
715 PendingRunnerEvent::DataCommand,
716 &mut process,
717 );
718 log::debug!("Drained {processed} remaining events during shutdown");
719 processed
720 }
721
722 pub(crate) async fn recv(&mut self) -> Option<PendingRunnerEvent> {
723 let commands_empty = self.channels.data_cmd_rx.is_empty();
724 let poll_time = !self.skip_time || (self.channels.data_evt_rx.is_empty() && commands_empty);
725 let poll_data = !self.skip_data || commands_empty;
726 tokio::select! {
727 biased;
728
729 Some(message) = self.channels.time_evt_rx.recv(), if poll_time => {
730 self.skip_time = true;
731 Some(PendingRunnerEvent::TimeEvent(message))
732 }
733 Some(event) = self.channels.system_evt_rx.recv() => {
734 self.skip_time = false;
735 Some(PendingRunnerEvent::SystemEvent(event))
736 }
737 Some(command) = self.channels.system_cmd_rx.recv() => {
738 self.skip_time = false;
739 Some(PendingRunnerEvent::SystemCommand(command))
740 }
741 Some(event) = self.channels.exec_evt_rx.recv() => {
742 self.skip_time = false;
743 Some(PendingRunnerEvent::ExecEvent(event))
744 }
745 Some(command) = self.channels.exec_cmd_rx.recv() => {
746 self.skip_time = false;
747 Some(PendingRunnerEvent::ExecCommand(command))
748 }
749 Some(event) = self.channels.data_evt_rx.recv(), if poll_data => {
750 self.skip_time = false;
751 self.skip_data = true;
752 Some(PendingRunnerEvent::DataEvent(event))
753 }
754 Some(command) = self.channels.data_cmd_rx.recv() => {
755 self.skip_time = false;
756 self.skip_data = false;
757 Some(PendingRunnerEvent::DataCommand(command))
758 }
759 else => None,
760 }
761 }
762}
763
764#[cfg(feature = "node")]
765fn node_stopped(handle: Option<&LiveNodeHandle>) -> bool {
766 handle.is_some_and(|handle| handle.state() == NodeState::Stopped)
767}
768
769#[cfg(feature = "node")]
770fn poll_channel<T>(
771 receiver: &mut tokio::sync::mpsc::UnboundedReceiver<T>,
772 pending: usize,
773 event: impl Fn(T) -> PendingRunnerEvent,
774 process: &mut impl FnMut(PendingRunnerEvent),
775) -> usize {
776 let mut processed = 0;
777
778 for _ in 0..pending {
779 let Ok(message) = receiver.try_recv() else {
780 break;
781 };
782
783 process(event(message));
784 processed += 1;
785 }
786
787 processed
788}
789
790#[cfg(test)]
791mod tests {
792 #[cfg(not(all(feature = "simulation", madsim)))]
793 use std::num::NonZeroU64;
794 use std::{cell::RefCell, rc::Rc, sync::Arc, time::Duration};
795
796 #[cfg(not(all(feature = "simulation", madsim)))]
797 use nautilus_common::live::LiveTimer;
798 use nautilus_common::{
799 actor,
800 cache::Cache,
801 clock::VirtualClock,
802 live::{
803 dispatch::DispatchMessage,
804 runner::{
805 get_data_event_sender, get_exec_event_sender, get_system_command_sender,
806 get_system_event_sender, try_get_system_command_sender,
807 try_get_system_event_sender,
808 },
809 },
810 messages::{
811 ExecutionEvent,
812 data::{SubscribeCommand, SubscribeCustomData},
813 execution::{CancelAllOrders, QueryAccount, TradingCommand},
814 system::{ReconnectSocket, SocketState, SocketStateChange},
815 },
816 msgbus::{Endpoint, MStr, TypedIntoHandler, stubs::get_typed_into_message_saving_handler},
817 runner::{
818 SyncTradingCommandSender, TimeEventMessage, drain_trading_cmd_queue,
819 get_data_cmd_sender, get_time_event_sender, get_trading_cmd_sender,
820 replace_exec_cmd_sender, try_get_time_event_sender, try_get_trading_cmd_sender,
821 },
822 timer::{TimeEvent, TimeEventCallback},
823 };
824 #[cfg(not(all(feature = "simulation", madsim)))]
825 use nautilus_core::time::get_atomic_clock_realtime;
826 use nautilus_core::{UUID4, UnixNanos};
827 use nautilus_execution::engine::ExecutionEngine;
828 #[cfg(feature = "node")]
829 use nautilus_model::types::Money;
830 use nautilus_model::{
831 data::{
832 Data, DataType, FundingRateUpdate, InstrumentStatus, OptionGreeks, quote::QuoteTick,
833 stubs::stub_instrument_status,
834 },
835 enums::{AccountType, OrderSide},
836 events::{OrderEventAny, account::state::AccountState, order::spec::OrderSubmittedSpec},
837 identifiers::{
838 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TraderId, Venue,
839 },
840 instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt},
841 types::{Price, Quantity},
842 };
843 use rstest::rstest;
844 use ustr::Ustr;
845
846 use super::*;
847
848 #[rstest]
849 fn test_runner_dispatches_typed_data_once_in_order() {
850 let mut runner = AsyncRunner::new();
851 let expected = vec![
852 Data::from(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt())),
853 Data::FundingRate(FundingRateUpdate::new(
854 InstrumentId::from("ETHUSDT.BINANCE"),
855 "0.00012".parse().unwrap(),
856 Some(480),
857 Some(UnixNanos::from(300)),
858 UnixNanos::from(100),
859 UnixNanos::from(200),
860 )),
861 Data::InstrumentStatus(InstrumentStatus {
862 ts_event: UnixNanos::from(400),
863 ts_init: UnixNanos::from(500),
864 ..stub_instrument_status()
865 }),
866 Data::OptionGreeks(OptionGreeks {
867 instrument_id: InstrumentId::from("BTC-20270101-50000-C.DERIBIT"),
868 mark_iv: Some(0.55),
869 bid_iv: Some(0.54),
870 ask_iv: Some(0.56),
871 underlying_price: Some(50001.0),
872 open_interest: Some(123.0),
873 ts_event: UnixNanos::from(600),
874 ts_init: UnixNanos::from(700),
875 ..OptionGreeks::default()
876 }),
877 ];
878 let received = Rc::new(RefCell::new(Vec::<Data>::new()));
879 let observed = received.clone();
880 msgbus::register_data_endpoint(
881 MessagingSwitchboard::data_engine_process_data(),
882 TypedIntoHandler::from(move |data| observed.borrow_mut().push(data)),
883 );
884
885 for data in &expected {
886 runner
887 .data_evt_tx
888 .send(DataEvent::Data(data.clone()).into())
889 .unwrap();
890 }
891
892 runner.flush_pending_data();
893
894 assert_eq!(
895 serde_json::to_value(&*received.borrow()).unwrap(),
896 serde_json::to_value(&expected).unwrap(),
897 );
898 }
899
900 #[rstest]
901 #[case(false)]
902 #[case(true)]
903 #[tokio::test]
904 async fn test_runner_callback_failure_stops_before_next_command(#[case] before_run: bool) {
905 actor::clear_callbacks().unwrap();
906 let mut runner = AsyncRunner::new();
907 let received = Rc::new(RefCell::new(Vec::new()));
908 let observed = received.clone();
909 msgbus::register_trading_command_endpoint(
910 MessagingSwitchboard::risk_engine_execute(),
911 TypedIntoHandler::from(move |command: TradingCommand| {
912 observed.borrow_mut().push(command.ts_init());
913 crate::dispatch::tests::latch_callback_failure();
914 }),
915 );
916
917 let sender = AsyncTradingCommandSender::new(runner.exec_cmd_tx.clone());
918 let seed_endpoint = ustr::Ustr::from("test.callback-root-seed");
919 msgbus::register_trading_command_endpoint(
920 seed_endpoint.into(),
921 TypedIntoHandler::from(move |command: TradingCommand| {
922 sender.execute(TradingCommandMessage::new(
923 MessagingSwitchboard::risk_engine_execute(),
924 command,
925 ));
926 }),
927 );
928
929 for timestamp in [17, 23] {
930 SyncTradingCommandSender.execute(TradingCommandMessage::new(
931 seed_endpoint.into(),
932 TradingCommand::QueryAccount(QueryAccount::new(
933 "TRADER-001".into(),
934 None,
935 "SIM-001".into(),
936 UUID4::new(),
937 timestamp.into(),
938 None,
939 None,
940 )),
941 ));
942 }
943
944 drain_trading_cmd_queue();
945 let first = runner.channels.exec_cmd_rx.try_recv().unwrap();
946 let second = runner.channels.exec_cmd_rx.try_recv().unwrap();
947 assert!(first.is_rooted());
948 assert!(second.is_rooted());
949 runner.exec_cmd_tx.send(first).unwrap();
950 runner.exec_cmd_tx.send(second).unwrap();
951
952 if before_run {
953 crate::dispatch::tests::latch_callback_failure();
954 }
955
956 let error = runner.run().await.unwrap_err();
957
958 assert_eq!(
959 error.downcast_ref::<actor::CallbackDispatchError>(),
960 Some(&actor::CallbackDispatchError::DeliveryUnwound)
961 );
962 assert_eq!(
963 *received.borrow(),
964 if before_run {
965 vec![]
966 } else {
967 vec![UnixNanos::from(17)]
968 }
969 );
970 assert_eq!(
971 runner.channels.exec_cmd_rx.len(),
972 if before_run { 2 } else { 1 }
973 );
974 assert!(!runner.exec_cmd_tx.is_closed());
975 assert_eq!(
976 actor::callback_failure(),
977 Some(actor::CallbackDispatchError::DeliveryUnwound)
978 );
979 drop(runner);
980 assert_eq!(actor::clear_callbacks(), Ok(()));
981 }
982
983 #[tokio::test]
984 async fn test_runner_stop_preserves_messages_and_channel_only_scheduling() {
985 let mut runner = AsyncRunner::new();
986 let received = Rc::new(RefCell::new(Vec::new()));
987 let stop = runner.signal_tx.clone();
988
989 for timestamp in 1..=65 {
990 let received = received.clone();
991 let stop = stop.clone();
992 runner
993 .time_evt_tx
994 .send(
995 TimeEventMessage::new(
996 TimeEvent::new(
997 "runner-resume".into(),
998 UUID4::new(),
999 timestamp.into(),
1000 timestamp.into(),
1001 ),
1002 TimeEventCallback::RustLocal(Rc::new(move |_| {
1003 received.borrow_mut().push(timestamp);
1004 if timestamp == 65 {
1005 stop.send(()).unwrap();
1006 }
1007 })),
1008 )
1009 .into(),
1010 )
1011 .unwrap();
1012 }
1013
1014 runner.stop();
1015 runner.run().await.unwrap();
1016
1017 assert!(received.borrow().is_empty());
1018 assert_eq!(runner.channels.time_evt_rx.len(), 65);
1019 assert!(!runner.time_evt_tx.is_closed());
1020
1021 let (result, at_yield) = tokio::join!(biased;
1022 runner.run(),
1023 async { received.borrow().clone() },
1024 );
1025 result.unwrap();
1026
1027 let expected: Vec<u64> = (1..=65).collect();
1028 assert_eq!(at_yield, expected);
1029 assert_eq!(*received.borrow(), expected);
1030 assert_eq!(runner.channels.time_evt_rx.len(), 0);
1031 assert!(!runner.time_evt_tx.is_closed());
1032 }
1033
1034 fn test_quote() -> QuoteTick {
1035 QuoteTick {
1036 instrument_id: InstrumentId::from("EUR/USD.SIM"),
1037 bid_price: Price::from("1.10000"),
1038 ask_price: Price::from("1.10001"),
1039 bid_size: Quantity::from(1_000_000),
1040 ask_size: Quantity::from(1_000_000),
1041 ts_event: UnixNanos::default(),
1042 ts_init: UnixNanos::default(),
1043 }
1044 }
1045
1046 fn test_system_event() -> SystemEvent {
1047 SystemEvent::SocketState(SocketStateChange::new(
1048 ClientId::from("BINANCE"),
1049 Some(Venue::from("BINANCE")),
1050 Ustr::from("binance-futures-market-streams"),
1051 SocketState::Connected,
1052 ))
1053 }
1054
1055 fn test_system_command() -> SystemCommand {
1056 SystemCommand::ReconnectSocket(ReconnectSocket::new(
1057 TraderId::from("TRADER-001"),
1058 ClientId::from("POLYMARKET"),
1059 Ustr::from("polymarket-market-streams"),
1060 UnixNanos::from(3),
1061 ))
1062 }
1063
1064 #[cfg(feature = "node")]
1065 #[rstest]
1066 fn test_poll_pending_processes_entry_snapshot_across_channels() {
1067 let mut runner = AsyncRunner::new();
1068 let time_evt_tx = runner.time_evt_tx.clone();
1069 let data_evt_tx = runner.data_evt_tx.clone();
1070 let data_cmd_tx = runner.data_cmd_tx.clone();
1071 let exec_evt_tx = runner.exec_evt_tx.clone();
1072 let exec_cmd_tx = runner.exec_cmd_tx.clone();
1073
1074 let time_event = TimeEvent::new(
1075 Ustr::from("test"),
1076 UUID4::new(),
1077 UnixNanos::from(1),
1078 UnixNanos::from(2),
1079 );
1080 time_evt_tx
1081 .send(
1082 (TimeEventMessage::new(time_event, TimeEventCallback::from(|_: TimeEvent| {})))
1083 .into(),
1084 )
1085 .unwrap();
1086 exec_evt_tx
1087 .send(
1088 (ExecutionEvent::Order(OrderEventAny::Submitted(
1089 OrderSubmittedSpec::builder()
1090 .client_order_id(ClientOrderId::from("O-POLL-001"))
1091 .build(),
1092 )))
1093 .into(),
1094 )
1095 .unwrap();
1096 exec_cmd_tx
1097 .send(
1098 TradingCommandMessage::new(
1099 MessagingSwitchboard::exec_engine_execute(),
1100 TradingCommand::CancelAllOrders(CancelAllOrders::new(
1101 TraderId::from("TRADER-001"),
1102 None,
1103 StrategyId::from("S-POLL-001"),
1104 InstrumentId::from("EUR/USD.SIM"),
1105 Some(OrderSide::Buy),
1106 UUID4::new(),
1107 UnixNanos::from(3),
1108 None,
1109 None,
1110 )),
1111 )
1112 .into(),
1113 )
1114 .unwrap();
1115 data_evt_tx
1116 .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1117 .unwrap();
1118 data_cmd_tx
1119 .send(
1120 DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1121 client_id: Some(ClientId::from("POLL")),
1122 venue: None,
1123 data_type: DataType::new("QuoteTick", None, None),
1124 command_id: UUID4::new(),
1125 ts_init: UnixNanos::from(4),
1126 correlation_id: None,
1127 params: None,
1128 }))
1129 .into(),
1130 )
1131 .unwrap();
1132
1133 runner.bind_senders();
1134 get_system_command_sender()
1135 .send(test_system_command())
1136 .unwrap();
1137 get_system_event_sender().send(test_system_event()).unwrap();
1138 get_system_event_sender().send(test_system_event()).unwrap();
1139 let mut processed_by_channel = [0; 7];
1140 let mut processed_order = Vec::new();
1141
1142 let first = runner.poll_pending(|event| match event {
1143 PendingRunnerEvent::TimeEvent(_) => {
1144 processed_by_channel[0] += 1;
1145 processed_order.push("time");
1146 }
1147 PendingRunnerEvent::SystemEvent(_) => {
1148 processed_by_channel[1] += 1;
1149 processed_order.push("system_event");
1150 }
1151 PendingRunnerEvent::SystemCommand(_) => {
1152 processed_by_channel[2] += 1;
1153 processed_order.push("system_command");
1154 }
1155 PendingRunnerEvent::ExecEvent(_) => {
1156 processed_by_channel[3] += 1;
1157 processed_order.push("exec_event");
1158 }
1159 PendingRunnerEvent::ExecCommand(_) => {
1160 processed_by_channel[4] += 1;
1161 processed_order.push("exec_command");
1162 }
1163 PendingRunnerEvent::DataEvent(_) => {
1164 processed_by_channel[5] += 1;
1165 processed_order.push("data_event");
1166 data_evt_tx
1167 .send((DataEvent::Data(Data::Quote(test_quote()))).into())
1168 .unwrap();
1169 }
1170 PendingRunnerEvent::DataCommand(_) => {
1171 processed_by_channel[6] += 1;
1172 processed_order.push("data_command");
1173 }
1174 });
1175
1176 let second = runner.poll_pending(|event| match event {
1177 PendingRunnerEvent::DataEvent(_) => {
1178 processed_by_channel[5] += 1;
1179 processed_order.push("data_event");
1180 }
1181 _ => panic!("Unexpected runner event"),
1182 });
1183
1184 assert_eq!(first, 8);
1185 assert_eq!(second, 1);
1186 assert_eq!(processed_by_channel, [1, 2, 1, 1, 1, 2, 1]);
1187 assert_eq!(
1188 processed_order,
1189 [
1190 "time",
1191 "system_event",
1192 "system_event",
1193 "system_command",
1194 "exec_event",
1195 "exec_command",
1196 "data_event",
1197 "data_command",
1198 "data_event",
1199 ]
1200 );
1201 }
1202
1203 #[cfg(feature = "node")]
1204 #[tokio::test]
1205 async fn test_recv_processes_system_event_before_command() {
1206 let mut runner = AsyncRunner::new();
1207
1208 runner
1209 .system_cmd_tx
1210 .send((test_system_command()).into())
1211 .unwrap();
1212 runner
1213 .system_evt_tx
1214 .send((test_system_event()).into())
1215 .unwrap();
1216
1217 assert!(matches!(
1218 runner.recv().await,
1219 Some(PendingRunnerEvent::SystemEvent(_))
1220 ));
1221 assert!(matches!(
1222 runner.recv().await,
1223 Some(PendingRunnerEvent::SystemCommand(_))
1224 ));
1225 }
1226
1227 #[cfg(feature = "node")]
1228 #[rstest]
1229 #[case::timers_and_data(true, true, false)]
1230 #[case::timers_only(true, false, false)]
1231 #[case::data_only(false, true, false)]
1232 #[case::execution_backlog(true, true, true)]
1233 #[tokio::test]
1234 async fn test_runner_dispatch_progress_under_load(
1235 #[case] timers: bool,
1236 #[case] data: bool,
1237 #[case] execution: bool,
1238 #[values(false, true)] receive: bool,
1239 ) {
1240 let mut runner = AsyncRunner::new();
1241 let stop = runner.signal_tx.clone();
1242 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1243 if event.ts_init == UnixNanos::from(8) {
1244 stop.send(()).unwrap();
1245 }
1246 }));
1247 let received = Rc::new(std::cell::Cell::new(0_u64));
1248 let observed = received.clone();
1249 let stop = runner.signal_tx.clone();
1250 let exec_event = || {
1251 ExecutionEvent::Order(OrderEventAny::Submitted(
1252 OrderSubmittedSpec::builder().build(),
1253 ))
1254 };
1255 let exec_tx = runner.exec_evt_tx.clone();
1256 msgbus::register_data_endpoint(
1257 MessagingSwitchboard::data_engine_process_data(),
1258 TypedIntoHandler::from(move |event| {
1259 let Data::Quote(quote) = event else {
1260 panic!("Expected quote")
1261 };
1262 observed.set(observed.get() + 1);
1263 assert_eq!(quote.ts_init.as_u64(), observed.get());
1264 exec_tx.send(exec_event().into()).unwrap();
1266
1267 if !timers && quote.ts_init == UnixNanos::from(8) {
1268 stop.send(()).unwrap();
1269 }
1270 }),
1271 );
1272 let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1273 client_id: Some(ClientId::from("TEST")),
1274 venue: None,
1275 data_type: DataType::new("QuoteTick", None, None),
1276 command_id: UUID4::new(),
1277 ts_init: UnixNanos::default(),
1278 correlation_id: None,
1279 params: None,
1280 }));
1281
1282 for ts in (1..=16_u64).map(UnixNanos::from) {
1284 if timers {
1285 let event = TimeEvent::new("fairness".into(), UUID4::new(), ts, ts);
1286 runner
1287 .time_evt_tx
1288 .send(TimeEventMessage::new(event, callback.clone()).into())
1289 .unwrap();
1290 }
1291
1292 if data {
1293 let mut quote = test_quote();
1294 quote.ts_init = ts;
1295 runner
1296 .data_evt_tx
1297 .send(DataEvent::Data(Data::Quote(quote)).into())
1298 .unwrap();
1299 }
1300
1301 if execution {
1302 runner.exec_evt_tx.send(exec_event().into()).unwrap();
1303 }
1304 runner.data_cmd_tx.send(command.clone().into()).unwrap();
1305 }
1306
1307 if receive {
1308 while runner.signal_rx.try_recv().is_err() {
1309 match runner.recv().await.unwrap() {
1310 PendingRunnerEvent::TimeEvent(event) => {
1311 let _ = AsyncRunner::handle_time_event(event);
1312 }
1313 PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
1314 PendingRunnerEvent::DataCommand(cmd) => AsyncRunner::handle_data_command(cmd),
1315 PendingRunnerEvent::ExecEvent(event) => AsyncRunner::dispatch_exec_event(event),
1316 _ => panic!("Unexpected runner event"),
1317 }
1318 }
1319 } else {
1320 runner.run().await.unwrap();
1321 }
1322
1323 if execution {
1324 assert!((1..16).contains(&runner.channels.exec_evt_rx.len()));
1325 assert_eq!(received.get(), 0);
1326 } else {
1327 assert!((1..16).contains(&runner.channels.data_cmd_rx.len()));
1328
1329 if data {
1330 assert!(received.get() > 0);
1331 assert!(!runner.channels.data_evt_rx.is_empty());
1332 }
1333 }
1334
1335 if timers {
1336 assert_eq!(runner.channels.time_evt_rx.len(), 8);
1337 }
1338 }
1339
1340 #[cfg(feature = "node")]
1341 #[rstest]
1342 fn test_runner_senders_shutdown_logging() {
1343 struct ErrorCapture(std::sync::Mutex<Vec<String>>);
1344
1345 impl log::Log for ErrorCapture {
1346 fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
1347 metadata.level() == log::Level::Error
1348 && metadata.target() == "nautilus_live::runner"
1349 }
1350
1351 fn log(&self, record: &log::Record<'_>) {
1352 if self.enabled(record.metadata()) {
1353 self.0.lock().unwrap().push(record.args().to_string());
1354 }
1355 }
1356
1357 fn flush(&self) {}
1358 }
1359
1360 static ERRORS: ErrorCapture = ErrorCapture(std::sync::Mutex::new(Vec::new()));
1361 log::set_logger(&ERRORS).unwrap();
1362 log::set_max_level(log::LevelFilter::Error);
1363
1364 let runner = AsyncRunner::new();
1365 let handle = LiveNodeHandle::new();
1366 handle.set_starting();
1367 runner.bind_senders_for_node(handle.clone());
1368 let sender = get_data_cmd_sender();
1369 let time_sender = get_time_event_sender();
1370 let mut channels = runner.take_channels();
1371
1372 let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1373 client_id: Some(ClientId::from("TEST")),
1374 venue: None,
1375 data_type: DataType::new("QuoteTick", None, None),
1376 command_id: UUID4::new(),
1377 ts_init: UnixNanos::default(),
1378 correlation_id: None,
1379 params: None,
1380 }));
1381
1382 let event = TimeEvent::new("shutdown-timer".into(), UUID4::new(), 17.into(), 23.into());
1383 let expected_events = vec![event.clone(); 2];
1384 let received = Rc::new(RefCell::new(Vec::new()));
1385 let observed = received.clone();
1386 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1387 observed.borrow_mut().push(event);
1388 }));
1389
1390 for stopped in [false, true] {
1392 if stopped {
1393 handle.set_stopped();
1394 } else {
1395 handle.set_shutting_down();
1396 }
1397
1398 sender.execute(command.clone());
1399 assert_eq!(
1400 channels
1401 .data_cmd_rx
1402 .try_recv()
1403 .unwrap()
1404 .dispatch(|command| command),
1405 command
1406 );
1407
1408 time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1409 assert!(AsyncRunner::handle_time_event(
1410 channels.time_evt_rx.try_recv().unwrap()
1411 ));
1412 }
1413
1414 assert_eq!(*received.borrow(), expected_events);
1415
1416 drop(channels);
1417 sender.execute(command.clone());
1418 time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1419 assert_eq!(*ERRORS.0.lock().unwrap(), Vec::<String>::new());
1420
1421 let runner = AsyncRunner::new();
1423 let handle = LiveNodeHandle::new();
1424 runner.bind_senders_for_node(handle.clone());
1425 let sender = get_data_cmd_sender();
1426 let time_sender = get_time_event_sender();
1427 drop(runner);
1428
1429 for shutting_down in [false, true] {
1430 if shutting_down {
1431 handle.set_shutting_down();
1432 } else {
1433 handle.set_starting();
1434 }
1435
1436 sender.execute(command.clone());
1437 time_sender.send(TimeEventMessage::new(event.clone(), callback.clone()));
1438 }
1439
1440 let expected_errors = vec![
1441 "Failed to send data command: channel closed",
1442 "Failed to send time event message: channel closed",
1443 ];
1444 assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors.repeat(2));
1445
1446 ERRORS.0.lock().unwrap().clear();
1447 let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1448 drop(rx);
1449 AsyncDataCommandSender::new(tx).execute(command);
1450 let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
1451 drop(rx);
1452 AsyncTimeEventSender::new(tx).send(TimeEventMessage::new(event, callback));
1453 assert_eq!(*ERRORS.0.lock().unwrap(), expected_errors);
1454 assert_eq!(*received.borrow(), expected_events);
1455 }
1456
1457 #[tokio::test]
1458 async fn test_async_data_command_sender_execute() {
1459 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1460 let sender = AsyncDataCommandSender::new(tx);
1461
1462 let command = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1463 client_id: Some(ClientId::from("TEST")),
1464 venue: None,
1465 data_type: DataType::new("QuoteTick", None, None),
1466 command_id: UUID4::new(),
1467 ts_init: UnixNanos::default(),
1468 correlation_id: None,
1469 params: None,
1470 }));
1471
1472 sender.execute(command.clone());
1473
1474 let received = rx.recv().await.unwrap().dispatch(|command| command);
1475 match (received, command) {
1476 (
1477 DataCommand::Subscribe(SubscribeCommand::Data(r)),
1478 DataCommand::Subscribe(SubscribeCommand::Data(c)),
1479 ) => {
1480 assert_eq!(r.client_id, c.client_id);
1481 assert_eq!(r.data_type, c.data_type);
1482 }
1483 _ => panic!("Command mismatch"),
1484 }
1485 }
1486
1487 #[rstest]
1488 #[case(false)]
1489 #[case(true)]
1490 fn test_async_command_senders_capture_owner_root(#[case] rooted: bool) {
1491 let runner = AsyncRunner::new();
1492 runner.bind_senders();
1493 let mut channels = runner.take_channels();
1494
1495 let data = DataCommand::Subscribe(SubscribeCommand::Data(SubscribeCustomData {
1496 client_id: Some(ClientId::from("ROOT")),
1497 venue: None,
1498 data_type: DataType::new("QuoteTick", None, None),
1499 command_id: UUID4::new(),
1500 ts_init: UnixNanos::from(7),
1501 correlation_id: None,
1502 params: None,
1503 }));
1504
1505 let trading = TradingCommand::CancelAllOrders(CancelAllOrders::new(
1506 TraderId::from("TRADER-001"),
1507 None,
1508 StrategyId::from("ROOT-001"),
1509 InstrumentId::from("EUR/USD.SIM"),
1510 Some(OrderSide::Sell),
1511 UUID4::new(),
1512 UnixNanos::from(11),
1513 None,
1514 None,
1515 ));
1516 let expected_data = data.clone();
1517 let expected_trading = trading.clone();
1518 let trigger = trading.clone();
1519
1520 let send = move || {
1521 get_data_cmd_sender().execute(data.clone());
1522 get_trading_cmd_sender().execute(TradingCommandMessage::new(
1523 MessagingSwitchboard::exec_engine_execute(),
1524 trading.clone(),
1525 ));
1526 };
1527
1528 if rooted {
1529 msgbus::register_trading_command_endpoint(
1530 MessagingSwitchboard::risk_engine_execute(),
1531 TypedIntoHandler::from(move |_| send()),
1532 );
1533 SyncTradingCommandSender.execute(TradingCommandMessage::new(
1534 MessagingSwitchboard::risk_engine_execute(),
1535 trigger,
1536 ));
1537 drain_trading_cmd_queue();
1538 } else {
1539 send();
1540 }
1541
1542 let data = channels.data_cmd_rx.try_recv().unwrap();
1543 let trading = channels.exec_cmd_rx.try_recv().unwrap();
1544
1545 let results = thread::spawn(move || {
1546 let data = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1547 data.dispatch(|command| command)
1548 }));
1549
1550 let trading = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1551 trading.dispatch(|message| (message.endpoint(), message.command().clone()))
1552 }));
1553
1554 (data, trading)
1555 })
1556 .join()
1557 .unwrap();
1558
1559 if rooted {
1560 for error in [results.0.unwrap_err(), results.1.unwrap_err()] {
1561 let message = error.downcast_ref::<String>().unwrap();
1562 assert!(message.contains("command context dispatched outside its owner thread"));
1563 }
1564 } else {
1565 assert_eq!(results.0.unwrap(), expected_data);
1566 assert_eq!(
1567 results.1.unwrap(),
1568 (
1569 MessagingSwitchboard::exec_engine_execute(),
1570 expected_trading
1571 ),
1572 );
1573 }
1574
1575 assert!(channels.data_cmd_rx.is_empty());
1576 assert!(channels.exec_cmd_rx.is_empty());
1577 }
1578
1579 #[rstest]
1580 #[case(false)]
1581 #[case(true)]
1582 fn test_event_senders_capture_owner_root(
1583 #[case] rooted: bool,
1584 #[values(false, true)] dispatch: bool,
1585 ) {
1586 let runner = AsyncRunner::new();
1587 runner.bind_senders();
1588 let mut channels = runner.take_channels();
1589 let expected_quote = test_quote();
1590 let expected_order = OrderSubmittedSpec::builder()
1591 .client_order_id(ClientOrderId::from("EVENT-017"))
1592 .build();
1593 let order = expected_order.clone();
1594
1595 let send = move || {
1596 get_data_event_sender()
1597 .send(DataEvent::Data(Data::Quote(expected_quote)))
1598 .unwrap();
1599 get_exec_event_sender()
1600 .send(ExecutionEvent::Order(OrderEventAny::Submitted(
1601 order.clone(),
1602 )))
1603 .unwrap();
1604 };
1605
1606 if rooted {
1607 msgbus::register_trading_command_endpoint(
1608 MessagingSwitchboard::risk_engine_execute(),
1609 TypedIntoHandler::from(move |_| send()),
1610 );
1611 SyncTradingCommandSender.execute(TradingCommandMessage::new(
1612 MessagingSwitchboard::risk_engine_execute(),
1613 TradingCommand::QueryAccount(QueryAccount::new(
1614 "TRADER-001".into(),
1615 None,
1616 "SIM-001".into(),
1617 UUID4::new(),
1618 19.into(),
1619 None,
1620 None,
1621 )),
1622 ));
1623 drain_trading_cmd_queue();
1624 } else {
1625 send();
1626 }
1627
1628 if dispatch {
1629 msgbus::register_data_endpoint(
1630 MessagingSwitchboard::data_engine_process_data(),
1631 TypedIntoHandler::from(move |data| {
1632 assert_eq!(data, Data::Quote(expected_quote));
1633 get_data_event_sender().send(DataEvent::Data(data)).unwrap();
1634 }),
1635 );
1636
1637 let order = expected_order.clone();
1638 msgbus::register_order_event_endpoint(
1639 MessagingSwitchboard::exec_engine_process(),
1640 TypedIntoHandler::from(move |event| {
1641 assert_eq!(event, OrderEventAny::Submitted(order.clone()));
1642 get_exec_event_sender()
1643 .send(ExecutionEvent::Order(event))
1644 .unwrap();
1645 }),
1646 );
1647
1648 AsyncRunner::dispatch_data_event(channels.data_evt_rx.try_recv().unwrap());
1649 AsyncRunner::dispatch_exec_event(channels.exec_evt_rx.try_recv().unwrap());
1650 }
1651
1652 let data = channels.data_evt_rx.try_recv().unwrap();
1653 let exec = channels.exec_evt_rx.try_recv().unwrap();
1654
1655 let (data, exec) = thread::spawn(move || {
1656 (
1657 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1658 data.dispatch(|event| event)
1659 })),
1660 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1661 exec.dispatch(|event| event)
1662 })),
1663 )
1664 })
1665 .join()
1666 .unwrap();
1667
1668 if rooted || dispatch {
1669 for error in [data.unwrap_err(), exec.unwrap_err()] {
1670 assert!(
1671 error
1672 .downcast_ref::<String>()
1673 .unwrap()
1674 .contains("command context dispatched outside its owner thread")
1675 );
1676 }
1677 } else {
1678 let DataEvent::Data(Data::Quote(quote)) = data.unwrap() else {
1679 panic!("expected quote")
1680 };
1681
1682 let ExecutionEvent::Order(OrderEventAny::Submitted(order)) = exec.unwrap() else {
1683 panic!("expected submitted order")
1684 };
1685
1686 assert_eq!(quote, expected_quote);
1687 assert_eq!(order, expected_order);
1688 }
1689
1690 assert!(channels.data_evt_rx.is_empty());
1691 assert!(channels.exec_evt_rx.is_empty());
1692 }
1693
1694 #[rstest]
1695 fn test_system_and_time_senders_capture_owner_root(#[values(false, true)] rooted: bool) {
1696 let runner = AsyncRunner::new();
1697 runner.bind_senders();
1698 let mut channels = runner.take_channels();
1699 let event = TimeEvent::new("system-time".into(), UUID4::new(), 17.into(), 23.into());
1700 let observed = Rc::new(RefCell::new(Vec::new()));
1701 let received = observed.clone();
1702 let expected_event = event.clone();
1703
1704 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1705 received.borrow_mut().push(event);
1706 get_system_command_sender()
1707 .send(test_system_command())
1708 .unwrap();
1709 }));
1710
1711 let send = move || {
1712 get_system_event_sender().send(test_system_event()).unwrap();
1713 get_system_command_sender()
1714 .send(test_system_command())
1715 .unwrap();
1716 get_time_event_sender().send(TimeEventMessage::new(event.clone(), callback.clone()));
1717 };
1718
1719 if rooted {
1720 msgbus::register_trading_command_endpoint(
1721 MessagingSwitchboard::risk_engine_execute(),
1722 TypedIntoHandler::from(move |_| send()),
1723 );
1724 SyncTradingCommandSender.execute(TradingCommandMessage::new(
1725 MessagingSwitchboard::risk_engine_execute(),
1726 TradingCommand::QueryAccount(QueryAccount::new(
1727 "TRADER-001".into(),
1728 None,
1729 "SIM-001".into(),
1730 UUID4::new(),
1731 31.into(),
1732 None,
1733 None,
1734 )),
1735 ));
1736 drain_trading_cmd_queue();
1737 } else {
1738 send();
1739 }
1740
1741 let system_event = channels.system_evt_rx.try_recv().unwrap();
1742 let system_command = channels.system_cmd_rx.try_recv().unwrap();
1743 let time_event = channels.time_evt_rx.try_recv().unwrap();
1744 assert_eq!(system_event.is_rooted(), rooted);
1745 assert_eq!(system_command.is_rooted(), rooted);
1746 assert_eq!(time_event.is_rooted(), rooted);
1747 assert!(AsyncRunner::handle_time_event(time_event));
1748 let child = channels.system_cmd_rx.try_recv().unwrap();
1749
1750 assert_eq!(system_event.dispatch(|event| event), test_system_event());
1751 assert_eq!(
1752 system_command.dispatch(|command| command),
1753 test_system_command()
1754 );
1755 assert_eq!(*observed.borrow(), vec![expected_event]);
1756 assert!(child.is_rooted());
1757 assert_eq!(child.dispatch(|command| command), test_system_command());
1758 assert!(channels.time_evt_rx.is_empty());
1759 assert!(channels.system_evt_rx.is_empty());
1760 assert!(channels.system_cmd_rx.is_empty());
1761 }
1762
1763 #[cfg(not(all(feature = "simulation", madsim)))]
1764 #[tokio::test]
1765 async fn test_live_timer_started_under_root_emits_independent_event() {
1766 let runner = AsyncRunner::new();
1767 runner.bind_senders();
1768 let mut channels = runner.take_channels();
1769 let now = get_atomic_clock_realtime().get_time_ns();
1770 let name = Ustr::from("ROOTED_TIMER");
1771 let owner = thread::current().id();
1772 let observed = Rc::new(RefCell::new(Vec::new()));
1773 let received = observed.clone();
1774
1775 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
1776 received
1777 .borrow_mut()
1778 .push((event.name, event.ts_event, thread::current().id()));
1779 }));
1780
1781 let timer = Rc::new(RefCell::new(None));
1782 let started = timer.clone();
1783 msgbus::register_trading_command_endpoint(
1784 MessagingSwitchboard::risk_engine_execute(),
1785 TypedIntoHandler::from(move |_| {
1786 get_system_command_sender()
1787 .send(test_system_command())
1788 .unwrap();
1789
1790 let mut timer = LiveTimer::new(
1791 name,
1792 NonZeroU64::new(1_000_000).unwrap(),
1793 now,
1794 Some(now),
1795 callback.clone(),
1796 true,
1797 Some(get_time_event_sender()),
1798 );
1799 timer.start();
1800 *started.borrow_mut() = Some(timer);
1801 }),
1802 );
1803
1804 SyncTradingCommandSender.execute(TradingCommandMessage::new(
1805 MessagingSwitchboard::risk_engine_execute(),
1806 TradingCommand::QueryAccount(QueryAccount::new(
1807 "TRADER-001".into(),
1808 None,
1809 "SIM-001".into(),
1810 UUID4::new(),
1811 31.into(),
1812 None,
1813 None,
1814 )),
1815 ));
1816 drain_trading_cmd_queue();
1817 let witness = channels.system_cmd_rx.try_recv().unwrap();
1818 let message = tokio::time::timeout(Duration::from_secs(2), channels.time_evt_rx.recv())
1819 .await
1820 .unwrap()
1821 .unwrap();
1822 timer.borrow_mut().take().unwrap().cancel();
1823
1824 assert!(witness.is_rooted());
1825 assert!(!message.is_rooted());
1826 assert!(observed.borrow().is_empty());
1827 assert!(AsyncRunner::handle_time_event(message));
1828 assert_eq!(*observed.borrow(), vec![(name, now, owner)]);
1829 assert!(channels.time_evt_rx.is_empty());
1830 }
1831
1832 #[tokio::test]
1833 async fn test_async_time_event_sender_send() {
1834 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1835 let sender = AsyncTimeEventSender::new(tx);
1836
1837 let event = TimeEvent::new(
1838 Ustr::from("test"),
1839 UUID4::new(),
1840 UnixNanos::from(1),
1841 UnixNanos::from(2),
1842 );
1843 let callback = TimeEventCallback::from(|_: TimeEvent| {});
1844 let message = TimeEventMessage::new(event, callback);
1845
1846 sender.send(message);
1847
1848 assert!(rx.recv().await.is_some());
1849 }
1850
1851 #[rstest]
1852 fn test_async_trading_command_sender_preserves_target_endpoints() {
1853 std::thread::spawn(|| {
1854 msgbus::get_message_bus().borrow_mut().dispose();
1855 let (risk_handler, risk_saving_handler) =
1856 get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1857 "RiskEngine.execute",
1858 )));
1859 msgbus::register_trading_command_endpoint(
1860 MessagingSwitchboard::risk_engine_execute(),
1861 risk_handler,
1862 );
1863 let (exec_handler, exec_saving_handler) =
1864 get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1865 "ExecEngine.execute",
1866 )));
1867 msgbus::register_trading_command_endpoint(
1868 MessagingSwitchboard::exec_engine_execute(),
1869 exec_handler,
1870 );
1871
1872 let (tx, mut rx) =
1873 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1874 let sender = AsyncTradingCommandSender::new(tx);
1875 sender.execute(TradingCommandMessage::new(
1876 MessagingSwitchboard::risk_engine_execute(),
1877 TradingCommand::CancelAllOrders(CancelAllOrders::new(
1878 TraderId::from("TRADER-001"),
1879 None,
1880 StrategyId::from("RISK-001"),
1881 InstrumentId::from("EUR/USD.SIM"),
1882 Some(OrderSide::Buy),
1883 UUID4::new(),
1884 UnixNanos::default(),
1885 None,
1886 None,
1887 )),
1888 ));
1889 sender.execute(TradingCommandMessage::new(
1890 MessagingSwitchboard::exec_engine_execute(),
1891 TradingCommand::CancelAllOrders(CancelAllOrders::new(
1892 TraderId::from("TRADER-001"),
1893 None,
1894 StrategyId::from("EXEC-001"),
1895 InstrumentId::from("EUR/USD.SIM"),
1896 Some(OrderSide::Sell),
1897 UUID4::new(),
1898 UnixNanos::default(),
1899 None,
1900 None,
1901 )),
1902 ));
1903
1904 AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1905 AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1906
1907 let risk_commands = risk_saving_handler.get_messages();
1908 let exec_commands = exec_saving_handler.get_messages();
1909 assert!(rx.try_recv().is_err());
1910 assert_eq!(risk_commands.len(), 1);
1911 assert_eq!(
1912 risk_commands[0].strategy_id(),
1913 Some(StrategyId::from("RISK-001"))
1914 );
1915 assert_eq!(exec_commands.len(), 1);
1916 assert_eq!(
1917 exec_commands[0].strategy_id(),
1918 Some(StrategyId::from("EXEC-001"))
1919 );
1920 })
1921 .join()
1922 .unwrap();
1923 }
1924
1925 #[rstest]
1926 fn test_async_runner_preserves_deferred_follow_up_order() {
1927 std::thread::spawn(|| {
1928 msgbus::get_message_bus().borrow_mut().dispose();
1929 let clock = Rc::new(RefCell::new(VirtualClock::new()));
1930 let cache = Rc::new(RefCell::new(Cache::default()));
1931 let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
1932 ExecutionEngine::register_msgbus_handlers(&exec_engine);
1933 msgbus::register_trading_command_endpoint(
1934 MessagingSwitchboard::risk_engine_execute(),
1935 TypedIntoHandler::from(|command: TradingCommand| {
1936 msgbus::send_trading_command(
1937 MessagingSwitchboard::exec_engine_queue_execute(),
1938 command,
1939 );
1940 }),
1941 );
1942
1943 let (exec_handler, exec_saving_handler) =
1944 get_typed_into_message_saving_handler::<TradingCommand>(Some(Ustr::from(
1945 "ExecEngine.execute",
1946 )));
1947 msgbus::register_trading_command_endpoint(
1948 MessagingSwitchboard::exec_engine_execute(),
1949 exec_handler,
1950 );
1951
1952 let (tx, mut rx) =
1953 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
1954 let sender = Arc::new(AsyncTradingCommandSender::new(tx));
1955 replace_exec_cmd_sender(sender.clone());
1956 sender.execute(TradingCommandMessage::new(
1957 MessagingSwitchboard::risk_engine_execute(),
1958 TradingCommand::CancelAllOrders(CancelAllOrders::new(
1959 TraderId::from("TRADER-001"),
1960 None,
1961 StrategyId::from("FIRST-001"),
1962 InstrumentId::from("EUR/USD.SIM"),
1963 Some(OrderSide::Buy),
1964 UUID4::new(),
1965 UnixNanos::default(),
1966 None,
1967 None,
1968 )),
1969 ));
1970 sender.execute(TradingCommandMessage::new(
1971 MessagingSwitchboard::exec_engine_execute(),
1972 TradingCommand::CancelAllOrders(CancelAllOrders::new(
1973 TraderId::from("TRADER-001"),
1974 None,
1975 StrategyId::from("SECOND-001"),
1976 InstrumentId::from("EUR/USD.SIM"),
1977 Some(OrderSide::Sell),
1978 UUID4::new(),
1979 UnixNanos::default(),
1980 None,
1981 None,
1982 )),
1983 ));
1984
1985 AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1986 AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
1987
1988 let commands = exec_saving_handler.get_messages();
1989 let strategy_ids = commands
1990 .iter()
1991 .map(TradingCommand::strategy_id)
1992 .collect::<Vec<_>>();
1993 assert!(rx.try_recv().is_err());
1994 assert_eq!(commands.len(), 2);
1995 assert_eq!(
1996 strategy_ids,
1997 vec![
1998 Some(StrategyId::from("FIRST-001")),
1999 Some(StrategyId::from("SECOND-001"))
2000 ]
2001 );
2002 })
2003 .join()
2004 .unwrap();
2005 }
2006
2007 #[rstest]
2008 fn test_async_runner_dispatches_deferred_exec_command_once() {
2009 std::thread::spawn(|| {
2010 msgbus::get_message_bus().borrow_mut().dispose();
2011 let clock = Rc::new(RefCell::new(VirtualClock::new()));
2012 let cache = Rc::new(RefCell::new(Cache::default()));
2013 let exec_engine = Rc::new(RefCell::new(ExecutionEngine::new(clock, cache, None)));
2014 ExecutionEngine::register_msgbus_handlers(&exec_engine);
2015
2016 let (tx, mut rx) =
2017 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
2018 replace_exec_cmd_sender(Arc::new(AsyncTradingCommandSender::new(tx)));
2019 let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2020 TraderId::from("TRADER-001"),
2021 None,
2022 StrategyId::from("EXEC-001"),
2023 InstrumentId::from("EUR/USD.SIM"),
2024 Some(OrderSide::Buy),
2025 UUID4::new(),
2026 UnixNanos::default(),
2027 None,
2028 None,
2029 ));
2030
2031 msgbus::send_trading_command(
2032 MessagingSwitchboard::exec_engine_queue_execute(),
2033 command,
2034 );
2035 assert_eq!(exec_engine.borrow().command_count(), 0);
2036
2037 AsyncRunner::handle_trading_command(rx.try_recv().unwrap());
2038
2039 assert!(rx.try_recv().is_err());
2040 assert_eq!(exec_engine.borrow().command_count(), 1);
2041 })
2042 .join()
2043 .unwrap();
2044 }
2045
2046 #[cfg(feature = "node")]
2047 #[rstest]
2048 #[case::cash(AccountType::Cash)]
2049 #[case::margin(AccountType::Margin)]
2050 fn test_account_event_publishes_after_portfolio_application(#[case] account_type: AccountType) {
2051 use nautilus_common::msgbus::{MessageBus, TypedHandler};
2052 use nautilus_model::{
2053 accounts::Account,
2054 types::{AccountBalance, Currency},
2055 };
2056 use nautilus_portfolio::Portfolio;
2057
2058 struct AccountLogCapture(std::sync::Mutex<Vec<String>>);
2060
2061 impl log::Log for AccountLogCapture {
2062 fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
2063 metadata.level() == log::Level::Info
2064 && metadata.target() == "nautilus_portfolio::portfolio"
2065 }
2066
2067 fn log(&self, record: &log::Record<'_>) {
2068 if self.enabled(record.metadata()) {
2069 self.0.lock().unwrap().push(record.args().to_string());
2070 }
2071 }
2072
2073 fn flush(&self) {}
2074 }
2075
2076 static LOGS: AccountLogCapture = AccountLogCapture(std::sync::Mutex::new(Vec::new()));
2077 log::set_logger(&LOGS).unwrap();
2078 log::set_max_level(log::LevelFilter::Info);
2079
2080 msgbus::set_message_bus(Rc::new(RefCell::new(MessageBus::new(
2081 TraderId::from("TESTER-001"),
2082 UUID4::new(),
2083 None,
2084 None,
2085 ))));
2086 let cache = Rc::new(RefCell::new(Cache::default()));
2087 let clock = Rc::new(RefCell::new(VirtualClock::new()));
2088 let _portfolio = Portfolio::new(clock, cache.clone(), None);
2089 let observed = Rc::new(RefCell::new(Vec::new()));
2090 let received = observed.clone();
2091 let source = cache.clone();
2092 let handler = TypedHandler::from(move |event: &AccountState| {
2093 let cache = source.borrow();
2094 let account = cache.account(&event.account_id).unwrap();
2095 assert_eq!(account.last_event().unwrap().event_id, event.event_id);
2096 assert_eq!(account.account_type(), event.account_type);
2097 assert_eq!(
2098 account.balances().into_values().collect::<Vec<_>>(),
2099 event.balances
2100 );
2101 received
2102 .borrow_mut()
2103 .push((event.event_id, account.events().len()));
2104 });
2105
2106 msgbus::subscribe_account_state("events.account.*".into(), handler.clone(), Some(20));
2108 let scoped = Rc::new(RefCell::new(Vec::new()));
2109 let scoped_received = scoped.clone();
2110 let scoped_handler = TypedHandler::from(move |event: &AccountState| {
2111 scoped_received.borrow_mut().push(event.event_id);
2112 });
2113 msgbus::subscribe_account_state(
2114 "events.account.SIM-001".into(),
2115 scoped_handler.clone(),
2116 None,
2117 );
2118 let mut expected = Vec::new();
2119 let mut expected_scoped = Vec::new();
2120 for (index, (account_id, total, locked, free, history_len)) in [
2121 ("SIM-001", "100 USD", "20 USD", "80 USD", 1),
2122 ("SIM-002", "200 USD", "50 USD", "150 USD", 1),
2123 ("SIM-001", "90 USD", "10 USD", "80 USD", 2),
2124 ("SIM-001", "0 USD", "0 USD", "0 USD", 3),
2125 ]
2126 .into_iter()
2127 .enumerate()
2128 {
2129 let event = AccountState::new(
2130 AccountId::from(account_id),
2131 account_type,
2132 vec![AccountBalance::new(
2133 Money::from(total),
2134 Money::from(locked),
2135 Money::from(free),
2136 )],
2137 vec![],
2138 true,
2139 UUID4::new(),
2140 UnixNanos::from(index as u64 + 1),
2141 UnixNanos::from(index as u64 + 1),
2142 Some(Currency::USD()),
2143 );
2144 expected.push((event.event_id, history_len));
2145 if account_id == "SIM-001" {
2146 expected_scoped.push(event.event_id);
2147 }
2148 let expected_log = format!("Updated {event}");
2149 AsyncRunner::handle_exec_event(ExecutionEvent::Account(event));
2150 assert_eq!(
2151 LOGS.0
2152 .lock()
2153 .unwrap()
2154 .iter()
2155 .filter(|line| **line == expected_log)
2156 .count(),
2157 1
2158 );
2159 }
2160 assert_eq!(*observed.borrow(), expected);
2161 assert_eq!(*scoped.borrow(), expected_scoped);
2162 let cache = cache.borrow();
2163 assert_eq!(
2164 cache
2165 .account(&AccountId::from("SIM-001"))
2166 .unwrap()
2167 .events()
2168 .len(),
2169 3
2170 );
2171 assert_eq!(
2172 cache
2173 .account(&AccountId::from("SIM-002"))
2174 .unwrap()
2175 .events()
2176 .len(),
2177 1
2178 );
2179 msgbus::unsubscribe_account_state("events.account.*".into(), &handler);
2180 msgbus::unsubscribe_account_state("events.account.SIM-001".into(), &scoped_handler);
2181 }
2182
2183 #[cfg(feature = "node")]
2184 #[rstest]
2185 #[case::cash_update(AccountType::Cash, true)]
2186 #[case::wallet_initial(AccountType::Wallet, false)]
2187 #[case::wallet_update(AccountType::Wallet, true)]
2188 fn test_rejected_account_event_is_not_published(
2189 #[case] account_type: AccountType,
2190 #[case] existing_account: bool,
2191 ) {
2192 use nautilus_common::msgbus::{MessageBus, TypedHandler};
2193 use nautilus_model::types::{AccountBalance, Currency};
2194 use nautilus_portfolio::Portfolio;
2195
2196 msgbus::set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
2197 let cache = Rc::new(RefCell::new(Cache::default()));
2198 let mut portfolio = Portfolio::new(
2199 Rc::new(RefCell::new(VirtualClock::new())),
2200 cache.clone(),
2201 None,
2202 );
2203 let state = |amount: &str| {
2204 AccountState::new(
2205 AccountId::from("SIM-001"),
2206 account_type,
2207 vec![AccountBalance::new(
2208 Money::from(amount),
2209 Money::zero(Currency::USD()),
2210 Money::from(amount),
2211 )],
2212 vec![],
2213 true,
2214 UUID4::new(),
2215 UnixNanos::from(1),
2216 UnixNanos::from(1),
2217 (account_type != AccountType::Wallet).then_some(Currency::USD()),
2218 )
2219 };
2220 let initial = state("100.00 USD");
2221
2222 if existing_account {
2223 portfolio.update_account(&initial);
2224 }
2225 let received = Rc::new(RefCell::new(Vec::new()));
2226 let handler = TypedHandler::from({
2227 let received = received.clone();
2228 move |event: &AccountState| received.borrow_mut().push(event.clone())
2229 });
2230 msgbus::subscribe_account_state("events.account.*".into(), handler.clone(), None);
2231 AsyncRunner::handle_exec_event(ExecutionEvent::Account(state("-1.00 USD")));
2232 assert!(received.borrow().is_empty());
2233 let cache = cache.borrow();
2234
2235 if existing_account {
2236 assert_eq!(
2237 cache.account(&initial.account_id).unwrap().last_event(),
2238 Some(initial)
2239 );
2240 } else {
2241 assert!(cache.account(&initial.account_id).is_none());
2242 }
2243 msgbus::unsubscribe_account_state("events.account.*".into(), &handler);
2244 }
2245
2246 #[rstest]
2247 #[case::risk(MessagingSwitchboard::risk_engine_execute())]
2248 #[case::execution(MessagingSwitchboard::exec_engine_execute())]
2249 #[tokio::test(flavor = "current_thread")]
2250 async fn test_runner_dispatches_commands_and_data_before_handle_stop(
2251 #[case] target: MStr<Endpoint>,
2252 ) {
2253 let count = 10_u64;
2254 let mut runner = AsyncRunner::new();
2255 let handle = runner.handle();
2256 let commands = Rc::new(RefCell::new(Vec::new()));
2257 let data = Rc::new(RefCell::new(Vec::new()));
2258 let (observed_tx, mut observed_rx) = tokio::sync::mpsc::unbounded_channel();
2259
2260 for endpoint in [
2261 MessagingSwitchboard::risk_engine_execute(),
2262 MessagingSwitchboard::exec_engine_execute(),
2263 ] {
2264 let observed_commands = commands.clone();
2265 let commands_tx = observed_tx.clone();
2266 msgbus::register_trading_command_endpoint(
2267 endpoint,
2268 TypedIntoHandler::from(move |command| {
2269 observed_commands.borrow_mut().push((endpoint, command));
2270 commands_tx.send(()).unwrap();
2271 }),
2272 );
2273 }
2274
2275 let observed_data = data.clone();
2276 msgbus::register_data_endpoint(
2277 MessagingSwitchboard::data_engine_process_data(),
2278 TypedIntoHandler::from(move |event| {
2279 observed_data.borrow_mut().push(event);
2280 observed_tx.send(()).unwrap();
2281 }),
2282 );
2283
2284 let mut expected_commands = Vec::new();
2285 let mut expected_data = Vec::new();
2286
2287 for index in 1..=count {
2288 let command = TradingCommand::CancelAllOrders(CancelAllOrders::new(
2289 TraderId::from("TRADER-001"),
2290 Some(ClientId::from("SIM")),
2291 StrategyId::from(format!("S-{index:03}")),
2292 InstrumentId::from("EUR/USD.SIM"),
2293 Some(OrderSide::Sell),
2294 UUID4::new(),
2295 UnixNanos::from(index),
2296 None,
2297 Some(UUID4::new()),
2298 ));
2299 runner
2300 .exec_cmd_tx
2301 .send(TradingCommandMessage::new(target, command.clone()).into())
2302 .unwrap();
2303 expected_commands.push((target, command));
2304 let mut quote = test_quote();
2305 quote.ts_init = UnixNanos::from(index);
2306 let event = Data::Quote(quote);
2307 runner
2308 .data_evt_tx
2309 .send(DataEvent::Data(event.clone()).into())
2310 .unwrap();
2311 expected_data.push(event);
2312 }
2313
2314 let driver = async {
2315 for _ in 0..count * 2 {
2316 observed_rx.recv().await.unwrap();
2317 }
2318
2319 handle.stop();
2320 };
2321
2322 let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
2323 tokio::join!(runner.run(), driver)
2324 })
2325 .await
2326 .expect("runner dispatches every message and stops");
2327
2328 result.unwrap();
2329
2330 assert_eq!(&*commands.borrow(), &expected_commands);
2331 assert_eq!(
2332 serde_json::to_value(&*data.borrow()).unwrap(),
2333 serde_json::to_value(expected_data).unwrap()
2334 );
2335 assert!(runner.channels.exec_cmd_rx.is_empty());
2336 assert!(runner.channels.data_evt_rx.is_empty());
2337 assert!(observed_rx.try_recv().is_err());
2338 }
2339
2340 #[tokio::test(flavor = "current_thread")]
2341 async fn test_runner_dispatches_every_concurrent_producer_in_order() {
2342 let mut runner = AsyncRunner::new();
2343 let handle = runner.handle();
2344 let (observed_tx, mut observed_rx) = tokio::sync::mpsc::unbounded_channel();
2345 msgbus::register_data_endpoint(
2346 MessagingSwitchboard::data_engine_process_data(),
2347 TypedIntoHandler::from(move |event| observed_tx.send(event).unwrap()),
2348 );
2349 let mut producers = Vec::new();
2350
2351 for producer in 0..5_u64 {
2352 let sender = runner.data_evt_tx.clone();
2353
2354 producers.push(tokio::spawn(async move {
2355 for index in 0..20 {
2356 let mut quote = test_quote();
2357 quote.ts_init = UnixNanos::from(producer * 20 + index);
2358 sender
2359 .send(DataEvent::Data(Data::Quote(quote)).into())
2360 .unwrap();
2361 tokio::task::yield_now().await;
2362 }
2363 }));
2364 }
2365
2366 let driver = async {
2367 for producer in producers {
2368 producer.await.unwrap();
2369 }
2370
2371 let mut observed = vec![Vec::new(); 5];
2372
2373 for _ in 0..100 {
2374 let Data::Quote(quote) = observed_rx.recv().await.unwrap() else {
2375 panic!("expected a quote");
2376 };
2377
2378 let timestamp = quote.ts_init.as_u64();
2379 let mut expected = test_quote();
2380 expected.ts_init = quote.ts_init;
2381 assert_eq!(quote, expected);
2382 observed[usize::try_from(timestamp / 20).unwrap()].push(timestamp % 20);
2383 }
2384
2385 handle.stop();
2386 observed
2387 };
2388
2389 let (result, observed) = tokio::time::timeout(Duration::from_secs(5), async {
2390 tokio::join!(runner.run(), driver)
2391 })
2392 .await
2393 .expect("runner receives every producer's messages and stops");
2394
2395 result.unwrap();
2396
2397 assert_eq!(observed, vec![(0..20_u64).collect::<Vec<_>>(); 5]);
2398 assert!(runner.channels.data_evt_rx.is_empty());
2399 assert!(observed_rx.try_recv().is_err());
2400 }
2401
2402 #[rstest]
2403 fn test_new_does_not_bind_tls() {
2404 std::thread::spawn(|| {
2405 let _runner = AsyncRunner::new();
2406 assert!(try_get_time_event_sender().is_none());
2407 assert!(try_get_system_command_sender().is_none());
2408 assert!(try_get_system_event_sender().is_none());
2409 assert!(try_get_trading_cmd_sender().is_none());
2410 })
2411 .join()
2412 .unwrap();
2413 }
2414
2415 #[rstest]
2416 fn test_bind_senders_routes_to_runner_channels() {
2417 std::thread::spawn(|| {
2418 let mut runner = AsyncRunner::new();
2419 runner.bind_senders();
2420
2421 get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2422 SubscribeCustomData {
2423 client_id: Some(ClientId::from("TEST")),
2424 venue: None,
2425 data_type: DataType::new("test", None, None),
2426 command_id: UUID4::new(),
2427 ts_init: UnixNanos::default(),
2428 correlation_id: None,
2429 params: None,
2430 },
2431 )));
2432
2433 assert!(runner.channels.data_cmd_rx.try_recv().is_ok());
2434
2435 get_trading_cmd_sender().execute(TradingCommandMessage::new(
2436 MessagingSwitchboard::exec_engine_execute(),
2437 TradingCommand::CancelAllOrders(CancelAllOrders::new(
2438 TraderId::from("TRADER-001"),
2439 None,
2440 StrategyId::from("S-001"),
2441 InstrumentId::from("EUR/USD.SIM"),
2442 Some(OrderSide::Buy),
2443 UUID4::new(),
2444 UnixNanos::default(),
2445 None,
2446 None, )),
2448 ));
2449 assert!(runner.channels.exec_cmd_rx.try_recv().is_ok());
2450
2451 let event = TimeEvent::new(
2452 Ustr::from("test"),
2453 UUID4::new(),
2454 UnixNanos::from(1),
2455 UnixNanos::from(2),
2456 );
2457 let callback = TimeEventCallback::from(|_: TimeEvent| {});
2458 get_time_event_sender().send(TimeEventMessage::new(event, callback));
2459 assert!(runner.channels.time_evt_rx.try_recv().is_ok());
2460
2461 get_system_event_sender().send(test_system_event()).unwrap();
2462 assert_eq!(
2463 runner
2464 .channels
2465 .system_evt_rx
2466 .try_recv()
2467 .unwrap()
2468 .dispatch(|value| value),
2469 test_system_event()
2470 );
2471
2472 get_system_command_sender()
2473 .send(test_system_command())
2474 .unwrap();
2475 assert_eq!(
2476 runner
2477 .channels
2478 .system_cmd_rx
2479 .try_recv()
2480 .unwrap()
2481 .dispatch(|value| value),
2482 test_system_command()
2483 );
2484
2485 get_data_event_sender()
2486 .send(DataEvent::Data(Data::Quote(test_quote())))
2487 .unwrap();
2488 assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2489
2490 let account = AccountState::new(
2491 AccountId::from("SIM-001"),
2492 AccountType::Cash,
2493 vec![],
2494 vec![],
2495 true,
2496 UUID4::new(),
2497 UnixNanos::from(1),
2498 UnixNanos::from(2),
2499 None,
2500 );
2501 get_exec_event_sender()
2502 .send(ExecutionEvent::Account(account))
2503 .unwrap();
2504 assert!(runner.channels.exec_evt_rx.try_recv().is_ok());
2505 })
2506 .join()
2507 .unwrap();
2508 }
2509
2510 #[cfg(feature = "node")]
2511 #[rstest]
2512 fn test_drain_pending_system_events_keeps_data_events_separate() {
2513 std::thread::spawn(|| {
2514 let mut runner = AsyncRunner::new();
2515 runner.bind_senders();
2516 let system_event = test_system_event();
2517
2518 get_system_event_sender().send(system_event).unwrap();
2519 get_data_event_sender()
2520 .send(DataEvent::Data(Data::Quote(test_quote())))
2521 .unwrap();
2522
2523 let system_events = runner
2524 .drain_pending_system_events()
2525 .into_iter()
2526 .map(|message| message.dispatch(|value| value))
2527 .collect::<Vec<_>>();
2528
2529 assert_eq!(system_events, vec![system_event]);
2530 assert!(runner.channels.system_evt_rx.try_recv().is_err());
2531 assert!(runner.channels.data_evt_rx.try_recv().is_ok());
2532 })
2533 .join()
2534 .unwrap();
2535 }
2536
2537 #[cfg(feature = "node")]
2538 #[rstest]
2539 fn test_drain_pending_system_commands_keeps_events_separate() {
2540 std::thread::spawn(|| {
2541 let mut runner = AsyncRunner::new();
2542 runner.bind_senders();
2543 let system_command = test_system_command();
2544
2545 get_system_command_sender().send(system_command).unwrap();
2546 get_system_event_sender().send(test_system_event()).unwrap();
2547
2548 let system_commands = runner
2549 .drain_pending_system_commands()
2550 .into_iter()
2551 .map(|message| message.dispatch(|value| value))
2552 .collect::<Vec<_>>();
2553
2554 assert_eq!(system_commands, vec![system_command]);
2555 assert!(runner.channels.system_cmd_rx.try_recv().is_err());
2556 assert!(runner.channels.system_evt_rx.try_recv().is_ok());
2557 })
2558 .join()
2559 .unwrap();
2560 }
2561
2562 #[rstest]
2563 fn test_bind_senders_reclaims_tls_from_previous_runner() {
2564 std::thread::spawn(|| {
2565 let mut runner1 = AsyncRunner::new();
2566 runner1.bind_senders();
2567
2568 let mut runner2 = AsyncRunner::new();
2569 runner2.bind_senders();
2570
2571 get_data_cmd_sender().execute(DataCommand::Subscribe(SubscribeCommand::Data(
2572 SubscribeCustomData {
2573 client_id: Some(ClientId::from("TEST")),
2574 venue: None,
2575 data_type: DataType::new("test", None, None),
2576 command_id: UUID4::new(),
2577 ts_init: UnixNanos::default(),
2578 correlation_id: None,
2579 params: None,
2580 },
2581 )));
2582
2583 assert!(runner2.channels.data_cmd_rx.try_recv().is_ok());
2584 assert!(runner1.channels.data_cmd_rx.try_recv().is_err());
2585 })
2586 .join()
2587 .unwrap();
2588 }
2589
2590 #[cfg(feature = "defi")]
2591 #[rstest]
2592 fn test_handle_data_event_routes_pool_snapshot_response_to_data_engine() {
2593 use nautilus_common::{
2594 messages::defi::PoolSnapshotResponse, msgbus::ShareableMessageHandler,
2595 };
2596 use nautilus_model::defi::{
2597 data::block::BlockPosition,
2598 pool_analysis::snapshot::{PoolAnalytics, PoolSnapshot, PoolState},
2599 };
2600
2601 let received = Rc::new(RefCell::new(Vec::new()));
2602 let sink = received.clone();
2603 msgbus::register_any(
2604 MessagingSwitchboard::data_engine_process(),
2605 ShareableMessageHandler::from_typed(move |response: &PoolSnapshotResponse| {
2606 sink.borrow_mut().push(response.correlation_id);
2607 }),
2608 );
2609
2610 let correlation_id = UUID4::new();
2611
2612 let snapshot = PoolSnapshot::new(
2613 InstrumentId::from("0x11b815efB8f581194ae79006d24E0d814B7697F6.Arbitrum:UniswapV3"),
2614 PoolState::default(),
2615 Vec::new(),
2616 Vec::new(),
2617 PoolAnalytics::default(),
2618 BlockPosition::new(1000, "0x0".to_string(), 0, 0),
2619 UnixNanos::default(),
2620 UnixNanos::default(),
2621 );
2622
2623 AsyncRunner::handle_data_event(DataEvent::PoolSnapshotResponse(PoolSnapshotResponse::new(
2624 correlation_id,
2625 snapshot,
2626 )));
2627
2628 assert_eq!(*received.borrow(), vec![correlation_id]);
2629 }
2630}