1use std::{
85 any::Any,
86 cell::RefCell,
87 collections::VecDeque,
88 fmt::Debug,
89 rc::{Rc, Weak},
90 time::Duration,
91};
92
93use anyhow::Context;
94use nautilus_common::{
95 actor::{self, Actor, DataActor, DataActorNative},
96 cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
97 clients::ExecutionClient,
98 component::Component,
99 enums::{Environment, LogColor},
100 live::{dispatch::DispatchMessage, dst},
101 log_info,
102 messages::{
103 DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
104 data::DataCommand,
105 execution::TradingCommand,
106 system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
107 },
108 msgbus::{self, BusMessage, MessagingSwitchboard},
109 runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
110};
111use nautilus_core::{
112 UUID4,
113 datetime::{mins_to_secs, secs_to_nanos_unchecked},
114};
115#[cfg(test)]
116use nautilus_model::reports::OrderStatusReport;
117use nautilus_model::{
118 events::OrderEventAny,
119 identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
120 orders::Order,
121};
122use nautilus_network::mode::ReconnectRequestOutcome;
123#[cfg(feature = "python")]
124use nautilus_system::trader::Trader;
125use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
126use nautilus_trading::{
127 ExecutionAlgorithm, ExecutionAlgorithmNative,
128 strategy::{Strategy, StrategyNative},
129};
130use tabled::{builder::Builder, settings::Style};
131
132use crate::{
133 dispatch::drain_callbacks,
134 execution::{
135 client::LiveExecutionClient,
136 manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
137 },
138 runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
139 socket::{SocketReconnectLookup, SocketReconnectRegistry},
140};
141
142pub mod builder;
143pub mod config;
144
145#[cfg(feature = "plugin")]
146pub mod plugin;
147
148mod metrics;
149mod queue;
150mod reconciliation;
151mod state;
152
153use builder::ExternalMessageBusIngress;
154pub use builder::LiveNodeBuilder;
155use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
156pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
157use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
158use queue::{QueueMonitor, QueueStateTransition};
159use reconciliation::{
160 OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
161 PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
162 ReportTaskOutcome, TargetedOrderReportTask,
163};
164use state::{EngineConnectionStatus, RunningTransition};
165pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
166
167const DISPATCHES_PER_YIELD: usize = 64;
173
174thread_local! {
175 static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
176}
177
178type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
179
180struct StreamProcessor(Box<StreamProcessorCallback>);
181
182impl Debug for StreamProcessor {
183 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
184 f.debug_struct(stringify!(StreamProcessor)).finish()
185 }
186}
187
188#[derive(Debug)]
197pub struct LiveNode {
198 kernel: NautilusKernel,
199 runner: Option<AsyncRunner>,
200 config: LiveNodeConfig,
201 handle: LiveNodeHandle,
202 exec_manager: ExecutionManager,
203 exec_clients: Vec<LiveExecutionClient>,
204 socket_registry: SocketReconnectRegistry,
205 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
206 external_msgbus: Option<ExternalMessageBusIngress>,
207 stream_processors: Vec<StreamProcessor>,
208 shutdown_deadline: Option<dst::time::Instant>,
209 #[cfg(feature = "plugin")]
210 plugins: plugin::NodePlugins,
211 _thread_owner: Rc<()>,
212}
213
214impl LiveNode {
215 #[must_use]
219 #[allow(
220 clippy::too_many_arguments,
221 reason = "builder components have distinct lifecycle roles"
222 )]
223 pub(crate) fn new_from_builder(
224 kernel: NautilusKernel,
225 runner: AsyncRunner,
226 config: LiveNodeConfig,
227 handle: LiveNodeHandle,
228 mut exec_manager: ExecutionManager,
229 exec_clients: Vec<LiveExecutionClient>,
230 socket_registry: SocketReconnectRegistry,
231 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
232 external_msgbus: Option<ExternalMessageBusIngress>,
233 thread_owner: Rc<()>,
234 ) -> Self {
235 for client in &exec_clients {
236 exec_manager.register_submission_retention(client);
237 }
238
239 Self {
240 kernel,
241 runner: Some(runner),
242 config,
243 handle,
244 exec_manager,
245 exec_clients,
246 socket_registry,
247 cache_database_factory,
248 external_msgbus,
249 stream_processors: Vec::new(),
250 shutdown_deadline: None,
251 #[cfg(feature = "plugin")]
252 plugins: plugin::NodePlugins,
253 _thread_owner: thread_owner,
254 }
255 }
256
257 pub fn builder(
263 trader_id: TraderId,
264 environment: Environment,
265 ) -> anyhow::Result<LiveNodeBuilder> {
266 LiveNodeBuilder::new(trader_id, environment)
267 }
268
269 pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
280 let thread_owner = Self::acquire_thread()?;
281 let config = config.unwrap_or_default();
282 validate_live_environment(config.environment())?;
283
284 config.validate_runtime_support()?;
285
286 if config.event_store.is_some() {
287 anyhow::bail!(
288 "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
289 use LiveNodeBuilder::with_event_store(...) instead"
290 );
291 }
292
293 let handle = LiveNodeHandle::new();
294 let runner = AsyncRunner::new();
295 runner.bind_senders_for_node(handle.clone());
296
297 let kernel = NautilusKernel::new(name, config.clone())?;
298 #[cfg(feature = "python")]
299 if let Some(controller) = config.controller.as_ref() {
300 Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
301 }
302
303 #[cfg(not(feature = "python"))]
304 if let Some(controller) = config.controller.as_ref() {
305 anyhow::bail!(
306 "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
307 controller.controller_path
308 );
309 }
310
311 let exec_manager_config =
312 ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
313
314 let exec_manager = ExecutionManager::new(
315 kernel.clock.clone(),
316 kernel.cache.clone(),
317 exec_manager_config,
318 )?;
319
320 let node = Self {
321 kernel,
322 runner: Some(runner),
323 config,
324 handle,
325 exec_manager,
326 exec_clients: Vec::new(),
327 socket_registry: SocketReconnectRegistry::default(),
328 cache_database_factory: None,
329 external_msgbus: None,
330 stream_processors: Vec::new(),
331 shutdown_deadline: None,
332 #[cfg(feature = "plugin")]
333 plugins: plugin::NodePlugins,
334 _thread_owner: thread_owner,
335 };
336
337 node.load_configured_plugins()?;
338
339 log::info!("LiveNode built successfully with kernel config");
340
341 Ok(node)
342 }
343
344 fn acquire_thread() -> anyhow::Result<Rc<()>> {
345 NODE_THREAD_OWNER.with(|slot| {
346 let mut owner = slot.borrow_mut();
347 anyhow::ensure!(
348 owner.upgrade().is_none(),
349 "A LiveNode already exists or is being built on this thread; drop it before building another"
350 );
351 let token = Rc::new(());
352 *owner = Rc::downgrade(&token);
353 Ok(token)
354 })
355 }
356
357 pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
363 if self.config.plugins.is_empty() {
364 return Ok(());
365 }
366
367 anyhow::bail!(
368 "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
369 )
370 }
371
372 #[expect(
378 clippy::needless_pass_by_value,
379 reason = "signature mirrors the host-enabled API"
380 )]
381 pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
382 #[cfg(feature = "plugin")]
383 config.validate_runtime_support(self.config.plugins.len())?;
384 #[cfg(not(feature = "plugin"))]
385 let _ = config;
386
387 anyhow::bail!(
388 "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
389 )
390 }
391
392 #[must_use]
394 pub fn handle(&self) -> LiveNodeHandle {
395 self.handle.clone()
396 }
397
398 pub fn add_stream_processor<F>(&mut self, callback: F)
415 where
416 F: Fn(&dyn Any) + 'static,
417 {
418 self.add_stream_processor_with_mapping(move |message, _| {
419 callback(message);
420 Ok(())
421 });
422 }
423
424 pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
425 where
426 F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
427 {
428 self.stream_processors
429 .push(StreamProcessor(Box::new(callback)));
430 }
431
432 pub async fn start(&mut self) -> anyhow::Result<()> {
443 if self.state().is_running() {
444 anyhow::bail!("Already running");
445 }
446
447 if self.external_msgbus.is_some() {
448 log::warn!(
449 "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
450 );
451 }
452
453 self.prepare_cache().await?;
454
455 if let Some(runner) = self.runner.as_ref() {
456 runner.bind_senders_for_node(self.handle.clone());
457 }
458
459 self.handle.set_starting();
460
461 self.kernel.reset_shutdown_flag();
462 self.kernel.start_async().await;
463
464 if self.kernel.is_event_store_replay() {
465 log::info!(
466 "Event-store replay loaded; skipping live client connection and reconciliation",
467 );
468
469 if !self.finish_startup_replay().await? {
470 return Ok(());
471 }
472
473 return Ok(());
474 }
475
476 if self.kernel.is_event_store_replay_configured() {
477 self.abort_startup("Event-store replay did not start")
478 .await?;
479 return Ok(());
480 }
481
482 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
483
484 if let Err(e) = self.connect_data_phase(connection_deadline).await {
486 return self
487 .abort_startup_with_error("Data client connection timed out", e)
488 .await;
489 }
490
491 let (startup_system_events, startup_system_commands) =
492 if let Some(runner) = self.runner.as_mut() {
493 runner.flush_pending_data();
494 (
495 runner.drain_pending_system_events(),
496 runner.drain_pending_system_commands(),
497 )
498 } else {
499 (Vec::new(), Vec::new())
500 };
501
502 if let Err(e) = self.connect_exec_clients(connection_deadline).await {
503 return self
504 .abort_startup_with_error("Execution client connection timed out", e)
505 .await;
506 }
507
508 if let Some(reason) = self.startup_abort_reason() {
509 self.abort_startup(reason).await?;
510 return Ok(());
511 }
512
513 match self.await_engines_connected(connection_deadline).await {
514 EngineConnectionStatus::Connected => {}
515 EngineConnectionStatus::TimedOut => {
516 return self
517 .abort_startup_with_error(
518 "Engine readiness timed out",
519 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
520 )
521 .await;
522 }
523 EngineConnectionStatus::StopRequested => {
524 self.abort_startup("Stop signal received during startup")
525 .await?;
526 return Ok(());
527 }
528 EngineConnectionStatus::ShutdownRequested => {
529 self.abort_startup("Shutdown signal received during startup")
530 .await?;
531 return Ok(());
532 }
533 }
534
535 if let Err(e) = self.perform_startup_reconciliation().await {
536 if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
537 anyhow::bail!(
538 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
539 );
540 }
541
542 return Err(e);
543 }
544
545 if let Some(reason) = self.startup_abort_reason() {
546 self.abort_startup(reason).await?;
547 return Ok(());
548 }
549
550 if let Err(e) = self.kernel.start_trader() {
551 return self.abort_after_trader_start_failure(e).await;
552 }
553
554 #[cfg(feature = "plugin")]
555 if let Err(e) = self.plugins.start_controllers() {
556 return self.abort_after_trader_start_failure(e).await;
557 }
558
559 self.process_system_events(startup_system_events);
560 self.process_system_commands(startup_system_commands);
561
562 if !self.finish_startup_trader(None).await? {
563 return Ok(());
564 }
565
566 Ok(())
567 }
568
569 pub async fn stop(&mut self) -> anyhow::Result<()> {
579 if !self.state().is_running() {
580 anyhow::bail!("Not running");
581 }
582
583 self.handle.set_shutting_down();
584
585 #[cfg(feature = "plugin")]
586 let controller_stop_result = self.plugins.stop_controllers();
587 #[cfg(not(feature = "plugin"))]
588 let controller_stop_result: anyhow::Result<()> = Ok(());
589
590 self.kernel.stop_trader();
591 let delay = self.kernel.delay_post_stop();
592 log::info!("Awaiting residual events ({delay:?})...");
593
594 let residual_events = self.process_runner_for(delay).await;
595 if residual_events > 0 {
596 log::debug!("Processed {residual_events} residual events during shutdown");
597 }
598
599 let stop_result = self.finalize_stop().await;
600 let drained_events = self.drain_runner_pending();
601 if drained_events > 0 {
602 log::info!("Drained {drained_events} remaining events during shutdown");
603 }
604
605 match (controller_stop_result, stop_result) {
606 (Ok(()), Ok(())) => Ok(()),
607 (Err(controller_err), Ok(())) => Err(controller_err),
608 (Ok(()), Err(stop_err)) => Err(stop_err),
609 (Err(controller_err), Err(stop_err)) => {
610 log::error!("Error stopping plug-in controllers: {controller_err}");
611 Err(stop_err)
612 }
613 }
614 }
615
616 pub fn dispose(&mut self) {
621 self.close_external_ingress();
622 self.handle.set_stopped();
623 self.kernel.dispose();
624 drop(self.runner.take());
625
626 if let Some(e) = actor::callback_failure() {
627 log::error!("Callback dispatch failed before disposal cleanup: {e}");
628 }
629
630 if let Err(e) = actor::clear_callbacks() {
631 log::error!("Failed to clear callback dispatch during disposal: {e}");
632 }
633 }
634
635 async fn process_runner_for(&mut self, duration: Duration) -> usize {
636 let Some(mut runner) = self.runner.take() else {
637 dst::time::sleep(duration).await;
638 return 0;
639 };
640
641 runner.bind_senders_for_node(self.handle.clone());
642 let deadline = dst::time::Instant::now() + duration;
643 let mut processed = 0;
644
645 loop {
646 tokio::select! {
647 biased;
648
649 () = dst::time::sleep_until(deadline) => break,
650 event = runner.recv() => {
651 let Some(event) = event else {
652 dst::time::sleep_until(deadline).await;
653 break;
654 };
655
656 self.process_runner_event(event);
657 processed += 1;
658 }
659 }
660 }
661
662 self.runner = Some(runner);
663 processed
664 }
665
666 fn drain_runner_pending(&mut self) -> usize {
667 let Some(mut runner) = self.runner.take() else {
668 return 0;
669 };
670
671 let processed = runner.poll_pending(|event| self.process_runner_event(event));
672 self.runner = Some(runner);
673 processed
674 }
675
676 fn process_runner_event(&mut self, event: PendingRunnerEvent) {
677 match event {
678 PendingRunnerEvent::TimeEvent(message) => {
679 let _ = AsyncRunner::handle_time_event(message);
680 }
681 PendingRunnerEvent::SystemEvent(event) => {
682 event.dispatch(|event| self.process_system_event(event));
683 }
684 PendingRunnerEvent::SystemCommand(command) => {
685 command.dispatch(|command| self.process_system_command(command));
686 }
687 PendingRunnerEvent::ExecEvent(event) => {
688 event.dispatch(|event| self.process_exec_event(event));
689 }
690 PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
691 PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
692 PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
693 }
694 }
695
696 fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
697 for event in events {
698 event.dispatch(|event| self.process_system_event(event));
699 }
700 }
701
702 fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
703 for command in commands {
704 command.dispatch(|command| self.process_system_command(command));
705 }
706 }
707
708 fn process_system_command(&self, command: SystemCommand) {
709 match command {
710 SystemCommand::ReconnectSocket(command) => {
711 self.process_socket_reconnect(command);
712 }
713 }
714 }
715
716 fn process_socket_reconnect(&self, command: ReconnectSocket) {
717 let outcome = if command.trader_id == self.config.trader_id {
718 Self::request_socket_reconnect(
719 self.socket_registry
720 .get(command.client_id, command.endpoint),
721 )
722 } else {
723 SocketReconnectDispatchOutcome::InvalidTrader
724 };
725
726 if outcome == SocketReconnectDispatchOutcome::Accepted {
727 log::info!(
728 "Requested socket reconnect for client {} endpoint {}",
729 command.client_id,
730 command.endpoint
731 );
732 } else {
733 log::warn!(
734 "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
735 command.client_id,
736 command.endpoint
737 );
738 }
739 }
740
741 fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
742 match lookup {
743 SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
744 ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
745 ReconnectRequestOutcome::AlreadyReconnecting => {
746 SocketReconnectDispatchOutcome::AlreadyReconnecting
747 }
748 ReconnectRequestOutcome::Disconnected => {
749 SocketReconnectDispatchOutcome::Disconnected
750 }
751 ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
752 ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
753 },
754 SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
755 SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
756 SocketReconnectLookup::EndpointNotFound => {
757 SocketReconnectDispatchOutcome::UnknownEndpoint
758 }
759 SocketReconnectLookup::AmbiguousEndpoint => {
760 SocketReconnectDispatchOutcome::AmbiguousEndpoint
761 }
762 }
763 }
764
765 fn process_system_event(&self, event: SystemEvent) {
766 match event {
767 SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
768 }
769 }
770
771 fn publish_socket_state_change(&self, change: SocketStateChange) {
772 let timestamp = self.kernel.generate_timestamp_ns();
773
774 let event = SocketStateChanged::new(
775 self.config.trader_id,
776 change.client_id,
777 change.venue,
778 change.endpoint,
779 change.state,
780 UUID4::new(),
781 timestamp,
782 timestamp,
783 );
784
785 msgbus::publish_any(
786 MessagingSwitchboard::socket_state_changed_topic(
787 event.client_id,
788 event.endpoint.as_str(),
789 ),
790 event.as_any(),
791 );
792 }
793
794 async fn await_engines_connected(
798 &self,
799 deadline: dst::time::Instant,
800 ) -> EngineConnectionStatus {
801 log::info!(
802 "Awaiting engine connections ({:?} timeout)...",
803 self.config.timeout_connection
804 );
805
806 let interval = Duration::from_millis(100);
807
808 loop {
809 if self.handle.should_stop() {
810 log::warn!("Stop signal received, aborting connection wait");
811 return EngineConnectionStatus::StopRequested;
812 }
813
814 if self.kernel.is_shutdown_requested() {
815 log::warn!("Shutdown signal received, aborting connection wait");
816 return EngineConnectionStatus::ShutdownRequested;
817 }
818
819 if self.kernel.check_engines_connected() {
820 log::info!("All engine clients connected");
821 return EngineConnectionStatus::Connected;
822 }
823
824 let now = dst::time::Instant::now();
825 if now >= deadline {
826 break;
827 }
828
829 dst::time::sleep(interval.min(deadline - now)).await;
830 }
831
832 self.log_connection_status();
833 EngineConnectionStatus::TimedOut
834 }
835
836 async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
840 log::info!(
841 "Awaiting engine disconnections ({:?} timeout)...",
842 self.config.timeout_disconnection
843 );
844
845 let timeout = self.config.timeout_disconnection;
846 let interval = Duration::from_millis(100);
847
848 loop {
849 if self.kernel.check_engines_disconnected() {
850 log::info!("All engine clients disconnected");
851 return Ok(());
852 }
853
854 let now = dst::time::Instant::now();
855 if now >= deadline {
856 break;
857 }
858
859 dst::time::sleep(interval.min(deadline - now)).await;
860 }
861
862 log::error!(
863 "Timed out ({:?}) waiting for engines to disconnect\n\
864 DataEngine.check_disconnected() == {}\n\
865 ExecEngine.check_disconnected() == {}",
866 timeout,
867 self.kernel.data_engine().check_disconnected(),
868 self.kernel.exec_engine().borrow().check_disconnected(),
869 );
870 anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
871 }
872
873 fn log_connection_status(&self) {
874 let data_status = self.kernel.data_client_connection_status();
875 let exec_status = self.kernel.exec_client_connection_status();
876
877 let mut rows: Vec<ClientStatus> = Vec::new();
878
879 for (client_id, connected) in data_status {
880 rows.push(ClientStatus {
881 client: client_id.to_string(),
882 client_type: "Data",
883 connected,
884 });
885 }
886
887 for (client_id, connected) in exec_status {
888 rows.push(ClientStatus {
889 client: client_id.to_string(),
890 client_type: "Execution",
891 connected,
892 });
893 }
894
895 let table = render_client_statuses(rows);
896
897 log::warn!(
898 "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
899 DataEngine.check_connected() == {}\n\
900 ExecEngine.check_connected() == {}",
901 self.config.timeout_connection,
902 self.kernel.data_engine().check_connected(),
903 self.kernel.exec_engine().borrow().check_connected(),
904 );
905 }
906
907 #[expect(clippy::await_holding_refcell_ref)] async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
917 if !self.config.exec_engine.reconciliation {
918 log::info!("Startup reconciliation disabled");
919 self.kernel
920 .portfolio
921 .borrow_mut()
922 .initialize_wallet_orders()?;
923 return Ok(());
924 }
925
926 log_info!(
927 "Starting execution state reconciliation...",
928 color = LogColor::Blue
929 );
930
931 let lookback_mins = self
932 .config
933 .exec_engine
934 .reconciliation_lookback_mins
935 .map(u64::from);
936
937 let timeout = self.config.timeout_reconciliation;
938 let start = dst::time::Instant::now();
939 let client_ids = self.kernel.exec_engine.borrow().client_ids();
940
941 for client_id in client_ids {
942 let elapsed = start.elapsed();
943 if elapsed >= timeout {
944 anyhow::bail!("Startup reconciliation timeout reached");
945 }
946
947 let remaining = timeout
948 .checked_sub(elapsed)
949 .expect("elapsed checked against reconciliation timeout");
950
951 log_info!(
952 "Requesting mass status from {}...",
953 client_id,
954 color = LogColor::Blue
955 );
956
957 let mass_status_result = dst::time::timeout(remaining, async {
958 self.kernel
959 .exec_engine
960 .borrow_mut()
961 .generate_mass_status(&client_id, lookback_mins)
962 .await
963 })
964 .await
965 .map_err(|_| {
966 anyhow::anyhow!(
967 "Startup reconciliation timeout reached while requesting mass status from {client_id}"
968 )
969 })?;
970
971 match mass_status_result {
972 Ok(Some(mass_status)) => {
973 log_info!(
974 "Reconciling ExecutionMassStatus for {}",
975 client_id,
976 color = LogColor::Blue
977 );
978
979 let result = self
980 .exec_manager
981 .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
982
983 anyhow::ensure!(
984 self.kernel
985 .exec_engine
986 .borrow()
987 .get_client(&client_id)
988 .is_some(),
989 "Execution client {client_id} disappeared during startup reconciliation",
990 );
991
992 anyhow::ensure!(
993 result.unresolved_positions.is_empty(),
994 "Unresolved positions during startup reconciliation for {client_id}: {}",
995 result.unresolved_positions.join("; "),
996 );
997
998 if result.events.is_empty() {
999 log_info!(
1000 "Reconciliation for {} succeeded",
1001 client_id,
1002 color = LogColor::Blue
1003 );
1004 } else {
1005 log::info!(
1006 color = LogColor::Blue as u8;
1007 "Reconciliation for {} processed {} events",
1008 client_id,
1009 result.events.len()
1010 );
1011 }
1012
1013 if !result.external_orders.is_empty() {
1015 let exec_engine = self.kernel.exec_engine.borrow();
1016
1017 let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1018 anyhow::anyhow!(
1019 "Execution client {client_id} disappeared during startup reconciliation"
1020 )
1021 })?;
1022
1023 for external in result.external_orders {
1024 source_client.register_external_order(
1025 external.client_order_id,
1026 external.venue_order_id,
1027 external.instrument_id,
1028 external.strategy_id,
1029 external.ts_init,
1030 );
1031 }
1032 }
1033 }
1034 Ok(None) => {
1035 log::warn!(
1036 "No mass status available from {client_id} \
1037 (likely adapter error when generating reports)"
1038 );
1039 }
1040 Err(e) => {
1041 return Err(e).context(format!("Failed to get mass status from {client_id}"));
1042 }
1043 }
1044 }
1045
1046 self.kernel.portfolio.borrow_mut().initialize_orders();
1047 self.kernel.portfolio.borrow_mut().initialize_positions();
1048 self.kernel
1049 .portfolio
1050 .borrow_mut()
1051 .initialize_wallet_orders()?;
1052
1053 let elapsed_secs = start.elapsed().as_secs_f64();
1054 log_info!(
1055 "Startup reconciliation completed in {:.2}s",
1056 elapsed_secs,
1057 color = LogColor::Blue
1058 );
1059
1060 Ok(())
1061 }
1062
1063 pub async fn run(&mut self) -> anyhow::Result<()> {
1086 self.run_with_mode(NodeRunMode::Owned).await
1087 }
1088
1089 pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1100 if self.state().is_running() {
1101 anyhow::bail!("Already running");
1102 }
1103
1104 if self.runner.is_none() {
1105 anyhow::bail!("Runner already consumed - run() called twice");
1106 }
1107
1108 self.prepare_cache().await?;
1109
1110 let Some(runner) = self.runner.take() else {
1111 anyhow::bail!("Runner already consumed - run() called twice");
1112 };
1113
1114 runner.bind_senders_for_node(self.handle.clone());
1115
1116 let AsyncRunnerChannels {
1117 mut time_evt_rx,
1118 mut system_evt_rx,
1119 mut system_cmd_rx,
1120 mut exec_evt_rx,
1121 mut exec_cmd_rx,
1122 mut data_evt_rx,
1123 mut data_cmd_rx,
1124 } = runner.take_channels();
1125
1126 log::info!("Event loop starting");
1127
1128 self.handle.set_starting();
1129 self.kernel.reset_shutdown_flag();
1130 self.kernel.start_async().await;
1131
1132 if self.kernel.is_event_store_replay() {
1133 log::info!(
1134 "Event-store replay loaded; skipping live client connection and reconciliation",
1135 );
1136
1137 if !self.finish_startup_replay().await? {
1138 return Ok(());
1139 }
1140
1141 return Ok(());
1142 }
1143
1144 if self.kernel.is_event_store_replay_configured() {
1145 self.abort_startup("Event-store replay did not start")
1146 .await?;
1147 return Ok(());
1148 }
1149
1150 let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1151 Ok(rx) => rx,
1152 Err(e) => {
1153 let result = self
1154 .abort_startup("External message bus ingress failed to start")
1155 .await;
1156 Self::drain_channels(
1157 &mut time_evt_rx,
1158 &mut system_evt_rx,
1159 &mut system_cmd_rx,
1160 &mut exec_evt_rx,
1161 &mut exec_cmd_rx,
1162 &mut data_evt_rx,
1163 &mut data_cmd_rx,
1164 );
1165 log::info!("Event loop stopped");
1166
1167 if let Err(finalize_err) = result {
1168 anyhow::bail!(
1169 "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1170 );
1171 }
1172
1173 return Err(e);
1174 }
1175 };
1176
1177 let stop_handle = self.handle.clone();
1178 let mut pending = PendingEvents::default();
1179 let mut startup_system_events = Vec::new();
1180 let mut startup_system_commands = Vec::new();
1181 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1182
1183 let mut pending_external = VecDeque::new();
1186 let data_connect_result = drive_with_event_buffering(
1187 drive_with_external_ingress(
1188 self.connect_data_phase(connection_deadline),
1189 &mut external_msgbus_rx,
1190 &mut pending_external,
1191 ),
1192 &mut pending,
1193 &mut time_evt_rx,
1194 &mut system_evt_rx,
1195 &mut system_cmd_rx,
1196 &mut exec_evt_rx,
1197 &mut exec_cmd_rx,
1198 &mut data_evt_rx,
1199 &mut data_cmd_rx,
1200 )
1201 .await;
1202
1203 self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1204
1205 if let Err(e) = data_connect_result {
1206 flush_all_pending(
1207 &mut pending,
1208 &mut time_evt_rx,
1209 &mut system_evt_rx,
1210 &mut system_cmd_rx,
1211 &mut exec_evt_rx,
1212 &mut exec_cmd_rx,
1213 &mut data_evt_rx,
1214 &mut data_cmd_rx,
1215 );
1216 let result = self
1217 .abort_startup_with_error("Data client connection timed out", e)
1218 .await;
1219 Self::drain_channels(
1220 &mut time_evt_rx,
1221 &mut system_evt_rx,
1222 &mut system_cmd_rx,
1223 &mut exec_evt_rx,
1224 &mut exec_cmd_rx,
1225 &mut data_evt_rx,
1226 &mut data_cmd_rx,
1227 );
1228 log::info!("Event loop stopped");
1229 return result;
1230 }
1231
1232 flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1236 startup_system_events.extend(pending.take_system_events());
1237 startup_system_commands.extend(pending.take_system_commands());
1238 debug_assert!(
1239 pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1240 "data must be drained into cache before exec clients connect",
1241 );
1242
1243 let engine_connection_result = drive_with_event_buffering(
1245 self.connect_exec_phase(connection_deadline),
1246 &mut pending,
1247 &mut time_evt_rx,
1248 &mut system_evt_rx,
1249 &mut system_cmd_rx,
1250 &mut exec_evt_rx,
1251 &mut exec_cmd_rx,
1252 &mut data_evt_rx,
1253 &mut data_cmd_rx,
1254 )
1255 .await;
1256
1257 flush_all_pending(
1259 &mut pending,
1260 &mut time_evt_rx,
1261 &mut system_evt_rx,
1262 &mut system_cmd_rx,
1263 &mut exec_evt_rx,
1264 &mut exec_cmd_rx,
1265 &mut data_evt_rx,
1266 &mut data_cmd_rx,
1267 );
1268 startup_system_events.extend(pending.take_system_events());
1269 startup_system_commands.extend(pending.take_system_commands());
1270 debug_assert!(
1271 pending.is_empty(),
1272 "all startup events must be processed before reconciliation",
1273 );
1274
1275 let engine_connection_status = match engine_connection_result {
1276 Ok(status) => status,
1277 Err(e) => {
1278 let result = self
1279 .abort_startup_with_error("Execution client connection timed out", e)
1280 .await;
1281 Self::drain_channels(
1282 &mut time_evt_rx,
1283 &mut system_evt_rx,
1284 &mut system_cmd_rx,
1285 &mut exec_evt_rx,
1286 &mut exec_cmd_rx,
1287 &mut data_evt_rx,
1288 &mut data_cmd_rx,
1289 );
1290 log::info!("Event loop stopped");
1291 return result;
1292 }
1293 };
1294
1295 if engine_connection_status == EngineConnectionStatus::TimedOut {
1296 let result = self
1297 .abort_startup_with_error(
1298 "Engine readiness timed out",
1299 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1300 )
1301 .await;
1302 Self::drain_channels(
1303 &mut time_evt_rx,
1304 &mut system_evt_rx,
1305 &mut system_cmd_rx,
1306 &mut exec_evt_rx,
1307 &mut exec_cmd_rx,
1308 &mut data_evt_rx,
1309 &mut data_cmd_rx,
1310 );
1311 log::info!("Event loop stopped");
1312 return result;
1313 }
1314
1315 if let Some(reason) = engine_connection_status
1316 .abort_reason()
1317 .or_else(|| self.startup_abort_reason())
1318 {
1319 self.abort_startup(reason).await?;
1320 Self::drain_channels(
1321 &mut time_evt_rx,
1322 &mut system_evt_rx,
1323 &mut system_cmd_rx,
1324 &mut exec_evt_rx,
1325 &mut exec_cmd_rx,
1326 &mut data_evt_rx,
1327 &mut data_cmd_rx,
1328 );
1329 log::info!("Event loop stopped");
1330 return Ok(());
1331 }
1332
1333 debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1334
1335 if let Err(e) = self.perform_startup_reconciliation().await {
1337 let result = self.abort_startup("Startup reconciliation failed").await;
1338 Self::drain_channels(
1339 &mut time_evt_rx,
1340 &mut system_evt_rx,
1341 &mut system_cmd_rx,
1342 &mut exec_evt_rx,
1343 &mut exec_cmd_rx,
1344 &mut data_evt_rx,
1345 &mut data_cmd_rx,
1346 );
1347 log::info!("Event loop stopped");
1348
1349 if let Err(finalize_err) = result {
1350 anyhow::bail!(
1351 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1352 );
1353 }
1354
1355 return Err(e);
1356 }
1357
1358 if let Some(reason) = self.startup_abort_reason() {
1359 let result = self.abort_startup(reason).await;
1360 Self::drain_channels(
1361 &mut time_evt_rx,
1362 &mut system_evt_rx,
1363 &mut system_cmd_rx,
1364 &mut exec_evt_rx,
1365 &mut exec_cmd_rx,
1366 &mut data_evt_rx,
1367 &mut data_cmd_rx,
1368 );
1369 log::info!("Event loop stopped");
1370 return result;
1371 }
1372
1373 if let Err(e) = self.kernel.start_trader() {
1374 let result = self.abort_after_trader_start_failure(e).await;
1375 Self::drain_channels(
1376 &mut time_evt_rx,
1377 &mut system_evt_rx,
1378 &mut system_cmd_rx,
1379 &mut exec_evt_rx,
1380 &mut exec_cmd_rx,
1381 &mut data_evt_rx,
1382 &mut data_cmd_rx,
1383 );
1384 log::info!("Event loop stopped");
1385 return result;
1386 }
1387
1388 #[cfg(feature = "plugin")]
1389 if let Err(e) = self.plugins.start_controllers() {
1390 let result = self.abort_after_trader_start_failure(e).await;
1391 Self::drain_channels(
1392 &mut time_evt_rx,
1393 &mut system_evt_rx,
1394 &mut system_cmd_rx,
1395 &mut exec_evt_rx,
1396 &mut exec_cmd_rx,
1397 &mut data_evt_rx,
1398 &mut data_cmd_rx,
1399 );
1400 log::info!("Event loop stopped");
1401 return result;
1402 }
1403
1404 self.process_system_events(startup_system_events);
1405 self.process_system_commands(startup_system_commands);
1406
1407 let finish_result = {
1408 let mut receivers = RunnerReceivers {
1409 time_evt: &mut time_evt_rx,
1410 system_evt: &mut system_evt_rx,
1411 system_cmd: &mut system_cmd_rx,
1412 data_evt: &mut data_evt_rx,
1413 data_cmd: &mut data_cmd_rx,
1414 exec_evt: &mut exec_evt_rx,
1415 exec_cmd: &mut exec_cmd_rx,
1416 };
1417
1418 self.finish_startup_trader(Some(&mut receivers)).await
1419 };
1420
1421 match finish_result {
1422 Ok(true) => {}
1423 result => {
1424 log::info!("Event loop stopped");
1425 return result.map(|_| ());
1426 }
1427 }
1428
1429 let exec_config = &self.config.exec_engine;
1430 let inflight_interval =
1431 Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1432
1433 let open_interval = exec_config
1434 .open_check_interval_secs
1435 .filter(|&s| s > 0.0)
1436 .map_or(Duration::ZERO, |secs| {
1437 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1438 });
1439
1440 let position_interval = exec_config
1441 .position_check_interval_secs
1442 .filter(|&s| s > 0.0)
1443 .map_or(Duration::ZERO, |secs| {
1444 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1445 });
1446
1447 let has_clients = !self
1448 .kernel
1449 .exec_engine
1450 .borrow()
1451 .get_all_clients()
1452 .is_empty();
1453 let recon_enabled = has_clients
1454 && (!inflight_interval.is_zero()
1455 || !open_interval.is_zero()
1456 || !position_interval.is_zero());
1457
1458 let recon_min_interval = if recon_enabled {
1459 let mut intervals = Vec::new();
1460
1461 if !inflight_interval.is_zero() {
1462 intervals.push(inflight_interval);
1463 }
1464
1465 if !open_interval.is_zero() {
1466 intervals.push(open_interval);
1467 }
1468
1469 if !position_interval.is_zero() {
1470 intervals.push(position_interval);
1471 }
1472
1473 intervals
1474 .into_iter()
1475 .min()
1476 .unwrap_or(Duration::from_secs(1))
1477 } else {
1478 Duration::from_secs(1) };
1480
1481 let startup_delay = if self.config.exec_engine.reconciliation {
1486 Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1487 } else {
1488 Duration::ZERO
1489 };
1490
1491 let recon_start = dst::time::Instant::now() + startup_delay;
1492
1493 let mut last_inflight_check = dst::time::Instant::now();
1494 let mut last_open_check = last_inflight_check;
1495 let mut last_position_check = last_inflight_check;
1496
1497 let far_future = Duration::from_hours(24 * 365 * 100);
1500
1501 let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1502 let dur = opt_dur.unwrap_or(far_future);
1503 (dur, recon_start + dur)
1504 };
1505
1506 let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1507 Some(recon_min_interval)
1508 } else {
1509 None
1510 });
1511
1512 let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1513 exec_config
1514 .purge_closed_orders_interval_mins
1515 .filter(|&m| m > 0)
1516 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1517 );
1518
1519 let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1520 exec_config
1521 .purge_closed_positions_interval_mins
1522 .filter(|&m| m > 0)
1523 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1524 );
1525
1526 let (purge_account_interval, mut purge_account_next) = make_schedule(
1527 exec_config
1528 .purge_account_events_interval_mins
1529 .filter(|&m| m > 0)
1530 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1531 );
1532
1533 let (own_books_interval, mut own_books_next) = make_schedule(
1534 exec_config
1535 .own_books_audit_interval_secs
1536 .filter(|&s| s > 0.0)
1537 .map(Duration::from_secs_f64),
1538 );
1539
1540 let (prune_fills_interval, mut prune_fills_next) =
1541 make_schedule(Some(Duration::from_mins(1)));
1542
1543 let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1544 maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1545
1546 let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1551 stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1552
1553 let mut residual_events = 0usize;
1555 let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1556 let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1557 let mut position_report_task: Option<PositionReportTask> = None;
1558
1559 let owns_signals = mode.owns_signals();
1562
1563 let ctrl_c = async move {
1564 if owns_signals {
1565 dst::signal::ctrl_c().await
1566 } else {
1567 std::future::pending::<std::io::Result<()>>().await
1568 }
1569 };
1570
1571 let terminate = async move {
1572 if owns_signals {
1573 dst::signal::terminate().await
1574 } else {
1575 std::future::pending::<std::io::Result<()>>().await
1576 }
1577 };
1578
1579 tokio::pin!(ctrl_c);
1580 tokio::pin!(terminate);
1581
1582 let metrics = self.handle.metrics.clone();
1583 let metrics_start = dst::time::Instant::now();
1584 metrics.reset();
1585
1586 let mut queue_monitor = self
1587 .config
1588 .queue_monitor
1589 .as_ref()
1590 .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1591 let mut dispatches_since_yield = 0usize;
1592 let mut skip_time = false;
1593 let mut skip_data = false;
1594
1595 let dispatch_result = loop {
1596 let callbacks_pending = match drain_callbacks().await {
1597 Ok(pending) => pending,
1598 Err(e) => {
1599 if self.state() == NodeState::Running {
1600 self.initiate_shutdown();
1601 }
1602
1603 log::warn!(
1604 "Skipping residual events and final buffered dispatch after callback failure"
1605 );
1606
1607 break Err(e);
1608 }
1609 };
1610
1611 let shutdown_deadline = self.shutdown_deadline;
1612 let is_shutting_down = self.state() == NodeState::ShuttingDown;
1613 let is_running = self.state() == NodeState::Running;
1614 let commands_empty = data_cmd_rx.is_empty();
1615 let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1616 let poll_data = !skip_data || commands_empty;
1617
1618 tokio::select! {
1619 biased;
1620
1621 result = &mut ctrl_c, if is_running => {
1623 match result {
1624 Ok(()) => log::info!("Received SIGINT, shutting down"),
1625 Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1626 }
1627 self.initiate_shutdown();
1628 }
1629 result = &mut terminate, if is_running => {
1630 match result {
1631 Ok(()) => log::info!("Received SIGTERM, shutting down"),
1632 Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1633 }
1634 self.initiate_shutdown();
1635 }
1636 _ = stop_check_timer.tick(), if is_running => {
1637 if stop_handle.should_stop() {
1638 log::info!("Received stop signal from handle");
1639 self.initiate_shutdown();
1640 } else if self.kernel.is_shutdown_requested() {
1641 log::info!("Received ShutdownSystem command, shutting down");
1642 self.initiate_shutdown();
1643 }
1644 }
1645 () = async {
1646 match shutdown_deadline {
1647 Some(deadline) => dst::time::sleep_until(deadline).await,
1648 None => std::future::pending::<()>().await,
1649 }
1650 }, if self.state() == NodeState::ShuttingDown => {
1651 break Ok(());
1652 }
1653 () = std::future::ready(()), if callbacks_pending => {},
1654 result = async {
1655 match open_order_report_task.as_mut() {
1656 Some(task) => task.future.as_mut().await,
1657 None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1658 }
1659 }, if open_order_report_task.is_some() => {
1660 let maintenance_start = dst::time::Instant::now();
1661
1662 drop(open_order_report_task.take());
1663
1664 match result {
1665 ReportTaskOutcome::Completed(result) => {
1666 let result = result.finish();
1667 let client_refs = self
1668 .exec_clients
1669 .iter()
1670 .map(|client| client as &dyn ExecutionClient)
1671 .collect::<Vec<_>>();
1672 let reconciliation = self.exec_manager.reconcile_open_order_reports(
1673 &result.check,
1674 result.reports,
1675 &result.queried_clients,
1676 &result.failed_clients,
1677 &client_refs,
1678 );
1679 self.process_reconciliation_events(&reconciliation.events);
1680 if !reconciliation.targeted_queries.is_empty() {
1681 if is_shutting_down {
1682 let planned_client_order_ids = reconciliation
1683 .targeted_queries
1684 .iter()
1685 .map(TargetedOrderQuery::client_order_id)
1686 .collect::<Vec<_>>();
1687 self.cleanup_cancelled_report_tasks(
1688 &planned_client_order_ids,
1689 );
1690 } else {
1691 targeted_order_report_task = Some(
1692 self.start_targeted_order_report_check(
1693 reconciliation.targeted_queries,
1694 ),
1695 );
1696 }
1697 }
1698 }
1699 ReportTaskOutcome::TimedOut => {
1700 self.cleanup_cancelled_report_tasks(&[]);
1701 log::warn!(
1702 "Open-order report collection expired after {:?}",
1703 self.config.timeout_reconciliation,
1704 );
1705 }
1706 }
1707 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1708 }
1709 result = async {
1710 match targeted_order_report_task.as_mut() {
1711 Some(task) => task.future.as_mut().await,
1712 None => std::future::pending::<ReportTaskOutcome<()>>().await,
1713 }
1714 }, if targeted_order_report_task.is_some() => {
1715 let maintenance_start = dst::time::Instant::now();
1716
1717 if let Some(task) = targeted_order_report_task.take() {
1718 self.complete_targeted_order_report_check(task, &result);
1719 }
1720 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1721 }
1722 result = async {
1723 match position_report_task.as_mut() {
1724 Some(task) => task.future.as_mut().await,
1725 None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1726 }
1727 }, if position_report_task.is_some() => {
1728 let maintenance_start = dst::time::Instant::now();
1729
1730 drop(position_report_task.take());
1731
1732 match result {
1733 ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1734 if is_shutting_down {
1735 self.cleanup_cancelled_report_tasks(&[]);
1736 } else {
1737 let result = result.finish();
1738 position_report_task = self.handle_position_report_result(result);
1739 }
1740 }
1741 ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1742 let reports = collection.finish();
1743 self.handle_position_fill_report_result(PositionFillReportResult {
1744 position_result,
1745 reports,
1746 });
1747 }
1748 ReportTaskOutcome::TimedOut => {
1749 self.cleanup_cancelled_report_tasks(&[]);
1750 log::warn!(
1751 "Position report collection expired after {:?}",
1752 self.config.timeout_reconciliation,
1753 );
1754 }
1755 }
1756 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1757 }
1758
1759 _ = maintenance_timer.tick(), if is_running => {
1762 let maintenance_start = dst::time::Instant::now();
1763 metrics.publish_queue_depths(
1764 RunnerChannelQueueDepths::from_receivers(
1765 &time_evt_rx,
1766 &exec_evt_rx,
1767 &exec_cmd_rx,
1768 &data_evt_rx,
1769 &data_cmd_rx,
1770 ),
1771 metrics_start.elapsed(),
1772 );
1773
1774 if let Some(queue_monitor) = queue_monitor.as_mut() {
1775 let transitions = queue_monitor.evaluate(metrics.snapshot());
1776 self.publish_queue_state_transitions(&transitions);
1777 }
1778
1779 let mut now = dst::time::Instant::now();
1780
1781 if recon_enabled && now >= recon_next {
1782 let recon_intervals = ReconciliationCheckIntervals {
1783 inflight: inflight_interval,
1784 open: open_interval,
1785 position: position_interval,
1786 };
1787 let mut recon_state = ReconciliationCheckState {
1788 last_inflight_check: &mut last_inflight_check,
1789 last_open_check: &mut last_open_check,
1790 last_position_check: &mut last_position_check,
1791 open_order_report_task: &mut open_order_report_task,
1792 targeted_order_report_task: &mut targeted_order_report_task,
1793 position_report_task: &mut position_report_task,
1794 };
1795
1796 self.run_reconciliation_checks(
1797 now,
1798 recon_intervals,
1799 &mut recon_state,
1800 );
1801
1802 now = dst::time::Instant::now();
1803 recon_next = now + recon_interval;
1804 }
1805
1806 if now >= purge_orders_next {
1807 self.exec_manager.purge_closed_orders();
1808 purge_orders_next = now + purge_orders_interval;
1809 }
1810
1811 if now >= purge_positions_next {
1812 self.exec_manager.purge_closed_positions();
1813 purge_positions_next = now + purge_positions_interval;
1814 }
1815
1816 if now >= purge_account_next {
1817 self.exec_manager.purge_account_events();
1818 purge_account_next = now + purge_account_interval;
1819 }
1820
1821 if now >= own_books_next {
1822 self.kernel.cache().borrow_mut().audit_own_order_books();
1823 own_books_next = now + own_books_interval;
1824 }
1825
1826 if now >= prune_fills_next {
1827 self.exec_manager.prune_recent_fills_cache(60.0);
1828 self.exec_manager.prune_processed_fills();
1829 self.exec_manager.prune_order_local_activity();
1830 prune_fills_next = now + prune_fills_interval;
1831 }
1832
1833 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1834 }
1835
1836 Some(handler) = time_evt_rx.recv(), if poll_time => {
1841 skip_time = true;
1842 let dispatch_start = dst::time::Instant::now();
1843 let dispatched = AsyncRunner::handle_time_event(handler);
1844
1845 if dispatched && is_shutting_down {
1846 log::debug!("Residual time event");
1847 residual_events += 1;
1848 }
1849
1850 if dispatched {
1851 record_runner_dispatch(
1852 &metrics,
1853 SystemChannel::TimeEvents,
1854 dispatch_start,
1855 metrics_start,
1856 );
1857 }
1858 }
1859 Some(event) = system_evt_rx.recv() => {
1860 skip_time = false;
1861
1862 if is_shutting_down {
1863 log::debug!("Residual system event: {event}");
1864 residual_events += 1;
1865 }
1866 event.dispatch(|event| self.process_system_event(event));
1867 }
1868 Some(command) = system_cmd_rx.recv() => {
1869 skip_time = false;
1870
1871 if is_shutting_down {
1872 log::debug!("Residual system command: {command}");
1873 residual_events += 1;
1874 }
1875 command.dispatch(|command| self.process_system_command(command));
1876 }
1877 Some(evt) = exec_evt_rx.recv() => {
1878 skip_time = false;
1879 let dispatch_start = dst::time::Instant::now();
1880
1881 if is_shutting_down {
1882 log::debug!("Residual exec event: {evt}");
1883 residual_events += 1;
1884 }
1885
1886 evt.dispatch(|evt| self.process_exec_event(evt));
1887 record_runner_dispatch(
1888 &metrics,
1889 SystemChannel::ExecEvents,
1890 dispatch_start,
1891 metrics_start,
1892 );
1893 }
1894 Some(cmd) = exec_cmd_rx.recv() => {
1895 skip_time = false;
1896 let dispatch_start = dst::time::Instant::now();
1897
1898 if is_shutting_down {
1899 log::debug!("Residual exec command: {cmd}");
1900 residual_events += 1;
1901 }
1902
1903 self.process_exec_command(cmd);
1904 record_runner_dispatch(
1905 &metrics,
1906 SystemChannel::ExecCommands,
1907 dispatch_start,
1908 metrics_start,
1909 );
1910 }
1911 message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1912 skip_time = false;
1913 let external_msgbus_start = dst::time::Instant::now();
1914
1915 match message {
1916 Some(message) => {
1917 if is_shutting_down {
1918 log::debug!("Residual external message bus message: {message}");
1919 residual_events += 1;
1920 }
1921 self.process_external_msgbus_message(&message);
1922 }
1923 None => {
1924 log::info!("External message bus ingress closed");
1925 external_msgbus_rx = None;
1926 self.close_external_ingress();
1927 }
1928 }
1929
1930 record_runner_external_msgbus(
1931 &metrics,
1932 external_msgbus_start,
1933 metrics_start,
1934 );
1935 }
1936 Some(evt) = data_evt_rx.recv(), if poll_data => {
1937 skip_time = false;
1938 skip_data = true;
1939 let dispatch_start = dst::time::Instant::now();
1940
1941 if is_shutting_down {
1942 log::debug!("Residual data event: {evt:?}");
1943 residual_events += 1;
1944 }
1945 AsyncRunner::dispatch_data_event(evt);
1946 record_runner_dispatch(
1947 &metrics,
1948 SystemChannel::DataEvents,
1949 dispatch_start,
1950 metrics_start,
1951 );
1952 }
1953 Some(cmd) = data_cmd_rx.recv() => {
1954 skip_time = false;
1955 skip_data = false;
1956 let dispatch_start = dst::time::Instant::now();
1957
1958 if is_shutting_down {
1959 log::debug!("Residual data command: {cmd:?}");
1960 residual_events += 1;
1961 }
1962 AsyncRunner::handle_data_command(cmd);
1963 record_runner_dispatch(
1964 &metrics,
1965 SystemChannel::DataCommands,
1966 dispatch_start,
1967 metrics_start,
1968 );
1969 }
1970 }
1971
1972 dispatches_since_yield += 1;
1973 if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1974 dispatches_since_yield = 0;
1975 tokio::task::yield_now().await;
1976 }
1977 };
1978
1979 if residual_events > 0 {
1980 log::debug!("Processed {residual_events} residual events during shutdown");
1981 }
1982
1983 self.cancel_report_tasks(
1984 &mut open_order_report_task,
1985 &mut targeted_order_report_task,
1986 &mut position_report_task,
1987 dispatch_result.is_ok(),
1988 );
1989
1990 drop(external_msgbus_rx.take());
1991 let _ = self.kernel.cache().borrow().check_residuals();
1992
1993 let stop_result = self.finalize_stop().await;
1994
1995 if let Err(e) = dispatch_result {
1996 if let Err(stop_err) = stop_result {
1997 log::error!("Failed to finalize node after callback failure: {stop_err}");
1998 let diagnostic = format!("{e}; {stop_err}");
2000 return Err(anyhow::Error::new(e).context(diagnostic));
2001 }
2002
2003 return Err(e.into());
2004 }
2005
2006 Self::drain_channels(
2008 &mut time_evt_rx,
2009 &mut system_evt_rx,
2010 &mut system_cmd_rx,
2011 &mut exec_evt_rx,
2012 &mut exec_cmd_rx,
2013 &mut data_evt_rx,
2014 &mut data_cmd_rx,
2015 );
2016
2017 log::info!("Event loop stopped");
2018
2019 stop_result
2020 }
2021
2022 fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2023 for transition in transitions {
2024 let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2025 let timestamp = self.kernel.generate_timestamp_ns();
2026
2027 let event = QueueStateChanged::new(
2028 self.config.trader_id,
2029 transition.channel,
2030 transition.condition,
2031 transition.state,
2032 transition.queue_depth,
2033 transition.mean_dispatch_ns,
2034 UUID4::new(),
2035 timestamp,
2036 timestamp,
2037 );
2038
2039 msgbus::publish_any(topic, event.as_any());
2040 }
2041 }
2042
2043 #[expect(
2044 clippy::await_holding_refcell_ref,
2045 reason = "cache loading is serialized before the single-threaded live node starts"
2046 )]
2047 async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2048 self.install_cache_database().await?;
2049
2050 let cache = self.kernel.cache();
2051 if !cache.borrow().has_backing() {
2052 return Ok(());
2053 }
2054
2055 if self
2056 .config
2057 .cache
2058 .as_ref()
2059 .is_some_and(|config| config.flush_on_start)
2060 {
2061 cache
2062 .borrow_mut()
2063 .flush_db()
2064 .context("Failed to flush persistent cache")?;
2065 return Ok(());
2066 }
2067
2068 if self.config.exec_engine.load_cache {
2069 self.kernel
2070 .exec_engine()
2071 .borrow_mut()
2072 .load_cache()
2073 .await
2074 .context("Failed to load persistent cache")?;
2075 }
2076
2077 Ok(())
2078 }
2079
2080 #[must_use]
2082 pub const fn has_pending_cache_database(&self) -> bool {
2083 self.cache_database_factory.is_some()
2084 }
2085
2086 async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2091 let Some(factory) = self.cache_database_factory.as_ref() else {
2092 return Ok(());
2093 };
2094
2095 let config = self.config.cache.clone().unwrap_or_default();
2096
2097 let database = factory
2100 .create(self.config.trader_id, self.kernel.instance_id, config)
2101 .await
2102 .context("failed to create cache database backing")?;
2103 self.kernel.cache().borrow_mut().set_database(database);
2104 self.cache_database_factory = None;
2105
2106 Ok(())
2107 }
2108
2109 fn take_external_ingress_receiver(
2110 &mut self,
2111 ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2112 let Some(external_ingress) = self.external_msgbus.as_mut() else {
2113 return Ok(None);
2114 };
2115
2116 let receiver = external_ingress.take_receiver()?;
2117 log::info!("External message bus ingress started");
2118 Ok(Some(receiver))
2119 }
2120
2121 fn republish_external_msgbus_message(message: &BusMessage) {
2122 if let Err(e) = msgbus::republish_external_message(message) {
2123 log::error!(
2124 "Failed to republish external message bus topic '{}': {e:#}",
2125 message.topic
2126 );
2127 }
2128 }
2129
2130 fn process_external_msgbus_message(&self, message: &BusMessage) {
2131 if self.stream_processors.is_empty() {
2132 Self::republish_external_msgbus_message(message);
2133 return;
2134 }
2135
2136 let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2137 for processor in &self.stream_processors {
2138 (processor.0)(value, mapping)?;
2139 }
2140
2141 Ok(())
2142 };
2143
2144 if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2145 log::error!(
2146 "Failed to process external message bus topic '{}': {e:#}",
2147 message.topic
2148 );
2149 }
2150 }
2151
2152 fn finish_external_ingress_connection(
2153 &mut self,
2154 receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2155 ) {
2156 if receiver.is_none() {
2157 self.close_external_ingress();
2158 }
2159 }
2160
2161 fn close_external_ingress(&mut self) {
2162 if let Some(external_ingress) = self.external_msgbus.as_mut()
2163 && !external_ingress.is_closed()
2164 {
2165 external_ingress.close();
2166 }
2167 }
2168
2169 fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2170 if events.is_empty() {
2171 self.publish_submission_recovery_exhaustions();
2172 return;
2173 }
2174
2175 log::info!(
2176 "Processing {} reconciliation event{}",
2177 events.len(),
2178 if events.len() == 1 { "" } else { "s" }
2179 );
2180
2181 for event in events {
2182 self.exec_manager
2183 .record_local_activity(event.client_order_id());
2184 if let OrderEventAny::Filled(fill) = event {
2185 self.exec_manager
2186 .record_position_activity(fill.instrument_id, fill.account_id);
2187 }
2188
2189 self.kernel.exec_engine.borrow_mut().process(event);
2190 self.exec_manager
2191 .confirm_submission_outcome(&event.client_order_id());
2192 if let OrderEventAny::Filled(fill) = event {
2193 self.exec_manager.commit_recent_fill_if_applied(fill);
2194 }
2195 }
2196
2197 self.publish_submission_recovery_exhaustions();
2198 }
2199
2200 fn publish_submission_recovery_exhaustions(&mut self) {
2201 for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2202 log::warn!(
2203 "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2204 diagnostic.client_order_id,
2205 diagnostic.source,
2206 diagnostic.retry_count,
2207 );
2208 msgbus::publish_any(
2209 MessagingSwitchboard::submission_recovery_exhausted_topic(),
2210 &diagnostic,
2211 );
2212 }
2213 }
2214
2215 fn process_exec_event(&mut self, event: ExecutionEvent) {
2216 let clear_closed_orders = matches!(
2217 &event,
2218 ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2219 );
2220 let closed_report_id = match &event {
2221 ExecutionEvent::Report(
2222 ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2223 ) if report.order_status.is_closed() => report.client_order_id,
2224 _ => None,
2225 };
2226 let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2227 return;
2228 };
2229
2230 self.dispatch_exec_event_and_commit_fill(event);
2231
2232 for client_order_id in &order_ids {
2233 self.exec_manager
2234 .confirm_submission_outcome(client_order_id);
2235 let is_closed = self
2236 .kernel
2237 .cache()
2238 .borrow()
2239 .order(client_order_id)
2240 .is_some_and(|order| order.is_closed());
2241
2242 if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2243 self.exec_manager
2244 .clear_recon_tracking(client_order_id, true);
2245 }
2246 }
2247 }
2248
2249 fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2250 message.dispatch_trading(|message| {
2251 if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2252 self.observe_exec_command_before_dispatch(message.command());
2253 }
2254 });
2255 }
2256
2257 fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2266 let recent_fill_candidate = match &evt {
2267 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2268 _ => None,
2269 };
2270
2271 AsyncRunner::handle_exec_event(evt);
2272
2273 if let Some(fill) = &recent_fill_candidate {
2274 self.exec_manager.commit_recent_fill_if_applied(fill);
2275 }
2276 }
2277
2278 async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2279 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2285 dst::time::timeout(remaining, self.kernel.connect_data_clients())
2286 .await
2287 .map_err(|_| anyhow::anyhow!("data-connect timeout"))
2288 }
2289
2290 async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2291 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2292 dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2293 .await
2294 .map_err(|_| anyhow::anyhow!("exec-connect timeout"))
2295 }
2296
2297 async fn connect_exec_phase(
2302 &mut self,
2303 deadline: dst::time::Instant,
2304 ) -> anyhow::Result<EngineConnectionStatus> {
2305 self.connect_exec_clients(deadline).await?;
2306 Ok(self.await_engines_connected(deadline).await)
2307 }
2308
2309 fn startup_abort_reason(&self) -> Option<&'static str> {
2310 if self.handle.should_stop() {
2311 Some("Stop signal received during startup")
2312 } else if self.kernel.is_shutdown_requested() {
2313 Some("Shutdown signal received during startup")
2314 } else {
2315 None
2316 }
2317 }
2318
2319 async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2320 match self.handle.try_set_running() {
2321 RunningTransition::Entered => Ok(true),
2322 RunningTransition::StopRequested => {
2323 self.abort_startup("Stop signal received during startup")
2324 .await?;
2325 Ok(false)
2326 }
2327 RunningTransition::Invalid(control) => {
2328 self.abort_startup_with_error(
2329 "Invalid lifecycle state during startup",
2330 anyhow::anyhow!(
2331 "Invalid LiveNode control state {control:#04x} while entering Running"
2332 ),
2333 )
2334 .await?;
2335 Ok(false)
2336 }
2337 }
2338 }
2339
2340 async fn finish_startup_trader(
2341 &mut self,
2342 receivers: Option<&mut RunnerReceivers<'_>>,
2343 ) -> anyhow::Result<bool> {
2344 match self.handle.try_set_running() {
2345 RunningTransition::Entered => Ok(true),
2346 RunningTransition::StopRequested => {
2347 self.abort_started_trader("Stop signal received during startup", receivers)
2348 .await?;
2349 Ok(false)
2350 }
2351 RunningTransition::Invalid(control) => {
2352 let state_err = anyhow::anyhow!(
2353 "Invalid LiveNode control state {control:#04x} while entering Running"
2354 );
2355
2356 match self
2357 .abort_started_trader("Invalid lifecycle state during startup", receivers)
2358 .await
2359 {
2360 Ok(()) => Err(state_err),
2361 Err(finalize_err) => {
2362 anyhow::bail!(
2363 "{state_err}; failed to finalize startup abort: {finalize_err}"
2364 )
2365 }
2366 }
2367 }
2368 }
2369 }
2370
2371 async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2372 log::info!("{reason}, aborting startup");
2373 self.handle.set_shutting_down();
2374 self.finalize_stop().await
2375 }
2376
2377 async fn abort_startup_with_error(
2378 &mut self,
2379 reason: &str,
2380 startup_err: anyhow::Error,
2381 ) -> anyhow::Result<()> {
2382 match self.abort_startup(reason).await {
2383 Ok(()) => Err(startup_err),
2384 Err(finalize_err) => {
2385 anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2386 }
2387 }
2388 }
2389
2390 async fn abort_started_trader(
2391 &mut self,
2392 reason: &str,
2393 mut receivers: Option<&mut RunnerReceivers<'_>>,
2394 ) -> anyhow::Result<()> {
2395 log::info!("{reason}, aborting startup");
2396 self.handle.set_shutting_down();
2397
2398 #[cfg(feature = "plugin")]
2399 let controller_stop_result = self.plugins.stop_controllers();
2400 #[cfg(not(feature = "plugin"))]
2401 let controller_stop_result: anyhow::Result<()> = Ok(());
2402
2403 let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2404 let delay = self.kernel.delay_post_stop();
2405 log::info!("Awaiting residual events ({delay:?})...");
2406
2407 let residual_events = match receivers.as_mut() {
2408 Some(receivers) => self.process_receivers_for(delay, receivers).await,
2409 None => self.process_runner_for(delay).await,
2410 };
2411
2412 if residual_events > 0 {
2413 log::debug!("Processed {residual_events} residual events during shutdown");
2414 }
2415
2416 let finalize_result = self.finalize_stop().await;
2417
2418 if let Some(receivers) = receivers {
2419 Self::drain_channels(
2420 receivers.time_evt,
2421 receivers.system_evt,
2422 receivers.system_cmd,
2423 receivers.exec_evt,
2424 receivers.exec_cmd,
2425 receivers.data_evt,
2426 receivers.data_cmd,
2427 );
2428 } else {
2429 let drained_events = self.drain_runner_pending();
2430 if drained_events > 0 {
2431 log::info!("Drained {drained_events} remaining events during shutdown");
2432 }
2433 }
2434
2435 let mut errors = Vec::new();
2436
2437 if let Err(e) = controller_stop_result {
2438 errors.push(format!("Failed to stop plug-in controllers: {e}"));
2439 }
2440
2441 if let Err(e) = trader_stop_result {
2442 errors.push(format!("Failed to stop trader: {e}"));
2443 }
2444
2445 if let Err(e) = finalize_result {
2446 errors.push(format!("Failed to finalize startup abort: {e}"));
2447 }
2448
2449 if errors.is_empty() {
2450 Ok(())
2451 } else {
2452 anyhow::bail!("{}", errors.join("; "))
2453 }
2454 }
2455
2456 async fn process_receivers_for(
2457 &mut self,
2458 duration: Duration,
2459 receivers: &mut RunnerReceivers<'_>,
2460 ) -> usize {
2461 let deadline = dst::time::Instant::now() + duration;
2462 let mut processed = 0;
2463
2464 loop {
2465 tokio::select! {
2466 biased;
2467
2468 () = dst::time::sleep_until(deadline) => break,
2469 Some(message) = receivers.time_evt.recv() => {
2470 let _ = AsyncRunner::handle_time_event(message);
2471 processed += 1;
2472 }
2473 Some(event) = receivers.system_evt.recv() => {
2474 event.dispatch(|event| self.process_system_event(event));
2475 processed += 1;
2476 }
2477 Some(command) = receivers.system_cmd.recv() => {
2478 command.dispatch(|command| self.process_system_command(command));
2479 processed += 1;
2480 }
2481 Some(event) = receivers.exec_evt.recv() => {
2482 event.dispatch(|event| self.process_exec_event(event));
2483 processed += 1;
2484 }
2485 Some(command) = receivers.exec_cmd.recv() => {
2486 self.process_exec_command(command);
2487 processed += 1;
2488 }
2489 Some(event) = receivers.data_evt.recv() => {
2490 AsyncRunner::dispatch_data_event(event);
2491 processed += 1;
2492 }
2493 Some(command) = receivers.data_cmd.recv() => {
2494 AsyncRunner::handle_data_command(command);
2495 processed += 1;
2496 }
2497 }
2498 }
2499
2500 processed
2501 }
2502
2503 async fn abort_after_trader_start_failure(
2504 &mut self,
2505 start_err: anyhow::Error,
2506 ) -> anyhow::Result<()> {
2507 log::info!("Trader startup failed, aborting startup");
2508 self.handle.set_shutting_down();
2509 let stop_result = self.kernel.stop_trader_after_start_failure();
2510 let finalize_result = self.finalize_stop().await;
2511
2512 match (stop_result, finalize_result) {
2513 (Ok(()), Ok(())) => Err(start_err),
2514 (Err(stop_err), Ok(())) => anyhow::bail!(
2515 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2516 {stop_err}"
2517 ),
2518 (Ok(()), Err(finalize_err)) => anyhow::bail!(
2519 "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2520 {finalize_err}"
2521 ),
2522 (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2523 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2524 {stop_err}; failed to finalize startup abort: {finalize_err}"
2525 ),
2526 }
2527 }
2528
2529 fn initiate_shutdown(&mut self) {
2530 #[cfg(feature = "plugin")]
2531 if let Err(e) = self.plugins.stop_controllers() {
2532 log::error!("Error stopping plug-in controllers: {e}");
2533 }
2534
2535 self.kernel.stop_trader();
2536 let delay = self.kernel.delay_post_stop();
2537 log::info!("Awaiting residual events ({delay:?})...");
2538
2539 self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2540 self.handle.set_shutting_down();
2541 }
2542
2543 async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2544 let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2547 self.close_external_ingress();
2548
2549 let timeout = self.config.timeout_disconnection;
2550 let deadline = dst::time::Instant::now() + timeout;
2551
2552 let mut errors = Vec::new();
2553 let (report_result, disconnect_result) = futures_util::join!(
2554 dst::time::timeout(
2555 timeout,
2556 Self::finish_report_tasks(&self.exec_clients, &mut errors)
2557 ),
2558 dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2559 );
2560
2561 if report_result.is_err() {
2562 errors.push("report collection shutdown timeout".to_string());
2563 }
2564
2565 let disconnect_result = match disconnect_result {
2566 Ok(result) => result,
2567 Err(_) => Err(anyhow::anyhow!(
2568 "disconnect timeout while disconnecting clients"
2569 )),
2570 };
2571
2572 if let Err(ref e) = disconnect_result {
2573 log::error!("Error disconnecting clients: {e}");
2574 }
2575
2576 let readiness_result = self.await_engines_disconnected(deadline).await;
2577 let kernel_result = self.kernel.finalize_stop().await;
2578
2579 self.handle.set_stopped();
2580
2581 if !unresolved_submissions.is_empty() {
2582 errors.push(format!(
2583 "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2584 ));
2585 }
2586
2587 if let Err(e) = disconnect_result {
2588 errors.push(e.to_string());
2589 }
2590
2591 if let Err(e) = readiness_result {
2592 errors.push(format!("failed while awaiting engine disconnection: {e}"));
2593 }
2594
2595 if let Err(e) = kernel_result {
2596 errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2597 }
2598
2599 if errors.is_empty() {
2600 Ok(())
2601 } else {
2602 anyhow::bail!("{}", errors.join("; "))
2603 }
2604 }
2605
2606 fn drain_channels(
2607 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2608 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2609 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2610 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2611 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2612 DispatchMessage<TradingCommandMessage>,
2613 >,
2614 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2615 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2616 ) {
2617 let mut drained = 0;
2618
2619 while let Ok(handler) = time_evt_rx.try_recv() {
2620 let _ = AsyncRunner::handle_time_event(handler);
2621 drained += 1;
2622 }
2623
2624 while system_evt_rx.try_recv().is_ok() {
2625 drained += 1;
2626 }
2627
2628 while system_cmd_rx.try_recv().is_ok() {
2629 drained += 1;
2630 }
2631
2632 while let Ok(evt) = data_evt_rx.try_recv() {
2633 AsyncRunner::dispatch_data_event(evt);
2634 drained += 1;
2635 }
2636
2637 while let Ok(cmd) = data_cmd_rx.try_recv() {
2638 AsyncRunner::handle_data_command(cmd);
2639 drained += 1;
2640 }
2641
2642 while let Ok(evt) = exec_evt_rx.try_recv() {
2643 AsyncRunner::dispatch_exec_event(evt);
2644 drained += 1;
2645 }
2646
2647 while let Ok(cmd) = exec_cmd_rx.try_recv() {
2648 AsyncRunner::handle_trading_command(cmd);
2649 drained += 1;
2650 }
2651
2652 if drained > 0 {
2653 log::info!("Drained {drained} remaining events during shutdown");
2654 }
2655 }
2656
2657 fn observe_exec_event_before_dispatch(
2658 &mut self,
2659 evt: &ExecutionEvent,
2660 ) -> Option<Vec<ClientOrderId>> {
2661 let mut order_ids = Vec::new();
2662
2663 match evt {
2664 ExecutionEvent::Order(order_evt) => {
2665 self.exec_manager.observe_order_event(order_evt);
2666 order_ids.push(order_evt.client_order_id());
2667 }
2668 ExecutionEvent::OrderSubmittedBatch(batch) => {
2669 for submitted in &batch.events {
2670 self.exec_manager
2671 .record_local_activity(submitted.client_order_id);
2672 }
2673 }
2674 ExecutionEvent::OrderAcceptedBatch(batch) => {
2675 for accepted in &batch.events {
2676 self.exec_manager
2677 .observe_order_event(&OrderEventAny::Accepted(*accepted));
2678 order_ids.push(accepted.client_order_id);
2679 }
2680 }
2681 ExecutionEvent::OrderCanceledBatch(batch) => {
2682 for canceled in &batch.events {
2683 self.exec_manager
2684 .observe_order_event(&OrderEventAny::Canceled(*canceled));
2685 order_ids.push(canceled.client_order_id);
2686 }
2687 }
2688 ExecutionEvent::Report(report) => {
2689 if let ExecutionReport::Fill(fill_report) = report
2690 && self.exec_manager.is_fill_recently_processed(
2691 fill_report.account_id,
2692 fill_report.instrument_id,
2693 fill_report.trade_id,
2694 )
2695 {
2696 log::debug!(
2697 "Skipping recently processed fill report: {}",
2698 fill_report.trade_id,
2699 );
2700 return None;
2701 }
2702
2703 self.exec_manager.observe_execution_report(report);
2704
2705 if let Some(client_order_id) = self.order_report_client_order_id(report) {
2706 order_ids.push(client_order_id);
2707 }
2708 }
2709 ExecutionEvent::Account(_) => {}
2710 }
2711
2712 Some(order_ids)
2713 }
2714
2715 fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2716 match report {
2717 ExecutionReport::Order(order_report)
2718 | ExecutionReport::OrderWithFills(order_report, _) => {
2719 order_report.client_order_id.or_else(|| {
2720 self.kernel
2721 .cache
2722 .borrow()
2723 .client_order_id(&order_report.venue_order_id)
2724 .copied()
2725 })
2726 }
2727 ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2728 self.kernel
2729 .cache
2730 .borrow()
2731 .client_order_id(&fill_report.venue_order_id)
2732 .copied()
2733 }),
2734 _ => None,
2735 }
2736 }
2737
2738 fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2739 let submission_client_id = if matches!(
2740 cmd,
2741 TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2742 ) {
2743 self.kernel
2744 .exec_engine
2745 .borrow()
2746 .find_client_for_command(cmd)
2747 .map(|client| client.client_id())
2748 } else {
2749 None
2750 };
2751
2752 match cmd {
2753 TradingCommand::SubmitOrder(submit) => {
2754 self.exec_manager.register_submission(
2755 &submit.order_init,
2756 submission_client_id.or(submit.client_id),
2757 );
2758 }
2759 TradingCommand::SubmitOrderList(submit) => {
2760 for order_init in &submit.order_inits {
2761 self.exec_manager
2762 .register_submission(order_init, submission_client_id.or(submit.client_id));
2763 }
2764 }
2765 TradingCommand::ModifyOrder(modify) => {
2766 self.exec_manager.register_inflight(modify.client_order_id);
2767 }
2768 TradingCommand::ModifyOrders(modify) => {
2769 for child in &modify.modifies {
2770 self.exec_manager.register_inflight(child.client_order_id);
2771 }
2772 }
2773 TradingCommand::CancelOrder(cancel) => {
2774 self.exec_manager.register_inflight(cancel.client_order_id);
2775 }
2776 TradingCommand::CancelOrders(cancel) => {
2777 for child in &cancel.cancels {
2778 self.exec_manager.register_inflight(child.client_order_id);
2779 }
2780 }
2781 _ => {}
2782 }
2783 }
2784
2785 #[must_use]
2787 pub fn environment(&self) -> Environment {
2788 self.kernel.environment()
2789 }
2790
2791 #[must_use]
2793 pub const fn kernel(&self) -> &NautilusKernel {
2794 &self.kernel
2795 }
2796
2797 #[must_use]
2799 pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2800 &mut self.kernel
2801 }
2802
2803 #[must_use]
2805 pub fn trader_id(&self) -> TraderId {
2806 self.kernel.trader_id()
2807 }
2808
2809 #[must_use]
2811 pub const fn instance_id(&self) -> UUID4 {
2812 self.kernel.instance_id()
2813 }
2814
2815 #[must_use]
2817 pub fn state(&self) -> NodeState {
2818 self.handle.state()
2819 }
2820
2821 #[must_use]
2823 pub fn is_running(&self) -> bool {
2824 self.state().is_running()
2825 }
2826
2827 pub fn set_cache_database(
2837 &mut self,
2838 database: Box<dyn CacheDatabaseAdapter>,
2839 ) -> anyhow::Result<()> {
2840 if self.state() != NodeState::Idle {
2841 anyhow::bail!(
2842 "Cannot set cache database while node is running, set it before running the node"
2843 );
2844 }
2845
2846 self.kernel.cache().borrow_mut().set_database(database);
2847 Ok(())
2848 }
2849
2850 #[must_use]
2852 pub fn exec_manager(&self) -> &ExecutionManager {
2853 &self.exec_manager
2854 }
2855
2856 #[must_use]
2858 pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2859 &mut self.exec_manager
2860 }
2861
2862 pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2875 where
2876 T: DataActor + DataActorNative + Component + Actor + 'static,
2877 {
2878 if self.state() != NodeState::Idle {
2879 anyhow::bail!(
2880 "Cannot add actor while node is running, add actors before running the node"
2881 );
2882 }
2883
2884 self.kernel.trader.borrow_mut().add_actor(actor)
2885 }
2886
2887 pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2899 where
2900 F: FnOnce() -> anyhow::Result<T>,
2901 T: DataActor + DataActorNative + Component + Actor + 'static,
2902 {
2903 if self.state() != NodeState::Idle {
2904 anyhow::bail!(
2905 "Cannot add actor while node is running, add actors before running the node"
2906 );
2907 }
2908
2909 self.kernel
2910 .trader
2911 .borrow_mut()
2912 .add_actor_from_factory(factory)
2913 }
2914
2915 pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2931 where
2932 T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2933 {
2934 if self.state() != NodeState::Idle {
2935 anyhow::bail!(
2936 "Cannot add strategy while node is running, add strategies before running the node"
2937 );
2938 }
2939
2940 let strategy_id = self
2942 .kernel
2943 .trader
2944 .borrow()
2945 .prepare_strategy_for_registration(&mut strategy)?;
2946 let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
2947 let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
2948
2949 if !instrument_ids.is_empty() {
2950 self.register_external_order_claims(strategy_id, &instrument_ids)?;
2951 }
2952
2953 let mut exec_engine = match oms_type
2954 .map(|_| {
2955 self.kernel
2956 .exec_engine
2957 .try_borrow_mut()
2958 .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
2959 })
2960 .transpose()
2961 {
2962 Ok(exec_engine) => exec_engine,
2963 Err(e) => {
2964 if !instrument_ids.is_empty()
2965 && let Err(rollback_error) =
2966 self.rollback_external_order_claims(strategy_id, &instrument_ids)
2967 {
2968 anyhow::bail!(
2969 "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
2970 );
2971 }
2972
2973 return Err(e);
2974 }
2975 };
2976
2977 if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
2978 drop(exec_engine);
2979
2980 if !instrument_ids.is_empty()
2981 && let Err(rollback_error) =
2982 self.rollback_external_order_claims(strategy_id, &instrument_ids)
2983 {
2984 anyhow::bail!(
2985 "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
2986 );
2987 }
2988
2989 return Err(add_error);
2990 }
2991
2992 if let Some(exec_engine) = &mut exec_engine
2993 && let Some(oms_type) = oms_type
2994 {
2995 exec_engine.register_oms_type(strategy_id, oms_type);
2996 }
2997
2998 Ok(())
2999 }
3000
3001 pub fn register_external_order_claims(
3012 &self,
3013 strategy_id: StrategyId,
3014 instrument_ids: &[InstrumentId],
3015 ) -> anyhow::Result<()> {
3016 self.kernel
3017 .cache
3018 .try_borrow_mut()
3019 .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3020 .register_external_order_claims(strategy_id, instrument_ids)?;
3021
3022 if !instrument_ids.is_empty() {
3023 log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3024 }
3025
3026 Ok(())
3027 }
3028
3029 pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3039 self.kernel
3040 .cache
3041 .try_borrow_mut()
3042 .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3043 .set_external_order_claims(strategy_id, &[])?;
3044
3045 Ok(())
3046 }
3047
3048 pub(crate) fn rollback_external_order_claims(
3049 &self,
3050 strategy_id: StrategyId,
3051 instrument_ids: &[InstrumentId],
3052 ) -> anyhow::Result<()> {
3053 let mut cache = self
3054 .kernel
3055 .cache
3056 .try_borrow_mut()
3057 .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3058 let retained: Vec<_> = cache
3059 .external_order_claim_instrument_ids(Some(strategy_id))
3060 .into_iter()
3061 .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3062 .collect();
3063 cache.set_external_order_claims(strategy_id, &retained)
3064 }
3065
3066 pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3077 where
3078 T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3079 {
3080 if self.state() != NodeState::Idle {
3081 anyhow::bail!(
3082 "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3083 );
3084 }
3085
3086 self.kernel
3087 .trader
3088 .borrow_mut()
3089 .add_exec_algorithm(exec_algorithm)
3090 }
3091
3092 }
3096
3097#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3098enum SocketReconnectDispatchOutcome {
3099 Accepted,
3100 AlreadyReconnecting,
3101 Disconnected,
3102 Closed,
3103 Unsupported,
3104 InvalidTrader,
3105 UnknownClient,
3106 UnknownEndpoint,
3107 AmbiguousEndpoint,
3108}
3109
3110fn record_runner_dispatch(
3111 metrics: &RunnerMetrics,
3112 channel: SystemChannel,
3113 dispatch_start: dst::time::Instant,
3114 metrics_start: dst::time::Instant,
3115) {
3116 let dispatch_end = dst::time::Instant::now();
3117 metrics.record_dispatch(
3118 channel,
3119 dispatch_end.duration_since(dispatch_start),
3120 dispatch_end.duration_since(metrics_start),
3121 );
3122}
3123
3124fn record_runner_maintenance(
3125 metrics: &RunnerMetrics,
3126 work_start: dst::time::Instant,
3127 metrics_start: dst::time::Instant,
3128) {
3129 let work_end = dst::time::Instant::now();
3130 metrics.record_maintenance(
3131 work_end.duration_since(work_start),
3132 work_end.duration_since(metrics_start),
3133 );
3134}
3135
3136fn record_runner_external_msgbus(
3137 metrics: &RunnerMetrics,
3138 work_start: dst::time::Instant,
3139 metrics_start: dst::time::Instant,
3140) {
3141 let work_end = dst::time::Instant::now();
3142 metrics.record_external_msgbus(
3143 work_end.duration_since(work_start),
3144 work_end.duration_since(metrics_start),
3145 );
3146}
3147
3148async fn drive_with_external_ingress<F, T>(
3149 future: F,
3150 receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3151 pending: &mut VecDeque<BusMessage>,
3152) -> T
3153where
3154 F: std::future::Future<Output = T>,
3155{
3156 tokio::pin!(future);
3157 let capacity = receiver
3158 .as_ref()
3159 .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3160
3161 loop {
3162 tokio::select! {
3163 biased;
3164
3165 result = &mut future => return result,
3166 message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3167 let Some(message) = message else {
3168 log::info!("External message bus ingress closed");
3169 *receiver = None;
3170 continue;
3171 };
3172
3173 match msgbus::republish_pending_external_response(&message) {
3174 Ok(true) => {},
3175 Ok(false) | Err(_) => pending.push_back(message),
3176 }
3177 }
3178 }
3179 }
3180}
3181
3182async fn recv_external_msgbus_message(
3183 rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3184 pending: &mut VecDeque<BusMessage>,
3185) -> Option<BusMessage> {
3186 if let Some(message) = pending.pop_front() {
3187 return Some(message);
3188 }
3189
3190 match rx {
3191 Some(rx) => rx.recv().await,
3192 None => std::future::pending::<Option<BusMessage>>().await,
3193 }
3194}
3195
3196struct RunnerReceivers<'a> {
3197 time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3198 system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3199 system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3200 exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3201 exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3202 data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3203 data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3204}
3205
3206fn flush_pending_data(
3213 pending: &mut PendingEvents,
3214 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3215 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3216) {
3217 loop {
3218 let mut progressed = pending.drain_data();
3219
3220 while let Ok(evt) = data_evt_rx.try_recv() {
3221 AsyncRunner::dispatch_data_event(evt);
3222 progressed = true;
3223 }
3224
3225 while let Ok(cmd) = data_cmd_rx.try_recv() {
3226 AsyncRunner::handle_data_command(cmd);
3227 progressed = true;
3228 }
3229
3230 if !progressed {
3231 break;
3232 }
3233 }
3234}
3235
3236#[expect(
3242 clippy::too_many_arguments,
3243 reason = "all runner receivers are drained together"
3244)]
3245fn flush_all_pending(
3246 pending: &mut PendingEvents,
3247 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3248 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3249 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3250 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3251 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3252 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3253 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3254) {
3255 while let Ok(handler) = time_evt_rx.try_recv() {
3257 let _ = AsyncRunner::handle_time_event(handler);
3258 }
3259
3260 while let Ok(event) = system_evt_rx.try_recv() {
3261 pending.system_events.push(event);
3262 }
3263
3264 while let Ok(command) = system_cmd_rx.try_recv() {
3265 pending.system_commands.push(command);
3266 }
3267
3268 while let Ok(evt) = data_evt_rx.try_recv() {
3269 pending.data_evts.push(evt);
3270 }
3271
3272 while let Ok(cmd) = data_cmd_rx.try_recv() {
3273 pending.data_cmds.push(cmd);
3274 }
3275
3276 while let Ok(evt) = exec_evt_rx.try_recv() {
3277 pending.push_exec_event(evt);
3278 }
3279
3280 while let Ok(cmd) = exec_cmd_rx.try_recv() {
3281 pending.exec_cmds.push(cmd);
3282 }
3283
3284 pending.drain();
3285}
3286
3287#[expect(
3292 clippy::too_many_arguments,
3293 reason = "startup buffering owns one future plus the pending state and all runner receivers"
3294)]
3295async fn drive_with_event_buffering<F: std::future::Future>(
3296 future: F,
3297 pending: &mut PendingEvents,
3298 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3299 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3300 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3301 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3302 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3303 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3304 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3305) -> F::Output {
3306 tokio::pin!(future);
3307
3308 loop {
3309 tokio::select! {
3310 biased;
3311
3312 result = &mut future => {
3313 break result;
3314 }
3315 Some(handler) = time_evt_rx.recv() => {
3316 let _ = AsyncRunner::handle_time_event(handler);
3317 }
3318 Some(event) = system_evt_rx.recv() => {
3319 pending.system_events.push(event);
3320 }
3321 Some(command) = system_cmd_rx.recv() => {
3322 pending.system_commands.push(command);
3323 }
3324 Some(evt) = exec_evt_rx.recv() => {
3325 pending.push_exec_event(evt);
3326 }
3327 Some(cmd) = exec_cmd_rx.recv() => {
3328 pending.exec_cmds.push(cmd);
3329 }
3330 Some(evt) = data_evt_rx.recv() => {
3331 pending.data_evts.push(evt);
3332 }
3333 Some(cmd) = data_cmd_rx.recv() => {
3334 pending.data_cmds.push(cmd);
3335 }
3336 }
3337 }
3338}
3339
3340#[derive(Default)]
3341struct PendingEvents {
3342 system_events: Vec<DispatchMessage<SystemEvent>>,
3343 system_commands: Vec<DispatchMessage<SystemCommand>>,
3344 data_evts: Vec<DispatchMessage<DataEvent>>,
3345 data_cmds: Vec<DispatchMessage<DataCommand>>,
3346 exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3347 order_evts: Vec<DispatchMessage<OrderEventAny>>,
3348 exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3349}
3350
3351impl PendingEvents {
3352 fn is_empty(&self) -> bool {
3353 self.system_events.is_empty()
3354 && self.system_commands.is_empty()
3355 && self.data_evts.is_empty()
3356 && self.data_cmds.is_empty()
3357 && self.exec_reports.is_empty()
3358 && self.order_evts.is_empty()
3359 && self.exec_cmds.is_empty()
3360 }
3361
3362 fn drain_data(&mut self) -> bool {
3366 let total = self.data_evts.len() + self.data_cmds.len();
3367
3368 if total > 0 {
3369 log::debug!(
3370 "Draining {total} data events/commands into cache \
3371 (data_evts={}, data_cmds={})",
3372 self.data_evts.len(),
3373 self.data_cmds.len(),
3374 );
3375 }
3376
3377 for evt in self.data_evts.drain(..) {
3378 AsyncRunner::dispatch_data_event(evt);
3379 }
3380
3381 for cmd in self.data_cmds.drain(..) {
3382 AsyncRunner::handle_data_command(cmd);
3383 }
3384
3385 total > 0
3386 }
3387
3388 fn drain(&mut self) {
3390 let total = self.data_evts.len()
3391 + self.data_cmds.len()
3392 + self.exec_reports.len()
3393 + self.order_evts.len()
3394 + self.exec_cmds.len();
3395
3396 if total > 0 {
3397 log::debug!(
3398 "Processing {total} events/commands queued during startup \
3399 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3400 self.data_evts.len(),
3401 self.data_cmds.len(),
3402 self.exec_reports.len(),
3403 self.order_evts.len(),
3404 self.exec_cmds.len()
3405 );
3406 }
3407
3408 for evt in self.data_evts.drain(..) {
3409 AsyncRunner::dispatch_data_event(evt);
3410 }
3411
3412 for cmd in self.data_cmds.drain(..) {
3413 AsyncRunner::handle_data_command(cmd);
3414 }
3415
3416 for report in self.exec_reports.drain(..) {
3417 report
3418 .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3419 }
3420
3421 for evt in self.order_evts.drain(..) {
3422 evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3423 }
3424
3425 for cmd in self.exec_cmds.drain(..) {
3426 AsyncRunner::handle_trading_command(cmd);
3427 }
3428 }
3429
3430 fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3431 let rooted = event.is_rooted();
3432 event.dispatch(|event| {
3433 let owner = std::thread::current().id();
3434
3435 let order = |event| {
3436 if rooted {
3437 DispatchMessage::new(event, owner)
3438 } else {
3439 DispatchMessage::from(event)
3440 }
3441 };
3442
3443 match event {
3446 ExecutionEvent::Account(_) => AsyncRunner::handle_exec_event(event),
3447 ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3448 DispatchMessage::new(report, owner)
3449 } else {
3450 report.into()
3451 }),
3452 ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3453 ExecutionEvent::OrderSubmittedBatch(batch) => {
3454 for event in batch {
3455 self.order_evts.push(order(OrderEventAny::Submitted(event)));
3456 }
3457 }
3458 ExecutionEvent::OrderAcceptedBatch(batch) => {
3459 for event in batch {
3460 self.order_evts.push(order(OrderEventAny::Accepted(event)));
3461 }
3462 }
3463 ExecutionEvent::OrderCanceledBatch(batch) => {
3464 for event in batch {
3465 self.order_evts.push(order(OrderEventAny::Canceled(event)));
3466 }
3467 }
3468 }
3469 });
3470 }
3471
3472 fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3473 std::mem::take(&mut self.system_events)
3474 }
3475
3476 fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3477 std::mem::take(&mut self.system_commands)
3478 }
3479}
3480
3481struct ClientStatus {
3482 client: String,
3483 client_type: &'static str,
3484 connected: bool,
3485}
3486
3487fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3488 let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3489 builder.push_record(["Client", "Type", "Connected"]);
3490
3491 for row in rows {
3492 builder.push_record([
3493 row.client,
3494 row.client_type.to_string(),
3495 row.connected.to_string(),
3496 ]);
3497 }
3498
3499 builder.build().with(Style::rounded()).to_string()
3500}
3501
3502#[cfg(test)]
3503mod tests {
3504 mod report_acceptance;
3505 mod shutdown;
3506
3507 use std::{
3508 cell::{Cell, RefCell},
3509 fmt::Debug,
3510 rc::Rc,
3511 sync::{
3512 Arc,
3513 atomic::{AtomicBool, Ordering},
3514 },
3515 };
3516
3517 use bytes::Bytes;
3518 use indexmap::{IndexMap, IndexSet};
3519 use log::{Level, LevelFilter, Log, Metadata, Record};
3520 #[cfg(feature = "python")]
3521 use nautilus_common::runner::{
3522 SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3523 };
3524 use nautilus_common::{
3525 actor::{
3526 self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3527 },
3528 cache::Cache,
3529 clock::{Clock, VirtualClock},
3530 enums::SerializationEncoding,
3531 live::{
3532 runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3533 sender::DispatchSender,
3534 },
3535 logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3536 messages::{
3537 data::{SubscribeCommand, SubscribeQuotes},
3538 execution::{
3539 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3540 GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3541 TradingCommand,
3542 },
3543 system::{
3544 QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3545 },
3546 },
3547 msgbus::{
3548 self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3549 MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3550 MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3551 },
3552 nautilus_actor,
3553 runner::{SyncTradingCommandSender, TradingCommandSender},
3554 testing::wait_until_async,
3555 timer::{TimeEvent, TimeEventCallback},
3556 };
3557 use nautilus_core::{Params, UUID4, UnixNanos};
3558 use nautilus_execution::{
3559 engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3560 reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3561 };
3562 use nautilus_model::{
3563 accounts::{AccountAny, MarginAccount},
3564 data::QuoteTick,
3565 enums::{
3566 AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3567 TimeInForce,
3568 },
3569 events::{
3570 AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3571 order::spec::{
3572 OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3573 },
3574 },
3575 identifiers::{
3576 AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3577 TradeId, TraderId, Venue, VenueOrderId,
3578 },
3579 instruments::{
3580 Instrument, InstrumentAny,
3581 stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3582 },
3583 orders::{
3584 Order, OrderAny, OrderList, OrderTestBuilder,
3585 stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3586 },
3587 position::PositionReplayEvent,
3588 reports::{FillReport, PositionStatusReport},
3589 types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3590 };
3591 use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3592 use nautilus_testkit::{
3593 cache::TestCacheDatabaseControl,
3594 components::{StateActor, StateStrategy},
3595 };
3596 use nautilus_trading::{
3597 nautilus_strategy,
3598 strategy::{config::StrategyConfig, core::StrategyCore},
3599 };
3600 use parking_lot::Mutex;
3601 use rstest::*;
3602 use rust_decimal::Decimal;
3603 use rust_decimal_macros::dec;
3604 use ustr::Ustr;
3605
3606 use super::{
3607 reconciliation::{
3608 PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3609 request_position_fill_reports,
3610 },
3611 *,
3612 };
3613 use crate::{
3614 execution::{
3615 client::LiveExecutionClient,
3616 manager::{PositionFillReportQuery, ReportClientCoverage},
3617 submission::{
3618 SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3619 },
3620 },
3621 socket::SocketControl,
3622 };
3623
3624 struct ExternalIngressLogCapture {
3625 messages: Mutex<Vec<String>>,
3626 }
3627
3628 static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3629 messages: Mutex::new(Vec::new()),
3630 };
3631
3632 #[derive(Debug)]
3633 enum FillReportClientOutcome {
3634 Reports(Vec<FillReport>),
3635 Failure,
3636 }
3637
3638 #[derive(Debug)]
3639 struct FillReportClient {
3640 client_id: ClientId,
3641 account_id: AccountId,
3642 venue: Venue,
3643 outcome: FillReportClientOutcome,
3644 commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3645 }
3646
3647 #[async_trait::async_trait(?Send)]
3648 impl ExecutionClient for FillReportClient {
3649 fn is_connected(&self) -> bool {
3650 true
3651 }
3652
3653 fn client_id(&self) -> ClientId {
3654 self.client_id
3655 }
3656
3657 fn account_id(&self) -> AccountId {
3658 self.account_id
3659 }
3660
3661 fn venue(&self) -> Venue {
3662 self.venue
3663 }
3664
3665 fn oms_type(&self) -> OmsType {
3666 OmsType::Netting
3667 }
3668
3669 fn get_account(&self) -> Option<AccountAny> {
3670 None
3671 }
3672
3673 fn generate_account_state(
3674 &self,
3675 _balances: Vec<AccountBalance>,
3676 _margins: Vec<MarginBalance>,
3677 _reported: bool,
3678 _ts_event: UnixNanos,
3679 _info: Option<Params>,
3680 ) -> anyhow::Result<()> {
3681 Ok(())
3682 }
3683
3684 fn start(&mut self) -> anyhow::Result<()> {
3685 Ok(())
3686 }
3687
3688 fn stop(&mut self) -> anyhow::Result<()> {
3689 Ok(())
3690 }
3691
3692 async fn generate_order_status_reports(
3693 &self,
3694 _cmd: &GenerateOrderStatusReports,
3695 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3696 match self.outcome {
3697 FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3698 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3699 }
3700 }
3701
3702 async fn generate_position_status_reports(
3703 &self,
3704 _cmd: &GeneratePositionStatusReports,
3705 ) -> anyhow::Result<Vec<PositionStatusReport>> {
3706 match self.outcome {
3707 FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3708 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3709 }
3710 }
3711
3712 async fn generate_fill_reports(
3713 &self,
3714 cmd: GenerateFillReports,
3715 ) -> anyhow::Result<Vec<FillReport>> {
3716 self.commands.borrow_mut().push(cmd);
3717
3718 match &self.outcome {
3719 FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3720 FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3721 }
3722 }
3723 }
3724
3725 #[derive(Debug)]
3726 struct OrderReportClient {
3727 client_id: ClientId,
3728 report: OrderStatusReport,
3729 targeted: bool,
3730 queries: Rc<RefCell<Vec<&'static str>>>,
3731 }
3732
3733 #[async_trait::async_trait(?Send)]
3734 impl ExecutionClient for OrderReportClient {
3735 fn is_connected(&self) -> bool {
3736 true
3737 }
3738
3739 fn client_id(&self) -> ClientId {
3740 self.client_id
3741 }
3742
3743 fn account_id(&self) -> AccountId {
3744 self.report.account_id
3745 }
3746
3747 fn venue(&self) -> Venue {
3748 self.report.instrument_id.venue
3749 }
3750
3751 fn oms_type(&self) -> OmsType {
3752 OmsType::Netting
3753 }
3754
3755 fn get_account(&self) -> Option<AccountAny> {
3756 None
3757 }
3758
3759 fn generate_account_state(
3760 &self,
3761 _balances: Vec<AccountBalance>,
3762 _margins: Vec<MarginBalance>,
3763 _reported: bool,
3764 _ts_event: UnixNanos,
3765 _info: Option<Params>,
3766 ) -> anyhow::Result<()> {
3767 Ok(())
3768 }
3769
3770 fn start(&mut self) -> anyhow::Result<()> {
3771 Ok(())
3772 }
3773
3774 fn stop(&mut self) -> anyhow::Result<()> {
3775 Ok(())
3776 }
3777
3778 async fn generate_order_status_reports(
3779 &self,
3780 _command: &GenerateOrderStatusReports,
3781 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3782 self.queries.borrow_mut().push("bulk");
3783 Ok(if self.targeted {
3784 Vec::new()
3785 } else {
3786 vec![self.report.clone()]
3787 })
3788 }
3789
3790 async fn generate_order_status_report(
3791 &self,
3792 _command: &GenerateOrderStatusReport,
3793 ) -> anyhow::Result<Option<OrderStatusReport>> {
3794 self.queries.borrow_mut().push("targeted");
3795 Ok(Some(self.report.clone()))
3796 }
3797 }
3798
3799 #[derive(Debug)]
3800 struct StartupSocketActor {
3801 core: DataActorCore,
3802 received: Rc<RefCell<Vec<SocketStateChanged>>>,
3803 }
3804
3805 impl StartupSocketActor {
3806 fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3807 Self {
3808 core: DataActorCore::new(DataActorConfig {
3809 actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3810 ..Default::default()
3811 }),
3812 received,
3813 }
3814 }
3815 }
3816
3817 impl DataActor for StartupSocketActor {
3818 fn on_start(&mut self) -> anyhow::Result<()> {
3819 self.subscribe_socket_state(None, None, None);
3820 Ok(())
3821 }
3822
3823 fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3824 self.received.borrow_mut().push(event.clone());
3825 Ok(())
3826 }
3827 }
3828
3829 nautilus_actor!(StartupSocketActor);
3830
3831 impl Log for ExternalIngressLogCapture {
3832 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3833 metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3834 }
3835
3836 fn log(&self, record: &Record<'_>) {
3837 if self.enabled(record.metadata()) {
3838 self.messages.lock().push(record.args().to_string());
3839 }
3840 }
3841
3842 fn flush(&self) {}
3843 }
3844
3845 #[derive(Debug)]
3846 struct FailingTimerActor {
3847 core: DataActorCore,
3848 received: Rc<RefCell<Vec<u64>>>,
3849 }
3850
3851 nautilus_actor!(FailingTimerActor);
3852
3853 impl DataActor for FailingTimerActor {
3854 fn on_start(&mut self) -> anyhow::Result<()> {
3855 for timestamp in [17, 23] {
3856 let received = self.received.clone();
3857
3858 let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3859 received.borrow_mut().push(timestamp);
3860
3861 if timestamp == 17 {
3862 crate::dispatch::tests::latch_callback_failure();
3863 }
3864 }));
3865
3866 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3867 TimeEvent::new(
3868 "callback-failure".into(),
3869 UUID4::new(),
3870 timestamp.into(),
3871 timestamp.into(),
3872 ),
3873 callback,
3874 ));
3875 }
3876
3877 Ok(())
3878 }
3879 }
3880
3881 #[rstest]
3882 #[tokio::test]
3883 async fn test_callback_failure_stops_later_live_events(
3884 #[values(false, true)] incomplete_submission: bool,
3885 ) {
3886 actor::clear_callbacks().unwrap();
3887
3888 let config = LiveNodeConfig {
3889 exec_engine: crate::config::LiveExecutionEngineConfig {
3890 reconciliation: false,
3891 submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3892 ..Default::default()
3893 },
3894 timeout_connection: Duration::ZERO,
3895 timeout_reconciliation: Duration::ZERO,
3896 timeout_portfolio: Duration::ZERO,
3897 timeout_disconnection: Duration::ZERO,
3898 delay_post_stop: Duration::ZERO,
3899 timeout_shutdown: Duration::ZERO,
3900 ..Default::default()
3901 };
3902
3903 let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3904
3905 if incomplete_submission {
3906 let order = OrderTestBuilder::new(OrderType::Market)
3907 .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3908 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3909 .quantity(Quantity::from("1"))
3910 .build();
3911 node.exec_manager
3912 .register_submission(order.init_event(), None);
3913 }
3914 let received = Rc::new(RefCell::new(Vec::new()));
3915 node.add_actor(FailingTimerActor {
3916 core: DataActorCore::new(DataActorConfig {
3917 actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3918 ..Default::default()
3919 }),
3920 received: received.clone(),
3921 })
3922 .unwrap();
3923
3924 let result = node.run_with_mode(NodeRunMode::Hosted).await;
3925
3926 let error = result.unwrap_err();
3927 assert!(error.to_string().contains("Callback delivery unwound"));
3928 if incomplete_submission {
3929 assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3930 }
3931 let state = node.state();
3932 let trader_stopped = node.kernel.trader.borrow().is_stopped();
3933 let failure = actor::callback_failure();
3934 node.dispose();
3935
3936 assert_eq!(
3937 error.downcast_ref::<CallbackDispatchError>(),
3938 Some(&CallbackDispatchError::DeliveryUnwound)
3939 );
3940 assert_eq!(*received.borrow(), [17]);
3941 assert_eq!(state, NodeState::Stopped);
3942 assert!(trader_stopped);
3943 assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
3944 assert_eq!(actor::callback_failure(), None);
3945 assert_eq!(actor::clear_callbacks(), Ok(()));
3946 }
3947
3948 #[rstest]
3949 fn test_render_client_statuses() {
3950 let rows = vec![
3951 ClientStatus {
3952 client: "BINANCE".to_string(),
3953 client_type: "Data",
3954 connected: true,
3955 },
3956 ClientStatus {
3957 client: "SIM".to_string(),
3958 client_type: "Execution",
3959 connected: false,
3960 },
3961 ];
3962
3963 let output = render_client_statuses(rows);
3964 let expected = "â•─────────┬───────────┬───────────╮\n\
3965│ Client │ Type │ Connected │\n\
3966├─────────┼───────────┼───────────┤\n\
3967│ BINANCE │ Data │ true │\n\
3968│ SIM │ Execution │ false │\n\
3969╰─────────┴───────────┴───────────╯";
3970
3971 assert_eq!(output, expected);
3972 }
3973
3974 #[rstest]
3975 fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
3976 log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
3977 log::set_max_level(LevelFilter::Error);
3978 EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
3979 let message = BusMessage::with_str_topic(
3980 "data.quotes.AUDUSD.SIM*",
3981 BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
3982 Bytes::new(),
3983 SerializationEncoding::Json,
3984 );
3985
3986 LiveNode::republish_external_msgbus_message(&message);
3987
3988 assert_eq!(
3989 *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
3990 vec![
3991 "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
3992 external message topic: Topic `value` contained invalid characters, was \
3993 data.quotes.AUDUSD.SIM*"
3994 .to_string()
3995 ],
3996 );
3997 }
3998
3999 #[rstest]
4000 #[tokio::test]
4001 async fn test_external_startup_prioritizes_completed_connection() {
4002 let message = BusMessage::with_str_topic(
4003 "data.trades.FIRST",
4004 BusPayloadType::TradeTick,
4005 bytes::Bytes::new(),
4006 SerializationEncoding::Json,
4007 );
4008 let (sender, receiver) = tokio::sync::mpsc::channel(1);
4009 sender.send(message.clone()).await.unwrap();
4010 let mut receiver = Some(receiver);
4011 let mut pending = VecDeque::new();
4012
4013 let result =
4014 drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4015 let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4016
4017 assert_eq!(result, 17);
4018 assert!(pending.is_empty());
4019 assert_eq!(
4020 serde_json::to_value(queued).unwrap(),
4021 serde_json::to_value(message).unwrap()
4022 );
4023 }
4024
4025 #[rstest]
4026 #[tokio::test]
4027 async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4028 use bytes::Bytes;
4029 use nautilus_common::messages::data::{
4030 CustomDataResponse, DataResponse, DataResponseEnvelope,
4031 };
4032 use nautilus_model::data::DataType;
4033
4034 let (egress, publications, _) = CapturingExternalEgress::new();
4035 let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4036 msgbus::set_message_bus(bus.clone());
4037 bus.borrow_mut()
4038 .set_external_egress_config(
4039 Box::new(egress),
4040 &MessageBusConfig {
4041 encoding: SerializationEncoding::MsgPack,
4042 external_streams: Some(vec!["external.test".into()]),
4043 ..Default::default()
4044 },
4045 )
4046 .unwrap();
4047
4048 bus.borrow_mut()
4049 .add_streaming_type(BusPayloadType::DataResponse);
4050 let id = UUID4::new();
4051 let response = DataResponse::Data(CustomDataResponse::new(
4052 id,
4053 "FEED".into(),
4054 None,
4055 DataType::new("Startup", None, None),
4056 Bytes::from_static(b"connected"),
4057 Some(101.into()),
4058 Some(103.into()),
4059 107.into(),
4060 None,
4061 ));
4062 msgbus::publish_any("responses.data.FEED".into(), &response);
4063 let message = BusMessage::with_str_topic(
4064 "responses.data.FEED",
4065 BusPayloadType::DataResponse,
4066 publications.borrow()[0].payload.clone(),
4067 SerializationEncoding::MsgPack,
4068 );
4069 let (done, connected) = tokio::sync::oneshot::channel();
4070 let done = Rc::new(RefCell::new(Some(done)));
4071 msgbus::register_response_handler(
4072 &id,
4073 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4074 let Some(DataResponse::Data(reply)) = envelope.response() else {
4075 panic!("expected custom reply");
4076 };
4077
4078 done.borrow_mut()
4079 .take()
4080 .unwrap()
4081 .send(reply.clone())
4082 .unwrap();
4083 }),
4084 );
4085
4086 msgbus::subscribe_any(
4087 message.topic.as_str().into(),
4088 ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4089 msgbus::send_response(reply.correlation_id(), reply);
4090 }),
4091 None,
4092 );
4093
4094 let unrelated = BusMessage::with_str_topic(
4095 "data.trades.FIRST",
4096 BusPayloadType::TradeTick,
4097 Bytes::new(),
4098 SerializationEncoding::Json,
4099 );
4100 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4101 sender.send(unrelated.clone()).await.unwrap();
4102 sender.send(message).await.unwrap();
4103 let mut receiver = Some(receiver);
4104 let mut pending = VecDeque::new();
4105 let actual = tokio::time::timeout(
4106 Duration::from_secs(1),
4107 drive_with_external_ingress(connected, &mut receiver, &mut pending),
4108 )
4109 .await
4110 .unwrap()
4111 .unwrap();
4112
4113 assert_eq!(actual.correlation_id, id);
4114 assert_eq!(actual.client_id, ClientId::from("FEED"));
4115 assert_eq!(actual.venue, None);
4116 assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4117 assert_eq!(
4118 actual.data.downcast_ref::<Bytes>(),
4119 Some(&Bytes::from_static(b"connected"))
4120 );
4121 assert_eq!(actual.start, Some(101.into()));
4122 assert_eq!(actual.end, Some(103.into()));
4123 assert_eq!(actual.ts_init, UnixNanos::from(107));
4124 assert_eq!(actual.params, None);
4125 assert_eq!(publications.borrow().len(), 1);
4126 assert_eq!(pending.len(), 1);
4127 assert_eq!(
4128 serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4129 serde_json::to_value(unrelated).unwrap()
4130 );
4131 assert!(receiver.as_ref().unwrap().is_empty());
4132 }
4133
4134 #[rstest]
4135 #[tokio::test]
4136 async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4137 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4138 let mut receiver = Some(receiver);
4139
4140 let message = |name: &str| {
4141 BusMessage::with_str_topic(
4142 name,
4143 BusPayloadType::TradeTick,
4144 bytes::Bytes::new(),
4145 SerializationEncoding::Json,
4146 )
4147 };
4148
4149 let first = message("data.trades.FIRST");
4150 let second = message("data.trades.SECOND");
4151 let third = message("data.trades.THIRD");
4152 sender.send(first.clone()).await.unwrap();
4153 sender.send(second.clone()).await.unwrap();
4154 let (done, connected) = tokio::sync::oneshot::channel();
4155 let mut pending = VecDeque::new();
4156 let mut connection = Box::pin(drive_with_external_ingress(
4157 connected,
4158 &mut receiver,
4159 &mut pending,
4160 ));
4161 assert!(futures_util::poll!(&mut connection).is_pending());
4162 sender.send(third.clone()).await.unwrap();
4163 assert!(futures_util::poll!(&mut connection).is_pending());
4164 done.send(()).unwrap();
4165 connection.await.unwrap();
4166
4167 assert_eq!(pending.len(), 2);
4168 assert_eq!(receiver.as_ref().unwrap().len(), 1);
4169
4170 for expected in [first, second, third] {
4171 let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4172 .await
4173 .unwrap();
4174 assert_eq!(
4175 serde_json::to_value(received).unwrap(),
4176 serde_json::to_value(expected).unwrap()
4177 );
4178 }
4179 }
4180
4181 #[rstest]
4182 #[tokio::test]
4183 async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4184 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4185 drop(sender);
4186 let mut receiver = Some(receiver);
4187 let mut pending = VecDeque::new();
4188 let (done, connected) = tokio::sync::oneshot::channel();
4189 let mut connection = Box::pin(drive_with_external_ingress(
4190 connected,
4191 &mut receiver,
4192 &mut pending,
4193 ));
4194 assert!(futures_util::poll!(&mut connection).is_pending());
4195 done.send(()).unwrap();
4196 connection.await.unwrap();
4197
4198 assert!(receiver.is_none());
4199 assert!(pending.is_empty());
4200 }
4201
4202 #[rstest]
4203 #[case(None, 2)]
4204 #[case(Some(SystemChannel::DataEvents), 1)]
4205 #[case(Some(SystemChannel::ExecCommands), 1)]
4206 fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4207 #[case] channel: Option<SystemChannel>,
4208 #[case] expected_count: usize,
4209 ) {
4210 let config = LiveNodeConfig {
4211 trader_id: TraderId::from("QUEUE-001"),
4212 exec_engine: crate::config::LiveExecutionEngineConfig {
4213 reconciliation: false,
4214 ..Default::default()
4215 },
4216 ..Default::default()
4217 };
4218
4219 let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4220 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4221
4222 let handler = ShareableMessageHandler::from_typed({
4223 let received = received.clone();
4224 move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4225 });
4226
4227 msgbus::subscribe_any(
4228 MessagingSwitchboard::queue_state_changed_pattern(channel),
4229 handler,
4230 None,
4231 );
4232
4233 let transitions = [
4234 QueueStateTransition {
4235 channel: SystemChannel::DataEvents,
4236 condition: QueueCondition::Backlogged,
4237 state: QueueState::Triggered,
4238 queue_depth: 17,
4239 mean_dispatch_ns: 23,
4240 },
4241 QueueStateTransition {
4242 channel: SystemChannel::ExecCommands,
4243 condition: QueueCondition::Slow,
4244 state: QueueState::Triggered,
4245 queue_depth: 17,
4246 mean_dispatch_ns: 23,
4247 },
4248 ];
4249
4250 node.publish_queue_state_transitions(&transitions);
4251
4252 let events = received.borrow();
4253 assert_eq!(events.len(), expected_count);
4254
4255 let expected = transitions
4256 .into_iter()
4257 .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4258
4259 for (event, transition) in events.iter().zip(expected) {
4260 assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4261 assert_eq!(event.channel, transition.channel);
4262 assert_eq!(event.condition, transition.condition);
4263 assert_eq!(event.state, transition.state);
4264 assert_eq!(event.queue_depth, transition.queue_depth);
4265 assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4266 assert_ne!(event.event_id, UUID4::default());
4267 assert_ne!(event.ts_event, UnixNanos::default());
4268 assert_eq!(event.ts_init, event.ts_event);
4269 }
4270
4271 if channel.is_none() {
4272 assert_ne!(events[0].event_id, events[1].event_id);
4273 }
4274
4275 drop(events);
4276 msgbus::get_message_bus().borrow_mut().dispose();
4277 }
4278
4279 #[rstest]
4280 #[case(None, None)]
4281 #[case(Some(ClientId::from("BINANCE")), None)]
4282 #[case(None, Some("binance-futures-market-streams"))]
4283 #[case(
4284 Some(ClientId::from("BINANCE")),
4285 Some("binance-futures-market-streams")
4286 )]
4287 fn test_process_socket_state_change_reaches_typed_subscriber(
4288 #[case] client_id: Option<ClientId>,
4289 #[case] endpoint: Option<&str>,
4290 ) {
4291 let config = LiveNodeConfig {
4292 trader_id: TraderId::from("SOCKET-001"),
4293 exec_engine: crate::config::LiveExecutionEngineConfig {
4294 reconciliation: false,
4295 ..Default::default()
4296 },
4297 ..Default::default()
4298 };
4299
4300 let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4301 let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4302
4303 let handler = ShareableMessageHandler::from_typed({
4304 let received = received.clone();
4305 move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4306 });
4307
4308 msgbus::subscribe_any(
4309 MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4310 handler,
4311 None,
4312 );
4313
4314 let change = SocketStateChange::new(
4315 ClientId::from("BINANCE"),
4316 Some(Venue::from("BINANCE")),
4317 Ustr::from("binance-futures-market-streams"),
4318 SocketState::Disconnected,
4319 );
4320
4321 node.process_system_event(SystemEvent::SocketState(change));
4322
4323 let events = received.borrow();
4324 assert_eq!(events.len(), 1);
4325 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4326 assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4327 assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4328 assert_eq!(
4329 events[0].endpoint,
4330 Ustr::from("binance-futures-market-streams")
4331 );
4332 assert_eq!(events[0].state, SocketState::Disconnected);
4333 assert_ne!(events[0].event_id, UUID4::default());
4334 assert_ne!(events[0].ts_event, UnixNanos::default());
4335 assert_eq!(events[0].ts_init, events[0].ts_event);
4336 drop(events);
4337 msgbus::get_message_bus().borrow_mut().dispose();
4338 }
4339
4340 #[rstest]
4341 #[case::accepted(
4342 ReconnectRequestOutcome::Accepted,
4343 SocketReconnectDispatchOutcome::Accepted
4344 )]
4345 #[case::already_reconnecting(
4346 ReconnectRequestOutcome::AlreadyReconnecting,
4347 SocketReconnectDispatchOutcome::AlreadyReconnecting
4348 )]
4349 #[case::disconnected(
4350 ReconnectRequestOutcome::Disconnected,
4351 SocketReconnectDispatchOutcome::Disconnected
4352 )]
4353 #[case::closed(
4354 ReconnectRequestOutcome::Closed,
4355 SocketReconnectDispatchOutcome::Closed
4356 )]
4357 #[case::unsupported(
4358 ReconnectRequestOutcome::Unsupported,
4359 SocketReconnectDispatchOutcome::Unsupported
4360 )]
4361 fn test_request_socket_reconnect_maps_transport_outcome(
4362 #[case] transport: ReconnectRequestOutcome,
4363 #[case] expected: SocketReconnectDispatchOutcome,
4364 ) {
4365 let registry = SocketReconnectRegistry::default();
4366 let client_id = ClientId::from("TEST");
4367 let endpoint = Ustr::from("test-streams");
4368 let control = SocketControl::with_registry(client_id, None, endpoint, ®istry);
4369 let _sink = control.sink();
4370 control.register(move || transport);
4371
4372 let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4373
4374 assert_eq!(outcome, expected);
4375 }
4376
4377 #[rstest]
4378 #[case::client_not_found(
4379 SocketReconnectLookup::ClientNotFound,
4380 SocketReconnectDispatchOutcome::UnknownClient
4381 )]
4382 #[case::unsupported(
4383 SocketReconnectLookup::Unsupported,
4384 SocketReconnectDispatchOutcome::Unsupported
4385 )]
4386 #[case::endpoint_not_found(
4387 SocketReconnectLookup::EndpointNotFound,
4388 SocketReconnectDispatchOutcome::UnknownEndpoint
4389 )]
4390 #[case::ambiguous(
4391 SocketReconnectLookup::AmbiguousEndpoint,
4392 SocketReconnectDispatchOutcome::AmbiguousEndpoint
4393 )]
4394 fn test_request_socket_reconnect_maps_lookup_failure(
4395 #[case] lookup: SocketReconnectLookup,
4396 #[case] expected: SocketReconnectDispatchOutcome,
4397 ) {
4398 assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4399 }
4400
4401 #[rstest]
4402 fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4403 let trader_id = TraderId::from("SOCKET-001");
4404
4405 let config = LiveNodeConfig {
4406 trader_id,
4407 ..Default::default()
4408 };
4409
4410 let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4411 let client_id = ClientId::from("TEST");
4412 let endpoint = Ustr::from("test-streams");
4413 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4414 let sender = DispatchSender::new(tx);
4415 let events = sender.clone();
4416 msgbus::subscribe_any(
4417 MessagingSwitchboard::socket_state_changed_pattern(
4418 Some(client_id),
4419 Some(endpoint.as_str()),
4420 ),
4421 ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4422 events.send(17u32).unwrap();
4423 }),
4424 None,
4425 );
4426
4427 let control =
4428 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4429 let _sink = control.sink();
4430 control.register(move || {
4431 sender.send(23u32).unwrap();
4432 ReconnectRequestOutcome::Accepted
4433 });
4434
4435 let event = SystemEvent::SocketState(SocketStateChange::new(
4436 client_id,
4437 None,
4438 endpoint,
4439 SocketState::Connected,
4440 ));
4441 let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4442 trader_id,
4443 client_id,
4444 endpoint,
4445 31.into(),
4446 ));
4447
4448 if startup {
4449 let mut pending = PendingEvents::default();
4450 pending.system_events.push(event.into());
4451 pending.system_commands.push(command.into());
4452 node.process_system_events(pending.take_system_events());
4453 node.process_system_commands(pending.take_system_commands());
4454 assert!(pending.is_empty());
4455 } else {
4456 node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4457 node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4458 }
4459
4460 let event_child = rx.try_recv().unwrap();
4461 let command_child = rx.try_recv().unwrap();
4462 assert!(event_child.is_rooted());
4463 assert!(command_child.is_rooted());
4464 assert_eq!(event_child.dispatch(|value| value), 17);
4465 assert_eq!(command_child.dispatch(|value| value), 23);
4466 assert!(rx.is_empty());
4467 msgbus::get_message_bus().borrow_mut().dispose();
4468 }
4469
4470 #[rstest]
4471 fn test_process_socket_reconnect_routes_only_matching_trader() {
4472 let trader_id = TraderId::from("SOCKET-001");
4473
4474 let config = LiveNodeConfig {
4475 trader_id,
4476 exec_engine: crate::config::LiveExecutionEngineConfig {
4477 reconciliation: false,
4478 ..Default::default()
4479 },
4480 ..Default::default()
4481 };
4482
4483 let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4484 let client_id = ClientId::from("TEST");
4485 let endpoint = Ustr::from("test-streams");
4486 let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4487 let request_count = Arc::clone(&requests);
4488 let control =
4489 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4490 let _sink = control.sink();
4491 control.register(move || {
4492 request_count.fetch_add(1, Ordering::SeqCst);
4493 ReconnectRequestOutcome::Accepted
4494 });
4495
4496 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4497 TraderId::from("OTHER-001"),
4498 client_id,
4499 endpoint,
4500 UnixNanos::default(),
4501 )));
4502 assert_eq!(requests.load(Ordering::SeqCst), 0);
4503
4504 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4505 trader_id,
4506 client_id,
4507 endpoint,
4508 UnixNanos::default(),
4509 )));
4510 assert_eq!(requests.load(Ordering::SeqCst), 1);
4511 }
4512
4513 #[rstest]
4514 #[tokio::test]
4515 async fn test_start_publishes_socket_change_after_actor_subscribes() {
4516 let config = LiveNodeConfig {
4517 trader_id: TraderId::from("SOCKET-STARTUP-001"),
4518 exec_engine: crate::config::LiveExecutionEngineConfig {
4519 reconciliation: false,
4520 ..Default::default()
4521 },
4522 timeout_connection: Duration::ZERO,
4523 timeout_reconciliation: Duration::ZERO,
4524 timeout_portfolio: Duration::ZERO,
4525 timeout_disconnection: Duration::ZERO,
4526 delay_post_stop: Duration::ZERO,
4527 timeout_shutdown: Duration::ZERO,
4528 ..Default::default()
4529 };
4530
4531 let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4532 let received = Rc::new(RefCell::new(Vec::new()));
4533 node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4534 .unwrap();
4535 node.runner.as_ref().unwrap().bind_senders();
4536
4537 let change = SocketStateChange::new(
4538 ClientId::from("BINANCE"),
4539 Some(Venue::from("BINANCE")),
4540 Ustr::from("binance-futures-market-streams"),
4541 SocketState::Connected,
4542 );
4543 get_system_event_sender()
4544 .send(SystemEvent::SocketState(change))
4545 .unwrap();
4546
4547 node.start().await.unwrap();
4548
4549 {
4550 let events = received.borrow();
4551 assert_eq!(events.len(), 1);
4552 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4553 assert_eq!(events[0].client_id, change.client_id);
4554 assert_eq!(events[0].venue, change.venue);
4555 assert_eq!(events[0].endpoint, change.endpoint);
4556 assert_eq!(events[0].state, change.state);
4557 assert_ne!(events[0].event_id, UUID4::default());
4558 assert_ne!(events[0].ts_event, UnixNanos::default());
4559 assert_eq!(events[0].ts_init, events[0].ts_event);
4560 }
4561
4562 node.stop().await.unwrap();
4563 node.dispose();
4564 }
4565
4566 #[rstest]
4567 #[tokio::test(flavor = "current_thread")]
4568 async fn test_run_publishes_queue_state_after_dispatch_sample() {
4569 let config = LiveNodeConfig {
4570 trader_id: TraderId::from("QUEUE-RUN-001"),
4571 queue_monitor: Some(crate::config::QueueMonitorConfig {
4572 queue_depth_trigger: usize::MAX,
4573 queue_depth_clear: 0,
4574 mean_dispatch_ns_trigger: 1,
4575 mean_dispatch_ns_clear: 0,
4576 }),
4577 exec_engine: crate::config::LiveExecutionEngineConfig {
4578 reconciliation: false,
4579 ..Default::default()
4580 },
4581 delay_post_stop: Duration::ZERO,
4582 ..Default::default()
4583 };
4584
4585 let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4586 let handle = node.handle();
4587 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4588
4589 let handler = ShareableMessageHandler::from_typed({
4590 let received = received.clone();
4591 let stop_handle = handle.clone();
4592
4593 move |event: &QueueStateChanged| {
4594 received.borrow_mut().push(event.clone());
4595 stop_handle.stop();
4596 }
4597 });
4598
4599 msgbus::subscribe_any(
4600 MessagingSwitchboard::queue_state_changed_pattern(None),
4601 handler,
4602 None,
4603 );
4604 let drive_handle = handle.clone();
4605
4606 let result = tokio::time::timeout(Duration::from_secs(5), async {
4607 let run = node.run();
4608 tokio::pin!(run);
4609
4610 let drive = async move {
4611 wait_until_async(
4612 || async { drive_handle.is_running() },
4613 Duration::from_secs(2),
4614 )
4615 .await;
4616 get_data_event_sender().send(stub_data_event()).unwrap();
4617 };
4618
4619 let (run_result, ()) = tokio::join!(run, drive);
4620 run_result
4621 })
4622 .await;
4623
4624 assert!(
4625 result.is_ok(),
4626 "queue state event should arrive before timeout"
4627 );
4628 assert!(result.unwrap().is_ok(), "run() should succeed");
4629 let events = received.borrow();
4630 assert_eq!(events.len(), 1);
4631 assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4632 assert_eq!(events[0].channel, SystemChannel::DataEvents);
4633 assert_eq!(events[0].condition, QueueCondition::Slow);
4634 assert_eq!(events[0].state, QueueState::Triggered);
4635 assert_eq!(events[0].queue_depth, 0);
4636 assert!(events[0].mean_dispatch_ns > 0);
4637 assert_ne!(events[0].event_id, UUID4::default());
4638 assert_ne!(events[0].ts_event, UnixNanos::default());
4639 assert_eq!(events[0].ts_init, events[0].ts_event);
4640 drop(events);
4641 msgbus::get_message_bus().borrow_mut().dispose();
4642 }
4643
4644 #[rstest]
4645 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4646 async fn test_run_dispatch_progress_under_load(
4647 #[values(false, true)] execution: bool,
4648 #[values(false, true)] slow: bool,
4649 ) {
4650 let config = LiveNodeConfig {
4651 delay_post_stop: Duration::ZERO,
4652 ..Default::default()
4653 };
4654 let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4655 let handle = node.handle();
4656 let stop = handle.clone();
4657 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4658 if slow {
4659 std::thread::sleep(Duration::from_millis(110));
4661 }
4662
4663 if event.ts_init == UnixNanos::from(8) {
4664 let snapshot = stop.metrics_snapshot();
4666 assert_eq!(snapshot.time_events.dispatched, 7);
4667
4668 if execution {
4669 assert!((1..16).contains(&snapshot.exec_events.dispatched));
4670 assert_eq!(snapshot.data_events.dispatched, 0);
4671 } else {
4672 assert!(snapshot.data_events.dispatched > 0);
4673 assert_eq!(snapshot.data_commands.dispatched, 1);
4674 }
4675 stop.stop();
4676 }
4677 }));
4678
4679 let drive = async {
4680 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4681
4682 for ts in (1..=16_u64).map(UnixNanos::from) {
4683 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4684 TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4685 callback.clone(),
4686 ));
4687 get_data_event_sender().send(stub_data_event()).unwrap();
4688
4689 if execution {
4690 get_exec_event_sender().send(stub_exec_event()).unwrap();
4691 }
4692 }
4693 nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4694 };
4695 tokio::time::timeout(Duration::from_secs(5), async {
4696 let (result, ()) = tokio::join!(node.run(), drive);
4697 result.unwrap();
4698 })
4699 .await
4700 .unwrap();
4701 assert!(handle.should_stop());
4702 }
4703
4704 #[rstest]
4705 fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4706 let config = LiveNodeConfig {
4707 exec_engine: crate::config::LiveExecutionEngineConfig {
4708 reconciliation: false,
4709 ..Default::default()
4710 },
4711 ..Default::default()
4712 };
4713
4714 let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4715 let event = stub_exec_event();
4716 let account_id = AccountId::from("TEST-001");
4717 let instrument_id = InstrumentId::from("TEST.VENUE");
4718 let trade_id = TradeId::from("T-001");
4719
4720 let close_ids = node.observe_exec_event_before_dispatch(&event);
4721 assert_eq!(close_ids, Some(Vec::new()));
4722 assert!(
4723 !node
4724 .exec_manager
4725 .is_fill_recently_processed(account_id, instrument_id, trade_id)
4726 );
4727
4728 node.exec_manager
4729 .mark_fill_processed(account_id, instrument_id, trade_id);
4730
4731 let close_ids = node.observe_exec_event_before_dispatch(&event);
4732 assert_eq!(close_ids, None);
4733 }
4734
4735 #[rstest]
4736 #[case(false, false, OrderStatus::Canceled, 1)]
4737 #[case(false, true, OrderStatus::Accepted, 0)]
4738 #[case(true, false, OrderStatus::Canceled, 1)]
4739 #[case(true, true, OrderStatus::Accepted, 0)]
4740 fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4741 #[case] with_fills: bool,
4742 #[case] superseded: bool,
4743 #[case] expected_status: OrderStatus,
4744 #[case] expected_query_count: usize,
4745 ) {
4746 let config = LiveNodeConfig {
4747 exec_engine: crate::config::LiveExecutionEngineConfig {
4748 reconciliation: true,
4749 open_check_threshold_ms: 5_000,
4750 single_order_query_delay_ms: 0,
4751 ..Default::default()
4752 },
4753 ..Default::default()
4754 };
4755
4756 let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4757 let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4758 let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4759 let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4760 let account_id = AccountId::from("TEST-001");
4761 let client_id = ClientId::from("TEST");
4762 let instrument = crypto_perpetual_ethusdt();
4763 let instrument_id = instrument.id();
4764 let account = AccountAny::Margin(MarginAccount::new(
4765 AccountState::new(
4766 account_id,
4767 AccountType::Margin,
4768 vec![AccountBalance::new(
4769 Money::from("1000000 USDT"),
4770 Money::from("0 USDT"),
4771 Money::from("1000000 USDT"),
4772 )],
4773 Vec::new(),
4774 true,
4775 UUID4::new(),
4776 UnixNanos::default(),
4777 UnixNanos::default(),
4778 Some(Currency::USDT()),
4779 ),
4780 true,
4781 ));
4782 node.kernel.cache.borrow_mut().add_account(account).unwrap();
4783 node.kernel
4784 .cache
4785 .borrow_mut()
4786 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4787 .unwrap();
4788 insert_accepted_limit_order_in_node(
4789 &node,
4790 account_id,
4791 client_id,
4792 instrument_id,
4793 client_order_id,
4794 old_venue_order_id,
4795 );
4796
4797 if superseded {
4798 let order = node
4799 .kernel
4800 .cache
4801 .borrow()
4802 .order_owned(&client_order_id)
4803 .unwrap();
4804 let pending_update = OrderPendingUpdateSpec::builder()
4805 .trader_id(order.trader_id())
4806 .strategy_id(order.strategy_id())
4807 .instrument_id(order.instrument_id())
4808 .client_order_id(client_order_id)
4809 .account_id(account_id)
4810 .venue_order_id(old_venue_order_id)
4811 .build();
4812 node.kernel
4813 .cache
4814 .borrow_mut()
4815 .update_order(&OrderEventAny::PendingUpdate(pending_update))
4816 .unwrap();
4817 let order = node
4818 .kernel
4819 .cache
4820 .borrow()
4821 .order_owned(&client_order_id)
4822 .unwrap();
4823 let updated = OrderUpdatedSpec::builder()
4824 .trader_id(order.trader_id())
4825 .strategy_id(order.strategy_id())
4826 .instrument_id(order.instrument_id())
4827 .client_order_id(client_order_id)
4828 .quantity(order.quantity())
4829 .venue_order_id(new_venue_order_id)
4830 .account_id(account_id)
4831 .build();
4832 node.kernel
4833 .cache
4834 .borrow_mut()
4835 .update_order(&OrderEventAny::Updated(updated))
4836 .unwrap();
4837 }
4838
4839 let report = OrderStatusReport::new(
4840 account_id,
4841 instrument_id,
4842 Some(client_order_id),
4843 old_venue_order_id,
4844 OrderSide::Buy.into(),
4845 OrderType::Limit,
4846 TimeInForce::Gtc,
4847 OrderStatus::Canceled,
4848 Quantity::from("10.0"),
4849 Quantity::from("0.0"),
4850 UnixNanos::from(1_000),
4851 UnixNanos::from(2_000),
4852 UnixNanos::from(3_000),
4853 None,
4854 );
4855
4856 let report = if with_fills {
4857 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4858 } else {
4859 ExecutionReport::Order(Box::new(report))
4860 };
4861
4862 let event = ExecutionEvent::Report(report);
4863
4864 node.process_exec_event(event);
4865
4866 let order = node
4867 .kernel
4868 .cache
4869 .borrow()
4870 .order_owned(&client_order_id)
4871 .unwrap();
4872
4873 let expected_venue_order_id = if superseded {
4874 new_venue_order_id
4875 } else {
4876 old_venue_order_id
4877 };
4878
4879 assert_eq!(order.status(), expected_status);
4880 assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4881
4882 if !superseded {
4883 let replacement = OrderTestBuilder::new(OrderType::Limit)
4884 .client_order_id(client_order_id)
4885 .instrument_id(instrument_id)
4886 .quantity(Quantity::from("20.0"))
4887 .price(Price::from("200.0"))
4888 .build();
4889 let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4890 node.kernel
4891 .cache
4892 .borrow_mut()
4893 .add_order(replacement, None, Some(client_id), true)
4894 .unwrap();
4895 let replacement = node
4896 .kernel
4897 .cache
4898 .borrow_mut()
4899 .update_order(&submitted)
4900 .unwrap();
4901 let accepted =
4902 TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4903 node.kernel
4904 .cache
4905 .borrow_mut()
4906 .update_order(&accepted)
4907 .unwrap();
4908 }
4909
4910 assert_eq!(
4911 node.exec_manager.check_open_order_queries().len(),
4912 expected_query_count,
4913 );
4914 }
4915
4916 #[rstest]
4917 fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4918 let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4919
4920 let OrderEventAny::Filled(fill) = &mut fill_event else {
4921 unreachable!();
4922 };
4923
4924 fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4925 fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4926 let fill = fill.clone();
4927 let report_event = fill_report_event(&fill);
4928 let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4929
4930 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4931 let marked_before_dispatch = is_recent_fill(&node, &fill);
4932
4933 node.dispatch_exec_event_and_commit_fill(event);
4934
4935 let marked_after_dispatch = is_recent_fill(&node, &fill);
4936 let later_report_is_eligible = node
4937 .observe_exec_event_before_dispatch(&report_event)
4938 .is_some();
4939 assert_eq!(
4940 (
4941 marked_before_dispatch,
4942 marked_after_dispatch,
4943 later_report_is_eligible,
4944 ),
4945 (false, false, true),
4946 );
4947 }
4948
4949 #[rstest]
4950 fn test_applied_direct_fill_commits_and_skips_later_report() {
4951 let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
4952
4953 let OrderEventAny::Filled(fill) = &fill_event else {
4954 unreachable!();
4955 };
4956
4957 let fill = fill.clone();
4958 let report_event = fill_report_event(&fill);
4959 let event = ExecutionEvent::Order(fill_event);
4960
4961 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4962 assert!(!is_recent_fill(&node, &fill));
4963
4964 node.dispatch_exec_event_and_commit_fill(event);
4965
4966 assert!(is_recent_fill(&node, &fill));
4967 assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
4968 }
4969
4970 #[rstest]
4971 fn test_canonical_duplicate_fill_counts_as_applied() {
4972 let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
4973
4974 let OrderEventAny::Filled(fill) = &fill_event else {
4975 unreachable!();
4976 };
4977
4978 let mut fill = fill.clone();
4979 node.kernel
4980 .cache
4981 .borrow_mut()
4982 .update_order(&fill_event)
4983 .unwrap();
4984 fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
4985
4986 node.exec_manager.commit_recent_fill_if_applied(&fill);
4987
4988 assert!(is_recent_fill(&node, &fill));
4989 }
4990
4991 #[rstest]
4992 #[case(false)]
4993 #[case(true)]
4994 fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
4995 let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
4996 "AppliedContinuousFillNode"
4997 } else {
4998 "RejectedContinuousFillNode"
4999 });
5000
5001 if !applied {
5002 let OrderEventAny::Filled(fill) = &mut fill_event else {
5003 unreachable!();
5004 };
5005
5006 fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5007 fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5008 }
5009
5010 let OrderEventAny::Filled(fill) = &fill_event else {
5011 unreachable!();
5012 };
5013
5014 let fill = fill.clone();
5015
5016 node.process_reconciliation_events(&[fill_event]);
5017
5018 assert_eq!(is_recent_fill(&node, &fill), applied);
5019 }
5020
5021 #[rstest]
5022 fn test_recent_fill_commit_requires_account_and_instrument_match() {
5023 let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5024 node.kernel
5025 .cache
5026 .borrow_mut()
5027 .update_order(&fill_event)
5028 .unwrap();
5029
5030 let OrderEventAny::Filled(fill) = fill_event else {
5031 unreachable!();
5032 };
5033
5034 let mut account_mismatch = fill.clone();
5035 account_mismatch.account_id = AccountId::from("OTHER-001");
5036 let mut instrument_mismatch = fill;
5037 instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5038
5039 node.exec_manager
5040 .commit_recent_fill_if_applied(&account_mismatch);
5041 node.exec_manager
5042 .commit_recent_fill_if_applied(&instrument_mismatch);
5043
5044 assert!(!is_recent_fill(&node, &account_mismatch));
5045 assert!(!is_recent_fill(&node, &instrument_mismatch));
5046 }
5047
5048 #[rstest]
5049 fn test_applied_inferred_fill_remains_recently_processed() {
5050 let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5051 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5052 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5053 let account_id = AccountId::from("TEST-001");
5054 let order = node
5055 .kernel
5056 .cache
5057 .borrow()
5058 .order_owned(&client_order_id)
5059 .unwrap();
5060 let report = OrderStatusReport::new(
5061 account_id,
5062 instrument.id(),
5063 Some(client_order_id),
5064 venue_order_id,
5065 OrderSide::Buy.into(),
5066 OrderType::Limit,
5067 TimeInForce::Gtc,
5068 OrderStatus::PartiallyFilled,
5069 Quantity::from("10.0"),
5070 Quantity::from("1.0"),
5071 UnixNanos::from(1_000),
5072 UnixNanos::from(1_000),
5073 UnixNanos::from(1_000),
5074 None,
5075 )
5076 .with_avg_px(dec!(100.0));
5077 let inferred = create_inferred_fill_for_qty(
5078 &order,
5079 &report,
5080 &account_id,
5081 &instrument,
5082 Quantity::from("1.0"),
5083 UnixNanos::from(1_000),
5084 None,
5085 )
5086 .unwrap();
5087
5088 let OrderEventAny::Filled(fill) = &inferred else {
5089 unreachable!();
5090 };
5091
5092 let fill = fill.clone();
5093
5094 node.process_reconciliation_events(&[inferred]);
5095
5096 assert!(fill.reconciliation);
5097 assert!(is_recent_fill(&node, &fill));
5098 }
5099
5100 #[rstest]
5101 #[tokio::test]
5102 async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5103 let client_id = ClientId::from("POSITION-FILLS");
5104 let account_id = AccountId::from("POSITION-FILLS-001");
5105 let instrument_id = crypto_perpetual_ethusdt().id();
5106 let commands = Rc::new(RefCell::new(Vec::new()));
5107
5108 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5109 client_id,
5110 account_id,
5111 venue: instrument_id.venue,
5112 outcome: FillReportClientOutcome::Failure,
5113 commands: commands.clone(),
5114 }));
5115
5116 let command = GenerateFillReports::new(
5117 UUID4::new(),
5118 UnixNanos::from(2_000),
5119 Some(instrument_id),
5120 None,
5121 Some(UnixNanos::from(1_000)),
5122 Some(UnixNanos::from(2_000)),
5123 None,
5124 None,
5125 );
5126
5127 let result = request_position_fill_reports(
5128 vec![client],
5129 vec![PositionFillReportQuery {
5130 key: (instrument_id, account_id),
5131 client_id,
5132 command: command.clone(),
5133 }],
5134 )
5135 .await
5136 .finish();
5137
5138 assert_eq!(*commands.borrow(), vec![command]);
5139 assert!(result.is_empty());
5140 }
5141
5142 #[rstest]
5143 #[tokio::test]
5144 async fn test_request_position_fill_reports_accepts_scoped_response() {
5145 let client_id = ClientId::from("POSITION-FILLS");
5146 let account_id = AccountId::from("POSITION-FILLS-001");
5147 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5148 let instrument_id = instrument.id();
5149
5150 let report_b = FillReport::new(
5151 account_id,
5152 instrument_id,
5153 VenueOrderId::from("V-POSITION-FILLS"),
5154 TradeId::from("T-POSITION-FILLS-B"),
5155 OrderSide::Buy,
5156 Quantity::from("1.0"),
5157 Price::from("100.0"),
5158 Money::zero(instrument.quote_currency()),
5159 LiquiditySide::Taker,
5160 Some(ClientOrderId::from("O-POSITION-FILLS")),
5161 None,
5162 UnixNanos::from(1_500),
5163 UnixNanos::from(2_000),
5164 None,
5165 );
5166 let mut report_a = report_b.clone();
5167 report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5168 let commands = Rc::new(RefCell::new(Vec::new()));
5169
5170 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5171 client_id,
5172 account_id,
5173 venue: instrument_id.venue,
5174 outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5175 commands,
5176 }));
5177
5178 let command = GenerateFillReports::new(
5179 UUID4::new(),
5180 UnixNanos::from(2_000),
5181 Some(instrument_id),
5182 None,
5183 Some(UnixNanos::from(1_000)),
5184 Some(UnixNanos::from(2_000)),
5185 None,
5186 None,
5187 );
5188 let key = (instrument_id, account_id);
5189
5190 let result = request_position_fill_reports(
5191 vec![client],
5192 vec![PositionFillReportQuery {
5193 key,
5194 client_id,
5195 command,
5196 }],
5197 )
5198 .await
5199 .finish();
5200
5201 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5202 assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5203 }
5204
5205 #[rstest]
5206 #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5207 #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5208 #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5209 #[tokio::test]
5210 async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5211 #[case] report_account: &str,
5212 #[case] report_instrument: &str,
5213 #[case] ts_event: u64,
5214 #[case] quantity: &str,
5215 ) {
5216 let client_id = ClientId::from("POSITION-FILLS");
5217 let account_id = AccountId::from("POSITION-FILLS-001");
5218 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5219 let instrument_id = instrument.id();
5220
5221 let report = FillReport::new(
5222 AccountId::from(report_account),
5223 InstrumentId::from(report_instrument),
5224 VenueOrderId::from("V-POSITION-FILLS"),
5225 TradeId::from("T-POSITION-FILLS"),
5226 OrderSide::Buy,
5227 Quantity::from(quantity),
5228 Price::from("100.0"),
5229 Money::zero(instrument.quote_currency()),
5230 LiquiditySide::Taker,
5231 Some(ClientOrderId::from("O-POSITION-FILLS")),
5232 None,
5233 UnixNanos::from(ts_event),
5234 UnixNanos::from(2_000),
5235 None,
5236 );
5237
5238 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5239 client_id,
5240 account_id,
5241 venue: instrument_id.venue,
5242 outcome: FillReportClientOutcome::Reports(vec![report]),
5243 commands: Rc::new(RefCell::new(Vec::new())),
5244 }));
5245
5246 let command = GenerateFillReports::new(
5247 UUID4::new(),
5248 UnixNanos::from(2_000),
5249 Some(instrument_id),
5250 None,
5251 Some(UnixNanos::from(1_000)),
5252 Some(UnixNanos::from(2_000)),
5253 None,
5254 None,
5255 );
5256 let key = (instrument_id, account_id);
5257
5258 let result = request_position_fill_reports(
5259 vec![client],
5260 vec![PositionFillReportQuery {
5261 key,
5262 client_id,
5263 command,
5264 }],
5265 )
5266 .await
5267 .finish();
5268
5269 assert!(result.is_empty());
5270 }
5271
5272 #[rstest]
5273 #[case(999)]
5274 #[case(2_001)]
5275 #[tokio::test]
5276 async fn test_request_position_fill_reports_filters_time_window_superset(
5277 #[case] ts_event: u64,
5278 ) {
5279 let client_id = ClientId::from("POSITION-FILLS");
5280 let account_id = AccountId::from("POSITION-FILLS-001");
5281 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5282 let instrument_id = instrument.id();
5283
5284 let report = FillReport::new(
5285 account_id,
5286 instrument_id,
5287 VenueOrderId::from("V-POSITION-FILLS"),
5288 TradeId::from("T-POSITION-FILLS"),
5289 OrderSide::Buy,
5290 Quantity::from("1.0"),
5291 Price::from("100.0"),
5292 Money::zero(instrument.quote_currency()),
5293 LiquiditySide::Taker,
5294 Some(ClientOrderId::from("O-POSITION-FILLS")),
5295 None,
5296 UnixNanos::from(ts_event),
5297 UnixNanos::from(2_000),
5298 None,
5299 );
5300
5301 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5302 client_id,
5303 account_id,
5304 venue: instrument_id.venue,
5305 outcome: FillReportClientOutcome::Reports(vec![report]),
5306 commands: Rc::new(RefCell::new(Vec::new())),
5307 }));
5308
5309 let command = GenerateFillReports::new(
5310 UUID4::new(),
5311 UnixNanos::from(2_000),
5312 Some(instrument_id),
5313 None,
5314 Some(UnixNanos::from(1_000)),
5315 Some(UnixNanos::from(2_000)),
5316 None,
5317 None,
5318 );
5319 let key = (instrument_id, account_id);
5320
5321 let result = request_position_fill_reports(
5322 vec![client],
5323 vec![PositionFillReportQuery {
5324 key,
5325 client_id,
5326 command,
5327 }],
5328 )
5329 .await
5330 .finish();
5331
5332 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5333 assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5334 }
5335
5336 #[rstest]
5337 #[case::query_error(false)]
5338 #[case::missing_client(true)]
5339 #[tokio::test]
5340 async fn test_position_fill_collection_discards_partial_failure(
5341 #[case] missing_client: bool,
5342 #[values(false, true)] failure_first: bool,
5343 position_fill_query: (FillReport, PositionFillReportQuery),
5344 ) {
5345 let (report, query) = position_fill_query;
5346 let failed_client_id = ClientId::from("FAILED-FILLS");
5347 let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5348 let mut healthy_report = report.clone();
5349 healthy_report.account_id = healthy_key.1;
5350 let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5351 let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5352 let empty_client_id = ClientId::from("EMPTY-FILLS");
5353 let mut clients = vec![
5354 LiveExecutionClient::new(Box::new(FillReportClient {
5355 client_id: query.client_id,
5356 account_id: query.key.1,
5357 venue: query.key.0.venue,
5358 outcome: FillReportClientOutcome::Reports(vec![report]),
5359 commands: Rc::new(RefCell::new(Vec::new())),
5360 })),
5361 LiveExecutionClient::new(Box::new(FillReportClient {
5362 client_id: healthy_client_id,
5363 account_id: healthy_key.1,
5364 venue: healthy_key.0.venue,
5365 outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5366 commands: Rc::new(RefCell::new(Vec::new())),
5367 })),
5368 LiveExecutionClient::new(Box::new(FillReportClient {
5369 client_id: empty_client_id,
5370 account_id: empty_key.1,
5371 venue: empty_key.0.venue,
5372 outcome: FillReportClientOutcome::Reports(Vec::new()),
5373 commands: Rc::new(RefCell::new(Vec::new())),
5374 })),
5375 ];
5376
5377 if !missing_client {
5378 clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5379 client_id: failed_client_id,
5380 account_id: query.key.1,
5381 venue: query.key.0.venue,
5382 outcome: FillReportClientOutcome::Failure,
5383 commands: Rc::new(RefCell::new(Vec::new())),
5384 })));
5385 }
5386
5387 let failed_query = PositionFillReportQuery {
5388 client_id: failed_client_id,
5389 key: query.key,
5390 command: query.command.clone(),
5391 };
5392
5393 let healthy_query = PositionFillReportQuery {
5394 key: healthy_key,
5395 client_id: healthy_client_id,
5396 command: query.command.clone(),
5397 };
5398
5399 let empty_query = PositionFillReportQuery {
5400 key: empty_key,
5401 client_id: empty_client_id,
5402 command: query.command.clone(),
5403 };
5404
5405 let mut queries = vec![query, failed_query];
5406
5407 if failure_first {
5408 queries.reverse();
5409 }
5410
5411 queries.insert(0, empty_query);
5412 queries.push(healthy_query);
5413
5414 let result = request_position_fill_reports(clients, queries)
5415 .await
5416 .finish();
5417
5418 assert_eq!(
5419 result.keys().copied().collect::<Vec<_>>(),
5420 vec![empty_key, healthy_key]
5421 );
5422 assert_eq!(
5423 result,
5424 IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5425 );
5426 }
5427
5428 #[rstest]
5429 #[case::same_economics("metadata", true)]
5430 #[case::venue_order("venue_order", false)]
5431 #[case::side("side", false)]
5432 #[case::quantity("quantity", false)]
5433 #[case::price("price", false)]
5434 #[case::commission("commission", false)]
5435 #[case::liquidity("liquidity", false)]
5436 #[case::average("average", false)]
5437 #[case::event_time("event_time", false)]
5438 #[case::client_order("client_order", false)]
5439 #[case::position("position", false)]
5440 #[tokio::test]
5441 async fn test_position_fill_collection_validates_duplicate_economics(
5442 #[case] changed: &str,
5443 #[case] accepted: bool,
5444 position_fill_query: (FillReport, PositionFillReportQuery),
5445 ) {
5446 let (report, query) = position_fill_query;
5447 let mut duplicate = report.clone();
5448
5449 match changed {
5450 "metadata" => {
5451 duplicate.report_id = UUID4::new();
5452 duplicate.ts_init = UnixNanos::from(3_000);
5453 }
5454 "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5455 "side" => duplicate.order_side = OrderSide::Sell,
5456 "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5457 "price" => duplicate.last_px = Price::from("101.0"),
5458 "commission" => duplicate.commission = Money::from("0.25 USDT"),
5459 "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5460 "average" => duplicate.avg_px = Some(dec!(100.5)),
5461 "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5462 "client_order" => duplicate.client_order_id = None,
5463 "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5464 _ => unreachable!(),
5465 }
5466
5467 let key = query.key;
5468
5469 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5470 client_id: query.client_id,
5471 account_id: key.1,
5472 venue: key.0.venue,
5473 outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5474 commands: Rc::new(RefCell::new(Vec::new())),
5475 }));
5476
5477 let result = request_position_fill_reports(vec![client], vec![query])
5478 .await
5479 .finish();
5480
5481 if accepted {
5482 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5483 assert_eq!(result, IndexMap::from([(key, vec![report])]));
5484 } else {
5485 assert!(result.is_empty());
5486 }
5487 }
5488
5489 #[fixture]
5490 fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5491 let account_id = AccountId::from("POSITION-FILLS-001");
5492 let instrument_id = crypto_perpetual_ethusdt().id();
5493
5494 let report = FillReport::new(
5495 account_id,
5496 instrument_id,
5497 VenueOrderId::from("V-POSITION-FILLS"),
5498 TradeId::from("T-POSITION-FILLS"),
5499 OrderSide::Buy,
5500 Quantity::from("1.0"),
5501 Price::from("100.0"),
5502 Money::from("0.10 USDT"),
5503 LiquiditySide::Taker,
5504 Some(ClientOrderId::from("O-POSITION-FILLS")),
5505 None,
5506 UnixNanos::from(1_500),
5507 UnixNanos::from(2_000),
5508 None,
5509 );
5510
5511 let query = PositionFillReportQuery {
5512 key: (instrument_id, account_id),
5513 client_id: ClientId::from("POSITION-FILLS"),
5514 command: GenerateFillReports::new(
5515 UUID4::new(),
5516 UnixNanos::from(2_000),
5517 Some(instrument_id),
5518 None,
5519 Some(UnixNanos::from(1_000)),
5520 Some(UnixNanos::from(2_000)),
5521 None,
5522 None,
5523 ),
5524 };
5525
5526 (report, query)
5527 }
5528
5529 #[rstest]
5530 #[case::failed_query(false)]
5531 #[case::local_activity(true)]
5532 fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5533 let (mut node, report, _) =
5534 position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5535 let active_key = (report.instrument_id, report.account_id);
5536 let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5537 let deferred_reports = vec![
5538 PositionStatusReport::new(
5539 deferred_key.1,
5540 deferred_key.0,
5541 PositionSide::Long,
5542 Quantity::from("2.0"),
5543 UnixNanos::from(1_000),
5544 UnixNanos::from(1_000),
5545 None,
5546 None,
5547 Some(dec!(100.0)),
5548 ),
5549 PositionStatusReport::new(
5550 deferred_key.1,
5551 deferred_key.0,
5552 PositionSide::Short,
5553 Quantity::from("1.0"),
5554 UnixNanos::from(1_000),
5555 UnixNanos::from(1_000),
5556 None,
5557 None,
5558 Some(dec!(100.0)),
5559 ),
5560 ];
5561 let mut check = node
5562 .exec_manager
5563 .prepare_position_report_check(UUID4::new(), &[]);
5564 check.client_coverage.clear();
5565 let events = node.exec_manager.reconcile_position_reports(
5566 &check,
5567 deferred_reports.clone(),
5568 &IndexSet::new(),
5569 &IndexSet::new(),
5570 );
5571 assert!(events.is_empty());
5572 assert_eq!(
5573 node.exec_manager.position_recon_retry_count(&deferred_key),
5574 1
5575 );
5576
5577 let mut position_result = position_report_result(&node, report);
5578 position_result.reports.extend(deferred_reports);
5579 position_result.check.client_coverage.insert(
5580 deferred_key,
5581 ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5582 );
5583 position_result
5584 .check
5585 .activity_revisions
5586 .insert(deferred_key, 0);
5587 position_result
5588 .queried_clients
5589 .insert(ClientId::from("SECOND"));
5590 let mut reports = IndexMap::from([(active_key, Vec::new())]);
5591 if local_activity {
5592 reports.insert(deferred_key, Vec::new());
5593 node.exec_manager
5594 .record_position_activity(deferred_key.0, deferred_key.1);
5595 }
5596
5597 node.handle_position_fill_report_result(PositionFillReportResult {
5598 position_result,
5599 reports,
5600 });
5601
5602 assert_eq!(
5603 node.exec_manager.position_recon_retry_count(&deferred_key),
5604 1
5605 );
5606 {
5607 let cache = node.kernel.cache.borrow();
5608 let positions =
5609 cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5610 assert_eq!(
5611 positions
5612 .iter()
5613 .map(|position| position.quantity)
5614 .collect::<Vec<_>>(),
5615 vec![Quantity::from("1.0"), Quantity::from("1.0")],
5616 );
5617 assert_eq!(
5618 cache
5619 .positions_open(
5620 None,
5621 Some(&deferred_key.0),
5622 None,
5623 Some(&deferred_key.1),
5624 None,
5625 )
5626 .len(),
5627 0
5628 );
5629 }
5630
5631 let mut fresh_check = node
5632 .exec_manager
5633 .prepare_position_report_check(UUID4::new(), &[]);
5634 node.exec_manager.plan_position_fill_reports(
5635 &mut fresh_check,
5636 &[],
5637 &IndexSet::new(),
5638 &IndexSet::new(),
5639 &[],
5640 );
5641
5642 assert_eq!(
5643 node.exec_manager.position_recon_retry_count(&deferred_key),
5644 0
5645 );
5646 }
5647
5648 #[rstest]
5649 fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5650 let (mut node, venue_report, fill_report) =
5651 position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5652 let key = (venue_report.instrument_id, venue_report.account_id);
5653 let revision = node.exec_manager.position_activity_revision(&key);
5654 let position_result = position_report_result(&node, venue_report);
5655
5656 node.handle_position_fill_report_result(PositionFillReportResult {
5657 position_result,
5658 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5659 });
5660
5661 let cache = node.kernel.cache.borrow();
5662 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5663 assert_eq!(
5664 positions
5665 .iter()
5666 .map(|position| position.quantity)
5667 .sum::<Quantity>(),
5668 Quantity::from("2.0")
5669 );
5670 assert_eq!(
5671 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5672 1
5673 );
5674 drop(positions);
5675 drop(cache);
5676 assert!(
5677 node.exec_manager
5678 .position_contains_fill_report(&fill_report)
5679 );
5680 assert_eq!(
5681 node.exec_manager.position_activity_revision(&key),
5682 revision + 1
5683 );
5684 }
5685
5686 #[rstest]
5687 fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5688 let (mut node, venue_report, mut first) =
5689 position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5690 first.commission = Money::from("0.13 USDT");
5691 let key = (first.instrument_id, first.account_id);
5692 let client_order_id = first.client_order_id.unwrap();
5693 let position_result = position_report_result(&node, venue_report.clone());
5694 node.handle_position_fill_report_result(PositionFillReportResult {
5695 position_result,
5696 reports: IndexMap::from([(key, vec![first.clone()])]),
5697 });
5698
5699 let mut second = first.clone();
5700 second.trade_id = TradeId::from("T-PREFIX-SECOND");
5701 second.last_qty = Quantity::from("0.75");
5702 second.commission = Money::from("0.29 USDT");
5703 let mut third = first.clone();
5704 third.trade_id = TradeId::from("T-PREFIX-THIRD");
5705 third.last_qty = Quantity::from("0.25");
5706 third.commission = Money::from("0.07 USDT");
5707 let revision = node.exec_manager.position_activity_revision(&key);
5708 let received = Rc::new(RefCell::new(Vec::new()));
5709 let cache = node.kernel.cache.clone();
5710 let callback_received = received.clone();
5711 let second_trade_id = second.trade_id;
5712
5713 let handler = TypedHandler::from(move |event: &OrderEventAny| {
5714 let OrderEventAny::Filled(fill) = event else {
5715 return;
5716 };
5717
5718 callback_received.borrow_mut().push(fill.trade_id);
5719 if fill.trade_id == second_trade_id {
5720 let mut cache = cache.borrow_mut();
5721 let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5722 let mut position = cache.position_mut(&position_id).unwrap();
5723 position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5724 }
5725 });
5726
5727 let strategy_id = node
5728 .kernel
5729 .cache
5730 .borrow()
5731 .order(&client_order_id)
5732 .unwrap()
5733 .strategy_id();
5734 let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5735 msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5736 let position_result = position_report_result(&node, venue_report);
5737 node.handle_position_fill_report_result(PositionFillReportResult {
5738 position_result,
5739 reports: IndexMap::from([(
5740 key,
5741 vec![first.clone(), first.clone(), second.clone(), third.clone()],
5742 )]),
5743 });
5744
5745 msgbus::unsubscribe_order_events(topic.into(), &handler);
5746 let cache = node.kernel.cache.borrow();
5747 let order = cache.order(&client_order_id).unwrap();
5748 let position_id = cache.position_id(&client_order_id).unwrap();
5749 let position = cache.position_ref(position_id).unwrap();
5750
5751 assert_eq!(*received.borrow(), vec![second.trade_id]);
5752 assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5753 assert_eq!(order.events().len(), 6);
5754 assert_eq!(
5755 order.trade_ids(),
5756 vec![
5757 &TradeId::from("T-POSITION-INITIAL"),
5758 &first.trade_id,
5759 &second.trade_id
5760 ]
5761 );
5762 assert_eq!(
5763 order.commissions().values().copied().collect::<Vec<_>>(),
5764 vec![Money::from("0.42 USDT")]
5765 );
5766 assert_eq!(position.quantity, Quantity::from("2.25"));
5767 assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5768 assert_eq!(position.replay_events.len(), 2);
5769 assert_eq!(
5770 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5771 1
5772 );
5773 assert_eq!(
5774 node.exec_manager.position_activity_revision(&key),
5775 revision + 1
5776 );
5777 assert!(node.exec_manager.position_contains_fill_report(&first));
5778 assert!(!node.exec_manager.position_contains_fill_report(&second));
5779 assert!(!node.exec_manager.position_contains_fill_report(&third));
5780 assert!(
5781 !node
5782 .exec_manager
5783 .is_fill_recently_processed(key.1, key.0, second.trade_id)
5784 );
5785 assert!(
5786 !node
5787 .exec_manager
5788 .is_fill_recently_processed(key.1, key.0, third.trade_id)
5789 );
5790 }
5791
5792 #[rstest]
5793 fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5794 let (mut node, venue_report, fill_report) =
5795 position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5796 let key = (venue_report.instrument_id, venue_report.account_id);
5797 let position_result = position_report_result(&node, venue_report);
5798
5799 node.handle_position_fill_report_result(PositionFillReportResult {
5800 position_result,
5801 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5802 });
5803
5804 let mut rest_report = fill_report;
5805 rest_report.ts_event = UnixNanos::from(2_000);
5806
5807 assert!(
5808 node.exec_manager
5809 .position_contains_fill_report(&rest_report)
5810 );
5811 }
5812
5813 #[rstest]
5814 fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5815 let (mut node, venue_report, fill_report) =
5816 position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5817 let key = (venue_report.instrument_id, venue_report.account_id);
5818 let client_order_id = fill_report.client_order_id.unwrap();
5819 apply_inferred_position_fill(&mut node, &fill_report);
5820
5821 let venue_report = PositionStatusReport::new(
5822 key.1,
5823 key.0,
5824 PositionSide::Long,
5825 Quantity::from("3.0"),
5826 venue_report.ts_last,
5827 venue_report.ts_init,
5828 None,
5829 None,
5830 venue_report.avg_px_open,
5831 );
5832 let mut later_fill_report = fill_report.clone();
5833 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5834 later_fill_report.ts_event = UnixNanos::from(1_001);
5835 let position_result = position_report_result(&node, venue_report);
5836
5837 node.handle_position_fill_report_result(PositionFillReportResult {
5838 position_result,
5839 reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5840 });
5841
5842 let cache = node.kernel.cache.borrow();
5843 let order = cache.order(&client_order_id).unwrap();
5844 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5845 assert_eq!(order.filled_qty(), Quantity::from("2.0"));
5846 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5847 assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
5848 assert_eq!(
5849 positions
5850 .iter()
5851 .map(|position| position.quantity)
5852 .sum::<Quantity>(),
5853 Quantity::from("3.0")
5854 );
5855 assert_eq!(
5856 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5857 2
5858 );
5859 }
5860
5861 #[rstest]
5862 fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
5863 let (mut node, _, _) =
5864 position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
5865 let account_id = AccountId::from("TEST-001");
5866 let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
5867 let key = (instrument.id(), account_id);
5868 let commission = Money::zero(instrument.quote_currency());
5869 node.kernel
5870 .cache
5871 .borrow_mut()
5872 .add_instrument(instrument.clone())
5873 .unwrap();
5874
5875 let order = OrderTestBuilder::new(OrderType::Market)
5876 .strategy_id(StrategyId::external())
5877 .instrument_id(instrument.id())
5878 .client_order_id(ClientOrderId::from("O-RECONCILED"))
5879 .quantity(Quantity::from("1.000000"))
5880 .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
5881 .build();
5882 let submitted = TestOrderEventStubs::submitted(&order, account_id);
5883 let accepted =
5884 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
5885 node.kernel
5886 .cache
5887 .borrow_mut()
5888 .add_order(
5889 order.clone(),
5890 None,
5891 Some(ClientId::from("POSITION-FILLS")),
5892 false,
5893 )
5894 .unwrap();
5895 node.process_reconciliation_events(&[submitted, accepted]);
5896 let order = node
5897 .kernel
5898 .cache
5899 .borrow()
5900 .order_owned(&order.client_order_id())
5901 .unwrap();
5902 let opening_fill = TestOrderEventStubs::filled(
5903 &order,
5904 &instrument,
5905 Some(TradeId::from("T-RECONCILED")),
5906 None,
5907 Some(Price::from("100.00")),
5908 Some(Quantity::from("1.000000")),
5909 Some(LiquiditySide::Taker),
5910 Some(commission),
5911 Some(UnixNanos::from(2_000)),
5912 Some(account_id),
5913 );
5914 node.process_reconciliation_events(&[opening_fill]);
5915
5916 let held_fill_report = FillReport::new(
5918 account_id,
5919 instrument.id(),
5920 VenueOrderId::from("V-HELD"),
5921 TradeId::from("T-HELD"),
5922 OrderSide::Buy,
5923 Quantity::from("1.000000"),
5924 Price::from("100.00"),
5925 commission,
5926 LiquiditySide::Taker,
5927 None,
5928 None,
5929 UnixNanos::from(1_000),
5930 UnixNanos::from(1_000),
5931 None,
5932 );
5933 node.kernel
5934 .exec_engine
5935 .borrow_mut()
5936 .reconcile_fill_report(&held_fill_report);
5937
5938 let venue_report = PositionStatusReport::new(
5939 account_id,
5940 instrument.id(),
5941 PositionSide::Long,
5942 Quantity::from("2.000000"),
5943 UnixNanos::from(3_000),
5944 UnixNanos::from(3_000),
5945 None,
5946 None,
5947 Some(dec!(100.00)),
5948 );
5949 let position_result = position_report_result(&node, venue_report);
5950
5951 node.handle_position_fill_report_result(PositionFillReportResult {
5952 position_result,
5953 reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
5954 });
5955
5956 let cache = node.kernel.cache.borrow();
5957 let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
5958 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5959 assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
5960 assert_eq!(
5961 positions
5962 .iter()
5963 .map(|position| position.quantity)
5964 .sum::<Quantity>(),
5965 Quantity::from("2.000000")
5966 );
5967 assert!(
5968 positions
5969 .iter()
5970 .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
5971 );
5972 }
5973
5974 #[rstest]
5975 fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
5976 let (mut node, _, mut fill_report) =
5977 position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
5978 apply_inferred_position_fill(&mut node, &fill_report);
5979 let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
5980 fill_report.venue_position_id = Some(conflicting_position_id);
5981
5982 let error = node
5983 .exec_manager
5984 .prepare_position_fill_report(&mut fill_report, &[])
5985 .unwrap_err();
5986
5987 assert!(
5988 error.to_string().contains(&format!(
5989 "position ID {conflicting_position_id} conflicts with cached order position"
5990 )),
5991 "{error:#}"
5992 );
5993 }
5994
5995 #[rstest]
5996 #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
5997 #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
5998 fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
5999 #[case] authoritative_qty: &str,
6000 #[case] client_order_id: &str,
6001 ) {
6002 let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6003 "UnappliedPositionFillNode",
6004 Quantity::from(authoritative_qty),
6005 );
6006 fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6007 let reports = [fill_report.clone()];
6008
6009 let quantities = (0..2)
6010 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6011 .collect::<Vec<_>>();
6012
6013 assert_eq!(
6014 quantities,
6015 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6016 );
6017 assert!(
6018 !node
6019 .exec_manager
6020 .position_contains_fill_report(&fill_report)
6021 );
6022 }
6023
6024 #[rstest]
6025 fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6026 let (mut node, venue_report, fill_report) =
6027 position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6028 let mut later_fill_report = fill_report.clone();
6029 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6030 later_fill_report.ts_event = UnixNanos::from(1_001);
6031 let reports = [fill_report.clone(), later_fill_report.clone()];
6032
6033 let quantities = (0..3)
6034 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6035 .collect::<Vec<_>>();
6036
6037 assert_eq!(
6038 quantities,
6039 vec![
6040 Quantity::from("1.0"),
6041 Quantity::from("1.0"),
6042 Quantity::from("2.0"),
6043 ]
6044 );
6045 assert!(
6046 !node
6047 .exec_manager
6048 .position_contains_fill_report(&fill_report)
6049 );
6050 assert!(
6051 !node
6052 .exec_manager
6053 .position_contains_fill_report(&later_fill_report)
6054 );
6055 }
6056
6057 #[rstest]
6058 #[case::within_grace(60_000, true)]
6059 #[case::after_grace(0, false)]
6060 fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6061 #[case] position_check_threshold_ms: u32,
6062 #[case] expected_applied: bool,
6063 ) {
6064 let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6065 "TransientPositionFillNode",
6066 Quantity::from("1.0"),
6067 position_check_threshold_ms,
6068 );
6069 let mut conflicting_report = fill_report.clone();
6070 conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6071
6072 let quantities = [conflicting_report, fill_report.clone()]
6073 .into_iter()
6074 .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6075 .collect::<Vec<_>>();
6076
6077 assert_eq!(
6078 quantities,
6079 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6080 );
6081 assert_eq!(
6082 node.exec_manager
6083 .position_contains_fill_report(&fill_report),
6084 expected_applied
6085 );
6086 }
6087
6088 #[rstest]
6089 fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6090 let (mut node, venue_report, fill_report) =
6091 position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6092 let key = (venue_report.instrument_id, venue_report.account_id);
6093 let first_result = position_report_result(&node, venue_report.clone());
6094
6095 node.handle_position_fill_report_result(PositionFillReportResult {
6096 position_result: first_result,
6097 reports: IndexMap::from([(key, vec![fill_report])]),
6098 });
6099
6100 let fresh_result = position_report_result(&node, venue_report);
6101 node.handle_position_fill_report_result(PositionFillReportResult {
6102 position_result: fresh_result,
6103 reports: IndexMap::from([(key, Vec::new())]),
6104 });
6105
6106 let cache = node.kernel.cache.borrow();
6107 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6108 assert_eq!(
6109 positions
6110 .iter()
6111 .map(|position| position.quantity)
6112 .sum::<Quantity>(),
6113 Quantity::from("2.0")
6114 );
6115 assert_eq!(
6116 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6117 2
6118 );
6119 }
6120
6121 #[rstest]
6122 #[case::below_limit(63)]
6123 #[case::at_limit(64)]
6124 #[case::above_limit(65)]
6125 fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6126 let (mut node, venue_report, fill_report) =
6127 position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6128 let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6129
6130 let venue_report = PositionStatusReport::new(
6131 venue_report.account_id,
6132 venue_report.instrument_id,
6133 PositionSide::Long,
6134 Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6135 venue_report.ts_last,
6136 venue_report.ts_init,
6137 None,
6138 None,
6139 venue_report.avg_px_open,
6140 );
6141 let key = (venue_report.instrument_id, venue_report.account_id);
6142 let client_order_id = fill_report.client_order_id.unwrap();
6143
6144 let reports = (0..count)
6145 .map(|index| {
6146 let mut report = fill_report.clone();
6147 report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6148 report.ts_event = UnixNanos::from(1_000 + index as u64);
6149 report
6150 })
6151 .collect::<Vec<_>>();
6152
6153 let position_result = position_report_result(&node, venue_report.clone());
6154
6155 node.handle_position_fill_report_result(PositionFillReportResult {
6156 position_result,
6157 reports: IndexMap::from([(key, reports.clone())]),
6158 });
6159
6160 {
6161 let cache = node.kernel.cache.borrow();
6162 let order = cache.order(&client_order_id).unwrap();
6163 let applied = count.min(64);
6164 assert_eq!(
6165 order.filled_qty().as_decimal(),
6166 dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6167 );
6168 assert_eq!(order.trade_ids().len(), applied + 1);
6169
6170 for (index, report) in reports.iter().enumerate() {
6171 assert_eq!(
6172 order.trade_ids().contains(&&report.trade_id),
6173 index < applied
6174 );
6175 }
6176
6177 assert_eq!(
6178 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6179 1
6180 );
6181 }
6182
6183 let position_result = position_report_result(&node, venue_report);
6185 node.handle_position_fill_report_result(PositionFillReportResult {
6186 position_result,
6187 reports: IndexMap::from([(key, reports)]),
6188 });
6189
6190 let cache = node.kernel.cache.borrow();
6191 let order = cache.order(&client_order_id).unwrap();
6192 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6193 assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6194 assert_eq!(order.trade_ids().len(), count + 1);
6195 assert_eq!(positions.len(), 1);
6196 assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6197 assert_eq!(positions[0].side, PositionSide::Long);
6198 assert_eq!(
6199 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6200 1
6201 );
6202 }
6203
6204 #[rstest]
6205 #[case::client_order(0)]
6206 #[case::venue_order(1)]
6207 #[case::side(2)]
6208 #[case::position(3)]
6209 fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6210 let (mut node, venue_report, valid_report) =
6211 position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6212 let key = (venue_report.instrument_id, venue_report.account_id);
6213 let client_order_id = valid_report.client_order_id.unwrap();
6214 let revision = node.exec_manager.position_activity_revision(&key);
6215 let position_result = position_report_result(&node, venue_report);
6216 let mut conflicting_report = valid_report.clone();
6217 conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6218
6219 match conflict {
6220 0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6221 1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6222 2 => conflicting_report.order_side = OrderSide::Sell,
6223 3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6224 _ => unreachable!(),
6225 }
6226
6227 node.handle_position_fill_report_result(PositionFillReportResult {
6228 position_result,
6229 reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6230 });
6231
6232 let cache = node.kernel.cache.borrow();
6233 let order = cache.order(&client_order_id).unwrap();
6234 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6235 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6236 assert_eq!(
6237 order.trade_ids(),
6238 vec![&TradeId::from("T-POSITION-INITIAL")]
6239 );
6240 assert_eq!(positions.len(), 1);
6241 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6242 assert_eq!(
6243 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6244 1
6245 );
6246 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6247 }
6248
6249 #[rstest]
6250 fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6251 let (mut node, venue_report, _) =
6252 position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6253 let key = (venue_report.instrument_id, venue_report.account_id);
6254 let position_result = position_report_result(&node, venue_report);
6255
6256 node.handle_position_fill_report_result(PositionFillReportResult {
6257 position_result,
6258 reports: IndexMap::new(),
6259 });
6260
6261 let cache = node.kernel.cache.borrow();
6262 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6263 assert_eq!(positions.len(), 1);
6264 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6265 assert_eq!(
6266 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6267 1
6268 );
6269 }
6270
6271 #[rstest]
6272 fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6273 let (mut node, mut venue_report, mut fill_report) =
6274 position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6275 node.kernel
6276 .exec_engine
6277 .borrow_mut()
6278 .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6279 let key = (venue_report.instrument_id, venue_report.account_id);
6280
6281 let position_id = {
6282 let cache = node.kernel.cache.borrow();
6283 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6284 assert_eq!(positions.len(), 1);
6285 positions[0].id
6286 };
6287
6288 venue_report.venue_position_id = Some(position_id);
6289 fill_report.client_order_id = None;
6290 fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6291 let position_result = position_report_result(&node, venue_report);
6292
6293 node.handle_position_fill_report_result(PositionFillReportResult {
6294 position_result,
6295 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6296 });
6297
6298 let cache = node.kernel.cache.borrow();
6299 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6300 assert_eq!(positions.len(), 1);
6301 assert_eq!(positions[0].id, position_id);
6302 assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6303 assert_eq!(
6304 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6305 2
6306 );
6307 drop(positions);
6308 drop(cache);
6309 assert!(
6310 !node
6311 .exec_manager
6312 .position_contains_fill_report(&fill_report)
6313 );
6314 }
6315
6316 #[rstest]
6317 fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6318 let (mut node, venue_report, fill_report) =
6319 position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6320 let key = (venue_report.instrument_id, venue_report.account_id);
6321 let client_order_id = fill_report.client_order_id.unwrap();
6322 let position_result = position_report_result(&node, venue_report);
6323 let revision = node.exec_manager.position_activity_revision(&key);
6324
6325 let order_report = OrderStatusReport::new(
6326 key.1,
6327 key.0,
6328 Some(client_order_id),
6329 fill_report.venue_order_id,
6330 OrderSide::Buy.into(),
6331 OrderType::Limit,
6332 TimeInForce::Gtc,
6333 OrderStatus::PartiallyFilled,
6334 Quantity::from("10.0"),
6335 Quantity::from("2.0"),
6336 UnixNanos::from(1_000),
6337 UnixNanos::from(1_000),
6338 UnixNanos::from(2_000),
6339 None,
6340 );
6341 let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6342 Box::new(order_report),
6343 vec![fill_report.clone()],
6344 ));
6345
6346 assert_eq!(
6347 node.observe_exec_event_before_dispatch(&event),
6348 Some(vec![client_order_id])
6349 );
6350
6351 assert_eq!(
6352 node.exec_manager.position_activity_revision(&key),
6353 revision + 1
6354 );
6355 assert!(
6356 !node
6357 .exec_manager
6358 .position_report_check_is_current(&position_result.check, &key)
6359 );
6360
6361 node.handle_position_fill_report_result(PositionFillReportResult {
6362 position_result,
6363 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6364 });
6365
6366 let cache = node.kernel.cache.borrow();
6367 let order = cache.order(&client_order_id).unwrap();
6368 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6369 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6370 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6371 assert_eq!(positions.len(), 1);
6372 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6373 assert_eq!(positions[0].side, PositionSide::Long);
6374 assert_eq!(
6375 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6376 1
6377 );
6378 }
6379
6380 #[rstest]
6381 fn test_position_fill_report_result_defers_after_local_position_activity() {
6382 let (mut node, venue_report, fill_report) =
6383 position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6384 let key = (venue_report.instrument_id, venue_report.account_id);
6385 let position_result = position_report_result(&node, venue_report);
6386 node.exec_manager.record_position_activity(key.0, key.1);
6387
6388 node.handle_position_fill_report_result(PositionFillReportResult {
6389 position_result,
6390 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6391 });
6392
6393 let cache = node.kernel.cache.borrow();
6394 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6395 assert_eq!(positions.len(), 1);
6396 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6397 assert_eq!(
6398 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6399 1
6400 );
6401 drop(positions);
6402 drop(cache);
6403 assert!(
6404 !node
6405 .exec_manager
6406 .position_contains_fill_report(&fill_report)
6407 );
6408 }
6409
6410 #[rstest]
6411 fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6412 let config = LiveNodeConfig {
6413 exec_engine: crate::config::LiveExecutionEngineConfig {
6414 reconciliation: true,
6415 open_check_threshold_ms: 5_000,
6416 single_order_query_delay_ms: 0,
6417 ..Default::default()
6418 },
6419 ..Default::default()
6420 };
6421
6422 let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6423 let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6424 let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6425 let instrument = crypto_perpetual_ethusdt();
6426 let instrument_id = instrument.id();
6427 let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6428 let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6429
6430 node.kernel
6431 .cache
6432 .borrow_mut()
6433 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6434 .unwrap();
6435 insert_accepted_limit_order_in_node(
6436 &node,
6437 account_id,
6438 client_id,
6439 instrument_id,
6440 client_order_id,
6441 venue_order_id,
6442 );
6443
6444 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6445
6446 let accepted = OrderAcceptedSpec::builder()
6447 .instrument_id(instrument_id)
6448 .client_order_id(client_order_id)
6449 .venue_order_id(venue_order_id)
6450 .account_id(account_id)
6451 .build();
6452 let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6453
6454 let close_ids = node.observe_exec_event_before_dispatch(&event);
6455
6456 assert_eq!(close_ids, Some(vec![client_order_id]));
6457 assert!(node.exec_manager.check_open_order_queries().is_empty());
6458 }
6459
6460 #[rstest]
6461 #[cfg_attr(
6462 not(all(feature = "simulation", madsim)),
6463 tokio::test(start_paused = true)
6464 )]
6465 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6466 async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6467 use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6468 use nautilus_model::{events::OrderPendingCancel, identifiers::ClientOrderId};
6469
6470 let config = LiveNodeConfig {
6471 exec_engine: crate::config::LiveExecutionEngineConfig {
6472 reconciliation: true,
6473 inflight_check_threshold_ms: 100,
6474 inflight_check_retries: 1,
6475 ..Default::default()
6476 },
6477 ..Default::default()
6478 };
6479
6480 let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6481 let trader_id = TraderId::from("TESTER-001");
6482 let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6483 let account_id = AccountId::from("TEST-001");
6484 let instrument = crypto_perpetual_ethusdt();
6485 let instrument_id = instrument.id();
6486 let child_ids = [
6487 ClientOrderId::from("O-BATCH-CANCEL-1"),
6488 ClientOrderId::from("O-BATCH-CANCEL-2"),
6489 ];
6490 node.kernel
6491 .cache
6492 .borrow_mut()
6493 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6494 .unwrap();
6495
6496 for client_order_id in child_ids {
6497 let order = OrderTestBuilder::new(OrderType::Limit)
6498 .trader_id(trader_id)
6499 .strategy_id(strategy_id)
6500 .client_order_id(client_order_id)
6501 .instrument_id(instrument_id)
6502 .quantity(Quantity::from("10.0"))
6503 .price(Price::from("100.0"))
6504 .build();
6505 let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6506 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6510 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6511 let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
6512 trader_id,
6513 strategy_id,
6514 instrument_id,
6515 client_order_id,
6516 Some(account_id),
6517 UUID4::new(),
6518 UnixNanos::default(),
6519 UnixNanos::default(),
6520 false,
6521 Some(venue_order_id),
6522 ));
6523 let mut cache = node.kernel.cache.borrow_mut();
6524 cache.add_order(order, None, None, false).unwrap();
6525 cache.update_order(&submitted).unwrap();
6526 cache.update_order(&accepted).unwrap();
6527 cache.update_order(&pending_cancel).unwrap();
6528 }
6529
6530 let cancels = child_ids
6531 .into_iter()
6532 .map(|client_order_id| {
6533 CancelOrder::new(
6534 trader_id,
6535 None,
6536 strategy_id,
6537 instrument_id,
6538 client_order_id,
6539 None,
6540 UUID4::new(),
6541 UnixNanos::default(),
6542 None,
6543 None,
6544 )
6545 })
6546 .collect();
6547
6548 let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6549 trader_id,
6550 None,
6551 strategy_id,
6552 instrument_id,
6553 cancels,
6554 UUID4::new(),
6555 UnixNanos::default(),
6556 None,
6557 None,
6558 ));
6559
6560 node.observe_exec_command_before_dispatch(&command);
6561 advance_clock(Duration::from_millis(101)).await;
6562 let result = node.exec_manager.check_inflight_orders();
6563 let timed_out_ids = result
6564 .events
6565 .iter()
6566 .map(OrderEventAny::client_order_id)
6567 .collect::<IndexSet<_>>();
6568
6569 assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6570 assert_eq!(result.events.len(), child_ids.len());
6571 assert!(
6572 result
6573 .events
6574 .iter()
6575 .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6576 "batch-cancel children must time out as Canceled events",
6577 );
6578 }
6579
6580 #[rstest]
6581 #[cfg_attr(
6582 not(all(feature = "simulation", madsim)),
6583 tokio::test(start_paused = true)
6584 )]
6585 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6586 async fn test_risk_bound_command_does_not_register_inflight() {
6587 let config = LiveNodeConfig {
6588 exec_engine: crate::config::LiveExecutionEngineConfig {
6589 reconciliation: true,
6590 inflight_check_threshold_ms: 100,
6591 inflight_check_retries: 2,
6592 ..Default::default()
6593 },
6594 ..Default::default()
6595 };
6596
6597 let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6598 msgbus::register_trading_command_endpoint(
6599 MessagingSwitchboard::risk_engine_execute(),
6600 TypedIntoHandler::from(|_: TradingCommand| {}),
6601 );
6602 let instrument = crypto_perpetual_ethusdt();
6603 let instrument_id = instrument.id();
6604 let order = OrderTestBuilder::new(OrderType::Limit)
6605 .trader_id(node.trader_id())
6606 .strategy_id(StrategyId::from("S-RISK-DENIED"))
6607 .instrument_id(instrument_id)
6608 .side(OrderSide::Buy)
6609 .quantity(Quantity::from("1.000"))
6610 .price(Price::from("100.00"))
6611 .build();
6612 let client_order_id = order.client_order_id();
6613
6614 {
6615 let mut cache = node.kernel.cache.borrow_mut();
6616 cache
6617 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6618 .unwrap();
6619 cache.add_order(order.clone(), None, None, false).unwrap();
6620 }
6621
6622 let submit_order = SubmitOrder::new(
6623 order.trader_id(),
6624 None,
6625 order.strategy_id(),
6626 instrument_id,
6627 client_order_id,
6628 order.init_event().clone(),
6629 None,
6630 None,
6631 None,
6632 UUID4::new(),
6633 UnixNanos::default(),
6634 None,
6635 );
6636 node.process_exec_command(
6637 TradingCommandMessage::new(
6638 MessagingSwitchboard::risk_engine_execute(),
6639 TradingCommand::SubmitOrder(submit_order),
6640 )
6641 .into(),
6642 );
6643
6644 advance_clock(Duration::from_millis(101)).await;
6645 let result = node.exec_manager.check_inflight_orders();
6646 let status = node
6647 .kernel
6648 .cache
6649 .borrow()
6650 .order(&client_order_id)
6651 .unwrap()
6652 .status();
6653
6654 assert_eq!(status, OrderStatus::Initialized);
6655 assert_eq!(
6656 node.exec_manager.recon_check_retry_count(&client_order_id),
6657 0
6658 );
6659 assert!(result.events.is_empty());
6660 assert!(result.queries.is_empty());
6661 }
6662
6663 #[rstest]
6664 #[cfg_attr(
6665 not(all(feature = "simulation", madsim)),
6666 tokio::test(start_paused = true)
6667 )]
6668 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6669 async fn test_risk_approved_command_registers_inflight() {
6670 let config = LiveNodeConfig {
6671 risk_engine: crate::config::LiveRiskEngineConfig {
6672 bypass: true,
6673 ..Default::default()
6674 },
6675 exec_engine: crate::config::LiveExecutionEngineConfig {
6676 reconciliation: true,
6677 inflight_check_threshold_ms: 100,
6678 inflight_check_retries: 2,
6679 ..Default::default()
6680 },
6681 ..Default::default()
6682 };
6683
6684 let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
6685 msgbus::register_trading_command_endpoint(
6686 MessagingSwitchboard::exec_engine_execute(),
6687 TypedIntoHandler::from(|_: TradingCommand| {}),
6688 );
6689 let instrument = crypto_perpetual_ethusdt();
6690 let instrument_id = instrument.id();
6691 let order = OrderTestBuilder::new(OrderType::Limit)
6692 .trader_id(node.trader_id())
6693 .strategy_id(StrategyId::from("S-RISK-APPROVED"))
6694 .instrument_id(instrument_id)
6695 .side(OrderSide::Buy)
6696 .quantity(Quantity::from("1.000"))
6697 .price(Price::from("100.00"))
6698 .build();
6699 let client_order_id = order.client_order_id();
6700
6701 {
6702 let mut cache = node.kernel.cache.borrow_mut();
6703 cache
6704 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6705 .unwrap();
6706 cache.add_order(order.clone(), None, None, false).unwrap();
6707 }
6708
6709 let submit_order = SubmitOrder::new(
6710 order.trader_id(),
6711 None,
6712 order.strategy_id(),
6713 instrument_id,
6714 client_order_id,
6715 order.init_event().clone(),
6716 None,
6717 None,
6718 None,
6719 UUID4::new(),
6720 UnixNanos::default(),
6721 None,
6722 );
6723 node.process_exec_command(
6724 TradingCommandMessage::new(
6725 MessagingSwitchboard::risk_engine_execute(),
6726 TradingCommand::SubmitOrder(submit_order),
6727 )
6728 .into(),
6729 );
6730
6731 advance_clock(Duration::from_millis(101)).await;
6732 let result = node.exec_manager.check_inflight_orders();
6733
6734 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6735 panic!("expected one query order command");
6736 };
6737
6738 assert_eq!(query.client_order_id, client_order_id);
6739 assert_eq!(
6740 node.exec_manager.recon_check_retry_count(&client_order_id),
6741 1
6742 );
6743 assert!(result.events.is_empty());
6744 }
6745
6746 #[rstest]
6747 fn test_live_node_builder_clock_factory_drives_kernel_clock() {
6748 let calls = Rc::new(Cell::new(0usize));
6749 let calls_in_factory = calls.clone();
6750 let sentinel = UnixNanos::from(123_456_789_u64);
6751
6752 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6753 .unwrap()
6754 .with_reconciliation(false)
6755 .with_clock_factory(move || {
6756 calls_in_factory.set(calls_in_factory.get() + 1);
6757 let mut clock = VirtualClock::new();
6758 clock.advance_time(sentinel, true);
6759 Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
6760 })
6761 .build()
6762 .unwrap();
6763
6764 assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
6765 assert_eq!(calls.get(), 1);
6766 }
6767
6768 #[derive(Debug)]
6769 struct ReplayKernelEventStore {
6770 fail_restore: bool,
6771 }
6772
6773 impl KernelEventStore for ReplayKernelEventStore {
6774 fn restore_parent_cache(
6775 &mut self,
6776 _instance_id: UUID4,
6777 _cache: &mut Cache,
6778 ) -> anyhow::Result<()> {
6779 if self.fail_restore {
6780 anyhow::bail!("replay restore failed");
6781 }
6782
6783 Ok(())
6784 }
6785
6786 fn open(
6787 &mut self,
6788 _instance_id: UUID4,
6789 _components: &RegisteredComponents,
6790 _environment: Environment,
6791 ) -> anyhow::Result<()> {
6792 Ok(())
6793 }
6794
6795 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
6796 None
6797 }
6798
6799 fn seal(&mut self, _ts_init: UnixNanos) {}
6800
6801 fn run_id(&self) -> Option<&str> {
6802 Some("replay-child")
6803 }
6804
6805 fn parent_run_id(&self) -> Option<&str> {
6806 Some("seed-run")
6807 }
6808
6809 fn is_event_store_replay_configured(&self) -> bool {
6810 true
6811 }
6812
6813 fn is_halted(&self) -> bool {
6814 false
6815 }
6816 }
6817
6818 #[derive(Debug)]
6819 struct TestStrategy {
6820 core: StrategyCore,
6821 }
6822
6823 impl TestStrategy {
6824 fn new(config: StrategyConfig) -> Self {
6825 Self {
6826 core: StrategyCore::new(config),
6827 }
6828 }
6829 }
6830
6831 impl DataActor for TestStrategy {}
6832
6833 nautilus_strategy!(TestStrategy, {
6834 fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
6835 self.core.config.external_order_instrument_ids.clone()
6836 }
6837 });
6838
6839 fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
6840 let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
6843 .unwrap()
6844 .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
6845 reconciliation: false,
6846 ..Default::default()
6847 })
6848 .with_load_state(true)
6849 .with_name("TestKernel")
6850 .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
6851 Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
6852 });
6853
6854 builder.build().unwrap()
6855 }
6856
6857 #[rstest]
6858 fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
6859 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6860 .unwrap()
6861 .with_reconciliation(false)
6862 .with_delay_post_stop_secs(0)
6863 .with_timeout_connection(1)
6864 .build()
6865 .unwrap();
6866 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6867 let strategy_id = StrategyId::from("CLAIMS-001");
6868
6869 node.add_strategy(TestStrategy::new(StrategyConfig {
6870 strategy_id: Some(strategy_id),
6871 external_order_instrument_ids: Some(vec![instrument_id]),
6872 ..Default::default()
6873 }))
6874 .unwrap();
6875
6876 assert_eq!(
6877 node.exec_manager.get_external_order_claim(&instrument_id),
6878 Some(strategy_id)
6879 );
6880
6881 {
6882 let exec_engine = node.kernel().exec_engine.borrow();
6883 assert_eq!(
6884 exec_engine.get_external_order_claim(&instrument_id),
6885 Some(strategy_id)
6886 );
6887 }
6888 }
6889
6890 #[rstest]
6891 fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
6892 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6893 .unwrap()
6894 .with_reconciliation(false)
6895 .build()
6896 .unwrap();
6897 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6898 let strategy_id = StrategyId::from("CLAIMS-001");
6899
6900 node.register_external_order_claims(strategy_id, &[instrument_id])
6901 .unwrap();
6902
6903 assert_eq!(
6904 node.exec_manager.get_external_order_claim(&instrument_id),
6905 Some(strategy_id)
6906 );
6907 assert_eq!(
6908 node.kernel
6909 .exec_engine
6910 .borrow()
6911 .get_external_order_claim(&instrument_id),
6912 Some(strategy_id)
6913 );
6914 }
6915
6916 #[rstest]
6917 #[tokio::test]
6918 async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
6919 let mut node = live_node_with_replay_store(false);
6920 let instrument_id = InstrumentId::from("AUDUSD.SIM");
6921 let strategy_id = StrategyId::from("CLAIMS-001");
6922
6923 node.start().await.unwrap();
6924 assert_eq!(node.state(), NodeState::Running);
6925
6926 node.register_external_order_claims(strategy_id, &[instrument_id])
6927 .unwrap();
6928
6929 assert_eq!(
6930 node.exec_manager.get_external_order_claim(&instrument_id),
6931 Some(strategy_id)
6932 );
6933 assert_eq!(
6934 node.kernel
6935 .exec_engine
6936 .borrow()
6937 .get_external_order_claim(&instrument_id),
6938 Some(strategy_id)
6939 );
6940 }
6941
6942 #[rstest]
6943 fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
6944 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6945 .unwrap()
6946 .with_reconciliation(false)
6947 .build()
6948 .unwrap();
6949 let existing_instrument = InstrumentId::from("AUDUSD.SIM");
6950 let new_instruments = [
6951 InstrumentId::from("EURUSD.SIM"),
6952 InstrumentId::from("GBPUSD.SIM"),
6953 ];
6954 let existing_strategy_id = StrategyId::from("CLAIMS-001");
6955 let new_strategy_id = StrategyId::from("CLAIMS-002");
6956 node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
6957 .unwrap();
6958
6959 let result = node.register_external_order_claims(
6960 new_strategy_id,
6961 &[new_instruments[0], existing_instrument, new_instruments[1]],
6962 );
6963
6964 assert!(result.is_err());
6965 assert_eq!(
6966 node.exec_manager
6967 .get_external_order_claim(&existing_instrument),
6968 Some(existing_strategy_id)
6969 );
6970
6971 for instrument_id in new_instruments {
6972 assert_eq!(
6973 node.exec_manager.get_external_order_claim(&instrument_id),
6974 None
6975 );
6976 assert_eq!(
6977 node.kernel
6978 .exec_engine
6979 .borrow()
6980 .get_external_order_claim(&instrument_id),
6981 None
6982 );
6983 }
6984 }
6985
6986 #[rstest]
6987 fn test_deregister_external_order_claims_allows_successor_to_claim() {
6988 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6989 .unwrap()
6990 .with_reconciliation(false)
6991 .build()
6992 .unwrap();
6993 let instruments = [
6994 InstrumentId::from("AUDUSD.SIM"),
6995 InstrumentId::from("EURUSD.SIM"),
6996 ];
6997 let first_strategy_id = StrategyId::from("CLAIMS-001");
6998 let successor_strategy_id = StrategyId::from("CLAIMS-002");
6999 node.register_external_order_claims(first_strategy_id, &instruments)
7000 .unwrap();
7001
7002 node.deregister_external_order_claims(first_strategy_id)
7003 .unwrap();
7004 node.register_external_order_claims(successor_strategy_id, &instruments)
7005 .unwrap();
7006
7007 for instrument_id in instruments {
7008 assert_eq!(
7009 node.exec_manager.get_external_order_claim(&instrument_id),
7010 Some(successor_strategy_id)
7011 );
7012 assert_eq!(
7013 node.kernel
7014 .exec_engine
7015 .borrow()
7016 .get_external_order_claim(&instrument_id),
7017 Some(successor_strategy_id)
7018 );
7019 }
7020 }
7021
7022 #[rstest]
7023 fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7024 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7025 .unwrap()
7026 .with_reconciliation(false)
7027 .build()
7028 .unwrap();
7029 let strategy_id = StrategyId::from("CLAIMS-001");
7030
7031 node.deregister_external_order_claims(strategy_id).unwrap();
7032 node.deregister_external_order_claims(strategy_id).unwrap();
7033 }
7034
7035 #[rstest]
7036 fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7037 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7038 .unwrap()
7039 .with_reconciliation(false)
7040 .with_delay_post_stop_secs(0)
7041 .with_timeout_connection(1)
7042 .build()
7043 .unwrap();
7044 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7045 let strategy_id = StrategyId::from("CLAIMS-001");
7046 let duplicate_strategy_id = StrategyId::from("OTHER-002");
7047
7048 node.add_strategy(TestStrategy::new(StrategyConfig {
7049 strategy_id: Some(strategy_id),
7050 external_order_instrument_ids: Some(vec![instrument_id]),
7051 ..Default::default()
7052 }))
7053 .unwrap();
7054
7055 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7056 strategy_id: Some(duplicate_strategy_id),
7057 external_order_instrument_ids: Some(vec![instrument_id]),
7058 ..Default::default()
7059 }));
7060
7061 assert!(result.is_err());
7062 assert!(
7063 result
7064 .unwrap_err()
7065 .to_string()
7066 .contains("already exists for CLAIMS-001")
7067 );
7068 assert_eq!(
7069 node.exec_manager.get_external_order_claim(&instrument_id),
7070 Some(strategy_id)
7071 );
7072
7073 {
7074 let exec_engine = node.kernel().exec_engine.borrow();
7075 assert_eq!(
7076 exec_engine.get_external_order_claim(&instrument_id),
7077 Some(strategy_id)
7078 );
7079 }
7080 }
7081
7082 #[rstest]
7083 fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7084 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7085 .unwrap()
7086 .with_reconciliation(false)
7087 .with_delay_post_stop_secs(0)
7088 .with_timeout_connection(1)
7089 .build()
7090 .unwrap();
7091 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7092 let strategy_id = StrategyId::from("CLAIMS-001");
7093
7094 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7095 strategy_id: Some(strategy_id),
7096 external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7097 ..Default::default()
7098 }));
7099
7100 assert!(result.is_err());
7101 assert!(
7102 result
7103 .unwrap_err()
7104 .to_string()
7105 .contains("appears more than once for CLAIMS-001")
7106 );
7107 assert_eq!(
7108 node.exec_manager.get_external_order_claim(&instrument_id),
7109 None
7110 );
7111
7112 {
7113 let exec_engine = node.kernel().exec_engine.borrow();
7114 assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7115 }
7116 }
7117
7118 #[rstest]
7119 fn test_add_strategy_failure_restores_external_order_claims() {
7120 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7121 .unwrap()
7122 .with_reconciliation(false)
7123 .with_delay_post_stop_secs(0)
7124 .with_timeout_connection(1)
7125 .build()
7126 .unwrap();
7127 let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7128 let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7129 let strategy_id = StrategyId::from("CLAIMS-001");
7130 node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7131 .unwrap();
7132
7133 let mut strategy = TestStrategy::new(StrategyConfig {
7134 strategy_id: Some(strategy_id),
7135 external_order_instrument_ids: Some(vec![configured_instrument_id]),
7136 ..Default::default()
7137 });
7138
7139 strategy
7140 .core
7141 .register(
7142 node.trader_id(),
7143 node.kernel.clock(),
7144 node.kernel.cache.clone(),
7145 node.kernel.portfolio.clone(),
7146 )
7147 .unwrap();
7148
7149 let result = node.add_strategy(strategy);
7150
7151 assert!(result.is_err());
7152 assert!(
7153 result
7154 .unwrap_err()
7155 .to_string()
7156 .contains("already registered with trader")
7157 );
7158 assert_eq!(
7159 node.exec_manager
7160 .get_external_order_claim(&existing_instrument_id),
7161 Some(strategy_id)
7162 );
7163 assert_eq!(
7164 node.kernel
7165 .exec_engine
7166 .borrow()
7167 .get_external_order_claim(&existing_instrument_id),
7168 Some(strategy_id)
7169 );
7170 assert_eq!(
7171 node.exec_manager
7172 .get_external_order_claim(&configured_instrument_id),
7173 None
7174 );
7175 assert_eq!(
7176 node.kernel
7177 .exec_engine
7178 .borrow()
7179 .get_external_order_claim(&configured_instrument_id),
7180 None
7181 );
7182 }
7183
7184 #[rstest]
7185 fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7186 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7187 .unwrap()
7188 .with_reconciliation(false)
7189 .build()
7190 .unwrap();
7191 let exec_engine = node.kernel.exec_engine.clone();
7192 let _engine_borrow = exec_engine.borrow_mut();
7193
7194 node.add_strategy(TestStrategy::new(StrategyConfig {
7195 strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7196 ..Default::default()
7197 }))
7198 .unwrap();
7199 }
7200
7201 #[rstest]
7202 fn test_add_strategy_registers_configured_hedging_oms_type() {
7203 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7204 .unwrap()
7205 .with_reconciliation(false)
7206 .with_delay_post_stop_secs(0)
7207 .with_timeout_connection(1)
7208 .build()
7209 .unwrap();
7210 let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7211
7212 node.add_strategy(TestStrategy::new(StrategyConfig {
7213 strategy_id: Some(strategy_id),
7214 oms_type: Some(OmsType::Hedging),
7215 ..Default::default()
7216 }))
7217 .unwrap();
7218
7219 let instrument = crypto_perpetual_ethusdt();
7220 let instrument_id = instrument.id();
7221 let client_id = ClientId::from("STUB");
7222
7223 node.kernel
7224 .cache
7225 .borrow_mut()
7226 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7227 .unwrap();
7228 node.kernel
7229 .exec_engine
7230 .borrow_mut()
7231 .register_client(Box::new(StubExecutionClient::new(
7232 client_id,
7233 AccountId::from("TEST-ACCOUNT"),
7234 instrument_id.venue,
7235 OmsType::Netting,
7236 None,
7237 )))
7238 .unwrap();
7239
7240 let order = OrderTestBuilder::new(OrderType::Market)
7241 .trader_id(node.trader_id())
7242 .strategy_id(strategy_id)
7243 .instrument_id(instrument_id)
7244 .quantity(Quantity::from("1.000"))
7245 .build();
7246 let position_id = PositionId::new("CUSTOM-POSITION-001");
7247
7248 node.kernel
7249 .cache
7250 .borrow_mut()
7251 .add_order(order.clone(), Some(position_id), Some(client_id), true)
7252 .unwrap();
7253
7254 let submit_order = SubmitOrder::new(
7255 order.trader_id(),
7256 Some(client_id),
7257 strategy_id,
7258 instrument_id,
7259 order.client_order_id(),
7260 order.init_event().clone(),
7261 order.exec_algorithm_id(),
7262 Some(position_id),
7263 None,
7264 UUID4::new(),
7265 UnixNanos::default(),
7266 None,
7267 );
7268
7269 node.kernel
7270 .exec_engine
7271 .borrow()
7272 .execute(TradingCommand::SubmitOrder(submit_order));
7273
7274 let exec_engine = node.kernel.exec_engine.borrow();
7275 let cache = exec_engine.cache().borrow();
7276 let cached_order = cache
7277 .order(&order.client_order_id())
7278 .expect("Order should be cached");
7279
7280 assert_eq!(cached_order.status(), OrderStatus::Initialized);
7281 }
7282
7283 #[cfg(all(feature = "simulation", madsim))]
7284 async fn advance_clock(d: Duration) {
7285 madsim::time::advance(d);
7286 madsim::task::yield_now().await;
7287 }
7288
7289 #[cfg(not(all(feature = "simulation", madsim)))]
7290 async fn advance_clock(d: Duration) {
7291 tokio::time::advance(d).await;
7292 }
7293
7294 #[rstest]
7295 #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7296 #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7297 #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7298 #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7299 #[case::unapplied_cancel_default(
7300 SubmissionRecoveryPolicy::ResolveLocally,
7301 false,
7302 Some(OrderStatus::PendingCancel),
7303 false
7304 )]
7305 #[case::unapplied_modify_default(
7306 SubmissionRecoveryPolicy::ResolveLocally,
7307 false,
7308 Some(OrderStatus::PendingUpdate),
7309 false
7310 )]
7311 #[case::unapplied_cancel_diagnostics(
7312 SubmissionRecoveryPolicy::RetainUnresolved,
7313 false,
7314 Some(OrderStatus::PendingCancel),
7315 false
7316 )]
7317 #[case::unapplied_modify_diagnostics(
7318 SubmissionRecoveryPolicy::RetainUnresolved,
7319 false,
7320 Some(OrderStatus::PendingUpdate),
7321 false
7322 )]
7323 #[case::first_fill_cancel_default(
7324 SubmissionRecoveryPolicy::ResolveLocally,
7325 false,
7326 Some(OrderStatus::PendingCancel),
7327 true
7328 )]
7329 #[case::first_fill_modify_default(
7330 SubmissionRecoveryPolicy::ResolveLocally,
7331 false,
7332 Some(OrderStatus::PendingUpdate),
7333 true
7334 )]
7335 #[case::first_fill_cancel_diagnostics(
7336 SubmissionRecoveryPolicy::RetainUnresolved,
7337 false,
7338 Some(OrderStatus::PendingCancel),
7339 true
7340 )]
7341 #[case::first_fill_modify_diagnostics(
7342 SubmissionRecoveryPolicy::RetainUnresolved,
7343 false,
7344 Some(OrderStatus::PendingUpdate),
7345 true
7346 )]
7347 #[cfg_attr(
7348 not(all(feature = "simulation", madsim)),
7349 tokio::test(start_paused = true)
7350 )]
7351 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7352 async fn test_submission_registry_publishes_after_native_resolution(
7353 #[case] policy: SubmissionRecoveryPolicy,
7354 #[case] order_list: bool,
7355 #[case] pending_status: Option<OrderStatus>,
7356 #[case] first_fill: bool,
7357 ) {
7358 let config = LiveNodeConfig {
7359 exec_engine: crate::config::LiveExecutionEngineConfig {
7360 inflight_check_threshold_ms: 100,
7361 inflight_check_retries: 2,
7362 submission_recovery_policy: policy,
7363 ..Default::default()
7364 },
7365 ..Default::default()
7366 };
7367 let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7368 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7369 let client_id = ClientId::from("TEST-SUBMISSION");
7370 let account_id = AccountId::from("TEST-001");
7371 let strategy_id = StrategyId::from("SUBMISSION-001");
7372 let account = AccountAny::Margin(MarginAccount::new(
7373 AccountState::new(
7374 account_id,
7375 AccountType::Margin,
7376 vec![AccountBalance::new(
7377 Money::from("1000000 USDT"),
7378 Money::from("0 USDT"),
7379 Money::from("1000000 USDT"),
7380 )],
7381 Vec::new(),
7382 true,
7383 UUID4::new(),
7384 UnixNanos::default(),
7385 UnixNanos::default(),
7386 Some(Currency::USDT()),
7387 ),
7388 true,
7389 ));
7390 node.kernel.cache.borrow_mut().add_account(account).unwrap();
7391 node.kernel
7392 .cache
7393 .borrow_mut()
7394 .add_instrument(instrument.clone())
7395 .unwrap();
7396 let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7397 .map(|index| {
7398 OrderTestBuilder::new(OrderType::Limit)
7399 .trader_id(node.trader_id())
7400 .strategy_id(strategy_id)
7401 .client_order_id(ClientOrderId::from(
7402 format!("O-SUBMISSION-{index}").as_str(),
7403 ))
7404 .instrument_id(instrument.id())
7405 .quantity(Quantity::from("1.000"))
7406 .price(Price::from("100.00"))
7407 .build()
7408 })
7409 .collect();
7410 let command = if order_list {
7411 TradingCommand::SubmitOrderList(SubmitOrderList::new(
7412 node.trader_id(),
7413 Some(client_id),
7414 strategy_id,
7415 OrderList::new(
7416 OrderListId::from("OL-SUBMISSION"),
7417 instrument.id(),
7418 strategy_id,
7419 orders.iter().map(Order::client_order_id).collect(),
7420 UnixNanos::default(),
7421 ),
7422 orders
7423 .iter()
7424 .map(|order| order.init_event().clone())
7425 .collect(),
7426 None,
7427 None,
7428 None,
7429 UUID4::new(),
7430 UnixNanos::default(),
7431 None,
7432 ))
7433 } else {
7434 TradingCommand::SubmitOrder(SubmitOrder::from_order(
7435 &orders[0],
7436 node.trader_id(),
7437 Some(client_id),
7438 None,
7439 UUID4::new(),
7440 UnixNanos::default(),
7441 ))
7442 };
7443 node.observe_exec_command_before_dispatch(&command);
7444
7445 for order in &orders {
7446 let submitted = TestOrderEventStubs::submitted(order, account_id);
7447 node.kernel
7448 .cache
7449 .borrow_mut()
7450 .add_order(order.clone(), None, Some(client_id), false)
7451 .unwrap();
7452 node.process_exec_event(ExecutionEvent::Order(submitted));
7453 }
7454
7455 let expected_status = match pending_status {
7456 Some(status) if !first_fill => status,
7457 Some(_) => OrderStatus::Canceled,
7458 None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7459 None => OrderStatus::Rejected,
7460 };
7461 let diagnostics = Rc::new(RefCell::new(Vec::new()));
7462 let handler = ShareableMessageHandler::from_typed({
7463 let diagnostics = diagnostics.clone();
7464 let cache = node.kernel.cache.clone();
7465 move |diagnostic: &SubmissionRecoveryExhausted| {
7466 let cache = cache.borrow_mut();
7467 let order = cache.order(&diagnostic.client_order_id).unwrap();
7468 assert_eq!(order.status(), expected_status);
7469 assert!(diagnostic.ts_event >= order.ts_last());
7470 diagnostics.borrow_mut().push(diagnostic.clone());
7471 }
7472 });
7473 let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7474 msgbus::subscribe_any(topic.into(), handler.clone(), None);
7475 let queries = Rc::new(RefCell::new(Vec::new()));
7476 msgbus::register_trading_command_endpoint(
7477 MessagingSwitchboard::exec_engine_execute(),
7478 TypedIntoHandler::from({
7479 let queries = queries.clone();
7480 move |command: TradingCommand| queries.borrow_mut().push(command)
7481 }),
7482 );
7483
7484 let mut last_inflight_check = dst::time::Instant::now();
7485 let mut last_open_check = last_inflight_check;
7486 let mut last_position_check = last_inflight_check;
7487 let mut open_order_report_task = None;
7488 let mut targeted_order_report_task = None;
7489 let mut position_report_task = None;
7490
7491 for check in 1..=3 {
7492 advance_clock(Duration::from_millis(101)).await;
7493 node.run_reconciliation_checks(
7494 dst::time::Instant::now(),
7495 ReconciliationCheckIntervals {
7496 inflight: Duration::from_millis(100),
7497 open: Duration::ZERO,
7498 position: Duration::ZERO,
7499 },
7500 &mut ReconciliationCheckState {
7501 last_inflight_check: &mut last_inflight_check,
7502 last_open_check: &mut last_open_check,
7503 last_position_check: &mut last_position_check,
7504 open_order_report_task: &mut open_order_report_task,
7505 targeted_order_report_task: &mut targeted_order_report_task,
7506 position_report_task: &mut position_report_task,
7507 },
7508 );
7509
7510 if check == 1 {
7511 assert!(diagnostics.borrow().is_empty());
7512
7513 if let Some(pending_status) = pending_status {
7514 let order = &orders[0];
7515 let pending = if pending_status == OrderStatus::PendingCancel {
7516 OrderEventAny::PendingCancel(
7517 OrderPendingCancelSpec::builder()
7518 .trader_id(node.trader_id())
7519 .strategy_id(strategy_id)
7520 .instrument_id(instrument.id())
7521 .client_order_id(order.client_order_id())
7522 .account_id(account_id)
7523 .build(),
7524 )
7525 } else {
7526 OrderEventAny::PendingUpdate(
7527 OrderPendingUpdateSpec::builder()
7528 .trader_id(node.trader_id())
7529 .strategy_id(strategy_id)
7530 .instrument_id(instrument.id())
7531 .client_order_id(order.client_order_id())
7532 .account_id(account_id)
7533 .build(),
7534 )
7535 };
7536 node.process_exec_event(ExecutionEvent::Order(pending));
7537 node.exec_manager.register_inflight(order.client_order_id());
7538 if first_fill {
7539 let cached = node
7540 .kernel
7541 .cache
7542 .borrow()
7543 .order(&order.client_order_id())
7544 .unwrap()
7545 .clone();
7546 let OrderEventAny::Filled(mut fill) =
7547 OrderFilledTestBuilder::new(&cached, &instrument)
7548 .last_qty(Quantity::from("0.400"))
7549 .last_px(Price::from("100.00"))
7550 .commission(Money::from("0 USDT"))
7551 .without_position_id()
7552 .build()
7553 else {
7554 unreachable!()
7555 };
7556 fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7557 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7558 } else {
7559 let report = OrderStatusReport::new(
7560 account_id,
7561 instrument.id(),
7562 Some(order.client_order_id()),
7563 VenueOrderId::from("V-SUBMISSION"),
7564 Some(order.order_side()),
7565 order.order_type(),
7566 order.time_in_force(),
7567 OrderStatus::Submitted,
7568 order.quantity(),
7569 Quantity::zero(3),
7570 UnixNanos::default(),
7571 UnixNanos::default(),
7572 UnixNanos::default(),
7573 None,
7574 )
7575 .with_price(Price::from("100.00"));
7576 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
7577 Box::new(report),
7578 )));
7579 }
7580 let cache = node.kernel.cache.borrow();
7581 let cached = cache.order(&order.client_order_id()).unwrap();
7582 assert_eq!(cached.status(), pending_status);
7583 assert_eq!(
7584 cached.filled_qty(),
7585 if first_fill {
7586 Quantity::from("0.400")
7587 } else {
7588 Quantity::zero(3)
7589 }
7590 );
7591 assert!(
7592 !cached
7593 .events()
7594 .iter()
7595 .any(|event| matches!(event, OrderEventAny::Accepted(_)))
7596 );
7597 }
7598 }
7599 }
7600
7601 assert_eq!(
7602 queries.borrow().len(),
7603 orders.len()
7604 + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
7605 );
7606
7607 for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
7608 assert!(matches!(command, TradingCommand::QueryOrder(query)
7609 if query.client_order_id == order.client_order_id()
7610 && query.client_id == Some(client_id)));
7611 assert_eq!(
7612 node.kernel
7613 .cache
7614 .borrow()
7615 .order(&order.client_order_id())
7616 .unwrap()
7617 .status(),
7618 if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
7619 pending_status.unwrap()
7620 } else {
7621 expected_status
7622 }
7623 );
7624 }
7625 let diagnostics = diagnostics.borrow();
7626 assert_eq!(
7627 diagnostics.len(),
7628 if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
7629 orders.len()
7630 } else {
7631 0
7632 }
7633 );
7634
7635 for (diagnostic, order) in diagnostics.iter().zip(&orders) {
7636 assert_eq!(diagnostic.trader_id, node.trader_id());
7637 assert_eq!(diagnostic.client_id, Some(client_id));
7638 assert_eq!(diagnostic.strategy_id, strategy_id);
7639 assert_eq!(diagnostic.instrument_id, instrument.id());
7640 assert_eq!(diagnostic.client_order_id, order.client_order_id());
7641 assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
7642 assert_eq!(diagnostic.retry_count, 2);
7643 }
7644 msgbus::unsubscribe_any(topic.into(), &handler);
7645 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7646 }
7647
7648 #[derive(Debug)]
7649 struct SubmissionCallbackStrategy {
7650 core: StrategyCore,
7651 cancel: bool,
7652 batch: bool,
7653 callback: &'static str,
7654 client_id: ClientId,
7655 }
7656
7657 impl DataActor for SubmissionCallbackStrategy {}
7658
7659 impl SubmissionCallbackStrategy {
7660 fn queue_command(&mut self, client_order_id: ClientOrderId) {
7661 match (self.cancel, self.batch) {
7662 (true, true) => {
7663 self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
7664 .unwrap();
7665 }
7666 (true, false) => {
7667 self.cancel_order(client_order_id, Some(self.client_id), None)
7668 .unwrap();
7669 }
7670 (false, true) => {
7671 self.modify_orders(
7672 vec![(client_order_id, None, Some(Price::from("101.00")), None)],
7673 Some(self.client_id),
7674 None,
7675 )
7676 .unwrap();
7677 }
7678 (false, false) => self
7679 .modify_order(
7680 client_order_id,
7681 None,
7682 Some(Price::from("101.00")),
7683 None,
7684 Some(self.client_id),
7685 None,
7686 )
7687 .unwrap(),
7688 }
7689 }
7690 }
7691
7692 nautilus_strategy!(SubmissionCallbackStrategy, {
7693 fn on_order_accepted(&mut self, event: OrderAccepted) {
7694 if self.callback == "accepted" {
7695 self.queue_command(event.client_order_id);
7696 }
7697 }
7698
7699 fn on_order_updated(&mut self, event: OrderUpdated) {
7700 if self.callback == "updated" {
7701 self.queue_command(event.client_order_id);
7702 }
7703 }
7704 });
7705
7706 #[rstest]
7707 #[case::native(false, "accepted")]
7708 #[case::report(true, "accepted")]
7709 #[case::updated(false, "updated")]
7710 #[cfg_attr(
7711 not(all(feature = "simulation", madsim)),
7712 tokio::test(start_paused = true)
7713 )]
7714 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7715 async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
7716 #[values(
7717 SubmissionRecoveryPolicy::ResolveLocally,
7718 SubmissionRecoveryPolicy::RetainUnresolved
7719 )]
7720 policy: SubmissionRecoveryPolicy,
7721 #[values(false, true)] cancel: bool,
7722 #[values(false, true)] batch: bool,
7723 #[case] report: bool,
7724 #[case] callback: &'static str,
7725 ) {
7726 let config = LiveNodeConfig {
7727 exec_engine: crate::config::LiveExecutionEngineConfig {
7728 inflight_check_threshold_ms: 100,
7729 inflight_check_retries: 2,
7730 submission_recovery_policy: policy,
7731 ..Default::default()
7732 },
7733 ..Default::default()
7734 };
7735 let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
7736 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7737 let account_id = AccountId::from("TEST-001");
7738 let strategy_id = StrategyId::from("CALLBACK-001");
7739 let client_id = ClientId::from("TEST-CALLBACK");
7740 let account = AccountAny::Margin(MarginAccount::new(
7741 AccountState::new(
7742 account_id,
7743 AccountType::Margin,
7744 vec![AccountBalance::new(
7745 Money::from("1000000 USDT"),
7746 Money::from("0 USDT"),
7747 Money::from("1000000 USDT"),
7748 )],
7749 Vec::new(),
7750 true,
7751 UUID4::new(),
7752 UnixNanos::default(),
7753 UnixNanos::default(),
7754 Some(Currency::USDT()),
7755 ),
7756 true,
7757 ));
7758 node.kernel.cache.borrow_mut().add_account(account).unwrap();
7759 node.kernel
7760 .cache
7761 .borrow_mut()
7762 .add_instrument(instrument.clone())
7763 .unwrap();
7764 node.kernel
7765 .exec_engine
7766 .borrow_mut()
7767 .register_client(Box::new(StubExecutionClient::new(
7768 client_id,
7769 account_id,
7770 instrument.id().venue,
7771 OmsType::Netting,
7772 None,
7773 )))
7774 .unwrap();
7775 node.add_strategy(SubmissionCallbackStrategy {
7776 core: StrategyCore::new(StrategyConfig {
7777 strategy_id: Some(strategy_id),
7778 ..Default::default()
7779 }),
7780 cancel,
7781 batch,
7782 callback,
7783 client_id,
7784 })
7785 .unwrap();
7786 node.kernel.trader.borrow_mut().initialize().unwrap();
7787 node.kernel.start_trader().unwrap();
7788 let runner = node.runner.take().unwrap();
7789 runner.bind_senders_for_node(node.handle.clone());
7790 let mut channels = runner.take_channels();
7791 let order = OrderTestBuilder::new(OrderType::Limit)
7792 .trader_id(node.trader_id())
7793 .strategy_id(strategy_id)
7794 .client_order_id(ClientOrderId::from("O-CALLBACK"))
7795 .instrument_id(instrument.id())
7796 .quantity(Quantity::from("1.000"))
7797 .price(Price::from("100.00"))
7798 .build();
7799 let client_order_id = order.client_order_id();
7800 let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
7801 &order,
7802 node.trader_id(),
7803 Some(client_id),
7804 None,
7805 UUID4::new(),
7806 UnixNanos::default(),
7807 ));
7808 node.observe_exec_command_before_dispatch(&submit);
7809 node.kernel
7810 .cache
7811 .borrow_mut()
7812 .add_order(order.clone(), None, Some(client_id), false)
7813 .unwrap();
7814 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
7815 &order, account_id,
7816 )));
7817 advance_clock(Duration::from_millis(101)).await;
7818 let first = node.exec_manager.check_inflight_orders();
7819 assert_eq!(first.queries.len(), 1);
7820 assert!(first.events.is_empty());
7821
7822 advance_clock(Duration::from_millis(101)).await;
7823 let accepted =
7824 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
7825
7826 if callback == "updated" {
7827 let mut update = OrderUpdatedSpec::builder()
7828 .trader_id(node.trader_id())
7829 .strategy_id(strategy_id)
7830 .instrument_id(instrument.id())
7831 .client_order_id(client_order_id)
7832 .account_id(account_id)
7833 .quantity(order.quantity())
7834 .price(order.price().unwrap())
7835 .build();
7836 update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
7837 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
7838 } else if report {
7839 let accepted = OrderStatusReport::new(
7840 account_id,
7841 instrument.id(),
7842 Some(client_order_id),
7843 VenueOrderId::from("V-CALLBACK"),
7844 Some(order.order_side()),
7845 order.order_type(),
7846 order.time_in_force(),
7847 OrderStatus::Accepted,
7848 order.quantity(),
7849 Quantity::zero(3),
7850 UnixNanos::default(),
7851 UnixNanos::default(),
7852 UnixNanos::default(),
7853 None,
7854 )
7855 .with_price(Price::from("100.00"));
7856 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
7857 accepted,
7858 ))));
7859 } else {
7860 node.process_exec_event(ExecutionEvent::Order(accepted));
7861 }
7862
7863 let pending_status = if cancel {
7864 OrderStatus::PendingCancel
7865 } else {
7866 OrderStatus::PendingUpdate
7867 };
7868 assert_eq!(channels.exec_cmd_rx.len(), 1);
7869 assert_eq!(
7870 node.kernel
7871 .cache
7872 .borrow()
7873 .order(&client_order_id)
7874 .unwrap()
7875 .status(),
7876 pending_status
7877 );
7878
7879 for _ in 0..3 {
7881 advance_clock(Duration::from_millis(101)).await;
7882 let maintenance = node.exec_manager.check_inflight_orders();
7883 assert!(maintenance.events.is_empty());
7884 assert!(maintenance.queries.is_empty());
7885 assert!(
7886 node.exec_manager
7887 .take_submission_recovery_exhaustions()
7888 .is_empty()
7889 );
7890 }
7891
7892 let command = channels.exec_cmd_rx.try_recv().unwrap();
7893 node.process_exec_command(command);
7894 assert!(channels.exec_cmd_rx.is_empty());
7895 let immediate = node.exec_manager.check_inflight_orders();
7896 assert!(immediate.events.is_empty());
7897 assert!(immediate.queries.is_empty());
7898 advance_clock(Duration::from_millis(101)).await;
7899 let first_command_check = node.exec_manager.check_inflight_orders();
7900 assert!(first_command_check.events.is_empty());
7901 assert_eq!(first_command_check.queries.len(), 1);
7902 advance_clock(Duration::from_millis(101)).await;
7903 let exhausted = node.exec_manager.check_inflight_orders();
7904 assert!(exhausted.queries.is_empty());
7905 assert!(
7906 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
7907 if event.client_order_id == client_order_id)
7908 );
7909 assert!(
7910 node.exec_manager
7911 .take_submission_recovery_exhaustions()
7912 .is_empty()
7913 );
7914 node.process_reconciliation_events(&exhausted.events);
7915 assert_eq!(
7916 node.kernel
7917 .cache
7918 .borrow()
7919 .order(&client_order_id)
7920 .unwrap()
7921 .status(),
7922 OrderStatus::Canceled
7923 );
7924 }
7925
7926 #[cfg_attr(
7927 not(all(feature = "simulation", madsim)),
7928 tokio::test(start_paused = true)
7929 )]
7930 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7931 async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
7932 let last = dst::time::Instant::now();
7933 let interval = Duration::from_millis(100);
7934
7935 assert!(!reconciliation_check_due(last, last, Duration::ZERO));
7936 assert!(!reconciliation_check_due(last, last, interval));
7937
7938 advance_clock(Duration::from_millis(99)).await;
7939 let before_interval = dst::time::Instant::now();
7940 assert!(!reconciliation_check_due(before_interval, last, interval));
7941
7942 advance_clock(Duration::from_millis(1)).await;
7943 let at_interval = dst::time::Instant::now();
7944 assert!(reconciliation_check_due(at_interval, last, interval));
7945
7946 assert!(!reconciliation_check_due(last, at_interval, interval));
7947 }
7948
7949 #[cfg_attr(
7950 not(all(feature = "simulation", madsim)),
7951 tokio::test(start_paused = true)
7952 )]
7953 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7954 async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
7955 let config = LiveNodeConfig {
7956 exec_engine: crate::config::LiveExecutionEngineConfig {
7957 reconciliation: true,
7958 open_check_interval_secs: Some(1.0),
7959 position_check_interval_secs: Some(1.0),
7960 max_single_order_queries_per_cycle: 5,
7961 ..Default::default()
7962 },
7963 ..Default::default()
7964 };
7965
7966 let mut node =
7967 LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
7968 let client_id = ClientId::from("TEST-QUERY");
7969 let account_id = AccountId::from("TEST-QUERY-001");
7970
7971 let trading_commands = Rc::new(RefCell::new(Vec::new()));
7972 msgbus::register_trading_command_endpoint(
7973 MessagingSwitchboard::exec_engine_execute(),
7974 TypedIntoHandler::from({
7975 let trading_commands = trading_commands.clone();
7976 move |command: TradingCommand| {
7977 trading_commands.borrow_mut().push(command);
7978 }
7979 }),
7980 );
7981
7982 let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
7983 let instrument = crypto_perpetual_ethusdt();
7984 let instrument_id = instrument.id();
7985 let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
7986
7987 node.kernel
7988 .cache
7989 .borrow_mut()
7990 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7991 .unwrap();
7992 insert_accepted_limit_order_in_node(
7993 &node,
7994 account_id,
7995 client_id,
7996 instrument_id,
7997 client_order_id,
7998 venue_order_id,
7999 );
8000
8001 let last = dst::time::Instant::now();
8002 advance_clock(Duration::from_nanos(1)).await;
8003 let now = dst::time::Instant::now();
8004 let mut last_inflight_check = last;
8005 let mut last_open_check = last;
8006 let mut last_position_check = last;
8007 let mut open_order_report_task = None;
8008 let mut targeted_order_report_task = None;
8009 let mut position_report_task = None;
8010
8011 node.run_reconciliation_checks(
8012 now,
8013 ReconciliationCheckIntervals {
8014 inflight: Duration::ZERO,
8015 open: Duration::from_nanos(1),
8016 position: Duration::ZERO,
8017 },
8018 &mut ReconciliationCheckState {
8019 last_inflight_check: &mut last_inflight_check,
8020 last_open_check: &mut last_open_check,
8021 last_position_check: &mut last_position_check,
8022 open_order_report_task: &mut open_order_report_task,
8023 targeted_order_report_task: &mut targeted_order_report_task,
8024 position_report_task: &mut position_report_task,
8025 },
8026 );
8027
8028 let commands = trading_commands.borrow();
8029
8030 assert!(commands.is_empty());
8031 assert!(open_order_report_task.is_none());
8032 assert!(targeted_order_report_task.is_none());
8033 assert!(position_report_task.is_none());
8034
8035 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8036 }
8037
8038 #[rstest]
8039 #[tokio::test]
8040 async fn test_recurring_report_failure_preserves_client_coverage(
8041 #[values(false, true)] positions: bool,
8042 #[values(false, true)] failure_first: bool,
8043 ) {
8044 let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8045 let failed_id = ClientId::from("FAILED");
8046 let healthy_id = ClientId::from("HEALTHY");
8047
8048 for (client_id, outcome) in [
8049 (failed_id, FillReportClientOutcome::Failure),
8050 (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8051 ] {
8052 node.exec_clients
8053 .push(LiveExecutionClient::new(Box::new(FillReportClient {
8054 client_id,
8055 account_id: AccountId::from("TEST-001"),
8056 venue: Venue::from("BINANCE"),
8057 outcome,
8058 commands: Rc::new(RefCell::new(Vec::new())),
8059 })));
8060 }
8061
8062 if !failure_first {
8063 node.exec_clients.reverse();
8064 }
8065
8066 let last = dst::time::Instant::now();
8067 let now = last + Duration::from_secs(1);
8068 let mut last_inflight = last;
8069 let mut last_open = last;
8070 let mut last_position = last;
8071 let mut open_task = None;
8072 let mut targeted_task = None;
8073 let mut position_task = None;
8074
8075 node.run_reconciliation_checks(
8076 now,
8077 ReconciliationCheckIntervals {
8078 inflight: Duration::ZERO,
8079 open: if positions {
8080 Duration::ZERO
8081 } else {
8082 Duration::from_secs(1)
8083 },
8084 position: if positions {
8085 Duration::from_secs(1)
8086 } else {
8087 Duration::ZERO
8088 },
8089 },
8090 &mut ReconciliationCheckState {
8091 last_inflight_check: &mut last_inflight,
8092 last_open_check: &mut last_open,
8093 last_position_check: &mut last_position,
8094 open_order_report_task: &mut open_task,
8095 targeted_order_report_task: &mut targeted_task,
8096 position_report_task: &mut position_task,
8097 },
8098 );
8099
8100 let (queried_clients, failed_clients) = if positions {
8101 let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8102 position_task.unwrap().future.await
8103 else {
8104 panic!("position collection should complete despite one client failure");
8105 };
8106
8107 let result = result.finish();
8108 assert!(result.reports.is_empty());
8109 assert!(open_task.is_none());
8110 (result.queried_clients, result.failed_clients)
8111 } else {
8112 let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8113 panic!("order collection should complete despite one client failure");
8114 };
8115
8116 let result = result.finish();
8117
8118 assert!(result.reports.is_empty());
8119 assert!(position_task.is_none());
8120 (result.queried_clients, result.failed_clients)
8121 };
8122
8123 assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8124 assert_eq!(failed_clients, IndexSet::from([failed_id]));
8125 assert!(targeted_task.is_none());
8126 }
8127
8128 #[rstest]
8129 #[case::inflight(true, false)]
8130 #[case::open(false, false)]
8131 #[case::positions(false, true)]
8132 #[tokio::test]
8133 async fn test_reconciliation_shutdown_preserves_pending_checks(
8134 #[case] inflight: bool,
8135 #[case] positions: bool,
8136 ) {
8137 let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8138 node.handle.set_shutting_down();
8139 let last = dst::time::Instant::now();
8140 let now = last + Duration::from_secs(1);
8141 let mut last_inflight = last;
8142 let mut last_open = last;
8143 let mut last_position = last;
8144 let mut open_task = None;
8145 let mut targeted_task = None;
8146 let mut position_task = None;
8147
8148 node.run_reconciliation_checks(
8149 now,
8150 ReconciliationCheckIntervals {
8151 inflight: if inflight {
8152 Duration::from_secs(1)
8153 } else {
8154 Duration::ZERO
8155 },
8156 open: if !inflight && !positions {
8157 Duration::from_secs(1)
8158 } else {
8159 Duration::ZERO
8160 },
8161 position: if positions {
8162 Duration::from_secs(1)
8163 } else {
8164 Duration::ZERO
8165 },
8166 },
8167 &mut ReconciliationCheckState {
8168 last_inflight_check: &mut last_inflight,
8169 last_open_check: &mut last_open,
8170 last_position_check: &mut last_position,
8171 open_order_report_task: &mut open_task,
8172 targeted_order_report_task: &mut targeted_task,
8173 position_report_task: &mut position_task,
8174 },
8175 );
8176
8177 assert_eq!(
8178 (last_inflight, last_open, last_position),
8179 (last, last, last)
8180 );
8181 assert!(open_task.is_none());
8182 assert!(targeted_task.is_none());
8183 assert!(position_task.is_none());
8184 }
8185
8186 fn insert_accepted_limit_order_in_node(
8187 node: &LiveNode,
8188 account_id: AccountId,
8189 client_id: ClientId,
8190 instrument_id: InstrumentId,
8191 client_order_id: ClientOrderId,
8192 venue_order_id: VenueOrderId,
8193 ) {
8194 let order = OrderTestBuilder::new(OrderType::Limit)
8195 .client_order_id(client_order_id)
8196 .instrument_id(instrument_id)
8197 .quantity(Quantity::from("10.0"))
8198 .price(Price::from("100.0"))
8199 .build();
8200 let submitted = TestOrderEventStubs::submitted(&order, account_id);
8201 node.kernel
8202 .cache
8203 .borrow_mut()
8204 .add_order(order, None, Some(client_id), false)
8205 .unwrap();
8206 let order = node
8207 .kernel
8208 .cache
8209 .borrow_mut()
8210 .update_order(&submitted)
8211 .unwrap();
8212 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8213 node.kernel
8214 .cache
8215 .borrow_mut()
8216 .update_order(&accepted)
8217 .unwrap();
8218 }
8219
8220 fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8221 let config = LiveNodeConfig {
8222 exec_engine: crate::config::LiveExecutionEngineConfig {
8223 reconciliation: true,
8224 ..Default::default()
8225 },
8226 ..Default::default()
8227 };
8228
8229 let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8230 let account_id = AccountId::from("TEST-001");
8231 let client_id = ClientId::from("TEST-RECENT-FILL");
8232 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8233 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8234 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8235 node.kernel
8236 .cache
8237 .borrow_mut()
8238 .add_instrument(instrument.clone())
8239 .unwrap();
8240 insert_accepted_limit_order_in_node(
8241 &node,
8242 account_id,
8243 client_id,
8244 instrument.id(),
8245 client_order_id,
8246 venue_order_id,
8247 );
8248 let order = node
8249 .kernel
8250 .cache
8251 .borrow()
8252 .order_owned(&client_order_id)
8253 .unwrap();
8254 let fill = TestOrderEventStubs::filled(
8255 &order,
8256 &instrument,
8257 Some(TradeId::from("T-RECENT-FILL")),
8258 None,
8259 Some(Price::from("100.0")),
8260 Some(Quantity::from("1.0")),
8261 Some(LiquiditySide::Taker),
8262 None,
8263 None,
8264 Some(account_id),
8265 );
8266
8267 (node, fill, instrument)
8268 }
8269
8270 fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8271 let client_order_id = fill_report.client_order_id.unwrap();
8272 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8273 let order = node
8274 .kernel
8275 .cache
8276 .borrow()
8277 .order_owned(&client_order_id)
8278 .unwrap();
8279 let order_report = OrderStatusReport::new(
8280 fill_report.account_id,
8281 fill_report.instrument_id,
8282 Some(client_order_id),
8283 fill_report.venue_order_id,
8284 OrderSide::Buy.into(),
8285 OrderType::Limit,
8286 TimeInForce::Gtc,
8287 OrderStatus::PartiallyFilled,
8288 Quantity::from("10.0"),
8289 Quantity::from("2.0"),
8290 UnixNanos::from(1_000),
8291 UnixNanos::from(1_000),
8292 UnixNanos::from(1_000),
8293 None,
8294 )
8295 .with_avg_px(dec!(100.0));
8296 let inferred = create_inferred_fill_for_qty(
8297 &order,
8298 &order_report,
8299 &fill_report.account_id,
8300 &instrument,
8301 Quantity::from("1.0"),
8302 UnixNanos::from(1_000),
8303 None,
8304 )
8305 .unwrap();
8306
8307 node.process_reconciliation_events(&[inferred]);
8308 }
8309
8310 #[rstest]
8311 #[case::rejected(false, "10.0", "10.0", "1.0")]
8312 #[case::dust(true, "9.004", "10.004", "10.004")]
8313 #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8314 fn test_position_check_overfill_raises_before_fill_when_client_allows(
8315 #[case] allow: bool,
8316 #[case] fill_qty: &str,
8317 #[case] expected_qty: &str,
8318 #[case] expected_filled: &str,
8319 ) {
8320 let (mut node, venue_report, mut fill_report) =
8321 position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8322 fill_report.commission = Money::from("1.25 USDT");
8323 let key = (fill_report.instrument_id, fill_report.account_id);
8324
8325 let client = StubExecutionClient::new(
8326 ClientId::from("POSITION-FILLS"),
8327 AccountId::from("TEST-001"),
8328 fill_report.instrument_id.venue,
8329 OmsType::Netting,
8330 None,
8331 );
8332
8333 let client = if allow {
8334 client.with_allows_reconciliation_overfill()
8335 } else {
8336 client
8337 };
8338
8339 node.kernel
8340 .exec_engine
8341 .borrow_mut()
8342 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8343 .unwrap();
8344
8345 let position_result = position_report_result(&node, venue_report);
8346 node.handle_position_fill_report_result(PositionFillReportResult {
8347 position_result,
8348 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8349 });
8350
8351 let order = node
8352 .kernel
8353 .cache
8354 .borrow()
8355 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8356 .unwrap();
8357 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8358 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8359 let applied = order
8360 .trade_ids()
8361 .iter()
8362 .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8363 assert_eq!(applied, allow);
8364 let revision = node.exec_manager.position_activity_revision(&key);
8365
8366 if !allow {
8367 return;
8368 }
8369
8370 assert_eq!(revision, 2);
8371
8372 {
8373 let cache = node.kernel.cache.borrow();
8374 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8375 assert_eq!(positions.len(), 1);
8376 assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8377 assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8378 assert_eq!(positions[0].trade_ids.len(), 2);
8379 }
8380
8381 let replay_report = position_report_result(
8382 &node,
8383 PositionStatusReport::new(
8384 key.1,
8385 key.0,
8386 PositionSide::Long,
8387 Quantity::from(expected_qty),
8388 UnixNanos::from(2_000),
8389 UnixNanos::from(2_000),
8390 None,
8391 None,
8392 Some(dec!(100.0)),
8393 ),
8394 );
8395 node.handle_position_fill_report_result(PositionFillReportResult {
8396 position_result: replay_report,
8397 reports: IndexMap::from([(key, vec![fill_report])]),
8398 });
8399
8400 let order = node
8401 .kernel
8402 .cache
8403 .borrow()
8404 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8405 .unwrap();
8406 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8407 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8408 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8409 }
8410
8411 #[rstest]
8412 fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8413 let (mut node, venue_report, mut overfill) =
8414 position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8415 overfill.commission = Money::from("0.20 USDT");
8416 let key = (overfill.instrument_id, overfill.account_id);
8417 let client = StubExecutionClient::new(
8418 ClientId::from("POSITION-FILLS"),
8419 AccountId::from("TEST-001"),
8420 overfill.instrument_id.venue,
8421 OmsType::Netting,
8422 None,
8423 )
8424 .with_allows_reconciliation_overfill();
8425 node.kernel
8426 .exec_engine
8427 .borrow_mut()
8428 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8429 .unwrap();
8430
8431 let mut fitting = overfill.clone();
8432 fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8433 fitting.last_qty = Quantity::from("8.0");
8434 fitting.commission = Money::from("0.80 USDT");
8435 let position_result = position_report_result(&node, venue_report);
8436 node.handle_position_fill_report_result(PositionFillReportResult {
8437 position_result,
8438 reports: IndexMap::from([(key, vec![fitting, overfill])]),
8439 });
8440
8441 let order = node
8442 .kernel
8443 .cache
8444 .borrow()
8445 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8446 .unwrap();
8447 assert_eq!(order.quantity(), Quantity::from("11.0"));
8448 assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8449 assert_eq!(order.trade_ids().len(), 3);
8450 assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8451 {
8452 let cache = node.kernel.cache.borrow();
8453 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8454 assert_eq!(positions.len(), 1);
8455 assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8456 assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8457 }
8458 }
8459
8460 #[rstest]
8461 fn test_position_check_second_overfill_does_not_raise_filled_order() {
8462 let (mut node, venue_report, mut first) =
8463 position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8464 first.commission = Money::from("1.20 USDT");
8465 let key = (first.instrument_id, first.account_id);
8466 let client = StubExecutionClient::new(
8467 ClientId::from("POSITION-FILLS"),
8468 AccountId::from("TEST-001"),
8469 first.instrument_id.venue,
8470 OmsType::Netting,
8471 None,
8472 )
8473 .with_allows_reconciliation_overfill();
8474 node.kernel
8475 .exec_engine
8476 .borrow_mut()
8477 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8478 .unwrap();
8479
8480 let mut second = first.clone();
8481 second.trade_id = TradeId::from("T-POSITION-LATER");
8482 second.last_qty = Quantity::from("1.0");
8483 second.commission = Money::from("0.10 USDT");
8484 second.ts_event = UnixNanos::from(1_001);
8485 let position_result = position_report_result(&node, venue_report);
8486 node.handle_position_fill_report_result(PositionFillReportResult {
8487 position_result,
8488 reports: IndexMap::from([(key, vec![first, second])]),
8489 });
8490
8491 let order = node
8492 .kernel
8493 .cache
8494 .borrow()
8495 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8496 .unwrap();
8497 assert_eq!(order.quantity(), Quantity::from("13.0"));
8498 assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8499 assert_eq!(order.status(), OrderStatus::Filled);
8500 assert_eq!(order.trade_ids().len(), 2);
8501 assert!(
8502 order
8503 .trade_ids()
8504 .iter()
8505 .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8506 );
8507 {
8508 let cache = node.kernel.cache.borrow();
8509 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8510 assert_eq!(positions.len(), 1);
8511 assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8512 assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8513 }
8514 }
8515
8516 #[rstest]
8517 fn test_allow_overfills_applies_fill_without_raising_quantity() {
8518 let config = LiveNodeConfig {
8519 exec_engine: crate::config::LiveExecutionEngineConfig {
8520 reconciliation: true,
8521 allow_overfills: true,
8522 ..Default::default()
8523 },
8524 ..Default::default()
8525 };
8526
8527 let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8528 let account_id = AccountId::from("TEST-001");
8529 let client_id = ClientId::from("POSITION-FILLS");
8530 let client_order_id = ClientOrderId::from("O-ALLOW");
8531 let venue_order_id = VenueOrderId::from("V-ALLOW");
8532 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8533 node.kernel
8534 .cache
8535 .borrow_mut()
8536 .add_instrument(instrument.clone())
8537 .unwrap();
8538 insert_accepted_limit_order_in_node(
8539 &node,
8540 account_id,
8541 client_id,
8542 instrument.id(),
8543 client_order_id,
8544 venue_order_id,
8545 );
8546 let client = StubExecutionClient::new(
8547 client_id,
8548 account_id,
8549 instrument.id().venue,
8550 OmsType::Netting,
8551 None,
8552 )
8553 .with_allows_reconciliation_overfill();
8554 node.kernel
8555 .exec_engine
8556 .borrow_mut()
8557 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8558 .unwrap();
8559
8560 let fill = FillReport::new(
8561 account_id,
8562 instrument.id(),
8563 venue_order_id,
8564 TradeId::from("T-ALLOW"),
8565 OrderSide::Buy,
8566 Quantity::from("12.0"),
8567 Price::from("100.0"),
8568 Money::from("1.00 USDT"),
8569 LiquiditySide::Taker,
8570 Some(client_order_id),
8571 None,
8572 UnixNanos::from(1_000),
8573 UnixNanos::from(1_000),
8574 None,
8575 );
8576 node.kernel
8577 .exec_engine
8578 .borrow_mut()
8579 .reconcile_fill_report(&fill);
8580
8581 let order = node
8582 .kernel
8583 .cache
8584 .borrow()
8585 .order_owned(&client_order_id)
8586 .unwrap();
8587 assert_eq!(order.quantity(), Quantity::from("10.0"));
8588 assert_eq!(order.filled_qty(), Quantity::from("12.0"));
8589 assert_eq!(order.trade_ids().len(), 1);
8590 }
8591
8592 fn position_fill_test_fixture(
8593 name: &str,
8594 authoritative_qty: Quantity,
8595 ) -> (LiveNode, PositionStatusReport, FillReport) {
8596 position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
8597 }
8598
8599 fn position_fill_test_fixture_with_threshold(
8600 name: &str,
8601 authoritative_qty: Quantity,
8602 position_check_threshold_ms: u32,
8603 ) -> (LiveNode, PositionStatusReport, FillReport) {
8604 let config = LiveNodeConfig {
8605 exec_engine: crate::config::LiveExecutionEngineConfig {
8606 reconciliation: true,
8607 position_check_threshold_ms,
8608 ..Default::default()
8609 },
8610 ..Default::default()
8611 };
8612
8613 let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8614 let account_id = AccountId::from("TEST-001");
8615 let client_id = ClientId::from("POSITION-FILLS");
8616 let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
8617 let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
8618 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8619 let account = AccountAny::Margin(MarginAccount::new(
8620 AccountState::new(
8621 account_id,
8622 AccountType::Margin,
8623 vec![AccountBalance::new(
8624 Money::from("1000000 USDT"),
8625 Money::from("0 USDT"),
8626 Money::from("1000000 USDT"),
8627 )],
8628 Vec::new(),
8629 true,
8630 UUID4::new(),
8631 UnixNanos::default(),
8632 UnixNanos::default(),
8633 Some(Currency::USDT()),
8634 ),
8635 true,
8636 ));
8637 node.kernel.cache.borrow_mut().add_account(account).unwrap();
8638 node.kernel
8639 .cache
8640 .borrow_mut()
8641 .add_instrument(instrument.clone())
8642 .unwrap();
8643 insert_accepted_limit_order_in_node(
8644 &node,
8645 account_id,
8646 client_id,
8647 instrument.id(),
8648 client_order_id,
8649 venue_order_id,
8650 );
8651 let order = node
8652 .kernel
8653 .cache
8654 .borrow()
8655 .order_owned(&client_order_id)
8656 .unwrap();
8657 let mut initial_fill = TestOrderEventStubs::filled(
8658 &order,
8659 &instrument,
8660 Some(TradeId::from("T-POSITION-INITIAL")),
8661 None,
8662 Some(Price::from("100.0")),
8663 Some(Quantity::from("1.0")),
8664 Some(LiquiditySide::Taker),
8665 None,
8666 None,
8667 Some(account_id),
8668 );
8669
8670 let OrderEventAny::Filled(fill) = &mut initial_fill else {
8671 unreachable!();
8672 };
8673
8674 fill.commission = Some(Money::zero(instrument.quote_currency()));
8675 node.process_reconciliation_events(&[initial_fill]);
8676
8677 let ts_event = UnixNanos::from(1_000);
8678
8679 let fill_report = FillReport::new(
8680 account_id,
8681 instrument.id(),
8682 venue_order_id,
8683 TradeId::from("T-POSITION-AUTHORITATIVE"),
8684 OrderSide::Buy,
8685 authoritative_qty,
8686 Price::from("100.0"),
8687 Money::zero(instrument.quote_currency()),
8688 LiquiditySide::Taker,
8689 Some(client_order_id),
8690 None,
8691 ts_event,
8692 ts_event,
8693 None,
8694 );
8695
8696 let venue_report = PositionStatusReport::new(
8697 account_id,
8698 instrument.id(),
8699 PositionSide::Long,
8700 Quantity::from("2.0"),
8701 ts_event,
8702 ts_event,
8703 None,
8704 None,
8705 Some(dec!(100.0)),
8706 );
8707
8708 (node, venue_report, fill_report)
8709 }
8710
8711 fn position_report_result(
8712 node: &LiveNode,
8713 report: PositionStatusReport,
8714 ) -> PositionReportResult {
8715 let client_id = ClientId::from("POSITION-FILLS");
8716 let key = (report.instrument_id, report.account_id);
8717 let mut check = node
8718 .exec_manager
8719 .prepare_position_report_check(UUID4::new(), &[]);
8720 check.client_coverage.insert(
8721 key,
8722 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
8723 );
8724
8725 PositionReportResult {
8726 check,
8727 reports: vec![report],
8728 queried_clients: IndexSet::from([client_id]),
8729 failed_clients: IndexSet::new(),
8730 }
8731 }
8732
8733 fn position_quantity_after_check(
8734 node: &mut LiveNode,
8735 venue_report: &PositionStatusReport,
8736 reports: &[FillReport],
8737 ) -> Quantity {
8738 let key = (venue_report.instrument_id, venue_report.account_id);
8739 let position_result = position_report_result(node, venue_report.clone());
8740 node.handle_position_fill_report_result(PositionFillReportResult {
8741 position_result,
8742 reports: IndexMap::from([(key, reports.to_vec())]),
8743 });
8744
8745 node.kernel
8746 .cache
8747 .borrow()
8748 .positions_open(None, Some(&key.0), None, Some(&key.1), None)
8749 .iter()
8750 .map(|position| position.quantity)
8751 .sum()
8752 }
8753
8754 fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
8755 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
8756 fill.account_id,
8757 fill.instrument_id,
8758 fill.venue_order_id,
8759 fill.trade_id,
8760 fill.order_side,
8761 fill.last_qty,
8762 fill.last_px,
8763 fill.commission
8764 .unwrap_or_else(|| Money::zero(fill.currency)),
8765 fill.liquidity_side,
8766 Some(fill.client_order_id),
8767 fill.position_id,
8768 fill.ts_event,
8769 fill.ts_init,
8770 None,
8771 ))))
8772 }
8773
8774 fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
8775 node.exec_manager.is_fill_recently_processed(
8776 fill.account_id,
8777 fill.instrument_id,
8778 fill.trade_id,
8779 )
8780 }
8781
8782 #[rstest]
8783 #[case(0, NodeState::Idle)]
8784 #[case(1, NodeState::Starting)]
8785 #[case(2, NodeState::Running)]
8786 #[case(3, NodeState::ShuttingDown)]
8787 #[case(4, NodeState::Stopped)]
8788 fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
8789 assert_eq!(NodeState::from_u8(value), expected);
8790 }
8791
8792 #[rstest]
8793 #[case(5)]
8794 #[case(255)]
8795 #[should_panic(expected = "Invalid NodeState value")]
8796 fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
8797 let _ = NodeState::from_u8(value);
8798 }
8799
8800 #[rstest]
8801 fn test_node_state_roundtrip() {
8802 for state in [
8803 NodeState::Idle,
8804 NodeState::Starting,
8805 NodeState::Running,
8806 NodeState::ShuttingDown,
8807 NodeState::Stopped,
8808 ] {
8809 assert_eq!(NodeState::from_u8(state.as_u8()), state);
8810 }
8811 }
8812
8813 #[rstest]
8814 fn test_node_state_is_running_only_for_running() {
8815 assert!(!NodeState::Idle.is_running());
8816 assert!(!NodeState::Starting.is_running());
8817 assert!(NodeState::Running.is_running());
8818 assert!(!NodeState::ShuttingDown.is_running());
8819 assert!(!NodeState::Stopped.is_running());
8820 }
8821
8822 #[rstest]
8823 #[tokio::test]
8824 async fn test_await_engines_connected_returns_stop_requested() {
8825 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8826 let handle = node.handle();
8827
8828 handle.stop();
8829
8830 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8831 let status = node.await_engines_connected(deadline).await;
8832
8833 assert_eq!(status, EngineConnectionStatus::StopRequested);
8834 assert!(handle.should_stop());
8835 }
8836
8837 #[rstest]
8838 #[tokio::test]
8839 async fn test_await_engines_connected_returns_shutdown_requested() {
8840 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8841
8842 node.kernel().shutdown_flag().set(true);
8843
8844 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8845 let status = node.await_engines_connected(deadline).await;
8846
8847 assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
8848 }
8849
8850 #[rstest]
8851 #[tokio::test]
8852 async fn test_start_stop_request_aborts_startup_without_running() {
8853 let config = LiveNodeConfig {
8854 exec_engine: crate::config::LiveExecutionEngineConfig {
8855 reconciliation: false,
8856 ..Default::default()
8857 },
8858 timeout_disconnection: Duration::from_millis(50),
8859 ..Default::default()
8860 };
8861
8862 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8863 let handle = node.handle();
8864
8865 handle.stop();
8866 node.start().await.unwrap();
8867
8868 assert_eq!(handle.state(), NodeState::Stopped);
8869 assert!(handle.should_stop());
8870 assert!(!handle.is_running());
8871 }
8872
8873 #[rstest]
8874 #[tokio::test(start_paused = true)]
8875 async fn test_stop_processes_residual_exec_event_during_grace_period() {
8876 let config = LiveNodeConfig {
8877 exec_engine: crate::config::LiveExecutionEngineConfig {
8878 reconciliation: false,
8879 ..Default::default()
8880 },
8881 timeout_connection: Duration::ZERO,
8882 timeout_reconciliation: Duration::ZERO,
8883 timeout_portfolio: Duration::ZERO,
8884 timeout_disconnection: Duration::ZERO,
8885 delay_post_stop: Duration::from_millis(20),
8886 timeout_shutdown: Duration::ZERO,
8887 ..Default::default()
8888 };
8889
8890 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8891 let order = OrderTestBuilder::new(OrderType::Market)
8892 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
8893 .quantity(Quantity::from("2"))
8894 .build();
8895 let client_order_id = order.client_order_id();
8896 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
8897
8898 node.kernel
8899 .cache()
8900 .borrow_mut()
8901 .add_order(order, None, None, false)
8902 .unwrap();
8903
8904 node.start().await.unwrap();
8905 let exec_event_sender = get_exec_event_sender();
8906
8907 let send_residual = tokio::spawn(async move {
8908 tokio::time::sleep(Duration::from_millis(1)).await;
8909 exec_event_sender
8910 .send(ExecutionEvent::Order(submitted))
8911 .unwrap();
8912 });
8913
8914 node.stop().await.unwrap();
8915 send_residual.await.unwrap();
8916
8917 assert_eq!(
8918 node.kernel
8919 .cache()
8920 .borrow()
8921 .order(&client_order_id)
8922 .unwrap()
8923 .status(),
8924 OrderStatus::Submitted
8925 );
8926
8927 node.dispose();
8928 }
8929
8930 #[rstest]
8931 #[tokio::test]
8932 async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
8933 let actor_id = ActorId::from("LIVE-STATE-ACTOR");
8934 let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
8935 let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
8936 let strategy_load =
8937 IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
8938 let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
8939 let strategy_save =
8940 IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
8941 let (database, control) = TestCacheDatabaseControl::create();
8942 control.set_actor_state(actor_id, &actor_load);
8943 control.set_strategy_state(strategy_id, &strategy_load);
8944
8945 let config = LiveNodeConfig {
8946 load_state: true,
8947 save_state: true,
8948 exec_engine: crate::config::LiveExecutionEngineConfig {
8949 reconciliation: false,
8950 ..Default::default()
8951 },
8952 timeout_connection: Duration::ZERO,
8953 timeout_reconciliation: Duration::ZERO,
8954 timeout_portfolio: Duration::ZERO,
8955 timeout_disconnection: Duration::ZERO,
8956 delay_post_stop: Duration::ZERO,
8957 timeout_shutdown: Duration::ZERO,
8958 ..Default::default()
8959 };
8960
8961 let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
8962 node.set_cache_database(Box::new(database)).unwrap();
8963 node.add_actor(StateActor::new(
8964 actor_id,
8965 control.clone(),
8966 actor_save.clone(),
8967 ))
8968 .unwrap();
8969 node.add_strategy(StateStrategy::new(
8970 strategy_id,
8971 control.clone(),
8972 strategy_save.clone(),
8973 ))
8974 .unwrap();
8975
8976 node.start().await.unwrap();
8977 node.stop().await.unwrap();
8978 node.dispose();
8979
8980 assert_eq!(
8981 control.events(),
8982 vec![
8983 "actor.load:LIVE-STATE-ACTOR",
8984 "actor.on_load",
8985 "strategy.load:LIVE-STATE-STRATEGY-001",
8986 "strategy.on_load",
8987 "actor.on_start",
8988 "strategy.on_start",
8989 "actor.on_stop",
8990 "strategy.on_stop",
8991 "actor.on_save",
8992 "actor.update:LIVE-STATE-ACTOR",
8993 "strategy.on_save",
8994 "strategy.update:LIVE-STATE-STRATEGY-001",
8995 "database.close",
8996 ]
8997 );
8998 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
8999 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9000 assert_eq!(node.state(), NodeState::Stopped);
9001 }
9002
9003 #[rstest]
9004 #[tokio::test]
9005 async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9006 let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9007 let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9008 let (database, control) = TestCacheDatabaseControl::create();
9009
9010 let config = LiveNodeConfig {
9011 save_state: true,
9012 exec_engine: crate::config::LiveExecutionEngineConfig {
9013 reconciliation: false,
9014 ..Default::default()
9015 },
9016 timeout_connection: Duration::ZERO,
9017 timeout_reconciliation: Duration::ZERO,
9018 timeout_portfolio: Duration::ZERO,
9019 timeout_disconnection: Duration::ZERO,
9020 delay_post_stop: Duration::ZERO,
9021 timeout_shutdown: Duration::ZERO,
9022 ..Default::default()
9023 };
9024
9025 let mut node =
9026 LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9027 node.set_cache_database(Box::new(database)).unwrap();
9028 node.add_actor(
9029 StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9030 )
9031 .unwrap();
9032 node.add_strategy(
9033 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9034 )
9035 .unwrap();
9036
9037 node.start().await.unwrap();
9038 let error = node.stop().await.unwrap_err();
9039 node.dispose();
9040
9041 assert_eq!(
9042 error.to_string(),
9043 "failed while finalizing kernel shutdown: Failed to save component state: actor \
9044 LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9045 LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9046 );
9047 assert_eq!(
9048 control.events(),
9049 vec![
9050 "actor.on_start",
9051 "strategy.on_start",
9052 "actor.on_stop",
9053 "strategy.on_stop",
9054 "actor.on_save",
9055 "strategy.on_save",
9056 "database.close",
9057 ]
9058 );
9059 assert_eq!(node.state(), NodeState::Stopped);
9060 }
9061
9062 #[rstest]
9063 #[tokio::test]
9064 async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9065 let config = LiveNodeConfig {
9066 exec_engine: crate::config::LiveExecutionEngineConfig {
9067 reconciliation: false,
9068 ..Default::default()
9069 },
9070 timeout_connection: Duration::ZERO,
9071 timeout_reconciliation: Duration::ZERO,
9072 timeout_portfolio: Duration::ZERO,
9073 timeout_disconnection: Duration::ZERO,
9074 delay_post_stop: Duration::ZERO,
9075 timeout_shutdown: Duration::ZERO,
9076 ..Default::default()
9077 };
9078
9079 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9080 let order = OrderTestBuilder::new(OrderType::Market)
9081 .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9082 .quantity(Quantity::from("3"))
9083 .build();
9084 let client_order_id = order.client_order_id();
9085 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9086
9087 node.kernel
9088 .cache()
9089 .borrow_mut()
9090 .add_order(order, None, None, false)
9091 .unwrap();
9092
9093 node.start().await.unwrap();
9094 get_exec_event_sender()
9095 .send(ExecutionEvent::Order(submitted))
9096 .unwrap();
9097
9098 node.stop().await.unwrap();
9099
9100 assert_eq!(
9101 node.kernel
9102 .cache()
9103 .borrow()
9104 .order(&client_order_id)
9105 .unwrap()
9106 .status(),
9107 OrderStatus::Submitted
9108 );
9109
9110 node.dispose();
9111 }
9112
9113 #[rstest]
9114 #[tokio::test]
9115 async fn test_start_event_store_replay_skips_live_connections() {
9116 let mut node = live_node_with_replay_store(false);
9117 let handle = node.handle();
9118
9119 node.start().await.unwrap();
9120
9121 assert_eq!(handle.state(), NodeState::Running);
9122 assert!(handle.is_running());
9123 assert!(node.kernel.is_event_store_replay());
9124 assert!(node.runner.is_some());
9125 }
9126
9127 #[rstest]
9128 #[tokio::test]
9129 async fn test_start_event_store_replay_preserves_stop_request() {
9130 let mut node = live_node_with_replay_store(false);
9131 let handle = node.handle();
9132 handle.stop();
9133
9134 node.start().await.unwrap();
9135
9136 assert_eq!(handle.state(), NodeState::Stopped);
9137 assert!(handle.should_stop());
9138 assert!(node.kernel.is_event_store_replay());
9139 assert!(node.runner.is_some());
9140 }
9141
9142 #[rstest]
9143 #[tokio::test]
9144 async fn test_start_event_store_replay_config_failure_aborts_startup() {
9145 let mut node = live_node_with_replay_store(true);
9146 let handle = node.handle();
9147
9148 node.start().await.unwrap();
9149
9150 assert_eq!(handle.state(), NodeState::Stopped);
9151 assert!(!handle.is_running());
9152 assert!(node.kernel.is_event_store_replay_configured());
9153 assert!(!node.kernel.is_event_store_replay());
9154 assert!(node.runner.is_some());
9155 }
9156
9157 #[rstest]
9158 #[tokio::test]
9159 async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9160 let mut node = live_node_with_replay_store(false);
9161 let handle = node.handle();
9162
9163 node.run().await.unwrap();
9164
9165 assert_eq!(handle.state(), NodeState::Running);
9166 assert!(handle.is_running());
9167 assert!(node.kernel.is_event_store_replay());
9168 assert!(node.runner.is_none());
9169 }
9170
9171 #[rstest]
9172 #[tokio::test]
9173 async fn test_run_event_store_replay_preserves_stop_request() {
9174 let mut node = live_node_with_replay_store(false);
9175 let handle = node.handle();
9176 handle.stop();
9177
9178 node.run().await.unwrap();
9179
9180 assert_eq!(handle.state(), NodeState::Stopped);
9181 assert!(handle.should_stop());
9182 assert!(node.kernel.is_event_store_replay());
9183 assert!(node.runner.is_none());
9184 }
9185
9186 #[rstest]
9187 #[tokio::test]
9188 async fn test_run_event_store_replay_config_failure_aborts_startup() {
9189 let mut node = live_node_with_replay_store(true);
9190 let handle = node.handle();
9191
9192 node.run().await.unwrap();
9193
9194 assert_eq!(handle.state(), NodeState::Stopped);
9195 assert!(!handle.is_running());
9196 assert!(node.kernel.is_event_store_replay_configured());
9197 assert!(!node.kernel.is_event_store_replay());
9198 assert!(node.runner.is_none());
9199 }
9200
9201 #[rstest]
9202 fn test_build_rejects_event_store_config_without_factory() {
9203 let config = LiveNodeConfig {
9204 event_store: Some(EventStoreConfig::default()),
9205 exec_engine: crate::config::LiveExecutionEngineConfig {
9206 reconciliation: false,
9207 ..Default::default()
9208 },
9209 ..Default::default()
9210 };
9211
9212 let err = LiveNodeBuilder::from_config(config)
9213 .expect("builder")
9214 .build()
9215 .expect_err("should reject event_store config without factory");
9216
9217 assert!(
9218 err.to_string().contains("with_event_store"),
9219 "error message should mention with_event_store, was: {err}"
9220 );
9221 }
9222
9223 #[rstest]
9224 fn test_direct_build_rejects_event_store_config() {
9225 let config = LiveNodeConfig {
9226 event_store: Some(EventStoreConfig::default()),
9227 exec_engine: crate::config::LiveExecutionEngineConfig {
9228 reconciliation: false,
9229 ..Default::default()
9230 },
9231 ..Default::default()
9232 };
9233
9234 let err = LiveNode::build("TestNode".to_string(), Some(config))
9235 .expect_err("LiveNode::build should reject event_store config");
9236
9237 assert!(
9238 err.to_string().contains("with_event_store"),
9239 "error message should mention with_event_store, was: {err}"
9240 );
9241 }
9242
9243 #[rstest]
9244 fn test_dispose_before_start_is_idempotent() {
9245 let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9246 node.add_strategy(TestStrategy::new(StrategyConfig {
9247 strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9248 ..Default::default()
9249 }))
9250 .unwrap();
9251
9252 node.dispose();
9253 node.dispose();
9254
9255 assert!(node.kernel.trader().borrow().is_disposed());
9256 assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9257 assert_eq!(node.state(), NodeState::Stopped);
9258 }
9259
9260 #[rstest]
9261 fn test_dispose_releases_retained_callback_roots(
9262 #[values(false, true)] fatal: bool,
9263 #[values(false, true)] external: bool,
9264 ) {
9265 actor::clear_callbacks().unwrap();
9266 let directory =
9267 std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9268
9269 let config = LiveNodeConfig {
9270 logging: LoggerConfig {
9271 fileout_level: LevelFilter::Info,
9272 file_config: Some(FileWriterConfig {
9273 directory: Some(directory.to_str().unwrap().to_string()),
9274 file_name: Some("disposal".to_string()),
9275 ..Default::default()
9276 }),
9277 ..Default::default()
9278 },
9279 ..Default::default()
9280 };
9281
9282 let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9283 let received = Rc::new(RefCell::new(Vec::new()));
9284 let observed = received.clone();
9285
9286 let retained = DispatchMessage::from(()).dispatch(|()| {
9287 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9288 TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9289 TimeEventCallback::RustLocal(Rc::new(move |_| {
9290 observed.borrow_mut().push("delivered");
9291 })),
9292 ));
9293
9294 external.then(|| DispatchMessage::new((), std::thread::current().id()))
9295 });
9296
9297 if let Some(retained) = &retained {
9298 assert!(retained.is_rooted());
9299 }
9300
9301 assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9302
9303 if fatal {
9304 crate::dispatch::tests::latch_callback_failure();
9305 }
9306
9307 node.dispose();
9308
9309 assert!(node.runner.is_none());
9310 assert!(received.borrow().is_empty());
9311 assert_eq!(Rc::strong_count(&received), 1);
9312 assert_eq!(
9313 actor::callback_failure(),
9314 (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9315 );
9316 assert_eq!(
9317 actor::clear_callbacks(),
9318 if external {
9319 Err(CallbackDispatchError::Active)
9320 } else {
9321 Ok(())
9322 }
9323 );
9324
9325 logging_sync_to_disk().unwrap();
9326 let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9327 drop(retained);
9328 node.dispose();
9329
9330 assert_eq!(actor::callback_failure(), None);
9331 assert_eq!(actor::clear_callbacks(), Ok(()));
9332 drop(node);
9333 std::fs::remove_dir_all(directory).unwrap();
9334
9335 let errors: Vec<_> = output
9336 .lines()
9337 .filter(|line| line.contains("[ERROR]"))
9338 .filter_map(|line| {
9339 line.split_once(".nautilus_live::node: ")
9340 .map(|(_, text)| text)
9341 })
9342 .collect();
9343
9344 let mut expected = Vec::new();
9345
9346 if fatal {
9347 expected.push(
9348 "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9349 );
9350 }
9351
9352 if external {
9353 expected.push(
9354 "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9355 );
9356 }
9357
9358 assert_eq!(errors, expected);
9359 }
9360
9361 #[tokio::test]
9362 async fn test_dispose_releases_stop_generated_callback_roots() {
9363 actor::clear_callbacks().unwrap();
9364
9365 let config = LiveNodeConfig {
9366 exec_engine: crate::config::LiveExecutionEngineConfig {
9367 reconciliation: false,
9368 ..Default::default()
9369 },
9370 timeout_connection: Duration::ZERO,
9371 timeout_reconciliation: Duration::ZERO,
9372 timeout_portfolio: Duration::ZERO,
9373 timeout_shutdown: Duration::ZERO,
9374 ..Default::default()
9375 };
9376
9377 let mut node =
9378 LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9379 let received = Rc::new(RefCell::new(Vec::new()));
9380 node.add_actor(StopCallbackActor {
9381 core: DataActorCore::new(DataActorConfig {
9382 actor_id: Some(ActorId::from("STOP-CALLBACK")),
9383 ..Default::default()
9384 }),
9385 received: received.clone(),
9386 })
9387 .unwrap();
9388
9389 node.start().await.unwrap();
9390
9391 assert!(node.kernel.trader().borrow().is_running());
9392 assert_eq!(actor::clear_callbacks(), Ok(()));
9393
9394 node.dispose();
9395
9396 assert_eq!(*received.borrow(), ["stop", "queued"]);
9397 assert_eq!(Rc::strong_count(&received), 1);
9398 assert!(node.runner.is_none());
9399 assert!(node.kernel.trader().borrow().is_disposed());
9400 assert_eq!(node.state(), NodeState::Stopped);
9401 assert_eq!(actor::clear_callbacks(), Ok(()));
9402 }
9403
9404 #[derive(Debug)]
9405 struct StopCallbackActor {
9406 core: DataActorCore,
9407 received: Rc<RefCell<Vec<&'static str>>>,
9408 }
9409
9410 nautilus_actor!(StopCallbackActor);
9411
9412 impl DataActor for StopCallbackActor {
9413 fn on_stop(&mut self) -> anyhow::Result<()> {
9414 self.received.borrow_mut().push("stop");
9415 let received = self.received.clone();
9416 DispatchMessage::from(()).dispatch(|()| {
9417 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9418 TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9419 TimeEventCallback::RustLocal(Rc::new(move |_| {
9420 received.borrow_mut().push("delivered");
9421 })),
9422 ));
9423 });
9424
9425 let clear_result = actor::clear_callbacks();
9426 anyhow::ensure!(
9427 clear_result == Err(CallbackDispatchError::Active),
9428 "Expected active callback roots during stop, received {clear_result:?}"
9429 );
9430 self.received.borrow_mut().push("queued");
9431 Ok(())
9432 }
9433 }
9434
9435 #[rstest]
9436 fn test_handle_initial_state() {
9437 let handle = LiveNodeHandle::new();
9438
9439 assert_eq!(handle.state(), NodeState::Idle);
9440 assert!(!handle.should_stop());
9441 assert!(!handle.is_running());
9442 }
9443
9444 #[rstest]
9445 fn test_handle_initial_metrics_snapshot_is_zero() {
9446 let handle = LiveNodeHandle::new();
9447
9448 assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9449 }
9450
9451 #[rstest]
9452 fn test_record_runner_dispatch_updates_selected_channel() {
9453 let metrics = RunnerMetrics::default();
9454 let dispatch_start = dst::time::Instant::now();
9455 let metrics_start = dispatch_start
9456 .checked_sub(Duration::from_micros(1))
9457 .expect("test instant should support a one-microsecond lookback");
9458
9459 record_runner_dispatch(
9460 &metrics,
9461 SystemChannel::DataCommands,
9462 dispatch_start,
9463 metrics_start,
9464 );
9465 let snapshot = metrics.snapshot();
9466
9467 assert_eq!(snapshot.time_events.dispatched, 0);
9468 assert_eq!(snapshot.exec_events.dispatched, 0);
9469 assert_eq!(snapshot.exec_commands.dispatched, 0);
9470 assert_eq!(snapshot.data_events.dispatched, 0);
9471 assert_eq!(snapshot.data_commands.dispatched, 1);
9472 assert_eq!(
9473 snapshot.data_commands.last_dispatch_at_ns,
9474 snapshot.elapsed_ns
9475 );
9476 assert_eq!(
9477 snapshot.data_commands.dispatch_busy_ns,
9478 snapshot.dispatch_busy_ns
9479 );
9480 assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9481 }
9482
9483 #[rstest]
9484 fn test_handle_stop_sets_flag() {
9485 let handle = LiveNodeHandle::new();
9486
9487 handle.stop();
9488
9489 assert!(handle.should_stop());
9490 }
9491
9492 #[rstest]
9493 fn test_handle_stop_blocks_running_transition() {
9494 let handle = LiveNodeHandle::new();
9495 handle.set_starting();
9496 handle.stop();
9497
9498 let transition = handle.try_set_running();
9499
9500 assert_eq!(transition, RunningTransition::StopRequested);
9501 assert_eq!(handle.state(), NodeState::Starting);
9502 assert!(handle.should_stop());
9503 assert!(!handle.is_running());
9504 }
9505
9506 #[rstest]
9507 fn test_handle_stop_after_running_transition_remains_pending() {
9508 let handle = LiveNodeHandle::new();
9509 handle.set_starting();
9510
9511 let transition = handle.try_set_running();
9512 handle.stop();
9513
9514 assert_eq!(transition, RunningTransition::Entered);
9515 assert_eq!(handle.state(), NodeState::Running);
9516 assert!(handle.should_stop());
9517 assert!(handle.is_running());
9518 }
9519
9520 #[rstest]
9521 fn test_handle_node_state_transitions() {
9522 let handle = LiveNodeHandle::new();
9523 assert_eq!(handle.state(), NodeState::Idle);
9524
9525 handle.set_starting();
9526 assert_eq!(handle.state(), NodeState::Starting);
9527 assert!(!handle.is_running());
9528
9529 assert_eq!(handle.try_set_running(), RunningTransition::Entered);
9530 assert_eq!(handle.state(), NodeState::Running);
9531 assert!(handle.is_running());
9532
9533 handle.set_shutting_down();
9534 assert_eq!(handle.state(), NodeState::ShuttingDown);
9535 assert!(!handle.is_running());
9536
9537 handle.set_stopped();
9538 assert_eq!(handle.state(), NodeState::Stopped);
9539 assert!(!handle.is_running());
9540 }
9541
9542 #[rstest]
9543 fn test_handle_clone_shares_state_bidirectionally() {
9544 let handle1 = LiveNodeHandle::new();
9545 let handle2 = handle1.clone();
9546
9547 handle1.set_starting();
9548 let transition = handle2.try_set_running();
9549 handle1.stop();
9550
9551 assert_eq!(transition, RunningTransition::Entered);
9552 assert_eq!(handle1.state(), NodeState::Running);
9553 assert!(handle2.should_stop());
9554 }
9555
9556 #[rstest]
9557 fn test_handle_stop_flag_survives_non_running_state_changes() {
9558 let handle = LiveNodeHandle::new();
9559
9560 handle.set_starting();
9561 handle.stop();
9562 handle.set_shutting_down();
9563 handle.set_stopped();
9564
9565 assert_eq!(handle.state(), NodeState::Stopped);
9566 assert!(handle.should_stop());
9567 }
9568
9569 #[rstest]
9570 fn test_builder_creation() {
9571 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9572
9573 assert!(result.is_ok());
9574 }
9575
9576 #[rstest]
9577 fn test_stream_processor_receives_unregistered_typed_payload() {
9578 let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9579 .unwrap()
9580 .build()
9581 .unwrap();
9582 let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
9583 InstrumentId::from("AUD/USD.SIM"),
9584 Some(ClientId::from("EXTERNAL")),
9585 Some(Venue::from("SIM")),
9586 UUID4::from("00000000-0000-4000-8000-000000000001"),
9587 UnixNanos::from(1),
9588 None,
9589 None,
9590 ));
9591 let expected = serde_json::to_value(&command).unwrap();
9592 let received = Rc::new(RefCell::new(Vec::new()));
9593 let steps = Rc::new(RefCell::new(Vec::new()));
9594 let received_processor = received.clone();
9595 let first_steps = steps.clone();
9596 node.add_stream_processor(move |message| {
9597 let command = message
9598 .downcast_ref::<SubscribeCommand>()
9599 .expect("processor must receive the decoded concrete command");
9600 received_processor
9601 .borrow_mut()
9602 .push(serde_json::to_value(command).unwrap());
9603 first_steps.borrow_mut().push(1);
9604 });
9605
9606 let second_steps = steps.clone();
9607 node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
9608 let republished = Rc::new(RefCell::new(Vec::new()));
9609 let republished_handler = republished.clone();
9610 let subscriber_steps = steps.clone();
9611 msgbus::subscribe_any(
9612 "external.test".into(),
9613 ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
9614 republished_handler
9615 .borrow_mut()
9616 .push(serde_json::to_value(command).unwrap());
9617 subscriber_steps.borrow_mut().push(3);
9618 }),
9619 None,
9620 );
9621
9622 let message = BusMessage::with_str_topic(
9623 "external.test",
9624 BusPayloadType::SubscribeCommand,
9625 Bytes::from(serde_json::to_vec(&command).unwrap()),
9626 SerializationEncoding::Json,
9627 );
9628
9629 assert!(
9630 !msgbus::get_message_bus()
9631 .borrow()
9632 .is_streaming_type(BusPayloadType::SubscribeCommand)
9633 );
9634 node.process_external_msgbus_message(&message);
9635
9636 assert_eq!(*received.borrow(), vec![expected.clone()]);
9637 assert_eq!(*steps.borrow(), vec![1, 2]);
9638 assert!(republished.borrow().is_empty());
9639
9640 received.borrow_mut().clear();
9641 steps.borrow_mut().clear();
9642 msgbus::get_message_bus()
9643 .borrow_mut()
9644 .add_streaming_type(BusPayloadType::SubscribeCommand);
9645 node.process_external_msgbus_message(&message);
9646
9647 assert_eq!(*received.borrow(), vec![expected.clone()]);
9648 assert_eq!(*republished.borrow(), vec![expected]);
9649 assert_eq!(*steps.borrow(), vec![1, 2, 3]);
9650 msgbus::get_message_bus().borrow_mut().dispose();
9651 }
9652
9653 #[rstest]
9654 fn test_builder_rejects_backtest() {
9655 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
9656
9657 assert!(result.is_err());
9658 assert!(result.unwrap_err().to_string().contains("Backtest"));
9659 }
9660
9661 #[rstest]
9662 fn test_builder_accepts_live_environment() {
9663 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
9664
9665 assert!(result.is_ok());
9666 }
9667
9668 #[rstest]
9669 fn test_builder_accepts_sandbox_environment() {
9670 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9671
9672 assert!(result.is_ok());
9673 }
9674
9675 #[rstest]
9676 fn test_builder_fluent_api_chaining() {
9677 let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
9678 .unwrap()
9679 .with_name("TestNode")
9680 .with_instance_id(UUID4::new())
9681 .with_load_state(false)
9682 .with_save_state(true)
9683 .with_timeout_connection(30)
9684 .with_timeout_reconciliation(60)
9685 .with_reconciliation(true)
9686 .with_reconciliation_lookback_mins(120)
9687 .with_timeout_portfolio(10)
9688 .with_timeout_disconnection_secs(5)
9689 .with_delay_post_stop_secs(3)
9690 .with_delay_shutdown_secs(10);
9691
9692 assert_eq!(builder.name(), "TestNode");
9693 }
9694
9695 #[rstest]
9696 fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
9697 let (external_egress, publications, closed) = CapturingExternalEgress::new();
9698
9699 let msgbus_config = MessageBusConfig {
9700 encoding: SerializationEncoding::Json,
9701 ..Default::default()
9702 };
9703
9704 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9705 .unwrap()
9706 .with_msgbus_config(msgbus_config)
9707 .with_external_msgbus_egress(Box::new(external_egress))
9708 .build()
9709 .expect("node builds with external message bus egress");
9710 let quote = QuoteTick::default();
9711
9712 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9713
9714 let publications = publications.borrow();
9715 assert_eq!(publications.len(), 1);
9716 assert_eq!(publications[0].topic, "data.quotes.TEST");
9717 assert_eq!(
9718 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9719 .expect("JSON payload must decode as QuoteTick"),
9720 quote
9721 );
9722 drop(publications);
9723
9724 msgbus::get_message_bus().borrow_mut().dispose();
9725 assert!(closed.get());
9726 drop(node);
9727 }
9728
9729 #[rstest]
9730 #[tokio::test(flavor = "current_thread")]
9731 async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
9732 let quote = QuoteTick::default();
9733 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9734 let publications = Arc::new(Mutex::new(Vec::new()));
9735 let closed = Arc::new(AtomicBool::new(false));
9736 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
9737
9738 let msgbus_config = MessageBusConfig {
9739 external_streams: Some(vec!["stream".to_string()]),
9740 ..Default::default()
9741 };
9742
9743 let config = LiveNodeConfig {
9744 environment: Environment::Sandbox,
9745 msgbus: Some(msgbus_config),
9746 exec_engine: crate::config::LiveExecutionEngineConfig {
9747 reconciliation: false,
9748 ..Default::default()
9749 },
9750 delay_post_stop: Duration::ZERO,
9751 timeout_connection: Duration::from_millis(500),
9752 timeout_disconnection: Duration::from_millis(500),
9753 ..Default::default()
9754 };
9755
9756 let mut node = LiveNodeBuilder::from_config(config)
9757 .unwrap()
9758 .with_external_msgbus_factory(Box::new(factory))
9759 .build()
9760 .expect("node builds with external message bus factory");
9761
9762 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9763 {
9764 let publications = publications.lock();
9765 assert_eq!(publications.len(), 1);
9766 assert_eq!(publications[0].topic, "data.quotes.TEST");
9767 assert_eq!(
9768 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9769 .expect("JSON payload must decode as QuoteTick"),
9770 quote
9771 );
9772 }
9773
9774 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9775 let handle = node.handle();
9776
9777 let handler = TypedHandler::from({
9780 let received = received.clone();
9781 move |quote: &QuoteTick| {
9782 received.borrow_mut().push(*quote);
9783 }
9784 });
9785
9786 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9787 msgbus::get_message_bus()
9788 .borrow_mut()
9789 .add_streaming_type(BusPayloadType::QuoteTick);
9790
9791 let payload =
9792 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
9793 let message = BusMessage::with_str_topic(
9794 "data.quotes.TEST",
9795 BusPayloadType::QuoteTick,
9796 payload,
9797 SerializationEncoding::Json,
9798 );
9799
9800 tokio::time::timeout(Duration::from_secs(30), async {
9801 let run = node.run();
9802 tokio::pin!(run);
9803
9804 let drive = async {
9805 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9806
9807 tx.send(message)
9808 .await
9809 .expect("external ingress receiver should be open");
9810
9811 wait_until_async(
9812 || async { received.borrow().len() == 1 },
9813 Duration::from_secs(10),
9814 )
9815 .await;
9816 assert_eq!(*received.borrow(), vec![quote]);
9817 handle.stop();
9818 };
9819
9820 tokio::select! {
9821 biased;
9822
9823 () = drive => {}
9824 result = &mut run => {
9825 panic!("node stopped before factory ingress was republished: {result:?}");
9826 }
9827 }
9828
9829 run.await.expect("node should stop cleanly");
9830 })
9831 .await
9832 .expect("live node should republish factory ingress and stop before timeout");
9833
9834 assert_eq!(handle.state(), NodeState::Stopped);
9835 assert!(closed.load(Ordering::Relaxed));
9836 msgbus::get_message_bus().borrow_mut().dispose();
9837 }
9838
9839 #[rstest]
9840 #[tokio::test(flavor = "current_thread")]
9841 async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
9842 let quote = QuoteTick::default();
9843 let publications = Arc::new(Mutex::new(Vec::new()));
9844 let closed = Arc::new(AtomicBool::new(false));
9845 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
9846
9847 let config = LiveNodeConfig {
9848 environment: Environment::Sandbox,
9849 msgbus: Some(MessageBusConfig::default()),
9850 exec_engine: crate::config::LiveExecutionEngineConfig {
9851 reconciliation: false,
9852 ..Default::default()
9853 },
9854 delay_post_stop: Duration::ZERO,
9855 timeout_connection: Duration::from_millis(500),
9856 timeout_disconnection: Duration::from_millis(500),
9857 ..Default::default()
9858 };
9859
9860 let mut node = LiveNodeBuilder::from_config(config)
9861 .unwrap()
9862 .with_external_msgbus_factory(Box::new(factory))
9863 .build()
9864 .expect("node builds with egress-only message bus factory");
9865 let handle = node.handle();
9866
9867 msgbus::publish_quote("data.quotes.TEST".into(), "e);
9868 {
9869 let publications = publications.lock();
9870 assert_eq!(publications.len(), 1);
9871 assert_eq!(publications[0].topic, "data.quotes.TEST");
9872 }
9873
9874 tokio::time::timeout(Duration::from_secs(30), async {
9875 let run = node.run();
9876 tokio::pin!(run);
9877
9878 let drive = async {
9879 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9880 handle.stop();
9881 };
9882
9883 tokio::select! {
9884 biased;
9885
9886 () = drive => {}
9887 result = &mut run => {
9888 panic!("node stopped before egress-only factory run was observed: {result:?}");
9889 }
9890 }
9891
9892 run.await.expect("node should stop cleanly");
9893 })
9894 .await
9895 .expect("live node should run without external ingress before timeout");
9896
9897 assert_eq!(handle.state(), NodeState::Stopped);
9898 msgbus::get_message_bus().borrow_mut().dispose();
9899 assert!(closed.load(Ordering::Relaxed));
9900 }
9901
9902 #[rstest]
9903 fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
9904 let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
9905
9906 let egress_factory = CapturingBackingFactory::new(
9907 Arc::new(Mutex::new(Vec::new())),
9908 Arc::new(AtomicBool::new(false)),
9909 None,
9910 );
9911 let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9912 .unwrap()
9913 .with_external_msgbus_factory(Box::new(egress_factory))
9914 .with_external_msgbus_egress(Box::new(external_egress))
9915 .build()
9916 .expect_err("builder should reject factory plus injected egress");
9917
9918 assert!(
9919 egress_error
9920 .to_string()
9921 .contains("cannot be combined with injected egress or ingress")
9922 );
9923
9924 let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9925
9926 let ingress_factory = CapturingBackingFactory::new(
9927 Arc::new(Mutex::new(Vec::new())),
9928 Arc::new(AtomicBool::new(false)),
9929 None,
9930 );
9931 let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
9932 let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9933 .unwrap()
9934 .with_external_msgbus_factory(Box::new(ingress_factory))
9935 .with_external_ingress(Box::new(ingress))
9936 .build()
9937 .expect_err("builder should reject factory plus injected ingress");
9938
9939 assert!(
9940 ingress_error
9941 .to_string()
9942 .contains("cannot be combined with injected egress or ingress")
9943 );
9944 }
9945
9946 #[rstest]
9947 #[tokio::test(flavor = "current_thread")]
9948 async fn test_run_republishes_external_ingress_on_local_msgbus() {
9949 let quote = QuoteTick::default();
9950 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9951 let payload =
9952 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
9953 let message = BusMessage::with_str_topic(
9954 "data.quotes.TEST",
9955 BusPayloadType::QuoteTick,
9956 payload,
9957 SerializationEncoding::Json,
9958 );
9959 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9960 let closed = Rc::new(Cell::new(false));
9961 let ingress = CapturingExternalIngress::new(rx, closed.clone());
9962
9963 let config = LiveNodeConfig {
9964 environment: Environment::Sandbox,
9965 exec_engine: crate::config::LiveExecutionEngineConfig {
9966 reconciliation: false,
9967 ..Default::default()
9968 },
9969 delay_post_stop: Duration::ZERO,
9970 timeout_connection: Duration::from_millis(500),
9971 timeout_disconnection: Duration::from_millis(500),
9972 ..Default::default()
9973 };
9974
9975 let mut node = LiveNodeBuilder::from_config(config)
9976 .unwrap()
9977 .with_external_ingress(Box::new(ingress))
9978 .build()
9979 .expect("node builds with external message bus ingress");
9980 let handle = node.handle();
9981
9982 let handler = TypedHandler::from({
9983 let received = received.clone();
9984 move |quote: &QuoteTick| {
9985 received.borrow_mut().push(*quote);
9986 }
9987 });
9988
9989 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9990 msgbus::get_message_bus()
9991 .borrow_mut()
9992 .add_streaming_type(BusPayloadType::QuoteTick);
9993
9994 tokio::time::timeout(Duration::from_secs(30), async {
9995 let run = node.run();
9996 tokio::pin!(run);
9997
9998 let drive = async {
9999 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10000
10001 tx.send(message)
10002 .await
10003 .expect("external ingress receiver should be open");
10004
10005 wait_until_async(
10006 || async { received.borrow().len() == 1 },
10007 Duration::from_secs(10),
10008 )
10009 .await;
10010 assert_eq!(*received.borrow(), vec![quote]);
10011 handle.stop();
10012 };
10013
10014 tokio::select! {
10015 biased;
10016
10017 () = drive => {}
10018 result = &mut run => {
10019 panic!("node stopped before external message was republished: {result:?}");
10020 }
10021 }
10022
10023 run.await.expect("node should stop cleanly");
10024 })
10025 .await
10026 .expect("live node should republish ingress and stop before timeout");
10027
10028 assert_eq!(handle.state(), NodeState::Stopped);
10029 assert!(closed.get());
10030 msgbus::get_message_bus().borrow_mut().dispose();
10031 }
10032
10033 #[rstest]
10034 #[tokio::test(flavor = "current_thread")]
10035 async fn test_run_closes_external_ingress_when_receiver_closes() {
10036 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10037 let closed = Rc::new(Cell::new(false));
10038 let ingress = CapturingExternalIngress::new(rx, closed.clone());
10039
10040 let config = LiveNodeConfig {
10041 environment: Environment::Sandbox,
10042 exec_engine: crate::config::LiveExecutionEngineConfig {
10043 reconciliation: false,
10044 ..Default::default()
10045 },
10046 delay_post_stop: Duration::ZERO,
10047 timeout_connection: Duration::from_millis(500),
10048 timeout_disconnection: Duration::from_millis(500),
10049 ..Default::default()
10050 };
10051
10052 let mut node = LiveNodeBuilder::from_config(config)
10053 .unwrap()
10054 .with_external_ingress(Box::new(ingress))
10055 .build()
10056 .expect("node builds with external message bus ingress");
10057 let handle = node.handle();
10058
10059 tokio::time::timeout(Duration::from_secs(30), async {
10060 let run = node.run();
10061 tokio::pin!(run);
10062
10063 let drive = async {
10064 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10065
10066 drop(tx);
10067
10068 wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10069 assert!(
10070 handle.is_running(),
10071 "node should keep running after ingress closes"
10072 );
10073 handle.stop();
10074 };
10075
10076 tokio::select! {
10077 biased;
10078
10079 () = drive => {}
10080 result = &mut run => {
10081 panic!("node stopped before ingress close was observed: {result:?}");
10082 }
10083 }
10084
10085 run.await.expect("node should stop cleanly");
10086 })
10087 .await
10088 .expect("live node should close ingress and stop before timeout");
10089
10090 assert_eq!(handle.state(), NodeState::Stopped);
10091 }
10092
10093 #[rstest]
10094 #[tokio::test(flavor = "current_thread")]
10095 async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10096 let closed = Rc::new(Cell::new(false));
10097 let ingress = FailingExternalIngress::new(closed.clone());
10098
10099 let config = LiveNodeConfig {
10100 environment: Environment::Sandbox,
10101 exec_engine: crate::config::LiveExecutionEngineConfig {
10102 reconciliation: false,
10103 ..Default::default()
10104 },
10105 delay_post_stop: Duration::ZERO,
10106 timeout_connection: Duration::from_millis(500),
10107 timeout_disconnection: Duration::from_millis(500),
10108 ..Default::default()
10109 };
10110
10111 let mut node = LiveNodeBuilder::from_config(config)
10112 .unwrap()
10113 .with_external_ingress(Box::new(ingress))
10114 .build()
10115 .expect("node builds with external message bus ingress");
10116 let handle = node.handle();
10117
10118 let err = node.run().await.expect_err("run should fail");
10119
10120 assert!(
10121 err.to_string()
10122 .contains("external ingress receiver unavailable")
10123 );
10124 assert_eq!(handle.state(), NodeState::Stopped);
10125 assert!(closed.get());
10126 }
10127
10128 #[rstest]
10129 fn test_node_build_rejects_existing_node_preserving_account_delivery(
10130 #[values(false, true)] first_builder: bool,
10131 #[values(false, true)] second_builder: bool,
10132 ) {
10133 let config = LiveNodeConfig {
10134 trader_id: TraderId::from("PROBE-001"),
10135 ..Default::default()
10136 };
10137
10138 let mut node = if first_builder {
10139 LiveNodeBuilder::from_config(config)
10140 .unwrap()
10141 .build()
10142 .unwrap()
10143 } else {
10144 LiveNode::build("Original".to_string(), Some(config)).unwrap()
10145 };
10146
10147 let bus = msgbus::get_message_bus();
10148
10149 let ExecutionEvent::Account(before) = stub_account_event() else {
10150 unreachable!()
10151 };
10152
10153 get_exec_event_sender()
10154 .send(ExecutionEvent::Account(before.clone()))
10155 .unwrap();
10156 assert_eq!(node.drain_runner_pending(), 1);
10157 assert_eq!(
10158 node.kernel
10159 .cache
10160 .borrow()
10161 .account(&before.account_id)
10162 .unwrap()
10163 .last_event(),
10164 Some(before)
10165 );
10166
10167 let result = if second_builder {
10168 LiveNodeBuilder::from_config(LiveNodeConfig::default())
10169 .unwrap()
10170 .build()
10171 } else {
10172 LiveNode::build("Extra".to_string(), None)
10173 };
10174
10175 assert_eq!(
10176 result.unwrap_err().to_string(),
10177 "A LiveNode already exists or is being built on this thread; drop it before building another"
10178 );
10179
10180 let ExecutionEvent::Account(after) = stub_account_event() else {
10181 unreachable!()
10182 };
10183
10184 get_exec_event_sender()
10185 .send(ExecutionEvent::Account(after.clone()))
10186 .unwrap();
10187 assert_eq!(node.drain_runner_pending(), 1);
10188 assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10189 assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10190 assert_eq!(
10191 node.kernel
10192 .cache
10193 .borrow()
10194 .account(&after.account_id)
10195 .unwrap()
10196 .last_event(),
10197 Some(after)
10198 );
10199 }
10200
10201 #[rstest]
10202 fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10203 let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10204 .unwrap()
10205 .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10206 let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10207 assert!(
10208 error
10209 .to_string()
10210 .contains("already exists or is being built")
10211 );
10212 anyhow::bail!("Event store construction failed")
10213 })
10214 .build();
10215
10216 assert_eq!(
10217 result.unwrap_err().to_string(),
10218 "Event store construction failed"
10219 );
10220
10221 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10222 assert_eq!(node.state(), NodeState::Idle);
10223 }
10224
10225 #[rstest]
10226 fn test_node_build_allows_separate_threads() {
10227 let node = LiveNode::build("First".to_string(), None).unwrap();
10228
10229 let other_state = std::thread::spawn(|| {
10230 let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10231 other.state()
10232 })
10233 .join()
10234 .unwrap();
10235
10236 assert_eq!(node.state(), NodeState::Idle);
10237 assert_eq!(other_state, NodeState::Idle);
10238 }
10239
10240 #[rstest]
10241 fn test_node_build_releases_thread_after_drop() {
10242 let mut node = LiveNode::build("First".to_string(), None).unwrap();
10243 node.dispose();
10244 assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10245 drop(node);
10246
10247 let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10248 assert_eq!(replacement.state(), NodeState::Idle);
10249 }
10250
10251 #[rstest]
10252 fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10253 let config = LiveNodeConfig {
10254 event_store: Some(EventStoreConfig::default()),
10255 ..Default::default()
10256 };
10257
10258 let result = if builder {
10259 LiveNodeBuilder::from_config(config).unwrap().build()
10260 } else {
10261 LiveNode::build("Failure".to_string(), Some(config))
10262 };
10263
10264 assert!(result.unwrap_err().to_string().contains("factory"));
10265
10266 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10267 assert_eq!(node.state(), NodeState::Idle);
10268 }
10269
10270 #[cfg(feature = "python")]
10271 #[rstest]
10272 fn test_node_build_and_initial_state() {
10273 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10274 .unwrap()
10275 .with_name("TestNode")
10276 .build()
10277 .unwrap();
10278
10279 assert_eq!(node.state(), NodeState::Idle);
10280 assert!(!node.is_running());
10281 assert_eq!(node.environment(), Environment::Sandbox);
10282 assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10283 }
10284
10285 #[cfg(feature = "python")]
10286 #[rstest]
10287 fn test_node_build_replaces_stale_runner_senders() {
10288 replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10289 replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10290
10291 let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10292 .unwrap()
10293 .with_name("FirstNode")
10294 .build()
10295 .unwrap();
10296
10297 assert_eq!(first.state(), NodeState::Idle);
10298 drop(first);
10299
10300 let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10301 .unwrap()
10302 .with_name("SecondNode")
10303 .build()
10304 .unwrap();
10305
10306 assert_eq!(second.state(), NodeState::Idle);
10307 assert!(!second.is_running());
10308 }
10309
10310 #[cfg(feature = "python")]
10311 #[rstest]
10312 fn test_node_handle_reflects_node_state() {
10313 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10314 .unwrap()
10315 .with_name("TestNode")
10316 .build()
10317 .unwrap();
10318
10319 let handle = node.handle();
10320
10321 assert_eq!(handle.state(), NodeState::Idle);
10322 assert!(!handle.is_running());
10323 }
10324
10325 #[rstest]
10326 fn test_pending_drain_data_returns_false_when_empty() {
10327 let mut pending = PendingEvents::default();
10328
10329 assert!(!pending.drain_data());
10330 }
10331
10332 #[rstest]
10333 fn test_pending_drain_data_returns_true_when_non_empty() {
10334 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10335
10336 let mut pending = PendingEvents::default();
10337 pending.data_evts.push(
10338 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10339 .into(),
10340 );
10341
10342 assert!(pending.drain_data());
10343 assert!(pending.data_evts.is_empty());
10344 }
10345
10346 fn stub_data_event() -> DataEvent {
10347 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10348
10349 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10350 }
10351
10352 fn stub_data_command() -> DataCommand {
10353 use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10354 use nautilus_core::{UUID4, UnixNanos};
10355 use nautilus_model::identifiers::Venue;
10356
10357 DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10358 None,
10359 Venue::from("TEST"),
10360 UUID4::new(),
10361 UnixNanos::default(),
10362 None,
10363 None,
10364 )))
10365 }
10366
10367 fn stub_system_command() -> SystemCommand {
10368 SystemCommand::ReconnectSocket(ReconnectSocket::new(
10369 TraderId::from("TRADER-001"),
10370 ClientId::from("POLYMARKET"),
10371 Ustr::from("polymarket-market-streams"),
10372 UnixNanos::default(),
10373 ))
10374 }
10375
10376 #[rstest]
10377 fn test_flush_pending_data_drains_events_and_commands() {
10378 let (evt_tx, mut evt_rx) =
10379 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10380 let (cmd_tx, mut cmd_rx) =
10381 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10382
10383 let mut pending = PendingEvents::default();
10384
10385 pending.data_evts.push((stub_data_event()).into());
10387 pending.data_cmds.push(stub_data_command().into());
10388
10389 evt_tx.send((stub_data_event()).into()).unwrap();
10391 cmd_tx.send(stub_data_command().into()).unwrap();
10392
10393 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10394
10395 assert!(pending.data_evts.is_empty());
10396 assert!(pending.data_cmds.is_empty());
10397 assert!(evt_rx.try_recv().is_err());
10398 assert!(cmd_rx.try_recv().is_err());
10399 }
10400
10401 #[rstest]
10402 fn test_flush_pending_data_drains_mixed_sources() {
10403 let (evt_tx, mut evt_rx) =
10404 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10405 let (cmd_tx, mut cmd_rx) =
10406 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10407
10408 let mut pending = PendingEvents::default();
10409
10410 pending.data_evts.push((stub_data_event()).into());
10412 cmd_tx.send(stub_data_command().into()).unwrap();
10413
10414 evt_tx.send((stub_data_event()).into()).unwrap();
10416 evt_tx.send((stub_data_event()).into()).unwrap();
10417 cmd_tx.send(stub_data_command().into()).unwrap();
10418
10419 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10420
10421 assert!(pending.data_evts.is_empty());
10422 assert!(pending.data_cmds.is_empty());
10423 assert!(evt_rx.try_recv().is_err());
10424 assert!(cmd_rx.try_recv().is_err());
10425 }
10426
10427 #[rstest]
10428 fn test_pending_system_events_stay_separate_from_data() {
10429 let mut pending = PendingEvents::default();
10430
10431 let change = SocketStateChange::new(
10432 ClientId::from("BINANCE"),
10433 Some(Venue::from("BINANCE")),
10434 ustr::Ustr::from("binance-futures-market-streams"),
10435 SocketState::Connected,
10436 );
10437
10438 pending
10439 .system_events
10440 .push(SystemEvent::SocketState(change).into());
10441 let system_events = pending
10442 .take_system_events()
10443 .into_iter()
10444 .map(|message| message.dispatch(|value| value))
10445 .collect::<Vec<_>>();
10446
10447 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10448 assert!(pending.is_empty());
10449 }
10450
10451 #[rstest]
10452 fn test_pending_system_commands_stay_separate_from_data() {
10453 let mut pending = PendingEvents::default();
10454 let command = stub_system_command();
10455
10456 pending.system_commands.push(command.into());
10457 let system_commands = pending
10458 .take_system_commands()
10459 .into_iter()
10460 .map(|message| message.dispatch(|value| value))
10461 .collect::<Vec<_>>();
10462
10463 assert_eq!(system_commands, vec![command]);
10464 assert!(pending.is_empty());
10465 }
10466
10467 fn stub_time_event_handler() -> TimeEventMessage {
10468 use std::rc::Rc;
10469
10470 use nautilus_common::{
10471 runner::TimeEventMessage,
10472 timer::{TimeEvent, TimeEventCallback},
10473 };
10474 use nautilus_core::{UUID4, UnixNanos};
10475 use ustr::Ustr;
10476
10477 TimeEventMessage::new(
10478 TimeEvent::new(
10479 Ustr::from("test-timer"),
10480 UUID4::new(),
10481 UnixNanos::default(),
10482 UnixNanos::default(),
10483 ),
10484 TimeEventCallback::RustLocal(Rc::new(|_| {})),
10485 )
10486 }
10487
10488 fn stub_trading_command_message() -> TradingCommandMessage {
10489 use nautilus_common::messages::execution::query::QueryAccount;
10490 use nautilus_core::{UUID4, UnixNanos};
10491 use nautilus_model::identifiers::AccountId;
10492
10493 TradingCommandMessage::new(
10494 MessagingSwitchboard::exec_engine_execute(),
10495 TradingCommand::QueryAccount(QueryAccount::new(
10496 TraderId::from("TESTER-001"),
10497 None,
10498 AccountId::from("TEST-001"),
10499 UUID4::new(),
10500 UnixNanos::default(),
10501 None,
10502 None, )),
10504 )
10505 }
10506
10507 fn stub_exec_event() -> ExecutionEvent {
10508 use nautilus_model::{
10509 enums::{LiquiditySide, OrderSide},
10510 identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
10511 reports::FillReport,
10512 types::{Money, Price, Quantity},
10513 };
10514
10515 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
10516 AccountId::from("TEST-001"),
10517 InstrumentId::from("TEST.VENUE"),
10518 VenueOrderId::from("V-001"),
10519 TradeId::from("T-001"),
10520 OrderSide::Buy,
10521 Quantity::from("1.0"),
10522 Price::from("100.0"),
10523 Money::from("0.01 USD"),
10524 LiquiditySide::Maker,
10525 None,
10526 None,
10527 nautilus_core::UnixNanos::default(),
10528 nautilus_core::UnixNanos::default(),
10529 None,
10530 ))))
10531 }
10532
10533 #[rstest]
10534 fn test_flush_all_pending_drains_buffered_channels() {
10535 let (time_tx, mut time_rx) =
10536 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10537 let (system_evt_tx, mut system_evt_rx) =
10538 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10539 let (system_cmd_tx, mut system_cmd_rx) =
10540 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10541 let (data_evt_tx, mut data_evt_rx) =
10542 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10543 let (data_cmd_tx, mut data_cmd_rx) =
10544 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10545 let (exec_evt_tx, mut exec_evt_rx) =
10546 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10547 let (exec_cmd_tx, mut exec_cmd_rx) =
10548 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10549
10550 let mut pending = PendingEvents::default();
10551
10552 pending.data_evts.push((stub_data_event()).into());
10554 pending.data_cmds.push(stub_data_command().into());
10555
10556 time_tx.send((stub_time_event_handler()).into()).unwrap();
10558
10559 let change = SocketStateChange::new(
10560 ClientId::from("BINANCE"),
10561 Some(Venue::from("BINANCE")),
10562 Ustr::from("binance-futures-market-streams"),
10563 SocketState::Connected,
10564 );
10565 system_evt_tx
10566 .send((SystemEvent::SocketState(change)).into())
10567 .unwrap();
10568 system_cmd_tx.send((stub_system_command()).into()).unwrap();
10569 data_evt_tx.send((stub_data_event()).into()).unwrap();
10570 data_cmd_tx.send(stub_data_command().into()).unwrap();
10571 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10572 exec_cmd_tx
10573 .send(stub_trading_command_message().into())
10574 .unwrap();
10575
10576 flush_all_pending(
10577 &mut pending,
10578 &mut time_rx,
10579 &mut system_evt_rx,
10580 &mut system_cmd_rx,
10581 &mut exec_evt_rx,
10582 &mut exec_cmd_rx,
10583 &mut data_evt_rx,
10584 &mut data_cmd_rx,
10585 );
10586
10587 let system_events = pending
10588 .take_system_events()
10589 .into_iter()
10590 .map(|message| message.dispatch(|value| value))
10591 .collect::<Vec<_>>();
10592 let system_commands = pending
10593 .take_system_commands()
10594 .into_iter()
10595 .map(|message| message.dispatch(|value| value))
10596 .collect::<Vec<_>>();
10597 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10598 assert_eq!(system_commands, vec![stub_system_command()]);
10599 assert!(pending.data_evts.is_empty());
10600 assert!(pending.data_cmds.is_empty());
10601 assert!(pending.exec_reports.is_empty());
10602 assert!(pending.exec_cmds.is_empty());
10603 assert!(pending.order_evts.is_empty());
10604 assert!(time_rx.try_recv().is_err());
10605 assert!(system_evt_rx.try_recv().is_err());
10606 assert!(system_cmd_rx.try_recv().is_err());
10607 assert!(data_evt_rx.try_recv().is_err());
10608 assert!(data_cmd_rx.try_recv().is_err());
10609 assert!(exec_evt_rx.try_recv().is_err());
10610 assert!(exec_cmd_rx.try_recv().is_err());
10611 }
10612
10613 fn stub_order_event() -> ExecutionEvent {
10614 use nautilus_model::events::order::spec::OrderSubmittedSpec;
10615
10616 ExecutionEvent::Order(OrderEventAny::Submitted(
10617 OrderSubmittedSpec::builder().build(),
10618 ))
10619 }
10620
10621 fn stub_account_event() -> ExecutionEvent {
10622 use nautilus_core::{UUID4, UnixNanos};
10623 use nautilus_model::{
10624 enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
10625 };
10626
10627 ExecutionEvent::Account(AccountState::new(
10628 AccountId::from("TEST-001"),
10629 AccountType::Cash,
10630 vec![],
10631 vec![],
10632 true,
10633 UUID4::new(),
10634 UnixNanos::default(),
10635 UnixNanos::default(),
10636 None,
10637 ))
10638 }
10639
10640 #[rstest]
10641 fn test_flush_all_pending_routes_order_event_to_order_evts() {
10642 let (_time_tx, mut time_rx) =
10643 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10644 let (_system_evt_tx, mut system_evt_rx) =
10645 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10646 let (_system_cmd_tx, mut system_cmd_rx) =
10647 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10648 let (_data_evt_tx, mut data_evt_rx) =
10649 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10650 let (_data_cmd_tx, mut data_cmd_rx) =
10651 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10652 let (exec_evt_tx, mut exec_evt_rx) =
10653 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10654 let (_exec_cmd_tx, mut exec_cmd_rx) =
10655 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10656
10657 let mut pending = PendingEvents::default();
10658
10659 exec_evt_tx.send((stub_order_event()).into()).unwrap();
10660 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10661
10662 flush_all_pending(
10663 &mut pending,
10664 &mut time_rx,
10665 &mut system_evt_rx,
10666 &mut system_cmd_rx,
10667 &mut exec_evt_rx,
10668 &mut exec_cmd_rx,
10669 &mut data_evt_rx,
10670 &mut data_cmd_rx,
10671 );
10672
10673 assert!(pending.order_evts.is_empty());
10675 assert!(pending.exec_reports.is_empty());
10676 assert!(exec_evt_rx.try_recv().is_err());
10677 }
10678
10679 #[rstest]
10680 fn test_flush_all_pending_routes_account_event_immediately() {
10681 let (_time_tx, mut time_rx) =
10682 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10683 let (_system_evt_tx, mut system_evt_rx) =
10684 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10685 let (_system_cmd_tx, mut system_cmd_rx) =
10686 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10687 let (_data_evt_tx, mut data_evt_rx) =
10688 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10689 let (_data_cmd_tx, mut data_cmd_rx) =
10690 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10691 let (exec_evt_tx, mut exec_evt_rx) =
10692 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10693 let (_exec_cmd_tx, mut exec_cmd_rx) =
10694 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10695
10696 let mut pending = PendingEvents::default();
10697
10698 exec_evt_tx.send((stub_account_event()).into()).unwrap();
10699
10700 flush_all_pending(
10701 &mut pending,
10702 &mut time_rx,
10703 &mut system_evt_rx,
10704 &mut system_cmd_rx,
10705 &mut exec_evt_rx,
10706 &mut exec_cmd_rx,
10707 &mut data_evt_rx,
10708 &mut data_cmd_rx,
10709 );
10710
10711 assert!(pending.exec_reports.is_empty());
10713 assert!(pending.order_evts.is_empty());
10714 assert!(pending.exec_cmds.is_empty());
10715 assert!(exec_evt_rx.try_recv().is_err());
10716 }
10717
10718 #[rstest]
10719 fn test_pending_is_empty_when_default() {
10720 let pending = PendingEvents::default();
10721
10722 assert!(pending.is_empty());
10723 }
10724
10725 #[rstest]
10726 fn test_pending_is_empty_false_with_data_evt() {
10727 let mut pending = PendingEvents::default();
10728 pending.data_evts.push((stub_data_event()).into());
10729
10730 assert!(!pending.is_empty());
10731 }
10732
10733 #[rstest]
10734 fn test_pending_is_empty_false_with_data_cmd() {
10735 let mut pending = PendingEvents::default();
10736 pending.data_cmds.push(stub_data_command().into());
10737
10738 assert!(!pending.is_empty());
10739 }
10740
10741 #[rstest]
10742 fn test_pending_is_empty_false_with_exec_cmd() {
10743 let mut pending = PendingEvents::default();
10744 pending
10745 .exec_cmds
10746 .push(stub_trading_command_message().into());
10747
10748 assert!(!pending.is_empty());
10749 }
10750
10751 #[rstest]
10752 fn test_pending_drain_preserves_trading_command_target() {
10753 std::thread::spawn(|| {
10754 msgbus::get_message_bus().borrow_mut().dispose();
10755 let risk_commands = Rc::new(RefCell::new(Vec::new()));
10756 let exec_commands = Rc::new(RefCell::new(Vec::new()));
10757
10758 let risk_commands_handler = risk_commands.clone();
10759 msgbus::register_trading_command_endpoint(
10760 MessagingSwitchboard::risk_engine_execute(),
10761 TypedIntoHandler::from(move |command: TradingCommand| {
10762 risk_commands_handler.borrow_mut().push(command);
10763 }),
10764 );
10765
10766 let exec_commands_handler = exec_commands.clone();
10767 msgbus::register_trading_command_endpoint(
10768 MessagingSwitchboard::exec_engine_execute(),
10769 TypedIntoHandler::from(move |command: TradingCommand| {
10770 exec_commands_handler.borrow_mut().push(command);
10771 }),
10772 );
10773
10774 let mut pending = PendingEvents::default();
10775 pending.exec_cmds.push(
10776 TradingCommandMessage::new(
10777 MessagingSwitchboard::risk_engine_execute(),
10778 TradingCommand::QueryAccount(QueryAccount::new(
10779 TraderId::from("TESTER-001"),
10780 None,
10781 AccountId::from("TEST-001"),
10782 UUID4::new(),
10783 UnixNanos::default(),
10784 None,
10785 None,
10786 )),
10787 )
10788 .into(),
10789 );
10790
10791 pending.drain();
10792
10793 assert!(pending.is_empty());
10794 assert_eq!(risk_commands.borrow().len(), 1);
10795 assert!(matches!(
10796 &risk_commands.borrow()[0],
10797 TradingCommand::QueryAccount(_)
10798 ));
10799 assert_eq!(exec_commands.borrow().as_slice(), &[]);
10800 })
10801 .join()
10802 .unwrap();
10803 }
10804
10805 #[rstest]
10806 fn test_pending_is_empty_false_with_exec_report() {
10807 let mut pending = PendingEvents::default();
10808
10809 if let ExecutionEvent::Report(report) = stub_exec_event() {
10810 pending.exec_reports.push((report).into());
10811 }
10812
10813 assert!(!pending.is_empty());
10814 }
10815
10816 #[rstest]
10817 fn test_pending_is_empty_false_with_order_evt() {
10818 let mut pending = PendingEvents::default();
10819
10820 if let ExecutionEvent::Order(order_evt) = stub_order_event() {
10821 pending.order_evts.push((order_evt).into());
10822 }
10823
10824 assert!(!pending.is_empty());
10825 }
10826
10827 fn stub_submitted_batch_event() -> ExecutionEvent {
10828 use nautilus_model::{
10829 events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
10830 identifiers::ClientOrderId,
10831 };
10832
10833 let events = vec![
10834 OrderSubmittedSpec::builder()
10835 .client_order_id(ClientOrderId::from("O-001"))
10836 .build(),
10837 OrderSubmittedSpec::builder()
10838 .client_order_id(ClientOrderId::from("O-002"))
10839 .build(),
10840 ];
10841
10842 ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
10843 }
10844
10845 fn stub_canceled_batch_event() -> ExecutionEvent {
10846 use nautilus_model::{
10847 events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
10848 identifiers::ClientOrderId,
10849 };
10850
10851 let events = vec![
10852 OrderCanceledSpec::builder()
10853 .client_order_id(ClientOrderId::from("O-001"))
10854 .build(),
10855 OrderCanceledSpec::builder()
10856 .client_order_id(ClientOrderId::from("O-002"))
10857 .build(),
10858 ];
10859
10860 ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
10861 }
10862
10863 #[rstest]
10864 fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
10865 let (_time_tx, mut time_rx) =
10866 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10867 let (_system_evt_tx, mut system_evt_rx) =
10868 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10869 let (_system_cmd_tx, mut system_cmd_rx) =
10870 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10871 let (_data_evt_tx, mut data_evt_rx) =
10872 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10873 let (_data_cmd_tx, mut data_cmd_rx) =
10874 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10875 let (exec_evt_tx, mut exec_evt_rx) =
10876 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10877 let (_exec_cmd_tx, mut exec_cmd_rx) =
10878 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10879
10880 let mut pending = PendingEvents::default();
10881
10882 exec_evt_tx
10883 .send((stub_submitted_batch_event()).into())
10884 .unwrap();
10885
10886 flush_all_pending(
10887 &mut pending,
10888 &mut time_rx,
10889 &mut system_evt_rx,
10890 &mut system_cmd_rx,
10891 &mut exec_evt_rx,
10892 &mut exec_cmd_rx,
10893 &mut data_evt_rx,
10894 &mut data_cmd_rx,
10895 );
10896
10897 assert!(pending.order_evts.is_empty());
10899 assert!(exec_evt_rx.try_recv().is_err());
10900 }
10901
10902 #[rstest]
10903 fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
10904 let (_time_tx, mut time_rx) =
10905 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10906 let (_system_evt_tx, mut system_evt_rx) =
10907 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10908 let (_system_cmd_tx, mut system_cmd_rx) =
10909 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10910 let (_data_evt_tx, mut data_evt_rx) =
10911 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10912 let (_data_cmd_tx, mut data_cmd_rx) =
10913 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10914 let (exec_evt_tx, mut exec_evt_rx) =
10915 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10916 let (_exec_cmd_tx, mut exec_cmd_rx) =
10917 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10918
10919 let mut pending = PendingEvents::default();
10920
10921 exec_evt_tx
10922 .send((stub_canceled_batch_event()).into())
10923 .unwrap();
10924
10925 flush_all_pending(
10926 &mut pending,
10927 &mut time_rx,
10928 &mut system_evt_rx,
10929 &mut system_cmd_rx,
10930 &mut exec_evt_rx,
10931 &mut exec_cmd_rx,
10932 &mut data_evt_rx,
10933 &mut data_cmd_rx,
10934 );
10935
10936 assert!(pending.order_evts.is_empty());
10938 assert!(exec_evt_rx.try_recv().is_err());
10939 }
10940
10941 #[rstest]
10942 fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
10943 use nautilus_model::identifiers::ClientOrderId;
10944
10945 let (exec_evt_tx, mut exec_evt_rx) =
10946 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10947
10948 exec_evt_tx
10949 .send((stub_canceled_batch_event()).into())
10950 .unwrap();
10951
10952 let mut pending = PendingEvents::default();
10953
10954 while let Ok(evt) = exec_evt_rx.try_recv() {
10955 pending.push_exec_event(evt);
10956 }
10957
10958 let events: Vec<_> = pending
10959 .order_evts
10960 .drain(..)
10961 .map(|event| event.dispatch(|event| event))
10962 .collect();
10963 assert_eq!(events.len(), 2);
10964 assert!(
10965 matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
10966 );
10967 assert!(
10968 matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
10969 );
10970 }
10971
10972 #[rstest]
10973 #[cfg_attr(
10974 not(all(feature = "simulation", madsim)),
10975 tokio::test(start_paused = true)
10976 )]
10977 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10978 async fn test_submission_confirmation_prevents_duplicate_tracking(
10979 #[values(false, true)] report_confirmation: bool,
10980 #[values(false, true)] exhausted: bool,
10981 ) {
10982 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
10983 let client_order_id = order.client_order_id();
10984 let account_id = AccountId::from("TEST-001");
10985 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10986 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10987 OrderPendingUpdateSpec::builder()
10988 .trader_id(order.trader_id())
10989 .strategy_id(order.strategy_id())
10990 .instrument_id(order.instrument_id())
10991 .client_order_id(client_order_id)
10992 .account_id(account_id)
10993 .build(),
10994 )));
10995 advance_clock(Duration::from_millis(101)).await;
10996 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10997
10998 if exhausted {
10999 advance_clock(Duration::from_millis(101)).await;
11000 let result = node.exec_manager.check_inflight_orders();
11001 assert!(result.events.is_empty());
11002 assert!(result.queries.is_empty());
11003 assert_eq!(
11004 node.exec_manager
11005 .take_submission_recovery_exhaustions()
11006 .len(),
11007 1
11008 );
11009 }
11010
11011 if report_confirmation {
11012 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11013 OrderStatusReport::new(
11014 account_id,
11015 order.instrument_id(),
11016 Some(client_order_id),
11017 venue_order_id,
11018 Some(order.order_side()),
11019 order.order_type(),
11020 order.time_in_force(),
11021 OrderStatus::Submitted,
11022 order.quantity(),
11023 Quantity::zero(3),
11024 UnixNanos::default(),
11025 UnixNanos::default(),
11026 UnixNanos::default(),
11027 None,
11028 )
11029 .with_price(order.price().unwrap()),
11030 ))));
11031 node.exec_manager
11032 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11033 advance_clock(Duration::from_millis(101)).await;
11034 let result = node.exec_manager.check_inflight_orders();
11035 assert!(result.queries.is_empty());
11036 assert!(result.events.is_empty());
11037 }
11038
11039 let mut update = OrderUpdatedSpec::builder()
11040 .trader_id(order.trader_id())
11041 .strategy_id(order.strategy_id())
11042 .instrument_id(order.instrument_id())
11043 .client_order_id(client_order_id)
11044 .account_id(account_id)
11045 .quantity(order.quantity())
11046 .price(order.price().unwrap())
11047 .build();
11048
11049 if !report_confirmation {
11050 update.venue_order_id = Some(venue_order_id);
11051 }
11052 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11053 assert_eq!(
11054 node.kernel
11055 .cache
11056 .borrow()
11057 .order(&client_order_id)
11058 .unwrap()
11059 .status(),
11060 OrderStatus::Submitted
11061 );
11062
11063 for _ in 0..3 {
11064 node.exec_manager
11065 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11066 advance_clock(Duration::from_millis(101)).await;
11067 let result = node.exec_manager.check_inflight_orders();
11068 assert!(result.queries.is_empty());
11069 assert!(result.events.is_empty());
11070 assert!(
11071 node.exec_manager
11072 .take_submission_recovery_exhaustions()
11073 .is_empty()
11074 );
11075 }
11076 }
11077
11078 #[rstest]
11079 #[cfg_attr(
11080 not(all(feature = "simulation", madsim)),
11081 tokio::test(start_paused = true)
11082 )]
11083 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11084 async fn test_submission_report_confirmation_preserves_later_command_budget(
11085 #[values(false, true)] cancel: bool,
11086 #[values(false, true)] later_fill: bool,
11087 #[values(false, true)] repeated_report: bool,
11088 #[values(
11089 SubmissionRecoveryPolicy::ResolveLocally,
11090 SubmissionRecoveryPolicy::RetainUnresolved
11091 )]
11092 policy: SubmissionRecoveryPolicy,
11093 ) {
11094 let (mut node, order, instrument) = venue_evidence_node(policy);
11095 let client_order_id = order.client_order_id();
11096 let account_id = AccountId::from("TEST-001");
11097 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11098 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11099 OrderPendingUpdateSpec::builder()
11100 .trader_id(order.trader_id())
11101 .strategy_id(order.strategy_id())
11102 .instrument_id(order.instrument_id())
11103 .client_order_id(client_order_id)
11104 .account_id(account_id)
11105 .build(),
11106 )));
11107 node.exec_manager.register_inflight(client_order_id);
11108 advance_clock(Duration::from_millis(101)).await;
11109 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11110 let report = ExecutionReport::Order(Box::new(
11111 OrderStatusReport::new(
11112 account_id,
11113 order.instrument_id(),
11114 Some(client_order_id),
11115 venue_order_id,
11116 Some(order.order_side()),
11117 order.order_type(),
11118 order.time_in_force(),
11119 OrderStatus::Submitted,
11120 order.quantity(),
11121 Quantity::zero(3),
11122 UnixNanos::default(),
11123 UnixNanos::default(),
11124 UnixNanos::default(),
11125 None,
11126 )
11127 .with_price(order.price().unwrap()),
11128 ));
11129 node.process_exec_event(ExecutionEvent::Report(report.clone()));
11130 advance_clock(Duration::from_millis(101)).await;
11131 let result = node.exec_manager.check_inflight_orders();
11132 assert!(result.events.is_empty());
11133 assert!(result.queries.is_empty());
11134
11135 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11136 OrderUpdatedSpec::builder()
11137 .trader_id(order.trader_id())
11138 .strategy_id(order.strategy_id())
11139 .instrument_id(order.instrument_id())
11140 .client_order_id(client_order_id)
11141 .account_id(account_id)
11142 .quantity(order.quantity())
11143 .price(order.price().unwrap())
11144 .build(),
11145 )));
11146 let pending = if cancel {
11147 OrderEventAny::PendingCancel(
11148 OrderPendingCancelSpec::builder()
11149 .trader_id(order.trader_id())
11150 .strategy_id(order.strategy_id())
11151 .instrument_id(order.instrument_id())
11152 .client_order_id(client_order_id)
11153 .account_id(account_id)
11154 .build(),
11155 )
11156 } else {
11157 OrderEventAny::PendingUpdate(
11158 OrderPendingUpdateSpec::builder()
11159 .trader_id(order.trader_id())
11160 .strategy_id(order.strategy_id())
11161 .instrument_id(order.instrument_id())
11162 .client_order_id(client_order_id)
11163 .account_id(account_id)
11164 .build(),
11165 )
11166 };
11167 node.process_exec_event(ExecutionEvent::Order(pending));
11168 node.exec_manager.register_inflight(client_order_id);
11169
11170 if repeated_report {
11171 node.process_exec_event(ExecutionEvent::Report(report));
11172 }
11173 advance_clock(Duration::from_millis(50)).await;
11174
11175 if later_fill {
11176 let cached = node
11177 .kernel
11178 .cache
11179 .borrow()
11180 .order_owned(&client_order_id)
11181 .unwrap();
11182 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11183 .last_qty(Quantity::from("0.400"))
11184 .last_px(order.price().unwrap())
11185 .commission(Money::from("0 USDT"))
11186 .without_position_id()
11187 .build()
11188 else {
11189 unreachable!()
11190 };
11191 fill.venue_order_id = venue_order_id;
11192 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11193 }
11194 let early = node.exec_manager.check_inflight_orders();
11195 assert!(early.events.is_empty());
11196 assert!(early.queries.is_empty());
11197 advance_clock(Duration::from_millis(51)).await;
11198 let first = node.exec_manager.check_inflight_orders();
11199 assert_eq!(first.queries.len(), usize::from(!repeated_report));
11200 assert!(first.events.is_empty());
11201 advance_clock(Duration::from_millis(101)).await;
11202 let second = node.exec_manager.check_inflight_orders();
11203 assert!(second.queries.is_empty());
11204 if repeated_report {
11205 assert!(second.events.is_empty());
11206 } else {
11207 assert!(
11208 matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
11209 );
11210 }
11211 assert!(
11212 node.exec_manager
11213 .take_submission_recovery_exhaustions()
11214 .is_empty()
11215 );
11216 }
11217
11218 #[rstest]
11219 #[cfg_attr(
11220 not(all(feature = "simulation", madsim)),
11221 tokio::test(start_paused = true)
11222 )]
11223 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11224 async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11225 #[values(false, true)] report_confirmation: bool,
11226 #[values(false, true)] cancel: bool,
11227 #[values(false, true)] batch: bool,
11228 #[values(false, true)] pending_before_dispatch: bool,
11229 #[values(false, true)] missing_order_check: bool,
11230 #[values(
11231 SubmissionRecoveryPolicy::ResolveLocally,
11232 SubmissionRecoveryPolicy::RetainUnresolved
11233 )]
11234 policy: SubmissionRecoveryPolicy,
11235 ) {
11236 let (mut node, order, instrument) = venue_evidence_node_with_config(
11237 crate::config::LiveExecutionEngineConfig {
11238 reconciliation: true,
11239 inflight_check_threshold_ms: 100,
11240 inflight_check_retries: 2,
11241 open_check_threshold_ms: 5_000,
11242 open_check_open_only: !missing_order_check,
11243 open_check_lookback_mins: None,
11244 open_check_missing_retries: 5,
11245 single_order_query_delay_ms: 0,
11246 submission_recovery_policy: policy,
11247 ..Default::default()
11248 },
11249 OrderType::Limit,
11250 true,
11251 );
11252 let client_order_id = order.client_order_id();
11253 let client_id = ClientId::from("TEST-EVIDENCE");
11254 let account_id = AccountId::from("TEST-001");
11255 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11256 node.kernel
11257 .exec_engine
11258 .borrow_mut()
11259 .register_client(Box::new(StubExecutionClient::new(
11260 client_id,
11261 account_id,
11262 instrument.id().venue,
11263 OmsType::Netting,
11264 None,
11265 )))
11266 .unwrap();
11267
11268 advance_clock(Duration::from_millis(101)).await;
11270 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11271
11272 if report_confirmation {
11273 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11274 OrderPendingUpdateSpec::builder()
11275 .trader_id(order.trader_id())
11276 .strategy_id(order.strategy_id())
11277 .instrument_id(order.instrument_id())
11278 .client_order_id(client_order_id)
11279 .account_id(account_id)
11280 .build(),
11281 )));
11282 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11283 OrderStatusReport::new(
11284 account_id,
11285 order.instrument_id(),
11286 Some(client_order_id),
11287 venue_order_id,
11288 Some(order.order_side()),
11289 order.order_type(),
11290 order.time_in_force(),
11291 OrderStatus::Submitted,
11292 order.quantity(),
11293 Quantity::zero(3),
11294 UnixNanos::default(),
11295 UnixNanos::default(),
11296 UnixNanos::default(),
11297 None,
11298 )
11299 .with_price(order.price().unwrap()),
11300 ))));
11301 }
11302
11303 let mut update = OrderUpdatedSpec::builder()
11304 .trader_id(order.trader_id())
11305 .strategy_id(order.strategy_id())
11306 .instrument_id(order.instrument_id())
11307 .client_order_id(client_order_id)
11308 .account_id(account_id)
11309 .quantity(order.quantity())
11310 .price(order.price().unwrap())
11311 .build();
11312
11313 if !report_confirmation {
11314 update.venue_order_id = Some(venue_order_id);
11315 }
11316 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11317 assert_eq!(
11318 node.kernel
11319 .cache
11320 .borrow()
11321 .order(&client_order_id)
11322 .unwrap()
11323 .status(),
11324 OrderStatus::Submitted,
11325 );
11326 advance_clock(Duration::from_millis(101)).await;
11327 let confirmed = node.exec_manager.check_inflight_orders();
11328 assert!(confirmed.queries.is_empty());
11329 assert!(confirmed.events.is_empty());
11330
11331 let pending = if cancel {
11332 OrderEventAny::PendingCancel(
11333 OrderPendingCancelSpec::builder()
11334 .trader_id(order.trader_id())
11335 .strategy_id(order.strategy_id())
11336 .instrument_id(order.instrument_id())
11337 .client_order_id(client_order_id)
11338 .account_id(account_id)
11339 .build(),
11340 )
11341 } else {
11342 OrderEventAny::PendingUpdate(
11343 OrderPendingUpdateSpec::builder()
11344 .trader_id(order.trader_id())
11345 .strategy_id(order.strategy_id())
11346 .instrument_id(order.instrument_id())
11347 .client_order_id(client_order_id)
11348 .account_id(account_id)
11349 .build(),
11350 )
11351 };
11352 let command = venue_evidence_command(&order, cancel, batch);
11353
11354 if pending_before_dispatch {
11355 node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11356 }
11357 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11358 MessagingSwitchboard::exec_engine_execute(),
11359 command,
11360 )));
11361 advance_clock(Duration::from_millis(50)).await;
11362
11363 if missing_order_check {
11364 let engine = node.kernel.exec_engine.borrow();
11365 let clients = engine.get_all_clients();
11366 let check = node
11367 .exec_manager
11368 .prepare_open_order_report_check(UUID4::new(), &clients);
11369 assert_eq!(check.filtered_orders.len(), 1);
11370 let missing = node.exec_manager.reconcile_open_order_reports(
11371 &check,
11372 Vec::new(),
11373 &IndexSet::from([client_id]),
11374 &IndexSet::new(),
11375 &clients,
11376 );
11377 assert!(missing.events.is_empty());
11378 assert!(missing.targeted_queries.is_empty());
11379 assert_eq!(
11380 node.exec_manager.recon_check_retry_count(&client_order_id),
11381 1
11382 );
11383 }
11384 let early = node.exec_manager.check_inflight_orders();
11385 assert!(early.queries.is_empty());
11386 assert!(early.events.is_empty());
11387
11388 if !pending_before_dispatch {
11389 assert_eq!(
11392 node.kernel
11393 .cache
11394 .borrow()
11395 .order(&client_order_id)
11396 .unwrap()
11397 .status(),
11398 OrderStatus::Submitted,
11399 );
11400 node.process_exec_event(ExecutionEvent::Order(pending));
11401 }
11402
11403 if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11404 node.exec_manager
11405 .register_submission(order.init_event(), Some(client_id));
11406 }
11407 advance_clock(Duration::from_millis(51)).await;
11408 let first = node.exec_manager.check_inflight_orders();
11409 assert_eq!(first.queries.len(), 1);
11410 assert!(
11411 matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11412 if query.client_order_id == client_order_id)
11413 );
11414 assert!(first.events.is_empty());
11415 advance_clock(Duration::from_millis(101)).await;
11416 let exhausted = node.exec_manager.check_inflight_orders();
11417 assert!(exhausted.queries.is_empty());
11418 assert!(
11419 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11420 if event.client_order_id == client_order_id)
11421 );
11422 assert!(
11423 node.exec_manager
11424 .take_submission_recovery_exhaustions()
11425 .is_empty()
11426 );
11427 node.process_reconciliation_events(&exhausted.events);
11428 assert_eq!(
11429 node.kernel
11430 .cache
11431 .borrow()
11432 .order(&client_order_id)
11433 .unwrap()
11434 .status(),
11435 OrderStatus::Canceled,
11436 );
11437 }
11438
11439 #[rstest]
11440 #[cfg_attr(
11441 not(all(feature = "simulation", madsim)),
11442 tokio::test(start_paused = true)
11443 )]
11444 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11445 async fn test_submission_command_report_cleanup_matches_default(
11446 #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11447 #[values(
11448 SubmissionRecoveryPolicy::ResolveLocally,
11449 SubmissionRecoveryPolicy::RetainUnresolved
11450 )]
11451 policy: SubmissionRecoveryPolicy,
11452 ) {
11453 let (mut node, order, instrument) = venue_evidence_node(policy);
11454 let client_order_id = order.client_order_id();
11455 let client_id = ClientId::from("TEST-EVIDENCE");
11456 let account_id = AccountId::from("TEST-001");
11457 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11458 node.kernel
11459 .exec_engine
11460 .borrow_mut()
11461 .register_client(Box::new(StubExecutionClient::new(
11462 client_id,
11463 account_id,
11464 instrument.id().venue,
11465 OmsType::Netting,
11466 None,
11467 )))
11468 .unwrap();
11469 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11470 &order,
11471 account_id,
11472 venue_order_id,
11473 )));
11474 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11475 MessagingSwitchboard::exec_engine_execute(),
11476 venue_evidence_command(&order, true, false),
11477 )));
11478 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11479 OrderPendingCancelSpec::builder()
11480 .trader_id(order.trader_id())
11481 .strategy_id(order.strategy_id())
11482 .instrument_id(order.instrument_id())
11483 .client_order_id(client_order_id)
11484 .account_id(account_id)
11485 .venue_order_id(venue_order_id)
11486 .build(),
11487 )));
11488 advance_clock(Duration::from_millis(101)).await;
11489 let first = node.exec_manager.check_inflight_orders();
11490 assert_eq!(first.queries.len(), 1);
11491 assert!(first.events.is_empty());
11492
11493 let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11494 Quantity::from("0.400")
11495 } else {
11496 Quantity::zero(3)
11497 };
11498
11499 let report = OrderStatusReport::new(
11500 account_id,
11501 order.instrument_id(),
11502 Some(client_order_id),
11503 venue_order_id,
11504 Some(order.order_side()),
11505 order.order_type(),
11506 order.time_in_force(),
11507 report_status,
11508 order.quantity(),
11509 filled_qty,
11510 UnixNanos::default(),
11511 UnixNanos::default(),
11512 UnixNanos::default(),
11513 None,
11514 )
11515 .with_price(order.price().unwrap())
11516 .with_avg_px(dec!(100.00));
11517 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11518 report,
11519 ))));
11520 advance_clock(Duration::from_millis(101)).await;
11521 let exhausted = node.exec_manager.check_inflight_orders();
11522
11523 assert!(exhausted.queries.is_empty());
11524 assert!(exhausted.events.is_empty());
11525 assert!(
11526 node.exec_manager
11527 .take_submission_recovery_exhaustions()
11528 .is_empty()
11529 );
11530 let cache = node.kernel.cache.borrow();
11531 let cached = cache.order_ref(&client_order_id).unwrap();
11532 assert_eq!(cached.status(), OrderStatus::PendingCancel);
11533 assert_eq!(cached.filled_qty(), filled_qty);
11534 }
11535
11536 #[rstest]
11537 #[case::accepted_modify(false, "accepted")]
11538 #[case::accepted_cancel(true, "accepted")]
11539 #[case::accepted_batch_modify(false, "accepted_batch")]
11540 #[case::accepted_batch_cancel(true, "accepted_batch")]
11541 #[case::updated_cancel(true, "updated")]
11542 #[case::triggered_cancel(true, "triggered")]
11543 #[cfg_attr(
11544 not(all(feature = "simulation", madsim)),
11545 tokio::test(start_paused = true)
11546 )]
11547 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11548 async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
11549 #[case] cancel: bool,
11550 #[case] evidence: &str,
11551 #[values(false, true)] batch: bool,
11552 #[values(false, true)] report_first: bool,
11553 #[values(
11554 SubmissionRecoveryPolicy::ResolveLocally,
11555 SubmissionRecoveryPolicy::RetainUnresolved
11556 )]
11557 policy: SubmissionRecoveryPolicy,
11558 ) {
11559 use nautilus_model::events::order::spec::OrderTriggeredSpec;
11560
11561 let triggered = evidence == "triggered";
11562
11563 let order_type = if triggered {
11564 OrderType::StopLimit
11565 } else {
11566 OrderType::Limit
11567 };
11568 let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
11569 let client_order_id = order.client_order_id();
11570 let account_id = AccountId::from("TEST-001");
11571 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11572 node.kernel
11573 .exec_engine
11574 .borrow_mut()
11575 .register_client(Box::new(StubExecutionClient::new(
11576 ClientId::from("TEST-EVIDENCE"),
11577 account_id,
11578 instrument.id().venue,
11579 OmsType::Netting,
11580 None,
11581 )))
11582 .unwrap();
11583 advance_clock(Duration::from_millis(101)).await;
11584 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11585
11586 if triggered || report_first {
11587 let previous_pending = if cancel {
11589 OrderEventAny::PendingUpdate(
11590 OrderPendingUpdateSpec::builder()
11591 .trader_id(order.trader_id())
11592 .strategy_id(order.strategy_id())
11593 .instrument_id(order.instrument_id())
11594 .client_order_id(client_order_id)
11595 .account_id(account_id)
11596 .build(),
11597 )
11598 } else {
11599 OrderEventAny::PendingCancel(
11600 OrderPendingCancelSpec::builder()
11601 .trader_id(order.trader_id())
11602 .strategy_id(order.strategy_id())
11603 .instrument_id(order.instrument_id())
11604 .client_order_id(client_order_id)
11605 .account_id(account_id)
11606 .build(),
11607 )
11608 };
11609 node.process_exec_event(ExecutionEvent::Order(previous_pending));
11610 }
11611 advance_clock(Duration::from_millis(10)).await;
11612 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11613 MessagingSwitchboard::exec_engine_execute(),
11614 venue_evidence_command(&order, cancel, batch),
11615 )));
11616
11617 if report_first {
11618 let report = OrderStatusReport::new(
11619 account_id,
11620 instrument.id(),
11621 Some(client_order_id),
11622 venue_order_id,
11623 Some(order.order_side()),
11624 order.order_type(),
11625 order.time_in_force(),
11626 OrderStatus::Submitted,
11627 order.quantity(),
11628 Quantity::zero(3),
11629 UnixNanos::default(),
11630 UnixNanos::default(),
11631 UnixNanos::default(),
11632 None,
11633 )
11634 .with_price(order.price().unwrap());
11635 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11636 report,
11637 ))));
11638 assert!(
11639 node.kernel
11640 .cache
11641 .borrow()
11642 .order(&client_order_id)
11643 .unwrap()
11644 .venue_order_id()
11645 .is_none()
11646 );
11647 }
11648 advance_clock(Duration::from_millis(40)).await;
11649 let acknowledgement = if triggered {
11650 OrderEventAny::Triggered(
11651 OrderTriggeredSpec::builder()
11652 .trader_id(order.trader_id())
11653 .strategy_id(order.strategy_id())
11654 .instrument_id(order.instrument_id())
11655 .client_order_id(client_order_id)
11656 .account_id(account_id)
11657 .venue_order_id(venue_order_id)
11658 .build(),
11659 )
11660 } else if evidence == "updated" {
11661 OrderEventAny::Updated(
11662 OrderUpdatedSpec::builder()
11663 .trader_id(order.trader_id())
11664 .strategy_id(order.strategy_id())
11665 .instrument_id(order.instrument_id())
11666 .client_order_id(client_order_id)
11667 .account_id(account_id)
11668 .venue_order_id(venue_order_id)
11669 .quantity(order.quantity())
11670 .price(order.price().unwrap())
11671 .build(),
11672 )
11673 } else {
11674 TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
11675 };
11676
11677 if evidence == "accepted_batch" {
11678 let OrderEventAny::Accepted(accepted) = acknowledgement else {
11679 unreachable!()
11680 };
11681 node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
11682 vec![accepted],
11683 )));
11684 } else {
11685 node.process_exec_event(ExecutionEvent::Order(acknowledgement));
11686 }
11687 assert_eq!(
11688 node.kernel
11689 .cache
11690 .borrow()
11691 .order(&client_order_id)
11692 .unwrap()
11693 .status(),
11694 match evidence {
11695 "triggered" => OrderStatus::Triggered,
11696 "updated" => OrderStatus::Submitted,
11697 _ => OrderStatus::Accepted,
11698 }
11699 );
11700 advance_clock(Duration::from_millis(20)).await;
11701 let pending = if cancel {
11702 OrderEventAny::PendingCancel(
11703 OrderPendingCancelSpec::builder()
11704 .trader_id(order.trader_id())
11705 .strategy_id(order.strategy_id())
11706 .instrument_id(order.instrument_id())
11707 .client_order_id(client_order_id)
11708 .account_id(account_id)
11709 .build(),
11710 )
11711 } else {
11712 OrderEventAny::PendingUpdate(
11713 OrderPendingUpdateSpec::builder()
11714 .trader_id(order.trader_id())
11715 .strategy_id(order.strategy_id())
11716 .instrument_id(order.instrument_id())
11717 .client_order_id(client_order_id)
11718 .account_id(account_id)
11719 .build(),
11720 )
11721 };
11722 node.process_exec_event(ExecutionEvent::Order(pending));
11723 let early = node.exec_manager.check_inflight_orders();
11724 assert!(early.queries.is_empty());
11725 assert!(early.events.is_empty());
11726 advance_clock(Duration::from_millis(41)).await;
11727 let first = node.exec_manager.check_inflight_orders();
11728 let recover_command =
11730 triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
11731 assert_eq!(first.queries.len(), usize::from(recover_command));
11732 assert!(first.events.is_empty());
11733 advance_clock(Duration::from_millis(101)).await;
11734 let exhausted = node.exec_manager.check_inflight_orders();
11735 assert!(exhausted.queries.is_empty());
11736 assert!(
11737 node.exec_manager
11738 .take_submission_recovery_exhaustions()
11739 .is_empty()
11740 );
11741
11742 if recover_command {
11743 assert!(
11744 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11745 if event.client_order_id == client_order_id)
11746 );
11747 } else {
11748 assert!(exhausted.events.is_empty());
11749 }
11750 }
11751
11752 #[rstest]
11753 #[case::submission_only(None, false)]
11754 #[case::modify(Some(false), false)]
11755 #[case::cancel(Some(true), false)]
11756 #[case::batch_modify(Some(false), true)]
11757 #[case::batch_cancel(Some(true), true)]
11758 #[cfg_attr(
11759 not(all(feature = "simulation", madsim)),
11760 tokio::test(start_paused = true)
11761 )]
11762 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11763 async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
11764 #[case] cancel: Option<bool>,
11765 #[case] batch: bool,
11766 #[values(false, true)] missing_order_check: bool,
11767 #[values(
11768 SubmissionRecoveryPolicy::ResolveLocally,
11769 SubmissionRecoveryPolicy::RetainUnresolved
11770 )]
11771 policy: SubmissionRecoveryPolicy,
11772 ) {
11773 let (mut node, order, instrument) = venue_evidence_node_with_config(
11774 crate::config::LiveExecutionEngineConfig {
11775 reconciliation: true,
11776 inflight_check_threshold_ms: 100,
11777 inflight_check_retries: 2,
11778 open_check_threshold_ms: 1,
11779 open_check_open_only: false,
11780 open_check_lookback_mins: None,
11781 open_check_missing_retries: 5,
11782 single_order_query_delay_ms: 0,
11783 submission_recovery_policy: policy,
11784 ..Default::default()
11785 },
11786 OrderType::Limit,
11787 false,
11788 );
11789 let client_order_id = order.client_order_id();
11790 let client_id = ClientId::from("TEST-EVIDENCE");
11791 let account_id = AccountId::from("TEST-001");
11792 node.kernel
11793 .exec_engine
11794 .borrow_mut()
11795 .register_client(Box::new(StubExecutionClient::new(
11796 client_id,
11797 account_id,
11798 instrument.id().venue,
11799 OmsType::Netting,
11800 None,
11801 )))
11802 .unwrap();
11803 let queries = Rc::new(RefCell::new(Vec::new()));
11804 let client = OrderReportClient {
11805 client_id,
11806 report: OrderStatusReport::new(
11807 account_id,
11808 instrument.id(),
11809 Some(client_order_id),
11810 VenueOrderId::from("V-VENUE-EVIDENCE"),
11811 Some(order.order_side()),
11812 order.order_type(),
11813 order.time_in_force(),
11814 OrderStatus::Submitted,
11815 order.quantity(),
11816 Quantity::zero(3),
11817 UnixNanos::default(),
11818 UnixNanos::default(),
11819 UnixNanos::default(),
11820 None,
11821 )
11822 .with_price(order.price().unwrap()),
11823 targeted: false,
11824 queries: queries.clone(),
11825 };
11826 advance_clock(Duration::from_millis(2)).await;
11827 let events = node.exec_manager.check_open_orders(&[&client]).await;
11828 assert!(events.is_empty());
11829 assert_eq!(*queries.borrow(), vec!["bulk"]);
11830 assert_eq!(
11831 node.kernel
11832 .cache
11833 .borrow()
11834 .order(&client_order_id)
11835 .unwrap()
11836 .status(),
11837 OrderStatus::Submitted
11838 );
11839 advance_clock(Duration::from_millis(2)).await;
11840
11841 if missing_order_check {
11842 let engine = node.kernel.exec_engine.borrow();
11843 let clients = engine.get_all_clients();
11844 let check = node
11845 .exec_manager
11846 .prepare_open_order_report_check(UUID4::new(), &clients);
11847 assert_eq!(check.filtered_orders.len(), 1);
11848 let missing = node.exec_manager.reconcile_open_order_reports(
11849 &check,
11850 Vec::new(),
11851 &IndexSet::from([client_id]),
11852 &IndexSet::new(),
11853 &clients,
11854 );
11855 assert!(missing.events.is_empty());
11856 assert!(missing.targeted_queries.is_empty());
11857 assert_eq!(
11858 node.exec_manager.recon_check_retry_count(&client_order_id),
11859 1
11860 );
11861 }
11862 node.exec_manager
11863 .register_submission(order.init_event(), Some(client_id));
11864 advance_clock(Duration::from_millis(101)).await;
11865 let duplicate = node.exec_manager.check_inflight_orders();
11866 assert!(duplicate.events.is_empty());
11867 assert_eq!(
11868 duplicate.queries.len(),
11869 usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
11870 );
11871
11872 if let Some(cancel) = cancel {
11873 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11874 MessagingSwitchboard::exec_engine_execute(),
11875 venue_evidence_command(&order, cancel, batch),
11876 )));
11877 let pending = if cancel {
11878 OrderEventAny::PendingCancel(
11879 OrderPendingCancelSpec::builder()
11880 .trader_id(order.trader_id())
11881 .strategy_id(order.strategy_id())
11882 .instrument_id(order.instrument_id())
11883 .client_order_id(client_order_id)
11884 .account_id(account_id)
11885 .build(),
11886 )
11887 } else {
11888 OrderEventAny::PendingUpdate(
11889 OrderPendingUpdateSpec::builder()
11890 .trader_id(order.trader_id())
11891 .strategy_id(order.strategy_id())
11892 .instrument_id(order.instrument_id())
11893 .client_order_id(client_order_id)
11894 .account_id(account_id)
11895 .build(),
11896 )
11897 };
11898 node.process_exec_event(ExecutionEvent::Order(pending));
11899 }
11900 advance_clock(Duration::from_millis(101)).await;
11901 let first = node.exec_manager.check_inflight_orders();
11902 assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
11903
11904 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11905 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11906 if event.client_order_id == client_order_id));
11907 } else {
11908 assert!(first.events.is_empty());
11909 }
11910 node.process_reconciliation_events(&first.events);
11911 advance_clock(Duration::from_millis(101)).await;
11912 let exhausted = node.exec_manager.check_inflight_orders();
11913 assert!(exhausted.queries.is_empty());
11914
11915 if cancel.is_some() {
11916 assert!(
11917 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11918 if event.client_order_id == client_order_id)
11919 );
11920 } else {
11921 assert!(exhausted.events.is_empty());
11922 }
11923 assert!(
11924 node.exec_manager
11925 .take_submission_recovery_exhaustions()
11926 .is_empty()
11927 );
11928 }
11929
11930 #[rstest]
11931 #[case::submission_only(None, false)]
11932 #[case::modify(Some(false), false)]
11933 #[case::cancel(Some(true), false)]
11934 #[case::batch_modify(Some(false), true)]
11935 #[case::batch_cancel(Some(true), true)]
11936 #[cfg_attr(
11937 not(all(feature = "simulation", madsim)),
11938 tokio::test(start_paused = true)
11939 )]
11940 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11941 async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
11942 #[case] cancel: Option<bool>,
11943 #[case] batch: bool,
11944 #[values(false, true)] targeted: bool,
11945 #[values(
11946 SubmissionRecoveryPolicy::ResolveLocally,
11947 SubmissionRecoveryPolicy::RetainUnresolved
11948 )]
11949 policy: SubmissionRecoveryPolicy,
11950 ) {
11951 let (mut node, order, instrument) = venue_evidence_node_with_config(
11952 crate::config::LiveExecutionEngineConfig {
11953 reconciliation: true,
11954 inflight_check_threshold_ms: 100,
11955 inflight_check_retries: 2,
11956 open_check_threshold_ms: 1,
11957 open_check_open_only: false,
11958 open_check_lookback_mins: None,
11959 open_check_missing_retries: 1,
11960 single_order_query_delay_ms: 0,
11961 submission_recovery_policy: policy,
11962 ..Default::default()
11963 },
11964 OrderType::Limit,
11965 true,
11966 );
11967 let client_order_id = order.client_order_id();
11968 let client_id = ClientId::from("TEST-EVIDENCE");
11969 let account_id = AccountId::from("TEST-001");
11970 node.kernel
11971 .exec_engine
11972 .borrow_mut()
11973 .register_client(Box::new(StubExecutionClient::new(
11974 client_id,
11975 account_id,
11976 instrument.id().venue,
11977 OmsType::Netting,
11978 None,
11979 )))
11980 .unwrap();
11981 advance_clock(Duration::from_millis(101)).await;
11982 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11983 advance_clock(Duration::from_millis(10)).await;
11984
11985 if let Some(cancel) = cancel {
11986 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11987 MessagingSwitchboard::exec_engine_execute(),
11988 venue_evidence_command(&order, cancel, batch),
11989 )));
11990 let pending = if cancel {
11991 OrderEventAny::PendingCancel(
11992 OrderPendingCancelSpec::builder()
11993 .trader_id(order.trader_id())
11994 .strategy_id(order.strategy_id())
11995 .instrument_id(order.instrument_id())
11996 .client_order_id(client_order_id)
11997 .account_id(account_id)
11998 .build(),
11999 )
12000 } else {
12001 OrderEventAny::PendingUpdate(
12002 OrderPendingUpdateSpec::builder()
12003 .trader_id(order.trader_id())
12004 .strategy_id(order.strategy_id())
12005 .instrument_id(order.instrument_id())
12006 .client_order_id(client_order_id)
12007 .account_id(account_id)
12008 .build(),
12009 )
12010 };
12011 node.process_exec_event(ExecutionEvent::Order(pending));
12012 }
12013 let queries = Rc::new(RefCell::new(Vec::new()));
12014 let client = OrderReportClient {
12015 client_id,
12016 report: OrderStatusReport::new(
12017 account_id,
12018 order.instrument_id(),
12019 Some(client_order_id),
12020 VenueOrderId::from("V-VENUE-EVIDENCE"),
12021 Some(order.order_side()),
12022 order.order_type(),
12023 order.time_in_force(),
12024 OrderStatus::Submitted,
12025 order.quantity(),
12026 Quantity::zero(3),
12027 UnixNanos::default(),
12028 UnixNanos::default(),
12029 UnixNanos::default(),
12030 None,
12031 )
12032 .with_price(order.price().unwrap()),
12033 targeted,
12034 queries: queries.clone(),
12035 };
12036 advance_clock(Duration::from_millis(40)).await;
12037 let events = node.exec_manager.check_open_orders(&[&client]).await;
12038 assert!(events.is_empty());
12039 assert_eq!(
12040 *queries.borrow(),
12041 if targeted {
12042 vec!["bulk", "targeted"]
12043 } else {
12044 vec!["bulk"]
12045 }
12046 );
12047 node.process_reconciliation_events(&events);
12048 advance_clock(Duration::from_millis(61)).await;
12049 let first = node.exec_manager.check_inflight_orders();
12050 let recover_command =
12051 cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12052 assert_eq!(first.queries.len(), usize::from(recover_command));
12053
12054 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12055 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12056 if event.client_order_id == client_order_id));
12057 } else {
12058 assert!(first.events.is_empty());
12059 }
12060 node.process_reconciliation_events(&first.events);
12061 advance_clock(Duration::from_millis(101)).await;
12062 let exhausted = node.exec_manager.check_inflight_orders();
12063 assert!(exhausted.queries.is_empty());
12064
12065 if recover_command {
12066 assert!(
12067 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12068 if event.client_order_id == client_order_id)
12069 );
12070 } else {
12071 assert!(exhausted.events.is_empty());
12072 }
12073 assert!(
12074 node.exec_manager
12075 .take_submission_recovery_exhaustions()
12076 .is_empty()
12077 );
12078 }
12079
12080 #[rstest]
12081 #[case::local_update_before_pending("updated")]
12082 #[case::modify_rejected_after_pending("modify_rejected")]
12083 #[case::cancel_rejected_after_pending("cancel_rejected")]
12084 #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12085 #[cfg_attr(
12086 not(all(feature = "simulation", madsim)),
12087 tokio::test(start_paused = true)
12088 )]
12089 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12090 async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12091 #[case] completion: &str,
12092 #[values(
12093 SubmissionRecoveryPolicy::ResolveLocally,
12094 SubmissionRecoveryPolicy::RetainUnresolved
12095 )]
12096 policy: SubmissionRecoveryPolicy,
12097 ) {
12098 use nautilus_model::events::order::spec::{
12099 OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12100 };
12101
12102 let (mut node, order, instrument) = venue_evidence_node(policy);
12103 let client_order_id = order.client_order_id();
12104 let client_id = ClientId::from("TEST-EVIDENCE");
12105 let account_id = AccountId::from("TEST-001");
12106 let cancel = completion == "cancel_rejected";
12107 let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12108 node.kernel
12109 .exec_engine
12110 .borrow_mut()
12111 .register_client(Box::new(StubExecutionClient::new(
12112 client_id,
12113 account_id,
12114 instrument.id().venue,
12115 OmsType::Netting,
12116 None,
12117 )))
12118 .unwrap();
12119 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12120 MessagingSwitchboard::exec_engine_execute(),
12121 venue_evidence_command(&order, cancel, false),
12122 )));
12123 advance_clock(Duration::from_millis(10)).await;
12124
12125 if completion != "updated" {
12126 let pending = if cancel {
12127 OrderEventAny::PendingCancel(
12128 OrderPendingCancelSpec::builder()
12129 .trader_id(order.trader_id())
12130 .strategy_id(order.strategy_id())
12131 .instrument_id(order.instrument_id())
12132 .client_order_id(client_order_id)
12133 .account_id(account_id)
12134 .build(),
12135 )
12136 } else {
12137 OrderEventAny::PendingUpdate(
12138 OrderPendingUpdateSpec::builder()
12139 .trader_id(order.trader_id())
12140 .strategy_id(order.strategy_id())
12141 .instrument_id(order.instrument_id())
12142 .client_order_id(client_order_id)
12143 .account_id(account_id)
12144 .build(),
12145 )
12146 };
12147 node.process_exec_event(ExecutionEvent::Order(pending));
12148 }
12149
12150 if later_cancel_pending {
12151 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12153 OrderPendingCancelSpec::builder()
12154 .trader_id(order.trader_id())
12155 .strategy_id(order.strategy_id())
12156 .instrument_id(order.instrument_id())
12157 .client_order_id(client_order_id)
12158 .account_id(account_id)
12159 .build(),
12160 )));
12161 }
12162 advance_clock(Duration::from_millis(10)).await;
12163 let completed = match completion {
12164 "updated" => OrderEventAny::Updated(
12165 OrderUpdatedSpec::builder()
12166 .trader_id(order.trader_id())
12167 .strategy_id(order.strategy_id())
12168 .instrument_id(order.instrument_id())
12169 .client_order_id(client_order_id)
12170 .account_id(account_id)
12171 .quantity(order.quantity())
12172 .price(Price::from("101.00"))
12173 .build(),
12174 ),
12175 "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12176 OrderEventAny::ModifyRejected(
12177 OrderModifyRejectedSpec::builder()
12178 .trader_id(order.trader_id())
12179 .strategy_id(order.strategy_id())
12180 .instrument_id(order.instrument_id())
12181 .client_order_id(client_order_id)
12182 .account_id(account_id)
12183 .build(),
12184 )
12185 }
12186 "cancel_rejected" => OrderEventAny::CancelRejected(
12187 OrderCancelRejectedSpec::builder()
12188 .trader_id(order.trader_id())
12189 .strategy_id(order.strategy_id())
12190 .instrument_id(order.instrument_id())
12191 .client_order_id(client_order_id)
12192 .account_id(account_id)
12193 .build(),
12194 ),
12195 _ => unreachable!(),
12196 };
12197 node.process_exec_event(ExecutionEvent::Order(completed));
12198 assert_eq!(
12199 node.kernel
12200 .cache
12201 .borrow()
12202 .order(&client_order_id)
12203 .unwrap()
12204 .status(),
12205 if later_cancel_pending {
12206 OrderStatus::PendingCancel
12207 } else {
12208 OrderStatus::Submitted
12209 }
12210 );
12211
12212 advance_clock(Duration::from_millis(10)).await;
12214
12215 if !later_cancel_pending {
12216 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12217 OrderPendingUpdateSpec::builder()
12218 .trader_id(order.trader_id())
12219 .strategy_id(order.strategy_id())
12220 .instrument_id(order.instrument_id())
12221 .client_order_id(client_order_id)
12222 .account_id(account_id)
12223 .build(),
12224 )));
12225 }
12226 advance_clock(Duration::from_millis(10)).await;
12227 let cached = node
12228 .kernel
12229 .cache
12230 .borrow()
12231 .order_owned(&client_order_id)
12232 .unwrap();
12233 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12234 .last_qty(Quantity::from("0.400"))
12235 .last_px(order.price().unwrap())
12236 .commission(Money::from("0 USDT"))
12237 .without_position_id()
12238 .build()
12239 else {
12240 unreachable!()
12241 };
12242 fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12243 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12244
12245 for _ in 0..3 {
12246 advance_clock(Duration::from_millis(101)).await;
12247 let result = node.exec_manager.check_inflight_orders();
12248 assert!(result.queries.is_empty());
12249 assert!(result.events.is_empty());
12250 assert!(
12251 node.exec_manager
12252 .take_submission_recovery_exhaustions()
12253 .is_empty()
12254 );
12255 }
12256 assert_eq!(
12257 node.kernel
12258 .cache
12259 .borrow()
12260 .order(&client_order_id)
12261 .unwrap()
12262 .status(),
12263 if later_cancel_pending {
12264 OrderStatus::PendingCancel
12265 } else {
12266 OrderStatus::PendingUpdate
12267 }
12268 );
12269 }
12270
12271 #[rstest]
12272 #[case::wrong_account("account")]
12273 #[case::wrong_instrument("instrument")]
12274 #[case::conflicting_report_venue("report_venue")]
12275 #[case::unapplied_venue_update("update_venue")]
12276 #[case::local_update("local_update")]
12277 #[cfg_attr(
12278 not(all(feature = "simulation", madsim)),
12279 tokio::test(start_paused = true)
12280 )]
12281 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12282 async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12283 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12284 let client_order_id = order.client_order_id();
12285 let account_id = AccountId::from("TEST-001");
12286 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12287
12288 if invalid.ends_with("venue") {
12289 insert_accepted_limit_order_in_node(
12290 &node,
12291 account_id,
12292 ClientId::from("TEST-EVIDENCE"),
12293 order.instrument_id(),
12294 ClientOrderId::from("O-OTHER"),
12295 venue_order_id,
12296 );
12297 }
12298 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12299 OrderPendingCancelSpec::builder()
12300 .trader_id(order.trader_id())
12301 .strategy_id(order.strategy_id())
12302 .instrument_id(order.instrument_id())
12303 .client_order_id(client_order_id)
12304 .account_id(account_id)
12305 .build(),
12306 )));
12307 advance_clock(Duration::from_millis(101)).await;
12308 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12309
12310 if invalid == "update_venue" || invalid == "local_update" {
12311 let mut updated = OrderUpdatedSpec::builder()
12312 .trader_id(order.trader_id())
12313 .strategy_id(order.strategy_id())
12314 .instrument_id(order.instrument_id())
12315 .client_order_id(client_order_id)
12316 .account_id(account_id)
12317 .quantity(order.quantity())
12318 .price(order.price().unwrap())
12319 .build();
12320
12321 if invalid == "update_venue" {
12322 updated.venue_order_id = Some(venue_order_id);
12323 }
12324 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12325 } else {
12326 let report = OrderStatusReport::new(
12327 if invalid == "account" {
12328 AccountId::from("OTHER-001")
12329 } else {
12330 account_id
12331 },
12332 if invalid == "instrument" {
12333 InstrumentId::from("OTHER.BINANCE")
12334 } else {
12335 order.instrument_id()
12336 },
12337 Some(client_order_id),
12338 venue_order_id,
12339 Some(order.order_side()),
12340 order.order_type(),
12341 order.time_in_force(),
12342 OrderStatus::Submitted,
12343 order.quantity(),
12344 Quantity::zero(3),
12345 UnixNanos::default(),
12346 UnixNanos::default(),
12347 UnixNanos::default(),
12348 None,
12349 )
12350 .with_price(order.price().unwrap());
12351 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12352 report,
12353 ))));
12354 }
12355 assert!(
12356 node.kernel
12357 .cache
12358 .borrow()
12359 .order(&client_order_id)
12360 .unwrap()
12361 .venue_order_id()
12362 .is_none()
12363 );
12364 advance_clock(Duration::from_millis(101)).await;
12365 let result = node.exec_manager.check_inflight_orders();
12366 assert!(result.events.is_empty());
12367 assert!(result.queries.is_empty());
12368 let cached = node
12369 .kernel
12370 .cache
12371 .borrow()
12372 .order_owned(&client_order_id)
12373 .unwrap();
12374 assert_eq!(
12375 cached.status(),
12376 if invalid == "local_update" {
12377 OrderStatus::Submitted
12378 } else {
12379 OrderStatus::PendingCancel
12380 }
12381 );
12382 let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12383 assert_eq!(diagnostics.len(), 1);
12384 assert_eq!(diagnostics[0].client_order_id, client_order_id);
12385 }
12386
12387 fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12388 use nautilus_common::messages::execution::{
12389 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12390 };
12391
12392 let client_id = ClientId::from("TEST-EVIDENCE");
12393 let client_order_id = order.client_order_id();
12394 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12395
12396 if cancel {
12397 let cancel = CancelOrder::new(
12398 order.trader_id(),
12399 Some(client_id),
12400 order.strategy_id(),
12401 order.instrument_id(),
12402 client_order_id,
12403 Some(venue_order_id),
12404 UUID4::new(),
12405 UnixNanos::default(),
12406 None,
12407 None,
12408 );
12409
12410 if batch {
12411 TradingCommand::CancelOrders(BatchCancelOrders::new(
12412 order.trader_id(),
12413 Some(client_id),
12414 order.strategy_id(),
12415 order.instrument_id(),
12416 vec![cancel],
12417 UUID4::new(),
12418 UnixNanos::default(),
12419 None,
12420 None,
12421 ))
12422 } else {
12423 TradingCommand::CancelOrder(cancel)
12424 }
12425 } else {
12426 let modify = ModifyOrder::new(
12427 order.trader_id(),
12428 Some(client_id),
12429 order.strategy_id(),
12430 order.instrument_id(),
12431 client_order_id,
12432 Some(venue_order_id),
12433 None,
12434 Some(Price::from("101.00")),
12435 None,
12436 UUID4::new(),
12437 UnixNanos::default(),
12438 None,
12439 None,
12440 );
12441
12442 if batch {
12443 TradingCommand::ModifyOrders(BatchModifyOrders::new(
12444 order.trader_id(),
12445 Some(client_id),
12446 order.strategy_id(),
12447 order.instrument_id(),
12448 vec![modify],
12449 UUID4::new(),
12450 UnixNanos::default(),
12451 None,
12452 None,
12453 ))
12454 } else {
12455 TradingCommand::ModifyOrder(modify)
12456 }
12457 }
12458 }
12459
12460 fn venue_evidence_node(
12461 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12462 ) -> (LiveNode, OrderAny, InstrumentAny) {
12463 venue_evidence_node_with_order_type(policy, OrderType::Limit)
12464 }
12465
12466 fn venue_evidence_node_with_order_type(
12467 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12468 order_type: OrderType,
12469 ) -> (LiveNode, OrderAny, InstrumentAny) {
12470 venue_evidence_node_with_config(
12471 crate::config::LiveExecutionEngineConfig {
12472 reconciliation: true,
12473 inflight_check_threshold_ms: 100,
12474 inflight_check_retries: 2,
12475 open_check_threshold_ms: 5_000,
12476 single_order_query_delay_ms: 0,
12477 submission_recovery_policy: policy,
12478 ..Default::default()
12479 },
12480 order_type,
12481 true,
12482 )
12483 }
12484
12485 fn venue_evidence_node_with_config(
12486 exec_engine: crate::config::LiveExecutionEngineConfig,
12487 order_type: OrderType,
12488 register_submission: bool,
12489 ) -> (LiveNode, OrderAny, InstrumentAny) {
12490 let config = LiveNodeConfig {
12491 exec_engine,
12492 ..Default::default()
12493 };
12494 let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12495 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12496 let account_id = AccountId::from("TEST-001");
12497 let client_id = ClientId::from("TEST-EVIDENCE");
12498 let account = AccountAny::Margin(MarginAccount::new(
12499 AccountState::new(
12500 account_id,
12501 AccountType::Margin,
12502 vec![AccountBalance::new(
12503 Money::from("1000000 USDT"),
12504 Money::from("0 USDT"),
12505 Money::from("1000000 USDT"),
12506 )],
12507 Vec::new(),
12508 true,
12509 UUID4::new(),
12510 UnixNanos::default(),
12511 UnixNanos::default(),
12512 Some(Currency::USDT()),
12513 ),
12514 true,
12515 ));
12516 node.kernel.cache.borrow_mut().add_account(account).unwrap();
12517 node.kernel
12518 .cache
12519 .borrow_mut()
12520 .add_instrument(instrument.clone())
12521 .unwrap();
12522 let mut builder = OrderTestBuilder::new(order_type);
12523 builder
12524 .trader_id(node.trader_id())
12525 .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
12526 .instrument_id(instrument.id())
12527 .quantity(Quantity::from("1.000"))
12528 .price(Price::from("100.00"));
12529
12530 if order_type == OrderType::StopLimit {
12531 builder.trigger_price(Price::from("110.00"));
12532 }
12533 let order = builder.build();
12534 let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
12535 &order,
12536 node.trader_id(),
12537 Some(client_id),
12538 None,
12539 UUID4::new(),
12540 UnixNanos::default(),
12541 ));
12542
12543 if register_submission {
12544 node.observe_exec_command_before_dispatch(&command);
12545 }
12546 node.kernel
12547 .cache
12548 .borrow_mut()
12549 .add_order(order.clone(), None, Some(client_id), false)
12550 .unwrap();
12551 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
12552 &order, account_id,
12553 )));
12554 (node, order, instrument)
12555 }
12556
12557 #[rstest]
12558 #[case::partial("0.400", OrderStatus::PartiallyFilled)]
12559 #[case::full("1.000", OrderStatus::Filled)]
12560 #[tokio::test(start_paused = true)]
12561 async fn test_retained_submission_resolves_late_fill(
12562 #[case] quantity: &str,
12563 #[case] expected_status: OrderStatus,
12564 ) {
12565 let (mut node, order, instrument) =
12566 venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12567 let client_order_id = order.client_order_id();
12568
12569 for _ in 0..2 {
12570 advance_clock(Duration::from_millis(101)).await;
12571 assert!(node.exec_manager.check_inflight_orders().events.is_empty());
12572 }
12573
12574 assert_eq!(
12575 node.exec_manager
12576 .take_submission_recovery_exhaustions()
12577 .len(),
12578 1
12579 );
12580 let fill = TestOrderEventStubs::filled(
12581 &order,
12582 &instrument,
12583 Some(TradeId::from("T-LATE-RETAINED")),
12584 None,
12585 Some(Price::from("100.00")),
12586 Some(Quantity::from(quantity)),
12587 Some(LiquiditySide::Taker),
12588 None,
12589 None,
12590 Some(AccountId::from("TEST-001")),
12591 );
12592 node.process_exec_event(ExecutionEvent::Order(fill));
12593 advance_clock(Duration::from_millis(101)).await;
12594 let result = node.exec_manager.check_inflight_orders();
12595 let cache = node.kernel.cache.borrow();
12596 let cached = cache.order_ref(&client_order_id).unwrap();
12597 assert_eq!(cached.status(), expected_status);
12598 assert_eq!(cached.filled_qty(), Quantity::from(quantity));
12599 assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
12600 assert!(result.events.is_empty());
12601 assert!(result.queries.is_empty());
12602 assert!(
12603 node.exec_manager
12604 .take_submission_recovery_exhaustions()
12605 .is_empty()
12606 );
12607 }
12608
12609 #[rstest]
12610 #[case::submitted_report("order")]
12611 #[case::submitted_report_with_fills("order_with_fills")]
12612 #[case::venue_update("updated")]
12613 #[cfg_attr(
12614 not(all(feature = "simulation", madsim)),
12615 tokio::test(start_paused = true)
12616 )]
12617 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12618 async fn test_venue_evidence_stops_submission_timeout(
12619 #[case] evidence: &str,
12620 #[values(
12621 OrderStatus::Submitted,
12622 OrderStatus::PendingCancel,
12623 OrderStatus::PendingUpdate
12624 )]
12625 status: OrderStatus,
12626 #[values(
12627 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12628 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12629 )]
12630 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12631 ) {
12632 let (mut node, order, _) = venue_evidence_node(policy);
12633 let client_order_id = order.client_order_id();
12634 let account_id = AccountId::from("TEST-001");
12635 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12636 let pending = match status {
12637 OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
12638 OrderPendingCancelSpec::builder()
12639 .trader_id(order.trader_id())
12640 .strategy_id(order.strategy_id())
12641 .instrument_id(order.instrument_id())
12642 .client_order_id(client_order_id)
12643 .account_id(account_id)
12644 .build(),
12645 )),
12646 OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
12647 OrderPendingUpdateSpec::builder()
12648 .trader_id(order.trader_id())
12649 .strategy_id(order.strategy_id())
12650 .instrument_id(order.instrument_id())
12651 .client_order_id(client_order_id)
12652 .account_id(account_id)
12653 .build(),
12654 )),
12655 _ => None,
12656 };
12657
12658 if let Some(pending) = pending {
12659 node.process_exec_event(ExecutionEvent::Order(pending));
12660 }
12661 advance_clock(Duration::from_millis(101)).await;
12662 let first = node.exec_manager.check_inflight_orders();
12663 assert_eq!(first.queries.len(), 1);
12664 assert!(first.events.is_empty());
12665 advance_clock(Duration::from_millis(94)).await;
12666
12667 let event = if evidence == "updated" {
12668 ExecutionEvent::Order(OrderEventAny::Updated(
12669 OrderUpdatedSpec::builder()
12670 .trader_id(order.trader_id())
12671 .strategy_id(order.strategy_id())
12672 .instrument_id(order.instrument_id())
12673 .client_order_id(client_order_id)
12674 .account_id(account_id)
12675 .venue_order_id(venue_order_id)
12676 .quantity(order.quantity())
12677 .price(Price::from("101.00"))
12678 .build(),
12679 ))
12680 } else {
12681 let report = OrderStatusReport::new(
12682 account_id,
12683 order.instrument_id(),
12684 Some(client_order_id),
12685 venue_order_id,
12686 Some(order.order_side()),
12687 order.order_type(),
12688 order.time_in_force(),
12689 OrderStatus::Submitted,
12690 order.quantity(),
12691 Quantity::zero(3),
12692 UnixNanos::default(),
12693 UnixNanos::default(),
12694 UnixNanos::default(),
12695 None,
12696 )
12697 .with_price(Price::from("100.00"));
12698 ExecutionEvent::Report(if evidence == "order" {
12699 ExecutionReport::Order(Box::new(report))
12700 } else {
12701 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
12702 })
12703 };
12704 node.process_exec_event(event);
12705 let expected_status = if evidence == "updated" {
12706 OrderStatus::Submitted
12707 } else if status == OrderStatus::Submitted {
12708 OrderStatus::Accepted
12709 } else {
12710 status
12711 };
12712 assert_eq!(
12713 node.kernel
12714 .cache
12715 .borrow()
12716 .order(&client_order_id)
12717 .unwrap()
12718 .status(),
12719 expected_status,
12720 );
12721
12722 for _ in 0..3 {
12723 advance_clock(Duration::from_millis(101)).await;
12724 let result = node.exec_manager.check_inflight_orders();
12725 assert!(result.queries.is_empty());
12726 assert!(result.events.is_empty());
12727 }
12728 assert_eq!(
12729 node.exec_manager.recon_check_retry_count(&client_order_id),
12730 0
12731 );
12732 assert!(node.exec_manager.check_open_order_queries().is_empty());
12733 advance_clock(Duration::from_secs(5)).await;
12734 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12735 }
12736
12737 #[rstest]
12738 #[case::fill("fill", false)]
12739 #[case::terminal_order("order", true)]
12740 #[case::terminal_order_with_fills("order_with_fills", true)]
12741 #[case::working_order_for_closed_cache("order", false)]
12742 #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
12743 #[cfg_attr(
12744 not(all(feature = "simulation", madsim)),
12745 tokio::test(start_paused = true)
12746 )]
12747 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12748 async fn test_report_cleanup_preserves_original_settling_predicate(
12749 #[case] report_kind: &str,
12750 #[case] terminal_report: bool,
12751 #[values(false, true)] explicit_client_id: bool,
12752 #[values(
12753 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12754 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12755 )]
12756 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12757 ) {
12758 let (mut node, order, instrument) = venue_evidence_node(policy);
12759 let client_order_id = order.client_order_id();
12760 let account_id = AccountId::from("TEST-001");
12761 let client_id = ClientId::from("TEST-EVIDENCE");
12762 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12763 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
12764 &order,
12765 account_id,
12766 venue_order_id,
12767 )));
12768 let accepted = node
12769 .kernel
12770 .cache
12771 .borrow()
12772 .order_owned(&client_order_id)
12773 .unwrap();
12774 let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
12775 .last_qty(accepted.quantity())
12776 .last_px(Price::from("100.00"))
12777 .commission(Money::from("0 USDT"))
12778 .without_position_id()
12779 .build()
12780 else {
12781 unreachable!()
12782 };
12783 let mut fill_report = FillReport::new(
12784 account_id,
12785 instrument.id(),
12786 venue_order_id,
12787 fill.trade_id,
12788 fill.order_side,
12789 fill.last_qty,
12790 fill.last_px,
12791 fill.commission.unwrap(),
12792 fill.liquidity_side,
12793 Some(client_order_id),
12794 None,
12795 fill.ts_event,
12796 fill.ts_init,
12797 None,
12798 );
12799
12800 if !explicit_client_id {
12801 fill_report.client_order_id = None;
12802 }
12803 let event = if report_kind == "fill" {
12804 ExecutionReport::Fill(Box::new(fill_report))
12805 } else {
12806 if !terminal_report {
12807 node.kernel
12808 .cache
12809 .borrow_mut()
12810 .update_order(&OrderEventAny::Filled(fill))
12811 .unwrap();
12812 }
12813 let report = OrderStatusReport::new(
12814 account_id,
12815 instrument.id(),
12816 explicit_client_id.then_some(client_order_id),
12817 venue_order_id,
12818 Some(order.order_side()),
12819 order.order_type(),
12820 order.time_in_force(),
12821 if terminal_report {
12822 OrderStatus::Filled
12823 } else {
12824 OrderStatus::Accepted
12825 },
12826 order.quantity(),
12827 order.quantity(),
12828 UnixNanos::default(),
12829 UnixNanos::default(),
12830 UnixNanos::default(),
12831 None,
12832 )
12833 .with_price(Price::from("100.00"))
12834 .with_avg_px(dec!(100.00));
12835
12836 if report_kind == "order" {
12837 ExecutionReport::Order(Box::new(report))
12838 } else {
12839 ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
12840 }
12841 };
12842 node.process_exec_event(ExecutionEvent::Report(event));
12843 assert_eq!(
12844 node.kernel
12845 .cache
12846 .borrow()
12847 .order(&client_order_id)
12848 .unwrap()
12849 .status(),
12850 OrderStatus::Filled
12851 );
12852
12853 node.kernel
12855 .cache
12856 .borrow_mut()
12857 .add_order(order.clone(), None, Some(client_id), true)
12858 .unwrap();
12859 node.kernel
12860 .cache
12861 .borrow_mut()
12862 .update_order(&TestOrderEventStubs::submitted(&order, account_id))
12863 .unwrap();
12864 node.kernel
12865 .cache
12866 .borrow_mut()
12867 .update_order(&TestOrderEventStubs::accepted(
12868 &order,
12869 account_id,
12870 venue_order_id,
12871 ))
12872 .unwrap();
12873 let should_clear = terminal_report && explicit_client_id;
12874 assert_eq!(
12875 node.exec_manager.check_open_order_queries().len(),
12876 usize::from(should_clear)
12877 );
12878
12879 if !should_clear {
12880 advance_clock(Duration::from_secs(5)).await;
12881 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12882 }
12883 }
12884
12885 #[rstest]
12886 #[case(stub_order_event, 1)]
12887 #[case(stub_submitted_batch_event, 2)]
12888 #[case(stub_accepted_batch_event, 2)]
12889 #[case(stub_canceled_batch_event, 2)]
12890 fn test_pending_events_preserve_roots_when_splitting_batches(
12891 #[values(false, true)] rooted: bool,
12892 #[case] make_event: fn() -> ExecutionEvent,
12893 #[case] orders: usize,
12894 ) {
12895 let runner = AsyncRunner::new();
12896 runner.bind_senders();
12897 let mut channels = runner.take_channels();
12898
12899 let send = move || {
12900 get_data_event_sender().send(stub_data_event()).unwrap();
12901 get_exec_event_sender().send(stub_exec_event()).unwrap();
12902 get_exec_event_sender().send(make_event()).unwrap();
12903 };
12904
12905 if rooted {
12906 msgbus::register_trading_command_endpoint(
12907 MessagingSwitchboard::risk_engine_execute(),
12908 TypedIntoHandler::from(move |_| send()),
12909 );
12910 TradingCommandSender::execute(
12911 &SyncTradingCommandSender,
12912 TradingCommandMessage::new(
12913 MessagingSwitchboard::risk_engine_execute(),
12914 TradingCommand::QueryAccount(QueryAccount::new(
12915 "TRADER-001".into(),
12916 None,
12917 "SIM-001".into(),
12918 UUID4::new(),
12919 37.into(),
12920 None,
12921 None,
12922 )),
12923 ),
12924 );
12925 nautilus_common::runner::drain_trading_cmd_queue();
12926 } else {
12927 send();
12928 }
12929
12930 let mut pending = PendingEvents::default();
12931 pending
12932 .data_evts
12933 .push(channels.data_evt_rx.try_recv().unwrap());
12934 while let Ok(event) = channels.exec_evt_rx.try_recv() {
12935 pending.push_exec_event(event);
12936 }
12937
12938 assert_eq!(pending.data_evts.len(), 1);
12939 assert_eq!(pending.exec_reports.len(), 1);
12940 assert_eq!(pending.order_evts.len(), orders);
12941 assert_eq!(pending.data_evts[0].is_rooted(), rooted);
12942 assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
12943 assert!(
12944 pending
12945 .order_evts
12946 .iter()
12947 .all(|event| event.is_rooted() == rooted)
12948 );
12949
12950 let results = std::thread::spawn(move || {
12951 let mut results = Vec::new();
12952
12953 for event in pending.data_evts {
12954 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12955 || event.dispatch(drop),
12956 )));
12957 }
12958
12959 for event in pending.exec_reports {
12960 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12961 || event.dispatch(drop),
12962 )));
12963 }
12964
12965 for event in pending.order_evts {
12966 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12967 || event.dispatch(drop),
12968 )));
12969 }
12970
12971 results
12972 })
12973 .join()
12974 .unwrap();
12975
12976 assert_eq!(results.len(), orders + 2);
12977
12978 for result in results {
12979 if rooted {
12980 let error = result.unwrap_err();
12981 assert!(
12982 error
12983 .downcast_ref::<String>()
12984 .unwrap()
12985 .contains("command context dispatched outside its owner thread")
12986 );
12987 } else {
12988 result.unwrap();
12989 }
12990 }
12991 }
12992
12993 fn stub_accepted_batch_event() -> ExecutionEvent {
12994 ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
12995 OrderAcceptedSpec::builder()
12996 .client_order_id(ClientOrderId::from("O-017"))
12997 .build(),
12998 OrderAcceptedSpec::builder()
12999 .client_order_id(ClientOrderId::from("O-023"))
13000 .build(),
13001 ]))
13002 }
13003
13004 #[derive(Debug)]
13005 struct CapturedEgressMessage {
13006 topic: String,
13007 payload: Bytes,
13008 }
13009
13010 type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13011 type SharedClosed = Rc<Cell<bool>>;
13012
13013 #[derive(Debug)]
13014 struct CapturingExternalIngress {
13015 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13016 closed: SharedClosed,
13017 }
13018
13019 impl CapturingExternalIngress {
13020 fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13021 Self {
13022 rx: Some(rx),
13023 closed,
13024 }
13025 }
13026 }
13027
13028 impl MessageBusExternalIngress for CapturingExternalIngress {
13029 fn is_closed(&self) -> bool {
13030 self.closed.get()
13031 }
13032
13033 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13034 self.rx
13035 .take()
13036 .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13037 }
13038
13039 fn close(&mut self) {
13040 self.closed.set(true);
13041 }
13042 }
13043
13044 #[derive(Debug)]
13045 struct FailingExternalIngress {
13046 closed: SharedClosed,
13047 }
13048
13049 impl FailingExternalIngress {
13050 fn new(closed: SharedClosed) -> Self {
13051 Self { closed }
13052 }
13053 }
13054
13055 impl MessageBusExternalIngress for FailingExternalIngress {
13056 fn is_closed(&self) -> bool {
13057 self.closed.get()
13058 }
13059
13060 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13061 anyhow::bail!("external ingress receiver unavailable")
13062 }
13063
13064 fn close(&mut self) {
13065 self.closed.set(true);
13066 }
13067 }
13068
13069 struct CapturingExternalEgress {
13070 publications: CapturedEgressMessages,
13071 closed: SharedClosed,
13072 }
13073
13074 impl CapturingExternalEgress {
13075 fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13076 let publications = Rc::new(RefCell::new(Vec::new()));
13077 let closed = Rc::new(Cell::new(false));
13078 (
13079 Self {
13080 publications: publications.clone(),
13081 closed: closed.clone(),
13082 },
13083 publications,
13084 closed,
13085 )
13086 }
13087 }
13088
13089 impl MessageBusExternalEgress for CapturingExternalEgress {
13090 fn is_closed(&self) -> bool {
13091 self.closed.get()
13092 }
13093
13094 fn publish(&self, message: BusMessage) {
13095 self.publications.borrow_mut().push(CapturedEgressMessage {
13096 topic: message.topic.to_string(),
13097 payload: message.payload,
13098 });
13099 }
13100
13101 fn close(&mut self) {
13102 self.closed.set(true);
13103 }
13104 }
13105
13106 struct CapturingBackingFactory {
13107 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13108 closed: Arc<AtomicBool>,
13109 rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13110 }
13111
13112 impl CapturingBackingFactory {
13113 fn new(
13114 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13115 closed: Arc<AtomicBool>,
13116 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13117 ) -> Self {
13118 Self {
13119 publications,
13120 closed,
13121 rx: Mutex::new(rx),
13122 }
13123 }
13124 }
13125
13126 impl Debug for CapturingBackingFactory {
13127 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13128 f.debug_struct(stringify!(CapturingBackingFactory))
13129 .finish_non_exhaustive()
13130 }
13131 }
13132
13133 impl MessageBusBackingFactory for CapturingBackingFactory {
13134 fn create(
13135 &self,
13136 _trader_id: TraderId,
13137 _instance_id: UUID4,
13138 _config: MessageBusConfig,
13139 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13140 let rx = self.rx.lock().take();
13141 Ok(Box::new(CapturingBacking {
13142 publications: self.publications.clone(),
13143 closed: self.closed.clone(),
13144 rx,
13145 }))
13146 }
13147 }
13148
13149 struct CapturingBacking {
13150 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13151 closed: Arc<AtomicBool>,
13152 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13153 }
13154
13155 impl MessageBusBacking for CapturingBacking {
13156 fn is_closed(&self) -> bool {
13157 self.closed.load(Ordering::Relaxed)
13158 }
13159
13160 fn publish(&self, message: BusMessage) {
13161 self.publications.lock().push(CapturedEgressMessage {
13162 topic: message.topic.to_string(),
13163 payload: message.payload,
13164 });
13165 }
13166
13167 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13168 self.rx
13169 .take()
13170 .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13171 }
13172
13173 fn close(&mut self) {
13174 self.closed.store(true, Ordering::Relaxed);
13175 }
13176 }
13177}