1use std::{
86 any::Any,
87 cell::RefCell,
88 collections::VecDeque,
89 fmt::Debug,
90 rc::{Rc, Weak},
91 time::Duration,
92};
93
94use anyhow::Context;
95use nautilus_common::{
96 actor::{self, Actor, DataActor, DataActorNative},
97 cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
98 clients::ExecutionClient,
99 component::Component,
100 enums::{Environment, LogColor},
101 live::{dispatch::DispatchMessage, dst},
102 log_info,
103 messages::{
104 DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
105 data::DataCommand,
106 execution::TradingCommand,
107 system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
108 },
109 msgbus::{self, BusMessage, MessagingSwitchboard},
110 runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
111};
112use nautilus_core::{
113 UUID4,
114 datetime::{mins_to_secs, secs_to_nanos_unchecked},
115};
116use nautilus_execution::client::ExecutionClientAdapter;
117#[cfg(test)]
118use nautilus_model::reports::OrderStatusReport;
119use nautilus_model::{
120 events::OrderEventAny,
121 identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
122 orders::Order,
123};
124use nautilus_network::mode::ReconnectRequestOutcome;
125#[cfg(feature = "python")]
126use nautilus_system::trader::Trader;
127use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
128use nautilus_trading::{
129 ExecutionAlgorithm, ExecutionAlgorithmNative,
130 strategy::{Strategy, StrategyNative},
131};
132use tabled::{builder::Builder, settings::Style};
133
134use crate::{
135 dispatch::drain_callbacks,
136 execution::{
137 client::LiveExecutionClient,
138 manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
139 },
140 runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
141 socket::{SocketReconnectLookup, SocketReconnectRegistry},
142};
143
144pub mod builder;
145pub mod config;
146
147mod metrics;
148mod queue;
149mod reconciliation;
150mod state;
151
152#[cfg(feature = "plugin")]
153pub mod plugin;
154
155use builder::ExternalMessageBusIngress;
156pub use builder::LiveNodeBuilder;
157use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
158pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
159use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
160use queue::{QueueMonitor, QueueStateTransition};
161use reconciliation::{
162 OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
163 PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
164 ReportTaskOutcome, TargetedOrderReportTask,
165};
166use state::{EngineConnectionStatus, RunningTransition};
167pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
168
169const DISPATCHES_PER_YIELD: usize = 64;
175
176thread_local! {
177 static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
178}
179
180type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
181
182struct StreamProcessor(Box<StreamProcessorCallback>);
183
184impl Debug for StreamProcessor {
185 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186 f.debug_struct(stringify!(StreamProcessor)).finish()
187 }
188}
189
190#[derive(Debug)]
199pub struct LiveNode {
200 kernel: NautilusKernel,
201 runner: Option<AsyncRunner>,
202 config: LiveNodeConfig,
203 handle: LiveNodeHandle,
204 exec_manager: ExecutionManager,
205 exec_clients: Vec<LiveExecutionClient>,
206 socket_registry: SocketReconnectRegistry,
207 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
208 external_msgbus: Option<ExternalMessageBusIngress>,
209 stream_processors: Vec<StreamProcessor>,
210 shutdown_deadline: Option<dst::time::Instant>,
211 #[cfg(feature = "plugin")]
212 plugins: plugin::NodePlugins,
213 _thread_owner: Rc<()>,
214}
215
216impl LiveNode {
217 #[must_use]
221 #[allow(
222 clippy::too_many_arguments,
223 reason = "builder components have distinct lifecycle roles"
224 )]
225 pub(crate) fn new_from_builder(
226 kernel: NautilusKernel,
227 runner: AsyncRunner,
228 config: LiveNodeConfig,
229 handle: LiveNodeHandle,
230 mut exec_manager: ExecutionManager,
231 exec_clients: Vec<LiveExecutionClient>,
232 socket_registry: SocketReconnectRegistry,
233 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
234 external_msgbus: Option<ExternalMessageBusIngress>,
235 thread_owner: Rc<()>,
236 ) -> Self {
237 for client in &exec_clients {
238 exec_manager.register_submission_retention(client);
239 }
240
241 Self {
242 kernel,
243 runner: Some(runner),
244 config,
245 handle,
246 exec_manager,
247 exec_clients,
248 socket_registry,
249 cache_database_factory,
250 external_msgbus,
251 stream_processors: Vec::new(),
252 shutdown_deadline: None,
253 #[cfg(feature = "plugin")]
254 plugins: plugin::NodePlugins,
255 _thread_owner: thread_owner,
256 }
257 }
258
259 pub fn builder(
265 trader_id: TraderId,
266 environment: Environment,
267 ) -> anyhow::Result<LiveNodeBuilder> {
268 LiveNodeBuilder::new(trader_id, environment)
269 }
270
271 pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
282 let thread_owner = Self::acquire_thread()?;
283 let config = config.unwrap_or_default();
284 validate_live_environment(config.environment())?;
285
286 config.validate_runtime_support()?;
287
288 if config.event_store.is_some() {
289 anyhow::bail!(
290 "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
291 use LiveNodeBuilder::with_event_store(...) instead"
292 );
293 }
294
295 let handle = LiveNodeHandle::new();
296 let runner = AsyncRunner::new();
297 runner.bind_senders_for_node(handle.clone());
298
299 let kernel = NautilusKernel::new(name, config.clone())?;
300 #[cfg(feature = "python")]
301 if let Some(controller) = config.controller.as_ref() {
302 Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
303 }
304
305 #[cfg(not(feature = "python"))]
306 if let Some(controller) = config.controller.as_ref() {
307 anyhow::bail!(
308 "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
309 controller.controller_path
310 );
311 }
312
313 let exec_manager_config =
314 ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
315
316 let exec_manager = ExecutionManager::new(
317 kernel.clock.clone(),
318 kernel.cache.clone(),
319 exec_manager_config,
320 )?;
321
322 let node = Self {
323 kernel,
324 runner: Some(runner),
325 config,
326 handle,
327 exec_manager,
328 exec_clients: Vec::new(),
329 socket_registry: SocketReconnectRegistry::default(),
330 cache_database_factory: None,
331 external_msgbus: None,
332 stream_processors: Vec::new(),
333 shutdown_deadline: None,
334 #[cfg(feature = "plugin")]
335 plugins: plugin::NodePlugins,
336 _thread_owner: thread_owner,
337 };
338
339 node.load_configured_plugins()?;
340
341 log::info!("LiveNode built successfully with kernel config");
342
343 Ok(node)
344 }
345
346 fn acquire_thread() -> anyhow::Result<Rc<()>> {
347 NODE_THREAD_OWNER.with(|slot| {
348 let mut owner = slot.borrow_mut();
349 anyhow::ensure!(
350 owner.upgrade().is_none(),
351 "A LiveNode already exists or is being built on this thread; drop it before building another"
352 );
353 let token = Rc::new(());
354 *owner = Rc::downgrade(&token);
355 Ok(token)
356 })
357 }
358
359 pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
365 if self.config.plugins.is_empty() {
366 return Ok(());
367 }
368
369 anyhow::bail!(
370 "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
371 )
372 }
373
374 #[expect(
380 clippy::needless_pass_by_value,
381 reason = "signature mirrors the host-enabled API"
382 )]
383 pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
384 #[cfg(feature = "plugin")]
385 config.validate_runtime_support(self.config.plugins.len())?;
386 #[cfg(not(feature = "plugin"))]
387 let _ = config;
388
389 anyhow::bail!(
390 "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
391 )
392 }
393
394 #[must_use]
396 pub fn handle(&self) -> LiveNodeHandle {
397 self.handle.clone()
398 }
399
400 pub fn add_stream_processor<F>(&mut self, callback: F)
417 where
418 F: Fn(&dyn Any) + 'static,
419 {
420 self.add_stream_processor_with_mapping(move |message, _| {
421 callback(message);
422 Ok(())
423 });
424 }
425
426 pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
427 where
428 F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
429 {
430 self.stream_processors
431 .push(StreamProcessor(Box::new(callback)));
432 }
433
434 pub async fn start(&mut self) -> anyhow::Result<()> {
445 if self.state().is_running() {
446 anyhow::bail!("Already running");
447 }
448
449 if self.external_msgbus.is_some() {
450 log::warn!(
451 "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
452 );
453 }
454
455 self.prepare_cache().await?;
456
457 if let Some(runner) = self.runner.as_ref() {
458 runner.bind_senders_for_node(self.handle.clone());
459 }
460
461 self.handle.set_starting();
462
463 self.kernel.reset_shutdown_flag();
464 self.kernel.start_async().await;
465
466 if self.kernel.is_event_store_replay() {
467 log::info!(
468 "Event-store replay loaded; skipping live client connection and reconciliation",
469 );
470
471 if !self.finish_startup_replay().await? {
472 return Ok(());
473 }
474
475 return Ok(());
476 }
477
478 if self.kernel.is_event_store_replay_configured() {
479 self.abort_startup("Event-store replay did not start")
480 .await?;
481 return Ok(());
482 }
483
484 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
485
486 if let Err(e) = self.connect_data_phase(connection_deadline).await {
488 return self
489 .abort_startup_with_error("Data client connection failed", e)
490 .await;
491 }
492
493 let (startup_system_events, startup_system_commands) =
494 if let Some(runner) = self.runner.as_mut() {
495 runner.flush_pending_data();
496 (
497 runner.drain_pending_system_events(),
498 runner.drain_pending_system_commands(),
499 )
500 } else {
501 (Vec::new(), Vec::new())
502 };
503
504 if let Err(e) = self.connect_exec_clients(connection_deadline).await {
505 return self
506 .abort_startup_with_error("Execution client connection failed", e)
507 .await;
508 }
509
510 if let Some(reason) = self.startup_abort_reason() {
511 self.abort_startup(reason).await?;
512 return Ok(());
513 }
514
515 match self.await_engines_connected(connection_deadline).await {
516 EngineConnectionStatus::Connected => {}
517 EngineConnectionStatus::TimedOut => {
518 return self
519 .abort_startup_with_error(
520 "Engine readiness timed out",
521 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
522 )
523 .await;
524 }
525 EngineConnectionStatus::StopRequested => {
526 self.abort_startup("Stop signal received during startup")
527 .await?;
528 return Ok(());
529 }
530 EngineConnectionStatus::ShutdownRequested => {
531 self.abort_startup("Shutdown signal received during startup")
532 .await?;
533 return Ok(());
534 }
535 }
536
537 if let Err(e) = self.perform_startup_reconciliation().await {
538 if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
539 anyhow::bail!(
540 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
541 );
542 }
543
544 return Err(e);
545 }
546
547 if let Some(reason) = self.startup_abort_reason() {
548 self.abort_startup(reason).await?;
549 return Ok(());
550 }
551
552 if let Err(e) = self.kernel.start_trader() {
553 return self.abort_after_trader_start_failure(e).await;
554 }
555
556 #[cfg(feature = "plugin")]
557 if let Err(e) = self.plugins.start_controllers() {
558 return self.abort_after_trader_start_failure(e).await;
559 }
560
561 self.process_system_events(startup_system_events);
562 self.process_system_commands(startup_system_commands);
563
564 if !self.finish_startup_trader(None).await? {
565 return Ok(());
566 }
567
568 Ok(())
569 }
570
571 pub async fn stop(&mut self) -> anyhow::Result<()> {
581 if !self.state().is_running() {
582 anyhow::bail!("Not running");
583 }
584
585 self.handle.set_shutting_down();
586
587 #[cfg(feature = "plugin")]
588 let controller_stop_result = self.plugins.stop_controllers();
589 #[cfg(not(feature = "plugin"))]
590 let controller_stop_result: anyhow::Result<()> = Ok(());
591
592 self.kernel.stop_trader();
593 let delay = self.kernel.delay_post_stop();
594 log::info!("Awaiting residual events ({delay:?})...");
595
596 let residual_events = self.process_runner_for(delay).await;
597 if residual_events > 0 {
598 log::debug!("Processed {residual_events} residual events during shutdown");
599 }
600
601 let stop_result = self.finalize_stop().await;
602 self.drain_runner_pending();
603
604 match (controller_stop_result, stop_result) {
605 (Ok(()), Ok(())) => Ok(()),
606 (Err(controller_err), Ok(())) => Err(controller_err),
607 (Ok(()), Err(stop_err)) => Err(stop_err),
608 (Err(controller_err), Err(stop_err)) => {
609 log::error!("Error stopping plug-in controllers: {controller_err}");
610 Err(stop_err)
611 }
612 }
613 }
614
615 pub fn dispose(&mut self) {
620 self.close_external_ingress();
621 self.handle.set_stopped();
622 self.kernel.dispose();
623 drop(self.runner.take());
624
625 if let Some(e) = actor::callback_failure() {
626 log::error!("Callback dispatch failed before disposal cleanup: {e}");
627 }
628
629 if let Err(e) = actor::clear_callbacks() {
630 log::error!("Failed to clear callback dispatch during disposal: {e}");
631 }
632 }
633
634 async fn process_runner_for(&mut self, duration: Duration) -> usize {
635 let Some(mut runner) = self.runner.take() else {
636 dst::time::sleep(duration).await;
637 return 0;
638 };
639
640 runner.bind_senders_for_node(self.handle.clone());
641 let deadline = dst::time::Instant::now() + duration;
642 let mut processed = 0;
643
644 loop {
645 tokio::select! {
646 biased;
647
648 () = dst::time::sleep_until(deadline) => break,
649 event = runner.recv() => {
650 let Some(event) = event else {
651 dst::time::sleep_until(deadline).await;
652 break;
653 };
654
655 self.process_runner_event(event);
656 processed += 1;
657 }
658 }
659 }
660
661 self.runner = Some(runner);
662 processed
663 }
664
665 fn drain_runner_pending(&mut self) -> usize {
666 let Some(mut runner) = self.runner.take() else {
667 return 0;
668 };
669
670 let processed = runner.poll_pending(|event| self.process_runner_event(event));
671 self.runner = Some(runner);
672 processed
673 }
674
675 fn process_runner_event(&mut self, event: PendingRunnerEvent) {
676 match event {
677 PendingRunnerEvent::TimeEvent(message) => {
678 let _ = AsyncRunner::handle_time_event(message);
679 }
680 PendingRunnerEvent::SystemEvent(event) => {
681 event.dispatch(|event| self.process_system_event(event));
682 }
683 PendingRunnerEvent::SystemCommand(command) => {
684 command.dispatch(|command| self.process_system_command(command));
685 }
686 PendingRunnerEvent::ExecEvent(event) => {
687 event.dispatch(|event| self.process_exec_event(event));
688 }
689 PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
690 PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
691 PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
692 }
693 }
694
695 fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
696 for event in events {
697 event.dispatch(|event| self.process_system_event(event));
698 }
699 }
700
701 fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
702 for command in commands {
703 command.dispatch(|command| self.process_system_command(command));
704 }
705 }
706
707 fn process_system_command(&self, command: SystemCommand) {
708 match command {
709 SystemCommand::ReconnectSocket(command) => {
710 self.process_socket_reconnect(command);
711 }
712 }
713 }
714
715 fn process_socket_reconnect(&self, command: ReconnectSocket) {
716 let outcome = if command.trader_id == self.config.trader_id {
717 Self::request_socket_reconnect(
718 self.socket_registry
719 .get(command.client_id, command.endpoint),
720 )
721 } else {
722 SocketReconnectDispatchOutcome::InvalidTrader
723 };
724
725 if outcome == SocketReconnectDispatchOutcome::Accepted {
726 log::info!(
727 "Requested socket reconnect for client {} endpoint {}",
728 command.client_id,
729 command.endpoint
730 );
731 } else {
732 log::warn!(
733 "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
734 command.client_id,
735 command.endpoint
736 );
737 }
738 }
739
740 fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
741 match lookup {
742 SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
743 ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
744 ReconnectRequestOutcome::AlreadyReconnecting => {
745 SocketReconnectDispatchOutcome::AlreadyReconnecting
746 }
747 ReconnectRequestOutcome::Disconnected => {
748 SocketReconnectDispatchOutcome::Disconnected
749 }
750 ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
751 ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
752 },
753 SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
754 SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
755 SocketReconnectLookup::EndpointNotFound => {
756 SocketReconnectDispatchOutcome::UnknownEndpoint
757 }
758 SocketReconnectLookup::AmbiguousEndpoint => {
759 SocketReconnectDispatchOutcome::AmbiguousEndpoint
760 }
761 }
762 }
763
764 fn process_system_event(&self, event: SystemEvent) {
765 match event {
766 SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
767 }
768 }
769
770 fn publish_socket_state_change(&self, change: SocketStateChange) {
771 let timestamp = self.kernel.generate_timestamp_ns();
772
773 let event = SocketStateChanged::new(
774 self.config.trader_id,
775 change.client_id,
776 change.venue,
777 change.endpoint,
778 change.state,
779 UUID4::new(),
780 timestamp,
781 timestamp,
782 );
783
784 msgbus::publish_any(
785 MessagingSwitchboard::socket_state_changed_topic(
786 event.client_id,
787 event.endpoint.as_str(),
788 ),
789 event.as_any(),
790 );
791 }
792
793 async fn await_engines_connected(
797 &self,
798 deadline: dst::time::Instant,
799 ) -> EngineConnectionStatus {
800 log::info!(
801 "Awaiting engine connections ({:?} timeout)...",
802 self.config.timeout_connection
803 );
804
805 let interval = Duration::from_millis(100);
806
807 loop {
808 if self.handle.should_stop() {
809 log::warn!("Stop signal received, aborting connection wait");
810 return EngineConnectionStatus::StopRequested;
811 }
812
813 if self.kernel.is_shutdown_requested() {
814 log::warn!("Shutdown signal received, aborting connection wait");
815 return EngineConnectionStatus::ShutdownRequested;
816 }
817
818 if self.kernel.check_engines_connected() {
819 log::info!("All engine clients connected");
820 return EngineConnectionStatus::Connected;
821 }
822
823 let now = dst::time::Instant::now();
824 if now >= deadline {
825 break;
826 }
827
828 dst::time::sleep(interval.min(deadline - now)).await;
829 }
830
831 self.log_connection_status();
832 EngineConnectionStatus::TimedOut
833 }
834
835 async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
839 log::info!(
840 "Awaiting engine disconnections ({:?} timeout)...",
841 self.config.timeout_disconnection
842 );
843
844 let timeout = self.config.timeout_disconnection;
845 let interval = Duration::from_millis(100);
846
847 loop {
848 if self.kernel.check_engines_disconnected() {
849 log::info!("All engine clients disconnected");
850 return Ok(());
851 }
852
853 let now = dst::time::Instant::now();
854 if now >= deadline {
855 break;
856 }
857
858 dst::time::sleep(interval.min(deadline - now)).await;
859 }
860
861 log::error!(
862 "Timed out ({:?}) waiting for engines to disconnect\n\
863 DataEngine.check_disconnected() == {}\n\
864 ExecEngine.check_disconnected() == {}",
865 timeout,
866 self.kernel.data_engine().check_disconnected(),
867 self.kernel.exec_engine().borrow().check_disconnected(),
868 );
869 anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
870 }
871
872 fn log_connection_status(&self) {
873 let data_status = self.kernel.data_client_connection_status();
874 let exec_status = self.kernel.exec_client_connection_status();
875
876 let mut rows: Vec<ClientStatus> = Vec::new();
877
878 for (client_id, connected) in data_status {
879 rows.push(ClientStatus {
880 client: client_id.to_string(),
881 client_type: "Data",
882 connected,
883 });
884 }
885
886 for (client_id, connected) in exec_status {
887 rows.push(ClientStatus {
888 client: client_id.to_string(),
889 client_type: "Execution",
890 connected,
891 });
892 }
893
894 let table = render_client_statuses(rows);
895
896 log::warn!(
897 "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
898 DataEngine.check_connected() == {}\n\
899 ExecEngine.check_connected() == {}",
900 self.config.timeout_connection,
901 self.kernel.data_engine().check_connected(),
902 self.kernel.exec_engine().borrow().check_connected(),
903 );
904 }
905
906 async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
915 if !self.config.exec_engine.reconciliation {
916 log::info!("Startup reconciliation disabled");
917 self.kernel
918 .portfolio
919 .borrow_mut()
920 .initialize_wallet_orders()?;
921 return Ok(());
922 }
923
924 log_info!(
925 "Starting execution state reconciliation...",
926 color = LogColor::Blue
927 );
928
929 let lookback_mins = self
930 .config
931 .exec_engine
932 .reconciliation_lookback_mins
933 .map(u64::from);
934
935 let timeout = self.config.timeout_reconciliation;
936 let start = dst::time::Instant::now();
937 let client_ids = self.kernel.exec_engine.borrow().client_ids();
938
939 for client_id in client_ids {
940 let elapsed = start.elapsed();
941 if elapsed >= timeout {
942 anyhow::bail!("Startup reconciliation timeout reached");
943 }
944
945 let remaining = timeout
946 .checked_sub(elapsed)
947 .expect("elapsed checked against reconciliation timeout");
948
949 log_info!(
950 "Requesting mass status from {}...",
951 client_id,
952 color = LogColor::Blue
953 );
954
955 let client = self
956 .exec_clients
957 .iter()
958 .find(|client| client.client_id() == client_id)
959 .ok_or_else(|| anyhow::anyhow!("Execution client {client_id} has no live handle"))?
960 .clone();
961 let client = Box::new(client);
962 let client = ExecutionClientAdapter::new(client);
963
964 let mass_status_query = client.generate_mass_status(lookback_mins);
965
966 let mass_status_result = dst::time::timeout(remaining, mass_status_query)
967 .await
968 .map_err(|_| {
969 anyhow::anyhow!(
970 "Startup reconciliation timeout reached while requesting mass status from {client_id}"
971 )
972 })?;
973
974 match mass_status_result {
975 Ok(Some(mass_status)) => {
976 log_info!(
977 "Reconciling ExecutionMassStatus for {}",
978 client_id,
979 color = LogColor::Blue
980 );
981
982 let result = self
983 .exec_manager
984 .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
985
986 anyhow::ensure!(
987 self.kernel
988 .exec_engine
989 .borrow()
990 .get_client(&client_id)
991 .is_some(),
992 "Execution client {client_id} disappeared during startup reconciliation",
993 );
994
995 anyhow::ensure!(
996 result.unresolved_positions.is_empty(),
997 "Unresolved positions during startup reconciliation for {client_id}: {}",
998 result.unresolved_positions.join("; "),
999 );
1000
1001 if result.events.is_empty() {
1002 log_info!(
1003 "Reconciliation for {} succeeded",
1004 client_id,
1005 color = LogColor::Blue
1006 );
1007 } else {
1008 log::info!(
1009 color = LogColor::Blue as u8;
1010 "Reconciliation for {} processed {} events",
1011 client_id,
1012 result.events.len()
1013 );
1014 }
1015
1016 if !result.external_orders.is_empty() {
1018 let exec_engine = self.kernel.exec_engine.borrow();
1019
1020 let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1021 anyhow::anyhow!(
1022 "Execution client {client_id} disappeared during startup reconciliation"
1023 )
1024 })?;
1025
1026 for external in result.external_orders {
1027 source_client.register_external_order(
1028 external.client_order_id,
1029 external.venue_order_id,
1030 external.instrument_id,
1031 external.strategy_id,
1032 external.ts_init,
1033 );
1034 }
1035 }
1036 }
1037 Ok(None) => {
1038 log::warn!(
1039 "No mass status available from {client_id} \
1040 (likely adapter error when generating reports)"
1041 );
1042 }
1043 Err(e) => {
1044 return Err(e).context(format!("Failed to get mass status from {client_id}"));
1045 }
1046 }
1047 }
1048
1049 self.kernel.portfolio.borrow_mut().initialize_orders();
1050 self.kernel.portfolio.borrow_mut().initialize_positions();
1051 self.kernel
1052 .portfolio
1053 .borrow_mut()
1054 .initialize_wallet_orders()?;
1055
1056 let elapsed_secs = start.elapsed().as_secs_f64();
1057 log_info!(
1058 "Startup reconciliation completed in {:.2}s",
1059 elapsed_secs,
1060 color = LogColor::Blue
1061 );
1062
1063 Ok(())
1064 }
1065
1066 pub async fn run(&mut self) -> anyhow::Result<()> {
1089 self.run_with_mode(NodeRunMode::Owned).await
1090 }
1091
1092 pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1103 if self.state().is_running() {
1104 anyhow::bail!("Already running");
1105 }
1106
1107 if self.runner.is_none() {
1108 anyhow::bail!("Runner already consumed - run() called twice");
1109 }
1110
1111 self.prepare_cache().await?;
1112
1113 let Some(runner) = self.runner.take() else {
1114 anyhow::bail!("Runner already consumed - run() called twice");
1115 };
1116
1117 runner.bind_senders_for_node(self.handle.clone());
1118
1119 let AsyncRunnerChannels {
1120 mut time_evt_rx,
1121 mut system_evt_rx,
1122 mut system_cmd_rx,
1123 mut exec_evt_rx,
1124 mut exec_cmd_rx,
1125 mut data_evt_rx,
1126 mut data_cmd_rx,
1127 } = runner.take_channels();
1128
1129 log::info!("Event loop starting");
1130
1131 self.handle.set_starting();
1132 self.kernel.reset_shutdown_flag();
1133 self.kernel.start_async().await;
1134
1135 if self.kernel.is_event_store_replay() {
1136 log::info!(
1137 "Event-store replay loaded; skipping live client connection and reconciliation",
1138 );
1139
1140 if !self.finish_startup_replay().await? {
1141 return Ok(());
1142 }
1143
1144 return Ok(());
1145 }
1146
1147 if self.kernel.is_event_store_replay_configured() {
1148 self.abort_startup("Event-store replay did not start")
1149 .await?;
1150 return Ok(());
1151 }
1152
1153 let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1154 Ok(rx) => rx,
1155 Err(e) => {
1156 let result = self
1157 .abort_startup("External message bus ingress failed to start")
1158 .await;
1159 Self::drain_channels(
1160 &mut time_evt_rx,
1161 &mut system_evt_rx,
1162 &mut system_cmd_rx,
1163 &mut exec_evt_rx,
1164 &mut exec_cmd_rx,
1165 &mut data_evt_rx,
1166 &mut data_cmd_rx,
1167 );
1168 log::info!("Event loop stopped");
1169
1170 if let Err(finalize_err) = result {
1171 anyhow::bail!(
1172 "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1173 );
1174 }
1175
1176 return Err(e);
1177 }
1178 };
1179
1180 let stop_handle = self.handle.clone();
1181 let mut pending = PendingEvents::default();
1182 let mut startup_system_events = Vec::new();
1183 let mut startup_system_commands = Vec::new();
1184 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1185
1186 let mut pending_external = VecDeque::new();
1189 let data_connect_result = drive_with_event_buffering(
1190 drive_with_external_ingress(
1191 self.connect_data_phase(connection_deadline),
1192 &mut external_msgbus_rx,
1193 &mut pending_external,
1194 ),
1195 &mut pending,
1196 &mut time_evt_rx,
1197 &mut system_evt_rx,
1198 &mut system_cmd_rx,
1199 &mut exec_evt_rx,
1200 &mut exec_cmd_rx,
1201 &mut data_evt_rx,
1202 &mut data_cmd_rx,
1203 )
1204 .await;
1205
1206 self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1207
1208 if let Err(e) = data_connect_result {
1209 flush_all_pending(
1210 &mut pending,
1211 &mut time_evt_rx,
1212 &mut system_evt_rx,
1213 &mut system_cmd_rx,
1214 &mut exec_evt_rx,
1215 &mut exec_cmd_rx,
1216 &mut data_evt_rx,
1217 &mut data_cmd_rx,
1218 );
1219 let result = self
1220 .abort_startup_with_error("Data client connection failed", e)
1221 .await;
1222 Self::drain_channels(
1223 &mut time_evt_rx,
1224 &mut system_evt_rx,
1225 &mut system_cmd_rx,
1226 &mut exec_evt_rx,
1227 &mut exec_cmd_rx,
1228 &mut data_evt_rx,
1229 &mut data_cmd_rx,
1230 );
1231 log::info!("Event loop stopped");
1232 return result;
1233 }
1234
1235 flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1239 startup_system_events.extend(pending.take_system_events());
1240 startup_system_commands.extend(pending.take_system_commands());
1241 debug_assert!(
1242 pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1243 "data must be drained into cache before exec clients connect",
1244 );
1245
1246 let engine_connection_result = drive_with_event_buffering(
1248 self.connect_exec_phase(connection_deadline),
1249 &mut pending,
1250 &mut time_evt_rx,
1251 &mut system_evt_rx,
1252 &mut system_cmd_rx,
1253 &mut exec_evt_rx,
1254 &mut exec_cmd_rx,
1255 &mut data_evt_rx,
1256 &mut data_cmd_rx,
1257 )
1258 .await;
1259
1260 flush_all_pending(
1262 &mut pending,
1263 &mut time_evt_rx,
1264 &mut system_evt_rx,
1265 &mut system_cmd_rx,
1266 &mut exec_evt_rx,
1267 &mut exec_cmd_rx,
1268 &mut data_evt_rx,
1269 &mut data_cmd_rx,
1270 );
1271 startup_system_events.extend(pending.take_system_events());
1272 startup_system_commands.extend(pending.take_system_commands());
1273 debug_assert!(
1274 pending.is_empty(),
1275 "all startup events must be processed before reconciliation",
1276 );
1277
1278 let engine_connection_status = match engine_connection_result {
1279 Ok(status) => status,
1280 Err(e) => {
1281 let result = self
1282 .abort_startup_with_error("Execution client connection failed", e)
1283 .await;
1284 Self::drain_channels(
1285 &mut time_evt_rx,
1286 &mut system_evt_rx,
1287 &mut system_cmd_rx,
1288 &mut exec_evt_rx,
1289 &mut exec_cmd_rx,
1290 &mut data_evt_rx,
1291 &mut data_cmd_rx,
1292 );
1293 log::info!("Event loop stopped");
1294 return result;
1295 }
1296 };
1297
1298 if engine_connection_status == EngineConnectionStatus::TimedOut {
1299 let result = self
1300 .abort_startup_with_error(
1301 "Engine readiness timed out",
1302 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1303 )
1304 .await;
1305 Self::drain_channels(
1306 &mut time_evt_rx,
1307 &mut system_evt_rx,
1308 &mut system_cmd_rx,
1309 &mut exec_evt_rx,
1310 &mut exec_cmd_rx,
1311 &mut data_evt_rx,
1312 &mut data_cmd_rx,
1313 );
1314 log::info!("Event loop stopped");
1315 return result;
1316 }
1317
1318 if let Some(reason) = engine_connection_status
1319 .abort_reason()
1320 .or_else(|| self.startup_abort_reason())
1321 {
1322 self.abort_startup(reason).await?;
1323 Self::drain_channels(
1324 &mut time_evt_rx,
1325 &mut system_evt_rx,
1326 &mut system_cmd_rx,
1327 &mut exec_evt_rx,
1328 &mut exec_cmd_rx,
1329 &mut data_evt_rx,
1330 &mut data_cmd_rx,
1331 );
1332 log::info!("Event loop stopped");
1333 return Ok(());
1334 }
1335
1336 debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1337
1338 if let Err(e) = self.perform_startup_reconciliation().await {
1340 let result = self.abort_startup("Startup reconciliation failed").await;
1341 Self::drain_channels(
1342 &mut time_evt_rx,
1343 &mut system_evt_rx,
1344 &mut system_cmd_rx,
1345 &mut exec_evt_rx,
1346 &mut exec_cmd_rx,
1347 &mut data_evt_rx,
1348 &mut data_cmd_rx,
1349 );
1350 log::info!("Event loop stopped");
1351
1352 if let Err(finalize_err) = result {
1353 anyhow::bail!(
1354 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1355 );
1356 }
1357
1358 return Err(e);
1359 }
1360
1361 if let Some(reason) = self.startup_abort_reason() {
1362 let result = self.abort_startup(reason).await;
1363 Self::drain_channels(
1364 &mut time_evt_rx,
1365 &mut system_evt_rx,
1366 &mut system_cmd_rx,
1367 &mut exec_evt_rx,
1368 &mut exec_cmd_rx,
1369 &mut data_evt_rx,
1370 &mut data_cmd_rx,
1371 );
1372 log::info!("Event loop stopped");
1373 return result;
1374 }
1375
1376 if let Err(e) = self.kernel.start_trader() {
1377 let result = self.abort_after_trader_start_failure(e).await;
1378 Self::drain_channels(
1379 &mut time_evt_rx,
1380 &mut system_evt_rx,
1381 &mut system_cmd_rx,
1382 &mut exec_evt_rx,
1383 &mut exec_cmd_rx,
1384 &mut data_evt_rx,
1385 &mut data_cmd_rx,
1386 );
1387 log::info!("Event loop stopped");
1388 return result;
1389 }
1390
1391 #[cfg(feature = "plugin")]
1392 if let Err(e) = self.plugins.start_controllers() {
1393 let result = self.abort_after_trader_start_failure(e).await;
1394 Self::drain_channels(
1395 &mut time_evt_rx,
1396 &mut system_evt_rx,
1397 &mut system_cmd_rx,
1398 &mut exec_evt_rx,
1399 &mut exec_cmd_rx,
1400 &mut data_evt_rx,
1401 &mut data_cmd_rx,
1402 );
1403 log::info!("Event loop stopped");
1404 return result;
1405 }
1406
1407 self.process_system_events(startup_system_events);
1408 self.process_system_commands(startup_system_commands);
1409
1410 let finish_result = {
1411 let mut receivers = RunnerReceivers {
1412 time_evt: &mut time_evt_rx,
1413 system_evt: &mut system_evt_rx,
1414 system_cmd: &mut system_cmd_rx,
1415 data_evt: &mut data_evt_rx,
1416 data_cmd: &mut data_cmd_rx,
1417 exec_evt: &mut exec_evt_rx,
1418 exec_cmd: &mut exec_cmd_rx,
1419 };
1420
1421 self.finish_startup_trader(Some(&mut receivers)).await
1422 };
1423
1424 match finish_result {
1425 Ok(true) => {}
1426 result => {
1427 log::info!("Event loop stopped");
1428 return result.map(|_| ());
1429 }
1430 }
1431
1432 let exec_config = &self.config.exec_engine;
1433 let inflight_interval =
1434 Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1435
1436 let open_interval = exec_config
1437 .open_check_interval_secs
1438 .filter(|&s| s > 0.0)
1439 .map_or(Duration::ZERO, |secs| {
1440 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1441 });
1442
1443 let position_interval = exec_config
1444 .position_check_interval_secs
1445 .filter(|&s| s > 0.0)
1446 .map_or(Duration::ZERO, |secs| {
1447 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1448 });
1449
1450 let has_clients = !self
1451 .kernel
1452 .exec_engine
1453 .borrow()
1454 .get_all_clients()
1455 .is_empty();
1456 let recon_enabled = has_clients
1457 && (!inflight_interval.is_zero()
1458 || !open_interval.is_zero()
1459 || !position_interval.is_zero());
1460
1461 let recon_min_interval = if recon_enabled {
1462 let mut intervals = Vec::new();
1463
1464 if !inflight_interval.is_zero() {
1465 intervals.push(inflight_interval);
1466 }
1467
1468 if !open_interval.is_zero() {
1469 intervals.push(open_interval);
1470 }
1471
1472 if !position_interval.is_zero() {
1473 intervals.push(position_interval);
1474 }
1475
1476 intervals
1477 .into_iter()
1478 .min()
1479 .unwrap_or(Duration::from_secs(1))
1480 } else {
1481 Duration::from_secs(1) };
1483
1484 let startup_delay = if self.config.exec_engine.reconciliation {
1489 Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1490 } else {
1491 Duration::ZERO
1492 };
1493
1494 let recon_start = dst::time::Instant::now() + startup_delay;
1495
1496 let mut last_inflight_check = dst::time::Instant::now();
1497 let mut last_open_check = last_inflight_check;
1498 let mut last_position_check = last_inflight_check;
1499
1500 let far_future = Duration::from_hours(24 * 365 * 100);
1503
1504 let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1505 let dur = opt_dur.unwrap_or(far_future);
1506 (dur, recon_start + dur)
1507 };
1508
1509 let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1510 Some(recon_min_interval)
1511 } else {
1512 None
1513 });
1514
1515 let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1516 exec_config
1517 .purge_closed_orders_interval_mins
1518 .filter(|&m| m > 0)
1519 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1520 );
1521
1522 let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1523 exec_config
1524 .purge_closed_positions_interval_mins
1525 .filter(|&m| m > 0)
1526 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1527 );
1528
1529 let (purge_account_interval, mut purge_account_next) = make_schedule(
1530 exec_config
1531 .purge_account_events_interval_mins
1532 .filter(|&m| m > 0)
1533 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1534 );
1535
1536 let (own_books_interval, mut own_books_next) = make_schedule(
1537 exec_config
1538 .own_books_audit_interval_secs
1539 .filter(|&s| s > 0.0)
1540 .map(Duration::from_secs_f64),
1541 );
1542
1543 let (prune_fills_interval, mut prune_fills_next) =
1544 make_schedule(Some(Duration::from_mins(1)));
1545
1546 let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1547 maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1548
1549 let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1554 stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1555
1556 let mut residual_events = 0usize;
1558 let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1559 let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1560 let mut position_report_task: Option<PositionReportTask> = None;
1561
1562 let owns_signals = mode.owns_signals();
1565
1566 let ctrl_c = async move {
1567 if owns_signals {
1568 dst::signal::ctrl_c().await
1569 } else {
1570 std::future::pending::<std::io::Result<()>>().await
1571 }
1572 };
1573
1574 let terminate = async move {
1575 if owns_signals {
1576 dst::signal::terminate().await
1577 } else {
1578 std::future::pending::<std::io::Result<()>>().await
1579 }
1580 };
1581
1582 tokio::pin!(ctrl_c);
1583 tokio::pin!(terminate);
1584
1585 let metrics = self.handle.metrics.clone();
1586 let metrics_start = dst::time::Instant::now();
1587 metrics.reset();
1588
1589 let mut queue_monitor = self
1590 .config
1591 .queue_monitor
1592 .as_ref()
1593 .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1594 let mut dispatches_since_yield = 0usize;
1595 let mut skip_time = false;
1596 let mut skip_data = false;
1597
1598 let dispatch_result = loop {
1599 let callbacks_pending = match drain_callbacks().await {
1600 Ok(pending) => pending,
1601 Err(e) => {
1602 if self.state() == NodeState::Running {
1603 self.initiate_shutdown();
1604 }
1605
1606 log::warn!(
1607 "Skipping residual events and final buffered dispatch after callback failure"
1608 );
1609
1610 break Err(e);
1611 }
1612 };
1613
1614 let shutdown_deadline = self.shutdown_deadline;
1615 let is_shutting_down = self.state() == NodeState::ShuttingDown;
1616 let is_running = self.state() == NodeState::Running;
1617 let commands_empty = data_cmd_rx.is_empty();
1618 let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1619 let poll_data = !skip_data || commands_empty;
1620
1621 tokio::select! {
1622 biased;
1623
1624 result = &mut ctrl_c, if is_running => {
1626 match result {
1627 Ok(()) => log::info!("Received SIGINT, shutting down"),
1628 Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1629 }
1630 self.initiate_shutdown();
1631 }
1632 result = &mut terminate, if is_running => {
1633 match result {
1634 Ok(()) => log::info!("Received SIGTERM, shutting down"),
1635 Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1636 }
1637 self.initiate_shutdown();
1638 }
1639 _ = stop_check_timer.tick(), if is_running => {
1640 if stop_handle.should_stop() {
1641 log::info!("Received stop signal from handle");
1642 self.initiate_shutdown();
1643 } else if self.kernel.is_shutdown_requested() {
1644 log::info!("Received ShutdownSystem command, shutting down");
1645 self.initiate_shutdown();
1646 }
1647 }
1648 () = async {
1649 match shutdown_deadline {
1650 Some(deadline) => dst::time::sleep_until(deadline).await,
1651 None => std::future::pending::<()>().await,
1652 }
1653 }, if self.state() == NodeState::ShuttingDown => {
1654 break Ok(());
1655 }
1656 () = std::future::ready(()), if callbacks_pending => {},
1657 result = async {
1658 match open_order_report_task.as_mut() {
1659 Some(task) => task.future.as_mut().await,
1660 None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1661 }
1662 }, if open_order_report_task.is_some() => {
1663 let maintenance_start = dst::time::Instant::now();
1664
1665 drop(open_order_report_task.take());
1666
1667 match result {
1668 ReportTaskOutcome::Completed(result) => {
1669 let result = result.finish();
1670 self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1671 let client_refs = self
1672 .exec_clients
1673 .iter()
1674 .map(|client| client as &dyn ExecutionClient)
1675 .collect::<Vec<_>>();
1676 let reconciliation = self.exec_manager.reconcile_open_order_reports(
1677 &result.check,
1678 result.reports,
1679 &result.queried_clients,
1680 &result.failed_clients,
1681 &client_refs,
1682 );
1683 self.process_reconciliation_events(&reconciliation.events);
1684 if !reconciliation.targeted_queries.is_empty() {
1685 if is_shutting_down {
1686 let planned_client_order_ids = reconciliation
1687 .targeted_queries
1688 .iter()
1689 .map(TargetedOrderQuery::client_order_id)
1690 .collect::<Vec<_>>();
1691 self.cleanup_cancelled_report_tasks(
1692 &planned_client_order_ids,
1693 );
1694 } else {
1695 targeted_order_report_task = Some(
1696 self.start_targeted_order_report_check(
1697 reconciliation.targeted_queries,
1698 ),
1699 );
1700 }
1701 }
1702 }
1703 ReportTaskOutcome::TimedOut => {
1704 self.cleanup_cancelled_report_tasks(&[]);
1705 log::warn!(
1706 "Open-order report collection expired after {:?}",
1707 self.config.timeout_reconciliation,
1708 );
1709 }
1710 }
1711 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1712 }
1713 result = async {
1714 match targeted_order_report_task.as_mut() {
1715 Some(task) => task.future.as_mut().await,
1716 None => std::future::pending::<ReportTaskOutcome<()>>().await,
1717 }
1718 }, if targeted_order_report_task.is_some() => {
1719 let maintenance_start = dst::time::Instant::now();
1720
1721 if let Some(task) = targeted_order_report_task.take() {
1722 self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1723 self.complete_targeted_order_report_check(task, &result);
1724 }
1725 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1726 }
1727 result = async {
1728 match position_report_task.as_mut() {
1729 Some(task) => task.future.as_mut().await,
1730 None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1731 }
1732 }, if position_report_task.is_some() => {
1733 let maintenance_start = dst::time::Instant::now();
1734
1735 drop(position_report_task.take());
1736
1737 match result {
1738 ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1739 if is_shutting_down {
1740 self.cleanup_cancelled_report_tasks(&[]);
1741 } else {
1742 let result = result.finish();
1743 position_report_task = self.handle_position_report_result(result);
1744 }
1745 }
1746 ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1747 let reports = collection.finish();
1748 self.handle_position_fill_report_result(PositionFillReportResult {
1749 position_result,
1750 reports,
1751 });
1752 }
1753 ReportTaskOutcome::TimedOut => {
1754 self.cleanup_cancelled_report_tasks(&[]);
1755 log::warn!(
1756 "Position report collection expired after {:?}",
1757 self.config.timeout_reconciliation,
1758 );
1759 }
1760 }
1761 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1762 }
1763
1764 _ = maintenance_timer.tick(), if is_running => {
1767 let maintenance_start = dst::time::Instant::now();
1768 metrics.publish_queue_depths(
1769 RunnerChannelQueueDepths::from_receivers(
1770 &time_evt_rx,
1771 &exec_evt_rx,
1772 &exec_cmd_rx,
1773 &data_evt_rx,
1774 &data_cmd_rx,
1775 ),
1776 metrics_start.elapsed(),
1777 );
1778
1779 if let Some(queue_monitor) = queue_monitor.as_mut() {
1780 let transitions = queue_monitor.evaluate(metrics.snapshot());
1781 self.publish_queue_state_transitions(&transitions);
1782 }
1783
1784 let mut now = dst::time::Instant::now();
1785
1786 if recon_enabled && now >= recon_next {
1787 let recon_intervals = ReconciliationCheckIntervals {
1788 inflight: inflight_interval,
1789 open: open_interval,
1790 position: position_interval,
1791 };
1792 let mut recon_state = ReconciliationCheckState {
1793 last_inflight_check: &mut last_inflight_check,
1794 last_open_check: &mut last_open_check,
1795 last_position_check: &mut last_position_check,
1796 open_order_report_task: &mut open_order_report_task,
1797 targeted_order_report_task: &mut targeted_order_report_task,
1798 position_report_task: &mut position_report_task,
1799 };
1800
1801 self.run_reconciliation_checks(
1802 now,
1803 recon_intervals,
1804 &mut recon_state,
1805 );
1806
1807 now = dst::time::Instant::now();
1808 recon_next = now + recon_interval;
1809 }
1810
1811 if now >= purge_orders_next {
1812 self.exec_manager.purge_closed_orders();
1813 purge_orders_next = now + purge_orders_interval;
1814 }
1815
1816 if now >= purge_positions_next {
1817 self.exec_manager.purge_closed_positions();
1818 purge_positions_next = now + purge_positions_interval;
1819 }
1820
1821 if now >= purge_account_next {
1822 self.exec_manager.purge_account_events();
1823 purge_account_next = now + purge_account_interval;
1824 }
1825
1826 if now >= own_books_next {
1827 self.kernel.cache().borrow_mut().audit_own_order_books();
1828 own_books_next = now + own_books_interval;
1829 }
1830
1831 if now >= prune_fills_next {
1832 self.exec_manager.prune_recent_fills_cache(60.0);
1833 self.exec_manager.prune_processed_fills();
1834 self.exec_manager.prune_order_local_activity();
1835 prune_fills_next = now + prune_fills_interval;
1836 }
1837
1838 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1839 }
1840
1841 Some(handler) = time_evt_rx.recv(), if poll_time => {
1846 skip_time = true;
1847 let dispatch_start = dst::time::Instant::now();
1848 let dispatched = AsyncRunner::handle_time_event(handler);
1849
1850 if dispatched && is_shutting_down {
1851 log::debug!("Residual time event");
1852 residual_events += 1;
1853 }
1854
1855 if dispatched {
1856 record_runner_dispatch(
1857 &metrics,
1858 SystemChannel::TimeEvents,
1859 dispatch_start,
1860 metrics_start,
1861 );
1862 }
1863 }
1864 Some(event) = system_evt_rx.recv() => {
1865 skip_time = false;
1866
1867 if is_shutting_down {
1868 log::debug!("Residual system event: {event}");
1869 residual_events += 1;
1870 }
1871 event.dispatch(|event| self.process_system_event(event));
1872 }
1873 Some(command) = system_cmd_rx.recv() => {
1874 skip_time = false;
1875
1876 if is_shutting_down {
1877 log::debug!("Residual system command: {command}");
1878 residual_events += 1;
1879 }
1880 command.dispatch(|command| self.process_system_command(command));
1881 }
1882 Some(evt) = exec_evt_rx.recv() => {
1883 skip_time = false;
1884 let dispatch_start = dst::time::Instant::now();
1885
1886 if is_shutting_down {
1887 log::debug!("Residual exec event: {evt}");
1888 residual_events += 1;
1889 }
1890
1891 evt.dispatch(|evt| self.process_exec_event(evt));
1892 record_runner_dispatch(
1893 &metrics,
1894 SystemChannel::ExecEvents,
1895 dispatch_start,
1896 metrics_start,
1897 );
1898 }
1899 Some(cmd) = exec_cmd_rx.recv() => {
1900 skip_time = false;
1901 let dispatch_start = dst::time::Instant::now();
1902
1903 if is_shutting_down {
1904 log::debug!("Residual exec command: {cmd}");
1905 residual_events += 1;
1906 }
1907
1908 self.process_exec_command(cmd);
1909 record_runner_dispatch(
1910 &metrics,
1911 SystemChannel::ExecCommands,
1912 dispatch_start,
1913 metrics_start,
1914 );
1915 }
1916 message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1917 skip_time = false;
1918 let external_msgbus_start = dst::time::Instant::now();
1919
1920 match message {
1921 Some(message) => {
1922 if is_shutting_down {
1923 log::debug!("Residual external message bus message: {message}");
1924 residual_events += 1;
1925 }
1926 self.process_external_msgbus_message(&message);
1927 }
1928 None => {
1929 log::info!("External message bus ingress closed");
1930 external_msgbus_rx = None;
1931 self.close_external_ingress();
1932 }
1933 }
1934
1935 record_runner_external_msgbus(
1936 &metrics,
1937 external_msgbus_start,
1938 metrics_start,
1939 );
1940 }
1941 Some(evt) = data_evt_rx.recv(), if poll_data => {
1942 skip_time = false;
1943 skip_data = true;
1944 let dispatch_start = dst::time::Instant::now();
1945
1946 if is_shutting_down {
1947 log::debug!("Residual data event: {evt:?}");
1948 residual_events += 1;
1949 }
1950 AsyncRunner::dispatch_data_event(evt);
1951 record_runner_dispatch(
1952 &metrics,
1953 SystemChannel::DataEvents,
1954 dispatch_start,
1955 metrics_start,
1956 );
1957 }
1958 Some(cmd) = data_cmd_rx.recv() => {
1959 skip_time = false;
1960 skip_data = false;
1961 let dispatch_start = dst::time::Instant::now();
1962
1963 if is_shutting_down {
1964 log::debug!("Residual data command: {cmd:?}");
1965 residual_events += 1;
1966 }
1967 AsyncRunner::handle_data_command(cmd);
1968 record_runner_dispatch(
1969 &metrics,
1970 SystemChannel::DataCommands,
1971 dispatch_start,
1972 metrics_start,
1973 );
1974 }
1975 }
1976
1977 dispatches_since_yield += 1;
1978 if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1979 dispatches_since_yield = 0;
1980 tokio::task::yield_now().await;
1981 }
1982 };
1983
1984 if residual_events > 0 {
1985 log::debug!("Processed {residual_events} residual events during shutdown");
1986 }
1987
1988 self.cancel_report_tasks(
1989 &mut open_order_report_task,
1990 &mut targeted_order_report_task,
1991 &mut position_report_task,
1992 dispatch_result.is_ok(),
1993 );
1994
1995 drop(external_msgbus_rx.take());
1996 let _ = self.kernel.cache().borrow().check_residuals();
1997
1998 let stop_result = self.finalize_stop().await;
1999
2000 if let Err(e) = dispatch_result {
2001 if let Err(stop_err) = stop_result {
2002 log::error!("Failed to finalize node after callback failure: {stop_err}");
2003 let diagnostic = format!("{e}; {stop_err}");
2005 return Err(anyhow::Error::new(e).context(diagnostic));
2006 }
2007
2008 return Err(e.into());
2009 }
2010
2011 Self::drain_channels(
2013 &mut time_evt_rx,
2014 &mut system_evt_rx,
2015 &mut system_cmd_rx,
2016 &mut exec_evt_rx,
2017 &mut exec_cmd_rx,
2018 &mut data_evt_rx,
2019 &mut data_cmd_rx,
2020 );
2021
2022 log::info!("Event loop stopped");
2023
2024 stop_result
2025 }
2026
2027 fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2028 for transition in transitions {
2029 let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2030 let timestamp = self.kernel.generate_timestamp_ns();
2031
2032 let event = QueueStateChanged::new(
2033 self.config.trader_id,
2034 transition.channel,
2035 transition.condition,
2036 transition.state,
2037 transition.queue_depth,
2038 transition.mean_dispatch_ns,
2039 UUID4::new(),
2040 timestamp,
2041 timestamp,
2042 );
2043
2044 msgbus::publish_any(topic, event.as_any());
2045 }
2046 }
2047
2048 #[expect(
2049 clippy::await_holding_refcell_ref,
2050 reason = "cache loading is serialized before the single-threaded live node starts"
2051 )]
2052 async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2053 self.install_cache_database().await?;
2054
2055 let cache = self.kernel.cache();
2056 if !cache.borrow().has_backing() {
2057 return Ok(());
2058 }
2059
2060 if self
2061 .config
2062 .cache
2063 .as_ref()
2064 .is_some_and(|config| config.flush_on_start)
2065 {
2066 cache
2067 .borrow_mut()
2068 .flush_db()
2069 .context("Failed to flush persistent cache")?;
2070 return Ok(());
2071 }
2072
2073 if self.config.exec_engine.load_cache {
2074 self.kernel
2075 .exec_engine()
2076 .borrow_mut()
2077 .load_cache()
2078 .await
2079 .context("Failed to load persistent cache")?;
2080 }
2081
2082 Ok(())
2083 }
2084
2085 #[must_use]
2087 pub const fn has_pending_cache_database(&self) -> bool {
2088 self.cache_database_factory.is_some()
2089 }
2090
2091 async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2096 let Some(factory) = self.cache_database_factory.as_ref() else {
2097 return Ok(());
2098 };
2099
2100 let config = self.config.cache.clone().unwrap_or_default();
2101
2102 let database = factory
2105 .create(self.config.trader_id, self.kernel.instance_id, config)
2106 .await
2107 .context("failed to create cache database backing")?;
2108 self.kernel.cache().borrow_mut().set_database(database);
2109 self.cache_database_factory = None;
2110
2111 Ok(())
2112 }
2113
2114 fn take_external_ingress_receiver(
2115 &mut self,
2116 ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2117 let Some(external_ingress) = self.external_msgbus.as_mut() else {
2118 return Ok(None);
2119 };
2120
2121 let receiver = external_ingress.take_receiver()?;
2122 log::info!("External message bus ingress started");
2123 Ok(Some(receiver))
2124 }
2125
2126 fn republish_external_msgbus_message(message: &BusMessage) {
2127 if let Err(e) = msgbus::republish_external_message(message) {
2128 log::error!(
2129 "Failed to republish external message bus topic '{}': {e:#}",
2130 message.topic
2131 );
2132 }
2133 }
2134
2135 fn process_external_msgbus_message(&self, message: &BusMessage) {
2136 if self.stream_processors.is_empty() {
2137 Self::republish_external_msgbus_message(message);
2138 return;
2139 }
2140
2141 let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2142 for processor in &self.stream_processors {
2143 (processor.0)(value, mapping)?;
2144 }
2145
2146 Ok(())
2147 };
2148
2149 if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2150 log::error!(
2151 "Failed to process external message bus topic '{}': {e:#}",
2152 message.topic
2153 );
2154 }
2155 }
2156
2157 fn finish_external_ingress_connection(
2158 &mut self,
2159 receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2160 ) {
2161 if receiver.is_none() {
2162 self.close_external_ingress();
2163 }
2164 }
2165
2166 fn close_external_ingress(&mut self) {
2167 if let Some(external_ingress) = self.external_msgbus.as_mut()
2168 && !external_ingress.is_closed()
2169 {
2170 external_ingress.close();
2171 }
2172 }
2173
2174 fn drain_exec_events_before_reconciliation(
2175 &mut self,
2176 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2177 ) {
2178 while let Ok(event) = exec_evt_rx.try_recv() {
2179 event.dispatch(|event| self.process_exec_event(event));
2180 }
2181 }
2182
2183 fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2184 if events.is_empty() {
2185 self.publish_submission_recovery_exhaustions();
2186 return;
2187 }
2188
2189 log::info!(
2190 "Processing {} reconciliation event{}",
2191 events.len(),
2192 if events.len() == 1 { "" } else { "s" }
2193 );
2194
2195 for event in events {
2196 self.exec_manager
2197 .record_local_activity(event.client_order_id());
2198 if let OrderEventAny::Filled(fill) = event {
2199 self.exec_manager
2200 .record_position_activity(fill.instrument_id, fill.account_id);
2201 }
2202
2203 self.kernel.exec_engine.borrow_mut().process(event);
2204 self.exec_manager
2205 .confirm_submission_outcome(&event.client_order_id());
2206 if let OrderEventAny::Filled(fill) = event {
2207 self.exec_manager.commit_recent_fill_if_applied(fill);
2208 }
2209 }
2210
2211 self.publish_submission_recovery_exhaustions();
2212 }
2213
2214 fn publish_submission_recovery_exhaustions(&mut self) {
2215 for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2216 log::warn!(
2217 "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2218 diagnostic.client_order_id,
2219 diagnostic.source,
2220 diagnostic.retry_count,
2221 );
2222 msgbus::publish_any(
2223 MessagingSwitchboard::submission_recovery_exhausted_topic(),
2224 &diagnostic,
2225 );
2226 }
2227 }
2228
2229 fn process_exec_event(&mut self, event: ExecutionEvent) {
2230 let clear_closed_orders = matches!(
2231 &event,
2232 ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2233 );
2234 let closed_report_id = match &event {
2235 ExecutionEvent::Report(
2236 ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2237 ) if report.order_status.is_closed() => report.client_order_id,
2238 _ => None,
2239 };
2240 let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2241 return;
2242 };
2243
2244 self.dispatch_exec_event_and_commit_fill(event);
2245
2246 for client_order_id in &order_ids {
2247 self.exec_manager
2248 .confirm_submission_outcome(client_order_id);
2249 let is_closed = self
2250 .kernel
2251 .cache()
2252 .borrow()
2253 .order(client_order_id)
2254 .is_some_and(|order| order.is_closed());
2255
2256 if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2257 self.exec_manager
2258 .clear_recon_tracking(client_order_id, true);
2259 }
2260 }
2261 }
2262
2263 fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2264 message.dispatch_trading(|message| {
2265 if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2266 self.observe_exec_command_before_dispatch(message.command());
2267 }
2268 });
2269 }
2270
2271 fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2280 let recent_fill_candidate = match &evt {
2281 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2282 _ => None,
2283 };
2284
2285 AsyncRunner::handle_exec_event(evt);
2286
2287 if let Some(fill) = &recent_fill_candidate {
2288 self.exec_manager.commit_recent_fill_if_applied(fill);
2289 }
2290 }
2291
2292 async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2293 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2299 dst::time::timeout(remaining, self.kernel.connect_data_clients())
2300 .await
2301 .map_err(|_| anyhow::anyhow!("data-connect timeout"))?
2302 }
2303
2304 async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2305 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2306 dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2307 .await
2308 .map_err(|_| anyhow::anyhow!("exec-connect timeout"))?
2309 }
2310
2311 async fn connect_exec_phase(
2316 &mut self,
2317 deadline: dst::time::Instant,
2318 ) -> anyhow::Result<EngineConnectionStatus> {
2319 self.connect_exec_clients(deadline).await?;
2320 Ok(self.await_engines_connected(deadline).await)
2321 }
2322
2323 fn startup_abort_reason(&self) -> Option<&'static str> {
2324 if self.handle.should_stop() {
2325 Some("Stop signal received during startup")
2326 } else if self.kernel.is_shutdown_requested() {
2327 Some("Shutdown signal received during startup")
2328 } else {
2329 None
2330 }
2331 }
2332
2333 async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2334 match self.handle.try_set_running() {
2335 RunningTransition::Entered => Ok(true),
2336 RunningTransition::StopRequested => {
2337 self.abort_startup("Stop signal received during startup")
2338 .await?;
2339 Ok(false)
2340 }
2341 RunningTransition::Invalid(control) => {
2342 self.abort_startup_with_error(
2343 "Invalid lifecycle state during startup",
2344 anyhow::anyhow!(
2345 "Invalid LiveNode control state {control:#04x} while entering Running"
2346 ),
2347 )
2348 .await?;
2349 Ok(false)
2350 }
2351 }
2352 }
2353
2354 async fn finish_startup_trader(
2355 &mut self,
2356 receivers: Option<&mut RunnerReceivers<'_>>,
2357 ) -> anyhow::Result<bool> {
2358 match self.handle.try_set_running() {
2359 RunningTransition::Entered => Ok(true),
2360 RunningTransition::StopRequested => {
2361 self.abort_started_trader("Stop signal received during startup", receivers)
2362 .await?;
2363 Ok(false)
2364 }
2365 RunningTransition::Invalid(control) => {
2366 let state_err = anyhow::anyhow!(
2367 "Invalid LiveNode control state {control:#04x} while entering Running"
2368 );
2369
2370 match self
2371 .abort_started_trader("Invalid lifecycle state during startup", receivers)
2372 .await
2373 {
2374 Ok(()) => Err(state_err),
2375 Err(finalize_err) => {
2376 anyhow::bail!(
2377 "{state_err}; failed to finalize startup abort: {finalize_err}"
2378 )
2379 }
2380 }
2381 }
2382 }
2383 }
2384
2385 async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2386 log::info!("{reason}, aborting startup");
2387 self.handle.set_shutting_down();
2388 self.finalize_stop().await
2389 }
2390
2391 async fn abort_startup_with_error(
2392 &mut self,
2393 reason: &str,
2394 startup_err: anyhow::Error,
2395 ) -> anyhow::Result<()> {
2396 match self.abort_startup(reason).await {
2397 Ok(()) => Err(startup_err),
2398 Err(finalize_err) => {
2399 anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2400 }
2401 }
2402 }
2403
2404 async fn abort_started_trader(
2405 &mut self,
2406 reason: &str,
2407 mut receivers: Option<&mut RunnerReceivers<'_>>,
2408 ) -> anyhow::Result<()> {
2409 log::info!("{reason}, aborting startup");
2410 self.handle.set_shutting_down();
2411
2412 #[cfg(feature = "plugin")]
2413 let controller_stop_result = self.plugins.stop_controllers();
2414 #[cfg(not(feature = "plugin"))]
2415 let controller_stop_result: anyhow::Result<()> = Ok(());
2416
2417 let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2418 let delay = self.kernel.delay_post_stop();
2419 log::info!("Awaiting residual events ({delay:?})...");
2420
2421 let residual_events = match receivers.as_mut() {
2422 Some(receivers) => self.process_receivers_for(delay, receivers).await,
2423 None => self.process_runner_for(delay).await,
2424 };
2425
2426 if residual_events > 0 {
2427 log::debug!("Processed {residual_events} residual events during shutdown");
2428 }
2429
2430 let finalize_result = self.finalize_stop().await;
2431
2432 if let Some(receivers) = receivers {
2433 Self::drain_channels(
2434 receivers.time_evt,
2435 receivers.system_evt,
2436 receivers.system_cmd,
2437 receivers.exec_evt,
2438 receivers.exec_cmd,
2439 receivers.data_evt,
2440 receivers.data_cmd,
2441 );
2442 } else {
2443 self.drain_runner_pending();
2444 }
2445
2446 let mut errors = Vec::new();
2447
2448 if let Err(e) = controller_stop_result {
2449 errors.push(format!("Failed to stop plug-in controllers: {e}"));
2450 }
2451
2452 if let Err(e) = trader_stop_result {
2453 errors.push(format!("Failed to stop trader: {e}"));
2454 }
2455
2456 if let Err(e) = finalize_result {
2457 errors.push(format!("Failed to finalize startup abort: {e}"));
2458 }
2459
2460 if errors.is_empty() {
2461 Ok(())
2462 } else {
2463 anyhow::bail!("{}", errors.join("; "))
2464 }
2465 }
2466
2467 async fn process_receivers_for(
2468 &mut self,
2469 duration: Duration,
2470 receivers: &mut RunnerReceivers<'_>,
2471 ) -> usize {
2472 let deadline = dst::time::Instant::now() + duration;
2473 let mut processed = 0;
2474
2475 loop {
2476 tokio::select! {
2477 biased;
2478
2479 () = dst::time::sleep_until(deadline) => break,
2480 Some(message) = receivers.time_evt.recv() => {
2481 let _ = AsyncRunner::handle_time_event(message);
2482 processed += 1;
2483 }
2484 Some(event) = receivers.system_evt.recv() => {
2485 event.dispatch(|event| self.process_system_event(event));
2486 processed += 1;
2487 }
2488 Some(command) = receivers.system_cmd.recv() => {
2489 command.dispatch(|command| self.process_system_command(command));
2490 processed += 1;
2491 }
2492 Some(event) = receivers.exec_evt.recv() => {
2493 event.dispatch(|event| self.process_exec_event(event));
2494 processed += 1;
2495 }
2496 Some(command) = receivers.exec_cmd.recv() => {
2497 self.process_exec_command(command);
2498 processed += 1;
2499 }
2500 Some(event) = receivers.data_evt.recv() => {
2501 AsyncRunner::dispatch_data_event(event);
2502 processed += 1;
2503 }
2504 Some(command) = receivers.data_cmd.recv() => {
2505 AsyncRunner::handle_data_command(command);
2506 processed += 1;
2507 }
2508 }
2509 }
2510
2511 processed
2512 }
2513
2514 async fn abort_after_trader_start_failure(
2515 &mut self,
2516 start_err: anyhow::Error,
2517 ) -> anyhow::Result<()> {
2518 log::info!("Trader startup failed, aborting startup");
2519 self.handle.set_shutting_down();
2520 let stop_result = self.kernel.stop_trader_after_start_failure();
2521 let finalize_result = self.finalize_stop().await;
2522
2523 match (stop_result, finalize_result) {
2524 (Ok(()), Ok(())) => Err(start_err),
2525 (Err(stop_err), Ok(())) => anyhow::bail!(
2526 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2527 {stop_err}"
2528 ),
2529 (Ok(()), Err(finalize_err)) => anyhow::bail!(
2530 "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2531 {finalize_err}"
2532 ),
2533 (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2534 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2535 {stop_err}; failed to finalize startup abort: {finalize_err}"
2536 ),
2537 }
2538 }
2539
2540 fn initiate_shutdown(&mut self) {
2541 #[cfg(feature = "plugin")]
2542 if let Err(e) = self.plugins.stop_controllers() {
2543 log::error!("Error stopping plug-in controllers: {e}");
2544 }
2545
2546 self.kernel.stop_trader();
2547 let delay = self.kernel.delay_post_stop();
2548 log::info!("Awaiting residual events ({delay:?})...");
2549
2550 self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2551 self.handle.set_shutting_down();
2552 }
2553
2554 async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2555 let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2558 self.close_external_ingress();
2559
2560 let timeout = self.config.timeout_disconnection;
2561 let deadline = dst::time::Instant::now() + timeout;
2562
2563 let mut errors = Vec::new();
2564 let (report_result, disconnect_result) = futures_util::join!(
2565 dst::time::timeout(
2566 timeout,
2567 Self::finish_report_tasks(&self.exec_clients, &mut errors)
2568 ),
2569 dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2570 );
2571
2572 if report_result.is_err() {
2573 errors.push("report collection shutdown timeout".to_string());
2574 }
2575
2576 let disconnect_result = match disconnect_result {
2577 Ok(result) => result,
2578 Err(_) => Err(anyhow::anyhow!(
2579 "disconnect timeout while disconnecting clients"
2580 )),
2581 };
2582
2583 if let Err(ref e) = disconnect_result {
2584 log::error!("Error disconnecting clients: {e}");
2585 }
2586
2587 let readiness_result = self.await_engines_disconnected(deadline).await;
2588 let kernel_result = self.kernel.finalize_stop().await;
2589
2590 self.handle.set_stopped();
2591
2592 if !unresolved_submissions.is_empty() {
2593 errors.push(format!(
2594 "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2595 ));
2596 }
2597
2598 if let Err(e) = disconnect_result {
2599 errors.push(e.to_string());
2600 }
2601
2602 if let Err(e) = readiness_result {
2603 errors.push(format!("failed while awaiting engine disconnection: {e}"));
2604 }
2605
2606 if let Err(e) = kernel_result {
2607 errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2608 }
2609
2610 if errors.is_empty() {
2611 Ok(())
2612 } else {
2613 anyhow::bail!("{}", errors.join("; "))
2614 }
2615 }
2616
2617 fn drain_channels(
2618 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2619 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2620 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2621 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2622 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2623 DispatchMessage<TradingCommandMessage>,
2624 >,
2625 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2626 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2627 ) {
2628 let pending = time_evt_rx.len()
2629 + system_evt_rx.len()
2630 + system_cmd_rx.len()
2631 + exec_evt_rx.len()
2632 + exec_cmd_rx.len()
2633 + data_evt_rx.len()
2634 + data_cmd_rx.len();
2635 log::debug!("Draining shutdown queues ({pending} events initially pending)");
2636
2637 let mut drained = 0;
2638 let log_progress = |processed: usize| {
2639 if processed.is_multiple_of(10_000) {
2640 log::debug!("Processed {processed} events during shutdown drain");
2641 }
2642 };
2643
2644 while let Ok(handler) = time_evt_rx.try_recv() {
2645 let _ = AsyncRunner::handle_time_event(handler);
2646 drained += 1;
2647 log_progress(drained);
2648 }
2649
2650 while system_evt_rx.try_recv().is_ok() {
2651 drained += 1;
2652 log_progress(drained);
2653 }
2654
2655 while system_cmd_rx.try_recv().is_ok() {
2656 drained += 1;
2657 log_progress(drained);
2658 }
2659
2660 while let Ok(evt) = data_evt_rx.try_recv() {
2661 AsyncRunner::dispatch_data_event(evt);
2662 drained += 1;
2663 log_progress(drained);
2664 }
2665
2666 while let Ok(cmd) = data_cmd_rx.try_recv() {
2667 AsyncRunner::handle_data_command(cmd);
2668 drained += 1;
2669 log_progress(drained);
2670 }
2671
2672 while let Ok(evt) = exec_evt_rx.try_recv() {
2673 AsyncRunner::dispatch_exec_event(evt);
2674 drained += 1;
2675 log_progress(drained);
2676 }
2677
2678 while let Ok(cmd) = exec_cmd_rx.try_recv() {
2679 AsyncRunner::handle_trading_command(cmd);
2680 drained += 1;
2681 log_progress(drained);
2682 }
2683
2684 log::debug!("Drained {drained} remaining events during shutdown");
2685 }
2686
2687 fn observe_exec_event_before_dispatch(
2688 &mut self,
2689 evt: &ExecutionEvent,
2690 ) -> Option<Vec<ClientOrderId>> {
2691 let mut order_ids = Vec::new();
2692
2693 match evt {
2694 ExecutionEvent::Order(order_evt) => {
2695 self.exec_manager.observe_order_event(order_evt);
2696 order_ids.push(order_evt.client_order_id());
2697 }
2698 ExecutionEvent::OrderSubmittedBatch(batch) => {
2699 for submitted in &batch.events {
2700 self.exec_manager
2701 .record_local_activity(submitted.client_order_id);
2702 }
2703 }
2704 ExecutionEvent::OrderAcceptedBatch(batch) => {
2705 for accepted in &batch.events {
2706 self.exec_manager
2707 .observe_order_event(&OrderEventAny::Accepted(*accepted));
2708 order_ids.push(accepted.client_order_id);
2709 }
2710 }
2711 ExecutionEvent::OrderCanceledBatch(batch) => {
2712 for canceled in &batch.events {
2713 self.exec_manager
2714 .observe_order_event(&OrderEventAny::Canceled(*canceled));
2715 order_ids.push(canceled.client_order_id);
2716 }
2717 }
2718 ExecutionEvent::Report(report) => {
2719 if let ExecutionReport::Fill(fill_report) = report
2720 && self.exec_manager.is_fill_recently_processed(
2721 fill_report.account_id,
2722 fill_report.instrument_id,
2723 fill_report.trade_id,
2724 )
2725 {
2726 log::debug!(
2727 "Skipping recently processed fill report: {}",
2728 fill_report.trade_id,
2729 );
2730 return None;
2731 }
2732
2733 self.exec_manager.observe_execution_report(report);
2734
2735 if let Some(client_order_id) = self.order_report_client_order_id(report) {
2736 order_ids.push(client_order_id);
2737 }
2738 }
2739 ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {}
2740 }
2741
2742 Some(order_ids)
2743 }
2744
2745 fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2746 match report {
2747 ExecutionReport::Order(order_report)
2748 | ExecutionReport::OrderWithFills(order_report, _) => {
2749 order_report.client_order_id.or_else(|| {
2750 self.kernel
2751 .cache
2752 .borrow()
2753 .client_order_id(&order_report.venue_order_id)
2754 .copied()
2755 })
2756 }
2757 ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2758 self.kernel
2759 .cache
2760 .borrow()
2761 .client_order_id(&fill_report.venue_order_id)
2762 .copied()
2763 }),
2764 _ => None,
2765 }
2766 }
2767
2768 fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2769 if matches!(
2770 cmd,
2771 TradingCommand::ModifyOrder(_)
2772 | TradingCommand::ModifyOrders(_)
2773 | TradingCommand::CancelOrder(_)
2774 | TradingCommand::CancelOrders(_)
2775 ) {
2776 let engine = self.kernel.exec_engine.borrow();
2777 let no_route = engine.find_client_for_command(cmd).is_none()
2778 && cmd.client_id().is_none_or(|client_id| {
2779 engine
2780 .config()
2781 .external_clients
2782 .as_ref()
2783 .is_none_or(|clients| !clients.contains(&client_id))
2784 });
2785
2786 if no_route {
2787 return;
2788 }
2789 }
2790
2791 let submission_client_id = if matches!(
2792 cmd,
2793 TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2794 ) {
2795 self.kernel
2796 .exec_engine
2797 .borrow()
2798 .find_client_for_command(cmd)
2799 .map(|client| client.client_id())
2800 } else {
2801 None
2802 };
2803
2804 match cmd {
2805 TradingCommand::SubmitOrder(submit) => {
2806 self.exec_manager.register_submission(
2807 &submit.order_init,
2808 submission_client_id.or(submit.client_id),
2809 );
2810 }
2811 TradingCommand::SubmitOrderList(submit) => {
2812 for order_init in &submit.order_inits {
2813 self.exec_manager
2814 .register_submission(order_init, submission_client_id.or(submit.client_id));
2815 }
2816 }
2817 TradingCommand::ModifyOrder(modify) => {
2818 self.exec_manager
2819 .register_inflight(modify.client_order_id, modify.command_id);
2820 }
2821 TradingCommand::ModifyOrders(modify) => {
2822 for child in &modify.modifies {
2823 self.exec_manager
2824 .register_inflight(child.client_order_id, child.command_id);
2825 }
2826 }
2827 TradingCommand::CancelOrder(cancel) => {
2828 self.exec_manager
2829 .register_inflight(cancel.client_order_id, cancel.command_id);
2830 }
2831 TradingCommand::CancelOrders(cancel) => {
2832 for child in &cancel.cancels {
2833 self.exec_manager
2834 .register_inflight(child.client_order_id, child.command_id);
2835 }
2836 }
2837 _ => {}
2838 }
2839 }
2840
2841 #[must_use]
2843 pub fn environment(&self) -> Environment {
2844 self.kernel.environment()
2845 }
2846
2847 #[must_use]
2849 pub const fn kernel(&self) -> &NautilusKernel {
2850 &self.kernel
2851 }
2852
2853 #[must_use]
2855 pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2856 &mut self.kernel
2857 }
2858
2859 #[must_use]
2861 pub fn trader_id(&self) -> TraderId {
2862 self.kernel.trader_id()
2863 }
2864
2865 #[must_use]
2867 pub const fn instance_id(&self) -> UUID4 {
2868 self.kernel.instance_id()
2869 }
2870
2871 #[must_use]
2873 pub fn state(&self) -> NodeState {
2874 self.handle.state()
2875 }
2876
2877 #[must_use]
2879 pub fn is_running(&self) -> bool {
2880 self.state().is_running()
2881 }
2882
2883 pub fn set_cache_database(
2893 &mut self,
2894 database: Box<dyn CacheDatabaseAdapter>,
2895 ) -> anyhow::Result<()> {
2896 if self.state() != NodeState::Idle {
2897 anyhow::bail!(
2898 "Cannot set cache database while node is running, set it before running the node"
2899 );
2900 }
2901
2902 self.kernel.cache().borrow_mut().set_database(database);
2903 Ok(())
2904 }
2905
2906 #[must_use]
2908 pub fn exec_manager(&self) -> &ExecutionManager {
2909 &self.exec_manager
2910 }
2911
2912 #[must_use]
2914 pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2915 &mut self.exec_manager
2916 }
2917
2918 pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2931 where
2932 T: DataActor + DataActorNative + Component + Actor + 'static,
2933 {
2934 if self.state() != NodeState::Idle {
2935 anyhow::bail!(
2936 "Cannot add actor while node is running, add actors before running the node"
2937 );
2938 }
2939
2940 self.kernel.trader.borrow_mut().add_actor(actor)
2941 }
2942
2943 pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2955 where
2956 F: FnOnce() -> anyhow::Result<T>,
2957 T: DataActor + DataActorNative + Component + Actor + 'static,
2958 {
2959 if self.state() != NodeState::Idle {
2960 anyhow::bail!(
2961 "Cannot add actor while node is running, add actors before running the node"
2962 );
2963 }
2964
2965 self.kernel
2966 .trader
2967 .borrow_mut()
2968 .add_actor_from_factory(factory)
2969 }
2970
2971 pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2987 where
2988 T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2989 {
2990 if self.state() != NodeState::Idle {
2991 anyhow::bail!(
2992 "Cannot add strategy while node is running, add strategies before running the node"
2993 );
2994 }
2995
2996 let strategy_id = self
2998 .kernel
2999 .trader
3000 .borrow()
3001 .prepare_strategy_for_registration(&mut strategy)?;
3002 let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
3003 let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
3004
3005 if !instrument_ids.is_empty() {
3006 self.register_external_order_claims(strategy_id, &instrument_ids)?;
3007 }
3008
3009 let mut exec_engine = match oms_type
3010 .map(|_| {
3011 self.kernel
3012 .exec_engine
3013 .try_borrow_mut()
3014 .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
3015 })
3016 .transpose()
3017 {
3018 Ok(exec_engine) => exec_engine,
3019 Err(e) => {
3020 if !instrument_ids.is_empty()
3021 && let Err(rollback_error) =
3022 self.rollback_external_order_claims(strategy_id, &instrument_ids)
3023 {
3024 anyhow::bail!(
3025 "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
3026 );
3027 }
3028
3029 return Err(e);
3030 }
3031 };
3032
3033 if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
3034 drop(exec_engine);
3035
3036 if !instrument_ids.is_empty()
3037 && let Err(rollback_error) =
3038 self.rollback_external_order_claims(strategy_id, &instrument_ids)
3039 {
3040 anyhow::bail!(
3041 "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
3042 );
3043 }
3044
3045 return Err(add_error);
3046 }
3047
3048 if let Some(exec_engine) = &mut exec_engine
3049 && let Some(oms_type) = oms_type
3050 {
3051 exec_engine.register_oms_type(strategy_id, oms_type);
3052 }
3053
3054 Ok(())
3055 }
3056
3057 pub fn register_external_order_claims(
3068 &self,
3069 strategy_id: StrategyId,
3070 instrument_ids: &[InstrumentId],
3071 ) -> anyhow::Result<()> {
3072 self.kernel
3073 .cache
3074 .try_borrow_mut()
3075 .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3076 .register_external_order_claims(strategy_id, instrument_ids)?;
3077
3078 if !instrument_ids.is_empty() {
3079 log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3080 }
3081
3082 Ok(())
3083 }
3084
3085 pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3095 self.kernel
3096 .cache
3097 .try_borrow_mut()
3098 .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3099 .set_external_order_claims(strategy_id, &[])?;
3100
3101 Ok(())
3102 }
3103
3104 pub(crate) fn rollback_external_order_claims(
3105 &self,
3106 strategy_id: StrategyId,
3107 instrument_ids: &[InstrumentId],
3108 ) -> anyhow::Result<()> {
3109 let mut cache = self
3110 .kernel
3111 .cache
3112 .try_borrow_mut()
3113 .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3114 let retained: Vec<_> = cache
3115 .external_order_claim_instrument_ids(Some(strategy_id))
3116 .into_iter()
3117 .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3118 .collect();
3119 cache.set_external_order_claims(strategy_id, &retained)
3120 }
3121
3122 pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3133 where
3134 T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3135 {
3136 if self.state() != NodeState::Idle {
3137 anyhow::bail!(
3138 "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3139 );
3140 }
3141
3142 self.kernel
3143 .trader
3144 .borrow_mut()
3145 .add_exec_algorithm(exec_algorithm)
3146 }
3147
3148 }
3152
3153#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3154enum SocketReconnectDispatchOutcome {
3155 Accepted,
3156 AlreadyReconnecting,
3157 Disconnected,
3158 Closed,
3159 Unsupported,
3160 InvalidTrader,
3161 UnknownClient,
3162 UnknownEndpoint,
3163 AmbiguousEndpoint,
3164}
3165
3166fn record_runner_dispatch(
3167 metrics: &RunnerMetrics,
3168 channel: SystemChannel,
3169 dispatch_start: dst::time::Instant,
3170 metrics_start: dst::time::Instant,
3171) {
3172 let dispatch_end = dst::time::Instant::now();
3173 metrics.record_dispatch(
3174 channel,
3175 dispatch_end.duration_since(dispatch_start),
3176 dispatch_end.duration_since(metrics_start),
3177 );
3178}
3179
3180fn record_runner_maintenance(
3181 metrics: &RunnerMetrics,
3182 work_start: dst::time::Instant,
3183 metrics_start: dst::time::Instant,
3184) {
3185 let work_end = dst::time::Instant::now();
3186 metrics.record_maintenance(
3187 work_end.duration_since(work_start),
3188 work_end.duration_since(metrics_start),
3189 );
3190}
3191
3192fn record_runner_external_msgbus(
3193 metrics: &RunnerMetrics,
3194 work_start: dst::time::Instant,
3195 metrics_start: dst::time::Instant,
3196) {
3197 let work_end = dst::time::Instant::now();
3198 metrics.record_external_msgbus(
3199 work_end.duration_since(work_start),
3200 work_end.duration_since(metrics_start),
3201 );
3202}
3203
3204async fn drive_with_external_ingress<F, T>(
3205 future: F,
3206 receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3207 pending: &mut VecDeque<BusMessage>,
3208) -> T
3209where
3210 F: std::future::Future<Output = T>,
3211{
3212 tokio::pin!(future);
3213 let capacity = receiver
3214 .as_ref()
3215 .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3216
3217 loop {
3218 tokio::select! {
3219 biased;
3220
3221 result = &mut future => return result,
3222 message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3223 let Some(message) = message else {
3224 log::info!("External message bus ingress closed");
3225 *receiver = None;
3226 continue;
3227 };
3228
3229 match msgbus::republish_pending_external_response(&message) {
3230 Ok(true) => {},
3231 Ok(false) | Err(_) => pending.push_back(message),
3232 }
3233 }
3234 }
3235 }
3236}
3237
3238async fn recv_external_msgbus_message(
3239 rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3240 pending: &mut VecDeque<BusMessage>,
3241) -> Option<BusMessage> {
3242 if let Some(message) = pending.pop_front() {
3243 return Some(message);
3244 }
3245
3246 match rx {
3247 Some(rx) => rx.recv().await,
3248 None => std::future::pending::<Option<BusMessage>>().await,
3249 }
3250}
3251
3252struct RunnerReceivers<'a> {
3253 time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3254 system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3255 system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3256 exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3257 exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3258 data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3259 data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3260}
3261
3262fn flush_pending_data(
3269 pending: &mut PendingEvents,
3270 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3271 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3272) {
3273 loop {
3274 let mut progressed = pending.drain_data();
3275
3276 while let Ok(evt) = data_evt_rx.try_recv() {
3277 AsyncRunner::dispatch_data_event(evt);
3278 progressed = true;
3279 }
3280
3281 while let Ok(cmd) = data_cmd_rx.try_recv() {
3282 AsyncRunner::handle_data_command(cmd);
3283 progressed = true;
3284 }
3285
3286 if !progressed {
3287 break;
3288 }
3289 }
3290}
3291
3292#[expect(
3298 clippy::too_many_arguments,
3299 reason = "all runner receivers are drained together"
3300)]
3301fn flush_all_pending(
3302 pending: &mut PendingEvents,
3303 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3304 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3305 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3306 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3307 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3308 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3309 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3310) {
3311 while let Ok(handler) = time_evt_rx.try_recv() {
3313 let _ = AsyncRunner::handle_time_event(handler);
3314 }
3315
3316 while let Ok(event) = system_evt_rx.try_recv() {
3317 pending.system_events.push(event);
3318 }
3319
3320 while let Ok(command) = system_cmd_rx.try_recv() {
3321 pending.system_commands.push(command);
3322 }
3323
3324 while let Ok(evt) = data_evt_rx.try_recv() {
3325 pending.data_evts.push(evt);
3326 }
3327
3328 while let Ok(cmd) = data_cmd_rx.try_recv() {
3329 pending.data_cmds.push(cmd);
3330 }
3331
3332 while let Ok(evt) = exec_evt_rx.try_recv() {
3333 pending.push_exec_event(evt);
3334 }
3335
3336 while let Ok(cmd) = exec_cmd_rx.try_recv() {
3337 pending.exec_cmds.push(cmd);
3338 }
3339
3340 pending.drain();
3341}
3342
3343#[expect(
3348 clippy::too_many_arguments,
3349 reason = "startup buffering owns one future plus the pending state and all runner receivers"
3350)]
3351async fn drive_with_event_buffering<F: std::future::Future>(
3352 future: F,
3353 pending: &mut PendingEvents,
3354 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3355 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3356 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3357 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3358 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3359 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3360 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3361) -> F::Output {
3362 tokio::pin!(future);
3363
3364 loop {
3365 tokio::select! {
3366 biased;
3367
3368 result = &mut future => {
3369 break result;
3370 }
3371 Some(handler) = time_evt_rx.recv() => {
3372 let _ = AsyncRunner::handle_time_event(handler);
3373 }
3374 Some(event) = system_evt_rx.recv() => {
3375 pending.system_events.push(event);
3376 }
3377 Some(command) = system_cmd_rx.recv() => {
3378 pending.system_commands.push(command);
3379 }
3380 Some(evt) = exec_evt_rx.recv() => {
3381 pending.push_exec_event(evt);
3382 }
3383 Some(cmd) = exec_cmd_rx.recv() => {
3384 pending.exec_cmds.push(cmd);
3385 }
3386 Some(evt) = data_evt_rx.recv() => {
3387 pending.data_evts.push(evt);
3388 }
3389 Some(cmd) = data_cmd_rx.recv() => {
3390 pending.data_cmds.push(cmd);
3391 }
3392 }
3393 }
3394}
3395
3396#[derive(Default)]
3397struct PendingEvents {
3398 system_events: Vec<DispatchMessage<SystemEvent>>,
3399 system_commands: Vec<DispatchMessage<SystemCommand>>,
3400 data_evts: Vec<DispatchMessage<DataEvent>>,
3401 data_cmds: Vec<DispatchMessage<DataCommand>>,
3402 exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3403 order_evts: Vec<DispatchMessage<OrderEventAny>>,
3404 exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3405}
3406
3407impl PendingEvents {
3408 fn is_empty(&self) -> bool {
3409 self.system_events.is_empty()
3410 && self.system_commands.is_empty()
3411 && self.data_evts.is_empty()
3412 && self.data_cmds.is_empty()
3413 && self.exec_reports.is_empty()
3414 && self.order_evts.is_empty()
3415 && self.exec_cmds.is_empty()
3416 }
3417
3418 fn drain_data(&mut self) -> bool {
3422 let total = self.data_evts.len() + self.data_cmds.len();
3423
3424 if total > 0 {
3425 log::debug!(
3426 "Draining {total} data events/commands into cache \
3427 (data_evts={}, data_cmds={})",
3428 self.data_evts.len(),
3429 self.data_cmds.len(),
3430 );
3431 }
3432
3433 for evt in self.data_evts.drain(..) {
3434 AsyncRunner::dispatch_data_event(evt);
3435 }
3436
3437 for cmd in self.data_cmds.drain(..) {
3438 AsyncRunner::handle_data_command(cmd);
3439 }
3440
3441 total > 0
3442 }
3443
3444 fn drain(&mut self) {
3446 let total = self.data_evts.len()
3447 + self.data_cmds.len()
3448 + self.exec_reports.len()
3449 + self.order_evts.len()
3450 + self.exec_cmds.len();
3451
3452 if total > 0 {
3453 log::debug!(
3454 "Processing {total} events/commands queued during startup \
3455 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3456 self.data_evts.len(),
3457 self.data_cmds.len(),
3458 self.exec_reports.len(),
3459 self.order_evts.len(),
3460 self.exec_cmds.len()
3461 );
3462 }
3463
3464 for evt in self.data_evts.drain(..) {
3465 AsyncRunner::dispatch_data_event(evt);
3466 }
3467
3468 for cmd in self.data_cmds.drain(..) {
3469 AsyncRunner::handle_data_command(cmd);
3470 }
3471
3472 for report in self.exec_reports.drain(..) {
3473 report
3474 .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3475 }
3476
3477 for evt in self.order_evts.drain(..) {
3478 evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3479 }
3480
3481 for cmd in self.exec_cmds.drain(..) {
3482 AsyncRunner::handle_trading_command(cmd);
3483 }
3484 }
3485
3486 fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3487 let rooted = event.is_rooted();
3488 event.dispatch(|event| {
3489 let owner = std::thread::current().id();
3490
3491 let order = |event| {
3492 if rooted {
3493 DispatchMessage::new(event, owner)
3494 } else {
3495 DispatchMessage::from(event)
3496 }
3497 };
3498
3499 match event {
3502 ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {
3503 AsyncRunner::handle_exec_event(event);
3504 }
3505 ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3506 DispatchMessage::new(report, owner)
3507 } else {
3508 report.into()
3509 }),
3510 ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3511 ExecutionEvent::OrderSubmittedBatch(batch) => {
3512 for event in batch {
3513 self.order_evts.push(order(OrderEventAny::Submitted(event)));
3514 }
3515 }
3516 ExecutionEvent::OrderAcceptedBatch(batch) => {
3517 for event in batch {
3518 self.order_evts.push(order(OrderEventAny::Accepted(event)));
3519 }
3520 }
3521 ExecutionEvent::OrderCanceledBatch(batch) => {
3522 for event in batch {
3523 self.order_evts.push(order(OrderEventAny::Canceled(event)));
3524 }
3525 }
3526 }
3527 });
3528 }
3529
3530 fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3531 std::mem::take(&mut self.system_events)
3532 }
3533
3534 fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3535 std::mem::take(&mut self.system_commands)
3536 }
3537}
3538
3539struct ClientStatus {
3540 client: String,
3541 client_type: &'static str,
3542 connected: bool,
3543}
3544
3545fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3546 let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3547 builder.push_record(["Client", "Type", "Connected"]);
3548
3549 for row in rows {
3550 builder.push_record([
3551 row.client,
3552 row.client_type.to_string(),
3553 row.connected.to_string(),
3554 ]);
3555 }
3556
3557 builder.build().with(Style::rounded()).to_string()
3558}
3559
3560#[cfg(test)]
3561mod tests {
3562 mod report_acceptance;
3563 mod shutdown;
3564
3565 use std::{
3566 cell::{Cell, RefCell},
3567 fmt::Debug,
3568 rc::Rc,
3569 sync::{
3570 Arc,
3571 atomic::{AtomicBool, Ordering},
3572 },
3573 };
3574
3575 use bytes::Bytes;
3576 use indexmap::{IndexMap, IndexSet};
3577 use log::{Level, LevelFilter, Log, Metadata, Record};
3578 #[cfg(feature = "python")]
3579 use nautilus_common::runner::{
3580 SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3581 };
3582 use nautilus_common::{
3583 actor::{
3584 self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3585 },
3586 cache::Cache,
3587 clock::{Clock, VirtualClock},
3588 enums::SerializationEncoding,
3589 live::{
3590 runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3591 sender::DispatchSender,
3592 },
3593 logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3594 messages::{
3595 data::{SubscribeCommand, SubscribeQuotes},
3596 execution::{
3597 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3598 GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3599 TradingCommand,
3600 },
3601 system::{
3602 QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3603 },
3604 },
3605 msgbus::{
3606 self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3607 MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3608 MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3609 },
3610 nautilus_actor,
3611 runner::{SyncTradingCommandSender, TradingCommandSender},
3612 testing::wait_until_async,
3613 timer::{TimeEvent, TimeEventCallback},
3614 };
3615 use nautilus_core::{Params, UUID4, UnixNanos};
3616 use nautilus_execution::{
3617 engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3618 reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3619 };
3620 use nautilus_model::{
3621 accounts::{AccountAny, MarginAccount},
3622 data::QuoteTick,
3623 enums::{
3624 AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3625 TimeInForce,
3626 },
3627 events::{
3628 AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3629 order::spec::{
3630 OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3631 },
3632 },
3633 identifiers::{
3634 AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3635 TradeId, TraderId, Venue, VenueOrderId,
3636 },
3637 instruments::{
3638 Instrument, InstrumentAny,
3639 stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3640 },
3641 orders::{
3642 Order, OrderAny, OrderList, OrderTestBuilder,
3643 stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3644 },
3645 position::PositionReplayEvent,
3646 reports::{FillReport, PositionStatusReport},
3647 types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3648 };
3649 use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3650 use nautilus_testkit::{
3651 cache::TestCacheDatabaseControl,
3652 components::{StateActor, StateStrategy},
3653 };
3654 use nautilus_trading::{
3655 nautilus_strategy,
3656 strategy::{config::StrategyConfig, core::StrategyCore},
3657 };
3658 use parking_lot::Mutex;
3659 use rstest::*;
3660 use rust_decimal::Decimal;
3661 use rust_decimal_macros::dec;
3662 use ustr::Ustr;
3663
3664 use super::{
3665 reconciliation::{
3666 PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3667 request_position_fill_reports,
3668 },
3669 *,
3670 };
3671 use crate::{
3672 execution::{
3673 client::LiveExecutionClient,
3674 manager::{PositionFillReportQuery, ReportClientCoverage, SourcedOrderStatusReport},
3675 submission::{
3676 SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3677 },
3678 },
3679 socket::SocketControl,
3680 };
3681
3682 struct ExternalIngressLogCapture {
3683 messages: Mutex<Vec<String>>,
3684 }
3685
3686 static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3687 messages: Mutex::new(Vec::new()),
3688 };
3689
3690 #[derive(Debug)]
3691 enum FillReportClientOutcome {
3692 Reports(Vec<FillReport>),
3693 Failure,
3694 }
3695
3696 #[derive(Debug)]
3697 struct FillReportClient {
3698 client_id: ClientId,
3699 account_id: AccountId,
3700 venue: Venue,
3701 outcome: FillReportClientOutcome,
3702 commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3703 }
3704
3705 #[async_trait::async_trait(?Send)]
3706 impl ExecutionClient for FillReportClient {
3707 fn is_connected(&self) -> bool {
3708 true
3709 }
3710
3711 fn client_id(&self) -> ClientId {
3712 self.client_id
3713 }
3714
3715 fn account_id(&self) -> AccountId {
3716 self.account_id
3717 }
3718
3719 fn venue(&self) -> Venue {
3720 self.venue
3721 }
3722
3723 fn oms_type(&self) -> OmsType {
3724 OmsType::Netting
3725 }
3726
3727 fn get_account(&self) -> Option<AccountAny> {
3728 None
3729 }
3730
3731 fn generate_account_state(
3732 &self,
3733 _balances: Vec<AccountBalance>,
3734 _margins: Vec<MarginBalance>,
3735 _reported: bool,
3736 _ts_event: UnixNanos,
3737 _info: Option<Params>,
3738 ) -> anyhow::Result<()> {
3739 Ok(())
3740 }
3741
3742 fn start(&mut self) -> anyhow::Result<()> {
3743 Ok(())
3744 }
3745
3746 fn stop(&mut self) -> anyhow::Result<()> {
3747 Ok(())
3748 }
3749
3750 async fn generate_order_status_reports(
3751 &self,
3752 _cmd: &GenerateOrderStatusReports,
3753 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3754 match self.outcome {
3755 FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3756 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3757 }
3758 }
3759
3760 async fn generate_position_status_reports(
3761 &self,
3762 _cmd: &GeneratePositionStatusReports,
3763 ) -> anyhow::Result<Vec<PositionStatusReport>> {
3764 match self.outcome {
3765 FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3766 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3767 }
3768 }
3769
3770 async fn generate_fill_reports(
3771 &self,
3772 cmd: GenerateFillReports,
3773 ) -> anyhow::Result<Vec<FillReport>> {
3774 self.commands.borrow_mut().push(cmd);
3775
3776 match &self.outcome {
3777 FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3778 FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3779 }
3780 }
3781 }
3782
3783 #[derive(Debug)]
3784 struct OrderReportClient {
3785 client_id: ClientId,
3786 report: OrderStatusReport,
3787 targeted: bool,
3788 queries: Rc<RefCell<Vec<&'static str>>>,
3789 }
3790
3791 #[async_trait::async_trait(?Send)]
3792 impl ExecutionClient for OrderReportClient {
3793 fn is_connected(&self) -> bool {
3794 true
3795 }
3796
3797 fn client_id(&self) -> ClientId {
3798 self.client_id
3799 }
3800
3801 fn account_id(&self) -> AccountId {
3802 self.report.account_id
3803 }
3804
3805 fn venue(&self) -> Venue {
3806 self.report.instrument_id.venue
3807 }
3808
3809 fn oms_type(&self) -> OmsType {
3810 OmsType::Netting
3811 }
3812
3813 fn get_account(&self) -> Option<AccountAny> {
3814 None
3815 }
3816
3817 fn generate_account_state(
3818 &self,
3819 _balances: Vec<AccountBalance>,
3820 _margins: Vec<MarginBalance>,
3821 _reported: bool,
3822 _ts_event: UnixNanos,
3823 _info: Option<Params>,
3824 ) -> anyhow::Result<()> {
3825 Ok(())
3826 }
3827
3828 fn start(&mut self) -> anyhow::Result<()> {
3829 Ok(())
3830 }
3831
3832 fn stop(&mut self) -> anyhow::Result<()> {
3833 Ok(())
3834 }
3835
3836 async fn generate_order_status_reports(
3837 &self,
3838 _command: &GenerateOrderStatusReports,
3839 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3840 self.queries.borrow_mut().push("bulk");
3841 Ok(if self.targeted {
3842 Vec::new()
3843 } else {
3844 vec![self.report.clone()]
3845 })
3846 }
3847
3848 async fn generate_order_status_report(
3849 &self,
3850 _command: &GenerateOrderStatusReport,
3851 ) -> anyhow::Result<Option<OrderStatusReport>> {
3852 self.queries.borrow_mut().push("targeted");
3853 Ok(Some(self.report.clone()))
3854 }
3855 }
3856
3857 #[derive(Debug)]
3858 struct StartupSocketActor {
3859 core: DataActorCore,
3860 received: Rc<RefCell<Vec<SocketStateChanged>>>,
3861 }
3862
3863 impl StartupSocketActor {
3864 fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3865 Self {
3866 core: DataActorCore::new(DataActorConfig {
3867 actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3868 ..Default::default()
3869 }),
3870 received,
3871 }
3872 }
3873 }
3874
3875 impl DataActor for StartupSocketActor {
3876 fn on_start(&mut self) -> anyhow::Result<()> {
3877 self.subscribe_socket_state(None, None, None);
3878 Ok(())
3879 }
3880
3881 fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3882 self.received.borrow_mut().push(event.clone());
3883 Ok(())
3884 }
3885 }
3886
3887 nautilus_actor!(StartupSocketActor);
3888
3889 impl Log for ExternalIngressLogCapture {
3890 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3891 metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3892 }
3893
3894 fn log(&self, record: &Record<'_>) {
3895 if self.enabled(record.metadata()) {
3896 self.messages.lock().push(record.args().to_string());
3897 }
3898 }
3899
3900 fn flush(&self) {}
3901 }
3902
3903 #[derive(Debug)]
3904 struct FailingTimerActor {
3905 core: DataActorCore,
3906 received: Rc<RefCell<Vec<u64>>>,
3907 }
3908
3909 nautilus_actor!(FailingTimerActor);
3910
3911 impl DataActor for FailingTimerActor {
3912 fn on_start(&mut self) -> anyhow::Result<()> {
3913 for timestamp in [17, 23] {
3914 let received = self.received.clone();
3915
3916 let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3917 received.borrow_mut().push(timestamp);
3918
3919 if timestamp == 17 {
3920 crate::dispatch::tests::latch_callback_failure();
3921 }
3922 }));
3923
3924 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3925 TimeEvent::new(
3926 "callback-failure".into(),
3927 UUID4::new(),
3928 timestamp.into(),
3929 timestamp.into(),
3930 ),
3931 callback,
3932 ));
3933 }
3934
3935 Ok(())
3936 }
3937 }
3938
3939 #[rstest]
3940 #[tokio::test]
3941 async fn test_callback_failure_stops_later_live_events(
3942 #[values(false, true)] incomplete_submission: bool,
3943 ) {
3944 actor::clear_callbacks().unwrap();
3945
3946 let config = LiveNodeConfig {
3947 exec_engine: crate::config::LiveExecutionEngineConfig {
3948 reconciliation: false,
3949 submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3950 ..Default::default()
3951 },
3952 timeout_connection: Duration::ZERO,
3953 timeout_reconciliation: Duration::ZERO,
3954 timeout_portfolio: Duration::ZERO,
3955 timeout_disconnection: Duration::ZERO,
3956 delay_post_stop: Duration::ZERO,
3957 timeout_shutdown: Duration::ZERO,
3958 ..Default::default()
3959 };
3960
3961 let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3962
3963 if incomplete_submission {
3964 let order = OrderTestBuilder::new(OrderType::Market)
3965 .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3966 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3967 .quantity(Quantity::from("1"))
3968 .build();
3969 node.exec_manager
3970 .register_submission(order.init_event(), None);
3971 }
3972 let received = Rc::new(RefCell::new(Vec::new()));
3973 node.add_actor(FailingTimerActor {
3974 core: DataActorCore::new(DataActorConfig {
3975 actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3976 ..Default::default()
3977 }),
3978 received: received.clone(),
3979 })
3980 .unwrap();
3981
3982 let result = node.run_with_mode(NodeRunMode::Hosted).await;
3983
3984 let error = result.unwrap_err();
3985 assert!(error.to_string().contains("Callback delivery unwound"));
3986 if incomplete_submission {
3987 assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3988 }
3989 let state = node.state();
3990 let trader_stopped = node.kernel.trader.borrow().is_stopped();
3991 let failure = actor::callback_failure();
3992 node.dispose();
3993
3994 assert_eq!(
3995 error.downcast_ref::<CallbackDispatchError>(),
3996 Some(&CallbackDispatchError::DeliveryUnwound)
3997 );
3998 assert_eq!(*received.borrow(), [17]);
3999 assert_eq!(state, NodeState::Stopped);
4000 assert!(trader_stopped);
4001 assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
4002 assert_eq!(actor::callback_failure(), None);
4003 assert_eq!(actor::clear_callbacks(), Ok(()));
4004 }
4005
4006 #[rstest]
4007 fn test_render_client_statuses() {
4008 let rows = vec![
4009 ClientStatus {
4010 client: "BINANCE".to_string(),
4011 client_type: "Data",
4012 connected: true,
4013 },
4014 ClientStatus {
4015 client: "SIM".to_string(),
4016 client_type: "Execution",
4017 connected: false,
4018 },
4019 ];
4020
4021 let output = render_client_statuses(rows);
4022 let expected = "â•─────────┬───────────┬───────────╮\n\
4023│ Client │ Type │ Connected │\n\
4024├─────────┼───────────┼───────────┤\n\
4025│ BINANCE │ Data │ true │\n\
4026│ SIM │ Execution │ false │\n\
4027╰─────────┴───────────┴───────────╯";
4028
4029 assert_eq!(output, expected);
4030 }
4031
4032 #[rstest]
4033 fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
4034 log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
4035 log::set_max_level(LevelFilter::Error);
4036 EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
4037 let message = BusMessage::with_str_topic(
4038 "data.quotes.AUDUSD.SIM*",
4039 BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
4040 Bytes::new(),
4041 SerializationEncoding::Json,
4042 );
4043
4044 LiveNode::republish_external_msgbus_message(&message);
4045
4046 assert_eq!(
4047 *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
4048 vec![
4049 "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
4050 external message topic: Topic `value` contained invalid characters, was \
4051 data.quotes.AUDUSD.SIM*"
4052 .to_string()
4053 ],
4054 );
4055 }
4056
4057 #[rstest]
4058 #[tokio::test]
4059 async fn test_external_startup_prioritizes_completed_connection() {
4060 let message = BusMessage::with_str_topic(
4061 "data.trades.FIRST",
4062 BusPayloadType::TradeTick,
4063 bytes::Bytes::new(),
4064 SerializationEncoding::Json,
4065 );
4066 let (sender, receiver) = tokio::sync::mpsc::channel(1);
4067 sender.send(message.clone()).await.unwrap();
4068 let mut receiver = Some(receiver);
4069 let mut pending = VecDeque::new();
4070
4071 let result =
4072 drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4073 let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4074
4075 assert_eq!(result, 17);
4076 assert!(pending.is_empty());
4077 assert_eq!(
4078 serde_json::to_value(queued).unwrap(),
4079 serde_json::to_value(message).unwrap()
4080 );
4081 }
4082
4083 #[rstest]
4084 #[tokio::test]
4085 async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4086 use bytes::Bytes;
4087 use nautilus_common::messages::data::{
4088 CustomDataResponse, DataResponse, DataResponseEnvelope,
4089 };
4090 use nautilus_model::data::DataType;
4091
4092 let (egress, publications, _) = CapturingExternalEgress::new();
4093 let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4094 msgbus::set_message_bus(bus.clone());
4095 bus.borrow_mut()
4096 .set_external_egress_config(
4097 Box::new(egress),
4098 &MessageBusConfig {
4099 encoding: SerializationEncoding::MsgPack,
4100 external_streams: Some(vec!["external.test".into()]),
4101 ..Default::default()
4102 },
4103 )
4104 .unwrap();
4105
4106 bus.borrow_mut()
4107 .add_streaming_type(BusPayloadType::DataResponse);
4108 let id = UUID4::new();
4109 let response = DataResponse::Data(CustomDataResponse::new(
4110 id,
4111 "FEED".into(),
4112 None,
4113 DataType::new("Startup", None, None),
4114 Bytes::from_static(b"connected"),
4115 Some(101.into()),
4116 Some(103.into()),
4117 107.into(),
4118 None,
4119 ));
4120 msgbus::publish_any("responses.data.FEED".into(), &response);
4121 let message = BusMessage::with_str_topic(
4122 "responses.data.FEED",
4123 BusPayloadType::DataResponse,
4124 publications.borrow()[0].payload.clone(),
4125 SerializationEncoding::MsgPack,
4126 );
4127 let (done, connected) = tokio::sync::oneshot::channel();
4128 let done = Rc::new(RefCell::new(Some(done)));
4129 msgbus::register_response_handler(
4130 &id,
4131 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4132 let Some(DataResponse::Data(reply)) = envelope.response() else {
4133 panic!("expected custom reply");
4134 };
4135
4136 done.borrow_mut()
4137 .take()
4138 .unwrap()
4139 .send(reply.clone())
4140 .unwrap();
4141 }),
4142 );
4143
4144 msgbus::subscribe_any(
4145 message.topic.as_str().into(),
4146 ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4147 msgbus::send_response(reply.correlation_id(), reply);
4148 }),
4149 None,
4150 );
4151
4152 let unrelated = BusMessage::with_str_topic(
4153 "data.trades.FIRST",
4154 BusPayloadType::TradeTick,
4155 Bytes::new(),
4156 SerializationEncoding::Json,
4157 );
4158 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4159 sender.send(unrelated.clone()).await.unwrap();
4160 sender.send(message).await.unwrap();
4161 let mut receiver = Some(receiver);
4162 let mut pending = VecDeque::new();
4163 let actual = tokio::time::timeout(
4164 Duration::from_secs(1),
4165 drive_with_external_ingress(connected, &mut receiver, &mut pending),
4166 )
4167 .await
4168 .unwrap()
4169 .unwrap();
4170
4171 assert_eq!(actual.correlation_id, id);
4172 assert_eq!(actual.client_id, ClientId::from("FEED"));
4173 assert_eq!(actual.venue, None);
4174 assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4175 assert_eq!(
4176 actual.data.downcast_ref::<Bytes>(),
4177 Some(&Bytes::from_static(b"connected"))
4178 );
4179 assert_eq!(actual.start, Some(101.into()));
4180 assert_eq!(actual.end, Some(103.into()));
4181 assert_eq!(actual.ts_init, UnixNanos::from(107));
4182 assert_eq!(actual.params, None);
4183 assert_eq!(publications.borrow().len(), 1);
4184 assert_eq!(pending.len(), 1);
4185 assert_eq!(
4186 serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4187 serde_json::to_value(unrelated).unwrap()
4188 );
4189 assert!(receiver.as_ref().unwrap().is_empty());
4190 }
4191
4192 #[rstest]
4193 #[tokio::test]
4194 async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4195 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4196 let mut receiver = Some(receiver);
4197
4198 let message = |name: &str| {
4199 BusMessage::with_str_topic(
4200 name,
4201 BusPayloadType::TradeTick,
4202 bytes::Bytes::new(),
4203 SerializationEncoding::Json,
4204 )
4205 };
4206
4207 let first = message("data.trades.FIRST");
4208 let second = message("data.trades.SECOND");
4209 let third = message("data.trades.THIRD");
4210 sender.send(first.clone()).await.unwrap();
4211 sender.send(second.clone()).await.unwrap();
4212 let (done, connected) = tokio::sync::oneshot::channel();
4213 let mut pending = VecDeque::new();
4214 let mut connection = Box::pin(drive_with_external_ingress(
4215 connected,
4216 &mut receiver,
4217 &mut pending,
4218 ));
4219 assert!(futures_util::poll!(&mut connection).is_pending());
4220 sender.send(third.clone()).await.unwrap();
4221 assert!(futures_util::poll!(&mut connection).is_pending());
4222 done.send(()).unwrap();
4223 connection.await.unwrap();
4224
4225 assert_eq!(pending.len(), 2);
4226 assert_eq!(receiver.as_ref().unwrap().len(), 1);
4227
4228 for expected in [first, second, third] {
4229 let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4230 .await
4231 .unwrap();
4232 assert_eq!(
4233 serde_json::to_value(received).unwrap(),
4234 serde_json::to_value(expected).unwrap()
4235 );
4236 }
4237 }
4238
4239 #[rstest]
4240 #[tokio::test]
4241 async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4242 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4243 drop(sender);
4244 let mut receiver = Some(receiver);
4245 let mut pending = VecDeque::new();
4246 let (done, connected) = tokio::sync::oneshot::channel();
4247 let mut connection = Box::pin(drive_with_external_ingress(
4248 connected,
4249 &mut receiver,
4250 &mut pending,
4251 ));
4252 assert!(futures_util::poll!(&mut connection).is_pending());
4253 done.send(()).unwrap();
4254 connection.await.unwrap();
4255
4256 assert!(receiver.is_none());
4257 assert!(pending.is_empty());
4258 }
4259
4260 #[rstest]
4261 #[case(None, 2)]
4262 #[case(Some(SystemChannel::DataEvents), 1)]
4263 #[case(Some(SystemChannel::ExecCommands), 1)]
4264 fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4265 #[case] channel: Option<SystemChannel>,
4266 #[case] expected_count: usize,
4267 ) {
4268 let config = LiveNodeConfig {
4269 trader_id: TraderId::from("QUEUE-001"),
4270 exec_engine: crate::config::LiveExecutionEngineConfig {
4271 reconciliation: false,
4272 ..Default::default()
4273 },
4274 ..Default::default()
4275 };
4276
4277 let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4278 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4279
4280 let handler = ShareableMessageHandler::from_typed({
4281 let received = received.clone();
4282 move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4283 });
4284
4285 msgbus::subscribe_any(
4286 MessagingSwitchboard::queue_state_changed_pattern(channel),
4287 handler,
4288 None,
4289 );
4290
4291 let transitions = [
4292 QueueStateTransition {
4293 channel: SystemChannel::DataEvents,
4294 condition: QueueCondition::Backlogged,
4295 state: QueueState::Triggered,
4296 queue_depth: 17,
4297 mean_dispatch_ns: 23,
4298 },
4299 QueueStateTransition {
4300 channel: SystemChannel::ExecCommands,
4301 condition: QueueCondition::Slow,
4302 state: QueueState::Triggered,
4303 queue_depth: 17,
4304 mean_dispatch_ns: 23,
4305 },
4306 ];
4307
4308 node.publish_queue_state_transitions(&transitions);
4309
4310 let events = received.borrow();
4311 assert_eq!(events.len(), expected_count);
4312
4313 let expected = transitions
4314 .into_iter()
4315 .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4316
4317 for (event, transition) in events.iter().zip(expected) {
4318 assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4319 assert_eq!(event.channel, transition.channel);
4320 assert_eq!(event.condition, transition.condition);
4321 assert_eq!(event.state, transition.state);
4322 assert_eq!(event.queue_depth, transition.queue_depth);
4323 assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4324 assert_ne!(event.event_id, UUID4::default());
4325 assert_ne!(event.ts_event, UnixNanos::default());
4326 assert_eq!(event.ts_init, event.ts_event);
4327 }
4328
4329 if channel.is_none() {
4330 assert_ne!(events[0].event_id, events[1].event_id);
4331 }
4332
4333 drop(events);
4334 msgbus::get_message_bus().borrow_mut().dispose();
4335 }
4336
4337 #[rstest]
4338 #[case(None, None)]
4339 #[case(Some(ClientId::from("BINANCE")), None)]
4340 #[case(None, Some("binance-futures-market-streams"))]
4341 #[case(
4342 Some(ClientId::from("BINANCE")),
4343 Some("binance-futures-market-streams")
4344 )]
4345 fn test_process_socket_state_change_reaches_typed_subscriber(
4346 #[case] client_id: Option<ClientId>,
4347 #[case] endpoint: Option<&str>,
4348 ) {
4349 let config = LiveNodeConfig {
4350 trader_id: TraderId::from("SOCKET-001"),
4351 exec_engine: crate::config::LiveExecutionEngineConfig {
4352 reconciliation: false,
4353 ..Default::default()
4354 },
4355 ..Default::default()
4356 };
4357
4358 let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4359 let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4360
4361 let handler = ShareableMessageHandler::from_typed({
4362 let received = received.clone();
4363 move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4364 });
4365
4366 msgbus::subscribe_any(
4367 MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4368 handler,
4369 None,
4370 );
4371
4372 let change = SocketStateChange::new(
4373 ClientId::from("BINANCE"),
4374 Some(Venue::from("BINANCE")),
4375 Ustr::from("binance-futures-market-streams"),
4376 SocketState::Disconnected,
4377 );
4378
4379 node.process_system_event(SystemEvent::SocketState(change));
4380
4381 let events = received.borrow();
4382 assert_eq!(events.len(), 1);
4383 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4384 assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4385 assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4386 assert_eq!(
4387 events[0].endpoint,
4388 Ustr::from("binance-futures-market-streams")
4389 );
4390 assert_eq!(events[0].state, SocketState::Disconnected);
4391 assert_ne!(events[0].event_id, UUID4::default());
4392 assert_ne!(events[0].ts_event, UnixNanos::default());
4393 assert_eq!(events[0].ts_init, events[0].ts_event);
4394 drop(events);
4395 msgbus::get_message_bus().borrow_mut().dispose();
4396 }
4397
4398 #[rstest]
4399 #[case::accepted(
4400 ReconnectRequestOutcome::Accepted,
4401 SocketReconnectDispatchOutcome::Accepted
4402 )]
4403 #[case::already_reconnecting(
4404 ReconnectRequestOutcome::AlreadyReconnecting,
4405 SocketReconnectDispatchOutcome::AlreadyReconnecting
4406 )]
4407 #[case::disconnected(
4408 ReconnectRequestOutcome::Disconnected,
4409 SocketReconnectDispatchOutcome::Disconnected
4410 )]
4411 #[case::closed(
4412 ReconnectRequestOutcome::Closed,
4413 SocketReconnectDispatchOutcome::Closed
4414 )]
4415 #[case::unsupported(
4416 ReconnectRequestOutcome::Unsupported,
4417 SocketReconnectDispatchOutcome::Unsupported
4418 )]
4419 fn test_request_socket_reconnect_maps_transport_outcome(
4420 #[case] transport: ReconnectRequestOutcome,
4421 #[case] expected: SocketReconnectDispatchOutcome,
4422 ) {
4423 let registry = SocketReconnectRegistry::default();
4424 let client_id = ClientId::from("TEST");
4425 let endpoint = Ustr::from("test-streams");
4426 let control = SocketControl::with_registry(client_id, None, endpoint, ®istry);
4427 let _sink = control.sink();
4428 control.register(move || transport);
4429
4430 let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4431
4432 assert_eq!(outcome, expected);
4433 }
4434
4435 #[rstest]
4436 #[case::client_not_found(
4437 SocketReconnectLookup::ClientNotFound,
4438 SocketReconnectDispatchOutcome::UnknownClient
4439 )]
4440 #[case::unsupported(
4441 SocketReconnectLookup::Unsupported,
4442 SocketReconnectDispatchOutcome::Unsupported
4443 )]
4444 #[case::endpoint_not_found(
4445 SocketReconnectLookup::EndpointNotFound,
4446 SocketReconnectDispatchOutcome::UnknownEndpoint
4447 )]
4448 #[case::ambiguous(
4449 SocketReconnectLookup::AmbiguousEndpoint,
4450 SocketReconnectDispatchOutcome::AmbiguousEndpoint
4451 )]
4452 fn test_request_socket_reconnect_maps_lookup_failure(
4453 #[case] lookup: SocketReconnectLookup,
4454 #[case] expected: SocketReconnectDispatchOutcome,
4455 ) {
4456 assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4457 }
4458
4459 #[rstest]
4460 fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4461 let trader_id = TraderId::from("SOCKET-001");
4462
4463 let config = LiveNodeConfig {
4464 trader_id,
4465 ..Default::default()
4466 };
4467
4468 let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4469 let client_id = ClientId::from("TEST");
4470 let endpoint = Ustr::from("test-streams");
4471 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4472 let sender = DispatchSender::new(tx);
4473 let events = sender.clone();
4474 msgbus::subscribe_any(
4475 MessagingSwitchboard::socket_state_changed_pattern(
4476 Some(client_id),
4477 Some(endpoint.as_str()),
4478 ),
4479 ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4480 events.send(17u32).unwrap();
4481 }),
4482 None,
4483 );
4484
4485 let control =
4486 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4487 let _sink = control.sink();
4488 control.register(move || {
4489 sender.send(23u32).unwrap();
4490 ReconnectRequestOutcome::Accepted
4491 });
4492
4493 let event = SystemEvent::SocketState(SocketStateChange::new(
4494 client_id,
4495 None,
4496 endpoint,
4497 SocketState::Connected,
4498 ));
4499 let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4500 trader_id,
4501 client_id,
4502 endpoint,
4503 31.into(),
4504 ));
4505
4506 if startup {
4507 let mut pending = PendingEvents::default();
4508 pending.system_events.push(event.into());
4509 pending.system_commands.push(command.into());
4510 node.process_system_events(pending.take_system_events());
4511 node.process_system_commands(pending.take_system_commands());
4512 assert!(pending.is_empty());
4513 } else {
4514 node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4515 node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4516 }
4517
4518 let event_child = rx.try_recv().unwrap();
4519 let command_child = rx.try_recv().unwrap();
4520 assert!(event_child.is_rooted());
4521 assert!(command_child.is_rooted());
4522 assert_eq!(event_child.dispatch(|value| value), 17);
4523 assert_eq!(command_child.dispatch(|value| value), 23);
4524 assert!(rx.is_empty());
4525 msgbus::get_message_bus().borrow_mut().dispose();
4526 }
4527
4528 #[rstest]
4529 fn test_process_socket_reconnect_routes_only_matching_trader() {
4530 let trader_id = TraderId::from("SOCKET-001");
4531
4532 let config = LiveNodeConfig {
4533 trader_id,
4534 exec_engine: crate::config::LiveExecutionEngineConfig {
4535 reconciliation: false,
4536 ..Default::default()
4537 },
4538 ..Default::default()
4539 };
4540
4541 let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4542 let client_id = ClientId::from("TEST");
4543 let endpoint = Ustr::from("test-streams");
4544 let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4545 let request_count = Arc::clone(&requests);
4546 let control =
4547 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4548 let _sink = control.sink();
4549 control.register(move || {
4550 request_count.fetch_add(1, Ordering::SeqCst);
4551 ReconnectRequestOutcome::Accepted
4552 });
4553
4554 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4555 TraderId::from("OTHER-001"),
4556 client_id,
4557 endpoint,
4558 UnixNanos::default(),
4559 )));
4560 assert_eq!(requests.load(Ordering::SeqCst), 0);
4561
4562 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4563 trader_id,
4564 client_id,
4565 endpoint,
4566 UnixNanos::default(),
4567 )));
4568 assert_eq!(requests.load(Ordering::SeqCst), 1);
4569 }
4570
4571 #[rstest]
4572 #[tokio::test]
4573 async fn test_start_publishes_socket_change_after_actor_subscribes() {
4574 let config = LiveNodeConfig {
4575 trader_id: TraderId::from("SOCKET-STARTUP-001"),
4576 exec_engine: crate::config::LiveExecutionEngineConfig {
4577 reconciliation: false,
4578 ..Default::default()
4579 },
4580 timeout_connection: Duration::ZERO,
4581 timeout_reconciliation: Duration::ZERO,
4582 timeout_portfolio: Duration::ZERO,
4583 timeout_disconnection: Duration::ZERO,
4584 delay_post_stop: Duration::ZERO,
4585 timeout_shutdown: Duration::ZERO,
4586 ..Default::default()
4587 };
4588
4589 let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4590 let received = Rc::new(RefCell::new(Vec::new()));
4591 node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4592 .unwrap();
4593 node.runner.as_ref().unwrap().bind_senders();
4594
4595 let change = SocketStateChange::new(
4596 ClientId::from("BINANCE"),
4597 Some(Venue::from("BINANCE")),
4598 Ustr::from("binance-futures-market-streams"),
4599 SocketState::Connected,
4600 );
4601 get_system_event_sender()
4602 .send(SystemEvent::SocketState(change))
4603 .unwrap();
4604
4605 node.start().await.unwrap();
4606
4607 {
4608 let events = received.borrow();
4609 assert_eq!(events.len(), 1);
4610 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4611 assert_eq!(events[0].client_id, change.client_id);
4612 assert_eq!(events[0].venue, change.venue);
4613 assert_eq!(events[0].endpoint, change.endpoint);
4614 assert_eq!(events[0].state, change.state);
4615 assert_ne!(events[0].event_id, UUID4::default());
4616 assert_ne!(events[0].ts_event, UnixNanos::default());
4617 assert_eq!(events[0].ts_init, events[0].ts_event);
4618 }
4619
4620 node.stop().await.unwrap();
4621 node.dispose();
4622 }
4623
4624 #[rstest]
4625 #[tokio::test(flavor = "current_thread")]
4626 async fn test_run_publishes_queue_state_after_dispatch_sample() {
4627 let config = LiveNodeConfig {
4628 trader_id: TraderId::from("QUEUE-RUN-001"),
4629 queue_monitor: Some(crate::config::QueueMonitorConfig {
4630 queue_depth_trigger: usize::MAX,
4631 queue_depth_clear: 0,
4632 mean_dispatch_ns_trigger: 1,
4633 mean_dispatch_ns_clear: 0,
4634 }),
4635 exec_engine: crate::config::LiveExecutionEngineConfig {
4636 reconciliation: false,
4637 ..Default::default()
4638 },
4639 delay_post_stop: Duration::ZERO,
4640 ..Default::default()
4641 };
4642
4643 let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4644 let handle = node.handle();
4645 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4646
4647 let handler = ShareableMessageHandler::from_typed({
4648 let received = received.clone();
4649 let stop_handle = handle.clone();
4650
4651 move |event: &QueueStateChanged| {
4652 received.borrow_mut().push(event.clone());
4653 stop_handle.stop();
4654 }
4655 });
4656
4657 msgbus::subscribe_any(
4658 MessagingSwitchboard::queue_state_changed_pattern(None),
4659 handler,
4660 None,
4661 );
4662 let drive_handle = handle.clone();
4663
4664 let result = tokio::time::timeout(Duration::from_secs(5), async {
4665 let run = node.run();
4666 tokio::pin!(run);
4667
4668 let drive = async move {
4669 wait_until_async(
4670 || async { drive_handle.is_running() },
4671 Duration::from_secs(2),
4672 )
4673 .await;
4674 get_data_event_sender().send(stub_data_event()).unwrap();
4675 };
4676
4677 let (run_result, ()) = tokio::join!(run, drive);
4678 run_result
4679 })
4680 .await;
4681
4682 assert!(
4683 result.is_ok(),
4684 "queue state event should arrive before timeout"
4685 );
4686 assert!(result.unwrap().is_ok(), "run() should succeed");
4687 let events = received.borrow();
4688 assert_eq!(events.len(), 1);
4689 assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4690 assert_eq!(events[0].channel, SystemChannel::DataEvents);
4691 assert_eq!(events[0].condition, QueueCondition::Slow);
4692 assert_eq!(events[0].state, QueueState::Triggered);
4693 assert_eq!(events[0].queue_depth, 0);
4694 assert!(events[0].mean_dispatch_ns > 0);
4695 assert_ne!(events[0].event_id, UUID4::default());
4696 assert_ne!(events[0].ts_event, UnixNanos::default());
4697 assert_eq!(events[0].ts_init, events[0].ts_event);
4698 drop(events);
4699 msgbus::get_message_bus().borrow_mut().dispose();
4700 }
4701
4702 #[rstest]
4703 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4704 async fn test_run_dispatch_progress_under_load(
4705 #[values(false, true)] execution: bool,
4706 #[values(false, true)] slow: bool,
4707 ) {
4708 let config = LiveNodeConfig {
4709 delay_post_stop: Duration::ZERO,
4710 ..Default::default()
4711 };
4712 let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4713 let handle = node.handle();
4714 let stop = handle.clone();
4715 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4716 if slow {
4717 std::thread::sleep(Duration::from_millis(110));
4719 }
4720
4721 if event.ts_init == UnixNanos::from(8) {
4722 let snapshot = stop.metrics_snapshot();
4724 assert_eq!(snapshot.time_events.dispatched, 7);
4725
4726 if execution {
4727 assert!((1..16).contains(&snapshot.exec_events.dispatched));
4728 assert_eq!(snapshot.data_events.dispatched, 0);
4729 } else {
4730 assert!(snapshot.data_events.dispatched > 0);
4731 assert_eq!(snapshot.data_commands.dispatched, 1);
4732 }
4733 stop.stop();
4734 }
4735 }));
4736
4737 let drive = async {
4738 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4739
4740 for ts in (1..=16_u64).map(UnixNanos::from) {
4741 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4742 TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4743 callback.clone(),
4744 ));
4745 get_data_event_sender().send(stub_data_event()).unwrap();
4746
4747 if execution {
4748 get_exec_event_sender().send(stub_exec_event()).unwrap();
4749 }
4750 }
4751 nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4752 };
4753 tokio::time::timeout(Duration::from_secs(5), async {
4754 let (result, ()) = tokio::join!(node.run(), drive);
4755 result.unwrap();
4756 })
4757 .await
4758 .unwrap();
4759 assert!(handle.should_stop());
4760 }
4761
4762 #[rstest]
4763 fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4764 let config = LiveNodeConfig {
4765 exec_engine: crate::config::LiveExecutionEngineConfig {
4766 reconciliation: false,
4767 ..Default::default()
4768 },
4769 ..Default::default()
4770 };
4771
4772 let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4773 let event = stub_exec_event();
4774 let account_id = AccountId::from("TEST-001");
4775 let instrument_id = InstrumentId::from("TEST.VENUE");
4776 let trade_id = TradeId::from("T-001");
4777
4778 let close_ids = node.observe_exec_event_before_dispatch(&event);
4779 assert_eq!(close_ids, Some(Vec::new()));
4780 assert!(
4781 !node
4782 .exec_manager
4783 .is_fill_recently_processed(account_id, instrument_id, trade_id)
4784 );
4785
4786 node.exec_manager
4787 .mark_fill_processed(account_id, instrument_id, trade_id);
4788
4789 let close_ids = node.observe_exec_event_before_dispatch(&event);
4790 assert_eq!(close_ids, None);
4791 }
4792
4793 #[rstest]
4794 #[case(false, false, OrderStatus::Canceled, 1)]
4795 #[case(false, true, OrderStatus::Accepted, 0)]
4796 #[case(true, false, OrderStatus::Canceled, 1)]
4797 #[case(true, true, OrderStatus::Accepted, 0)]
4798 fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4799 #[case] with_fills: bool,
4800 #[case] superseded: bool,
4801 #[case] expected_status: OrderStatus,
4802 #[case] expected_query_count: usize,
4803 ) {
4804 let config = LiveNodeConfig {
4805 exec_engine: crate::config::LiveExecutionEngineConfig {
4806 reconciliation: true,
4807 open_check_threshold_ms: 5_000,
4808 single_order_query_delay_ms: 0,
4809 ..Default::default()
4810 },
4811 ..Default::default()
4812 };
4813
4814 let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4815 let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4816 let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4817 let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4818 let account_id = AccountId::from("TEST-001");
4819 let client_id = ClientId::from("TEST");
4820 let instrument = crypto_perpetual_ethusdt();
4821 let instrument_id = instrument.id();
4822 let account = AccountAny::Margin(MarginAccount::new(
4823 AccountState::new(
4824 account_id,
4825 AccountType::Margin,
4826 vec![AccountBalance::new(
4827 Money::from("1000000 USDT"),
4828 Money::from("0 USDT"),
4829 Money::from("1000000 USDT"),
4830 )],
4831 Vec::new(),
4832 true,
4833 UUID4::new(),
4834 UnixNanos::default(),
4835 UnixNanos::default(),
4836 Some(Currency::USDT()),
4837 ),
4838 true,
4839 ));
4840 node.kernel.cache.borrow_mut().add_account(account).unwrap();
4841 node.kernel
4842 .cache
4843 .borrow_mut()
4844 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4845 .unwrap();
4846 insert_accepted_limit_order_in_node(
4847 &node,
4848 account_id,
4849 client_id,
4850 instrument_id,
4851 client_order_id,
4852 old_venue_order_id,
4853 );
4854
4855 if superseded {
4856 let order = node
4857 .kernel
4858 .cache
4859 .borrow()
4860 .order_owned(&client_order_id)
4861 .unwrap();
4862 let pending_update = OrderPendingUpdateSpec::builder()
4863 .trader_id(order.trader_id())
4864 .strategy_id(order.strategy_id())
4865 .instrument_id(order.instrument_id())
4866 .client_order_id(client_order_id)
4867 .account_id(account_id)
4868 .venue_order_id(old_venue_order_id)
4869 .build();
4870 node.kernel
4871 .cache
4872 .borrow_mut()
4873 .update_order(&OrderEventAny::PendingUpdate(pending_update))
4874 .unwrap();
4875 let order = node
4876 .kernel
4877 .cache
4878 .borrow()
4879 .order_owned(&client_order_id)
4880 .unwrap();
4881 let updated = OrderUpdatedSpec::builder()
4882 .trader_id(order.trader_id())
4883 .strategy_id(order.strategy_id())
4884 .instrument_id(order.instrument_id())
4885 .client_order_id(client_order_id)
4886 .quantity(order.quantity())
4887 .venue_order_id(new_venue_order_id)
4888 .account_id(account_id)
4889 .build();
4890 node.kernel
4891 .cache
4892 .borrow_mut()
4893 .update_order(&OrderEventAny::Updated(updated))
4894 .unwrap();
4895 }
4896
4897 let report = OrderStatusReport::new(
4898 account_id,
4899 instrument_id,
4900 Some(client_order_id),
4901 old_venue_order_id,
4902 OrderSide::Buy.into(),
4903 OrderType::Limit,
4904 TimeInForce::Gtc,
4905 OrderStatus::Canceled,
4906 Quantity::from("10.0"),
4907 Quantity::from("0.0"),
4908 UnixNanos::from(1_000),
4909 UnixNanos::from(2_000),
4910 UnixNanos::from(3_000),
4911 None,
4912 );
4913
4914 let report = if with_fills {
4915 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4916 } else {
4917 ExecutionReport::Order(Box::new(report))
4918 };
4919
4920 let event = ExecutionEvent::Report(report);
4921
4922 node.process_exec_event(event);
4923
4924 let order = node
4925 .kernel
4926 .cache
4927 .borrow()
4928 .order_owned(&client_order_id)
4929 .unwrap();
4930
4931 let expected_venue_order_id = if superseded {
4932 new_venue_order_id
4933 } else {
4934 old_venue_order_id
4935 };
4936
4937 assert_eq!(order.status(), expected_status);
4938 assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4939
4940 if !superseded {
4941 let replacement = OrderTestBuilder::new(OrderType::Limit)
4942 .client_order_id(client_order_id)
4943 .instrument_id(instrument_id)
4944 .quantity(Quantity::from("20.0"))
4945 .price(Price::from("200.0"))
4946 .build();
4947 let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4948 node.kernel
4949 .cache
4950 .borrow_mut()
4951 .add_order(replacement, None, Some(client_id), true)
4952 .unwrap();
4953 let replacement = node
4954 .kernel
4955 .cache
4956 .borrow_mut()
4957 .update_order(&submitted)
4958 .unwrap();
4959 let accepted =
4960 TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4961 node.kernel
4962 .cache
4963 .borrow_mut()
4964 .update_order(&accepted)
4965 .unwrap();
4966 }
4967
4968 assert_eq!(
4969 node.exec_manager.check_open_order_queries().len(),
4970 expected_query_count,
4971 );
4972 }
4973
4974 #[rstest]
4975 fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4976 let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4977
4978 let OrderEventAny::Filled(fill) = &mut fill_event else {
4979 unreachable!();
4980 };
4981
4982 fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4983 fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4984 let fill = fill.clone();
4985 let report_event = fill_report_event(&fill);
4986 let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4987
4988 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4989 let marked_before_dispatch = is_recent_fill(&node, &fill);
4990
4991 node.dispatch_exec_event_and_commit_fill(event);
4992
4993 let marked_after_dispatch = is_recent_fill(&node, &fill);
4994 let later_report_is_eligible = node
4995 .observe_exec_event_before_dispatch(&report_event)
4996 .is_some();
4997 assert_eq!(
4998 (
4999 marked_before_dispatch,
5000 marked_after_dispatch,
5001 later_report_is_eligible,
5002 ),
5003 (false, false, true),
5004 );
5005 }
5006
5007 #[rstest]
5008 fn test_applied_direct_fill_commits_and_skips_later_report() {
5009 let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
5010
5011 let OrderEventAny::Filled(fill) = &fill_event else {
5012 unreachable!();
5013 };
5014
5015 let fill = fill.clone();
5016 let report_event = fill_report_event(&fill);
5017 let event = ExecutionEvent::Order(fill_event);
5018
5019 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
5020 assert!(!is_recent_fill(&node, &fill));
5021
5022 node.dispatch_exec_event_and_commit_fill(event);
5023
5024 assert!(is_recent_fill(&node, &fill));
5025 assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
5026 }
5027
5028 #[rstest]
5029 fn test_canonical_duplicate_fill_counts_as_applied() {
5030 let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
5031
5032 let OrderEventAny::Filled(fill) = &fill_event else {
5033 unreachable!();
5034 };
5035
5036 let mut fill = fill.clone();
5037 node.kernel
5038 .cache
5039 .borrow_mut()
5040 .update_order(&fill_event)
5041 .unwrap();
5042 fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
5043
5044 node.exec_manager.commit_recent_fill_if_applied(&fill);
5045
5046 assert!(is_recent_fill(&node, &fill));
5047 }
5048
5049 #[rstest]
5050 #[case(false)]
5051 #[case(true)]
5052 fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
5053 let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
5054 "AppliedContinuousFillNode"
5055 } else {
5056 "RejectedContinuousFillNode"
5057 });
5058
5059 if !applied {
5060 let OrderEventAny::Filled(fill) = &mut fill_event else {
5061 unreachable!();
5062 };
5063
5064 fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5065 fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5066 }
5067
5068 let OrderEventAny::Filled(fill) = &fill_event else {
5069 unreachable!();
5070 };
5071
5072 let fill = fill.clone();
5073
5074 node.process_reconciliation_events(&[fill_event]);
5075
5076 assert_eq!(is_recent_fill(&node, &fill), applied);
5077 }
5078
5079 #[rstest]
5080 fn test_recent_fill_commit_requires_account_and_instrument_match() {
5081 let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5082 node.kernel
5083 .cache
5084 .borrow_mut()
5085 .update_order(&fill_event)
5086 .unwrap();
5087
5088 let OrderEventAny::Filled(fill) = fill_event else {
5089 unreachable!();
5090 };
5091
5092 let mut account_mismatch = fill.clone();
5093 account_mismatch.account_id = AccountId::from("OTHER-001");
5094 let mut instrument_mismatch = fill;
5095 instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5096
5097 node.exec_manager
5098 .commit_recent_fill_if_applied(&account_mismatch);
5099 node.exec_manager
5100 .commit_recent_fill_if_applied(&instrument_mismatch);
5101
5102 assert!(!is_recent_fill(&node, &account_mismatch));
5103 assert!(!is_recent_fill(&node, &instrument_mismatch));
5104 }
5105
5106 #[rstest]
5107 fn test_applied_inferred_fill_remains_recently_processed() {
5108 let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5109 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5110 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5111 let account_id = AccountId::from("TEST-001");
5112 let order = node
5113 .kernel
5114 .cache
5115 .borrow()
5116 .order_owned(&client_order_id)
5117 .unwrap();
5118 let report = OrderStatusReport::new(
5119 account_id,
5120 instrument.id(),
5121 Some(client_order_id),
5122 venue_order_id,
5123 OrderSide::Buy.into(),
5124 OrderType::Limit,
5125 TimeInForce::Gtc,
5126 OrderStatus::PartiallyFilled,
5127 Quantity::from("10.0"),
5128 Quantity::from("1.0"),
5129 UnixNanos::from(1_000),
5130 UnixNanos::from(1_000),
5131 UnixNanos::from(1_000),
5132 None,
5133 )
5134 .with_avg_px(dec!(100.0));
5135 let inferred = create_inferred_fill_for_qty(
5136 &order,
5137 &report,
5138 &account_id,
5139 &instrument,
5140 Quantity::from("1.0"),
5141 UnixNanos::from(1_000),
5142 None,
5143 )
5144 .unwrap();
5145
5146 let OrderEventAny::Filled(fill) = &inferred else {
5147 unreachable!();
5148 };
5149
5150 let fill = fill.clone();
5151
5152 node.process_reconciliation_events(&[inferred]);
5153
5154 assert!(fill.reconciliation);
5155 assert!(is_recent_fill(&node, &fill));
5156 }
5157
5158 #[rstest]
5159 fn test_drain_exec_events_before_reconciliation_applies_real_fill_first() {
5160 let (mut node, fill_event, instrument) = recent_fill_test_fixture("DeferredFillNode");
5161
5162 let OrderEventAny::Filled(fill) = fill_event else {
5163 unreachable!();
5164 };
5165
5166 let account_id = fill.account_id;
5167 let venue_order_id = fill.venue_order_id;
5168 let client_id = ClientId::from("TEST-EVIDENCE");
5169 node.kernel
5170 .exec_engine
5171 .borrow_mut()
5172 .register_client(Box::new(StubExecutionClient::new(
5173 client_id,
5174 account_id,
5175 instrument.id().venue,
5176 OmsType::Netting,
5177 None,
5178 )))
5179 .unwrap();
5180
5181 let report = FillReport::new(
5182 account_id,
5183 instrument.id(),
5184 venue_order_id,
5185 fill.trade_id,
5186 fill.order_side,
5187 fill.last_qty,
5188 fill.last_px,
5189 fill.commission.unwrap(),
5190 fill.liquidity_side,
5191 Some(fill.client_order_id),
5192 None,
5193 fill.ts_event,
5194 fill.ts_init,
5195 None,
5196 );
5197 let (exec_evt_tx, mut exec_evt_rx) =
5198 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
5199 exec_evt_tx
5200 .send(ExecutionEvent::Report(ExecutionReport::Fill(Box::new(report))).into())
5201 .unwrap();
5202
5203 node.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
5204
5205 assert!(exec_evt_rx.try_recv().is_err());
5206 let order = node
5207 .kernel
5208 .cache
5209 .borrow()
5210 .order_owned(&fill.client_order_id)
5211 .unwrap();
5212 assert_eq!(order.filled_qty(), fill.last_qty);
5213 assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5214
5215 let report = OrderStatusReport::new(
5216 account_id,
5217 instrument.id(),
5218 Some(fill.client_order_id),
5219 venue_order_id,
5220 OrderSide::Buy.into(),
5221 OrderType::Limit,
5222 TimeInForce::Gtc,
5223 OrderStatus::PartiallyFilled,
5224 Quantity::from("10.0"),
5225 fill.last_qty,
5226 UnixNanos::from(1_000),
5227 UnixNanos::from(1_000),
5228 UnixNanos::from(1_000),
5229 None,
5230 )
5231 .with_avg_px(dec!(100.0));
5232 let engine = node.kernel.exec_engine.borrow();
5233 let clients = engine.get_all_clients();
5234 let check = node
5235 .exec_manager
5236 .prepare_open_order_report_check(UUID4::new(), &clients);
5237 let reconciliation = node.exec_manager.reconcile_open_order_reports(
5238 &check,
5239 vec![SourcedOrderStatusReport { client_id, report }],
5240 &IndexSet::new(),
5241 &IndexSet::new(),
5242 &clients,
5243 );
5244 drop(engine);
5245 node.process_reconciliation_events(&reconciliation.events);
5246
5247 let order = node
5248 .kernel
5249 .cache
5250 .borrow()
5251 .order_owned(&fill.client_order_id)
5252 .unwrap();
5253 assert_eq!(order.filled_qty(), fill.last_qty);
5254 assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5255 }
5256
5257 #[rstest]
5258 #[tokio::test]
5259 async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5260 let client_id = ClientId::from("POSITION-FILLS");
5261 let account_id = AccountId::from("POSITION-FILLS-001");
5262 let instrument_id = crypto_perpetual_ethusdt().id();
5263 let commands = Rc::new(RefCell::new(Vec::new()));
5264
5265 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5266 client_id,
5267 account_id,
5268 venue: instrument_id.venue,
5269 outcome: FillReportClientOutcome::Failure,
5270 commands: commands.clone(),
5271 }));
5272
5273 let command = GenerateFillReports::new(
5274 UUID4::new(),
5275 UnixNanos::from(2_000),
5276 Some(instrument_id),
5277 None,
5278 Some(UnixNanos::from(1_000)),
5279 Some(UnixNanos::from(2_000)),
5280 None,
5281 None,
5282 );
5283
5284 let result = request_position_fill_reports(
5285 vec![client],
5286 vec![PositionFillReportQuery {
5287 key: (instrument_id, account_id),
5288 client_id,
5289 command: command.clone(),
5290 }],
5291 )
5292 .await
5293 .finish();
5294
5295 assert_eq!(*commands.borrow(), vec![command]);
5296 assert!(result.is_empty());
5297 }
5298
5299 #[rstest]
5300 #[tokio::test]
5301 async fn test_request_position_fill_reports_accepts_scoped_response() {
5302 let client_id = ClientId::from("POSITION-FILLS");
5303 let account_id = AccountId::from("POSITION-FILLS-001");
5304 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5305 let instrument_id = instrument.id();
5306
5307 let report_b = FillReport::new(
5308 account_id,
5309 instrument_id,
5310 VenueOrderId::from("V-POSITION-FILLS"),
5311 TradeId::from("T-POSITION-FILLS-B"),
5312 OrderSide::Buy,
5313 Quantity::from("1.0"),
5314 Price::from("100.0"),
5315 Money::zero(instrument.quote_currency()),
5316 LiquiditySide::Taker,
5317 Some(ClientOrderId::from("O-POSITION-FILLS")),
5318 None,
5319 UnixNanos::from(1_500),
5320 UnixNanos::from(2_000),
5321 None,
5322 );
5323 let mut report_a = report_b.clone();
5324 report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5325 let commands = Rc::new(RefCell::new(Vec::new()));
5326
5327 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5328 client_id,
5329 account_id,
5330 venue: instrument_id.venue,
5331 outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5332 commands,
5333 }));
5334
5335 let command = GenerateFillReports::new(
5336 UUID4::new(),
5337 UnixNanos::from(2_000),
5338 Some(instrument_id),
5339 None,
5340 Some(UnixNanos::from(1_000)),
5341 Some(UnixNanos::from(2_000)),
5342 None,
5343 None,
5344 );
5345 let key = (instrument_id, account_id);
5346
5347 let result = request_position_fill_reports(
5348 vec![client],
5349 vec![PositionFillReportQuery {
5350 key,
5351 client_id,
5352 command,
5353 }],
5354 )
5355 .await
5356 .finish();
5357
5358 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5359 assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5360 }
5361
5362 #[rstest]
5363 #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5364 #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5365 #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5366 #[tokio::test]
5367 async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5368 #[case] report_account: &str,
5369 #[case] report_instrument: &str,
5370 #[case] ts_event: u64,
5371 #[case] quantity: &str,
5372 ) {
5373 let client_id = ClientId::from("POSITION-FILLS");
5374 let account_id = AccountId::from("POSITION-FILLS-001");
5375 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5376 let instrument_id = instrument.id();
5377
5378 let report = FillReport::new(
5379 AccountId::from(report_account),
5380 InstrumentId::from(report_instrument),
5381 VenueOrderId::from("V-POSITION-FILLS"),
5382 TradeId::from("T-POSITION-FILLS"),
5383 OrderSide::Buy,
5384 Quantity::from(quantity),
5385 Price::from("100.0"),
5386 Money::zero(instrument.quote_currency()),
5387 LiquiditySide::Taker,
5388 Some(ClientOrderId::from("O-POSITION-FILLS")),
5389 None,
5390 UnixNanos::from(ts_event),
5391 UnixNanos::from(2_000),
5392 None,
5393 );
5394
5395 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5396 client_id,
5397 account_id,
5398 venue: instrument_id.venue,
5399 outcome: FillReportClientOutcome::Reports(vec![report]),
5400 commands: Rc::new(RefCell::new(Vec::new())),
5401 }));
5402
5403 let command = GenerateFillReports::new(
5404 UUID4::new(),
5405 UnixNanos::from(2_000),
5406 Some(instrument_id),
5407 None,
5408 Some(UnixNanos::from(1_000)),
5409 Some(UnixNanos::from(2_000)),
5410 None,
5411 None,
5412 );
5413 let key = (instrument_id, account_id);
5414
5415 let result = request_position_fill_reports(
5416 vec![client],
5417 vec![PositionFillReportQuery {
5418 key,
5419 client_id,
5420 command,
5421 }],
5422 )
5423 .await
5424 .finish();
5425
5426 assert!(result.is_empty());
5427 }
5428
5429 #[rstest]
5430 #[case(999)]
5431 #[case(2_001)]
5432 #[tokio::test]
5433 async fn test_request_position_fill_reports_filters_time_window_superset(
5434 #[case] ts_event: u64,
5435 ) {
5436 let client_id = ClientId::from("POSITION-FILLS");
5437 let account_id = AccountId::from("POSITION-FILLS-001");
5438 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5439 let instrument_id = instrument.id();
5440
5441 let report = FillReport::new(
5442 account_id,
5443 instrument_id,
5444 VenueOrderId::from("V-POSITION-FILLS"),
5445 TradeId::from("T-POSITION-FILLS"),
5446 OrderSide::Buy,
5447 Quantity::from("1.0"),
5448 Price::from("100.0"),
5449 Money::zero(instrument.quote_currency()),
5450 LiquiditySide::Taker,
5451 Some(ClientOrderId::from("O-POSITION-FILLS")),
5452 None,
5453 UnixNanos::from(ts_event),
5454 UnixNanos::from(2_000),
5455 None,
5456 );
5457
5458 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5459 client_id,
5460 account_id,
5461 venue: instrument_id.venue,
5462 outcome: FillReportClientOutcome::Reports(vec![report]),
5463 commands: Rc::new(RefCell::new(Vec::new())),
5464 }));
5465
5466 let command = GenerateFillReports::new(
5467 UUID4::new(),
5468 UnixNanos::from(2_000),
5469 Some(instrument_id),
5470 None,
5471 Some(UnixNanos::from(1_000)),
5472 Some(UnixNanos::from(2_000)),
5473 None,
5474 None,
5475 );
5476 let key = (instrument_id, account_id);
5477
5478 let result = request_position_fill_reports(
5479 vec![client],
5480 vec![PositionFillReportQuery {
5481 key,
5482 client_id,
5483 command,
5484 }],
5485 )
5486 .await
5487 .finish();
5488
5489 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5490 assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5491 }
5492
5493 #[rstest]
5494 #[case::query_error(false)]
5495 #[case::missing_client(true)]
5496 #[tokio::test]
5497 async fn test_position_fill_collection_discards_partial_failure(
5498 #[case] missing_client: bool,
5499 #[values(false, true)] failure_first: bool,
5500 position_fill_query: (FillReport, PositionFillReportQuery),
5501 ) {
5502 let (report, query) = position_fill_query;
5503 let failed_client_id = ClientId::from("FAILED-FILLS");
5504 let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5505 let mut healthy_report = report.clone();
5506 healthy_report.account_id = healthy_key.1;
5507 let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5508 let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5509 let empty_client_id = ClientId::from("EMPTY-FILLS");
5510 let mut clients = vec![
5511 LiveExecutionClient::new(Box::new(FillReportClient {
5512 client_id: query.client_id,
5513 account_id: query.key.1,
5514 venue: query.key.0.venue,
5515 outcome: FillReportClientOutcome::Reports(vec![report]),
5516 commands: Rc::new(RefCell::new(Vec::new())),
5517 })),
5518 LiveExecutionClient::new(Box::new(FillReportClient {
5519 client_id: healthy_client_id,
5520 account_id: healthy_key.1,
5521 venue: healthy_key.0.venue,
5522 outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5523 commands: Rc::new(RefCell::new(Vec::new())),
5524 })),
5525 LiveExecutionClient::new(Box::new(FillReportClient {
5526 client_id: empty_client_id,
5527 account_id: empty_key.1,
5528 venue: empty_key.0.venue,
5529 outcome: FillReportClientOutcome::Reports(Vec::new()),
5530 commands: Rc::new(RefCell::new(Vec::new())),
5531 })),
5532 ];
5533
5534 if !missing_client {
5535 clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5536 client_id: failed_client_id,
5537 account_id: query.key.1,
5538 venue: query.key.0.venue,
5539 outcome: FillReportClientOutcome::Failure,
5540 commands: Rc::new(RefCell::new(Vec::new())),
5541 })));
5542 }
5543
5544 let failed_query = PositionFillReportQuery {
5545 client_id: failed_client_id,
5546 key: query.key,
5547 command: query.command.clone(),
5548 };
5549
5550 let healthy_query = PositionFillReportQuery {
5551 key: healthy_key,
5552 client_id: healthy_client_id,
5553 command: query.command.clone(),
5554 };
5555
5556 let empty_query = PositionFillReportQuery {
5557 key: empty_key,
5558 client_id: empty_client_id,
5559 command: query.command.clone(),
5560 };
5561
5562 let mut queries = vec![query, failed_query];
5563
5564 if failure_first {
5565 queries.reverse();
5566 }
5567
5568 queries.insert(0, empty_query);
5569 queries.push(healthy_query);
5570
5571 let result = request_position_fill_reports(clients, queries)
5572 .await
5573 .finish();
5574
5575 assert_eq!(
5576 result.keys().copied().collect::<Vec<_>>(),
5577 vec![empty_key, healthy_key]
5578 );
5579 assert_eq!(
5580 result,
5581 IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5582 );
5583 }
5584
5585 #[rstest]
5586 #[case::same_economics("metadata", true)]
5587 #[case::venue_order("venue_order", false)]
5588 #[case::side("side", false)]
5589 #[case::quantity("quantity", false)]
5590 #[case::price("price", false)]
5591 #[case::commission("commission", false)]
5592 #[case::liquidity("liquidity", false)]
5593 #[case::average("average", false)]
5594 #[case::event_time("event_time", false)]
5595 #[case::client_order("client_order", false)]
5596 #[case::position("position", false)]
5597 #[tokio::test]
5598 async fn test_position_fill_collection_validates_duplicate_economics(
5599 #[case] changed: &str,
5600 #[case] accepted: bool,
5601 position_fill_query: (FillReport, PositionFillReportQuery),
5602 ) {
5603 let (report, query) = position_fill_query;
5604 let mut duplicate = report.clone();
5605
5606 match changed {
5607 "metadata" => {
5608 duplicate.report_id = UUID4::new();
5609 duplicate.ts_init = UnixNanos::from(3_000);
5610 }
5611 "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5612 "side" => duplicate.order_side = OrderSide::Sell,
5613 "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5614 "price" => duplicate.last_px = Price::from("101.0"),
5615 "commission" => duplicate.commission = Money::from("0.25 USDT"),
5616 "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5617 "average" => duplicate.avg_px = Some(dec!(100.5)),
5618 "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5619 "client_order" => duplicate.client_order_id = None,
5620 "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5621 _ => unreachable!(),
5622 }
5623
5624 let key = query.key;
5625
5626 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5627 client_id: query.client_id,
5628 account_id: key.1,
5629 venue: key.0.venue,
5630 outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5631 commands: Rc::new(RefCell::new(Vec::new())),
5632 }));
5633
5634 let result = request_position_fill_reports(vec![client], vec![query])
5635 .await
5636 .finish();
5637
5638 if accepted {
5639 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5640 assert_eq!(result, IndexMap::from([(key, vec![report])]));
5641 } else {
5642 assert!(result.is_empty());
5643 }
5644 }
5645
5646 #[fixture]
5647 fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5648 let account_id = AccountId::from("POSITION-FILLS-001");
5649 let instrument_id = crypto_perpetual_ethusdt().id();
5650
5651 let report = FillReport::new(
5652 account_id,
5653 instrument_id,
5654 VenueOrderId::from("V-POSITION-FILLS"),
5655 TradeId::from("T-POSITION-FILLS"),
5656 OrderSide::Buy,
5657 Quantity::from("1.0"),
5658 Price::from("100.0"),
5659 Money::from("0.10 USDT"),
5660 LiquiditySide::Taker,
5661 Some(ClientOrderId::from("O-POSITION-FILLS")),
5662 None,
5663 UnixNanos::from(1_500),
5664 UnixNanos::from(2_000),
5665 None,
5666 );
5667
5668 let query = PositionFillReportQuery {
5669 key: (instrument_id, account_id),
5670 client_id: ClientId::from("POSITION-FILLS"),
5671 command: GenerateFillReports::new(
5672 UUID4::new(),
5673 UnixNanos::from(2_000),
5674 Some(instrument_id),
5675 None,
5676 Some(UnixNanos::from(1_000)),
5677 Some(UnixNanos::from(2_000)),
5678 None,
5679 None,
5680 ),
5681 };
5682
5683 (report, query)
5684 }
5685
5686 #[rstest]
5687 #[case::failed_query(false)]
5688 #[case::local_activity(true)]
5689 fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5690 let (mut node, report, _) =
5691 position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5692 let active_key = (report.instrument_id, report.account_id);
5693 let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5694 let deferred_reports = vec![
5695 PositionStatusReport::new(
5696 deferred_key.1,
5697 deferred_key.0,
5698 PositionSide::Long,
5699 Quantity::from("2.0"),
5700 UnixNanos::from(1_000),
5701 UnixNanos::from(1_000),
5702 None,
5703 None,
5704 Some(dec!(100.0)),
5705 ),
5706 PositionStatusReport::new(
5707 deferred_key.1,
5708 deferred_key.0,
5709 PositionSide::Short,
5710 Quantity::from("1.0"),
5711 UnixNanos::from(1_000),
5712 UnixNanos::from(1_000),
5713 None,
5714 None,
5715 Some(dec!(100.0)),
5716 ),
5717 ];
5718 let mut check = node
5719 .exec_manager
5720 .prepare_position_report_check(UUID4::new(), &[]);
5721 check.client_coverage.clear();
5722 let events = node.exec_manager.reconcile_position_reports(
5723 &check,
5724 deferred_reports.clone(),
5725 &IndexSet::new(),
5726 &IndexSet::new(),
5727 );
5728 assert!(events.is_empty());
5729 assert_eq!(
5730 node.exec_manager.position_recon_retry_count(&deferred_key),
5731 1
5732 );
5733
5734 let mut position_result = position_report_result(&node, report);
5735 position_result.reports.extend(deferred_reports);
5736 position_result.check.client_coverage.insert(
5737 deferred_key,
5738 ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5739 );
5740 position_result
5741 .check
5742 .activity_revisions
5743 .insert(deferred_key, 0);
5744 position_result
5745 .queried_clients
5746 .insert(ClientId::from("SECOND"));
5747 let mut reports = IndexMap::from([(active_key, Vec::new())]);
5748 if local_activity {
5749 reports.insert(deferred_key, Vec::new());
5750 node.exec_manager
5751 .record_position_activity(deferred_key.0, deferred_key.1);
5752 }
5753
5754 node.handle_position_fill_report_result(PositionFillReportResult {
5755 position_result,
5756 reports,
5757 });
5758
5759 assert_eq!(
5760 node.exec_manager.position_recon_retry_count(&deferred_key),
5761 1
5762 );
5763 {
5764 let cache = node.kernel.cache.borrow();
5765 let positions =
5766 cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5767 assert_eq!(
5768 positions
5769 .iter()
5770 .map(|position| position.quantity)
5771 .collect::<Vec<_>>(),
5772 vec![Quantity::from("1.0"), Quantity::from("1.0")],
5773 );
5774 assert_eq!(
5775 cache
5776 .positions_open(
5777 None,
5778 Some(&deferred_key.0),
5779 None,
5780 Some(&deferred_key.1),
5781 None,
5782 )
5783 .len(),
5784 0
5785 );
5786 }
5787
5788 let mut fresh_check = node
5789 .exec_manager
5790 .prepare_position_report_check(UUID4::new(), &[]);
5791 node.exec_manager.plan_position_fill_reports(
5792 &mut fresh_check,
5793 &[],
5794 &IndexSet::new(),
5795 &IndexSet::new(),
5796 &[],
5797 );
5798
5799 assert_eq!(
5800 node.exec_manager.position_recon_retry_count(&deferred_key),
5801 0
5802 );
5803 }
5804
5805 #[rstest]
5806 fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5807 let (mut node, venue_report, fill_report) =
5808 position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5809 let key = (venue_report.instrument_id, venue_report.account_id);
5810 let revision = node.exec_manager.position_activity_revision(&key);
5811 let position_result = position_report_result(&node, venue_report);
5812
5813 node.handle_position_fill_report_result(PositionFillReportResult {
5814 position_result,
5815 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5816 });
5817
5818 let cache = node.kernel.cache.borrow();
5819 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5820 assert_eq!(
5821 positions
5822 .iter()
5823 .map(|position| position.quantity)
5824 .sum::<Quantity>(),
5825 Quantity::from("2.0")
5826 );
5827 assert_eq!(
5828 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5829 1
5830 );
5831 drop(positions);
5832 drop(cache);
5833 assert!(
5834 node.exec_manager
5835 .position_contains_fill_report(&fill_report)
5836 );
5837 assert_eq!(
5838 node.exec_manager.position_activity_revision(&key),
5839 revision + 1
5840 );
5841 }
5842
5843 #[rstest]
5844 fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5845 let (mut node, venue_report, mut first) =
5846 position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5847 first.commission = Money::from("0.13 USDT");
5848 let key = (first.instrument_id, first.account_id);
5849 let client_order_id = first.client_order_id.unwrap();
5850 let position_result = position_report_result(&node, venue_report.clone());
5851 node.handle_position_fill_report_result(PositionFillReportResult {
5852 position_result,
5853 reports: IndexMap::from([(key, vec![first.clone()])]),
5854 });
5855
5856 let mut second = first.clone();
5857 second.trade_id = TradeId::from("T-PREFIX-SECOND");
5858 second.last_qty = Quantity::from("0.75");
5859 second.commission = Money::from("0.29 USDT");
5860 let mut third = first.clone();
5861 third.trade_id = TradeId::from("T-PREFIX-THIRD");
5862 third.last_qty = Quantity::from("0.25");
5863 third.commission = Money::from("0.07 USDT");
5864 let revision = node.exec_manager.position_activity_revision(&key);
5865 let received = Rc::new(RefCell::new(Vec::new()));
5866 let cache = node.kernel.cache.clone();
5867 let callback_received = received.clone();
5868 let second_trade_id = second.trade_id;
5869
5870 let handler = TypedHandler::from(move |event: &OrderEventAny| {
5871 let OrderEventAny::Filled(fill) = event else {
5872 return;
5873 };
5874
5875 callback_received.borrow_mut().push(fill.trade_id);
5876 if fill.trade_id == second_trade_id {
5877 let mut cache = cache.borrow_mut();
5878 let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5879 let mut position = cache.position_mut(&position_id).unwrap();
5880 position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5881 }
5882 });
5883
5884 let strategy_id = node
5885 .kernel
5886 .cache
5887 .borrow()
5888 .order(&client_order_id)
5889 .unwrap()
5890 .strategy_id();
5891 let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5892 msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5893 let position_result = position_report_result(&node, venue_report);
5894 node.handle_position_fill_report_result(PositionFillReportResult {
5895 position_result,
5896 reports: IndexMap::from([(
5897 key,
5898 vec![first.clone(), first.clone(), second.clone(), third.clone()],
5899 )]),
5900 });
5901
5902 msgbus::unsubscribe_order_events(topic.into(), &handler);
5903 let cache = node.kernel.cache.borrow();
5904 let order = cache.order(&client_order_id).unwrap();
5905 let position_id = cache.position_id(&client_order_id).unwrap();
5906 let position = cache.position_ref(position_id).unwrap();
5907
5908 assert_eq!(*received.borrow(), vec![second.trade_id]);
5909 assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5910 assert_eq!(order.events().len(), 6);
5911 assert_eq!(
5912 order.trade_ids(),
5913 vec![
5914 &TradeId::from("T-POSITION-INITIAL"),
5915 &first.trade_id,
5916 &second.trade_id
5917 ]
5918 );
5919 assert_eq!(
5920 order.commissions().values().copied().collect::<Vec<_>>(),
5921 vec![Money::from("0.42 USDT")]
5922 );
5923 assert_eq!(position.quantity, Quantity::from("2.25"));
5924 assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5925 assert_eq!(position.replay_events.len(), 2);
5926 assert_eq!(
5927 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5928 1
5929 );
5930 assert_eq!(
5931 node.exec_manager.position_activity_revision(&key),
5932 revision + 1
5933 );
5934 assert!(node.exec_manager.position_contains_fill_report(&first));
5935 assert!(!node.exec_manager.position_contains_fill_report(&second));
5936 assert!(!node.exec_manager.position_contains_fill_report(&third));
5937 assert!(
5938 !node
5939 .exec_manager
5940 .is_fill_recently_processed(key.1, key.0, second.trade_id)
5941 );
5942 assert!(
5943 !node
5944 .exec_manager
5945 .is_fill_recently_processed(key.1, key.0, third.trade_id)
5946 );
5947 }
5948
5949 #[rstest]
5950 fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5951 let (mut node, venue_report, fill_report) =
5952 position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5953 let key = (venue_report.instrument_id, venue_report.account_id);
5954 let position_result = position_report_result(&node, venue_report);
5955
5956 node.handle_position_fill_report_result(PositionFillReportResult {
5957 position_result,
5958 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5959 });
5960
5961 let mut rest_report = fill_report;
5962 rest_report.ts_event = UnixNanos::from(2_000);
5963
5964 assert!(
5965 node.exec_manager
5966 .position_contains_fill_report(&rest_report)
5967 );
5968 }
5969
5970 #[rstest]
5971 fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5972 let (mut node, venue_report, fill_report) =
5973 position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5974 let key = (venue_report.instrument_id, venue_report.account_id);
5975 let client_order_id = fill_report.client_order_id.unwrap();
5976 apply_inferred_position_fill(&mut node, &fill_report);
5977
5978 let venue_report = PositionStatusReport::new(
5979 key.1,
5980 key.0,
5981 PositionSide::Long,
5982 Quantity::from("3.0"),
5983 venue_report.ts_last,
5984 venue_report.ts_init,
5985 None,
5986 None,
5987 venue_report.avg_px_open,
5988 );
5989 let mut later_fill_report = fill_report.clone();
5990 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5991 later_fill_report.ts_event = UnixNanos::from(1_001);
5992 let position_result = position_report_result(&node, venue_report);
5993
5994 node.handle_position_fill_report_result(PositionFillReportResult {
5995 position_result,
5996 reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5997 });
5998
5999 let cache = node.kernel.cache.borrow();
6000 let order = cache.order(&client_order_id).unwrap();
6001 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6002 assert_eq!(order.filled_qty(), Quantity::from("2.0"));
6003 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6004 assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
6005 assert_eq!(
6006 positions
6007 .iter()
6008 .map(|position| position.quantity)
6009 .sum::<Quantity>(),
6010 Quantity::from("3.0")
6011 );
6012 assert_eq!(
6013 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6014 2
6015 );
6016 }
6017
6018 #[rstest]
6019 fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
6020 let (mut node, _, _) =
6021 position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
6022 let account_id = AccountId::from("TEST-001");
6023 let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
6024 let key = (instrument.id(), account_id);
6025 let commission = Money::zero(instrument.quote_currency());
6026 node.kernel
6027 .cache
6028 .borrow_mut()
6029 .add_instrument(instrument.clone())
6030 .unwrap();
6031
6032 let order = OrderTestBuilder::new(OrderType::Market)
6033 .strategy_id(StrategyId::external())
6034 .instrument_id(instrument.id())
6035 .client_order_id(ClientOrderId::from("O-RECONCILED"))
6036 .quantity(Quantity::from("1.000000"))
6037 .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
6038 .build();
6039 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6040 let accepted =
6041 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
6042 node.kernel
6043 .cache
6044 .borrow_mut()
6045 .add_order(
6046 order.clone(),
6047 None,
6048 Some(ClientId::from("POSITION-FILLS")),
6049 false,
6050 )
6051 .unwrap();
6052 node.process_reconciliation_events(&[submitted, accepted]);
6053 let order = node
6054 .kernel
6055 .cache
6056 .borrow()
6057 .order_owned(&order.client_order_id())
6058 .unwrap();
6059 let opening_fill = TestOrderEventStubs::filled(
6060 &order,
6061 &instrument,
6062 Some(TradeId::from("T-RECONCILED")),
6063 None,
6064 Some(Price::from("100.00")),
6065 Some(Quantity::from("1.000000")),
6066 Some(LiquiditySide::Taker),
6067 Some(commission),
6068 Some(UnixNanos::from(2_000)),
6069 Some(account_id),
6070 );
6071 node.process_reconciliation_events(&[opening_fill]);
6072
6073 let held_fill_report = FillReport::new(
6075 account_id,
6076 instrument.id(),
6077 VenueOrderId::from("V-HELD"),
6078 TradeId::from("T-HELD"),
6079 OrderSide::Buy,
6080 Quantity::from("1.000000"),
6081 Price::from("100.00"),
6082 commission,
6083 LiquiditySide::Taker,
6084 None,
6085 None,
6086 UnixNanos::from(1_000),
6087 UnixNanos::from(1_000),
6088 None,
6089 );
6090 node.kernel
6091 .exec_engine
6092 .borrow_mut()
6093 .reconcile_fill_report(&held_fill_report);
6094
6095 let venue_report = PositionStatusReport::new(
6096 account_id,
6097 instrument.id(),
6098 PositionSide::Long,
6099 Quantity::from("2.000000"),
6100 UnixNanos::from(3_000),
6101 UnixNanos::from(3_000),
6102 None,
6103 None,
6104 Some(dec!(100.00)),
6105 );
6106 let position_result = position_report_result(&node, venue_report);
6107
6108 node.handle_position_fill_report_result(PositionFillReportResult {
6109 position_result,
6110 reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
6111 });
6112
6113 let cache = node.kernel.cache.borrow();
6114 let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
6115 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6116 assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
6117 assert_eq!(
6118 positions
6119 .iter()
6120 .map(|position| position.quantity)
6121 .sum::<Quantity>(),
6122 Quantity::from("2.000000")
6123 );
6124 assert!(
6125 positions
6126 .iter()
6127 .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
6128 );
6129 }
6130
6131 #[rstest]
6132 fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
6133 let (mut node, _, mut fill_report) =
6134 position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
6135 apply_inferred_position_fill(&mut node, &fill_report);
6136 let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
6137 fill_report.venue_position_id = Some(conflicting_position_id);
6138
6139 let error = node
6140 .exec_manager
6141 .prepare_position_fill_report(&mut fill_report, &[])
6142 .unwrap_err();
6143
6144 assert!(
6145 error.to_string().contains(&format!(
6146 "position ID {conflicting_position_id} conflicts with cached order position"
6147 )),
6148 "{error:#}"
6149 );
6150 }
6151
6152 #[rstest]
6153 #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
6154 #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
6155 fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
6156 #[case] authoritative_qty: &str,
6157 #[case] client_order_id: &str,
6158 ) {
6159 let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6160 "UnappliedPositionFillNode",
6161 Quantity::from(authoritative_qty),
6162 );
6163 fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6164 let reports = [fill_report.clone()];
6165
6166 let quantities = (0..2)
6167 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6168 .collect::<Vec<_>>();
6169
6170 assert_eq!(
6171 quantities,
6172 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6173 );
6174 assert!(
6175 !node
6176 .exec_manager
6177 .position_contains_fill_report(&fill_report)
6178 );
6179 }
6180
6181 #[rstest]
6182 fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6183 let (mut node, venue_report, fill_report) =
6184 position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6185 let mut later_fill_report = fill_report.clone();
6186 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6187 later_fill_report.ts_event = UnixNanos::from(1_001);
6188 let reports = [fill_report.clone(), later_fill_report.clone()];
6189
6190 let quantities = (0..3)
6191 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6192 .collect::<Vec<_>>();
6193
6194 assert_eq!(
6195 quantities,
6196 vec![
6197 Quantity::from("1.0"),
6198 Quantity::from("1.0"),
6199 Quantity::from("2.0"),
6200 ]
6201 );
6202 assert!(
6203 !node
6204 .exec_manager
6205 .position_contains_fill_report(&fill_report)
6206 );
6207 assert!(
6208 !node
6209 .exec_manager
6210 .position_contains_fill_report(&later_fill_report)
6211 );
6212 }
6213
6214 #[rstest]
6215 #[case::within_grace(60_000, true)]
6216 #[case::after_grace(0, false)]
6217 fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6218 #[case] position_check_threshold_ms: u32,
6219 #[case] expected_applied: bool,
6220 ) {
6221 let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6222 "TransientPositionFillNode",
6223 Quantity::from("1.0"),
6224 position_check_threshold_ms,
6225 );
6226 let mut conflicting_report = fill_report.clone();
6227 conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6228
6229 let quantities = [conflicting_report, fill_report.clone()]
6230 .into_iter()
6231 .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6232 .collect::<Vec<_>>();
6233
6234 assert_eq!(
6235 quantities,
6236 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6237 );
6238 assert_eq!(
6239 node.exec_manager
6240 .position_contains_fill_report(&fill_report),
6241 expected_applied
6242 );
6243 }
6244
6245 #[rstest]
6246 fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6247 let (mut node, venue_report, fill_report) =
6248 position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6249 let key = (venue_report.instrument_id, venue_report.account_id);
6250 let first_result = position_report_result(&node, venue_report.clone());
6251
6252 node.handle_position_fill_report_result(PositionFillReportResult {
6253 position_result: first_result,
6254 reports: IndexMap::from([(key, vec![fill_report])]),
6255 });
6256
6257 let fresh_result = position_report_result(&node, venue_report);
6258 node.handle_position_fill_report_result(PositionFillReportResult {
6259 position_result: fresh_result,
6260 reports: IndexMap::from([(key, Vec::new())]),
6261 });
6262
6263 let cache = node.kernel.cache.borrow();
6264 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6265 assert_eq!(
6266 positions
6267 .iter()
6268 .map(|position| position.quantity)
6269 .sum::<Quantity>(),
6270 Quantity::from("2.0")
6271 );
6272 assert_eq!(
6273 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6274 2
6275 );
6276 }
6277
6278 #[rstest]
6279 #[case::below_limit(63)]
6280 #[case::at_limit(64)]
6281 #[case::above_limit(65)]
6282 fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6283 let (mut node, venue_report, fill_report) =
6284 position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6285 let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6286
6287 let venue_report = PositionStatusReport::new(
6288 venue_report.account_id,
6289 venue_report.instrument_id,
6290 PositionSide::Long,
6291 Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6292 venue_report.ts_last,
6293 venue_report.ts_init,
6294 None,
6295 None,
6296 venue_report.avg_px_open,
6297 );
6298 let key = (venue_report.instrument_id, venue_report.account_id);
6299 let client_order_id = fill_report.client_order_id.unwrap();
6300
6301 let reports = (0..count)
6302 .map(|index| {
6303 let mut report = fill_report.clone();
6304 report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6305 report.ts_event = UnixNanos::from(1_000 + index as u64);
6306 report
6307 })
6308 .collect::<Vec<_>>();
6309
6310 let position_result = position_report_result(&node, venue_report.clone());
6311
6312 node.handle_position_fill_report_result(PositionFillReportResult {
6313 position_result,
6314 reports: IndexMap::from([(key, reports.clone())]),
6315 });
6316
6317 {
6318 let cache = node.kernel.cache.borrow();
6319 let order = cache.order(&client_order_id).unwrap();
6320 let applied = count.min(64);
6321 assert_eq!(
6322 order.filled_qty().as_decimal(),
6323 dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6324 );
6325 assert_eq!(order.trade_ids().len(), applied + 1);
6326
6327 for (index, report) in reports.iter().enumerate() {
6328 assert_eq!(
6329 order.trade_ids().contains(&&report.trade_id),
6330 index < applied
6331 );
6332 }
6333
6334 assert_eq!(
6335 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6336 1
6337 );
6338 }
6339
6340 let position_result = position_report_result(&node, venue_report);
6342 node.handle_position_fill_report_result(PositionFillReportResult {
6343 position_result,
6344 reports: IndexMap::from([(key, reports)]),
6345 });
6346
6347 let cache = node.kernel.cache.borrow();
6348 let order = cache.order(&client_order_id).unwrap();
6349 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6350 assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6351 assert_eq!(order.trade_ids().len(), count + 1);
6352 assert_eq!(positions.len(), 1);
6353 assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6354 assert_eq!(positions[0].side, PositionSide::Long);
6355 assert_eq!(
6356 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6357 1
6358 );
6359 }
6360
6361 #[rstest]
6362 #[case::client_order(0)]
6363 #[case::venue_order(1)]
6364 #[case::side(2)]
6365 #[case::position(3)]
6366 fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6367 let (mut node, venue_report, valid_report) =
6368 position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6369 let key = (venue_report.instrument_id, venue_report.account_id);
6370 let client_order_id = valid_report.client_order_id.unwrap();
6371 let revision = node.exec_manager.position_activity_revision(&key);
6372 let position_result = position_report_result(&node, venue_report);
6373 let mut conflicting_report = valid_report.clone();
6374 conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6375
6376 match conflict {
6377 0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6378 1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6379 2 => conflicting_report.order_side = OrderSide::Sell,
6380 3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6381 _ => unreachable!(),
6382 }
6383
6384 node.handle_position_fill_report_result(PositionFillReportResult {
6385 position_result,
6386 reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6387 });
6388
6389 let cache = node.kernel.cache.borrow();
6390 let order = cache.order(&client_order_id).unwrap();
6391 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6392 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6393 assert_eq!(
6394 order.trade_ids(),
6395 vec![&TradeId::from("T-POSITION-INITIAL")]
6396 );
6397 assert_eq!(positions.len(), 1);
6398 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6399 assert_eq!(
6400 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6401 1
6402 );
6403 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6404 }
6405
6406 #[rstest]
6407 fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6408 let (mut node, venue_report, _) =
6409 position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6410 let key = (venue_report.instrument_id, venue_report.account_id);
6411 let position_result = position_report_result(&node, venue_report);
6412
6413 node.handle_position_fill_report_result(PositionFillReportResult {
6414 position_result,
6415 reports: IndexMap::new(),
6416 });
6417
6418 let cache = node.kernel.cache.borrow();
6419 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6420 assert_eq!(positions.len(), 1);
6421 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6422 assert_eq!(
6423 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6424 1
6425 );
6426 }
6427
6428 #[rstest]
6429 fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6430 let (mut node, mut venue_report, mut fill_report) =
6431 position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6432 node.kernel
6433 .exec_engine
6434 .borrow_mut()
6435 .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6436 let key = (venue_report.instrument_id, venue_report.account_id);
6437
6438 let position_id = {
6439 let cache = node.kernel.cache.borrow();
6440 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6441 assert_eq!(positions.len(), 1);
6442 positions[0].id
6443 };
6444
6445 venue_report.venue_position_id = Some(position_id);
6446 fill_report.client_order_id = None;
6447 fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6448 let position_result = position_report_result(&node, venue_report);
6449
6450 node.handle_position_fill_report_result(PositionFillReportResult {
6451 position_result,
6452 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6453 });
6454
6455 let cache = node.kernel.cache.borrow();
6456 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6457 assert_eq!(positions.len(), 1);
6458 assert_eq!(positions[0].id, position_id);
6459 assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6460 assert_eq!(
6461 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6462 2
6463 );
6464 drop(positions);
6465 drop(cache);
6466 assert!(
6467 !node
6468 .exec_manager
6469 .position_contains_fill_report(&fill_report)
6470 );
6471 }
6472
6473 #[rstest]
6474 fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6475 let (mut node, venue_report, fill_report) =
6476 position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6477 let key = (venue_report.instrument_id, venue_report.account_id);
6478 let client_order_id = fill_report.client_order_id.unwrap();
6479 let position_result = position_report_result(&node, venue_report);
6480 let revision = node.exec_manager.position_activity_revision(&key);
6481
6482 let order_report = OrderStatusReport::new(
6483 key.1,
6484 key.0,
6485 Some(client_order_id),
6486 fill_report.venue_order_id,
6487 OrderSide::Buy.into(),
6488 OrderType::Limit,
6489 TimeInForce::Gtc,
6490 OrderStatus::PartiallyFilled,
6491 Quantity::from("10.0"),
6492 Quantity::from("2.0"),
6493 UnixNanos::from(1_000),
6494 UnixNanos::from(1_000),
6495 UnixNanos::from(2_000),
6496 None,
6497 );
6498 let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6499 Box::new(order_report),
6500 vec![fill_report.clone()],
6501 ));
6502
6503 assert_eq!(
6504 node.observe_exec_event_before_dispatch(&event),
6505 Some(vec![client_order_id])
6506 );
6507
6508 assert_eq!(
6509 node.exec_manager.position_activity_revision(&key),
6510 revision + 1
6511 );
6512 assert!(
6513 !node
6514 .exec_manager
6515 .position_report_check_is_current(&position_result.check, &key)
6516 );
6517
6518 node.handle_position_fill_report_result(PositionFillReportResult {
6519 position_result,
6520 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6521 });
6522
6523 let cache = node.kernel.cache.borrow();
6524 let order = cache.order(&client_order_id).unwrap();
6525 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6526 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6527 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6528 assert_eq!(positions.len(), 1);
6529 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6530 assert_eq!(positions[0].side, PositionSide::Long);
6531 assert_eq!(
6532 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6533 1
6534 );
6535 }
6536
6537 #[rstest]
6538 fn test_position_fill_report_result_defers_after_local_position_activity() {
6539 let (mut node, venue_report, fill_report) =
6540 position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6541 let key = (venue_report.instrument_id, venue_report.account_id);
6542 let position_result = position_report_result(&node, venue_report);
6543 node.exec_manager.record_position_activity(key.0, key.1);
6544
6545 node.handle_position_fill_report_result(PositionFillReportResult {
6546 position_result,
6547 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6548 });
6549
6550 let cache = node.kernel.cache.borrow();
6551 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6552 assert_eq!(positions.len(), 1);
6553 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6554 assert_eq!(
6555 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6556 1
6557 );
6558 drop(positions);
6559 drop(cache);
6560 assert!(
6561 !node
6562 .exec_manager
6563 .position_contains_fill_report(&fill_report)
6564 );
6565 }
6566
6567 #[rstest]
6568 fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6569 let config = LiveNodeConfig {
6570 exec_engine: crate::config::LiveExecutionEngineConfig {
6571 reconciliation: true,
6572 open_check_threshold_ms: 5_000,
6573 single_order_query_delay_ms: 0,
6574 ..Default::default()
6575 },
6576 ..Default::default()
6577 };
6578
6579 let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6580 let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6581 let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6582 let instrument = crypto_perpetual_ethusdt();
6583 let instrument_id = instrument.id();
6584 let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6585 let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6586
6587 node.kernel
6588 .cache
6589 .borrow_mut()
6590 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6591 .unwrap();
6592 insert_accepted_limit_order_in_node(
6593 &node,
6594 account_id,
6595 client_id,
6596 instrument_id,
6597 client_order_id,
6598 venue_order_id,
6599 );
6600
6601 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6602
6603 let accepted = OrderAcceptedSpec::builder()
6604 .instrument_id(instrument_id)
6605 .client_order_id(client_order_id)
6606 .venue_order_id(venue_order_id)
6607 .account_id(account_id)
6608 .build();
6609 let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6610
6611 let close_ids = node.observe_exec_event_before_dispatch(&event);
6612
6613 assert_eq!(close_ids, Some(vec![client_order_id]));
6614 assert!(node.exec_manager.check_open_order_queries().is_empty());
6615 }
6616
6617 #[rstest]
6618 #[cfg_attr(
6619 not(all(feature = "simulation", madsim)),
6620 tokio::test(start_paused = true)
6621 )]
6622 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6623 async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6624 use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6625 use nautilus_model::{
6626 events::order::spec::OrderPendingCancelSpec, identifiers::ClientOrderId,
6627 };
6628
6629 let config = LiveNodeConfig {
6630 exec_engine: crate::config::LiveExecutionEngineConfig {
6631 reconciliation: true,
6632 inflight_check_threshold_ms: 100,
6633 inflight_check_retries: 1,
6634 ..Default::default()
6635 },
6636 ..Default::default()
6637 };
6638
6639 let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6640 let trader_id = TraderId::from("TESTER-001");
6641 let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6642 let account_id = AccountId::from("TEST-001");
6643 let instrument = crypto_perpetual_ethusdt();
6644 let instrument_id = instrument.id();
6645 node.kernel
6646 .exec_engine
6647 .borrow_mut()
6648 .register_client(Box::new(StubExecutionClient::new(
6649 ClientId::from("STUB"),
6650 account_id,
6651 instrument_id.venue,
6652 OmsType::Netting,
6653 None,
6654 )))
6655 .unwrap();
6656 node.kernel
6657 .exec_engine
6658 .borrow_mut()
6659 .set_default_client(ClientId::from("STUB"))
6660 .unwrap();
6661 let child_ids = [
6662 ClientOrderId::from("O-BATCH-CANCEL-1"),
6663 ClientOrderId::from("O-BATCH-CANCEL-2"),
6664 ];
6665 node.kernel
6666 .cache
6667 .borrow_mut()
6668 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6669 .unwrap();
6670
6671 for client_order_id in child_ids {
6672 let order = OrderTestBuilder::new(OrderType::Limit)
6673 .trader_id(trader_id)
6674 .strategy_id(strategy_id)
6675 .client_order_id(client_order_id)
6676 .instrument_id(instrument_id)
6677 .quantity(Quantity::from("10.0"))
6678 .price(Price::from("100.0"))
6679 .build();
6680 let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6681 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6685 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6686 let pending_cancel = OrderEventAny::PendingCancel(
6687 OrderPendingCancelSpec::builder()
6688 .trader_id(trader_id)
6689 .strategy_id(strategy_id)
6690 .instrument_id(instrument_id)
6691 .client_order_id(client_order_id)
6692 .account_id(account_id)
6693 .venue_order_id(venue_order_id)
6694 .build(),
6695 );
6696 let mut cache = node.kernel.cache.borrow_mut();
6697 cache.add_order(order, None, None, false).unwrap();
6698 cache.update_order(&submitted).unwrap();
6699 cache.update_order(&accepted).unwrap();
6700 cache.update_order(&pending_cancel).unwrap();
6701 }
6702
6703 let cancels = child_ids
6704 .into_iter()
6705 .map(|client_order_id| {
6706 CancelOrder::new(
6707 trader_id,
6708 None,
6709 strategy_id,
6710 instrument_id,
6711 client_order_id,
6712 None,
6713 UUID4::new(),
6714 UnixNanos::default(),
6715 None,
6716 None,
6717 )
6718 })
6719 .collect();
6720
6721 let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6722 trader_id,
6723 None,
6724 strategy_id,
6725 instrument_id,
6726 cancels,
6727 UUID4::new(),
6728 UnixNanos::default(),
6729 None,
6730 None,
6731 ));
6732
6733 node.observe_exec_command_before_dispatch(&command);
6734 advance_clock(Duration::from_millis(101)).await;
6735 let result = node.exec_manager.check_inflight_orders();
6736 let timed_out_ids = result
6737 .events
6738 .iter()
6739 .map(OrderEventAny::client_order_id)
6740 .collect::<IndexSet<_>>();
6741
6742 assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6743 assert_eq!(result.events.len(), child_ids.len());
6744 assert!(
6745 result
6746 .events
6747 .iter()
6748 .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6749 "batch-cancel children must time out as Canceled events",
6750 );
6751 }
6752
6753 #[rstest]
6754 #[cfg_attr(
6755 not(all(feature = "simulation", madsim)),
6756 tokio::test(start_paused = true)
6757 )]
6758 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6759 async fn test_risk_bound_command_does_not_register_inflight() {
6760 let config = LiveNodeConfig {
6761 exec_engine: crate::config::LiveExecutionEngineConfig {
6762 reconciliation: true,
6763 inflight_check_threshold_ms: 100,
6764 inflight_check_retries: 2,
6765 ..Default::default()
6766 },
6767 ..Default::default()
6768 };
6769
6770 let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6771 msgbus::register_trading_command_endpoint(
6772 MessagingSwitchboard::risk_engine_execute(),
6773 TypedIntoHandler::from(|_: TradingCommand| {}),
6774 );
6775 let instrument = crypto_perpetual_ethusdt();
6776 let instrument_id = instrument.id();
6777 let order = OrderTestBuilder::new(OrderType::Limit)
6778 .trader_id(node.trader_id())
6779 .strategy_id(StrategyId::from("S-RISK-DENIED"))
6780 .instrument_id(instrument_id)
6781 .side(OrderSide::Buy)
6782 .quantity(Quantity::from("1.000"))
6783 .price(Price::from("100.00"))
6784 .build();
6785 let client_order_id = order.client_order_id();
6786
6787 {
6788 let mut cache = node.kernel.cache.borrow_mut();
6789 cache
6790 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6791 .unwrap();
6792 cache.add_order(order.clone(), None, None, false).unwrap();
6793 }
6794
6795 let submit_order = SubmitOrder::new(
6796 order.trader_id(),
6797 None,
6798 order.strategy_id(),
6799 instrument_id,
6800 client_order_id,
6801 order.init_event().clone(),
6802 None,
6803 None,
6804 None,
6805 UUID4::new(),
6806 UnixNanos::default(),
6807 None,
6808 );
6809 node.process_exec_command(
6810 TradingCommandMessage::new(
6811 MessagingSwitchboard::risk_engine_execute(),
6812 TradingCommand::SubmitOrder(submit_order),
6813 )
6814 .into(),
6815 );
6816
6817 advance_clock(Duration::from_millis(101)).await;
6818 let result = node.exec_manager.check_inflight_orders();
6819 let status = node
6820 .kernel
6821 .cache
6822 .borrow()
6823 .order(&client_order_id)
6824 .unwrap()
6825 .status();
6826
6827 assert_eq!(status, OrderStatus::Initialized);
6828 assert_eq!(
6829 node.exec_manager.recon_check_retry_count(&client_order_id),
6830 0
6831 );
6832 assert!(result.events.is_empty());
6833 assert!(result.queries.is_empty());
6834 }
6835
6836 #[rstest]
6837 #[cfg_attr(
6838 not(all(feature = "simulation", madsim)),
6839 tokio::test(start_paused = true)
6840 )]
6841 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6842 async fn test_external_cancel_command_still_registers_inflight() {
6843 use nautilus_common::messages::execution::CancelOrder;
6844 let external = ClientId::from("EXTERNAL");
6845
6846 let config = LiveNodeConfig {
6847 exec_engine: crate::config::LiveExecutionEngineConfig {
6848 reconciliation: true,
6849 inflight_check_threshold_ms: 100,
6850 inflight_check_retries: 2,
6851 external_clients: Some(vec![external]),
6852 ..Default::default()
6853 },
6854 ..Default::default()
6855 };
6856
6857 let mut node = LiveNode::build("ExternalCancelNode".to_string(), Some(config)).unwrap();
6858 let instrument = crypto_perpetual_ethusdt();
6859 let mut order = OrderTestBuilder::new(OrderType::Limit)
6860 .trader_id(node.trader_id())
6861 .strategy_id(StrategyId::from("S-EXTERNAL"))
6862 .instrument_id(instrument.id())
6863 .quantity(Quantity::from("9.003"))
6864 .price(Price::from("113.47"))
6865 .build();
6866 let venue = VenueOrderId::from("V-EXTERNAL");
6867 order
6868 .apply(TestOrderEventStubs::accepted(
6869 &order,
6870 AccountId::from("SIM-001"),
6871 venue,
6872 ))
6873 .unwrap();
6874 let id = order.client_order_id();
6875 let command = TradingCommand::CancelOrder(CancelOrder::new(
6876 order.trader_id(),
6877 Some(external),
6878 order.strategy_id(),
6879 order.instrument_id(),
6880 id,
6881 Some(venue),
6882 UUID4::new(),
6883 17.into(),
6884 None,
6885 None,
6886 ));
6887 {
6888 let mut cache = node.kernel.cache.borrow_mut();
6889 cache.add_instrument(instrument.into()).unwrap();
6890 cache
6891 .add_order(order.clone(), None, Some(external), false)
6892 .unwrap();
6893 }
6894
6895 node.process_exec_command(
6896 TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
6897 );
6898 advance_clock(Duration::from_millis(101)).await;
6899 let result = node.exec_manager.check_inflight_orders();
6900
6901 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6902 panic!("expected external query");
6903 };
6904
6905 assert_eq!(
6906 (query.client_order_id, query.venue_order_id, query.client_id),
6907 (id, Some(venue), Some(external))
6908 );
6909 assert!(result.events.is_empty());
6910 assert_eq!(
6911 serde_json::to_value(node.kernel.cache.borrow().order(&id).unwrap().clone()).unwrap(),
6912 serde_json::to_value(&order).unwrap()
6913 );
6914 }
6915
6916 #[rstest]
6917 #[case::modify(false, false)]
6918 #[case::batch_modify(false, true)]
6919 #[case::cancel(true, false)]
6920 #[case::batch_cancel(true, true)]
6921 #[cfg_attr(
6922 not(all(feature = "simulation", madsim)),
6923 tokio::test(start_paused = true)
6924 )]
6925 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6926 async fn test_native_command_denial_does_not_leave_inflight_tracking(
6927 #[case] cancel: bool,
6928 #[case] batch: bool,
6929 #[values(false, true)] old_pending: bool,
6930 ) {
6931 use nautilus_common::messages::execution::{
6932 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
6933 };
6934 use nautilus_model::events::order::spec::{OrderModifyDeniedSpec, OrderPendingUpdateSpec};
6935
6936 let config = LiveNodeConfig {
6937 exec_engine: crate::config::LiveExecutionEngineConfig {
6938 reconciliation: true,
6939 inflight_check_threshold_ms: 100,
6940 inflight_check_retries: 2,
6941 ..Default::default()
6942 },
6943 ..Default::default()
6944 };
6945
6946 let mut node = LiveNode::build("LocalDenialNode".to_string(), Some(config)).unwrap();
6947 let instrument = crypto_perpetual_ethusdt();
6948 let mut order = OrderTestBuilder::new(OrderType::Limit)
6949 .trader_id(node.trader_id())
6950 .strategy_id(StrategyId::from("S-LOCAL-DENIED"))
6951 .instrument_id(instrument.id())
6952 .side(OrderSide::Buy)
6953 .quantity(Quantity::from("9.003"))
6954 .price(Price::from("113.47"))
6955 .build();
6956 let account_id = AccountId::from("SIM-001");
6957 let venue_id = VenueOrderId::from("V-LOCAL-DENIED");
6958 order
6959 .apply(TestOrderEventStubs::accepted(&order, account_id, venue_id))
6960 .unwrap();
6961 let id = order.client_order_id();
6962 let old_cause = UUID4::new();
6963
6964 if old_pending {
6965 let mut marker = OrderPendingUpdateSpec::builder()
6966 .trader_id(order.trader_id())
6967 .strategy_id(order.strategy_id())
6968 .instrument_id(order.instrument_id())
6969 .client_order_id(id)
6970 .venue_order_id(venue_id)
6971 .account_id(account_id)
6972 .build();
6973 marker.causation_id = Some(old_cause);
6974 order.apply(OrderEventAny::PendingUpdate(marker)).unwrap();
6975 node.exec_manager.register_inflight(id, old_cause);
6976 }
6977
6978 let expected_status = order.status();
6979 let events_before = order.event_count();
6980 {
6981 let mut cache = node.kernel.cache.borrow_mut();
6982 cache.add_instrument(instrument.into()).unwrap();
6983 cache.add_order(order.clone(), None, None, false).unwrap();
6984 }
6985
6986 let cause = UUID4::new();
6987
6988 let command = if cancel {
6989 let command = CancelOrder::new(
6990 order.trader_id(),
6991 None,
6992 order.strategy_id(),
6993 order.instrument_id(),
6994 id,
6995 Some(venue_id),
6996 cause,
6997 17.into(),
6998 None,
6999 None,
7000 );
7001
7002 if batch {
7003 TradingCommand::CancelOrders(BatchCancelOrders::new(
7004 order.trader_id(),
7005 None,
7006 order.strategy_id(),
7007 order.instrument_id(),
7008 vec![command],
7009 UUID4::new(),
7010 19.into(),
7011 None,
7012 None,
7013 ))
7014 } else {
7015 TradingCommand::CancelOrder(command)
7016 }
7017 } else {
7018 let command = ModifyOrder::new(
7019 order.trader_id(),
7020 None,
7021 order.strategy_id(),
7022 order.instrument_id(),
7023 id,
7024 Some(venue_id),
7025 Some(Quantity::from("8.001")),
7026 None,
7027 None,
7028 cause,
7029 17.into(),
7030 None,
7031 None,
7032 );
7033
7034 if batch {
7035 TradingCommand::ModifyOrders(BatchModifyOrders::new(
7036 order.trader_id(),
7037 None,
7038 order.strategy_id(),
7039 order.instrument_id(),
7040 vec![command],
7041 UUID4::new(),
7042 19.into(),
7043 None,
7044 None,
7045 ))
7046 } else {
7047 TradingCommand::ModifyOrder(command)
7048 }
7049 };
7050
7051 node.process_exec_command(
7052 TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
7053 );
7054 {
7055 let cache = node.kernel.cache.borrow();
7056 let cached = cache.order(&id).unwrap();
7057 match cached.last_event() {
7058 OrderEventAny::ModifyDenied(event) if !cancel => {
7059 assert_eq!(event.causation_id, Some(cause));
7060 }
7061 OrderEventAny::CancelDenied(event) if cancel => {
7062 assert_eq!(event.causation_id, Some(cause));
7063 }
7064 other => panic!("expected native command denial, was {other:?}"),
7065 }
7066
7067 assert_eq!(cached.status(), expected_status);
7068 assert_eq!(cached.quantity(), order.quantity());
7069 assert_eq!(cached.price(), order.price());
7070 assert_eq!(cached.filled_qty(), order.filled_qty());
7071 assert_eq!(cached.event_count(), events_before + 1);
7072 }
7073
7074 advance_clock(Duration::from_millis(101)).await;
7075 let result = node.exec_manager.check_inflight_orders();
7076 assert_eq!(result.queries.len(), usize::from(old_pending));
7077 assert!(result.events.is_empty());
7078
7079 if old_pending {
7080 let refusal = OrderModifyDeniedSpec::builder()
7081 .trader_id(order.trader_id())
7082 .strategy_id(order.strategy_id())
7083 .instrument_id(order.instrument_id())
7084 .client_order_id(id)
7085 .venue_order_id(venue_id)
7086 .account_id(account_id)
7087 .causation_id(old_cause)
7088 .build();
7089 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::ModifyDenied(refusal)));
7090 }
7091
7092 assert_eq!(node.exec_manager.recon_check_retry_count(&id), 0);
7093 }
7094
7095 #[rstest]
7096 #[cfg_attr(
7097 not(all(feature = "simulation", madsim)),
7098 tokio::test(start_paused = true)
7099 )]
7100 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7101 async fn test_risk_approved_command_registers_inflight() {
7102 let config = LiveNodeConfig {
7103 risk_engine: crate::config::LiveRiskEngineConfig {
7104 bypass: true,
7105 ..Default::default()
7106 },
7107 exec_engine: crate::config::LiveExecutionEngineConfig {
7108 reconciliation: true,
7109 inflight_check_threshold_ms: 100,
7110 inflight_check_retries: 2,
7111 ..Default::default()
7112 },
7113 ..Default::default()
7114 };
7115
7116 let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
7117 msgbus::register_trading_command_endpoint(
7118 MessagingSwitchboard::exec_engine_execute(),
7119 TypedIntoHandler::from(|_: TradingCommand| {}),
7120 );
7121 let instrument = crypto_perpetual_ethusdt();
7122 let instrument_id = instrument.id();
7123 let order = OrderTestBuilder::new(OrderType::Limit)
7124 .trader_id(node.trader_id())
7125 .strategy_id(StrategyId::from("S-RISK-APPROVED"))
7126 .instrument_id(instrument_id)
7127 .side(OrderSide::Buy)
7128 .quantity(Quantity::from("1.000"))
7129 .price(Price::from("100.00"))
7130 .build();
7131 let client_order_id = order.client_order_id();
7132
7133 {
7134 let mut cache = node.kernel.cache.borrow_mut();
7135 cache
7136 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7137 .unwrap();
7138 cache.add_order(order.clone(), None, None, false).unwrap();
7139 }
7140
7141 let submit_order = SubmitOrder::new(
7142 order.trader_id(),
7143 None,
7144 order.strategy_id(),
7145 instrument_id,
7146 client_order_id,
7147 order.init_event().clone(),
7148 None,
7149 None,
7150 None,
7151 UUID4::new(),
7152 UnixNanos::default(),
7153 None,
7154 );
7155 node.process_exec_command(
7156 TradingCommandMessage::new(
7157 MessagingSwitchboard::risk_engine_execute(),
7158 TradingCommand::SubmitOrder(submit_order),
7159 )
7160 .into(),
7161 );
7162
7163 advance_clock(Duration::from_millis(101)).await;
7164 let result = node.exec_manager.check_inflight_orders();
7165
7166 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
7167 panic!("expected one query order command");
7168 };
7169
7170 assert_eq!(query.client_order_id, client_order_id);
7171 assert_eq!(
7172 node.exec_manager.recon_check_retry_count(&client_order_id),
7173 1
7174 );
7175 assert!(result.events.is_empty());
7176 }
7177
7178 #[rstest]
7179 fn test_live_node_builder_clock_factory_drives_kernel_clock() {
7180 let calls = Rc::new(Cell::new(0usize));
7181 let calls_in_factory = calls.clone();
7182 let sentinel = UnixNanos::from(123_456_789_u64);
7183
7184 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7185 .unwrap()
7186 .with_reconciliation(false)
7187 .with_clock_factory(move || {
7188 calls_in_factory.set(calls_in_factory.get() + 1);
7189 let mut clock = VirtualClock::new();
7190 clock.advance_time(sentinel, true);
7191 Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
7192 })
7193 .build()
7194 .unwrap();
7195
7196 assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
7197 assert_eq!(calls.get(), 1);
7198 }
7199
7200 #[derive(Debug)]
7201 struct ReplayKernelEventStore {
7202 fail_restore: bool,
7203 }
7204
7205 impl KernelEventStore for ReplayKernelEventStore {
7206 fn restore_parent_cache(
7207 &mut self,
7208 _instance_id: UUID4,
7209 _cache: &mut Cache,
7210 ) -> anyhow::Result<()> {
7211 if self.fail_restore {
7212 anyhow::bail!("replay restore failed");
7213 }
7214
7215 Ok(())
7216 }
7217
7218 fn open(
7219 &mut self,
7220 _instance_id: UUID4,
7221 _components: &RegisteredComponents,
7222 _environment: Environment,
7223 ) -> anyhow::Result<()> {
7224 Ok(())
7225 }
7226
7227 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
7228 None
7229 }
7230
7231 fn seal(&mut self, _ts_init: UnixNanos) {}
7232
7233 fn run_id(&self) -> Option<&str> {
7234 Some("replay-child")
7235 }
7236
7237 fn parent_run_id(&self) -> Option<&str> {
7238 Some("seed-run")
7239 }
7240
7241 fn is_event_store_replay_configured(&self) -> bool {
7242 true
7243 }
7244
7245 fn is_halted(&self) -> bool {
7246 false
7247 }
7248 }
7249
7250 #[derive(Debug)]
7251 struct TestStrategy {
7252 core: StrategyCore,
7253 }
7254
7255 impl TestStrategy {
7256 fn new(config: StrategyConfig) -> Self {
7257 Self {
7258 core: StrategyCore::new(config),
7259 }
7260 }
7261 }
7262
7263 impl DataActor for TestStrategy {}
7264
7265 nautilus_strategy!(TestStrategy, {
7266 fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
7267 self.core.config.external_order_instrument_ids.clone()
7268 }
7269 });
7270
7271 fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
7272 let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
7275 .unwrap()
7276 .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
7277 reconciliation: false,
7278 ..Default::default()
7279 })
7280 .with_load_state(true)
7281 .with_name("TestKernel")
7282 .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
7283 Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
7284 });
7285
7286 builder.build().unwrap()
7287 }
7288
7289 #[rstest]
7290 fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
7291 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7292 .unwrap()
7293 .with_reconciliation(false)
7294 .with_delay_post_stop_secs(0)
7295 .with_timeout_connection(1)
7296 .build()
7297 .unwrap();
7298 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7299 let strategy_id = StrategyId::from("CLAIMS-001");
7300
7301 node.add_strategy(TestStrategy::new(StrategyConfig {
7302 strategy_id: Some(strategy_id),
7303 external_order_instrument_ids: Some(vec![instrument_id]),
7304 ..Default::default()
7305 }))
7306 .unwrap();
7307
7308 assert_eq!(
7309 node.exec_manager.get_external_order_claim(&instrument_id),
7310 Some(strategy_id)
7311 );
7312
7313 {
7314 let exec_engine = node.kernel().exec_engine.borrow();
7315 assert_eq!(
7316 exec_engine.get_external_order_claim(&instrument_id),
7317 Some(strategy_id)
7318 );
7319 }
7320 }
7321
7322 #[rstest]
7323 fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
7324 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7325 .unwrap()
7326 .with_reconciliation(false)
7327 .build()
7328 .unwrap();
7329 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7330 let strategy_id = StrategyId::from("CLAIMS-001");
7331
7332 node.register_external_order_claims(strategy_id, &[instrument_id])
7333 .unwrap();
7334
7335 assert_eq!(
7336 node.exec_manager.get_external_order_claim(&instrument_id),
7337 Some(strategy_id)
7338 );
7339 assert_eq!(
7340 node.kernel
7341 .exec_engine
7342 .borrow()
7343 .get_external_order_claim(&instrument_id),
7344 Some(strategy_id)
7345 );
7346 }
7347
7348 #[rstest]
7349 #[tokio::test]
7350 async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
7351 let mut node = live_node_with_replay_store(false);
7352 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7353 let strategy_id = StrategyId::from("CLAIMS-001");
7354
7355 node.start().await.unwrap();
7356 assert_eq!(node.state(), NodeState::Running);
7357
7358 node.register_external_order_claims(strategy_id, &[instrument_id])
7359 .unwrap();
7360
7361 assert_eq!(
7362 node.exec_manager.get_external_order_claim(&instrument_id),
7363 Some(strategy_id)
7364 );
7365 assert_eq!(
7366 node.kernel
7367 .exec_engine
7368 .borrow()
7369 .get_external_order_claim(&instrument_id),
7370 Some(strategy_id)
7371 );
7372 }
7373
7374 #[rstest]
7375 fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
7376 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7377 .unwrap()
7378 .with_reconciliation(false)
7379 .build()
7380 .unwrap();
7381 let existing_instrument = InstrumentId::from("AUDUSD.SIM");
7382 let new_instruments = [
7383 InstrumentId::from("EURUSD.SIM"),
7384 InstrumentId::from("GBPUSD.SIM"),
7385 ];
7386 let existing_strategy_id = StrategyId::from("CLAIMS-001");
7387 let new_strategy_id = StrategyId::from("CLAIMS-002");
7388 node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
7389 .unwrap();
7390
7391 let result = node.register_external_order_claims(
7392 new_strategy_id,
7393 &[new_instruments[0], existing_instrument, new_instruments[1]],
7394 );
7395
7396 assert!(result.is_err());
7397 assert_eq!(
7398 node.exec_manager
7399 .get_external_order_claim(&existing_instrument),
7400 Some(existing_strategy_id)
7401 );
7402
7403 for instrument_id in new_instruments {
7404 assert_eq!(
7405 node.exec_manager.get_external_order_claim(&instrument_id),
7406 None
7407 );
7408 assert_eq!(
7409 node.kernel
7410 .exec_engine
7411 .borrow()
7412 .get_external_order_claim(&instrument_id),
7413 None
7414 );
7415 }
7416 }
7417
7418 #[rstest]
7419 fn test_deregister_external_order_claims_allows_successor_to_claim() {
7420 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7421 .unwrap()
7422 .with_reconciliation(false)
7423 .build()
7424 .unwrap();
7425 let instruments = [
7426 InstrumentId::from("AUDUSD.SIM"),
7427 InstrumentId::from("EURUSD.SIM"),
7428 ];
7429 let first_strategy_id = StrategyId::from("CLAIMS-001");
7430 let successor_strategy_id = StrategyId::from("CLAIMS-002");
7431 node.register_external_order_claims(first_strategy_id, &instruments)
7432 .unwrap();
7433
7434 node.deregister_external_order_claims(first_strategy_id)
7435 .unwrap();
7436 node.register_external_order_claims(successor_strategy_id, &instruments)
7437 .unwrap();
7438
7439 for instrument_id in instruments {
7440 assert_eq!(
7441 node.exec_manager.get_external_order_claim(&instrument_id),
7442 Some(successor_strategy_id)
7443 );
7444 assert_eq!(
7445 node.kernel
7446 .exec_engine
7447 .borrow()
7448 .get_external_order_claim(&instrument_id),
7449 Some(successor_strategy_id)
7450 );
7451 }
7452 }
7453
7454 #[rstest]
7455 fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7456 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7457 .unwrap()
7458 .with_reconciliation(false)
7459 .build()
7460 .unwrap();
7461 let strategy_id = StrategyId::from("CLAIMS-001");
7462
7463 node.deregister_external_order_claims(strategy_id).unwrap();
7464 node.deregister_external_order_claims(strategy_id).unwrap();
7465 }
7466
7467 #[rstest]
7468 fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7469 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7470 .unwrap()
7471 .with_reconciliation(false)
7472 .with_delay_post_stop_secs(0)
7473 .with_timeout_connection(1)
7474 .build()
7475 .unwrap();
7476 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7477 let strategy_id = StrategyId::from("CLAIMS-001");
7478 let duplicate_strategy_id = StrategyId::from("OTHER-002");
7479
7480 node.add_strategy(TestStrategy::new(StrategyConfig {
7481 strategy_id: Some(strategy_id),
7482 external_order_instrument_ids: Some(vec![instrument_id]),
7483 ..Default::default()
7484 }))
7485 .unwrap();
7486
7487 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7488 strategy_id: Some(duplicate_strategy_id),
7489 external_order_instrument_ids: Some(vec![instrument_id]),
7490 ..Default::default()
7491 }));
7492
7493 assert!(result.is_err());
7494 assert!(
7495 result
7496 .unwrap_err()
7497 .to_string()
7498 .contains("already exists for CLAIMS-001")
7499 );
7500 assert_eq!(
7501 node.exec_manager.get_external_order_claim(&instrument_id),
7502 Some(strategy_id)
7503 );
7504
7505 {
7506 let exec_engine = node.kernel().exec_engine.borrow();
7507 assert_eq!(
7508 exec_engine.get_external_order_claim(&instrument_id),
7509 Some(strategy_id)
7510 );
7511 }
7512 }
7513
7514 #[rstest]
7515 fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7516 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7517 .unwrap()
7518 .with_reconciliation(false)
7519 .with_delay_post_stop_secs(0)
7520 .with_timeout_connection(1)
7521 .build()
7522 .unwrap();
7523 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7524 let strategy_id = StrategyId::from("CLAIMS-001");
7525
7526 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7527 strategy_id: Some(strategy_id),
7528 external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7529 ..Default::default()
7530 }));
7531
7532 assert!(result.is_err());
7533 assert!(
7534 result
7535 .unwrap_err()
7536 .to_string()
7537 .contains("appears more than once for CLAIMS-001")
7538 );
7539 assert_eq!(
7540 node.exec_manager.get_external_order_claim(&instrument_id),
7541 None
7542 );
7543
7544 {
7545 let exec_engine = node.kernel().exec_engine.borrow();
7546 assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7547 }
7548 }
7549
7550 #[rstest]
7551 fn test_add_strategy_failure_restores_external_order_claims() {
7552 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7553 .unwrap()
7554 .with_reconciliation(false)
7555 .with_delay_post_stop_secs(0)
7556 .with_timeout_connection(1)
7557 .build()
7558 .unwrap();
7559 let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7560 let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7561 let strategy_id = StrategyId::from("CLAIMS-001");
7562 node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7563 .unwrap();
7564
7565 let mut strategy = TestStrategy::new(StrategyConfig {
7566 strategy_id: Some(strategy_id),
7567 external_order_instrument_ids: Some(vec![configured_instrument_id]),
7568 ..Default::default()
7569 });
7570
7571 strategy
7572 .core
7573 .register(
7574 node.trader_id(),
7575 node.kernel.clock(),
7576 node.kernel.cache.clone(),
7577 node.kernel.portfolio.clone(),
7578 )
7579 .unwrap();
7580
7581 let result = node.add_strategy(strategy);
7582
7583 assert!(result.is_err());
7584 assert!(
7585 result
7586 .unwrap_err()
7587 .to_string()
7588 .contains("already registered with trader")
7589 );
7590 assert_eq!(
7591 node.exec_manager
7592 .get_external_order_claim(&existing_instrument_id),
7593 Some(strategy_id)
7594 );
7595 assert_eq!(
7596 node.kernel
7597 .exec_engine
7598 .borrow()
7599 .get_external_order_claim(&existing_instrument_id),
7600 Some(strategy_id)
7601 );
7602 assert_eq!(
7603 node.exec_manager
7604 .get_external_order_claim(&configured_instrument_id),
7605 None
7606 );
7607 assert_eq!(
7608 node.kernel
7609 .exec_engine
7610 .borrow()
7611 .get_external_order_claim(&configured_instrument_id),
7612 None
7613 );
7614 }
7615
7616 #[rstest]
7617 fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7618 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7619 .unwrap()
7620 .with_reconciliation(false)
7621 .build()
7622 .unwrap();
7623 let exec_engine = node.kernel.exec_engine.clone();
7624 let _engine_borrow = exec_engine.borrow_mut();
7625
7626 node.add_strategy(TestStrategy::new(StrategyConfig {
7627 strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7628 ..Default::default()
7629 }))
7630 .unwrap();
7631 }
7632
7633 #[rstest]
7634 fn test_add_strategy_registers_configured_hedging_oms_type() {
7635 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7636 .unwrap()
7637 .with_reconciliation(false)
7638 .with_delay_post_stop_secs(0)
7639 .with_timeout_connection(1)
7640 .build()
7641 .unwrap();
7642 let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7643
7644 node.add_strategy(TestStrategy::new(StrategyConfig {
7645 strategy_id: Some(strategy_id),
7646 oms_type: Some(OmsType::Hedging),
7647 ..Default::default()
7648 }))
7649 .unwrap();
7650
7651 let instrument = crypto_perpetual_ethusdt();
7652 let instrument_id = instrument.id();
7653 let client_id = ClientId::from("STUB");
7654
7655 node.kernel
7656 .cache
7657 .borrow_mut()
7658 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7659 .unwrap();
7660 node.kernel
7661 .exec_engine
7662 .borrow_mut()
7663 .register_client(Box::new(StubExecutionClient::new(
7664 client_id,
7665 AccountId::from("TEST-ACCOUNT"),
7666 instrument_id.venue,
7667 OmsType::Netting,
7668 None,
7669 )))
7670 .unwrap();
7671
7672 let order = OrderTestBuilder::new(OrderType::Market)
7673 .trader_id(node.trader_id())
7674 .strategy_id(strategy_id)
7675 .instrument_id(instrument_id)
7676 .quantity(Quantity::from("1.000"))
7677 .build();
7678 let position_id = PositionId::new("CUSTOM-POSITION-001");
7679
7680 node.kernel
7681 .cache
7682 .borrow_mut()
7683 .add_order(order.clone(), Some(position_id), Some(client_id), true)
7684 .unwrap();
7685
7686 let submit_order = SubmitOrder::new(
7687 order.trader_id(),
7688 Some(client_id),
7689 strategy_id,
7690 instrument_id,
7691 order.client_order_id(),
7692 order.init_event().clone(),
7693 order.exec_algorithm_id(),
7694 Some(position_id),
7695 None,
7696 UUID4::new(),
7697 UnixNanos::default(),
7698 None,
7699 );
7700
7701 node.kernel
7702 .exec_engine
7703 .borrow()
7704 .execute(TradingCommand::SubmitOrder(submit_order));
7705
7706 let exec_engine = node.kernel.exec_engine.borrow();
7707 let cache = exec_engine.cache().borrow();
7708 let cached_order = cache
7709 .order(&order.client_order_id())
7710 .expect("Order should be cached");
7711
7712 assert_eq!(cached_order.status(), OrderStatus::Initialized);
7713 }
7714
7715 #[cfg(all(feature = "simulation", madsim))]
7716 async fn advance_clock(d: Duration) {
7717 madsim::time::advance(d);
7718 madsim::task::yield_now().await;
7719 }
7720
7721 #[cfg(not(all(feature = "simulation", madsim)))]
7722 async fn advance_clock(d: Duration) {
7723 tokio::time::advance(d).await;
7724 }
7725
7726 #[rstest]
7727 #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7728 #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7729 #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7730 #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7731 #[case::unapplied_cancel_default(
7732 SubmissionRecoveryPolicy::ResolveLocally,
7733 false,
7734 Some(OrderStatus::PendingCancel),
7735 false
7736 )]
7737 #[case::unapplied_modify_default(
7738 SubmissionRecoveryPolicy::ResolveLocally,
7739 false,
7740 Some(OrderStatus::PendingUpdate),
7741 false
7742 )]
7743 #[case::unapplied_cancel_diagnostics(
7744 SubmissionRecoveryPolicy::RetainUnresolved,
7745 false,
7746 Some(OrderStatus::PendingCancel),
7747 false
7748 )]
7749 #[case::unapplied_modify_diagnostics(
7750 SubmissionRecoveryPolicy::RetainUnresolved,
7751 false,
7752 Some(OrderStatus::PendingUpdate),
7753 false
7754 )]
7755 #[case::first_fill_cancel_default(
7756 SubmissionRecoveryPolicy::ResolveLocally,
7757 false,
7758 Some(OrderStatus::PendingCancel),
7759 true
7760 )]
7761 #[case::first_fill_modify_default(
7762 SubmissionRecoveryPolicy::ResolveLocally,
7763 false,
7764 Some(OrderStatus::PendingUpdate),
7765 true
7766 )]
7767 #[case::first_fill_cancel_diagnostics(
7768 SubmissionRecoveryPolicy::RetainUnresolved,
7769 false,
7770 Some(OrderStatus::PendingCancel),
7771 true
7772 )]
7773 #[case::first_fill_modify_diagnostics(
7774 SubmissionRecoveryPolicy::RetainUnresolved,
7775 false,
7776 Some(OrderStatus::PendingUpdate),
7777 true
7778 )]
7779 #[cfg_attr(
7780 not(all(feature = "simulation", madsim)),
7781 tokio::test(start_paused = true)
7782 )]
7783 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7784 async fn test_submission_registry_publishes_after_native_resolution(
7785 #[case] policy: SubmissionRecoveryPolicy,
7786 #[case] order_list: bool,
7787 #[case] pending_status: Option<OrderStatus>,
7788 #[case] first_fill: bool,
7789 ) {
7790 let config = LiveNodeConfig {
7791 exec_engine: crate::config::LiveExecutionEngineConfig {
7792 inflight_check_threshold_ms: 100,
7793 inflight_check_retries: 2,
7794 submission_recovery_policy: policy,
7795 ..Default::default()
7796 },
7797 ..Default::default()
7798 };
7799 let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7800 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7801 let client_id = ClientId::from("TEST-SUBMISSION");
7802 let account_id = AccountId::from("TEST-001");
7803 let strategy_id = StrategyId::from("SUBMISSION-001");
7804 let account = AccountAny::Margin(MarginAccount::new(
7805 AccountState::new(
7806 account_id,
7807 AccountType::Margin,
7808 vec![AccountBalance::new(
7809 Money::from("1000000 USDT"),
7810 Money::from("0 USDT"),
7811 Money::from("1000000 USDT"),
7812 )],
7813 Vec::new(),
7814 true,
7815 UUID4::new(),
7816 UnixNanos::default(),
7817 UnixNanos::default(),
7818 Some(Currency::USDT()),
7819 ),
7820 true,
7821 ));
7822 node.kernel.cache.borrow_mut().add_account(account).unwrap();
7823 node.kernel
7824 .cache
7825 .borrow_mut()
7826 .add_instrument(instrument.clone())
7827 .unwrap();
7828 let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7829 .map(|index| {
7830 OrderTestBuilder::new(OrderType::Limit)
7831 .trader_id(node.trader_id())
7832 .strategy_id(strategy_id)
7833 .client_order_id(ClientOrderId::from(
7834 format!("O-SUBMISSION-{index}").as_str(),
7835 ))
7836 .instrument_id(instrument.id())
7837 .quantity(Quantity::from("1.000"))
7838 .price(Price::from("100.00"))
7839 .build()
7840 })
7841 .collect();
7842 let command = if order_list {
7843 TradingCommand::SubmitOrderList(SubmitOrderList::new(
7844 node.trader_id(),
7845 Some(client_id),
7846 strategy_id,
7847 OrderList::new(
7848 OrderListId::from("OL-SUBMISSION"),
7849 instrument.id(),
7850 strategy_id,
7851 orders.iter().map(Order::client_order_id).collect(),
7852 UnixNanos::default(),
7853 ),
7854 orders
7855 .iter()
7856 .map(|order| order.init_event().clone())
7857 .collect(),
7858 None,
7859 None,
7860 None,
7861 UUID4::new(),
7862 UnixNanos::default(),
7863 None,
7864 ))
7865 } else {
7866 TradingCommand::SubmitOrder(SubmitOrder::from_order(
7867 &orders[0],
7868 node.trader_id(),
7869 Some(client_id),
7870 None,
7871 UUID4::new(),
7872 UnixNanos::default(),
7873 ))
7874 };
7875 node.observe_exec_command_before_dispatch(&command);
7876
7877 for order in &orders {
7878 let submitted = TestOrderEventStubs::submitted(order, account_id);
7879 node.kernel
7880 .cache
7881 .borrow_mut()
7882 .add_order(order.clone(), None, Some(client_id), false)
7883 .unwrap();
7884 node.process_exec_event(ExecutionEvent::Order(submitted));
7885 }
7886
7887 let expected_status = match pending_status {
7888 Some(status) if !first_fill => status,
7889 Some(_) => OrderStatus::Canceled,
7890 None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7891 None => OrderStatus::Rejected,
7892 };
7893 let diagnostics = Rc::new(RefCell::new(Vec::new()));
7894 let handler = ShareableMessageHandler::from_typed({
7895 let diagnostics = diagnostics.clone();
7896 let cache = node.kernel.cache.clone();
7897 move |diagnostic: &SubmissionRecoveryExhausted| {
7898 let cache = cache.borrow_mut();
7899 let order = cache.order(&diagnostic.client_order_id).unwrap();
7900 assert_eq!(order.status(), expected_status);
7901 assert!(diagnostic.ts_event >= order.ts_last());
7902 diagnostics.borrow_mut().push(diagnostic.clone());
7903 }
7904 });
7905 let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7906 msgbus::subscribe_any(topic.into(), handler.clone(), None);
7907 let queries = Rc::new(RefCell::new(Vec::new()));
7908 msgbus::register_trading_command_endpoint(
7909 MessagingSwitchboard::exec_engine_execute(),
7910 TypedIntoHandler::from({
7911 let queries = queries.clone();
7912 move |command: TradingCommand| queries.borrow_mut().push(command)
7913 }),
7914 );
7915
7916 let mut last_inflight_check = dst::time::Instant::now();
7917 let mut last_open_check = last_inflight_check;
7918 let mut last_position_check = last_inflight_check;
7919 let mut open_order_report_task = None;
7920 let mut targeted_order_report_task = None;
7921 let mut position_report_task = None;
7922
7923 for check in 1..=3 {
7924 advance_clock(Duration::from_millis(101)).await;
7925 node.run_reconciliation_checks(
7926 dst::time::Instant::now(),
7927 ReconciliationCheckIntervals {
7928 inflight: Duration::from_millis(100),
7929 open: Duration::ZERO,
7930 position: Duration::ZERO,
7931 },
7932 &mut ReconciliationCheckState {
7933 last_inflight_check: &mut last_inflight_check,
7934 last_open_check: &mut last_open_check,
7935 last_position_check: &mut last_position_check,
7936 open_order_report_task: &mut open_order_report_task,
7937 targeted_order_report_task: &mut targeted_order_report_task,
7938 position_report_task: &mut position_report_task,
7939 },
7940 );
7941
7942 if check == 1 {
7943 assert!(diagnostics.borrow().is_empty());
7944
7945 if let Some(pending_status) = pending_status {
7946 let order = &orders[0];
7947 let pending = if pending_status == OrderStatus::PendingCancel {
7948 OrderEventAny::PendingCancel(
7949 OrderPendingCancelSpec::builder()
7950 .trader_id(node.trader_id())
7951 .strategy_id(strategy_id)
7952 .instrument_id(instrument.id())
7953 .client_order_id(order.client_order_id())
7954 .account_id(account_id)
7955 .build(),
7956 )
7957 } else {
7958 OrderEventAny::PendingUpdate(
7959 OrderPendingUpdateSpec::builder()
7960 .trader_id(node.trader_id())
7961 .strategy_id(strategy_id)
7962 .instrument_id(instrument.id())
7963 .client_order_id(order.client_order_id())
7964 .account_id(account_id)
7965 .build(),
7966 )
7967 };
7968 node.process_exec_event(ExecutionEvent::Order(pending));
7969 node.exec_manager
7970 .register_inflight(order.client_order_id(), UUID4::new());
7971
7972 if first_fill {
7973 let cached = node
7974 .kernel
7975 .cache
7976 .borrow()
7977 .order(&order.client_order_id())
7978 .unwrap()
7979 .clone();
7980 let OrderEventAny::Filled(mut fill) =
7981 OrderFilledTestBuilder::new(&cached, &instrument)
7982 .last_qty(Quantity::from("0.400"))
7983 .last_px(Price::from("100.00"))
7984 .commission(Money::from("0 USDT"))
7985 .without_position_id()
7986 .build()
7987 else {
7988 unreachable!()
7989 };
7990 fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7991 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7992 } else {
7993 let report = OrderStatusReport::new(
7994 account_id,
7995 instrument.id(),
7996 Some(order.client_order_id()),
7997 VenueOrderId::from("V-SUBMISSION"),
7998 Some(order.order_side()),
7999 order.order_type(),
8000 order.time_in_force(),
8001 OrderStatus::Submitted,
8002 order.quantity(),
8003 Quantity::zero(3),
8004 UnixNanos::default(),
8005 UnixNanos::default(),
8006 UnixNanos::default(),
8007 None,
8008 )
8009 .with_price(Price::from("100.00"));
8010 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
8011 Box::new(report),
8012 )));
8013 }
8014 let cache = node.kernel.cache.borrow();
8015 let cached = cache.order(&order.client_order_id()).unwrap();
8016 assert_eq!(cached.status(), pending_status);
8017 assert_eq!(
8018 cached.filled_qty(),
8019 if first_fill {
8020 Quantity::from("0.400")
8021 } else {
8022 Quantity::zero(3)
8023 }
8024 );
8025 assert!(
8026 !cached
8027 .events()
8028 .iter()
8029 .any(|event| matches!(event, OrderEventAny::Accepted(_)))
8030 );
8031 }
8032 }
8033 }
8034
8035 assert_eq!(
8036 queries.borrow().len(),
8037 orders.len()
8038 + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
8039 );
8040
8041 for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
8042 assert!(matches!(command, TradingCommand::QueryOrder(query)
8043 if query.client_order_id == order.client_order_id()
8044 && query.client_id == Some(client_id)));
8045 assert_eq!(
8046 node.kernel
8047 .cache
8048 .borrow()
8049 .order(&order.client_order_id())
8050 .unwrap()
8051 .status(),
8052 if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
8053 pending_status.unwrap()
8054 } else {
8055 expected_status
8056 }
8057 );
8058 }
8059 let diagnostics = diagnostics.borrow();
8060 assert_eq!(
8061 diagnostics.len(),
8062 if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
8063 orders.len()
8064 } else {
8065 0
8066 }
8067 );
8068
8069 for (diagnostic, order) in diagnostics.iter().zip(&orders) {
8070 assert_eq!(diagnostic.trader_id, node.trader_id());
8071 assert_eq!(diagnostic.client_id, Some(client_id));
8072 assert_eq!(diagnostic.strategy_id, strategy_id);
8073 assert_eq!(diagnostic.instrument_id, instrument.id());
8074 assert_eq!(diagnostic.client_order_id, order.client_order_id());
8075 assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
8076 assert_eq!(diagnostic.retry_count, 2);
8077 }
8078 msgbus::unsubscribe_any(topic.into(), &handler);
8079 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8080 }
8081
8082 #[derive(Debug)]
8083 struct SubmissionCallbackStrategy {
8084 core: StrategyCore,
8085 cancel: bool,
8086 batch: bool,
8087 callback: &'static str,
8088 client_id: ClientId,
8089 }
8090
8091 impl DataActor for SubmissionCallbackStrategy {}
8092
8093 impl SubmissionCallbackStrategy {
8094 fn queue_command(&mut self, client_order_id: ClientOrderId) {
8095 match (self.cancel, self.batch) {
8096 (true, true) => {
8097 self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
8098 .unwrap();
8099 }
8100 (true, false) => {
8101 self.cancel_order(client_order_id, Some(self.client_id), None)
8102 .unwrap();
8103 }
8104 (false, true) => {
8105 self.modify_orders(
8106 vec![(client_order_id, None, Some(Price::from("101.00")), None)],
8107 Some(self.client_id),
8108 None,
8109 )
8110 .unwrap();
8111 }
8112 (false, false) => self
8113 .modify_order(
8114 client_order_id,
8115 None,
8116 Some(Price::from("101.00")),
8117 None,
8118 Some(self.client_id),
8119 None,
8120 )
8121 .unwrap(),
8122 }
8123 }
8124 }
8125
8126 nautilus_strategy!(SubmissionCallbackStrategy, {
8127 fn on_order_accepted(&mut self, event: OrderAccepted) {
8128 if self.callback == "accepted" {
8129 self.queue_command(event.client_order_id);
8130 }
8131 }
8132
8133 fn on_order_updated(&mut self, event: OrderUpdated) {
8134 if self.callback == "updated" {
8135 self.queue_command(event.client_order_id);
8136 }
8137 }
8138 });
8139
8140 #[rstest]
8141 #[case::native(false, "accepted")]
8142 #[case::report(true, "accepted")]
8143 #[case::updated(false, "updated")]
8144 #[cfg_attr(
8145 not(all(feature = "simulation", madsim)),
8146 tokio::test(start_paused = true)
8147 )]
8148 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8149 async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
8150 #[values(
8151 SubmissionRecoveryPolicy::ResolveLocally,
8152 SubmissionRecoveryPolicy::RetainUnresolved
8153 )]
8154 policy: SubmissionRecoveryPolicy,
8155 #[values(false, true)] cancel: bool,
8156 #[values(false, true)] batch: bool,
8157 #[case] report: bool,
8158 #[case] callback: &'static str,
8159 ) {
8160 let config = LiveNodeConfig {
8161 exec_engine: crate::config::LiveExecutionEngineConfig {
8162 inflight_check_threshold_ms: 100,
8163 inflight_check_retries: 2,
8164 submission_recovery_policy: policy,
8165 ..Default::default()
8166 },
8167 ..Default::default()
8168 };
8169 let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
8170 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8171 let account_id = AccountId::from("TEST-001");
8172 let strategy_id = StrategyId::from("CALLBACK-001");
8173 let client_id = ClientId::from("TEST-CALLBACK");
8174 let account = AccountAny::Margin(MarginAccount::new(
8175 AccountState::new(
8176 account_id,
8177 AccountType::Margin,
8178 vec![AccountBalance::new(
8179 Money::from("1000000 USDT"),
8180 Money::from("0 USDT"),
8181 Money::from("1000000 USDT"),
8182 )],
8183 Vec::new(),
8184 true,
8185 UUID4::new(),
8186 UnixNanos::default(),
8187 UnixNanos::default(),
8188 Some(Currency::USDT()),
8189 ),
8190 true,
8191 ));
8192 node.kernel.cache.borrow_mut().add_account(account).unwrap();
8193 node.kernel
8194 .cache
8195 .borrow_mut()
8196 .add_instrument(instrument.clone())
8197 .unwrap();
8198 node.kernel
8199 .exec_engine
8200 .borrow_mut()
8201 .register_client(Box::new(StubExecutionClient::new(
8202 client_id,
8203 account_id,
8204 instrument.id().venue,
8205 OmsType::Netting,
8206 None,
8207 )))
8208 .unwrap();
8209 node.add_strategy(SubmissionCallbackStrategy {
8210 core: StrategyCore::new(StrategyConfig {
8211 strategy_id: Some(strategy_id),
8212 ..Default::default()
8213 }),
8214 cancel,
8215 batch,
8216 callback,
8217 client_id,
8218 })
8219 .unwrap();
8220 node.kernel.trader.borrow_mut().initialize().unwrap();
8221 node.kernel.start_trader().unwrap();
8222 let runner = node.runner.take().unwrap();
8223 runner.bind_senders_for_node(node.handle.clone());
8224 let mut channels = runner.take_channels();
8225 let order = OrderTestBuilder::new(OrderType::Limit)
8226 .trader_id(node.trader_id())
8227 .strategy_id(strategy_id)
8228 .client_order_id(ClientOrderId::from("O-CALLBACK"))
8229 .instrument_id(instrument.id())
8230 .quantity(Quantity::from("1.000"))
8231 .price(Price::from("100.00"))
8232 .build();
8233 let client_order_id = order.client_order_id();
8234 let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
8235 &order,
8236 node.trader_id(),
8237 Some(client_id),
8238 None,
8239 UUID4::new(),
8240 UnixNanos::default(),
8241 ));
8242 node.observe_exec_command_before_dispatch(&submit);
8243 node.kernel
8244 .cache
8245 .borrow_mut()
8246 .add_order(order.clone(), None, Some(client_id), false)
8247 .unwrap();
8248 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
8249 &order, account_id,
8250 )));
8251 advance_clock(Duration::from_millis(101)).await;
8252 let first = node.exec_manager.check_inflight_orders();
8253 assert_eq!(first.queries.len(), 1);
8254 assert!(first.events.is_empty());
8255
8256 advance_clock(Duration::from_millis(101)).await;
8257 let accepted =
8258 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
8259
8260 if callback == "updated" {
8261 let mut update = OrderUpdatedSpec::builder()
8262 .trader_id(node.trader_id())
8263 .strategy_id(strategy_id)
8264 .instrument_id(instrument.id())
8265 .client_order_id(client_order_id)
8266 .account_id(account_id)
8267 .quantity(order.quantity())
8268 .price(order.price().unwrap())
8269 .build();
8270 update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
8271 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
8272 } else if report {
8273 let accepted = OrderStatusReport::new(
8274 account_id,
8275 instrument.id(),
8276 Some(client_order_id),
8277 VenueOrderId::from("V-CALLBACK"),
8278 Some(order.order_side()),
8279 order.order_type(),
8280 order.time_in_force(),
8281 OrderStatus::Accepted,
8282 order.quantity(),
8283 Quantity::zero(3),
8284 UnixNanos::default(),
8285 UnixNanos::default(),
8286 UnixNanos::default(),
8287 None,
8288 )
8289 .with_price(Price::from("100.00"));
8290 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
8291 accepted,
8292 ))));
8293 } else {
8294 node.process_exec_event(ExecutionEvent::Order(accepted));
8295 }
8296
8297 let pending_status = if cancel {
8298 OrderStatus::PendingCancel
8299 } else {
8300 OrderStatus::PendingUpdate
8301 };
8302 assert_eq!(channels.exec_cmd_rx.len(), 1);
8303 assert_eq!(
8304 node.kernel
8305 .cache
8306 .borrow()
8307 .order(&client_order_id)
8308 .unwrap()
8309 .status(),
8310 pending_status
8311 );
8312
8313 for _ in 0..3 {
8315 advance_clock(Duration::from_millis(101)).await;
8316 let maintenance = node.exec_manager.check_inflight_orders();
8317 assert!(maintenance.events.is_empty());
8318 assert!(maintenance.queries.is_empty());
8319 assert!(
8320 node.exec_manager
8321 .take_submission_recovery_exhaustions()
8322 .is_empty()
8323 );
8324 }
8325
8326 let command = channels.exec_cmd_rx.try_recv().unwrap();
8327 node.process_exec_command(command);
8328 assert!(channels.exec_cmd_rx.is_empty());
8329 let immediate = node.exec_manager.check_inflight_orders();
8330 assert!(immediate.events.is_empty());
8331 assert!(immediate.queries.is_empty());
8332 advance_clock(Duration::from_millis(101)).await;
8333 let first_command_check = node.exec_manager.check_inflight_orders();
8334 assert!(first_command_check.events.is_empty());
8335 assert_eq!(first_command_check.queries.len(), 1);
8336 advance_clock(Duration::from_millis(101)).await;
8337 let exhausted = node.exec_manager.check_inflight_orders();
8338 assert!(exhausted.queries.is_empty());
8339 assert!(
8340 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
8341 if event.client_order_id == client_order_id)
8342 );
8343 assert!(
8344 node.exec_manager
8345 .take_submission_recovery_exhaustions()
8346 .is_empty()
8347 );
8348 node.process_reconciliation_events(&exhausted.events);
8349 assert_eq!(
8350 node.kernel
8351 .cache
8352 .borrow()
8353 .order(&client_order_id)
8354 .unwrap()
8355 .status(),
8356 OrderStatus::Canceled
8357 );
8358 }
8359
8360 #[cfg_attr(
8361 not(all(feature = "simulation", madsim)),
8362 tokio::test(start_paused = true)
8363 )]
8364 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8365 async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
8366 let last = dst::time::Instant::now();
8367 let interval = Duration::from_millis(100);
8368
8369 assert!(!reconciliation_check_due(last, last, Duration::ZERO));
8370 assert!(!reconciliation_check_due(last, last, interval));
8371
8372 advance_clock(Duration::from_millis(99)).await;
8373 let before_interval = dst::time::Instant::now();
8374 assert!(!reconciliation_check_due(before_interval, last, interval));
8375
8376 advance_clock(Duration::from_millis(1)).await;
8377 let at_interval = dst::time::Instant::now();
8378 assert!(reconciliation_check_due(at_interval, last, interval));
8379
8380 assert!(!reconciliation_check_due(last, at_interval, interval));
8381 }
8382
8383 #[cfg_attr(
8384 not(all(feature = "simulation", madsim)),
8385 tokio::test(start_paused = true)
8386 )]
8387 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8388 async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
8389 let config = LiveNodeConfig {
8390 exec_engine: crate::config::LiveExecutionEngineConfig {
8391 reconciliation: true,
8392 open_check_interval_secs: Some(1.0),
8393 position_check_interval_secs: Some(1.0),
8394 max_single_order_queries_per_cycle: 5,
8395 ..Default::default()
8396 },
8397 ..Default::default()
8398 };
8399
8400 let mut node =
8401 LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
8402 let client_id = ClientId::from("TEST-QUERY");
8403 let account_id = AccountId::from("TEST-QUERY-001");
8404
8405 let trading_commands = Rc::new(RefCell::new(Vec::new()));
8406 msgbus::register_trading_command_endpoint(
8407 MessagingSwitchboard::exec_engine_execute(),
8408 TypedIntoHandler::from({
8409 let trading_commands = trading_commands.clone();
8410 move |command: TradingCommand| {
8411 trading_commands.borrow_mut().push(command);
8412 }
8413 }),
8414 );
8415
8416 let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
8417 let instrument = crypto_perpetual_ethusdt();
8418 let instrument_id = instrument.id();
8419 let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
8420
8421 node.kernel
8422 .cache
8423 .borrow_mut()
8424 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
8425 .unwrap();
8426 insert_accepted_limit_order_in_node(
8427 &node,
8428 account_id,
8429 client_id,
8430 instrument_id,
8431 client_order_id,
8432 venue_order_id,
8433 );
8434
8435 let last = dst::time::Instant::now();
8436 advance_clock(Duration::from_nanos(1)).await;
8437 let now = dst::time::Instant::now();
8438 let mut last_inflight_check = last;
8439 let mut last_open_check = last;
8440 let mut last_position_check = last;
8441 let mut open_order_report_task = None;
8442 let mut targeted_order_report_task = None;
8443 let mut position_report_task = None;
8444
8445 node.run_reconciliation_checks(
8446 now,
8447 ReconciliationCheckIntervals {
8448 inflight: Duration::ZERO,
8449 open: Duration::from_nanos(1),
8450 position: Duration::ZERO,
8451 },
8452 &mut ReconciliationCheckState {
8453 last_inflight_check: &mut last_inflight_check,
8454 last_open_check: &mut last_open_check,
8455 last_position_check: &mut last_position_check,
8456 open_order_report_task: &mut open_order_report_task,
8457 targeted_order_report_task: &mut targeted_order_report_task,
8458 position_report_task: &mut position_report_task,
8459 },
8460 );
8461
8462 let commands = trading_commands.borrow();
8463
8464 assert!(commands.is_empty());
8465 assert!(open_order_report_task.is_none());
8466 assert!(targeted_order_report_task.is_none());
8467 assert!(position_report_task.is_none());
8468
8469 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8470 }
8471
8472 #[rstest]
8473 #[tokio::test]
8474 async fn test_recurring_report_failure_preserves_client_coverage(
8475 #[values(false, true)] positions: bool,
8476 #[values(false, true)] failure_first: bool,
8477 ) {
8478 let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8479 let failed_id = ClientId::from("FAILED");
8480 let healthy_id = ClientId::from("HEALTHY");
8481
8482 for (client_id, outcome) in [
8483 (failed_id, FillReportClientOutcome::Failure),
8484 (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8485 ] {
8486 node.exec_clients
8487 .push(LiveExecutionClient::new(Box::new(FillReportClient {
8488 client_id,
8489 account_id: AccountId::from("TEST-001"),
8490 venue: Venue::from("BINANCE"),
8491 outcome,
8492 commands: Rc::new(RefCell::new(Vec::new())),
8493 })));
8494 }
8495
8496 if !failure_first {
8497 node.exec_clients.reverse();
8498 }
8499
8500 let last = dst::time::Instant::now();
8501 let now = last + Duration::from_secs(1);
8502 let mut last_inflight = last;
8503 let mut last_open = last;
8504 let mut last_position = last;
8505 let mut open_task = None;
8506 let mut targeted_task = None;
8507 let mut position_task = None;
8508
8509 node.run_reconciliation_checks(
8510 now,
8511 ReconciliationCheckIntervals {
8512 inflight: Duration::ZERO,
8513 open: if positions {
8514 Duration::ZERO
8515 } else {
8516 Duration::from_secs(1)
8517 },
8518 position: if positions {
8519 Duration::from_secs(1)
8520 } else {
8521 Duration::ZERO
8522 },
8523 },
8524 &mut ReconciliationCheckState {
8525 last_inflight_check: &mut last_inflight,
8526 last_open_check: &mut last_open,
8527 last_position_check: &mut last_position,
8528 open_order_report_task: &mut open_task,
8529 targeted_order_report_task: &mut targeted_task,
8530 position_report_task: &mut position_task,
8531 },
8532 );
8533
8534 let (queried_clients, failed_clients) = if positions {
8535 let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8536 position_task.unwrap().future.await
8537 else {
8538 panic!("position collection should complete despite one client failure");
8539 };
8540
8541 let result = result.finish();
8542 assert!(result.reports.is_empty());
8543 assert!(open_task.is_none());
8544 (result.queried_clients, result.failed_clients)
8545 } else {
8546 let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8547 panic!("order collection should complete despite one client failure");
8548 };
8549
8550 let result = result.finish();
8551
8552 assert!(result.reports.is_empty());
8553 assert!(position_task.is_none());
8554 (result.queried_clients, result.failed_clients)
8555 };
8556
8557 assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8558 assert_eq!(failed_clients, IndexSet::from([failed_id]));
8559 assert!(targeted_task.is_none());
8560 }
8561
8562 #[rstest]
8563 #[case::inflight(true, false)]
8564 #[case::open(false, false)]
8565 #[case::positions(false, true)]
8566 #[tokio::test]
8567 async fn test_reconciliation_shutdown_preserves_pending_checks(
8568 #[case] inflight: bool,
8569 #[case] positions: bool,
8570 ) {
8571 let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8572 node.handle.set_shutting_down();
8573 let last = dst::time::Instant::now();
8574 let now = last + Duration::from_secs(1);
8575 let mut last_inflight = last;
8576 let mut last_open = last;
8577 let mut last_position = last;
8578 let mut open_task = None;
8579 let mut targeted_task = None;
8580 let mut position_task = None;
8581
8582 node.run_reconciliation_checks(
8583 now,
8584 ReconciliationCheckIntervals {
8585 inflight: if inflight {
8586 Duration::from_secs(1)
8587 } else {
8588 Duration::ZERO
8589 },
8590 open: if !inflight && !positions {
8591 Duration::from_secs(1)
8592 } else {
8593 Duration::ZERO
8594 },
8595 position: if positions {
8596 Duration::from_secs(1)
8597 } else {
8598 Duration::ZERO
8599 },
8600 },
8601 &mut ReconciliationCheckState {
8602 last_inflight_check: &mut last_inflight,
8603 last_open_check: &mut last_open,
8604 last_position_check: &mut last_position,
8605 open_order_report_task: &mut open_task,
8606 targeted_order_report_task: &mut targeted_task,
8607 position_report_task: &mut position_task,
8608 },
8609 );
8610
8611 assert_eq!(
8612 (last_inflight, last_open, last_position),
8613 (last, last, last)
8614 );
8615 assert!(open_task.is_none());
8616 assert!(targeted_task.is_none());
8617 assert!(position_task.is_none());
8618 }
8619
8620 fn insert_accepted_limit_order_in_node(
8621 node: &LiveNode,
8622 account_id: AccountId,
8623 client_id: ClientId,
8624 instrument_id: InstrumentId,
8625 client_order_id: ClientOrderId,
8626 venue_order_id: VenueOrderId,
8627 ) {
8628 let order = OrderTestBuilder::new(OrderType::Limit)
8629 .client_order_id(client_order_id)
8630 .instrument_id(instrument_id)
8631 .quantity(Quantity::from("10.0"))
8632 .price(Price::from("100.0"))
8633 .build();
8634 let submitted = TestOrderEventStubs::submitted(&order, account_id);
8635 node.kernel
8636 .cache
8637 .borrow_mut()
8638 .add_order(order, None, Some(client_id), false)
8639 .unwrap();
8640 let order = node
8641 .kernel
8642 .cache
8643 .borrow_mut()
8644 .update_order(&submitted)
8645 .unwrap();
8646 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8647 node.kernel
8648 .cache
8649 .borrow_mut()
8650 .update_order(&accepted)
8651 .unwrap();
8652 }
8653
8654 fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8655 let config = LiveNodeConfig {
8656 exec_engine: crate::config::LiveExecutionEngineConfig {
8657 reconciliation: true,
8658 ..Default::default()
8659 },
8660 ..Default::default()
8661 };
8662
8663 let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8664 let account_id = AccountId::from("TEST-001");
8665 let client_id = ClientId::from("TEST-RECENT-FILL");
8666 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8667 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8668 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8669 node.kernel
8670 .cache
8671 .borrow_mut()
8672 .add_instrument(instrument.clone())
8673 .unwrap();
8674 insert_accepted_limit_order_in_node(
8675 &node,
8676 account_id,
8677 client_id,
8678 instrument.id(),
8679 client_order_id,
8680 venue_order_id,
8681 );
8682 let order = node
8683 .kernel
8684 .cache
8685 .borrow()
8686 .order_owned(&client_order_id)
8687 .unwrap();
8688 let fill = TestOrderEventStubs::filled(
8689 &order,
8690 &instrument,
8691 Some(TradeId::from("T-RECENT-FILL")),
8692 None,
8693 Some(Price::from("100.0")),
8694 Some(Quantity::from("1.0")),
8695 Some(LiquiditySide::Taker),
8696 None,
8697 None,
8698 Some(account_id),
8699 );
8700
8701 (node, fill, instrument)
8702 }
8703
8704 fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8705 let client_order_id = fill_report.client_order_id.unwrap();
8706 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8707 let order = node
8708 .kernel
8709 .cache
8710 .borrow()
8711 .order_owned(&client_order_id)
8712 .unwrap();
8713 let order_report = OrderStatusReport::new(
8714 fill_report.account_id,
8715 fill_report.instrument_id,
8716 Some(client_order_id),
8717 fill_report.venue_order_id,
8718 OrderSide::Buy.into(),
8719 OrderType::Limit,
8720 TimeInForce::Gtc,
8721 OrderStatus::PartiallyFilled,
8722 Quantity::from("10.0"),
8723 Quantity::from("2.0"),
8724 UnixNanos::from(1_000),
8725 UnixNanos::from(1_000),
8726 UnixNanos::from(1_000),
8727 None,
8728 )
8729 .with_avg_px(dec!(100.0));
8730 let inferred = create_inferred_fill_for_qty(
8731 &order,
8732 &order_report,
8733 &fill_report.account_id,
8734 &instrument,
8735 Quantity::from("1.0"),
8736 UnixNanos::from(1_000),
8737 None,
8738 )
8739 .unwrap();
8740
8741 node.process_reconciliation_events(&[inferred]);
8742 }
8743
8744 #[rstest]
8745 #[case::rejected(false, "10.0", "10.0", "1.0")]
8746 #[case::dust(true, "9.004", "10.004", "10.004")]
8747 #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8748 fn test_position_check_overfill_raises_before_fill_when_client_allows(
8749 #[case] allow: bool,
8750 #[case] fill_qty: &str,
8751 #[case] expected_qty: &str,
8752 #[case] expected_filled: &str,
8753 ) {
8754 let (mut node, venue_report, mut fill_report) =
8755 position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8756 fill_report.commission = Money::from("1.25 USDT");
8757 let key = (fill_report.instrument_id, fill_report.account_id);
8758
8759 let client = StubExecutionClient::new(
8760 ClientId::from("POSITION-FILLS"),
8761 AccountId::from("TEST-001"),
8762 fill_report.instrument_id.venue,
8763 OmsType::Netting,
8764 None,
8765 );
8766
8767 let client = if allow {
8768 client.with_allows_reconciliation_overfill()
8769 } else {
8770 client
8771 };
8772
8773 node.kernel
8774 .exec_engine
8775 .borrow_mut()
8776 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8777 .unwrap();
8778
8779 let position_result = position_report_result(&node, venue_report);
8780 node.handle_position_fill_report_result(PositionFillReportResult {
8781 position_result,
8782 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8783 });
8784
8785 let order = node
8786 .kernel
8787 .cache
8788 .borrow()
8789 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8790 .unwrap();
8791 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8792 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8793 let applied = order
8794 .trade_ids()
8795 .iter()
8796 .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8797 assert_eq!(applied, allow);
8798 let revision = node.exec_manager.position_activity_revision(&key);
8799
8800 if !allow {
8801 return;
8802 }
8803
8804 assert_eq!(revision, 2);
8805
8806 {
8807 let cache = node.kernel.cache.borrow();
8808 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8809 assert_eq!(positions.len(), 1);
8810 assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8811 assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8812 assert_eq!(positions[0].trade_ids.len(), 2);
8813 }
8814
8815 let replay_report = position_report_result(
8816 &node,
8817 PositionStatusReport::new(
8818 key.1,
8819 key.0,
8820 PositionSide::Long,
8821 Quantity::from(expected_qty),
8822 UnixNanos::from(2_000),
8823 UnixNanos::from(2_000),
8824 None,
8825 None,
8826 Some(dec!(100.0)),
8827 ),
8828 );
8829 node.handle_position_fill_report_result(PositionFillReportResult {
8830 position_result: replay_report,
8831 reports: IndexMap::from([(key, vec![fill_report])]),
8832 });
8833
8834 let order = node
8835 .kernel
8836 .cache
8837 .borrow()
8838 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8839 .unwrap();
8840 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8841 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8842 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8843 }
8844
8845 #[rstest]
8846 fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8847 let (mut node, venue_report, mut overfill) =
8848 position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8849 overfill.commission = Money::from("0.20 USDT");
8850 let key = (overfill.instrument_id, overfill.account_id);
8851 let client = StubExecutionClient::new(
8852 ClientId::from("POSITION-FILLS"),
8853 AccountId::from("TEST-001"),
8854 overfill.instrument_id.venue,
8855 OmsType::Netting,
8856 None,
8857 )
8858 .with_allows_reconciliation_overfill();
8859 node.kernel
8860 .exec_engine
8861 .borrow_mut()
8862 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8863 .unwrap();
8864
8865 let mut fitting = overfill.clone();
8866 fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8867 fitting.last_qty = Quantity::from("8.0");
8868 fitting.commission = Money::from("0.80 USDT");
8869 let position_result = position_report_result(&node, venue_report);
8870 node.handle_position_fill_report_result(PositionFillReportResult {
8871 position_result,
8872 reports: IndexMap::from([(key, vec![fitting, overfill])]),
8873 });
8874
8875 let order = node
8876 .kernel
8877 .cache
8878 .borrow()
8879 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8880 .unwrap();
8881 assert_eq!(order.quantity(), Quantity::from("11.0"));
8882 assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8883 assert_eq!(order.trade_ids().len(), 3);
8884 assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8885 {
8886 let cache = node.kernel.cache.borrow();
8887 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8888 assert_eq!(positions.len(), 1);
8889 assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8890 assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8891 }
8892 }
8893
8894 #[rstest]
8895 fn test_position_check_second_overfill_does_not_raise_filled_order() {
8896 let (mut node, venue_report, mut first) =
8897 position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8898 first.commission = Money::from("1.20 USDT");
8899 let key = (first.instrument_id, first.account_id);
8900 let client = StubExecutionClient::new(
8901 ClientId::from("POSITION-FILLS"),
8902 AccountId::from("TEST-001"),
8903 first.instrument_id.venue,
8904 OmsType::Netting,
8905 None,
8906 )
8907 .with_allows_reconciliation_overfill();
8908 node.kernel
8909 .exec_engine
8910 .borrow_mut()
8911 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8912 .unwrap();
8913
8914 let mut second = first.clone();
8915 second.trade_id = TradeId::from("T-POSITION-LATER");
8916 second.last_qty = Quantity::from("1.0");
8917 second.commission = Money::from("0.10 USDT");
8918 second.ts_event = UnixNanos::from(1_001);
8919 let position_result = position_report_result(&node, venue_report);
8920 node.handle_position_fill_report_result(PositionFillReportResult {
8921 position_result,
8922 reports: IndexMap::from([(key, vec![first, second])]),
8923 });
8924
8925 let order = node
8926 .kernel
8927 .cache
8928 .borrow()
8929 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8930 .unwrap();
8931 assert_eq!(order.quantity(), Quantity::from("13.0"));
8932 assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8933 assert_eq!(order.status(), OrderStatus::Filled);
8934 assert_eq!(order.trade_ids().len(), 2);
8935 assert!(
8936 order
8937 .trade_ids()
8938 .iter()
8939 .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8940 );
8941 {
8942 let cache = node.kernel.cache.borrow();
8943 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8944 assert_eq!(positions.len(), 1);
8945 assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8946 assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8947 }
8948 }
8949
8950 #[rstest]
8951 fn test_allow_overfills_applies_fill_without_raising_quantity() {
8952 let config = LiveNodeConfig {
8953 exec_engine: crate::config::LiveExecutionEngineConfig {
8954 reconciliation: true,
8955 allow_overfills: true,
8956 ..Default::default()
8957 },
8958 ..Default::default()
8959 };
8960
8961 let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8962 let account_id = AccountId::from("TEST-001");
8963 let client_id = ClientId::from("POSITION-FILLS");
8964 let client_order_id = ClientOrderId::from("O-ALLOW");
8965 let venue_order_id = VenueOrderId::from("V-ALLOW");
8966 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8967 node.kernel
8968 .cache
8969 .borrow_mut()
8970 .add_instrument(instrument.clone())
8971 .unwrap();
8972 insert_accepted_limit_order_in_node(
8973 &node,
8974 account_id,
8975 client_id,
8976 instrument.id(),
8977 client_order_id,
8978 venue_order_id,
8979 );
8980 let client = StubExecutionClient::new(
8981 client_id,
8982 account_id,
8983 instrument.id().venue,
8984 OmsType::Netting,
8985 None,
8986 )
8987 .with_allows_reconciliation_overfill();
8988 node.kernel
8989 .exec_engine
8990 .borrow_mut()
8991 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8992 .unwrap();
8993
8994 let fill = FillReport::new(
8995 account_id,
8996 instrument.id(),
8997 venue_order_id,
8998 TradeId::from("T-ALLOW"),
8999 OrderSide::Buy,
9000 Quantity::from("12.0"),
9001 Price::from("100.0"),
9002 Money::from("1.00 USDT"),
9003 LiquiditySide::Taker,
9004 Some(client_order_id),
9005 None,
9006 UnixNanos::from(1_000),
9007 UnixNanos::from(1_000),
9008 None,
9009 );
9010 node.kernel
9011 .exec_engine
9012 .borrow_mut()
9013 .reconcile_fill_report(&fill);
9014
9015 let order = node
9016 .kernel
9017 .cache
9018 .borrow()
9019 .order_owned(&client_order_id)
9020 .unwrap();
9021 assert_eq!(order.quantity(), Quantity::from("10.0"));
9022 assert_eq!(order.filled_qty(), Quantity::from("12.0"));
9023 assert_eq!(order.trade_ids().len(), 1);
9024 }
9025
9026 fn position_fill_test_fixture(
9027 name: &str,
9028 authoritative_qty: Quantity,
9029 ) -> (LiveNode, PositionStatusReport, FillReport) {
9030 position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
9031 }
9032
9033 fn position_fill_test_fixture_with_threshold(
9034 name: &str,
9035 authoritative_qty: Quantity,
9036 position_check_threshold_ms: u32,
9037 ) -> (LiveNode, PositionStatusReport, FillReport) {
9038 let config = LiveNodeConfig {
9039 exec_engine: crate::config::LiveExecutionEngineConfig {
9040 reconciliation: true,
9041 position_check_threshold_ms,
9042 ..Default::default()
9043 },
9044 ..Default::default()
9045 };
9046
9047 let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
9048 let account_id = AccountId::from("TEST-001");
9049 let client_id = ClientId::from("POSITION-FILLS");
9050 let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
9051 let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
9052 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
9053 let account = AccountAny::Margin(MarginAccount::new(
9054 AccountState::new(
9055 account_id,
9056 AccountType::Margin,
9057 vec![AccountBalance::new(
9058 Money::from("1000000 USDT"),
9059 Money::from("0 USDT"),
9060 Money::from("1000000 USDT"),
9061 )],
9062 Vec::new(),
9063 true,
9064 UUID4::new(),
9065 UnixNanos::default(),
9066 UnixNanos::default(),
9067 Some(Currency::USDT()),
9068 ),
9069 true,
9070 ));
9071 node.kernel.cache.borrow_mut().add_account(account).unwrap();
9072 node.kernel
9073 .cache
9074 .borrow_mut()
9075 .add_instrument(instrument.clone())
9076 .unwrap();
9077 insert_accepted_limit_order_in_node(
9078 &node,
9079 account_id,
9080 client_id,
9081 instrument.id(),
9082 client_order_id,
9083 venue_order_id,
9084 );
9085 let order = node
9086 .kernel
9087 .cache
9088 .borrow()
9089 .order_owned(&client_order_id)
9090 .unwrap();
9091 let mut initial_fill = TestOrderEventStubs::filled(
9092 &order,
9093 &instrument,
9094 Some(TradeId::from("T-POSITION-INITIAL")),
9095 None,
9096 Some(Price::from("100.0")),
9097 Some(Quantity::from("1.0")),
9098 Some(LiquiditySide::Taker),
9099 None,
9100 None,
9101 Some(account_id),
9102 );
9103
9104 let OrderEventAny::Filled(fill) = &mut initial_fill else {
9105 unreachable!();
9106 };
9107
9108 fill.commission = Some(Money::zero(instrument.quote_currency()));
9109 node.process_reconciliation_events(&[initial_fill]);
9110
9111 let ts_event = UnixNanos::from(1_000);
9112
9113 let fill_report = FillReport::new(
9114 account_id,
9115 instrument.id(),
9116 venue_order_id,
9117 TradeId::from("T-POSITION-AUTHORITATIVE"),
9118 OrderSide::Buy,
9119 authoritative_qty,
9120 Price::from("100.0"),
9121 Money::zero(instrument.quote_currency()),
9122 LiquiditySide::Taker,
9123 Some(client_order_id),
9124 None,
9125 ts_event,
9126 ts_event,
9127 None,
9128 );
9129
9130 let venue_report = PositionStatusReport::new(
9131 account_id,
9132 instrument.id(),
9133 PositionSide::Long,
9134 Quantity::from("2.0"),
9135 ts_event,
9136 ts_event,
9137 None,
9138 None,
9139 Some(dec!(100.0)),
9140 );
9141
9142 (node, venue_report, fill_report)
9143 }
9144
9145 fn position_report_result(
9146 node: &LiveNode,
9147 report: PositionStatusReport,
9148 ) -> PositionReportResult {
9149 let client_id = ClientId::from("POSITION-FILLS");
9150 let key = (report.instrument_id, report.account_id);
9151 let mut check = node
9152 .exec_manager
9153 .prepare_position_report_check(UUID4::new(), &[]);
9154 check.client_coverage.insert(
9155 key,
9156 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
9157 );
9158
9159 PositionReportResult {
9160 check,
9161 reports: vec![report],
9162 queried_clients: IndexSet::from([client_id]),
9163 failed_clients: IndexSet::new(),
9164 }
9165 }
9166
9167 fn position_quantity_after_check(
9168 node: &mut LiveNode,
9169 venue_report: &PositionStatusReport,
9170 reports: &[FillReport],
9171 ) -> Quantity {
9172 let key = (venue_report.instrument_id, venue_report.account_id);
9173 let position_result = position_report_result(node, venue_report.clone());
9174 node.handle_position_fill_report_result(PositionFillReportResult {
9175 position_result,
9176 reports: IndexMap::from([(key, reports.to_vec())]),
9177 });
9178
9179 node.kernel
9180 .cache
9181 .borrow()
9182 .positions_open(None, Some(&key.0), None, Some(&key.1), None)
9183 .iter()
9184 .map(|position| position.quantity)
9185 .sum()
9186 }
9187
9188 fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
9189 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
9190 fill.account_id,
9191 fill.instrument_id,
9192 fill.venue_order_id,
9193 fill.trade_id,
9194 fill.order_side,
9195 fill.last_qty,
9196 fill.last_px,
9197 fill.commission
9198 .unwrap_or_else(|| Money::zero(fill.currency)),
9199 fill.liquidity_side,
9200 Some(fill.client_order_id),
9201 fill.position_id,
9202 fill.ts_event,
9203 fill.ts_init,
9204 None,
9205 ))))
9206 }
9207
9208 fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
9209 node.exec_manager.is_fill_recently_processed(
9210 fill.account_id,
9211 fill.instrument_id,
9212 fill.trade_id,
9213 )
9214 }
9215
9216 #[rstest]
9217 #[case(0, NodeState::Idle)]
9218 #[case(1, NodeState::Starting)]
9219 #[case(2, NodeState::Running)]
9220 #[case(3, NodeState::ShuttingDown)]
9221 #[case(4, NodeState::Stopped)]
9222 fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
9223 assert_eq!(NodeState::from_u8(value), expected);
9224 }
9225
9226 #[rstest]
9227 #[case(5)]
9228 #[case(255)]
9229 #[should_panic(expected = "Invalid NodeState value")]
9230 fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
9231 let _ = NodeState::from_u8(value);
9232 }
9233
9234 #[rstest]
9235 fn test_node_state_roundtrip() {
9236 for state in [
9237 NodeState::Idle,
9238 NodeState::Starting,
9239 NodeState::Running,
9240 NodeState::ShuttingDown,
9241 NodeState::Stopped,
9242 ] {
9243 assert_eq!(NodeState::from_u8(state.as_u8()), state);
9244 }
9245 }
9246
9247 #[rstest]
9248 fn test_node_state_is_running_only_for_running() {
9249 assert!(!NodeState::Idle.is_running());
9250 assert!(!NodeState::Starting.is_running());
9251 assert!(NodeState::Running.is_running());
9252 assert!(!NodeState::ShuttingDown.is_running());
9253 assert!(!NodeState::Stopped.is_running());
9254 }
9255
9256 #[rstest]
9257 #[tokio::test]
9258 async fn test_await_engines_connected_returns_stop_requested() {
9259 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9260 let handle = node.handle();
9261
9262 handle.stop();
9263
9264 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9265 let status = node.await_engines_connected(deadline).await;
9266
9267 assert_eq!(status, EngineConnectionStatus::StopRequested);
9268 assert!(handle.should_stop());
9269 }
9270
9271 #[rstest]
9272 #[tokio::test]
9273 async fn test_await_engines_connected_returns_shutdown_requested() {
9274 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9275
9276 node.kernel().shutdown_flag().set(true);
9277
9278 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9279 let status = node.await_engines_connected(deadline).await;
9280
9281 assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
9282 }
9283
9284 #[rstest]
9285 #[tokio::test]
9286 async fn test_start_stop_request_aborts_startup_without_running() {
9287 let config = LiveNodeConfig {
9288 exec_engine: crate::config::LiveExecutionEngineConfig {
9289 reconciliation: false,
9290 ..Default::default()
9291 },
9292 timeout_disconnection: Duration::from_millis(50),
9293 ..Default::default()
9294 };
9295
9296 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9297 let handle = node.handle();
9298
9299 handle.stop();
9300 node.start().await.unwrap();
9301
9302 assert_eq!(handle.state(), NodeState::Stopped);
9303 assert!(handle.should_stop());
9304 assert!(!handle.is_running());
9305 }
9306
9307 #[rstest]
9308 #[tokio::test(start_paused = true)]
9309 async fn test_stop_processes_residual_exec_event_during_grace_period() {
9310 let config = LiveNodeConfig {
9311 exec_engine: crate::config::LiveExecutionEngineConfig {
9312 reconciliation: false,
9313 ..Default::default()
9314 },
9315 timeout_connection: Duration::ZERO,
9316 timeout_reconciliation: Duration::ZERO,
9317 timeout_portfolio: Duration::ZERO,
9318 timeout_disconnection: Duration::ZERO,
9319 delay_post_stop: Duration::from_millis(20),
9320 timeout_shutdown: Duration::ZERO,
9321 ..Default::default()
9322 };
9323
9324 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9325 let order = OrderTestBuilder::new(OrderType::Market)
9326 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
9327 .quantity(Quantity::from("2"))
9328 .build();
9329 let client_order_id = order.client_order_id();
9330 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
9331
9332 node.kernel
9333 .cache()
9334 .borrow_mut()
9335 .add_order(order, None, None, false)
9336 .unwrap();
9337
9338 node.start().await.unwrap();
9339 let exec_event_sender = get_exec_event_sender();
9340
9341 let send_residual = tokio::spawn(async move {
9342 tokio::time::sleep(Duration::from_millis(1)).await;
9343 exec_event_sender
9344 .send(ExecutionEvent::Order(submitted))
9345 .unwrap();
9346 });
9347
9348 node.stop().await.unwrap();
9349 send_residual.await.unwrap();
9350
9351 assert_eq!(
9352 node.kernel
9353 .cache()
9354 .borrow()
9355 .order(&client_order_id)
9356 .unwrap()
9357 .status(),
9358 OrderStatus::Submitted
9359 );
9360
9361 node.dispose();
9362 }
9363
9364 #[rstest]
9365 #[tokio::test]
9366 async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
9367 let actor_id = ActorId::from("LIVE-STATE-ACTOR");
9368 let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
9369 let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
9370 let strategy_load =
9371 IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
9372 let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
9373 let strategy_save =
9374 IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
9375 let (database, control) = TestCacheDatabaseControl::create();
9376 control.set_actor_state(actor_id, &actor_load);
9377 control.set_strategy_state(strategy_id, &strategy_load);
9378
9379 let config = LiveNodeConfig {
9380 load_state: true,
9381 save_state: true,
9382 exec_engine: crate::config::LiveExecutionEngineConfig {
9383 reconciliation: false,
9384 ..Default::default()
9385 },
9386 timeout_connection: Duration::ZERO,
9387 timeout_reconciliation: Duration::ZERO,
9388 timeout_portfolio: Duration::ZERO,
9389 timeout_disconnection: Duration::ZERO,
9390 delay_post_stop: Duration::ZERO,
9391 timeout_shutdown: Duration::ZERO,
9392 ..Default::default()
9393 };
9394
9395 let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
9396 node.set_cache_database(Box::new(database)).unwrap();
9397 node.add_actor(StateActor::new(
9398 actor_id,
9399 control.clone(),
9400 actor_save.clone(),
9401 ))
9402 .unwrap();
9403 node.add_strategy(StateStrategy::new(
9404 strategy_id,
9405 control.clone(),
9406 strategy_save.clone(),
9407 ))
9408 .unwrap();
9409
9410 node.start().await.unwrap();
9411 node.stop().await.unwrap();
9412 node.dispose();
9413
9414 assert_eq!(
9415 control.events(),
9416 vec![
9417 "actor.load:LIVE-STATE-ACTOR",
9418 "actor.on_load",
9419 "strategy.load:LIVE-STATE-STRATEGY-001",
9420 "strategy.on_load",
9421 "actor.on_start",
9422 "strategy.on_start",
9423 "actor.on_stop",
9424 "strategy.on_stop",
9425 "actor.on_save",
9426 "actor.update:LIVE-STATE-ACTOR",
9427 "strategy.on_save",
9428 "strategy.update:LIVE-STATE-STRATEGY-001",
9429 "database.close",
9430 ]
9431 );
9432 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
9433 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9434 assert_eq!(node.state(), NodeState::Stopped);
9435 }
9436
9437 #[rstest]
9438 #[tokio::test]
9439 async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9440 let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9441 let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9442 let (database, control) = TestCacheDatabaseControl::create();
9443
9444 let config = LiveNodeConfig {
9445 save_state: true,
9446 exec_engine: crate::config::LiveExecutionEngineConfig {
9447 reconciliation: false,
9448 ..Default::default()
9449 },
9450 timeout_connection: Duration::ZERO,
9451 timeout_reconciliation: Duration::ZERO,
9452 timeout_portfolio: Duration::ZERO,
9453 timeout_disconnection: Duration::ZERO,
9454 delay_post_stop: Duration::ZERO,
9455 timeout_shutdown: Duration::ZERO,
9456 ..Default::default()
9457 };
9458
9459 let mut node =
9460 LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9461 node.set_cache_database(Box::new(database)).unwrap();
9462 node.add_actor(
9463 StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9464 )
9465 .unwrap();
9466 node.add_strategy(
9467 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9468 )
9469 .unwrap();
9470
9471 node.start().await.unwrap();
9472 let error = node.stop().await.unwrap_err();
9473 node.dispose();
9474
9475 assert_eq!(
9476 error.to_string(),
9477 "failed while finalizing kernel shutdown: Failed to save component state: actor \
9478 LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9479 LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9480 );
9481 assert_eq!(
9482 control.events(),
9483 vec![
9484 "actor.on_start",
9485 "strategy.on_start",
9486 "actor.on_stop",
9487 "strategy.on_stop",
9488 "actor.on_save",
9489 "strategy.on_save",
9490 "database.close",
9491 ]
9492 );
9493 assert_eq!(node.state(), NodeState::Stopped);
9494 }
9495
9496 #[rstest]
9497 #[tokio::test]
9498 async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9499 let config = LiveNodeConfig {
9500 exec_engine: crate::config::LiveExecutionEngineConfig {
9501 reconciliation: false,
9502 ..Default::default()
9503 },
9504 timeout_connection: Duration::ZERO,
9505 timeout_reconciliation: Duration::ZERO,
9506 timeout_portfolio: Duration::ZERO,
9507 timeout_disconnection: Duration::ZERO,
9508 delay_post_stop: Duration::ZERO,
9509 timeout_shutdown: Duration::ZERO,
9510 ..Default::default()
9511 };
9512
9513 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9514 let order = OrderTestBuilder::new(OrderType::Market)
9515 .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9516 .quantity(Quantity::from("3"))
9517 .build();
9518 let client_order_id = order.client_order_id();
9519 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9520
9521 node.kernel
9522 .cache()
9523 .borrow_mut()
9524 .add_order(order, None, None, false)
9525 .unwrap();
9526
9527 node.start().await.unwrap();
9528 get_exec_event_sender()
9529 .send(ExecutionEvent::Order(submitted))
9530 .unwrap();
9531
9532 node.stop().await.unwrap();
9533
9534 assert_eq!(
9535 node.kernel
9536 .cache()
9537 .borrow()
9538 .order(&client_order_id)
9539 .unwrap()
9540 .status(),
9541 OrderStatus::Submitted
9542 );
9543
9544 node.dispose();
9545 }
9546
9547 #[rstest]
9548 #[case::manual(false)]
9549 #[case::integrated(true)]
9550 fn test_shutdown_drain_logs_progress(
9551 #[case] integrated: bool,
9552 #[values(0, 9_999, 10_000, 20_001)] pending: usize,
9553 ) {
9554 struct DrainLogCapture(Mutex<Vec<String>>);
9555
9556 impl Log for DrainLogCapture {
9557 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
9558 metadata.level() == Level::Debug
9559 }
9560
9561 fn log(&self, record: &Record<'_>) {
9562 assert_eq!(record.level(), Level::Debug);
9563 self.0.lock().push(record.args().to_string());
9564 }
9565
9566 fn flush(&self) {}
9567 }
9568
9569 static LOGS: DrainLogCapture = DrainLogCapture(Mutex::new(Vec::new()));
9570 log::set_logger(&LOGS).unwrap();
9571 log::set_max_level(LevelFilter::Debug);
9572
9573 let mut runner = AsyncRunner::new();
9574 runner.bind_senders();
9575 let event = TimeEvent::new("drain".into(), UUID4::new(), 0.into(), 0.into());
9576
9577 for _ in 0..pending {
9578 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9579 event.clone(),
9580 TimeEventCallback::from(|_: TimeEvent| {}),
9581 ));
9582 }
9583
9584 if integrated {
9585 let mut channels = runner.take_channels();
9586 LiveNode::drain_channels(
9587 &mut channels.time_evt_rx,
9588 &mut channels.system_evt_rx,
9589 &mut channels.system_cmd_rx,
9590 &mut channels.exec_evt_rx,
9591 &mut channels.exec_cmd_rx,
9592 &mut channels.data_evt_rx,
9593 &mut channels.data_cmd_rx,
9594 );
9595 } else {
9596 assert_eq!(runner.poll_pending(|_| {}), pending);
9597 }
9598
9599 let mut expected = vec![format!(
9600 "Draining shutdown queues ({pending} events initially pending)"
9601 )];
9602 expected.extend(
9603 (10_000..=pending)
9604 .step_by(10_000)
9605 .map(|processed| format!("Processed {processed} events during shutdown drain")),
9606 );
9607 expected.push(format!(
9608 "Drained {pending} remaining events during shutdown"
9609 ));
9610 assert_eq!(*LOGS.0.lock(), expected);
9611 }
9612
9613 #[rstest]
9614 #[tokio::test]
9615 async fn test_start_event_store_replay_skips_live_connections() {
9616 let mut node = live_node_with_replay_store(false);
9617 let handle = node.handle();
9618
9619 node.start().await.unwrap();
9620
9621 assert_eq!(handle.state(), NodeState::Running);
9622 assert!(handle.is_running());
9623 assert!(node.kernel.is_event_store_replay());
9624 assert!(node.runner.is_some());
9625 }
9626
9627 #[rstest]
9628 #[tokio::test]
9629 async fn test_start_event_store_replay_preserves_stop_request() {
9630 let mut node = live_node_with_replay_store(false);
9631 let handle = node.handle();
9632 handle.stop();
9633
9634 node.start().await.unwrap();
9635
9636 assert_eq!(handle.state(), NodeState::Stopped);
9637 assert!(handle.should_stop());
9638 assert!(node.kernel.is_event_store_replay());
9639 assert!(node.runner.is_some());
9640 }
9641
9642 #[rstest]
9643 #[tokio::test]
9644 async fn test_start_event_store_replay_config_failure_aborts_startup() {
9645 let mut node = live_node_with_replay_store(true);
9646 let handle = node.handle();
9647
9648 node.start().await.unwrap();
9649
9650 assert_eq!(handle.state(), NodeState::Stopped);
9651 assert!(!handle.is_running());
9652 assert!(node.kernel.is_event_store_replay_configured());
9653 assert!(!node.kernel.is_event_store_replay());
9654 assert!(node.runner.is_some());
9655 }
9656
9657 #[rstest]
9658 #[tokio::test]
9659 async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9660 let mut node = live_node_with_replay_store(false);
9661 let handle = node.handle();
9662
9663 node.run().await.unwrap();
9664
9665 assert_eq!(handle.state(), NodeState::Running);
9666 assert!(handle.is_running());
9667 assert!(node.kernel.is_event_store_replay());
9668 assert!(node.runner.is_none());
9669 }
9670
9671 #[rstest]
9672 #[tokio::test]
9673 async fn test_run_event_store_replay_preserves_stop_request() {
9674 let mut node = live_node_with_replay_store(false);
9675 let handle = node.handle();
9676 handle.stop();
9677
9678 node.run().await.unwrap();
9679
9680 assert_eq!(handle.state(), NodeState::Stopped);
9681 assert!(handle.should_stop());
9682 assert!(node.kernel.is_event_store_replay());
9683 assert!(node.runner.is_none());
9684 }
9685
9686 #[rstest]
9687 #[tokio::test]
9688 async fn test_run_event_store_replay_config_failure_aborts_startup() {
9689 let mut node = live_node_with_replay_store(true);
9690 let handle = node.handle();
9691
9692 node.run().await.unwrap();
9693
9694 assert_eq!(handle.state(), NodeState::Stopped);
9695 assert!(!handle.is_running());
9696 assert!(node.kernel.is_event_store_replay_configured());
9697 assert!(!node.kernel.is_event_store_replay());
9698 assert!(node.runner.is_none());
9699 }
9700
9701 #[rstest]
9702 fn test_build_rejects_event_store_config_without_factory() {
9703 let config = LiveNodeConfig {
9704 event_store: Some(EventStoreConfig::default()),
9705 exec_engine: crate::config::LiveExecutionEngineConfig {
9706 reconciliation: false,
9707 ..Default::default()
9708 },
9709 ..Default::default()
9710 };
9711
9712 let err = LiveNodeBuilder::from_config(config)
9713 .expect("builder")
9714 .build()
9715 .expect_err("should reject event_store config without factory");
9716
9717 assert!(
9718 err.to_string().contains("with_event_store"),
9719 "error message should mention with_event_store, was: {err}"
9720 );
9721 }
9722
9723 #[rstest]
9724 fn test_direct_build_rejects_event_store_config() {
9725 let config = LiveNodeConfig {
9726 event_store: Some(EventStoreConfig::default()),
9727 exec_engine: crate::config::LiveExecutionEngineConfig {
9728 reconciliation: false,
9729 ..Default::default()
9730 },
9731 ..Default::default()
9732 };
9733
9734 let err = LiveNode::build("TestNode".to_string(), Some(config))
9735 .expect_err("LiveNode::build should reject event_store config");
9736
9737 assert!(
9738 err.to_string().contains("with_event_store"),
9739 "error message should mention with_event_store, was: {err}"
9740 );
9741 }
9742
9743 #[rstest]
9744 fn test_dispose_before_start_is_idempotent() {
9745 let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9746 node.add_strategy(TestStrategy::new(StrategyConfig {
9747 strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9748 ..Default::default()
9749 }))
9750 .unwrap();
9751
9752 node.dispose();
9753 node.dispose();
9754
9755 assert!(node.kernel.trader().borrow().is_disposed());
9756 assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9757 assert_eq!(node.state(), NodeState::Stopped);
9758 }
9759
9760 #[rstest]
9761 fn test_dispose_releases_retained_callback_roots(
9762 #[values(false, true)] fatal: bool,
9763 #[values(false, true)] external: bool,
9764 ) {
9765 actor::clear_callbacks().unwrap();
9766 let directory =
9767 std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9768
9769 let config = LiveNodeConfig {
9770 logging: LoggerConfig {
9771 fileout_level: LevelFilter::Info,
9772 file_config: Some(FileWriterConfig {
9773 directory: Some(directory.to_str().unwrap().to_string()),
9774 file_name: Some("disposal".to_string()),
9775 ..Default::default()
9776 }),
9777 ..Default::default()
9778 },
9779 ..Default::default()
9780 };
9781
9782 let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9783 let received = Rc::new(RefCell::new(Vec::new()));
9784 let observed = received.clone();
9785
9786 let retained = DispatchMessage::from(()).dispatch(|()| {
9787 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9788 TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9789 TimeEventCallback::RustLocal(Rc::new(move |_| {
9790 observed.borrow_mut().push("delivered");
9791 })),
9792 ));
9793
9794 external.then(|| DispatchMessage::new((), std::thread::current().id()))
9795 });
9796
9797 if let Some(retained) = &retained {
9798 assert!(retained.is_rooted());
9799 }
9800
9801 assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9802
9803 if fatal {
9804 crate::dispatch::tests::latch_callback_failure();
9805 }
9806
9807 node.dispose();
9808
9809 assert!(node.runner.is_none());
9810 assert!(received.borrow().is_empty());
9811 assert_eq!(Rc::strong_count(&received), 1);
9812 assert_eq!(
9813 actor::callback_failure(),
9814 (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9815 );
9816 assert_eq!(
9817 actor::clear_callbacks(),
9818 if external {
9819 Err(CallbackDispatchError::Active)
9820 } else {
9821 Ok(())
9822 }
9823 );
9824
9825 logging_sync_to_disk().unwrap();
9826 let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9827 drop(retained);
9828 node.dispose();
9829
9830 assert_eq!(actor::callback_failure(), None);
9831 assert_eq!(actor::clear_callbacks(), Ok(()));
9832 drop(node);
9833 std::fs::remove_dir_all(directory).unwrap();
9834
9835 let errors: Vec<_> = output
9836 .lines()
9837 .filter(|line| line.contains("[ERROR]"))
9838 .filter_map(|line| {
9839 line.split_once(".nautilus_live::node: ")
9840 .map(|(_, text)| text)
9841 })
9842 .collect();
9843
9844 let mut expected = Vec::new();
9845
9846 if fatal {
9847 expected.push(
9848 "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9849 );
9850 }
9851
9852 if external {
9853 expected.push(
9854 "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9855 );
9856 }
9857
9858 assert_eq!(errors, expected);
9859 }
9860
9861 #[tokio::test]
9862 async fn test_dispose_releases_stop_generated_callback_roots() {
9863 actor::clear_callbacks().unwrap();
9864
9865 let config = LiveNodeConfig {
9866 exec_engine: crate::config::LiveExecutionEngineConfig {
9867 reconciliation: false,
9868 ..Default::default()
9869 },
9870 timeout_connection: Duration::ZERO,
9871 timeout_reconciliation: Duration::ZERO,
9872 timeout_portfolio: Duration::ZERO,
9873 timeout_shutdown: Duration::ZERO,
9874 ..Default::default()
9875 };
9876
9877 let mut node =
9878 LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9879 let received = Rc::new(RefCell::new(Vec::new()));
9880 node.add_actor(StopCallbackActor {
9881 core: DataActorCore::new(DataActorConfig {
9882 actor_id: Some(ActorId::from("STOP-CALLBACK")),
9883 ..Default::default()
9884 }),
9885 received: received.clone(),
9886 })
9887 .unwrap();
9888
9889 node.start().await.unwrap();
9890
9891 assert!(node.kernel.trader().borrow().is_running());
9892 assert_eq!(actor::clear_callbacks(), Ok(()));
9893
9894 node.dispose();
9895
9896 assert_eq!(*received.borrow(), ["stop", "queued"]);
9897 assert_eq!(Rc::strong_count(&received), 1);
9898 assert!(node.runner.is_none());
9899 assert!(node.kernel.trader().borrow().is_disposed());
9900 assert_eq!(node.state(), NodeState::Stopped);
9901 assert_eq!(actor::clear_callbacks(), Ok(()));
9902 }
9903
9904 #[derive(Debug)]
9905 struct StopCallbackActor {
9906 core: DataActorCore,
9907 received: Rc<RefCell<Vec<&'static str>>>,
9908 }
9909
9910 nautilus_actor!(StopCallbackActor);
9911
9912 impl DataActor for StopCallbackActor {
9913 fn on_stop(&mut self) -> anyhow::Result<()> {
9914 self.received.borrow_mut().push("stop");
9915 let received = self.received.clone();
9916 DispatchMessage::from(()).dispatch(|()| {
9917 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9918 TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9919 TimeEventCallback::RustLocal(Rc::new(move |_| {
9920 received.borrow_mut().push("delivered");
9921 })),
9922 ));
9923 });
9924
9925 let clear_result = actor::clear_callbacks();
9926 anyhow::ensure!(
9927 clear_result == Err(CallbackDispatchError::Active),
9928 "Expected active callback roots during stop, received {clear_result:?}"
9929 );
9930 self.received.borrow_mut().push("queued");
9931 Ok(())
9932 }
9933 }
9934
9935 #[rstest]
9936 fn test_handle_initial_state() {
9937 let handle = LiveNodeHandle::new();
9938
9939 assert_eq!(handle.state(), NodeState::Idle);
9940 assert!(!handle.should_stop());
9941 assert!(!handle.is_running());
9942 }
9943
9944 #[rstest]
9945 fn test_handle_initial_metrics_snapshot_is_zero() {
9946 let handle = LiveNodeHandle::new();
9947
9948 assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9949 }
9950
9951 #[rstest]
9952 fn test_record_runner_dispatch_updates_selected_channel() {
9953 let metrics = RunnerMetrics::default();
9954 let dispatch_start = dst::time::Instant::now();
9955 let metrics_start = dispatch_start
9956 .checked_sub(Duration::from_micros(1))
9957 .expect("test instant should support a one-microsecond lookback");
9958
9959 record_runner_dispatch(
9960 &metrics,
9961 SystemChannel::DataCommands,
9962 dispatch_start,
9963 metrics_start,
9964 );
9965 let snapshot = metrics.snapshot();
9966
9967 assert_eq!(snapshot.time_events.dispatched, 0);
9968 assert_eq!(snapshot.exec_events.dispatched, 0);
9969 assert_eq!(snapshot.exec_commands.dispatched, 0);
9970 assert_eq!(snapshot.data_events.dispatched, 0);
9971 assert_eq!(snapshot.data_commands.dispatched, 1);
9972 assert_eq!(
9973 snapshot.data_commands.last_dispatch_at_ns,
9974 snapshot.elapsed_ns
9975 );
9976 assert_eq!(
9977 snapshot.data_commands.dispatch_busy_ns,
9978 snapshot.dispatch_busy_ns
9979 );
9980 assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9981 }
9982
9983 #[rstest]
9984 fn test_handle_stop_sets_flag() {
9985 let handle = LiveNodeHandle::new();
9986
9987 handle.stop();
9988
9989 assert!(handle.should_stop());
9990 }
9991
9992 #[rstest]
9993 fn test_handle_stop_blocks_running_transition() {
9994 let handle = LiveNodeHandle::new();
9995 handle.set_starting();
9996 handle.stop();
9997
9998 let transition = handle.try_set_running();
9999
10000 assert_eq!(transition, RunningTransition::StopRequested);
10001 assert_eq!(handle.state(), NodeState::Starting);
10002 assert!(handle.should_stop());
10003 assert!(!handle.is_running());
10004 }
10005
10006 #[rstest]
10007 fn test_handle_stop_after_running_transition_remains_pending() {
10008 let handle = LiveNodeHandle::new();
10009 handle.set_starting();
10010
10011 let transition = handle.try_set_running();
10012 handle.stop();
10013
10014 assert_eq!(transition, RunningTransition::Entered);
10015 assert_eq!(handle.state(), NodeState::Running);
10016 assert!(handle.should_stop());
10017 assert!(handle.is_running());
10018 }
10019
10020 #[rstest]
10021 fn test_handle_node_state_transitions() {
10022 let handle = LiveNodeHandle::new();
10023 assert_eq!(handle.state(), NodeState::Idle);
10024
10025 handle.set_starting();
10026 assert_eq!(handle.state(), NodeState::Starting);
10027 assert!(!handle.is_running());
10028
10029 assert_eq!(handle.try_set_running(), RunningTransition::Entered);
10030 assert_eq!(handle.state(), NodeState::Running);
10031 assert!(handle.is_running());
10032
10033 handle.set_shutting_down();
10034 assert_eq!(handle.state(), NodeState::ShuttingDown);
10035 assert!(!handle.is_running());
10036
10037 handle.set_stopped();
10038 assert_eq!(handle.state(), NodeState::Stopped);
10039 assert!(!handle.is_running());
10040 }
10041
10042 #[rstest]
10043 fn test_handle_clone_shares_state_bidirectionally() {
10044 let handle1 = LiveNodeHandle::new();
10045 let handle2 = handle1.clone();
10046
10047 handle1.set_starting();
10048 let transition = handle2.try_set_running();
10049 handle1.stop();
10050
10051 assert_eq!(transition, RunningTransition::Entered);
10052 assert_eq!(handle1.state(), NodeState::Running);
10053 assert!(handle2.should_stop());
10054 }
10055
10056 #[rstest]
10057 fn test_handle_stop_flag_survives_non_running_state_changes() {
10058 let handle = LiveNodeHandle::new();
10059
10060 handle.set_starting();
10061 handle.stop();
10062 handle.set_shutting_down();
10063 handle.set_stopped();
10064
10065 assert_eq!(handle.state(), NodeState::Stopped);
10066 assert!(handle.should_stop());
10067 }
10068
10069 #[rstest]
10070 fn test_builder_creation() {
10071 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10072
10073 assert!(result.is_ok());
10074 }
10075
10076 #[rstest]
10077 fn test_stream_processor_receives_unregistered_typed_payload() {
10078 let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10079 .unwrap()
10080 .build()
10081 .unwrap();
10082 let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
10083 InstrumentId::from("AUD/USD.SIM"),
10084 Some(ClientId::from("EXTERNAL")),
10085 Some(Venue::from("SIM")),
10086 UUID4::from("00000000-0000-4000-8000-000000000001"),
10087 UnixNanos::from(1),
10088 None,
10089 None,
10090 ));
10091 let expected = serde_json::to_value(&command).unwrap();
10092 let received = Rc::new(RefCell::new(Vec::new()));
10093 let steps = Rc::new(RefCell::new(Vec::new()));
10094 let received_processor = received.clone();
10095 let first_steps = steps.clone();
10096 node.add_stream_processor(move |message| {
10097 let command = message
10098 .downcast_ref::<SubscribeCommand>()
10099 .expect("processor must receive the decoded concrete command");
10100 received_processor
10101 .borrow_mut()
10102 .push(serde_json::to_value(command).unwrap());
10103 first_steps.borrow_mut().push(1);
10104 });
10105
10106 let second_steps = steps.clone();
10107 node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
10108 let republished = Rc::new(RefCell::new(Vec::new()));
10109 let republished_handler = republished.clone();
10110 let subscriber_steps = steps.clone();
10111 msgbus::subscribe_any(
10112 "external.test".into(),
10113 ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
10114 republished_handler
10115 .borrow_mut()
10116 .push(serde_json::to_value(command).unwrap());
10117 subscriber_steps.borrow_mut().push(3);
10118 }),
10119 None,
10120 );
10121
10122 let message = BusMessage::with_str_topic(
10123 "external.test",
10124 BusPayloadType::SubscribeCommand,
10125 Bytes::from(serde_json::to_vec(&command).unwrap()),
10126 SerializationEncoding::Json,
10127 );
10128
10129 assert!(
10130 !msgbus::get_message_bus()
10131 .borrow()
10132 .is_streaming_type(BusPayloadType::SubscribeCommand)
10133 );
10134 node.process_external_msgbus_message(&message);
10135
10136 assert_eq!(*received.borrow(), vec![expected.clone()]);
10137 assert_eq!(*steps.borrow(), vec![1, 2]);
10138 assert!(republished.borrow().is_empty());
10139
10140 received.borrow_mut().clear();
10141 steps.borrow_mut().clear();
10142 msgbus::get_message_bus()
10143 .borrow_mut()
10144 .add_streaming_type(BusPayloadType::SubscribeCommand);
10145 node.process_external_msgbus_message(&message);
10146
10147 assert_eq!(*received.borrow(), vec![expected.clone()]);
10148 assert_eq!(*republished.borrow(), vec![expected]);
10149 assert_eq!(*steps.borrow(), vec![1, 2, 3]);
10150 msgbus::get_message_bus().borrow_mut().dispose();
10151 }
10152
10153 #[rstest]
10154 fn test_builder_rejects_backtest() {
10155 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
10156
10157 assert!(result.is_err());
10158 assert!(result.unwrap_err().to_string().contains("Backtest"));
10159 }
10160
10161 #[rstest]
10162 fn test_builder_accepts_live_environment() {
10163 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
10164
10165 assert!(result.is_ok());
10166 }
10167
10168 #[rstest]
10169 fn test_builder_accepts_sandbox_environment() {
10170 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10171
10172 assert!(result.is_ok());
10173 }
10174
10175 #[rstest]
10176 fn test_builder_fluent_api_chaining() {
10177 let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
10178 .unwrap()
10179 .with_name("TestNode")
10180 .with_instance_id(UUID4::new())
10181 .with_load_state(false)
10182 .with_save_state(true)
10183 .with_timeout_connection(30)
10184 .with_timeout_reconciliation(60)
10185 .with_reconciliation(true)
10186 .with_reconciliation_lookback_mins(120)
10187 .with_timeout_portfolio(10)
10188 .with_timeout_disconnection_secs(5)
10189 .with_delay_post_stop_secs(3)
10190 .with_delay_shutdown_secs(10);
10191
10192 assert_eq!(builder.name(), "TestNode");
10193 }
10194
10195 #[rstest]
10196 fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
10197 let (external_egress, publications, closed) = CapturingExternalEgress::new();
10198
10199 let msgbus_config = MessageBusConfig {
10200 encoding: SerializationEncoding::Json,
10201 ..Default::default()
10202 };
10203
10204 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10205 .unwrap()
10206 .with_msgbus_config(msgbus_config)
10207 .with_external_msgbus_egress(Box::new(external_egress))
10208 .build()
10209 .expect("node builds with external message bus egress");
10210 let quote = QuoteTick::default();
10211
10212 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10213
10214 let publications = publications.borrow();
10215 assert_eq!(publications.len(), 1);
10216 assert_eq!(publications[0].topic, "data.quotes.TEST");
10217 assert_eq!(
10218 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10219 .expect("JSON payload must decode as QuoteTick"),
10220 quote
10221 );
10222 drop(publications);
10223
10224 msgbus::get_message_bus().borrow_mut().dispose();
10225 assert!(closed.get());
10226 drop(node);
10227 }
10228
10229 #[rstest]
10230 #[tokio::test(flavor = "current_thread")]
10231 async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
10232 let quote = QuoteTick::default();
10233 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10234 let publications = Arc::new(Mutex::new(Vec::new()));
10235 let closed = Arc::new(AtomicBool::new(false));
10236 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
10237
10238 let msgbus_config = MessageBusConfig {
10239 external_streams: Some(vec!["stream".to_string()]),
10240 ..Default::default()
10241 };
10242
10243 let config = LiveNodeConfig {
10244 environment: Environment::Sandbox,
10245 msgbus: Some(msgbus_config),
10246 exec_engine: crate::config::LiveExecutionEngineConfig {
10247 reconciliation: false,
10248 ..Default::default()
10249 },
10250 delay_post_stop: Duration::ZERO,
10251 timeout_connection: Duration::from_millis(500),
10252 timeout_disconnection: Duration::from_millis(500),
10253 ..Default::default()
10254 };
10255
10256 let mut node = LiveNodeBuilder::from_config(config)
10257 .unwrap()
10258 .with_external_msgbus_factory(Box::new(factory))
10259 .build()
10260 .expect("node builds with external message bus factory");
10261
10262 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10263 {
10264 let publications = publications.lock();
10265 assert_eq!(publications.len(), 1);
10266 assert_eq!(publications[0].topic, "data.quotes.TEST");
10267 assert_eq!(
10268 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10269 .expect("JSON payload must decode as QuoteTick"),
10270 quote
10271 );
10272 }
10273
10274 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10275 let handle = node.handle();
10276
10277 let handler = TypedHandler::from({
10280 let received = received.clone();
10281 move |quote: &QuoteTick| {
10282 received.borrow_mut().push(*quote);
10283 }
10284 });
10285
10286 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10287 msgbus::get_message_bus()
10288 .borrow_mut()
10289 .add_streaming_type(BusPayloadType::QuoteTick);
10290
10291 let payload =
10292 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
10293 let message = BusMessage::with_str_topic(
10294 "data.quotes.TEST",
10295 BusPayloadType::QuoteTick,
10296 payload,
10297 SerializationEncoding::Json,
10298 );
10299
10300 tokio::time::timeout(Duration::from_secs(30), async {
10301 let run = node.run();
10302 tokio::pin!(run);
10303
10304 let drive = async {
10305 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10306
10307 tx.send(message)
10308 .await
10309 .expect("external ingress receiver should be open");
10310
10311 wait_until_async(
10312 || async { received.borrow().len() == 1 },
10313 Duration::from_secs(10),
10314 )
10315 .await;
10316 assert_eq!(*received.borrow(), vec![quote]);
10317 handle.stop();
10318 };
10319
10320 tokio::select! {
10321 biased;
10322
10323 () = drive => {}
10324 result = &mut run => {
10325 panic!("node stopped before factory ingress was republished: {result:?}");
10326 }
10327 }
10328
10329 run.await.expect("node should stop cleanly");
10330 })
10331 .await
10332 .expect("live node should republish factory ingress and stop before timeout");
10333
10334 assert_eq!(handle.state(), NodeState::Stopped);
10335 assert!(closed.load(Ordering::Relaxed));
10336 msgbus::get_message_bus().borrow_mut().dispose();
10337 }
10338
10339 #[rstest]
10340 #[tokio::test(flavor = "current_thread")]
10341 async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
10342 let quote = QuoteTick::default();
10343 let publications = Arc::new(Mutex::new(Vec::new()));
10344 let closed = Arc::new(AtomicBool::new(false));
10345 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
10346
10347 let config = LiveNodeConfig {
10348 environment: Environment::Sandbox,
10349 msgbus: Some(MessageBusConfig::default()),
10350 exec_engine: crate::config::LiveExecutionEngineConfig {
10351 reconciliation: false,
10352 ..Default::default()
10353 },
10354 delay_post_stop: Duration::ZERO,
10355 timeout_connection: Duration::from_millis(500),
10356 timeout_disconnection: Duration::from_millis(500),
10357 ..Default::default()
10358 };
10359
10360 let mut node = LiveNodeBuilder::from_config(config)
10361 .unwrap()
10362 .with_external_msgbus_factory(Box::new(factory))
10363 .build()
10364 .expect("node builds with egress-only message bus factory");
10365 let handle = node.handle();
10366
10367 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10368 {
10369 let publications = publications.lock();
10370 assert_eq!(publications.len(), 1);
10371 assert_eq!(publications[0].topic, "data.quotes.TEST");
10372 }
10373
10374 tokio::time::timeout(Duration::from_secs(30), async {
10375 let run = node.run();
10376 tokio::pin!(run);
10377
10378 let drive = async {
10379 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10380 handle.stop();
10381 };
10382
10383 tokio::select! {
10384 biased;
10385
10386 () = drive => {}
10387 result = &mut run => {
10388 panic!("node stopped before egress-only factory run was observed: {result:?}");
10389 }
10390 }
10391
10392 run.await.expect("node should stop cleanly");
10393 })
10394 .await
10395 .expect("live node should run without external ingress before timeout");
10396
10397 assert_eq!(handle.state(), NodeState::Stopped);
10398 msgbus::get_message_bus().borrow_mut().dispose();
10399 assert!(closed.load(Ordering::Relaxed));
10400 }
10401
10402 #[rstest]
10403 fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
10404 let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
10405
10406 let egress_factory = CapturingBackingFactory::new(
10407 Arc::new(Mutex::new(Vec::new())),
10408 Arc::new(AtomicBool::new(false)),
10409 None,
10410 );
10411 let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10412 .unwrap()
10413 .with_external_msgbus_factory(Box::new(egress_factory))
10414 .with_external_msgbus_egress(Box::new(external_egress))
10415 .build()
10416 .expect_err("builder should reject factory plus injected egress");
10417
10418 assert!(
10419 egress_error
10420 .to_string()
10421 .contains("cannot be combined with injected egress or ingress")
10422 );
10423
10424 let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10425
10426 let ingress_factory = CapturingBackingFactory::new(
10427 Arc::new(Mutex::new(Vec::new())),
10428 Arc::new(AtomicBool::new(false)),
10429 None,
10430 );
10431 let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
10432 let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10433 .unwrap()
10434 .with_external_msgbus_factory(Box::new(ingress_factory))
10435 .with_external_ingress(Box::new(ingress))
10436 .build()
10437 .expect_err("builder should reject factory plus injected ingress");
10438
10439 assert!(
10440 ingress_error
10441 .to_string()
10442 .contains("cannot be combined with injected egress or ingress")
10443 );
10444 }
10445
10446 #[rstest]
10447 #[tokio::test(flavor = "current_thread")]
10448 async fn test_run_republishes_external_ingress_on_local_msgbus() {
10449 let quote = QuoteTick::default();
10450 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10451 let payload =
10452 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
10453 let message = BusMessage::with_str_topic(
10454 "data.quotes.TEST",
10455 BusPayloadType::QuoteTick,
10456 payload,
10457 SerializationEncoding::Json,
10458 );
10459 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10460 let closed = Rc::new(Cell::new(false));
10461 let ingress = CapturingExternalIngress::new(rx, closed.clone());
10462
10463 let config = LiveNodeConfig {
10464 environment: Environment::Sandbox,
10465 exec_engine: crate::config::LiveExecutionEngineConfig {
10466 reconciliation: false,
10467 ..Default::default()
10468 },
10469 delay_post_stop: Duration::ZERO,
10470 timeout_connection: Duration::from_millis(500),
10471 timeout_disconnection: Duration::from_millis(500),
10472 ..Default::default()
10473 };
10474
10475 let mut node = LiveNodeBuilder::from_config(config)
10476 .unwrap()
10477 .with_external_ingress(Box::new(ingress))
10478 .build()
10479 .expect("node builds with external message bus ingress");
10480 let handle = node.handle();
10481
10482 let handler = TypedHandler::from({
10483 let received = received.clone();
10484 move |quote: &QuoteTick| {
10485 received.borrow_mut().push(*quote);
10486 }
10487 });
10488
10489 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10490 msgbus::get_message_bus()
10491 .borrow_mut()
10492 .add_streaming_type(BusPayloadType::QuoteTick);
10493
10494 tokio::time::timeout(Duration::from_secs(30), async {
10495 let run = node.run();
10496 tokio::pin!(run);
10497
10498 let drive = async {
10499 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10500
10501 tx.send(message)
10502 .await
10503 .expect("external ingress receiver should be open");
10504
10505 wait_until_async(
10506 || async { received.borrow().len() == 1 },
10507 Duration::from_secs(10),
10508 )
10509 .await;
10510 assert_eq!(*received.borrow(), vec![quote]);
10511 handle.stop();
10512 };
10513
10514 tokio::select! {
10515 biased;
10516
10517 () = drive => {}
10518 result = &mut run => {
10519 panic!("node stopped before external message was republished: {result:?}");
10520 }
10521 }
10522
10523 run.await.expect("node should stop cleanly");
10524 })
10525 .await
10526 .expect("live node should republish ingress and stop before timeout");
10527
10528 assert_eq!(handle.state(), NodeState::Stopped);
10529 assert!(closed.get());
10530 msgbus::get_message_bus().borrow_mut().dispose();
10531 }
10532
10533 #[rstest]
10534 #[tokio::test(flavor = "current_thread")]
10535 async fn test_run_closes_external_ingress_when_receiver_closes() {
10536 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10537 let closed = Rc::new(Cell::new(false));
10538 let ingress = CapturingExternalIngress::new(rx, closed.clone());
10539
10540 let config = LiveNodeConfig {
10541 environment: Environment::Sandbox,
10542 exec_engine: crate::config::LiveExecutionEngineConfig {
10543 reconciliation: false,
10544 ..Default::default()
10545 },
10546 delay_post_stop: Duration::ZERO,
10547 timeout_connection: Duration::from_millis(500),
10548 timeout_disconnection: Duration::from_millis(500),
10549 ..Default::default()
10550 };
10551
10552 let mut node = LiveNodeBuilder::from_config(config)
10553 .unwrap()
10554 .with_external_ingress(Box::new(ingress))
10555 .build()
10556 .expect("node builds with external message bus ingress");
10557 let handle = node.handle();
10558
10559 tokio::time::timeout(Duration::from_secs(30), async {
10560 let run = node.run();
10561 tokio::pin!(run);
10562
10563 let drive = async {
10564 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10565
10566 drop(tx);
10567
10568 wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10569 assert!(
10570 handle.is_running(),
10571 "node should keep running after ingress closes"
10572 );
10573 handle.stop();
10574 };
10575
10576 tokio::select! {
10577 biased;
10578
10579 () = drive => {}
10580 result = &mut run => {
10581 panic!("node stopped before ingress close was observed: {result:?}");
10582 }
10583 }
10584
10585 run.await.expect("node should stop cleanly");
10586 })
10587 .await
10588 .expect("live node should close ingress and stop before timeout");
10589
10590 assert_eq!(handle.state(), NodeState::Stopped);
10591 }
10592
10593 #[rstest]
10594 #[tokio::test(flavor = "current_thread")]
10595 async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10596 let closed = Rc::new(Cell::new(false));
10597 let ingress = FailingExternalIngress::new(closed.clone());
10598
10599 let config = LiveNodeConfig {
10600 environment: Environment::Sandbox,
10601 exec_engine: crate::config::LiveExecutionEngineConfig {
10602 reconciliation: false,
10603 ..Default::default()
10604 },
10605 delay_post_stop: Duration::ZERO,
10606 timeout_connection: Duration::from_millis(500),
10607 timeout_disconnection: Duration::from_millis(500),
10608 ..Default::default()
10609 };
10610
10611 let mut node = LiveNodeBuilder::from_config(config)
10612 .unwrap()
10613 .with_external_ingress(Box::new(ingress))
10614 .build()
10615 .expect("node builds with external message bus ingress");
10616 let handle = node.handle();
10617
10618 let err = node.run().await.expect_err("run should fail");
10619
10620 assert!(
10621 err.to_string()
10622 .contains("external ingress receiver unavailable")
10623 );
10624 assert_eq!(handle.state(), NodeState::Stopped);
10625 assert!(closed.get());
10626 }
10627
10628 #[rstest]
10629 fn test_node_build_rejects_existing_node_preserving_account_delivery(
10630 #[values(false, true)] first_builder: bool,
10631 #[values(false, true)] second_builder: bool,
10632 ) {
10633 let config = LiveNodeConfig {
10634 trader_id: TraderId::from("PROBE-001"),
10635 ..Default::default()
10636 };
10637
10638 let mut node = if first_builder {
10639 LiveNodeBuilder::from_config(config)
10640 .unwrap()
10641 .build()
10642 .unwrap()
10643 } else {
10644 LiveNode::build("Original".to_string(), Some(config)).unwrap()
10645 };
10646
10647 let bus = msgbus::get_message_bus();
10648
10649 let ExecutionEvent::Account(before) = stub_account_event() else {
10650 unreachable!()
10651 };
10652
10653 get_exec_event_sender()
10654 .send(ExecutionEvent::Account(before.clone()))
10655 .unwrap();
10656 assert_eq!(node.drain_runner_pending(), 1);
10657 assert_eq!(
10658 node.kernel
10659 .cache
10660 .borrow()
10661 .account(&before.account_id)
10662 .unwrap()
10663 .last_event(),
10664 Some(before)
10665 );
10666
10667 let result = if second_builder {
10668 LiveNodeBuilder::from_config(LiveNodeConfig::default())
10669 .unwrap()
10670 .build()
10671 } else {
10672 LiveNode::build("Extra".to_string(), None)
10673 };
10674
10675 assert_eq!(
10676 result.unwrap_err().to_string(),
10677 "A LiveNode already exists or is being built on this thread; drop it before building another"
10678 );
10679
10680 let ExecutionEvent::Account(after) = stub_account_event() else {
10681 unreachable!()
10682 };
10683
10684 get_exec_event_sender()
10685 .send(ExecutionEvent::Account(after.clone()))
10686 .unwrap();
10687 assert_eq!(node.drain_runner_pending(), 1);
10688 assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10689 assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10690 assert_eq!(
10691 node.kernel
10692 .cache
10693 .borrow()
10694 .account(&after.account_id)
10695 .unwrap()
10696 .last_event(),
10697 Some(after)
10698 );
10699 }
10700
10701 #[rstest]
10702 fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10703 let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10704 .unwrap()
10705 .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10706 let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10707 assert!(
10708 error
10709 .to_string()
10710 .contains("already exists or is being built")
10711 );
10712 anyhow::bail!("Event store construction failed")
10713 })
10714 .build();
10715
10716 assert_eq!(
10717 result.unwrap_err().to_string(),
10718 "Event store construction failed"
10719 );
10720
10721 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10722 assert_eq!(node.state(), NodeState::Idle);
10723 }
10724
10725 #[rstest]
10726 fn test_node_build_allows_separate_threads() {
10727 let node = LiveNode::build("First".to_string(), None).unwrap();
10728
10729 let other_state = std::thread::spawn(|| {
10730 let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10731 other.state()
10732 })
10733 .join()
10734 .unwrap();
10735
10736 assert_eq!(node.state(), NodeState::Idle);
10737 assert_eq!(other_state, NodeState::Idle);
10738 }
10739
10740 #[rstest]
10741 fn test_node_build_releases_thread_after_drop() {
10742 let mut node = LiveNode::build("First".to_string(), None).unwrap();
10743 node.dispose();
10744 assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10745 drop(node);
10746
10747 let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10748 assert_eq!(replacement.state(), NodeState::Idle);
10749 }
10750
10751 #[rstest]
10752 fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10753 let config = LiveNodeConfig {
10754 event_store: Some(EventStoreConfig::default()),
10755 ..Default::default()
10756 };
10757
10758 let result = if builder {
10759 LiveNodeBuilder::from_config(config).unwrap().build()
10760 } else {
10761 LiveNode::build("Failure".to_string(), Some(config))
10762 };
10763
10764 assert!(result.unwrap_err().to_string().contains("factory"));
10765
10766 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10767 assert_eq!(node.state(), NodeState::Idle);
10768 }
10769
10770 #[cfg(feature = "python")]
10771 #[rstest]
10772 fn test_node_build_and_initial_state() {
10773 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10774 .unwrap()
10775 .with_name("TestNode")
10776 .build()
10777 .unwrap();
10778
10779 assert_eq!(node.state(), NodeState::Idle);
10780 assert!(!node.is_running());
10781 assert_eq!(node.environment(), Environment::Sandbox);
10782 assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10783 }
10784
10785 #[cfg(feature = "python")]
10786 #[rstest]
10787 fn test_node_build_replaces_stale_runner_senders() {
10788 replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10789 replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10790
10791 let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10792 .unwrap()
10793 .with_name("FirstNode")
10794 .build()
10795 .unwrap();
10796
10797 assert_eq!(first.state(), NodeState::Idle);
10798 drop(first);
10799
10800 let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10801 .unwrap()
10802 .with_name("SecondNode")
10803 .build()
10804 .unwrap();
10805
10806 assert_eq!(second.state(), NodeState::Idle);
10807 assert!(!second.is_running());
10808 }
10809
10810 #[cfg(feature = "python")]
10811 #[rstest]
10812 fn test_node_handle_reflects_node_state() {
10813 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10814 .unwrap()
10815 .with_name("TestNode")
10816 .build()
10817 .unwrap();
10818
10819 let handle = node.handle();
10820
10821 assert_eq!(handle.state(), NodeState::Idle);
10822 assert!(!handle.is_running());
10823 }
10824
10825 #[rstest]
10826 fn test_pending_drain_data_returns_false_when_empty() {
10827 let mut pending = PendingEvents::default();
10828
10829 assert!(!pending.drain_data());
10830 }
10831
10832 #[rstest]
10833 fn test_pending_drain_data_returns_true_when_non_empty() {
10834 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10835
10836 let mut pending = PendingEvents::default();
10837 pending.data_evts.push(
10838 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10839 .into(),
10840 );
10841
10842 assert!(pending.drain_data());
10843 assert!(pending.data_evts.is_empty());
10844 }
10845
10846 fn stub_data_event() -> DataEvent {
10847 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10848
10849 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10850 }
10851
10852 fn stub_data_command() -> DataCommand {
10853 use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10854 use nautilus_core::{UUID4, UnixNanos};
10855 use nautilus_model::identifiers::Venue;
10856
10857 DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10858 None,
10859 Venue::from("TEST"),
10860 UUID4::new(),
10861 UnixNanos::default(),
10862 None,
10863 None,
10864 )))
10865 }
10866
10867 fn stub_system_command() -> SystemCommand {
10868 SystemCommand::ReconnectSocket(ReconnectSocket::new(
10869 TraderId::from("TRADER-001"),
10870 ClientId::from("POLYMARKET"),
10871 Ustr::from("polymarket-market-streams"),
10872 UnixNanos::default(),
10873 ))
10874 }
10875
10876 #[rstest]
10877 fn test_flush_pending_data_drains_events_and_commands() {
10878 let (evt_tx, mut evt_rx) =
10879 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10880 let (cmd_tx, mut cmd_rx) =
10881 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10882
10883 let mut pending = PendingEvents::default();
10884
10885 pending.data_evts.push((stub_data_event()).into());
10887 pending.data_cmds.push(stub_data_command().into());
10888
10889 evt_tx.send((stub_data_event()).into()).unwrap();
10891 cmd_tx.send(stub_data_command().into()).unwrap();
10892
10893 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10894
10895 assert!(pending.data_evts.is_empty());
10896 assert!(pending.data_cmds.is_empty());
10897 assert!(evt_rx.try_recv().is_err());
10898 assert!(cmd_rx.try_recv().is_err());
10899 }
10900
10901 #[rstest]
10902 fn test_flush_pending_data_drains_mixed_sources() {
10903 let (evt_tx, mut evt_rx) =
10904 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10905 let (cmd_tx, mut cmd_rx) =
10906 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10907
10908 let mut pending = PendingEvents::default();
10909
10910 pending.data_evts.push((stub_data_event()).into());
10912 cmd_tx.send(stub_data_command().into()).unwrap();
10913
10914 evt_tx.send((stub_data_event()).into()).unwrap();
10916 evt_tx.send((stub_data_event()).into()).unwrap();
10917 cmd_tx.send(stub_data_command().into()).unwrap();
10918
10919 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10920
10921 assert!(pending.data_evts.is_empty());
10922 assert!(pending.data_cmds.is_empty());
10923 assert!(evt_rx.try_recv().is_err());
10924 assert!(cmd_rx.try_recv().is_err());
10925 }
10926
10927 #[rstest]
10928 fn test_pending_system_events_stay_separate_from_data() {
10929 let mut pending = PendingEvents::default();
10930
10931 let change = SocketStateChange::new(
10932 ClientId::from("BINANCE"),
10933 Some(Venue::from("BINANCE")),
10934 ustr::Ustr::from("binance-futures-market-streams"),
10935 SocketState::Connected,
10936 );
10937
10938 pending
10939 .system_events
10940 .push(SystemEvent::SocketState(change).into());
10941 let system_events = pending
10942 .take_system_events()
10943 .into_iter()
10944 .map(|message| message.dispatch(|value| value))
10945 .collect::<Vec<_>>();
10946
10947 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10948 assert!(pending.is_empty());
10949 }
10950
10951 #[rstest]
10952 fn test_pending_system_commands_stay_separate_from_data() {
10953 let mut pending = PendingEvents::default();
10954 let command = stub_system_command();
10955
10956 pending.system_commands.push(command.into());
10957 let system_commands = pending
10958 .take_system_commands()
10959 .into_iter()
10960 .map(|message| message.dispatch(|value| value))
10961 .collect::<Vec<_>>();
10962
10963 assert_eq!(system_commands, vec![command]);
10964 assert!(pending.is_empty());
10965 }
10966
10967 fn stub_time_event_handler() -> TimeEventMessage {
10968 use std::rc::Rc;
10969
10970 use nautilus_common::{
10971 runner::TimeEventMessage,
10972 timer::{TimeEvent, TimeEventCallback},
10973 };
10974 use nautilus_core::{UUID4, UnixNanos};
10975 use ustr::Ustr;
10976
10977 TimeEventMessage::new(
10978 TimeEvent::new(
10979 Ustr::from("test-timer"),
10980 UUID4::new(),
10981 UnixNanos::default(),
10982 UnixNanos::default(),
10983 ),
10984 TimeEventCallback::RustLocal(Rc::new(|_| {})),
10985 )
10986 }
10987
10988 fn stub_trading_command_message() -> TradingCommandMessage {
10989 use nautilus_common::messages::execution::query::QueryAccount;
10990 use nautilus_core::{UUID4, UnixNanos};
10991 use nautilus_model::identifiers::AccountId;
10992
10993 TradingCommandMessage::new(
10994 MessagingSwitchboard::exec_engine_execute(),
10995 TradingCommand::QueryAccount(QueryAccount::new(
10996 TraderId::from("TESTER-001"),
10997 None,
10998 AccountId::from("TEST-001"),
10999 UUID4::new(),
11000 UnixNanos::default(),
11001 None,
11002 None, )),
11004 )
11005 }
11006
11007 fn stub_exec_event() -> ExecutionEvent {
11008 use nautilus_model::{
11009 enums::{LiquiditySide, OrderSide},
11010 identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
11011 reports::FillReport,
11012 types::{Money, Price, Quantity},
11013 };
11014
11015 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
11016 AccountId::from("TEST-001"),
11017 InstrumentId::from("TEST.VENUE"),
11018 VenueOrderId::from("V-001"),
11019 TradeId::from("T-001"),
11020 OrderSide::Buy,
11021 Quantity::from("1.0"),
11022 Price::from("100.0"),
11023 Money::from("0.01 USD"),
11024 LiquiditySide::Maker,
11025 None,
11026 None,
11027 nautilus_core::UnixNanos::default(),
11028 nautilus_core::UnixNanos::default(),
11029 None,
11030 ))))
11031 }
11032
11033 #[rstest]
11034 fn test_flush_all_pending_drains_buffered_channels() {
11035 let (time_tx, mut time_rx) =
11036 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11037 let (system_evt_tx, mut system_evt_rx) =
11038 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11039 let (system_cmd_tx, mut system_cmd_rx) =
11040 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11041 let (data_evt_tx, mut data_evt_rx) =
11042 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11043 let (data_cmd_tx, mut data_cmd_rx) =
11044 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11045 let (exec_evt_tx, mut exec_evt_rx) =
11046 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11047 let (exec_cmd_tx, mut exec_cmd_rx) =
11048 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11049
11050 let mut pending = PendingEvents::default();
11051
11052 pending.data_evts.push((stub_data_event()).into());
11054 pending.data_cmds.push(stub_data_command().into());
11055
11056 time_tx.send((stub_time_event_handler()).into()).unwrap();
11058
11059 let change = SocketStateChange::new(
11060 ClientId::from("BINANCE"),
11061 Some(Venue::from("BINANCE")),
11062 Ustr::from("binance-futures-market-streams"),
11063 SocketState::Connected,
11064 );
11065 system_evt_tx
11066 .send((SystemEvent::SocketState(change)).into())
11067 .unwrap();
11068 system_cmd_tx.send((stub_system_command()).into()).unwrap();
11069 data_evt_tx.send((stub_data_event()).into()).unwrap();
11070 data_cmd_tx.send(stub_data_command().into()).unwrap();
11071 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11072 exec_cmd_tx
11073 .send(stub_trading_command_message().into())
11074 .unwrap();
11075
11076 flush_all_pending(
11077 &mut pending,
11078 &mut time_rx,
11079 &mut system_evt_rx,
11080 &mut system_cmd_rx,
11081 &mut exec_evt_rx,
11082 &mut exec_cmd_rx,
11083 &mut data_evt_rx,
11084 &mut data_cmd_rx,
11085 );
11086
11087 let system_events = pending
11088 .take_system_events()
11089 .into_iter()
11090 .map(|message| message.dispatch(|value| value))
11091 .collect::<Vec<_>>();
11092 let system_commands = pending
11093 .take_system_commands()
11094 .into_iter()
11095 .map(|message| message.dispatch(|value| value))
11096 .collect::<Vec<_>>();
11097 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
11098 assert_eq!(system_commands, vec![stub_system_command()]);
11099 assert!(pending.data_evts.is_empty());
11100 assert!(pending.data_cmds.is_empty());
11101 assert!(pending.exec_reports.is_empty());
11102 assert!(pending.exec_cmds.is_empty());
11103 assert!(pending.order_evts.is_empty());
11104 assert!(time_rx.try_recv().is_err());
11105 assert!(system_evt_rx.try_recv().is_err());
11106 assert!(system_cmd_rx.try_recv().is_err());
11107 assert!(data_evt_rx.try_recv().is_err());
11108 assert!(data_cmd_rx.try_recv().is_err());
11109 assert!(exec_evt_rx.try_recv().is_err());
11110 assert!(exec_cmd_rx.try_recv().is_err());
11111 }
11112
11113 fn stub_order_event() -> ExecutionEvent {
11114 use nautilus_model::events::order::spec::OrderSubmittedSpec;
11115
11116 ExecutionEvent::Order(OrderEventAny::Submitted(
11117 OrderSubmittedSpec::builder().build(),
11118 ))
11119 }
11120
11121 fn stub_account_event() -> ExecutionEvent {
11122 use nautilus_core::{UUID4, UnixNanos};
11123 use nautilus_model::{
11124 enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
11125 };
11126
11127 ExecutionEvent::Account(AccountState::new(
11128 AccountId::from("TEST-001"),
11129 AccountType::Cash,
11130 vec![],
11131 vec![],
11132 true,
11133 UUID4::new(),
11134 UnixNanos::default(),
11135 UnixNanos::default(),
11136 None,
11137 ))
11138 }
11139
11140 #[rstest]
11141 fn test_flush_all_pending_routes_order_event_to_order_evts() {
11142 let (_time_tx, mut time_rx) =
11143 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11144 let (_system_evt_tx, mut system_evt_rx) =
11145 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11146 let (_system_cmd_tx, mut system_cmd_rx) =
11147 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11148 let (_data_evt_tx, mut data_evt_rx) =
11149 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11150 let (_data_cmd_tx, mut data_cmd_rx) =
11151 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11152 let (exec_evt_tx, mut exec_evt_rx) =
11153 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11154 let (_exec_cmd_tx, mut exec_cmd_rx) =
11155 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11156
11157 let mut pending = PendingEvents::default();
11158
11159 exec_evt_tx.send((stub_order_event()).into()).unwrap();
11160 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11161
11162 flush_all_pending(
11163 &mut pending,
11164 &mut time_rx,
11165 &mut system_evt_rx,
11166 &mut system_cmd_rx,
11167 &mut exec_evt_rx,
11168 &mut exec_cmd_rx,
11169 &mut data_evt_rx,
11170 &mut data_cmd_rx,
11171 );
11172
11173 assert!(pending.order_evts.is_empty());
11175 assert!(pending.exec_reports.is_empty());
11176 assert!(exec_evt_rx.try_recv().is_err());
11177 }
11178
11179 #[rstest]
11180 fn test_flush_all_pending_routes_account_event_immediately() {
11181 let (_time_tx, mut time_rx) =
11182 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11183 let (_system_evt_tx, mut system_evt_rx) =
11184 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11185 let (_system_cmd_tx, mut system_cmd_rx) =
11186 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11187 let (_data_evt_tx, mut data_evt_rx) =
11188 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11189 let (_data_cmd_tx, mut data_cmd_rx) =
11190 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11191 let (exec_evt_tx, mut exec_evt_rx) =
11192 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11193 let (_exec_cmd_tx, mut exec_cmd_rx) =
11194 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11195
11196 let mut pending = PendingEvents::default();
11197
11198 exec_evt_tx.send((stub_account_event()).into()).unwrap();
11199
11200 flush_all_pending(
11201 &mut pending,
11202 &mut time_rx,
11203 &mut system_evt_rx,
11204 &mut system_cmd_rx,
11205 &mut exec_evt_rx,
11206 &mut exec_cmd_rx,
11207 &mut data_evt_rx,
11208 &mut data_cmd_rx,
11209 );
11210
11211 assert!(pending.exec_reports.is_empty());
11213 assert!(pending.order_evts.is_empty());
11214 assert!(pending.exec_cmds.is_empty());
11215 assert!(exec_evt_rx.try_recv().is_err());
11216 }
11217
11218 #[rstest]
11219 fn test_pending_is_empty_when_default() {
11220 let pending = PendingEvents::default();
11221
11222 assert!(pending.is_empty());
11223 }
11224
11225 #[rstest]
11226 fn test_pending_is_empty_false_with_data_evt() {
11227 let mut pending = PendingEvents::default();
11228 pending.data_evts.push((stub_data_event()).into());
11229
11230 assert!(!pending.is_empty());
11231 }
11232
11233 #[rstest]
11234 fn test_pending_is_empty_false_with_data_cmd() {
11235 let mut pending = PendingEvents::default();
11236 pending.data_cmds.push(stub_data_command().into());
11237
11238 assert!(!pending.is_empty());
11239 }
11240
11241 #[rstest]
11242 fn test_pending_is_empty_false_with_exec_cmd() {
11243 let mut pending = PendingEvents::default();
11244 pending
11245 .exec_cmds
11246 .push(stub_trading_command_message().into());
11247
11248 assert!(!pending.is_empty());
11249 }
11250
11251 #[rstest]
11252 fn test_pending_drain_preserves_trading_command_target() {
11253 std::thread::spawn(|| {
11254 msgbus::get_message_bus().borrow_mut().dispose();
11255 let risk_commands = Rc::new(RefCell::new(Vec::new()));
11256 let exec_commands = Rc::new(RefCell::new(Vec::new()));
11257
11258 let risk_commands_handler = risk_commands.clone();
11259 msgbus::register_trading_command_endpoint(
11260 MessagingSwitchboard::risk_engine_execute(),
11261 TypedIntoHandler::from(move |command: TradingCommand| {
11262 risk_commands_handler.borrow_mut().push(command);
11263 }),
11264 );
11265
11266 let exec_commands_handler = exec_commands.clone();
11267 msgbus::register_trading_command_endpoint(
11268 MessagingSwitchboard::exec_engine_execute(),
11269 TypedIntoHandler::from(move |command: TradingCommand| {
11270 exec_commands_handler.borrow_mut().push(command);
11271 }),
11272 );
11273
11274 let mut pending = PendingEvents::default();
11275 pending.exec_cmds.push(
11276 TradingCommandMessage::new(
11277 MessagingSwitchboard::risk_engine_execute(),
11278 TradingCommand::QueryAccount(QueryAccount::new(
11279 TraderId::from("TESTER-001"),
11280 None,
11281 AccountId::from("TEST-001"),
11282 UUID4::new(),
11283 UnixNanos::default(),
11284 None,
11285 None,
11286 )),
11287 )
11288 .into(),
11289 );
11290
11291 pending.drain();
11292
11293 assert!(pending.is_empty());
11294 assert_eq!(risk_commands.borrow().len(), 1);
11295 assert!(matches!(
11296 &risk_commands.borrow()[0],
11297 TradingCommand::QueryAccount(_)
11298 ));
11299 assert_eq!(exec_commands.borrow().as_slice(), &[]);
11300 })
11301 .join()
11302 .unwrap();
11303 }
11304
11305 #[rstest]
11306 fn test_pending_is_empty_false_with_exec_report() {
11307 let mut pending = PendingEvents::default();
11308
11309 if let ExecutionEvent::Report(report) = stub_exec_event() {
11310 pending.exec_reports.push((report).into());
11311 }
11312
11313 assert!(!pending.is_empty());
11314 }
11315
11316 #[rstest]
11317 fn test_pending_is_empty_false_with_order_evt() {
11318 let mut pending = PendingEvents::default();
11319
11320 if let ExecutionEvent::Order(order_evt) = stub_order_event() {
11321 pending.order_evts.push((order_evt).into());
11322 }
11323
11324 assert!(!pending.is_empty());
11325 }
11326
11327 fn stub_submitted_batch_event() -> ExecutionEvent {
11328 use nautilus_model::{
11329 events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
11330 identifiers::ClientOrderId,
11331 };
11332
11333 let events = vec![
11334 OrderSubmittedSpec::builder()
11335 .client_order_id(ClientOrderId::from("O-001"))
11336 .build(),
11337 OrderSubmittedSpec::builder()
11338 .client_order_id(ClientOrderId::from("O-002"))
11339 .build(),
11340 ];
11341
11342 ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
11343 }
11344
11345 fn stub_canceled_batch_event() -> ExecutionEvent {
11346 use nautilus_model::{
11347 events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
11348 identifiers::ClientOrderId,
11349 };
11350
11351 let events = vec![
11352 OrderCanceledSpec::builder()
11353 .client_order_id(ClientOrderId::from("O-001"))
11354 .build(),
11355 OrderCanceledSpec::builder()
11356 .client_order_id(ClientOrderId::from("O-002"))
11357 .build(),
11358 ];
11359
11360 ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
11361 }
11362
11363 #[rstest]
11364 fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
11365 let (_time_tx, mut time_rx) =
11366 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11367 let (_system_evt_tx, mut system_evt_rx) =
11368 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11369 let (_system_cmd_tx, mut system_cmd_rx) =
11370 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11371 let (_data_evt_tx, mut data_evt_rx) =
11372 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11373 let (_data_cmd_tx, mut data_cmd_rx) =
11374 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11375 let (exec_evt_tx, mut exec_evt_rx) =
11376 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11377 let (_exec_cmd_tx, mut exec_cmd_rx) =
11378 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11379
11380 let mut pending = PendingEvents::default();
11381
11382 exec_evt_tx
11383 .send((stub_submitted_batch_event()).into())
11384 .unwrap();
11385
11386 flush_all_pending(
11387 &mut pending,
11388 &mut time_rx,
11389 &mut system_evt_rx,
11390 &mut system_cmd_rx,
11391 &mut exec_evt_rx,
11392 &mut exec_cmd_rx,
11393 &mut data_evt_rx,
11394 &mut data_cmd_rx,
11395 );
11396
11397 assert!(pending.order_evts.is_empty());
11399 assert!(exec_evt_rx.try_recv().is_err());
11400 }
11401
11402 #[rstest]
11403 fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
11404 let (_time_tx, mut time_rx) =
11405 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11406 let (_system_evt_tx, mut system_evt_rx) =
11407 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11408 let (_system_cmd_tx, mut system_cmd_rx) =
11409 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11410 let (_data_evt_tx, mut data_evt_rx) =
11411 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11412 let (_data_cmd_tx, mut data_cmd_rx) =
11413 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11414 let (exec_evt_tx, mut exec_evt_rx) =
11415 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11416 let (_exec_cmd_tx, mut exec_cmd_rx) =
11417 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11418
11419 let mut pending = PendingEvents::default();
11420
11421 exec_evt_tx
11422 .send((stub_canceled_batch_event()).into())
11423 .unwrap();
11424
11425 flush_all_pending(
11426 &mut pending,
11427 &mut time_rx,
11428 &mut system_evt_rx,
11429 &mut system_cmd_rx,
11430 &mut exec_evt_rx,
11431 &mut exec_cmd_rx,
11432 &mut data_evt_rx,
11433 &mut data_cmd_rx,
11434 );
11435
11436 assert!(pending.order_evts.is_empty());
11438 assert!(exec_evt_rx.try_recv().is_err());
11439 }
11440
11441 #[rstest]
11442 fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
11443 use nautilus_model::identifiers::ClientOrderId;
11444
11445 let (exec_evt_tx, mut exec_evt_rx) =
11446 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11447
11448 exec_evt_tx
11449 .send((stub_canceled_batch_event()).into())
11450 .unwrap();
11451
11452 let mut pending = PendingEvents::default();
11453
11454 while let Ok(evt) = exec_evt_rx.try_recv() {
11455 pending.push_exec_event(evt);
11456 }
11457
11458 let events: Vec<_> = pending
11459 .order_evts
11460 .drain(..)
11461 .map(|event| event.dispatch(|event| event))
11462 .collect();
11463 assert_eq!(events.len(), 2);
11464 assert!(
11465 matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
11466 );
11467 assert!(
11468 matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
11469 );
11470 }
11471
11472 #[rstest]
11473 #[cfg_attr(
11474 not(all(feature = "simulation", madsim)),
11475 tokio::test(start_paused = true)
11476 )]
11477 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11478 async fn test_submission_confirmation_prevents_duplicate_tracking(
11479 #[values(false, true)] report_confirmation: bool,
11480 #[values(false, true)] exhausted: bool,
11481 ) {
11482 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11483 let client_order_id = order.client_order_id();
11484 let account_id = AccountId::from("TEST-001");
11485 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11486 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11487 OrderPendingUpdateSpec::builder()
11488 .trader_id(order.trader_id())
11489 .strategy_id(order.strategy_id())
11490 .instrument_id(order.instrument_id())
11491 .client_order_id(client_order_id)
11492 .account_id(account_id)
11493 .build(),
11494 )));
11495 advance_clock(Duration::from_millis(101)).await;
11496 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11497
11498 if exhausted {
11499 advance_clock(Duration::from_millis(101)).await;
11500 let result = node.exec_manager.check_inflight_orders();
11501 assert!(result.events.is_empty());
11502 assert!(result.queries.is_empty());
11503 assert_eq!(
11504 node.exec_manager
11505 .take_submission_recovery_exhaustions()
11506 .len(),
11507 1
11508 );
11509 }
11510
11511 if report_confirmation {
11512 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11513 OrderStatusReport::new(
11514 account_id,
11515 order.instrument_id(),
11516 Some(client_order_id),
11517 venue_order_id,
11518 Some(order.order_side()),
11519 order.order_type(),
11520 order.time_in_force(),
11521 OrderStatus::Submitted,
11522 order.quantity(),
11523 Quantity::zero(3),
11524 UnixNanos::default(),
11525 UnixNanos::default(),
11526 UnixNanos::default(),
11527 None,
11528 )
11529 .with_price(order.price().unwrap()),
11530 ))));
11531 node.exec_manager
11532 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11533 advance_clock(Duration::from_millis(101)).await;
11534 let result = node.exec_manager.check_inflight_orders();
11535 assert!(result.queries.is_empty());
11536 assert!(result.events.is_empty());
11537 }
11538
11539 let mut update = OrderUpdatedSpec::builder()
11540 .trader_id(order.trader_id())
11541 .strategy_id(order.strategy_id())
11542 .instrument_id(order.instrument_id())
11543 .client_order_id(client_order_id)
11544 .account_id(account_id)
11545 .quantity(order.quantity())
11546 .price(order.price().unwrap())
11547 .build();
11548
11549 if !report_confirmation {
11550 update.venue_order_id = Some(venue_order_id);
11551 }
11552 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11553 assert_eq!(
11554 node.kernel
11555 .cache
11556 .borrow()
11557 .order(&client_order_id)
11558 .unwrap()
11559 .status(),
11560 OrderStatus::Submitted
11561 );
11562
11563 for _ in 0..3 {
11564 node.exec_manager
11565 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11566 advance_clock(Duration::from_millis(101)).await;
11567 let result = node.exec_manager.check_inflight_orders();
11568 assert!(result.queries.is_empty());
11569 assert!(result.events.is_empty());
11570 assert!(
11571 node.exec_manager
11572 .take_submission_recovery_exhaustions()
11573 .is_empty()
11574 );
11575 }
11576 }
11577
11578 #[rstest]
11579 #[cfg_attr(
11580 not(all(feature = "simulation", madsim)),
11581 tokio::test(start_paused = true)
11582 )]
11583 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11584 async fn test_submission_report_confirmation_preserves_later_command_budget(
11585 #[values(false, true)] cancel: bool,
11586 #[values(false, true)] later_fill: bool,
11587 #[values(false, true)] repeated_report: bool,
11588 #[values(
11589 SubmissionRecoveryPolicy::ResolveLocally,
11590 SubmissionRecoveryPolicy::RetainUnresolved
11591 )]
11592 policy: SubmissionRecoveryPolicy,
11593 ) {
11594 let (mut node, order, instrument) = venue_evidence_node(policy);
11595 let client_order_id = order.client_order_id();
11596 let account_id = AccountId::from("TEST-001");
11597 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11598 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11599 OrderPendingUpdateSpec::builder()
11600 .trader_id(order.trader_id())
11601 .strategy_id(order.strategy_id())
11602 .instrument_id(order.instrument_id())
11603 .client_order_id(client_order_id)
11604 .account_id(account_id)
11605 .build(),
11606 )));
11607 node.exec_manager
11608 .register_inflight(client_order_id, UUID4::new());
11609 advance_clock(Duration::from_millis(101)).await;
11610 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11611 let report = ExecutionReport::Order(Box::new(
11612 OrderStatusReport::new(
11613 account_id,
11614 order.instrument_id(),
11615 Some(client_order_id),
11616 venue_order_id,
11617 Some(order.order_side()),
11618 order.order_type(),
11619 order.time_in_force(),
11620 OrderStatus::Submitted,
11621 order.quantity(),
11622 Quantity::zero(3),
11623 UnixNanos::default(),
11624 UnixNanos::default(),
11625 UnixNanos::default(),
11626 None,
11627 )
11628 .with_price(order.price().unwrap()),
11629 ));
11630 node.process_exec_event(ExecutionEvent::Report(report.clone()));
11631 advance_clock(Duration::from_millis(101)).await;
11632 let result = node.exec_manager.check_inflight_orders();
11633 assert!(result.events.is_empty());
11634 assert!(result.queries.is_empty());
11635
11636 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11637 OrderUpdatedSpec::builder()
11638 .trader_id(order.trader_id())
11639 .strategy_id(order.strategy_id())
11640 .instrument_id(order.instrument_id())
11641 .client_order_id(client_order_id)
11642 .account_id(account_id)
11643 .quantity(order.quantity())
11644 .price(order.price().unwrap())
11645 .build(),
11646 )));
11647 let pending = if cancel {
11648 OrderEventAny::PendingCancel(
11649 OrderPendingCancelSpec::builder()
11650 .trader_id(order.trader_id())
11651 .strategy_id(order.strategy_id())
11652 .instrument_id(order.instrument_id())
11653 .client_order_id(client_order_id)
11654 .account_id(account_id)
11655 .build(),
11656 )
11657 } else {
11658 OrderEventAny::PendingUpdate(
11659 OrderPendingUpdateSpec::builder()
11660 .trader_id(order.trader_id())
11661 .strategy_id(order.strategy_id())
11662 .instrument_id(order.instrument_id())
11663 .client_order_id(client_order_id)
11664 .account_id(account_id)
11665 .build(),
11666 )
11667 };
11668 node.process_exec_event(ExecutionEvent::Order(pending));
11669 node.exec_manager
11670 .register_inflight(client_order_id, UUID4::new());
11671
11672 if repeated_report {
11673 node.process_exec_event(ExecutionEvent::Report(report));
11674 }
11675 advance_clock(Duration::from_millis(50)).await;
11676
11677 if later_fill {
11678 let cached = node
11679 .kernel
11680 .cache
11681 .borrow()
11682 .order_owned(&client_order_id)
11683 .unwrap();
11684 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11685 .last_qty(Quantity::from("0.400"))
11686 .last_px(order.price().unwrap())
11687 .commission(Money::from("0 USDT"))
11688 .without_position_id()
11689 .build()
11690 else {
11691 unreachable!()
11692 };
11693 fill.venue_order_id = venue_order_id;
11694 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11695 }
11696 let early = node.exec_manager.check_inflight_orders();
11697 assert!(early.events.is_empty());
11698 assert!(early.queries.is_empty());
11699 advance_clock(Duration::from_millis(51)).await;
11700 let first = node.exec_manager.check_inflight_orders();
11701 assert_eq!(first.queries.len(), usize::from(!repeated_report));
11702 assert!(first.events.is_empty());
11703 advance_clock(Duration::from_millis(101)).await;
11704 let second = node.exec_manager.check_inflight_orders();
11705 assert!(second.queries.is_empty());
11706 if repeated_report {
11707 assert!(second.events.is_empty());
11708 } else {
11709 assert!(
11710 matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
11711 );
11712 }
11713 assert!(
11714 node.exec_manager
11715 .take_submission_recovery_exhaustions()
11716 .is_empty()
11717 );
11718 }
11719
11720 #[rstest]
11721 #[cfg_attr(
11722 not(all(feature = "simulation", madsim)),
11723 tokio::test(start_paused = true)
11724 )]
11725 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11726 async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11727 #[values(false, true)] report_confirmation: bool,
11728 #[values(false, true)] cancel: bool,
11729 #[values(false, true)] batch: bool,
11730 #[values(false, true)] pending_before_dispatch: bool,
11731 #[values(false, true)] missing_order_check: bool,
11732 #[values(
11733 SubmissionRecoveryPolicy::ResolveLocally,
11734 SubmissionRecoveryPolicy::RetainUnresolved
11735 )]
11736 policy: SubmissionRecoveryPolicy,
11737 ) {
11738 let (mut node, order, instrument) = venue_evidence_node_with_config(
11739 crate::config::LiveExecutionEngineConfig {
11740 reconciliation: true,
11741 inflight_check_threshold_ms: 100,
11742 inflight_check_retries: 2,
11743 open_check_threshold_ms: 5_000,
11744 open_check_open_only: !missing_order_check,
11745 open_check_lookback_mins: None,
11746 open_check_missing_retries: 5,
11747 single_order_query_delay_ms: 0,
11748 submission_recovery_policy: policy,
11749 ..Default::default()
11750 },
11751 OrderType::Limit,
11752 true,
11753 );
11754 let client_order_id = order.client_order_id();
11755 let client_id = ClientId::from("TEST-EVIDENCE");
11756 let account_id = AccountId::from("TEST-001");
11757 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11758 node.kernel
11759 .exec_engine
11760 .borrow_mut()
11761 .register_client(Box::new(StubExecutionClient::new(
11762 client_id,
11763 account_id,
11764 instrument.id().venue,
11765 OmsType::Netting,
11766 None,
11767 )))
11768 .unwrap();
11769
11770 advance_clock(Duration::from_millis(101)).await;
11772 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11773
11774 if report_confirmation {
11775 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11776 OrderPendingUpdateSpec::builder()
11777 .trader_id(order.trader_id())
11778 .strategy_id(order.strategy_id())
11779 .instrument_id(order.instrument_id())
11780 .client_order_id(client_order_id)
11781 .account_id(account_id)
11782 .build(),
11783 )));
11784 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11785 OrderStatusReport::new(
11786 account_id,
11787 order.instrument_id(),
11788 Some(client_order_id),
11789 venue_order_id,
11790 Some(order.order_side()),
11791 order.order_type(),
11792 order.time_in_force(),
11793 OrderStatus::Submitted,
11794 order.quantity(),
11795 Quantity::zero(3),
11796 UnixNanos::default(),
11797 UnixNanos::default(),
11798 UnixNanos::default(),
11799 None,
11800 )
11801 .with_price(order.price().unwrap()),
11802 ))));
11803 }
11804
11805 let mut update = OrderUpdatedSpec::builder()
11806 .trader_id(order.trader_id())
11807 .strategy_id(order.strategy_id())
11808 .instrument_id(order.instrument_id())
11809 .client_order_id(client_order_id)
11810 .account_id(account_id)
11811 .quantity(order.quantity())
11812 .price(order.price().unwrap())
11813 .build();
11814
11815 if !report_confirmation {
11816 update.venue_order_id = Some(venue_order_id);
11817 }
11818 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11819 assert_eq!(
11820 node.kernel
11821 .cache
11822 .borrow()
11823 .order(&client_order_id)
11824 .unwrap()
11825 .status(),
11826 OrderStatus::Submitted,
11827 );
11828 advance_clock(Duration::from_millis(101)).await;
11829 let confirmed = node.exec_manager.check_inflight_orders();
11830 assert!(confirmed.queries.is_empty());
11831 assert!(confirmed.events.is_empty());
11832
11833 let pending = if cancel {
11834 OrderEventAny::PendingCancel(
11835 OrderPendingCancelSpec::builder()
11836 .trader_id(order.trader_id())
11837 .strategy_id(order.strategy_id())
11838 .instrument_id(order.instrument_id())
11839 .client_order_id(client_order_id)
11840 .account_id(account_id)
11841 .build(),
11842 )
11843 } else {
11844 OrderEventAny::PendingUpdate(
11845 OrderPendingUpdateSpec::builder()
11846 .trader_id(order.trader_id())
11847 .strategy_id(order.strategy_id())
11848 .instrument_id(order.instrument_id())
11849 .client_order_id(client_order_id)
11850 .account_id(account_id)
11851 .build(),
11852 )
11853 };
11854 let command = venue_evidence_command(&order, cancel, batch);
11855
11856 if pending_before_dispatch {
11857 node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11858 }
11859 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11860 MessagingSwitchboard::exec_engine_execute(),
11861 command,
11862 )));
11863 advance_clock(Duration::from_millis(50)).await;
11864
11865 if missing_order_check {
11866 let engine = node.kernel.exec_engine.borrow();
11867 let clients = engine.get_all_clients();
11868 let check = node
11869 .exec_manager
11870 .prepare_open_order_report_check(UUID4::new(), &clients);
11871 assert_eq!(check.filtered_orders.len(), 1);
11872 let missing = node.exec_manager.reconcile_open_order_reports(
11873 &check,
11874 Vec::new(),
11875 &IndexSet::from([client_id]),
11876 &IndexSet::new(),
11877 &clients,
11878 );
11879 assert!(missing.events.is_empty());
11880 assert!(missing.targeted_queries.is_empty());
11881 assert_eq!(
11882 node.exec_manager.recon_check_retry_count(&client_order_id),
11883 1
11884 );
11885 }
11886 let early = node.exec_manager.check_inflight_orders();
11887 assert!(early.queries.is_empty());
11888 assert!(early.events.is_empty());
11889
11890 if !pending_before_dispatch {
11891 assert_eq!(
11894 node.kernel
11895 .cache
11896 .borrow()
11897 .order(&client_order_id)
11898 .unwrap()
11899 .status(),
11900 OrderStatus::Submitted,
11901 );
11902 node.process_exec_event(ExecutionEvent::Order(pending));
11903 }
11904
11905 if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11906 node.exec_manager
11907 .register_submission(order.init_event(), Some(client_id));
11908 }
11909 advance_clock(Duration::from_millis(51)).await;
11910 let first = node.exec_manager.check_inflight_orders();
11911 assert_eq!(first.queries.len(), 1);
11912 assert!(
11913 matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11914 if query.client_order_id == client_order_id)
11915 );
11916 assert!(first.events.is_empty());
11917 advance_clock(Duration::from_millis(101)).await;
11918 let exhausted = node.exec_manager.check_inflight_orders();
11919 assert!(exhausted.queries.is_empty());
11920 assert!(
11921 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11922 if event.client_order_id == client_order_id)
11923 );
11924 assert!(
11925 node.exec_manager
11926 .take_submission_recovery_exhaustions()
11927 .is_empty()
11928 );
11929 node.process_reconciliation_events(&exhausted.events);
11930 assert_eq!(
11931 node.kernel
11932 .cache
11933 .borrow()
11934 .order(&client_order_id)
11935 .unwrap()
11936 .status(),
11937 OrderStatus::Canceled,
11938 );
11939 }
11940
11941 #[rstest]
11942 #[cfg_attr(
11943 not(all(feature = "simulation", madsim)),
11944 tokio::test(start_paused = true)
11945 )]
11946 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11947 async fn test_submission_command_report_cleanup_matches_default(
11948 #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11949 #[values(
11950 SubmissionRecoveryPolicy::ResolveLocally,
11951 SubmissionRecoveryPolicy::RetainUnresolved
11952 )]
11953 policy: SubmissionRecoveryPolicy,
11954 ) {
11955 let (mut node, order, instrument) = venue_evidence_node(policy);
11956 let client_order_id = order.client_order_id();
11957 let client_id = ClientId::from("TEST-EVIDENCE");
11958 let account_id = AccountId::from("TEST-001");
11959 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11960 node.kernel
11961 .exec_engine
11962 .borrow_mut()
11963 .register_client(Box::new(StubExecutionClient::new(
11964 client_id,
11965 account_id,
11966 instrument.id().venue,
11967 OmsType::Netting,
11968 None,
11969 )))
11970 .unwrap();
11971 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11972 &order,
11973 account_id,
11974 venue_order_id,
11975 )));
11976 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11977 MessagingSwitchboard::exec_engine_execute(),
11978 venue_evidence_command(&order, true, false),
11979 )));
11980 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11981 OrderPendingCancelSpec::builder()
11982 .trader_id(order.trader_id())
11983 .strategy_id(order.strategy_id())
11984 .instrument_id(order.instrument_id())
11985 .client_order_id(client_order_id)
11986 .account_id(account_id)
11987 .venue_order_id(venue_order_id)
11988 .build(),
11989 )));
11990 advance_clock(Duration::from_millis(101)).await;
11991 let first = node.exec_manager.check_inflight_orders();
11992 assert_eq!(first.queries.len(), 1);
11993 assert!(first.events.is_empty());
11994
11995 let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11996 Quantity::from("0.400")
11997 } else {
11998 Quantity::zero(3)
11999 };
12000
12001 let report = OrderStatusReport::new(
12002 account_id,
12003 order.instrument_id(),
12004 Some(client_order_id),
12005 venue_order_id,
12006 Some(order.order_side()),
12007 order.order_type(),
12008 order.time_in_force(),
12009 report_status,
12010 order.quantity(),
12011 filled_qty,
12012 UnixNanos::default(),
12013 UnixNanos::default(),
12014 UnixNanos::default(),
12015 None,
12016 )
12017 .with_price(order.price().unwrap())
12018 .with_avg_px(dec!(100.00));
12019 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12020 report,
12021 ))));
12022 advance_clock(Duration::from_millis(101)).await;
12023 let exhausted = node.exec_manager.check_inflight_orders();
12024
12025 assert!(exhausted.queries.is_empty());
12026 assert!(exhausted.events.is_empty());
12027 assert!(
12028 node.exec_manager
12029 .take_submission_recovery_exhaustions()
12030 .is_empty()
12031 );
12032 let cache = node.kernel.cache.borrow();
12033 let cached = cache.order_ref(&client_order_id).unwrap();
12034 assert_eq!(cached.status(), OrderStatus::PendingCancel);
12035 assert_eq!(cached.filled_qty(), filled_qty);
12036 }
12037
12038 #[rstest]
12039 #[case::accepted_modify(false, "accepted")]
12040 #[case::accepted_cancel(true, "accepted")]
12041 #[case::accepted_batch_modify(false, "accepted_batch")]
12042 #[case::accepted_batch_cancel(true, "accepted_batch")]
12043 #[case::updated_cancel(true, "updated")]
12044 #[case::triggered_cancel(true, "triggered")]
12045 #[cfg_attr(
12046 not(all(feature = "simulation", madsim)),
12047 tokio::test(start_paused = true)
12048 )]
12049 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12050 async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
12051 #[case] cancel: bool,
12052 #[case] evidence: &str,
12053 #[values(false, true)] batch: bool,
12054 #[values(false, true)] report_first: bool,
12055 #[values(
12056 SubmissionRecoveryPolicy::ResolveLocally,
12057 SubmissionRecoveryPolicy::RetainUnresolved
12058 )]
12059 policy: SubmissionRecoveryPolicy,
12060 ) {
12061 use nautilus_model::events::order::spec::OrderTriggeredSpec;
12062
12063 let triggered = evidence == "triggered";
12064
12065 let order_type = if triggered {
12066 OrderType::StopLimit
12067 } else {
12068 OrderType::Limit
12069 };
12070 let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
12071 let client_order_id = order.client_order_id();
12072 let account_id = AccountId::from("TEST-001");
12073 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12074 node.kernel
12075 .exec_engine
12076 .borrow_mut()
12077 .register_client(Box::new(StubExecutionClient::new(
12078 ClientId::from("TEST-EVIDENCE"),
12079 account_id,
12080 instrument.id().venue,
12081 OmsType::Netting,
12082 None,
12083 )))
12084 .unwrap();
12085 advance_clock(Duration::from_millis(101)).await;
12086 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12087
12088 if triggered || report_first {
12089 let previous_pending = if cancel {
12091 OrderEventAny::PendingUpdate(
12092 OrderPendingUpdateSpec::builder()
12093 .trader_id(order.trader_id())
12094 .strategy_id(order.strategy_id())
12095 .instrument_id(order.instrument_id())
12096 .client_order_id(client_order_id)
12097 .account_id(account_id)
12098 .build(),
12099 )
12100 } else {
12101 OrderEventAny::PendingCancel(
12102 OrderPendingCancelSpec::builder()
12103 .trader_id(order.trader_id())
12104 .strategy_id(order.strategy_id())
12105 .instrument_id(order.instrument_id())
12106 .client_order_id(client_order_id)
12107 .account_id(account_id)
12108 .build(),
12109 )
12110 };
12111 node.process_exec_event(ExecutionEvent::Order(previous_pending));
12112 }
12113 advance_clock(Duration::from_millis(10)).await;
12114 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12115 MessagingSwitchboard::exec_engine_execute(),
12116 venue_evidence_command(&order, cancel, batch),
12117 )));
12118
12119 if report_first {
12120 let report = OrderStatusReport::new(
12121 account_id,
12122 instrument.id(),
12123 Some(client_order_id),
12124 venue_order_id,
12125 Some(order.order_side()),
12126 order.order_type(),
12127 order.time_in_force(),
12128 OrderStatus::Submitted,
12129 order.quantity(),
12130 Quantity::zero(3),
12131 UnixNanos::default(),
12132 UnixNanos::default(),
12133 UnixNanos::default(),
12134 None,
12135 )
12136 .with_price(order.price().unwrap());
12137 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12138 report,
12139 ))));
12140 assert!(
12141 node.kernel
12142 .cache
12143 .borrow()
12144 .order(&client_order_id)
12145 .unwrap()
12146 .venue_order_id()
12147 .is_none()
12148 );
12149 }
12150 advance_clock(Duration::from_millis(40)).await;
12151 let acknowledgement = if triggered {
12152 OrderEventAny::Triggered(
12153 OrderTriggeredSpec::builder()
12154 .trader_id(order.trader_id())
12155 .strategy_id(order.strategy_id())
12156 .instrument_id(order.instrument_id())
12157 .client_order_id(client_order_id)
12158 .account_id(account_id)
12159 .venue_order_id(venue_order_id)
12160 .build(),
12161 )
12162 } else if evidence == "updated" {
12163 OrderEventAny::Updated(
12164 OrderUpdatedSpec::builder()
12165 .trader_id(order.trader_id())
12166 .strategy_id(order.strategy_id())
12167 .instrument_id(order.instrument_id())
12168 .client_order_id(client_order_id)
12169 .account_id(account_id)
12170 .venue_order_id(venue_order_id)
12171 .quantity(order.quantity())
12172 .price(order.price().unwrap())
12173 .build(),
12174 )
12175 } else {
12176 TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
12177 };
12178
12179 if evidence == "accepted_batch" {
12180 let OrderEventAny::Accepted(accepted) = acknowledgement else {
12181 unreachable!()
12182 };
12183 node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
12184 vec![accepted],
12185 )));
12186 } else {
12187 node.process_exec_event(ExecutionEvent::Order(acknowledgement));
12188 }
12189 assert_eq!(
12190 node.kernel
12191 .cache
12192 .borrow()
12193 .order(&client_order_id)
12194 .unwrap()
12195 .status(),
12196 match evidence {
12197 "triggered" => OrderStatus::Triggered,
12198 "updated" => OrderStatus::Submitted,
12199 _ => OrderStatus::Accepted,
12200 }
12201 );
12202 advance_clock(Duration::from_millis(20)).await;
12203 let pending = if cancel {
12204 OrderEventAny::PendingCancel(
12205 OrderPendingCancelSpec::builder()
12206 .trader_id(order.trader_id())
12207 .strategy_id(order.strategy_id())
12208 .instrument_id(order.instrument_id())
12209 .client_order_id(client_order_id)
12210 .account_id(account_id)
12211 .build(),
12212 )
12213 } else {
12214 OrderEventAny::PendingUpdate(
12215 OrderPendingUpdateSpec::builder()
12216 .trader_id(order.trader_id())
12217 .strategy_id(order.strategy_id())
12218 .instrument_id(order.instrument_id())
12219 .client_order_id(client_order_id)
12220 .account_id(account_id)
12221 .build(),
12222 )
12223 };
12224 node.process_exec_event(ExecutionEvent::Order(pending));
12225 let early = node.exec_manager.check_inflight_orders();
12226 assert!(early.queries.is_empty());
12227 assert!(early.events.is_empty());
12228 advance_clock(Duration::from_millis(41)).await;
12229 let first = node.exec_manager.check_inflight_orders();
12230 let recover_command =
12232 triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
12233 assert_eq!(first.queries.len(), usize::from(recover_command));
12234 assert!(first.events.is_empty());
12235 advance_clock(Duration::from_millis(101)).await;
12236 let exhausted = node.exec_manager.check_inflight_orders();
12237 assert!(exhausted.queries.is_empty());
12238 assert!(
12239 node.exec_manager
12240 .take_submission_recovery_exhaustions()
12241 .is_empty()
12242 );
12243
12244 if recover_command {
12245 assert!(
12246 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12247 if event.client_order_id == client_order_id)
12248 );
12249 } else {
12250 assert!(exhausted.events.is_empty());
12251 }
12252 }
12253
12254 #[rstest]
12255 #[case::submission_only(None, false)]
12256 #[case::modify(Some(false), false)]
12257 #[case::cancel(Some(true), false)]
12258 #[case::batch_modify(Some(false), true)]
12259 #[case::batch_cancel(Some(true), true)]
12260 #[cfg_attr(
12261 not(all(feature = "simulation", madsim)),
12262 tokio::test(start_paused = true)
12263 )]
12264 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12265 async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
12266 #[case] cancel: Option<bool>,
12267 #[case] batch: bool,
12268 #[values(false, true)] missing_order_check: bool,
12269 #[values(
12270 SubmissionRecoveryPolicy::ResolveLocally,
12271 SubmissionRecoveryPolicy::RetainUnresolved
12272 )]
12273 policy: SubmissionRecoveryPolicy,
12274 ) {
12275 let (mut node, order, instrument) = venue_evidence_node_with_config(
12276 crate::config::LiveExecutionEngineConfig {
12277 reconciliation: true,
12278 inflight_check_threshold_ms: 100,
12279 inflight_check_retries: 2,
12280 open_check_threshold_ms: 1,
12281 open_check_open_only: false,
12282 open_check_lookback_mins: None,
12283 open_check_missing_retries: 5,
12284 single_order_query_delay_ms: 0,
12285 submission_recovery_policy: policy,
12286 ..Default::default()
12287 },
12288 OrderType::Limit,
12289 false,
12290 );
12291 let client_order_id = order.client_order_id();
12292 let client_id = ClientId::from("TEST-EVIDENCE");
12293 let account_id = AccountId::from("TEST-001");
12294 node.kernel
12295 .exec_engine
12296 .borrow_mut()
12297 .register_client(Box::new(StubExecutionClient::new(
12298 client_id,
12299 account_id,
12300 instrument.id().venue,
12301 OmsType::Netting,
12302 None,
12303 )))
12304 .unwrap();
12305 let queries = Rc::new(RefCell::new(Vec::new()));
12306 let client = OrderReportClient {
12307 client_id,
12308 report: OrderStatusReport::new(
12309 account_id,
12310 instrument.id(),
12311 Some(client_order_id),
12312 VenueOrderId::from("V-VENUE-EVIDENCE"),
12313 Some(order.order_side()),
12314 order.order_type(),
12315 order.time_in_force(),
12316 OrderStatus::Submitted,
12317 order.quantity(),
12318 Quantity::zero(3),
12319 UnixNanos::default(),
12320 UnixNanos::default(),
12321 UnixNanos::default(),
12322 None,
12323 )
12324 .with_price(order.price().unwrap()),
12325 targeted: false,
12326 queries: queries.clone(),
12327 };
12328 advance_clock(Duration::from_millis(2)).await;
12329 let events = node.exec_manager.check_open_orders(&[&client]).await;
12330 assert!(events.is_empty());
12331 assert_eq!(*queries.borrow(), vec!["bulk"]);
12332 assert_eq!(
12333 node.kernel
12334 .cache
12335 .borrow()
12336 .order(&client_order_id)
12337 .unwrap()
12338 .status(),
12339 OrderStatus::Submitted
12340 );
12341 advance_clock(Duration::from_millis(2)).await;
12342
12343 if missing_order_check {
12344 let engine = node.kernel.exec_engine.borrow();
12345 let clients = engine.get_all_clients();
12346 let check = node
12347 .exec_manager
12348 .prepare_open_order_report_check(UUID4::new(), &clients);
12349 assert_eq!(check.filtered_orders.len(), 1);
12350 let missing = node.exec_manager.reconcile_open_order_reports(
12351 &check,
12352 Vec::new(),
12353 &IndexSet::from([client_id]),
12354 &IndexSet::new(),
12355 &clients,
12356 );
12357 assert!(missing.events.is_empty());
12358 assert!(missing.targeted_queries.is_empty());
12359 assert_eq!(
12360 node.exec_manager.recon_check_retry_count(&client_order_id),
12361 1
12362 );
12363 }
12364 node.exec_manager
12365 .register_submission(order.init_event(), Some(client_id));
12366 advance_clock(Duration::from_millis(101)).await;
12367 let duplicate = node.exec_manager.check_inflight_orders();
12368 assert!(duplicate.events.is_empty());
12369 assert_eq!(
12370 duplicate.queries.len(),
12371 usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
12372 );
12373
12374 if let Some(cancel) = cancel {
12375 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12376 MessagingSwitchboard::exec_engine_execute(),
12377 venue_evidence_command(&order, cancel, batch),
12378 )));
12379 let pending = if cancel {
12380 OrderEventAny::PendingCancel(
12381 OrderPendingCancelSpec::builder()
12382 .trader_id(order.trader_id())
12383 .strategy_id(order.strategy_id())
12384 .instrument_id(order.instrument_id())
12385 .client_order_id(client_order_id)
12386 .account_id(account_id)
12387 .build(),
12388 )
12389 } else {
12390 OrderEventAny::PendingUpdate(
12391 OrderPendingUpdateSpec::builder()
12392 .trader_id(order.trader_id())
12393 .strategy_id(order.strategy_id())
12394 .instrument_id(order.instrument_id())
12395 .client_order_id(client_order_id)
12396 .account_id(account_id)
12397 .build(),
12398 )
12399 };
12400 node.process_exec_event(ExecutionEvent::Order(pending));
12401 }
12402 advance_clock(Duration::from_millis(101)).await;
12403 let first = node.exec_manager.check_inflight_orders();
12404 assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
12405
12406 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12407 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12408 if event.client_order_id == client_order_id));
12409 } else {
12410 assert!(first.events.is_empty());
12411 }
12412 node.process_reconciliation_events(&first.events);
12413 advance_clock(Duration::from_millis(101)).await;
12414 let exhausted = node.exec_manager.check_inflight_orders();
12415 assert!(exhausted.queries.is_empty());
12416
12417 if cancel.is_some() {
12418 assert!(
12419 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12420 if event.client_order_id == client_order_id)
12421 );
12422 } else {
12423 assert!(exhausted.events.is_empty());
12424 }
12425 assert!(
12426 node.exec_manager
12427 .take_submission_recovery_exhaustions()
12428 .is_empty()
12429 );
12430 }
12431
12432 #[rstest]
12433 #[case::submission_only(None, false)]
12434 #[case::modify(Some(false), false)]
12435 #[case::cancel(Some(true), false)]
12436 #[case::batch_modify(Some(false), true)]
12437 #[case::batch_cancel(Some(true), true)]
12438 #[cfg_attr(
12439 not(all(feature = "simulation", madsim)),
12440 tokio::test(start_paused = true)
12441 )]
12442 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12443 async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
12444 #[case] cancel: Option<bool>,
12445 #[case] batch: bool,
12446 #[values(false, true)] targeted: bool,
12447 #[values(
12448 SubmissionRecoveryPolicy::ResolveLocally,
12449 SubmissionRecoveryPolicy::RetainUnresolved
12450 )]
12451 policy: SubmissionRecoveryPolicy,
12452 ) {
12453 let (mut node, order, instrument) = venue_evidence_node_with_config(
12454 crate::config::LiveExecutionEngineConfig {
12455 reconciliation: true,
12456 inflight_check_threshold_ms: 100,
12457 inflight_check_retries: 2,
12458 open_check_threshold_ms: 1,
12459 open_check_open_only: false,
12460 open_check_lookback_mins: None,
12461 open_check_missing_retries: 1,
12462 single_order_query_delay_ms: 0,
12463 submission_recovery_policy: policy,
12464 ..Default::default()
12465 },
12466 OrderType::Limit,
12467 true,
12468 );
12469 let client_order_id = order.client_order_id();
12470 let client_id = ClientId::from("TEST-EVIDENCE");
12471 let account_id = AccountId::from("TEST-001");
12472 node.kernel
12473 .exec_engine
12474 .borrow_mut()
12475 .register_client(Box::new(StubExecutionClient::new(
12476 client_id,
12477 account_id,
12478 instrument.id().venue,
12479 OmsType::Netting,
12480 None,
12481 )))
12482 .unwrap();
12483 advance_clock(Duration::from_millis(101)).await;
12484 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12485 advance_clock(Duration::from_millis(10)).await;
12486
12487 if let Some(cancel) = cancel {
12488 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12489 MessagingSwitchboard::exec_engine_execute(),
12490 venue_evidence_command(&order, cancel, batch),
12491 )));
12492 let pending = if cancel {
12493 OrderEventAny::PendingCancel(
12494 OrderPendingCancelSpec::builder()
12495 .trader_id(order.trader_id())
12496 .strategy_id(order.strategy_id())
12497 .instrument_id(order.instrument_id())
12498 .client_order_id(client_order_id)
12499 .account_id(account_id)
12500 .build(),
12501 )
12502 } else {
12503 OrderEventAny::PendingUpdate(
12504 OrderPendingUpdateSpec::builder()
12505 .trader_id(order.trader_id())
12506 .strategy_id(order.strategy_id())
12507 .instrument_id(order.instrument_id())
12508 .client_order_id(client_order_id)
12509 .account_id(account_id)
12510 .build(),
12511 )
12512 };
12513 node.process_exec_event(ExecutionEvent::Order(pending));
12514 }
12515 let queries = Rc::new(RefCell::new(Vec::new()));
12516 let client = OrderReportClient {
12517 client_id,
12518 report: OrderStatusReport::new(
12519 account_id,
12520 order.instrument_id(),
12521 Some(client_order_id),
12522 VenueOrderId::from("V-VENUE-EVIDENCE"),
12523 Some(order.order_side()),
12524 order.order_type(),
12525 order.time_in_force(),
12526 OrderStatus::Submitted,
12527 order.quantity(),
12528 Quantity::zero(3),
12529 UnixNanos::default(),
12530 UnixNanos::default(),
12531 UnixNanos::default(),
12532 None,
12533 )
12534 .with_price(order.price().unwrap()),
12535 targeted,
12536 queries: queries.clone(),
12537 };
12538 advance_clock(Duration::from_millis(40)).await;
12539 let events = node.exec_manager.check_open_orders(&[&client]).await;
12540 assert!(events.is_empty());
12541 assert_eq!(
12542 *queries.borrow(),
12543 if targeted {
12544 vec!["bulk", "targeted"]
12545 } else {
12546 vec!["bulk"]
12547 }
12548 );
12549 node.process_reconciliation_events(&events);
12550 advance_clock(Duration::from_millis(61)).await;
12551 let first = node.exec_manager.check_inflight_orders();
12552 let recover_command =
12553 cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12554 assert_eq!(first.queries.len(), usize::from(recover_command));
12555
12556 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12557 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12558 if event.client_order_id == client_order_id));
12559 } else {
12560 assert!(first.events.is_empty());
12561 }
12562 node.process_reconciliation_events(&first.events);
12563 advance_clock(Duration::from_millis(101)).await;
12564 let exhausted = node.exec_manager.check_inflight_orders();
12565 assert!(exhausted.queries.is_empty());
12566
12567 if recover_command {
12568 assert!(
12569 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12570 if event.client_order_id == client_order_id)
12571 );
12572 } else {
12573 assert!(exhausted.events.is_empty());
12574 }
12575 assert!(
12576 node.exec_manager
12577 .take_submission_recovery_exhaustions()
12578 .is_empty()
12579 );
12580 }
12581
12582 #[rstest]
12583 #[case::local_update_before_pending("updated")]
12584 #[case::modify_rejected_after_pending("modify_rejected")]
12585 #[case::cancel_rejected_after_pending("cancel_rejected")]
12586 #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12587 #[cfg_attr(
12588 not(all(feature = "simulation", madsim)),
12589 tokio::test(start_paused = true)
12590 )]
12591 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12592 async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12593 #[case] completion: &str,
12594 #[values(
12595 SubmissionRecoveryPolicy::ResolveLocally,
12596 SubmissionRecoveryPolicy::RetainUnresolved
12597 )]
12598 policy: SubmissionRecoveryPolicy,
12599 ) {
12600 use nautilus_model::events::order::spec::{
12601 OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12602 };
12603
12604 let (mut node, order, instrument) = venue_evidence_node(policy);
12605 let client_order_id = order.client_order_id();
12606 let client_id = ClientId::from("TEST-EVIDENCE");
12607 let account_id = AccountId::from("TEST-001");
12608 let cancel = completion == "cancel_rejected";
12609 let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12610 node.kernel
12611 .exec_engine
12612 .borrow_mut()
12613 .register_client(Box::new(StubExecutionClient::new(
12614 client_id,
12615 account_id,
12616 instrument.id().venue,
12617 OmsType::Netting,
12618 None,
12619 )))
12620 .unwrap();
12621 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12622 MessagingSwitchboard::exec_engine_execute(),
12623 venue_evidence_command(&order, cancel, false),
12624 )));
12625 advance_clock(Duration::from_millis(10)).await;
12626
12627 if completion != "updated" {
12628 let pending = if cancel {
12629 OrderEventAny::PendingCancel(
12630 OrderPendingCancelSpec::builder()
12631 .trader_id(order.trader_id())
12632 .strategy_id(order.strategy_id())
12633 .instrument_id(order.instrument_id())
12634 .client_order_id(client_order_id)
12635 .account_id(account_id)
12636 .build(),
12637 )
12638 } else {
12639 OrderEventAny::PendingUpdate(
12640 OrderPendingUpdateSpec::builder()
12641 .trader_id(order.trader_id())
12642 .strategy_id(order.strategy_id())
12643 .instrument_id(order.instrument_id())
12644 .client_order_id(client_order_id)
12645 .account_id(account_id)
12646 .build(),
12647 )
12648 };
12649 node.process_exec_event(ExecutionEvent::Order(pending));
12650 }
12651
12652 if later_cancel_pending {
12653 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12655 OrderPendingCancelSpec::builder()
12656 .trader_id(order.trader_id())
12657 .strategy_id(order.strategy_id())
12658 .instrument_id(order.instrument_id())
12659 .client_order_id(client_order_id)
12660 .account_id(account_id)
12661 .build(),
12662 )));
12663 }
12664 advance_clock(Duration::from_millis(10)).await;
12665 let completed = match completion {
12666 "updated" => OrderEventAny::Updated(
12667 OrderUpdatedSpec::builder()
12668 .trader_id(order.trader_id())
12669 .strategy_id(order.strategy_id())
12670 .instrument_id(order.instrument_id())
12671 .client_order_id(client_order_id)
12672 .account_id(account_id)
12673 .quantity(order.quantity())
12674 .price(Price::from("101.00"))
12675 .build(),
12676 ),
12677 "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12678 OrderEventAny::ModifyRejected(
12679 OrderModifyRejectedSpec::builder()
12680 .trader_id(order.trader_id())
12681 .strategy_id(order.strategy_id())
12682 .instrument_id(order.instrument_id())
12683 .client_order_id(client_order_id)
12684 .account_id(account_id)
12685 .build(),
12686 )
12687 }
12688 "cancel_rejected" => OrderEventAny::CancelRejected(
12689 OrderCancelRejectedSpec::builder()
12690 .trader_id(order.trader_id())
12691 .strategy_id(order.strategy_id())
12692 .instrument_id(order.instrument_id())
12693 .client_order_id(client_order_id)
12694 .account_id(account_id)
12695 .build(),
12696 ),
12697 _ => unreachable!(),
12698 };
12699 node.process_exec_event(ExecutionEvent::Order(completed));
12700 assert_eq!(
12701 node.kernel
12702 .cache
12703 .borrow()
12704 .order(&client_order_id)
12705 .unwrap()
12706 .status(),
12707 if later_cancel_pending {
12708 OrderStatus::PendingCancel
12709 } else {
12710 OrderStatus::Submitted
12711 }
12712 );
12713
12714 advance_clock(Duration::from_millis(10)).await;
12716
12717 if !later_cancel_pending {
12718 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12719 OrderPendingUpdateSpec::builder()
12720 .trader_id(order.trader_id())
12721 .strategy_id(order.strategy_id())
12722 .instrument_id(order.instrument_id())
12723 .client_order_id(client_order_id)
12724 .account_id(account_id)
12725 .build(),
12726 )));
12727 }
12728 advance_clock(Duration::from_millis(10)).await;
12729 let cached = node
12730 .kernel
12731 .cache
12732 .borrow()
12733 .order_owned(&client_order_id)
12734 .unwrap();
12735 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12736 .last_qty(Quantity::from("0.400"))
12737 .last_px(order.price().unwrap())
12738 .commission(Money::from("0 USDT"))
12739 .without_position_id()
12740 .build()
12741 else {
12742 unreachable!()
12743 };
12744 fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12745 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12746
12747 for _ in 0..3 {
12748 advance_clock(Duration::from_millis(101)).await;
12749 let result = node.exec_manager.check_inflight_orders();
12750 assert!(result.queries.is_empty());
12751 assert!(result.events.is_empty());
12752 assert!(
12753 node.exec_manager
12754 .take_submission_recovery_exhaustions()
12755 .is_empty()
12756 );
12757 }
12758 assert_eq!(
12759 node.kernel
12760 .cache
12761 .borrow()
12762 .order(&client_order_id)
12763 .unwrap()
12764 .status(),
12765 if later_cancel_pending {
12766 OrderStatus::PendingCancel
12767 } else {
12768 OrderStatus::PendingUpdate
12769 }
12770 );
12771 }
12772
12773 #[rstest]
12774 #[case::wrong_account("account")]
12775 #[case::wrong_instrument("instrument")]
12776 #[case::conflicting_report_venue("report_venue")]
12777 #[case::unapplied_venue_update("update_venue")]
12778 #[case::local_update("local_update")]
12779 #[cfg_attr(
12780 not(all(feature = "simulation", madsim)),
12781 tokio::test(start_paused = true)
12782 )]
12783 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12784 async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12785 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12786 let client_order_id = order.client_order_id();
12787 let account_id = AccountId::from("TEST-001");
12788 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12789
12790 if invalid.ends_with("venue") {
12791 insert_accepted_limit_order_in_node(
12792 &node,
12793 account_id,
12794 ClientId::from("TEST-EVIDENCE"),
12795 order.instrument_id(),
12796 ClientOrderId::from("O-OTHER"),
12797 venue_order_id,
12798 );
12799 }
12800 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12801 OrderPendingCancelSpec::builder()
12802 .trader_id(order.trader_id())
12803 .strategy_id(order.strategy_id())
12804 .instrument_id(order.instrument_id())
12805 .client_order_id(client_order_id)
12806 .account_id(account_id)
12807 .build(),
12808 )));
12809 advance_clock(Duration::from_millis(101)).await;
12810 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12811
12812 if invalid == "update_venue" || invalid == "local_update" {
12813 let mut updated = OrderUpdatedSpec::builder()
12814 .trader_id(order.trader_id())
12815 .strategy_id(order.strategy_id())
12816 .instrument_id(order.instrument_id())
12817 .client_order_id(client_order_id)
12818 .account_id(account_id)
12819 .quantity(order.quantity())
12820 .price(order.price().unwrap())
12821 .build();
12822
12823 if invalid == "update_venue" {
12824 updated.venue_order_id = Some(venue_order_id);
12825 }
12826 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12827 } else {
12828 let report = OrderStatusReport::new(
12829 if invalid == "account" {
12830 AccountId::from("OTHER-001")
12831 } else {
12832 account_id
12833 },
12834 if invalid == "instrument" {
12835 InstrumentId::from("OTHER.BINANCE")
12836 } else {
12837 order.instrument_id()
12838 },
12839 Some(client_order_id),
12840 venue_order_id,
12841 Some(order.order_side()),
12842 order.order_type(),
12843 order.time_in_force(),
12844 OrderStatus::Submitted,
12845 order.quantity(),
12846 Quantity::zero(3),
12847 UnixNanos::default(),
12848 UnixNanos::default(),
12849 UnixNanos::default(),
12850 None,
12851 )
12852 .with_price(order.price().unwrap());
12853 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12854 report,
12855 ))));
12856 }
12857 assert!(
12858 node.kernel
12859 .cache
12860 .borrow()
12861 .order(&client_order_id)
12862 .unwrap()
12863 .venue_order_id()
12864 .is_none()
12865 );
12866 advance_clock(Duration::from_millis(101)).await;
12867 let result = node.exec_manager.check_inflight_orders();
12868 assert!(result.events.is_empty());
12869 assert!(result.queries.is_empty());
12870 let cached = node
12871 .kernel
12872 .cache
12873 .borrow()
12874 .order_owned(&client_order_id)
12875 .unwrap();
12876 assert_eq!(cached.status(), OrderStatus::PendingCancel);
12877 let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12878 assert_eq!(diagnostics.len(), 1);
12879 assert_eq!(diagnostics[0].client_order_id, client_order_id);
12880 }
12881
12882 fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12883 use nautilus_common::messages::execution::{
12884 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12885 };
12886
12887 let client_id = ClientId::from("TEST-EVIDENCE");
12888 let client_order_id = order.client_order_id();
12889 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12890
12891 if cancel {
12892 let cancel = CancelOrder::new(
12893 order.trader_id(),
12894 Some(client_id),
12895 order.strategy_id(),
12896 order.instrument_id(),
12897 client_order_id,
12898 Some(venue_order_id),
12899 UUID4::new(),
12900 UnixNanos::default(),
12901 None,
12902 None,
12903 );
12904
12905 if batch {
12906 TradingCommand::CancelOrders(BatchCancelOrders::new(
12907 order.trader_id(),
12908 Some(client_id),
12909 order.strategy_id(),
12910 order.instrument_id(),
12911 vec![cancel],
12912 UUID4::new(),
12913 UnixNanos::default(),
12914 None,
12915 None,
12916 ))
12917 } else {
12918 TradingCommand::CancelOrder(cancel)
12919 }
12920 } else {
12921 let modify = ModifyOrder::new(
12922 order.trader_id(),
12923 Some(client_id),
12924 order.strategy_id(),
12925 order.instrument_id(),
12926 client_order_id,
12927 Some(venue_order_id),
12928 None,
12929 Some(Price::from("101.00")),
12930 None,
12931 UUID4::new(),
12932 UnixNanos::default(),
12933 None,
12934 None,
12935 );
12936
12937 if batch {
12938 TradingCommand::ModifyOrders(BatchModifyOrders::new(
12939 order.trader_id(),
12940 Some(client_id),
12941 order.strategy_id(),
12942 order.instrument_id(),
12943 vec![modify],
12944 UUID4::new(),
12945 UnixNanos::default(),
12946 None,
12947 None,
12948 ))
12949 } else {
12950 TradingCommand::ModifyOrder(modify)
12951 }
12952 }
12953 }
12954
12955 fn venue_evidence_node(
12956 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12957 ) -> (LiveNode, OrderAny, InstrumentAny) {
12958 venue_evidence_node_with_order_type(policy, OrderType::Limit)
12959 }
12960
12961 fn venue_evidence_node_with_order_type(
12962 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12963 order_type: OrderType,
12964 ) -> (LiveNode, OrderAny, InstrumentAny) {
12965 venue_evidence_node_with_config(
12966 crate::config::LiveExecutionEngineConfig {
12967 reconciliation: true,
12968 inflight_check_threshold_ms: 100,
12969 inflight_check_retries: 2,
12970 open_check_threshold_ms: 5_000,
12971 single_order_query_delay_ms: 0,
12972 submission_recovery_policy: policy,
12973 ..Default::default()
12974 },
12975 order_type,
12976 true,
12977 )
12978 }
12979
12980 fn venue_evidence_node_with_config(
12981 exec_engine: crate::config::LiveExecutionEngineConfig,
12982 order_type: OrderType,
12983 register_submission: bool,
12984 ) -> (LiveNode, OrderAny, InstrumentAny) {
12985 let config = LiveNodeConfig {
12986 exec_engine,
12987 ..Default::default()
12988 };
12989 let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12990 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12991 let account_id = AccountId::from("TEST-001");
12992 let client_id = ClientId::from("TEST-EVIDENCE");
12993 let account = AccountAny::Margin(MarginAccount::new(
12994 AccountState::new(
12995 account_id,
12996 AccountType::Margin,
12997 vec![AccountBalance::new(
12998 Money::from("1000000 USDT"),
12999 Money::from("0 USDT"),
13000 Money::from("1000000 USDT"),
13001 )],
13002 Vec::new(),
13003 true,
13004 UUID4::new(),
13005 UnixNanos::default(),
13006 UnixNanos::default(),
13007 Some(Currency::USDT()),
13008 ),
13009 true,
13010 ));
13011 node.kernel.cache.borrow_mut().add_account(account).unwrap();
13012 node.kernel
13013 .cache
13014 .borrow_mut()
13015 .add_instrument(instrument.clone())
13016 .unwrap();
13017 let mut builder = OrderTestBuilder::new(order_type);
13018 builder
13019 .trader_id(node.trader_id())
13020 .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
13021 .instrument_id(instrument.id())
13022 .quantity(Quantity::from("1.000"))
13023 .price(Price::from("100.00"));
13024
13025 if order_type == OrderType::StopLimit {
13026 builder.trigger_price(Price::from("110.00"));
13027 }
13028 let order = builder.build();
13029 let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
13030 &order,
13031 node.trader_id(),
13032 Some(client_id),
13033 None,
13034 UUID4::new(),
13035 UnixNanos::default(),
13036 ));
13037
13038 if register_submission {
13039 node.observe_exec_command_before_dispatch(&command);
13040 }
13041 node.kernel
13042 .cache
13043 .borrow_mut()
13044 .add_order(order.clone(), None, Some(client_id), false)
13045 .unwrap();
13046 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
13047 &order, account_id,
13048 )));
13049 (node, order, instrument)
13050 }
13051
13052 #[rstest]
13053 #[case::partial("0.400", OrderStatus::PartiallyFilled)]
13054 #[case::full("1.000", OrderStatus::Filled)]
13055 #[tokio::test(start_paused = true)]
13056 async fn test_retained_submission_resolves_late_fill(
13057 #[case] quantity: &str,
13058 #[case] expected_status: OrderStatus,
13059 ) {
13060 let (mut node, order, instrument) =
13061 venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
13062 let client_order_id = order.client_order_id();
13063
13064 for _ in 0..2 {
13065 advance_clock(Duration::from_millis(101)).await;
13066 assert!(node.exec_manager.check_inflight_orders().events.is_empty());
13067 }
13068
13069 assert_eq!(
13070 node.exec_manager
13071 .take_submission_recovery_exhaustions()
13072 .len(),
13073 1
13074 );
13075 let fill = TestOrderEventStubs::filled(
13076 &order,
13077 &instrument,
13078 Some(TradeId::from("T-LATE-RETAINED")),
13079 None,
13080 Some(Price::from("100.00")),
13081 Some(Quantity::from(quantity)),
13082 Some(LiquiditySide::Taker),
13083 None,
13084 None,
13085 Some(AccountId::from("TEST-001")),
13086 );
13087 node.process_exec_event(ExecutionEvent::Order(fill));
13088 advance_clock(Duration::from_millis(101)).await;
13089 let result = node.exec_manager.check_inflight_orders();
13090 let cache = node.kernel.cache.borrow();
13091 let cached = cache.order_ref(&client_order_id).unwrap();
13092 assert_eq!(cached.status(), expected_status);
13093 assert_eq!(cached.filled_qty(), Quantity::from(quantity));
13094 assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
13095 assert!(result.events.is_empty());
13096 assert!(result.queries.is_empty());
13097 assert!(
13098 node.exec_manager
13099 .take_submission_recovery_exhaustions()
13100 .is_empty()
13101 );
13102 }
13103
13104 #[rstest]
13105 #[case::submitted_report("order")]
13106 #[case::submitted_report_with_fills("order_with_fills")]
13107 #[case::venue_update("updated")]
13108 #[cfg_attr(
13109 not(all(feature = "simulation", madsim)),
13110 tokio::test(start_paused = true)
13111 )]
13112 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13113 async fn test_venue_evidence_stops_submission_timeout(
13114 #[case] evidence: &str,
13115 #[values(
13116 OrderStatus::Submitted,
13117 OrderStatus::PendingCancel,
13118 OrderStatus::PendingUpdate
13119 )]
13120 status: OrderStatus,
13121 #[values(
13122 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13123 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13124 )]
13125 policy: crate::execution::submission::SubmissionRecoveryPolicy,
13126 ) {
13127 let (mut node, order, _) = venue_evidence_node(policy);
13128 let client_order_id = order.client_order_id();
13129 let account_id = AccountId::from("TEST-001");
13130 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13131 let pending = match status {
13132 OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
13133 OrderPendingCancelSpec::builder()
13134 .trader_id(order.trader_id())
13135 .strategy_id(order.strategy_id())
13136 .instrument_id(order.instrument_id())
13137 .client_order_id(client_order_id)
13138 .account_id(account_id)
13139 .build(),
13140 )),
13141 OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
13142 OrderPendingUpdateSpec::builder()
13143 .trader_id(order.trader_id())
13144 .strategy_id(order.strategy_id())
13145 .instrument_id(order.instrument_id())
13146 .client_order_id(client_order_id)
13147 .account_id(account_id)
13148 .build(),
13149 )),
13150 _ => None,
13151 };
13152
13153 if let Some(pending) = pending {
13154 node.process_exec_event(ExecutionEvent::Order(pending));
13155 }
13156 advance_clock(Duration::from_millis(101)).await;
13157 let first = node.exec_manager.check_inflight_orders();
13158 assert_eq!(first.queries.len(), 1);
13159 assert!(first.events.is_empty());
13160 advance_clock(Duration::from_millis(94)).await;
13161
13162 let event = if evidence == "updated" {
13163 ExecutionEvent::Order(OrderEventAny::Updated(
13164 OrderUpdatedSpec::builder()
13165 .trader_id(order.trader_id())
13166 .strategy_id(order.strategy_id())
13167 .instrument_id(order.instrument_id())
13168 .client_order_id(client_order_id)
13169 .account_id(account_id)
13170 .venue_order_id(venue_order_id)
13171 .quantity(order.quantity())
13172 .price(Price::from("101.00"))
13173 .build(),
13174 ))
13175 } else {
13176 let report = OrderStatusReport::new(
13177 account_id,
13178 order.instrument_id(),
13179 Some(client_order_id),
13180 venue_order_id,
13181 Some(order.order_side()),
13182 order.order_type(),
13183 order.time_in_force(),
13184 OrderStatus::Submitted,
13185 order.quantity(),
13186 Quantity::zero(3),
13187 UnixNanos::default(),
13188 UnixNanos::default(),
13189 UnixNanos::default(),
13190 None,
13191 )
13192 .with_price(Price::from("100.00"));
13193 ExecutionEvent::Report(if evidence == "order" {
13194 ExecutionReport::Order(Box::new(report))
13195 } else {
13196 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
13197 })
13198 };
13199 node.process_exec_event(event);
13200
13201 let expected_status = if evidence == "updated" && status != OrderStatus::PendingCancel {
13202 OrderStatus::Submitted
13203 } else if status == OrderStatus::Submitted {
13204 OrderStatus::Accepted
13205 } else {
13206 status
13207 };
13208 assert_eq!(
13209 node.kernel
13210 .cache
13211 .borrow()
13212 .order(&client_order_id)
13213 .unwrap()
13214 .status(),
13215 expected_status,
13216 );
13217
13218 for _ in 0..3 {
13219 advance_clock(Duration::from_millis(101)).await;
13220 let result = node.exec_manager.check_inflight_orders();
13221 assert!(result.queries.is_empty());
13222 assert!(result.events.is_empty());
13223 }
13224 assert_eq!(
13225 node.exec_manager.recon_check_retry_count(&client_order_id),
13226 0
13227 );
13228 assert!(node.exec_manager.check_open_order_queries().is_empty());
13229 advance_clock(Duration::from_secs(5)).await;
13230 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13231 }
13232
13233 #[rstest]
13234 #[case::fill("fill", false)]
13235 #[case::terminal_order("order", true)]
13236 #[case::terminal_order_with_fills("order_with_fills", true)]
13237 #[case::working_order_for_closed_cache("order", false)]
13238 #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
13239 #[cfg_attr(
13240 not(all(feature = "simulation", madsim)),
13241 tokio::test(start_paused = true)
13242 )]
13243 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13244 async fn test_report_cleanup_preserves_original_settling_predicate(
13245 #[case] report_kind: &str,
13246 #[case] terminal_report: bool,
13247 #[values(false, true)] explicit_client_id: bool,
13248 #[values(
13249 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13250 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13251 )]
13252 policy: crate::execution::submission::SubmissionRecoveryPolicy,
13253 ) {
13254 let (mut node, order, instrument) = venue_evidence_node(policy);
13255 let client_order_id = order.client_order_id();
13256 let account_id = AccountId::from("TEST-001");
13257 let client_id = ClientId::from("TEST-EVIDENCE");
13258 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13259 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
13260 &order,
13261 account_id,
13262 venue_order_id,
13263 )));
13264 let accepted = node
13265 .kernel
13266 .cache
13267 .borrow()
13268 .order_owned(&client_order_id)
13269 .unwrap();
13270 let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
13271 .last_qty(accepted.quantity())
13272 .last_px(Price::from("100.00"))
13273 .commission(Money::from("0 USDT"))
13274 .without_position_id()
13275 .build()
13276 else {
13277 unreachable!()
13278 };
13279 let mut fill_report = FillReport::new(
13280 account_id,
13281 instrument.id(),
13282 venue_order_id,
13283 fill.trade_id,
13284 fill.order_side,
13285 fill.last_qty,
13286 fill.last_px,
13287 fill.commission.unwrap(),
13288 fill.liquidity_side,
13289 Some(client_order_id),
13290 None,
13291 fill.ts_event,
13292 fill.ts_init,
13293 None,
13294 );
13295
13296 if !explicit_client_id {
13297 fill_report.client_order_id = None;
13298 }
13299 let event = if report_kind == "fill" {
13300 ExecutionReport::Fill(Box::new(fill_report))
13301 } else {
13302 if !terminal_report {
13303 node.kernel
13304 .cache
13305 .borrow_mut()
13306 .update_order(&OrderEventAny::Filled(fill))
13307 .unwrap();
13308 }
13309 let report = OrderStatusReport::new(
13310 account_id,
13311 instrument.id(),
13312 explicit_client_id.then_some(client_order_id),
13313 venue_order_id,
13314 Some(order.order_side()),
13315 order.order_type(),
13316 order.time_in_force(),
13317 if terminal_report {
13318 OrderStatus::Filled
13319 } else {
13320 OrderStatus::Accepted
13321 },
13322 order.quantity(),
13323 order.quantity(),
13324 UnixNanos::default(),
13325 UnixNanos::default(),
13326 UnixNanos::default(),
13327 None,
13328 )
13329 .with_price(Price::from("100.00"))
13330 .with_avg_px(dec!(100.00));
13331
13332 if report_kind == "order" {
13333 ExecutionReport::Order(Box::new(report))
13334 } else {
13335 ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
13336 }
13337 };
13338 node.process_exec_event(ExecutionEvent::Report(event));
13339 assert_eq!(
13340 node.kernel
13341 .cache
13342 .borrow()
13343 .order(&client_order_id)
13344 .unwrap()
13345 .status(),
13346 OrderStatus::Filled
13347 );
13348
13349 node.kernel
13351 .cache
13352 .borrow_mut()
13353 .add_order(order.clone(), None, Some(client_id), true)
13354 .unwrap();
13355 node.kernel
13356 .cache
13357 .borrow_mut()
13358 .update_order(&TestOrderEventStubs::submitted(&order, account_id))
13359 .unwrap();
13360 node.kernel
13361 .cache
13362 .borrow_mut()
13363 .update_order(&TestOrderEventStubs::accepted(
13364 &order,
13365 account_id,
13366 venue_order_id,
13367 ))
13368 .unwrap();
13369 let should_clear = terminal_report && explicit_client_id;
13370 assert_eq!(
13371 node.exec_manager.check_open_order_queries().len(),
13372 usize::from(should_clear)
13373 );
13374
13375 if !should_clear {
13376 advance_clock(Duration::from_secs(5)).await;
13377 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13378 }
13379 }
13380
13381 #[rstest]
13382 #[case(stub_order_event, 1)]
13383 #[case(stub_submitted_batch_event, 2)]
13384 #[case(stub_accepted_batch_event, 2)]
13385 #[case(stub_canceled_batch_event, 2)]
13386 fn test_pending_events_preserve_roots_when_splitting_batches(
13387 #[values(false, true)] rooted: bool,
13388 #[case] make_event: fn() -> ExecutionEvent,
13389 #[case] orders: usize,
13390 ) {
13391 let runner = AsyncRunner::new();
13392 runner.bind_senders();
13393 let mut channels = runner.take_channels();
13394
13395 let send = move || {
13396 get_data_event_sender().send(stub_data_event()).unwrap();
13397 get_exec_event_sender().send(stub_exec_event()).unwrap();
13398 get_exec_event_sender().send(make_event()).unwrap();
13399 };
13400
13401 if rooted {
13402 msgbus::register_trading_command_endpoint(
13403 MessagingSwitchboard::risk_engine_execute(),
13404 TypedIntoHandler::from(move |_| send()),
13405 );
13406 TradingCommandSender::execute(
13407 &SyncTradingCommandSender,
13408 TradingCommandMessage::new(
13409 MessagingSwitchboard::risk_engine_execute(),
13410 TradingCommand::QueryAccount(QueryAccount::new(
13411 "TRADER-001".into(),
13412 None,
13413 "SIM-001".into(),
13414 UUID4::new(),
13415 37.into(),
13416 None,
13417 None,
13418 )),
13419 ),
13420 );
13421 nautilus_common::runner::drain_trading_cmd_queue();
13422 } else {
13423 send();
13424 }
13425
13426 let mut pending = PendingEvents::default();
13427 pending
13428 .data_evts
13429 .push(channels.data_evt_rx.try_recv().unwrap());
13430 while let Ok(event) = channels.exec_evt_rx.try_recv() {
13431 pending.push_exec_event(event);
13432 }
13433
13434 assert_eq!(pending.data_evts.len(), 1);
13435 assert_eq!(pending.exec_reports.len(), 1);
13436 assert_eq!(pending.order_evts.len(), orders);
13437 assert_eq!(pending.data_evts[0].is_rooted(), rooted);
13438 assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
13439 assert!(
13440 pending
13441 .order_evts
13442 .iter()
13443 .all(|event| event.is_rooted() == rooted)
13444 );
13445
13446 let results = std::thread::spawn(move || {
13447 let mut results = Vec::new();
13448
13449 for event in pending.data_evts {
13450 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13451 || event.dispatch(drop),
13452 )));
13453 }
13454
13455 for event in pending.exec_reports {
13456 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13457 || event.dispatch(drop),
13458 )));
13459 }
13460
13461 for event in pending.order_evts {
13462 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13463 || event.dispatch(drop),
13464 )));
13465 }
13466
13467 results
13468 })
13469 .join()
13470 .unwrap();
13471
13472 assert_eq!(results.len(), orders + 2);
13473
13474 for result in results {
13475 if rooted {
13476 let error = result.unwrap_err();
13477 assert!(
13478 error
13479 .downcast_ref::<String>()
13480 .unwrap()
13481 .contains("command context dispatched outside its owner thread")
13482 );
13483 } else {
13484 result.unwrap();
13485 }
13486 }
13487 }
13488
13489 fn stub_accepted_batch_event() -> ExecutionEvent {
13490 ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
13491 OrderAcceptedSpec::builder()
13492 .client_order_id(ClientOrderId::from("O-017"))
13493 .build(),
13494 OrderAcceptedSpec::builder()
13495 .client_order_id(ClientOrderId::from("O-023"))
13496 .build(),
13497 ]))
13498 }
13499
13500 #[derive(Debug)]
13501 struct CapturedEgressMessage {
13502 topic: String,
13503 payload: Bytes,
13504 }
13505
13506 type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13507 type SharedClosed = Rc<Cell<bool>>;
13508
13509 #[derive(Debug)]
13510 struct CapturingExternalIngress {
13511 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13512 closed: SharedClosed,
13513 }
13514
13515 impl CapturingExternalIngress {
13516 fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13517 Self {
13518 rx: Some(rx),
13519 closed,
13520 }
13521 }
13522 }
13523
13524 impl MessageBusExternalIngress for CapturingExternalIngress {
13525 fn is_closed(&self) -> bool {
13526 self.closed.get()
13527 }
13528
13529 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13530 self.rx
13531 .take()
13532 .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13533 }
13534
13535 fn close(&mut self) {
13536 self.closed.set(true);
13537 }
13538 }
13539
13540 #[derive(Debug)]
13541 struct FailingExternalIngress {
13542 closed: SharedClosed,
13543 }
13544
13545 impl FailingExternalIngress {
13546 fn new(closed: SharedClosed) -> Self {
13547 Self { closed }
13548 }
13549 }
13550
13551 impl MessageBusExternalIngress for FailingExternalIngress {
13552 fn is_closed(&self) -> bool {
13553 self.closed.get()
13554 }
13555
13556 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13557 anyhow::bail!("external ingress receiver unavailable")
13558 }
13559
13560 fn close(&mut self) {
13561 self.closed.set(true);
13562 }
13563 }
13564
13565 struct CapturingExternalEgress {
13566 publications: CapturedEgressMessages,
13567 closed: SharedClosed,
13568 }
13569
13570 impl CapturingExternalEgress {
13571 fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13572 let publications = Rc::new(RefCell::new(Vec::new()));
13573 let closed = Rc::new(Cell::new(false));
13574 (
13575 Self {
13576 publications: publications.clone(),
13577 closed: closed.clone(),
13578 },
13579 publications,
13580 closed,
13581 )
13582 }
13583 }
13584
13585 impl MessageBusExternalEgress for CapturingExternalEgress {
13586 fn is_closed(&self) -> bool {
13587 self.closed.get()
13588 }
13589
13590 fn publish(&self, message: BusMessage) {
13591 self.publications.borrow_mut().push(CapturedEgressMessage {
13592 topic: message.topic.to_string(),
13593 payload: message.payload,
13594 });
13595 }
13596
13597 fn close(&mut self) {
13598 self.closed.set(true);
13599 }
13600 }
13601
13602 struct CapturingBackingFactory {
13603 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13604 closed: Arc<AtomicBool>,
13605 rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13606 }
13607
13608 impl CapturingBackingFactory {
13609 fn new(
13610 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13611 closed: Arc<AtomicBool>,
13612 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13613 ) -> Self {
13614 Self {
13615 publications,
13616 closed,
13617 rx: Mutex::new(rx),
13618 }
13619 }
13620 }
13621
13622 impl Debug for CapturingBackingFactory {
13623 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13624 f.debug_struct(stringify!(CapturingBackingFactory))
13625 .finish_non_exhaustive()
13626 }
13627 }
13628
13629 impl MessageBusBackingFactory for CapturingBackingFactory {
13630 fn create(
13631 &self,
13632 _trader_id: TraderId,
13633 _instance_id: UUID4,
13634 _config: MessageBusConfig,
13635 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13636 let rx = self.rx.lock().take();
13637 Ok(Box::new(CapturingBacking {
13638 publications: self.publications.clone(),
13639 closed: self.closed.clone(),
13640 rx,
13641 }))
13642 }
13643 }
13644
13645 struct CapturingBacking {
13646 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13647 closed: Arc<AtomicBool>,
13648 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13649 }
13650
13651 impl MessageBusBacking for CapturingBacking {
13652 fn is_closed(&self) -> bool {
13653 self.closed.load(Ordering::Relaxed)
13654 }
13655
13656 fn publish(&self, message: BusMessage) {
13657 self.publications.lock().push(CapturedEgressMessage {
13658 topic: message.topic.to_string(),
13659 payload: message.payload,
13660 });
13661 }
13662
13663 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13664 self.rx
13665 .take()
13666 .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13667 }
13668
13669 fn close(&mut self) {
13670 self.closed.store(true, Ordering::Relaxed);
13671 }
13672 }
13673}