1use std::{
86 any::Any,
87 cell::RefCell,
88 collections::VecDeque,
89 fmt::Debug,
90 rc::{Rc, Weak},
91 time::Duration,
92};
93
94use anyhow::Context;
95use nautilus_common::{
96 actor::{self, Actor, DataActor, DataActorNative},
97 cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
98 clients::ExecutionClient,
99 component::Component,
100 enums::{Environment, LogColor},
101 live::{dispatch::DispatchMessage, dst},
102 log_info,
103 messages::{
104 DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
105 data::DataCommand,
106 execution::TradingCommand,
107 system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
108 },
109 msgbus::{self, BusMessage, MessagingSwitchboard},
110 runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
111};
112use nautilus_core::{
113 UUID4,
114 datetime::{mins_to_secs, secs_to_nanos_unchecked},
115};
116use nautilus_execution::client::ExecutionClientAdapter;
117#[cfg(test)]
118use nautilus_model::reports::OrderStatusReport;
119use nautilus_model::{
120 events::OrderEventAny,
121 identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
122 orders::Order,
123};
124use nautilus_network::mode::ReconnectRequestOutcome;
125#[cfg(feature = "python")]
126use nautilus_system::trader::Trader;
127use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
128use nautilus_trading::{
129 ExecutionAlgorithm, ExecutionAlgorithmNative,
130 strategy::{Strategy, StrategyNative},
131};
132use tabled::{builder::Builder, settings::Style};
133
134use crate::{
135 dispatch::drain_callbacks,
136 execution::{
137 client::LiveExecutionClient,
138 manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
139 },
140 runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
141 socket::{SocketReconnectLookup, SocketReconnectRegistry},
142};
143
144pub mod builder;
145pub mod config;
146
147mod metrics;
148mod queue;
149mod reconciliation;
150mod state;
151
152#[cfg(feature = "plugin")]
153pub mod plugin;
154
155use builder::ExternalMessageBusIngress;
156pub use builder::LiveNodeBuilder;
157use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
158pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
159use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
160use queue::{QueueMonitor, QueueStateTransition};
161use reconciliation::{
162 OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
163 PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
164 ReportTaskOutcome, TargetedOrderReportTask,
165};
166use state::{EngineConnectionStatus, RunningTransition};
167pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
168
169const DISPATCHES_PER_YIELD: usize = 64;
175
176thread_local! {
177 static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
178}
179
180type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
181
182struct StreamProcessor(Box<StreamProcessorCallback>);
183
184impl Debug for StreamProcessor {
185 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186 f.debug_struct(stringify!(StreamProcessor)).finish()
187 }
188}
189
190#[derive(Debug)]
199pub struct LiveNode {
200 kernel: NautilusKernel,
201 runner: Option<AsyncRunner>,
202 config: LiveNodeConfig,
203 handle: LiveNodeHandle,
204 exec_manager: ExecutionManager,
205 exec_clients: Vec<LiveExecutionClient>,
206 socket_registry: SocketReconnectRegistry,
207 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
208 external_msgbus: Option<ExternalMessageBusIngress>,
209 stream_processors: Vec<StreamProcessor>,
210 shutdown_deadline: Option<dst::time::Instant>,
211 #[cfg(feature = "plugin")]
212 plugins: plugin::NodePlugins,
213 _thread_owner: Rc<()>,
214}
215
216impl LiveNode {
217 #[must_use]
221 #[allow(
222 clippy::too_many_arguments,
223 reason = "builder components have distinct lifecycle roles"
224 )]
225 pub(crate) fn new_from_builder(
226 kernel: NautilusKernel,
227 runner: AsyncRunner,
228 config: LiveNodeConfig,
229 handle: LiveNodeHandle,
230 mut exec_manager: ExecutionManager,
231 exec_clients: Vec<LiveExecutionClient>,
232 socket_registry: SocketReconnectRegistry,
233 cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
234 external_msgbus: Option<ExternalMessageBusIngress>,
235 thread_owner: Rc<()>,
236 ) -> Self {
237 for client in &exec_clients {
238 exec_manager.register_submission_retention(client);
239 }
240
241 Self {
242 kernel,
243 runner: Some(runner),
244 config,
245 handle,
246 exec_manager,
247 exec_clients,
248 socket_registry,
249 cache_database_factory,
250 external_msgbus,
251 stream_processors: Vec::new(),
252 shutdown_deadline: None,
253 #[cfg(feature = "plugin")]
254 plugins: plugin::NodePlugins,
255 _thread_owner: thread_owner,
256 }
257 }
258
259 pub fn builder(
265 trader_id: TraderId,
266 environment: Environment,
267 ) -> anyhow::Result<LiveNodeBuilder> {
268 LiveNodeBuilder::new(trader_id, environment)
269 }
270
271 pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
282 let thread_owner = Self::acquire_thread()?;
283 let config = config.unwrap_or_default();
284 validate_live_environment(config.environment())?;
285
286 config.validate_runtime_support()?;
287
288 if config.event_store.is_some() {
289 anyhow::bail!(
290 "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
291 use LiveNodeBuilder::with_event_store(...) instead"
292 );
293 }
294
295 let handle = LiveNodeHandle::new();
296 let runner = AsyncRunner::new();
297 runner.bind_senders_for_node(handle.clone());
298
299 let kernel = NautilusKernel::new(name, config.clone())?;
300 #[cfg(feature = "python")]
301 if let Some(controller) = config.controller.as_ref() {
302 Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
303 }
304
305 #[cfg(not(feature = "python"))]
306 if let Some(controller) = config.controller.as_ref() {
307 anyhow::bail!(
308 "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
309 controller.controller_path
310 );
311 }
312
313 let exec_manager_config =
314 ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
315
316 let exec_manager = ExecutionManager::new(
317 kernel.clock.clone(),
318 kernel.cache.clone(),
319 exec_manager_config,
320 )?;
321
322 let node = Self {
323 kernel,
324 runner: Some(runner),
325 config,
326 handle,
327 exec_manager,
328 exec_clients: Vec::new(),
329 socket_registry: SocketReconnectRegistry::default(),
330 cache_database_factory: None,
331 external_msgbus: None,
332 stream_processors: Vec::new(),
333 shutdown_deadline: None,
334 #[cfg(feature = "plugin")]
335 plugins: plugin::NodePlugins,
336 _thread_owner: thread_owner,
337 };
338
339 node.load_configured_plugins()?;
340
341 log::info!("LiveNode built successfully with kernel config");
342
343 Ok(node)
344 }
345
346 fn acquire_thread() -> anyhow::Result<Rc<()>> {
347 NODE_THREAD_OWNER.with(|slot| {
348 let mut owner = slot.borrow_mut();
349 anyhow::ensure!(
350 owner.upgrade().is_none(),
351 "A LiveNode already exists or is being built on this thread; drop it before building another"
352 );
353 let token = Rc::new(());
354 *owner = Rc::downgrade(&token);
355 Ok(token)
356 })
357 }
358
359 pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
365 if self.config.plugins.is_empty() {
366 return Ok(());
367 }
368
369 anyhow::bail!(
370 "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
371 )
372 }
373
374 #[expect(
380 clippy::needless_pass_by_value,
381 reason = "signature mirrors the host-enabled API"
382 )]
383 pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
384 #[cfg(feature = "plugin")]
385 config.validate_runtime_support(self.config.plugins.len())?;
386 #[cfg(not(feature = "plugin"))]
387 let _ = config;
388
389 anyhow::bail!(
390 "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
391 )
392 }
393
394 #[must_use]
396 pub fn handle(&self) -> LiveNodeHandle {
397 self.handle.clone()
398 }
399
400 pub fn add_stream_processor<F>(&mut self, callback: F)
417 where
418 F: Fn(&dyn Any) + 'static,
419 {
420 self.add_stream_processor_with_mapping(move |message, _| {
421 callback(message);
422 Ok(())
423 });
424 }
425
426 pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
427 where
428 F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
429 {
430 self.stream_processors
431 .push(StreamProcessor(Box::new(callback)));
432 }
433
434 pub async fn start(&mut self) -> anyhow::Result<()> {
445 if self.state().is_running() {
446 anyhow::bail!("Already running");
447 }
448
449 if self.external_msgbus.is_some() {
450 log::warn!(
451 "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
452 );
453 }
454
455 self.prepare_cache().await?;
456
457 if let Some(runner) = self.runner.as_ref() {
458 runner.bind_senders_for_node(self.handle.clone());
459 }
460
461 self.handle.set_starting();
462
463 self.kernel.reset_shutdown_flag();
464 self.kernel.start_async().await;
465
466 if self.kernel.is_event_store_replay() {
467 log::info!(
468 "Event-store replay loaded; skipping live client connection and reconciliation",
469 );
470
471 if !self.finish_startup_replay().await? {
472 return Ok(());
473 }
474
475 return Ok(());
476 }
477
478 if self.kernel.is_event_store_replay_configured() {
479 self.abort_startup("Event-store replay did not start")
480 .await?;
481 return Ok(());
482 }
483
484 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
485
486 if let Err(e) = self.connect_data_phase(connection_deadline).await {
488 return self
489 .abort_startup_with_error("Data client connection failed", e)
490 .await;
491 }
492
493 let (startup_system_events, startup_system_commands) =
494 if let Some(runner) = self.runner.as_mut() {
495 runner.flush_pending_data();
496 (
497 runner.drain_pending_system_events(),
498 runner.drain_pending_system_commands(),
499 )
500 } else {
501 (Vec::new(), Vec::new())
502 };
503
504 if let Err(e) = self.connect_exec_clients(connection_deadline).await {
505 return self
506 .abort_startup_with_error("Execution client connection failed", e)
507 .await;
508 }
509
510 if let Some(reason) = self.startup_abort_reason() {
511 self.abort_startup(reason).await?;
512 return Ok(());
513 }
514
515 match self.await_engines_connected(connection_deadline).await {
516 EngineConnectionStatus::Connected => {}
517 EngineConnectionStatus::TimedOut => {
518 return self
519 .abort_startup_with_error(
520 "Engine readiness timed out",
521 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
522 )
523 .await;
524 }
525 EngineConnectionStatus::StopRequested => {
526 self.abort_startup("Stop signal received during startup")
527 .await?;
528 return Ok(());
529 }
530 EngineConnectionStatus::ShutdownRequested => {
531 self.abort_startup("Shutdown signal received during startup")
532 .await?;
533 return Ok(());
534 }
535 }
536
537 if let Err(e) = self.perform_startup_reconciliation().await {
538 if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
539 anyhow::bail!(
540 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
541 );
542 }
543
544 return Err(e);
545 }
546
547 if let Some(reason) = self.startup_abort_reason() {
548 self.abort_startup(reason).await?;
549 return Ok(());
550 }
551
552 if let Err(e) = self.kernel.start_trader() {
553 return self.abort_after_trader_start_failure(e).await;
554 }
555
556 #[cfg(feature = "plugin")]
557 if let Err(e) = self.plugins.start_controllers() {
558 return self.abort_after_trader_start_failure(e).await;
559 }
560
561 self.process_system_events(startup_system_events);
562 self.process_system_commands(startup_system_commands);
563
564 if !self.finish_startup_trader(None).await? {
565 return Ok(());
566 }
567
568 Ok(())
569 }
570
571 pub async fn stop(&mut self) -> anyhow::Result<()> {
581 if !self.state().is_running() {
582 anyhow::bail!("Not running");
583 }
584
585 self.handle.set_shutting_down();
586
587 #[cfg(feature = "plugin")]
588 let controller_stop_result = self.plugins.stop_controllers();
589 #[cfg(not(feature = "plugin"))]
590 let controller_stop_result: anyhow::Result<()> = Ok(());
591
592 self.kernel.stop_trader();
593 let delay = self.kernel.delay_post_stop();
594 log::info!("Awaiting residual events ({delay:?})...");
595
596 let residual_events = self.process_runner_for(delay).await;
597 if residual_events > 0 {
598 log::debug!("Processed {residual_events} residual events during shutdown");
599 }
600
601 let stop_result = self.finalize_stop().await;
602 self.drain_runner_pending();
603
604 match (controller_stop_result, stop_result) {
605 (Ok(()), Ok(())) => Ok(()),
606 (Err(controller_err), Ok(())) => Err(controller_err),
607 (Ok(()), Err(stop_err)) => Err(stop_err),
608 (Err(controller_err), Err(stop_err)) => {
609 log::error!("Error stopping plug-in controllers: {controller_err}");
610 Err(stop_err)
611 }
612 }
613 }
614
615 pub fn dispose(&mut self) {
620 self.close_external_ingress();
621 self.handle.set_stopped();
622 self.kernel.dispose();
623 drop(self.runner.take());
624
625 if let Some(e) = actor::callback_failure() {
626 log::error!("Callback dispatch failed before disposal cleanup: {e}");
627 }
628
629 if let Err(e) = actor::clear_callbacks() {
630 log::error!("Failed to clear callback dispatch during disposal: {e}");
631 }
632 }
633
634 async fn process_runner_for(&mut self, duration: Duration) -> usize {
635 let Some(mut runner) = self.runner.take() else {
636 dst::time::sleep(duration).await;
637 return 0;
638 };
639
640 runner.bind_senders_for_node(self.handle.clone());
641 let deadline = dst::time::Instant::now() + duration;
642 let mut processed = 0;
643
644 loop {
645 tokio::select! {
646 biased;
647
648 () = dst::time::sleep_until(deadline) => break,
649 event = runner.recv() => {
650 let Some(event) = event else {
651 dst::time::sleep_until(deadline).await;
652 break;
653 };
654
655 self.process_runner_event(event);
656 processed += 1;
657 }
658 }
659 }
660
661 self.runner = Some(runner);
662 processed
663 }
664
665 fn drain_runner_pending(&mut self) -> usize {
666 let Some(mut runner) = self.runner.take() else {
667 return 0;
668 };
669
670 let processed = runner.poll_pending(|event| self.process_runner_event(event));
671 self.runner = Some(runner);
672 processed
673 }
674
675 fn process_runner_event(&mut self, event: PendingRunnerEvent) {
676 match event {
677 PendingRunnerEvent::TimeEvent(message) => {
678 let _ = AsyncRunner::handle_time_event(message);
679 }
680 PendingRunnerEvent::SystemEvent(event) => {
681 event.dispatch(|event| self.process_system_event(event));
682 }
683 PendingRunnerEvent::SystemCommand(command) => {
684 command.dispatch(|command| self.process_system_command(command));
685 }
686 PendingRunnerEvent::ExecEvent(event) => {
687 event.dispatch(|event| self.process_exec_event(event));
688 }
689 PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
690 PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
691 PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
692 }
693 }
694
695 fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
696 for event in events {
697 event.dispatch(|event| self.process_system_event(event));
698 }
699 }
700
701 fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
702 for command in commands {
703 command.dispatch(|command| self.process_system_command(command));
704 }
705 }
706
707 fn process_system_command(&self, command: SystemCommand) {
708 match command {
709 SystemCommand::ReconnectSocket(command) => {
710 self.process_socket_reconnect(command);
711 }
712 }
713 }
714
715 fn process_socket_reconnect(&self, command: ReconnectSocket) {
716 let outcome = if command.trader_id == self.config.trader_id {
717 Self::request_socket_reconnect(
718 self.socket_registry
719 .get(command.client_id, command.endpoint),
720 )
721 } else {
722 SocketReconnectDispatchOutcome::InvalidTrader
723 };
724
725 if outcome == SocketReconnectDispatchOutcome::Accepted {
726 log::info!(
727 "Requested socket reconnect for client {} endpoint {}",
728 command.client_id,
729 command.endpoint
730 );
731 } else {
732 log::warn!(
733 "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
734 command.client_id,
735 command.endpoint
736 );
737 }
738 }
739
740 fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
741 match lookup {
742 SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
743 ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
744 ReconnectRequestOutcome::AlreadyReconnecting => {
745 SocketReconnectDispatchOutcome::AlreadyReconnecting
746 }
747 ReconnectRequestOutcome::Disconnected => {
748 SocketReconnectDispatchOutcome::Disconnected
749 }
750 ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
751 ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
752 },
753 SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
754 SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
755 SocketReconnectLookup::EndpointNotFound => {
756 SocketReconnectDispatchOutcome::UnknownEndpoint
757 }
758 SocketReconnectLookup::AmbiguousEndpoint => {
759 SocketReconnectDispatchOutcome::AmbiguousEndpoint
760 }
761 }
762 }
763
764 fn process_system_event(&self, event: SystemEvent) {
765 match event {
766 SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
767 }
768 }
769
770 fn publish_socket_state_change(&self, change: SocketStateChange) {
771 let timestamp = self.kernel.generate_timestamp_ns();
772
773 let event = SocketStateChanged::new(
774 self.config.trader_id,
775 change.client_id,
776 change.venue,
777 change.endpoint,
778 change.state,
779 UUID4::new(),
780 timestamp,
781 timestamp,
782 );
783
784 msgbus::publish_any(
785 MessagingSwitchboard::socket_state_changed_topic(
786 event.client_id,
787 event.endpoint.as_str(),
788 ),
789 event.as_any(),
790 );
791 }
792
793 async fn await_engines_connected(
797 &self,
798 deadline: dst::time::Instant,
799 ) -> EngineConnectionStatus {
800 log::info!(
801 "Awaiting engine connections ({:?} timeout)...",
802 self.config.timeout_connection
803 );
804
805 let interval = Duration::from_millis(100);
806
807 loop {
808 if self.handle.should_stop() {
809 log::warn!("Stop signal received, aborting connection wait");
810 return EngineConnectionStatus::StopRequested;
811 }
812
813 if self.kernel.is_shutdown_requested() {
814 log::warn!("Shutdown signal received, aborting connection wait");
815 return EngineConnectionStatus::ShutdownRequested;
816 }
817
818 if self.kernel.check_engines_connected() {
819 log::info!("All engine clients connected");
820 return EngineConnectionStatus::Connected;
821 }
822
823 let now = dst::time::Instant::now();
824 if now >= deadline {
825 break;
826 }
827
828 dst::time::sleep(interval.min(deadline - now)).await;
829 }
830
831 self.log_connection_status();
832 EngineConnectionStatus::TimedOut
833 }
834
835 async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
839 log::info!(
840 "Awaiting engine disconnections ({:?} timeout)...",
841 self.config.timeout_disconnection
842 );
843
844 let timeout = self.config.timeout_disconnection;
845 let interval = Duration::from_millis(100);
846
847 loop {
848 if self.kernel.check_engines_disconnected() {
849 log::info!("All engine clients disconnected");
850 return Ok(());
851 }
852
853 let now = dst::time::Instant::now();
854 if now >= deadline {
855 break;
856 }
857
858 dst::time::sleep(interval.min(deadline - now)).await;
859 }
860
861 log::error!(
862 "Timed out ({:?}) waiting for engines to disconnect\n\
863 DataEngine.check_disconnected() == {}\n\
864 ExecEngine.check_disconnected() == {}",
865 timeout,
866 self.kernel.data_engine().check_disconnected(),
867 self.kernel.exec_engine().borrow().check_disconnected(),
868 );
869 anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
870 }
871
872 fn log_connection_status(&self) {
873 let data_status = self.kernel.data_client_connection_status();
874 let exec_status = self.kernel.exec_client_connection_status();
875
876 let mut rows: Vec<ClientStatus> = Vec::new();
877
878 for (client_id, connected) in data_status {
879 rows.push(ClientStatus {
880 client: client_id.to_string(),
881 client_type: "Data",
882 connected,
883 });
884 }
885
886 for (client_id, connected) in exec_status {
887 rows.push(ClientStatus {
888 client: client_id.to_string(),
889 client_type: "Execution",
890 connected,
891 });
892 }
893
894 let table = render_client_statuses(rows);
895
896 log::warn!(
897 "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
898 DataEngine.check_connected() == {}\n\
899 ExecEngine.check_connected() == {}",
900 self.config.timeout_connection,
901 self.kernel.data_engine().check_connected(),
902 self.kernel.exec_engine().borrow().check_connected(),
903 );
904 }
905
906 async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
915 if !self.config.exec_engine.reconciliation {
916 log::info!("Startup reconciliation disabled");
917 self.kernel
918 .portfolio
919 .borrow_mut()
920 .initialize_wallet_orders()?;
921 return Ok(());
922 }
923
924 log_info!(
925 "Starting execution state reconciliation...",
926 color = LogColor::Blue
927 );
928
929 let lookback_mins = self
930 .config
931 .exec_engine
932 .reconciliation_lookback_mins
933 .map(u64::from);
934
935 let timeout = self.config.timeout_reconciliation;
936 let start = dst::time::Instant::now();
937 let client_ids = self.kernel.exec_engine.borrow().client_ids();
938
939 for client_id in client_ids {
940 let elapsed = start.elapsed();
941 if elapsed >= timeout {
942 anyhow::bail!("Startup reconciliation timeout reached");
943 }
944
945 let remaining = timeout
946 .checked_sub(elapsed)
947 .expect("elapsed checked against reconciliation timeout");
948
949 log_info!(
950 "Requesting mass status from {}...",
951 client_id,
952 color = LogColor::Blue
953 );
954
955 let client = self
956 .exec_clients
957 .iter()
958 .find(|client| client.client_id() == client_id)
959 .ok_or_else(|| anyhow::anyhow!("Execution client {client_id} has no live handle"))?
960 .clone();
961 let client = Box::new(client);
962 let client = ExecutionClientAdapter::new(client);
963
964 let mass_status_query = client.generate_mass_status(lookback_mins);
965
966 let mass_status_result = dst::time::timeout(remaining, mass_status_query)
967 .await
968 .map_err(|_| {
969 anyhow::anyhow!(
970 "Startup reconciliation timeout reached while requesting mass status from {client_id}"
971 )
972 })?;
973
974 match mass_status_result {
975 Ok(Some(mass_status)) => {
976 log_info!(
977 "Reconciling ExecutionMassStatus for {}",
978 client_id,
979 color = LogColor::Blue
980 );
981
982 let result = self
983 .exec_manager
984 .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
985
986 anyhow::ensure!(
987 self.kernel
988 .exec_engine
989 .borrow()
990 .get_client(&client_id)
991 .is_some(),
992 "Execution client {client_id} disappeared during startup reconciliation",
993 );
994
995 anyhow::ensure!(
996 result.unresolved_positions.is_empty(),
997 "Unresolved positions during startup reconciliation for {client_id}: {}",
998 result.unresolved_positions.join("; "),
999 );
1000
1001 if result.events.is_empty() {
1002 log_info!(
1003 "Reconciliation for {} succeeded",
1004 client_id,
1005 color = LogColor::Blue
1006 );
1007 } else {
1008 log::info!(
1009 color = LogColor::Blue as u8;
1010 "Reconciliation for {} processed {} events",
1011 client_id,
1012 result.events.len()
1013 );
1014 }
1015
1016 if !result.external_orders.is_empty() {
1018 let exec_engine = self.kernel.exec_engine.borrow();
1019
1020 let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1021 anyhow::anyhow!(
1022 "Execution client {client_id} disappeared during startup reconciliation"
1023 )
1024 })?;
1025
1026 for external in result.external_orders {
1027 source_client.register_external_order(
1028 external.client_order_id,
1029 external.venue_order_id,
1030 external.instrument_id,
1031 external.strategy_id,
1032 external.ts_init,
1033 );
1034 }
1035 }
1036 }
1037 Ok(None) => {
1038 log::warn!(
1039 "No mass status available from {client_id} \
1040 (likely adapter error when generating reports)"
1041 );
1042 }
1043 Err(e) => {
1044 return Err(e).context(format!("Failed to get mass status from {client_id}"));
1045 }
1046 }
1047 }
1048
1049 self.kernel.portfolio.borrow_mut().initialize_orders();
1050 self.kernel.portfolio.borrow_mut().initialize_positions();
1051 self.kernel
1052 .portfolio
1053 .borrow_mut()
1054 .initialize_wallet_orders()?;
1055
1056 let elapsed_secs = start.elapsed().as_secs_f64();
1057 log_info!(
1058 "Startup reconciliation completed in {:.2}s",
1059 elapsed_secs,
1060 color = LogColor::Blue
1061 );
1062
1063 Ok(())
1064 }
1065
1066 pub async fn run(&mut self) -> anyhow::Result<()> {
1089 self.run_with_mode(NodeRunMode::Owned).await
1090 }
1091
1092 pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1103 if self.state().is_running() {
1104 anyhow::bail!("Already running");
1105 }
1106
1107 if self.runner.is_none() {
1108 anyhow::bail!("Runner already consumed - run() called twice");
1109 }
1110
1111 self.prepare_cache().await?;
1112
1113 let Some(runner) = self.runner.take() else {
1114 anyhow::bail!("Runner already consumed - run() called twice");
1115 };
1116
1117 runner.bind_senders_for_node(self.handle.clone());
1118
1119 let AsyncRunnerChannels {
1120 mut time_evt_rx,
1121 mut system_evt_rx,
1122 mut system_cmd_rx,
1123 mut exec_evt_rx,
1124 mut exec_cmd_rx,
1125 mut data_evt_rx,
1126 mut data_cmd_rx,
1127 } = runner.take_channels();
1128
1129 log::info!("Event loop starting");
1130
1131 self.handle.set_starting();
1132 self.kernel.reset_shutdown_flag();
1133 self.kernel.start_async().await;
1134
1135 if self.kernel.is_event_store_replay() {
1136 log::info!(
1137 "Event-store replay loaded; skipping live client connection and reconciliation",
1138 );
1139
1140 if !self.finish_startup_replay().await? {
1141 return Ok(());
1142 }
1143
1144 return Ok(());
1145 }
1146
1147 if self.kernel.is_event_store_replay_configured() {
1148 self.abort_startup("Event-store replay did not start")
1149 .await?;
1150 return Ok(());
1151 }
1152
1153 let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1154 Ok(rx) => rx,
1155 Err(e) => {
1156 let result = self
1157 .abort_startup("External message bus ingress failed to start")
1158 .await;
1159 Self::drain_channels(
1160 &mut time_evt_rx,
1161 &mut system_evt_rx,
1162 &mut system_cmd_rx,
1163 &mut exec_evt_rx,
1164 &mut exec_cmd_rx,
1165 &mut data_evt_rx,
1166 &mut data_cmd_rx,
1167 );
1168 log::info!("Event loop stopped");
1169
1170 if let Err(finalize_err) = result {
1171 anyhow::bail!(
1172 "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1173 );
1174 }
1175
1176 return Err(e);
1177 }
1178 };
1179
1180 let stop_handle = self.handle.clone();
1181 let mut pending = PendingEvents::default();
1182 let mut startup_system_events = Vec::new();
1183 let mut startup_system_commands = Vec::new();
1184 let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1185
1186 let mut pending_external = VecDeque::new();
1189 let data_connect_result = drive_with_event_buffering(
1190 drive_with_external_ingress(
1191 self.connect_data_phase(connection_deadline),
1192 &mut external_msgbus_rx,
1193 &mut pending_external,
1194 ),
1195 &mut pending,
1196 &mut time_evt_rx,
1197 &mut system_evt_rx,
1198 &mut system_cmd_rx,
1199 &mut exec_evt_rx,
1200 &mut exec_cmd_rx,
1201 &mut data_evt_rx,
1202 &mut data_cmd_rx,
1203 )
1204 .await;
1205
1206 self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1207
1208 if let Err(e) = data_connect_result {
1209 flush_all_pending(
1210 &mut pending,
1211 &mut time_evt_rx,
1212 &mut system_evt_rx,
1213 &mut system_cmd_rx,
1214 &mut exec_evt_rx,
1215 &mut exec_cmd_rx,
1216 &mut data_evt_rx,
1217 &mut data_cmd_rx,
1218 );
1219 let result = self
1220 .abort_startup_with_error("Data client connection failed", e)
1221 .await;
1222 Self::drain_channels(
1223 &mut time_evt_rx,
1224 &mut system_evt_rx,
1225 &mut system_cmd_rx,
1226 &mut exec_evt_rx,
1227 &mut exec_cmd_rx,
1228 &mut data_evt_rx,
1229 &mut data_cmd_rx,
1230 );
1231 log::info!("Event loop stopped");
1232 return result;
1233 }
1234
1235 flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1239 startup_system_events.extend(pending.take_system_events());
1240 startup_system_commands.extend(pending.take_system_commands());
1241 debug_assert!(
1242 pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1243 "data must be drained into cache before exec clients connect",
1244 );
1245
1246 let engine_connection_result = drive_with_event_buffering(
1248 self.connect_exec_phase(connection_deadline),
1249 &mut pending,
1250 &mut time_evt_rx,
1251 &mut system_evt_rx,
1252 &mut system_cmd_rx,
1253 &mut exec_evt_rx,
1254 &mut exec_cmd_rx,
1255 &mut data_evt_rx,
1256 &mut data_cmd_rx,
1257 )
1258 .await;
1259
1260 flush_all_pending(
1262 &mut pending,
1263 &mut time_evt_rx,
1264 &mut system_evt_rx,
1265 &mut system_cmd_rx,
1266 &mut exec_evt_rx,
1267 &mut exec_cmd_rx,
1268 &mut data_evt_rx,
1269 &mut data_cmd_rx,
1270 );
1271 startup_system_events.extend(pending.take_system_events());
1272 startup_system_commands.extend(pending.take_system_commands());
1273 debug_assert!(
1274 pending.is_empty(),
1275 "all startup events must be processed before reconciliation",
1276 );
1277
1278 let engine_connection_status = match engine_connection_result {
1279 Ok(status) => status,
1280 Err(e) => {
1281 let result = self
1282 .abort_startup_with_error("Execution client connection failed", e)
1283 .await;
1284 Self::drain_channels(
1285 &mut time_evt_rx,
1286 &mut system_evt_rx,
1287 &mut system_cmd_rx,
1288 &mut exec_evt_rx,
1289 &mut exec_cmd_rx,
1290 &mut data_evt_rx,
1291 &mut data_cmd_rx,
1292 );
1293 log::info!("Event loop stopped");
1294 return result;
1295 }
1296 };
1297
1298 if engine_connection_status == EngineConnectionStatus::TimedOut {
1299 let result = self
1300 .abort_startup_with_error(
1301 "Engine readiness timed out",
1302 anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1303 )
1304 .await;
1305 Self::drain_channels(
1306 &mut time_evt_rx,
1307 &mut system_evt_rx,
1308 &mut system_cmd_rx,
1309 &mut exec_evt_rx,
1310 &mut exec_cmd_rx,
1311 &mut data_evt_rx,
1312 &mut data_cmd_rx,
1313 );
1314 log::info!("Event loop stopped");
1315 return result;
1316 }
1317
1318 if let Some(reason) = engine_connection_status
1319 .abort_reason()
1320 .or_else(|| self.startup_abort_reason())
1321 {
1322 self.abort_startup(reason).await?;
1323 Self::drain_channels(
1324 &mut time_evt_rx,
1325 &mut system_evt_rx,
1326 &mut system_cmd_rx,
1327 &mut exec_evt_rx,
1328 &mut exec_cmd_rx,
1329 &mut data_evt_rx,
1330 &mut data_cmd_rx,
1331 );
1332 log::info!("Event loop stopped");
1333 return Ok(());
1334 }
1335
1336 debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1337
1338 if let Err(e) = self.perform_startup_reconciliation().await {
1340 let result = self.abort_startup("Startup reconciliation failed").await;
1341 Self::drain_channels(
1342 &mut time_evt_rx,
1343 &mut system_evt_rx,
1344 &mut system_cmd_rx,
1345 &mut exec_evt_rx,
1346 &mut exec_cmd_rx,
1347 &mut data_evt_rx,
1348 &mut data_cmd_rx,
1349 );
1350 log::info!("Event loop stopped");
1351
1352 if let Err(finalize_err) = result {
1353 anyhow::bail!(
1354 "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1355 );
1356 }
1357
1358 return Err(e);
1359 }
1360
1361 if let Some(reason) = self.startup_abort_reason() {
1362 let result = self.abort_startup(reason).await;
1363 Self::drain_channels(
1364 &mut time_evt_rx,
1365 &mut system_evt_rx,
1366 &mut system_cmd_rx,
1367 &mut exec_evt_rx,
1368 &mut exec_cmd_rx,
1369 &mut data_evt_rx,
1370 &mut data_cmd_rx,
1371 );
1372 log::info!("Event loop stopped");
1373 return result;
1374 }
1375
1376 if let Err(e) = self.kernel.start_trader() {
1377 let result = self.abort_after_trader_start_failure(e).await;
1378 Self::drain_channels(
1379 &mut time_evt_rx,
1380 &mut system_evt_rx,
1381 &mut system_cmd_rx,
1382 &mut exec_evt_rx,
1383 &mut exec_cmd_rx,
1384 &mut data_evt_rx,
1385 &mut data_cmd_rx,
1386 );
1387 log::info!("Event loop stopped");
1388 return result;
1389 }
1390
1391 #[cfg(feature = "plugin")]
1392 if let Err(e) = self.plugins.start_controllers() {
1393 let result = self.abort_after_trader_start_failure(e).await;
1394 Self::drain_channels(
1395 &mut time_evt_rx,
1396 &mut system_evt_rx,
1397 &mut system_cmd_rx,
1398 &mut exec_evt_rx,
1399 &mut exec_cmd_rx,
1400 &mut data_evt_rx,
1401 &mut data_cmd_rx,
1402 );
1403 log::info!("Event loop stopped");
1404 return result;
1405 }
1406
1407 self.process_system_events(startup_system_events);
1408 self.process_system_commands(startup_system_commands);
1409
1410 let finish_result = {
1411 let mut receivers = RunnerReceivers {
1412 time_evt: &mut time_evt_rx,
1413 system_evt: &mut system_evt_rx,
1414 system_cmd: &mut system_cmd_rx,
1415 data_evt: &mut data_evt_rx,
1416 data_cmd: &mut data_cmd_rx,
1417 exec_evt: &mut exec_evt_rx,
1418 exec_cmd: &mut exec_cmd_rx,
1419 };
1420
1421 self.finish_startup_trader(Some(&mut receivers)).await
1422 };
1423
1424 match finish_result {
1425 Ok(true) => {}
1426 result => {
1427 log::info!("Event loop stopped");
1428 return result.map(|_| ());
1429 }
1430 }
1431
1432 let exec_config = &self.config.exec_engine;
1433 let inflight_interval =
1434 Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1435
1436 let open_interval = exec_config
1437 .open_check_interval_secs
1438 .filter(|&s| s > 0.0)
1439 .map_or(Duration::ZERO, |secs| {
1440 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1441 });
1442
1443 let position_interval = exec_config
1444 .position_check_interval_secs
1445 .filter(|&s| s > 0.0)
1446 .map_or(Duration::ZERO, |secs| {
1447 Duration::from_nanos(secs_to_nanos_unchecked(secs))
1448 });
1449
1450 let has_clients = !self
1451 .kernel
1452 .exec_engine
1453 .borrow()
1454 .get_all_clients()
1455 .is_empty();
1456 let recon_enabled = has_clients
1457 && (!inflight_interval.is_zero()
1458 || !open_interval.is_zero()
1459 || !position_interval.is_zero());
1460
1461 let recon_min_interval = if recon_enabled {
1462 let mut intervals = Vec::new();
1463
1464 if !inflight_interval.is_zero() {
1465 intervals.push(inflight_interval);
1466 }
1467
1468 if !open_interval.is_zero() {
1469 intervals.push(open_interval);
1470 }
1471
1472 if !position_interval.is_zero() {
1473 intervals.push(position_interval);
1474 }
1475
1476 intervals
1477 .into_iter()
1478 .min()
1479 .unwrap_or(Duration::from_secs(1))
1480 } else {
1481 Duration::from_secs(1) };
1483
1484 let startup_delay = if self.config.exec_engine.reconciliation {
1489 Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1490 } else {
1491 Duration::ZERO
1492 };
1493
1494 let recon_start = dst::time::Instant::now() + startup_delay;
1495
1496 let mut last_inflight_check = dst::time::Instant::now();
1497 let mut last_open_check = last_inflight_check;
1498 let mut last_position_check = last_inflight_check;
1499
1500 let far_future = Duration::from_hours(24 * 365 * 100);
1503
1504 let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1505 let dur = opt_dur.unwrap_or(far_future);
1506 (dur, recon_start + dur)
1507 };
1508
1509 let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1510 Some(recon_min_interval)
1511 } else {
1512 None
1513 });
1514
1515 let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1516 exec_config
1517 .purge_closed_orders_interval_mins
1518 .filter(|&m| m > 0)
1519 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1520 );
1521
1522 let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1523 exec_config
1524 .purge_closed_positions_interval_mins
1525 .filter(|&m| m > 0)
1526 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1527 );
1528
1529 let (purge_account_interval, mut purge_account_next) = make_schedule(
1530 exec_config
1531 .purge_account_events_interval_mins
1532 .filter(|&m| m > 0)
1533 .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1534 );
1535
1536 let (own_books_interval, mut own_books_next) = make_schedule(
1537 exec_config
1538 .own_books_audit_interval_secs
1539 .filter(|&s| s > 0.0)
1540 .map(Duration::from_secs_f64),
1541 );
1542
1543 let (prune_fills_interval, mut prune_fills_next) =
1544 make_schedule(Some(Duration::from_mins(1)));
1545
1546 let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1547 maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1548
1549 let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1554 stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1555
1556 let mut residual_events = 0usize;
1558 let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1559 let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1560 let mut position_report_task: Option<PositionReportTask> = None;
1561
1562 let owns_signals = mode.owns_signals();
1565
1566 let ctrl_c = async move {
1567 if owns_signals {
1568 dst::signal::ctrl_c().await
1569 } else {
1570 std::future::pending::<std::io::Result<()>>().await
1571 }
1572 };
1573
1574 let terminate = async move {
1575 if owns_signals {
1576 dst::signal::terminate().await
1577 } else {
1578 std::future::pending::<std::io::Result<()>>().await
1579 }
1580 };
1581
1582 tokio::pin!(ctrl_c);
1583 tokio::pin!(terminate);
1584
1585 let metrics = self.handle.metrics.clone();
1586 let metrics_start = dst::time::Instant::now();
1587 metrics.reset();
1588
1589 let mut queue_monitor = self
1590 .config
1591 .queue_monitor
1592 .as_ref()
1593 .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1594 let mut dispatches_since_yield = 0usize;
1595 let mut skip_time = false;
1596 let mut skip_data = false;
1597
1598 let dispatch_result = loop {
1599 let callbacks_pending = match drain_callbacks().await {
1600 Ok(pending) => pending,
1601 Err(e) => {
1602 if self.state() == NodeState::Running {
1603 self.initiate_shutdown();
1604 }
1605
1606 log::warn!(
1607 "Skipping residual events and final buffered dispatch after callback failure"
1608 );
1609
1610 break Err(e);
1611 }
1612 };
1613
1614 let shutdown_deadline = self.shutdown_deadline;
1615 let is_shutting_down = self.state() == NodeState::ShuttingDown;
1616 let is_running = self.state() == NodeState::Running;
1617 let commands_empty = data_cmd_rx.is_empty();
1618 let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1619 let poll_data = !skip_data || commands_empty;
1620
1621 tokio::select! {
1622 biased;
1623
1624 result = &mut ctrl_c, if is_running => {
1626 match result {
1627 Ok(()) => log::info!("Received SIGINT, shutting down"),
1628 Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1629 }
1630 self.initiate_shutdown();
1631 }
1632 result = &mut terminate, if is_running => {
1633 match result {
1634 Ok(()) => log::info!("Received SIGTERM, shutting down"),
1635 Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1636 }
1637 self.initiate_shutdown();
1638 }
1639 _ = stop_check_timer.tick(), if is_running => {
1640 if stop_handle.should_stop() {
1641 log::info!("Received stop signal from handle");
1642 self.initiate_shutdown();
1643 } else if self.kernel.is_shutdown_requested() {
1644 log::info!("Received ShutdownSystem command, shutting down");
1645 self.initiate_shutdown();
1646 }
1647 }
1648 () = async {
1649 match shutdown_deadline {
1650 Some(deadline) => dst::time::sleep_until(deadline).await,
1651 None => std::future::pending::<()>().await,
1652 }
1653 }, if self.state() == NodeState::ShuttingDown => {
1654 break Ok(());
1655 }
1656 () = std::future::ready(()), if callbacks_pending => {},
1657 result = async {
1658 match open_order_report_task.as_mut() {
1659 Some(task) => task.future.as_mut().await,
1660 None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1661 }
1662 }, if open_order_report_task.is_some() => {
1663 let maintenance_start = dst::time::Instant::now();
1664
1665 drop(open_order_report_task.take());
1666
1667 match result {
1668 ReportTaskOutcome::Completed(result) => {
1669 let result = result.finish();
1670 self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1671 let client_refs = self
1672 .exec_clients
1673 .iter()
1674 .map(|client| client as &dyn ExecutionClient)
1675 .collect::<Vec<_>>();
1676 let reconciliation = self.exec_manager.reconcile_open_order_reports(
1677 &result.check,
1678 result.reports,
1679 &result.queried_clients,
1680 &result.failed_clients,
1681 &client_refs,
1682 );
1683 self.process_reconciliation_events(&reconciliation.events);
1684 if !reconciliation.targeted_queries.is_empty() {
1685 if is_shutting_down {
1686 let planned_client_order_ids = reconciliation
1687 .targeted_queries
1688 .iter()
1689 .map(TargetedOrderQuery::client_order_id)
1690 .collect::<Vec<_>>();
1691 self.cleanup_cancelled_report_tasks(
1692 &planned_client_order_ids,
1693 );
1694 } else {
1695 targeted_order_report_task = Some(
1696 self.start_targeted_order_report_check(
1697 reconciliation.targeted_queries,
1698 ),
1699 );
1700 }
1701 }
1702 }
1703 ReportTaskOutcome::TimedOut => {
1704 self.cleanup_cancelled_report_tasks(&[]);
1705 log::warn!(
1706 "Open-order report collection expired after {:?}",
1707 self.config.timeout_reconciliation,
1708 );
1709 }
1710 }
1711 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1712 }
1713 result = async {
1714 match targeted_order_report_task.as_mut() {
1715 Some(task) => task.future.as_mut().await,
1716 None => std::future::pending::<ReportTaskOutcome<()>>().await,
1717 }
1718 }, if targeted_order_report_task.is_some() => {
1719 let maintenance_start = dst::time::Instant::now();
1720
1721 if let Some(task) = targeted_order_report_task.take() {
1722 self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1723 self.complete_targeted_order_report_check(task, &result);
1724 }
1725 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1726 }
1727 result = async {
1728 match position_report_task.as_mut() {
1729 Some(task) => task.future.as_mut().await,
1730 None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1731 }
1732 }, if position_report_task.is_some() => {
1733 let maintenance_start = dst::time::Instant::now();
1734
1735 drop(position_report_task.take());
1736
1737 match result {
1738 ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1739 if is_shutting_down {
1740 self.cleanup_cancelled_report_tasks(&[]);
1741 } else {
1742 let result = result.finish();
1743 position_report_task = self.handle_position_report_result(result);
1744 }
1745 }
1746 ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1747 let reports = collection.finish();
1748 self.handle_position_fill_report_result(PositionFillReportResult {
1749 position_result,
1750 reports,
1751 });
1752 }
1753 ReportTaskOutcome::TimedOut => {
1754 self.cleanup_cancelled_report_tasks(&[]);
1755 log::warn!(
1756 "Position report collection expired after {:?}",
1757 self.config.timeout_reconciliation,
1758 );
1759 }
1760 }
1761 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1762 }
1763
1764 _ = maintenance_timer.tick(), if is_running => {
1767 let maintenance_start = dst::time::Instant::now();
1768 metrics.publish_queue_depths(
1769 RunnerChannelQueueDepths::from_receivers(
1770 &time_evt_rx,
1771 &exec_evt_rx,
1772 &exec_cmd_rx,
1773 &data_evt_rx,
1774 &data_cmd_rx,
1775 ),
1776 metrics_start.elapsed(),
1777 );
1778
1779 if let Some(queue_monitor) = queue_monitor.as_mut() {
1780 let transitions = queue_monitor.evaluate(metrics.snapshot());
1781 self.publish_queue_state_transitions(&transitions);
1782 }
1783
1784 let mut now = dst::time::Instant::now();
1785
1786 if recon_enabled && now >= recon_next {
1787 let recon_intervals = ReconciliationCheckIntervals {
1788 inflight: inflight_interval,
1789 open: open_interval,
1790 position: position_interval,
1791 };
1792 let mut recon_state = ReconciliationCheckState {
1793 last_inflight_check: &mut last_inflight_check,
1794 last_open_check: &mut last_open_check,
1795 last_position_check: &mut last_position_check,
1796 open_order_report_task: &mut open_order_report_task,
1797 targeted_order_report_task: &mut targeted_order_report_task,
1798 position_report_task: &mut position_report_task,
1799 };
1800
1801 self.run_reconciliation_checks(
1802 now,
1803 recon_intervals,
1804 &mut recon_state,
1805 );
1806
1807 now = dst::time::Instant::now();
1808 recon_next = now + recon_interval;
1809 }
1810
1811 if now >= purge_orders_next {
1812 self.exec_manager.purge_closed_orders();
1813 purge_orders_next = now + purge_orders_interval;
1814 }
1815
1816 if now >= purge_positions_next {
1817 self.exec_manager.purge_closed_positions();
1818 purge_positions_next = now + purge_positions_interval;
1819 }
1820
1821 if now >= purge_account_next {
1822 self.exec_manager.purge_account_events();
1823 purge_account_next = now + purge_account_interval;
1824 }
1825
1826 if now >= own_books_next {
1827 self.kernel.cache().borrow_mut().audit_own_order_books();
1828 own_books_next = now + own_books_interval;
1829 }
1830
1831 if now >= prune_fills_next {
1832 self.exec_manager.prune_recent_fills_cache(60.0);
1833 self.exec_manager.prune_processed_fills();
1834 self.exec_manager.prune_order_local_activity();
1835 prune_fills_next = now + prune_fills_interval;
1836 }
1837
1838 record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1839 }
1840
1841 Some(handler) = time_evt_rx.recv(), if poll_time => {
1846 skip_time = true;
1847 let dispatch_start = dst::time::Instant::now();
1848 let dispatched = AsyncRunner::handle_time_event(handler);
1849
1850 if dispatched && is_shutting_down {
1851 log::debug!("Residual time event");
1852 residual_events += 1;
1853 }
1854
1855 if dispatched {
1856 record_runner_dispatch(
1857 &metrics,
1858 SystemChannel::TimeEvents,
1859 dispatch_start,
1860 metrics_start,
1861 );
1862 }
1863 }
1864 Some(event) = system_evt_rx.recv() => {
1865 skip_time = false;
1866
1867 if is_shutting_down {
1868 log::debug!("Residual system event: {event}");
1869 residual_events += 1;
1870 }
1871 event.dispatch(|event| self.process_system_event(event));
1872 }
1873 Some(command) = system_cmd_rx.recv() => {
1874 skip_time = false;
1875
1876 if is_shutting_down {
1877 log::debug!("Residual system command: {command}");
1878 residual_events += 1;
1879 }
1880 command.dispatch(|command| self.process_system_command(command));
1881 }
1882 Some(evt) = exec_evt_rx.recv() => {
1883 skip_time = false;
1884 let dispatch_start = dst::time::Instant::now();
1885
1886 if is_shutting_down {
1887 log::debug!("Residual exec event: {evt}");
1888 residual_events += 1;
1889 }
1890
1891 evt.dispatch(|evt| self.process_exec_event(evt));
1892 record_runner_dispatch(
1893 &metrics,
1894 SystemChannel::ExecEvents,
1895 dispatch_start,
1896 metrics_start,
1897 );
1898 }
1899 Some(cmd) = exec_cmd_rx.recv() => {
1900 skip_time = false;
1901 let dispatch_start = dst::time::Instant::now();
1902
1903 if is_shutting_down {
1904 log::debug!("Residual exec command: {cmd}");
1905 residual_events += 1;
1906 }
1907
1908 self.process_exec_command(cmd);
1909 record_runner_dispatch(
1910 &metrics,
1911 SystemChannel::ExecCommands,
1912 dispatch_start,
1913 metrics_start,
1914 );
1915 }
1916 message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1917 skip_time = false;
1918 let external_msgbus_start = dst::time::Instant::now();
1919
1920 match message {
1921 Some(message) => {
1922 if is_shutting_down {
1923 log::debug!("Residual external message bus message: {message}");
1924 residual_events += 1;
1925 }
1926 self.process_external_msgbus_message(&message);
1927 }
1928 None => {
1929 log::info!("External message bus ingress closed");
1930 external_msgbus_rx = None;
1931 self.close_external_ingress();
1932 }
1933 }
1934
1935 record_runner_external_msgbus(
1936 &metrics,
1937 external_msgbus_start,
1938 metrics_start,
1939 );
1940 }
1941 Some(evt) = data_evt_rx.recv(), if poll_data => {
1942 skip_time = false;
1943 skip_data = true;
1944 let dispatch_start = dst::time::Instant::now();
1945
1946 if is_shutting_down {
1947 log::debug!("Residual data event: {evt:?}");
1948 residual_events += 1;
1949 }
1950 AsyncRunner::dispatch_data_event(evt);
1951 record_runner_dispatch(
1952 &metrics,
1953 SystemChannel::DataEvents,
1954 dispatch_start,
1955 metrics_start,
1956 );
1957 }
1958 Some(cmd) = data_cmd_rx.recv() => {
1959 skip_time = false;
1960 skip_data = false;
1961 let dispatch_start = dst::time::Instant::now();
1962
1963 if is_shutting_down {
1964 log::debug!("Residual data command: {cmd:?}");
1965 residual_events += 1;
1966 }
1967 AsyncRunner::handle_data_command(cmd);
1968 record_runner_dispatch(
1969 &metrics,
1970 SystemChannel::DataCommands,
1971 dispatch_start,
1972 metrics_start,
1973 );
1974 }
1975 }
1976
1977 dispatches_since_yield += 1;
1978 if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1979 dispatches_since_yield = 0;
1980 dst::task::yield_now().await;
1981 }
1982 };
1983
1984 if residual_events > 0 {
1985 log::debug!("Processed {residual_events} residual events during shutdown");
1986 }
1987
1988 self.cancel_report_tasks(
1989 &mut open_order_report_task,
1990 &mut targeted_order_report_task,
1991 &mut position_report_task,
1992 dispatch_result.is_ok(),
1993 );
1994
1995 drop(external_msgbus_rx.take());
1996 let _ = self.kernel.cache().borrow().check_residuals();
1997
1998 let stop_result = self.finalize_stop().await;
1999
2000 if let Err(e) = dispatch_result {
2001 if let Err(stop_err) = stop_result {
2002 log::error!("Failed to finalize node after callback failure: {stop_err}");
2003 let diagnostic = format!("{e}; {stop_err}");
2005 return Err(anyhow::Error::new(e).context(diagnostic));
2006 }
2007
2008 return Err(e.into());
2009 }
2010
2011 Self::drain_channels(
2013 &mut time_evt_rx,
2014 &mut system_evt_rx,
2015 &mut system_cmd_rx,
2016 &mut exec_evt_rx,
2017 &mut exec_cmd_rx,
2018 &mut data_evt_rx,
2019 &mut data_cmd_rx,
2020 );
2021
2022 log::info!("Event loop stopped");
2023
2024 stop_result
2025 }
2026
2027 fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2028 for transition in transitions {
2029 let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2030 let timestamp = self.kernel.generate_timestamp_ns();
2031
2032 let event = QueueStateChanged::new(
2033 self.config.trader_id,
2034 transition.channel,
2035 transition.condition,
2036 transition.state,
2037 transition.queue_depth,
2038 transition.mean_dispatch_ns,
2039 UUID4::new(),
2040 timestamp,
2041 timestamp,
2042 );
2043
2044 msgbus::publish_any(topic, event.as_any());
2045 }
2046 }
2047
2048 #[expect(
2049 clippy::await_holding_refcell_ref,
2050 reason = "cache loading is serialized before the single-threaded live node starts"
2051 )]
2052 async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2053 self.install_cache_database().await?;
2054
2055 let cache = self.kernel.cache();
2056 if !cache.borrow().has_backing() {
2057 return Ok(());
2058 }
2059
2060 if self
2061 .config
2062 .cache
2063 .as_ref()
2064 .is_some_and(|config| config.flush_on_start)
2065 {
2066 cache
2067 .borrow_mut()
2068 .flush_db()
2069 .context("Failed to flush persistent cache")?;
2070 return Ok(());
2071 }
2072
2073 if self.config.exec_engine.load_cache {
2074 self.kernel
2075 .exec_engine()
2076 .borrow_mut()
2077 .load_cache()
2078 .await
2079 .context("Failed to load persistent cache")?;
2080 }
2081
2082 Ok(())
2083 }
2084
2085 #[must_use]
2087 pub const fn has_pending_cache_database(&self) -> bool {
2088 self.cache_database_factory.is_some()
2089 }
2090
2091 async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2096 let Some(factory) = self.cache_database_factory.as_ref() else {
2097 return Ok(());
2098 };
2099
2100 let config = self.config.cache.clone().unwrap_or_default();
2101
2102 let database = factory
2105 .create(self.config.trader_id, self.kernel.instance_id, config)
2106 .await
2107 .context("failed to create cache database backing")?;
2108 self.kernel.cache().borrow_mut().set_database(database);
2109 self.cache_database_factory = None;
2110
2111 Ok(())
2112 }
2113
2114 fn take_external_ingress_receiver(
2115 &mut self,
2116 ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2117 let Some(external_ingress) = self.external_msgbus.as_mut() else {
2118 return Ok(None);
2119 };
2120
2121 let receiver = external_ingress.take_receiver()?;
2122 log::info!("External message bus ingress started");
2123 Ok(Some(receiver))
2124 }
2125
2126 fn republish_external_msgbus_message(message: &BusMessage) {
2127 if let Err(e) = msgbus::republish_external_message(message) {
2128 log::error!(
2129 "Failed to republish external message bus topic '{}': {e:#}",
2130 message.topic
2131 );
2132 }
2133 }
2134
2135 fn process_external_msgbus_message(&self, message: &BusMessage) {
2136 if self.stream_processors.is_empty() {
2137 Self::republish_external_msgbus_message(message);
2138 return;
2139 }
2140
2141 let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2142 for processor in &self.stream_processors {
2143 (processor.0)(value, mapping)?;
2144 }
2145
2146 Ok(())
2147 };
2148
2149 if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2150 log::error!(
2151 "Failed to process external message bus topic '{}': {e:#}",
2152 message.topic
2153 );
2154 }
2155 }
2156
2157 fn finish_external_ingress_connection(
2158 &mut self,
2159 receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2160 ) {
2161 if receiver.is_none() {
2162 self.close_external_ingress();
2163 }
2164 }
2165
2166 fn close_external_ingress(&mut self) {
2167 if let Some(external_ingress) = self.external_msgbus.as_mut()
2168 && !external_ingress.is_closed()
2169 {
2170 external_ingress.close();
2171 }
2172 }
2173
2174 fn drain_exec_events_before_reconciliation(
2175 &mut self,
2176 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2177 ) {
2178 while let Ok(event) = exec_evt_rx.try_recv() {
2179 event.dispatch(|event| self.process_exec_event(event));
2180 }
2181 }
2182
2183 fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2184 if events.is_empty() {
2185 self.publish_submission_recovery_exhaustions();
2186 return;
2187 }
2188
2189 log::info!(
2190 "Processing {} reconciliation event{}",
2191 events.len(),
2192 if events.len() == 1 { "" } else { "s" }
2193 );
2194
2195 for event in events {
2196 self.exec_manager
2197 .record_local_activity(event.client_order_id());
2198 if let OrderEventAny::Filled(fill) = event {
2199 self.exec_manager
2200 .record_position_activity(fill.instrument_id, fill.account_id);
2201 }
2202
2203 self.kernel.exec_engine.borrow_mut().process(event);
2204 self.exec_manager
2205 .confirm_submission_outcome(&event.client_order_id());
2206 if let OrderEventAny::Filled(fill) = event {
2207 self.exec_manager.commit_recent_fill_if_applied(fill);
2208 }
2209 }
2210
2211 self.publish_submission_recovery_exhaustions();
2212 }
2213
2214 fn publish_submission_recovery_exhaustions(&mut self) {
2215 for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2216 log::warn!(
2217 "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2218 diagnostic.client_order_id,
2219 diagnostic.source,
2220 diagnostic.retry_count,
2221 );
2222 msgbus::publish_any(
2223 MessagingSwitchboard::submission_recovery_exhausted_topic(),
2224 &diagnostic,
2225 );
2226 }
2227 }
2228
2229 fn process_exec_event(&mut self, event: ExecutionEvent) {
2230 let clear_closed_orders = matches!(
2231 &event,
2232 ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2233 );
2234 let closed_report_id = match &event {
2235 ExecutionEvent::Report(
2236 ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2237 ) if report.order_status.is_closed() => report.client_order_id,
2238 _ => None,
2239 };
2240 let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2241 return;
2242 };
2243
2244 self.dispatch_exec_event_and_commit_fill(event);
2245
2246 for client_order_id in &order_ids {
2247 self.exec_manager
2248 .confirm_submission_outcome(client_order_id);
2249 let is_closed = self
2250 .kernel
2251 .cache()
2252 .borrow()
2253 .order(client_order_id)
2254 .is_some_and(|order| order.is_closed());
2255
2256 if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2257 self.exec_manager
2258 .clear_recon_tracking(client_order_id, true);
2259 }
2260 }
2261 }
2262
2263 fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2264 message.dispatch_trading(|message| {
2265 if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2266 self.observe_exec_command_before_dispatch(message.command());
2267 }
2268 });
2269 }
2270
2271 fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2280 let recent_fill_candidate = match &evt {
2281 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2282 _ => None,
2283 };
2284
2285 AsyncRunner::handle_exec_event(evt);
2286
2287 if let Some(fill) = &recent_fill_candidate {
2288 self.exec_manager.commit_recent_fill_if_applied(fill);
2289 }
2290 }
2291
2292 async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2293 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2299 dst::time::timeout(remaining, self.kernel.connect_data_clients())
2300 .await
2301 .map_err(|_| anyhow::anyhow!("data-connect timeout"))?
2302 }
2303
2304 async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2305 let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2306 dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2307 .await
2308 .map_err(|_| anyhow::anyhow!("exec-connect timeout"))?
2309 }
2310
2311 async fn connect_exec_phase(
2316 &mut self,
2317 deadline: dst::time::Instant,
2318 ) -> anyhow::Result<EngineConnectionStatus> {
2319 self.connect_exec_clients(deadline).await?;
2320 Ok(self.await_engines_connected(deadline).await)
2321 }
2322
2323 fn startup_abort_reason(&self) -> Option<&'static str> {
2324 if self.handle.should_stop() {
2325 Some("Stop signal received during startup")
2326 } else if self.kernel.is_shutdown_requested() {
2327 Some("Shutdown signal received during startup")
2328 } else {
2329 None
2330 }
2331 }
2332
2333 async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2334 match self.handle.try_set_running() {
2335 RunningTransition::Entered => Ok(true),
2336 RunningTransition::StopRequested => {
2337 self.abort_startup("Stop signal received during startup")
2338 .await?;
2339 Ok(false)
2340 }
2341 RunningTransition::Invalid(control) => {
2342 self.abort_startup_with_error(
2343 "Invalid lifecycle state during startup",
2344 anyhow::anyhow!(
2345 "Invalid LiveNode control state {control:#04x} while entering Running"
2346 ),
2347 )
2348 .await?;
2349 Ok(false)
2350 }
2351 }
2352 }
2353
2354 async fn finish_startup_trader(
2355 &mut self,
2356 receivers: Option<&mut RunnerReceivers<'_>>,
2357 ) -> anyhow::Result<bool> {
2358 match self.handle.try_set_running() {
2359 RunningTransition::Entered => Ok(true),
2360 RunningTransition::StopRequested => {
2361 self.abort_started_trader("Stop signal received during startup", receivers)
2362 .await?;
2363 Ok(false)
2364 }
2365 RunningTransition::Invalid(control) => {
2366 let state_err = anyhow::anyhow!(
2367 "Invalid LiveNode control state {control:#04x} while entering Running"
2368 );
2369
2370 match self
2371 .abort_started_trader("Invalid lifecycle state during startup", receivers)
2372 .await
2373 {
2374 Ok(()) => Err(state_err),
2375 Err(finalize_err) => {
2376 anyhow::bail!(
2377 "{state_err}; failed to finalize startup abort: {finalize_err}"
2378 )
2379 }
2380 }
2381 }
2382 }
2383 }
2384
2385 async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2386 log::info!("{reason}, aborting startup");
2387 self.handle.set_shutting_down();
2388 self.finalize_stop().await
2389 }
2390
2391 async fn abort_startup_with_error(
2392 &mut self,
2393 reason: &str,
2394 startup_err: anyhow::Error,
2395 ) -> anyhow::Result<()> {
2396 match self.abort_startup(reason).await {
2397 Ok(()) => Err(startup_err),
2398 Err(finalize_err) => {
2399 anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2400 }
2401 }
2402 }
2403
2404 async fn abort_started_trader(
2405 &mut self,
2406 reason: &str,
2407 mut receivers: Option<&mut RunnerReceivers<'_>>,
2408 ) -> anyhow::Result<()> {
2409 log::info!("{reason}, aborting startup");
2410 self.handle.set_shutting_down();
2411
2412 #[cfg(feature = "plugin")]
2413 let controller_stop_result = self.plugins.stop_controllers();
2414 #[cfg(not(feature = "plugin"))]
2415 let controller_stop_result: anyhow::Result<()> = Ok(());
2416
2417 let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2418 let delay = self.kernel.delay_post_stop();
2419 log::info!("Awaiting residual events ({delay:?})...");
2420
2421 let residual_events = match receivers.as_mut() {
2422 Some(receivers) => self.process_receivers_for(delay, receivers).await,
2423 None => self.process_runner_for(delay).await,
2424 };
2425
2426 if residual_events > 0 {
2427 log::debug!("Processed {residual_events} residual events during shutdown");
2428 }
2429
2430 let finalize_result = self.finalize_stop().await;
2431
2432 if let Some(receivers) = receivers {
2433 Self::drain_channels(
2434 receivers.time_evt,
2435 receivers.system_evt,
2436 receivers.system_cmd,
2437 receivers.exec_evt,
2438 receivers.exec_cmd,
2439 receivers.data_evt,
2440 receivers.data_cmd,
2441 );
2442 } else {
2443 self.drain_runner_pending();
2444 }
2445
2446 let mut errors = Vec::new();
2447
2448 if let Err(e) = controller_stop_result {
2449 errors.push(format!("Failed to stop plug-in controllers: {e}"));
2450 }
2451
2452 if let Err(e) = trader_stop_result {
2453 errors.push(format!("Failed to stop trader: {e}"));
2454 }
2455
2456 if let Err(e) = finalize_result {
2457 errors.push(format!("Failed to finalize startup abort: {e}"));
2458 }
2459
2460 if errors.is_empty() {
2461 Ok(())
2462 } else {
2463 anyhow::bail!("{}", errors.join("; "))
2464 }
2465 }
2466
2467 async fn process_receivers_for(
2468 &mut self,
2469 duration: Duration,
2470 receivers: &mut RunnerReceivers<'_>,
2471 ) -> usize {
2472 let deadline = dst::time::Instant::now() + duration;
2473 let mut processed = 0;
2474
2475 loop {
2476 tokio::select! {
2477 biased;
2478
2479 () = dst::time::sleep_until(deadline) => break,
2480 Some(message) = receivers.time_evt.recv() => {
2481 let _ = AsyncRunner::handle_time_event(message);
2482 processed += 1;
2483 }
2484 Some(event) = receivers.system_evt.recv() => {
2485 event.dispatch(|event| self.process_system_event(event));
2486 processed += 1;
2487 }
2488 Some(command) = receivers.system_cmd.recv() => {
2489 command.dispatch(|command| self.process_system_command(command));
2490 processed += 1;
2491 }
2492 Some(event) = receivers.exec_evt.recv() => {
2493 event.dispatch(|event| self.process_exec_event(event));
2494 processed += 1;
2495 }
2496 Some(command) = receivers.exec_cmd.recv() => {
2497 self.process_exec_command(command);
2498 processed += 1;
2499 }
2500 Some(event) = receivers.data_evt.recv() => {
2501 AsyncRunner::dispatch_data_event(event);
2502 processed += 1;
2503 }
2504 Some(command) = receivers.data_cmd.recv() => {
2505 AsyncRunner::handle_data_command(command);
2506 processed += 1;
2507 }
2508 }
2509 }
2510
2511 processed
2512 }
2513
2514 async fn abort_after_trader_start_failure(
2515 &mut self,
2516 start_err: anyhow::Error,
2517 ) -> anyhow::Result<()> {
2518 log::info!("Trader startup failed, aborting startup");
2519 self.handle.set_shutting_down();
2520 let stop_result = self.kernel.stop_trader_after_start_failure();
2521 let finalize_result = self.finalize_stop().await;
2522
2523 match (stop_result, finalize_result) {
2524 (Ok(()), Ok(())) => Err(start_err),
2525 (Err(stop_err), Ok(())) => anyhow::bail!(
2526 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2527 {stop_err}"
2528 ),
2529 (Ok(()), Err(finalize_err)) => anyhow::bail!(
2530 "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2531 {finalize_err}"
2532 ),
2533 (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2534 "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2535 {stop_err}; failed to finalize startup abort: {finalize_err}"
2536 ),
2537 }
2538 }
2539
2540 fn initiate_shutdown(&mut self) {
2541 #[cfg(feature = "plugin")]
2542 if let Err(e) = self.plugins.stop_controllers() {
2543 log::error!("Error stopping plug-in controllers: {e}");
2544 }
2545
2546 self.kernel.stop_trader();
2547 let delay = self.kernel.delay_post_stop();
2548 log::info!("Awaiting residual events ({delay:?})...");
2549
2550 self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2551 self.handle.set_shutting_down();
2552 }
2553
2554 async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2555 let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2558 self.close_external_ingress();
2559
2560 let timeout = self.config.timeout_disconnection;
2561 let deadline = dst::time::Instant::now() + timeout;
2562
2563 let mut errors = Vec::new();
2564 let (report_result, disconnect_result) = futures_util::join!(
2565 dst::time::timeout(
2566 timeout,
2567 Self::finish_report_tasks(&self.exec_clients, &mut errors)
2568 ),
2569 dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2570 );
2571
2572 if report_result.is_err() {
2573 errors.push("report collection shutdown timeout".to_string());
2574 }
2575
2576 let disconnect_result = match disconnect_result {
2577 Ok(result) => result,
2578 Err(_) => Err(anyhow::anyhow!(
2579 "disconnect timeout while disconnecting clients"
2580 )),
2581 };
2582
2583 if let Err(ref e) = disconnect_result {
2584 log::error!("Error disconnecting clients: {e}");
2585 }
2586
2587 let readiness_result = self.await_engines_disconnected(deadline).await;
2588 let kernel_result = self.kernel.finalize_stop().await;
2589
2590 self.handle.set_stopped();
2591
2592 if !unresolved_submissions.is_empty() {
2593 errors.push(format!(
2594 "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2595 ));
2596 }
2597
2598 if let Err(e) = disconnect_result {
2599 errors.push(e.to_string());
2600 }
2601
2602 if let Err(e) = readiness_result {
2603 errors.push(format!("failed while awaiting engine disconnection: {e}"));
2604 }
2605
2606 if let Err(e) = kernel_result {
2607 errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2608 }
2609
2610 if errors.is_empty() {
2611 Ok(())
2612 } else {
2613 anyhow::bail!("{}", errors.join("; "))
2614 }
2615 }
2616
2617 fn drain_channels(
2618 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2619 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2620 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2621 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2622 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2623 DispatchMessage<TradingCommandMessage>,
2624 >,
2625 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2626 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2627 ) {
2628 let pending = time_evt_rx.len()
2629 + system_evt_rx.len()
2630 + system_cmd_rx.len()
2631 + exec_evt_rx.len()
2632 + exec_cmd_rx.len()
2633 + data_evt_rx.len()
2634 + data_cmd_rx.len();
2635 log::debug!("Draining shutdown queues ({pending} events initially pending)");
2636
2637 let mut drained = 0;
2638 let log_progress = |processed: usize| {
2639 if processed.is_multiple_of(10_000) {
2640 log::debug!("Processed {processed} events during shutdown drain");
2641 }
2642 };
2643
2644 while let Ok(handler) = time_evt_rx.try_recv() {
2645 let _ = AsyncRunner::handle_time_event(handler);
2646 drained += 1;
2647 log_progress(drained);
2648 }
2649
2650 while system_evt_rx.try_recv().is_ok() {
2651 drained += 1;
2652 log_progress(drained);
2653 }
2654
2655 while system_cmd_rx.try_recv().is_ok() {
2656 drained += 1;
2657 log_progress(drained);
2658 }
2659
2660 while let Ok(evt) = data_evt_rx.try_recv() {
2661 AsyncRunner::dispatch_data_event(evt);
2662 drained += 1;
2663 log_progress(drained);
2664 }
2665
2666 while let Ok(cmd) = data_cmd_rx.try_recv() {
2667 AsyncRunner::handle_data_command(cmd);
2668 drained += 1;
2669 log_progress(drained);
2670 }
2671
2672 while let Ok(evt) = exec_evt_rx.try_recv() {
2673 AsyncRunner::dispatch_exec_event(evt);
2674 drained += 1;
2675 log_progress(drained);
2676 }
2677
2678 while let Ok(cmd) = exec_cmd_rx.try_recv() {
2679 AsyncRunner::handle_trading_command(cmd);
2680 drained += 1;
2681 log_progress(drained);
2682 }
2683
2684 log::debug!("Drained {drained} remaining events during shutdown");
2685 }
2686
2687 fn observe_exec_event_before_dispatch(
2688 &mut self,
2689 evt: &ExecutionEvent,
2690 ) -> Option<Vec<ClientOrderId>> {
2691 let mut order_ids = Vec::new();
2692
2693 match evt {
2694 ExecutionEvent::Order(order_evt) => {
2695 self.exec_manager.observe_order_event(order_evt);
2696 order_ids.push(order_evt.client_order_id());
2697 }
2698 ExecutionEvent::OrderSubmittedBatch(batch) => {
2699 for submitted in &batch.events {
2700 self.exec_manager
2701 .record_local_activity(submitted.client_order_id);
2702 }
2703 }
2704 ExecutionEvent::OrderAcceptedBatch(batch) => {
2705 for accepted in &batch.events {
2706 self.exec_manager
2707 .observe_order_event(&OrderEventAny::Accepted(*accepted));
2708 order_ids.push(accepted.client_order_id);
2709 }
2710 }
2711 ExecutionEvent::OrderCanceledBatch(batch) => {
2712 for canceled in &batch.events {
2713 self.exec_manager
2714 .observe_order_event(&OrderEventAny::Canceled(*canceled));
2715 order_ids.push(canceled.client_order_id);
2716 }
2717 }
2718 ExecutionEvent::Report(report) => {
2719 if let ExecutionReport::Fill(fill_report) = report
2720 && self.exec_manager.is_fill_recently_processed(
2721 fill_report.account_id,
2722 fill_report.instrument_id,
2723 fill_report.trade_id,
2724 )
2725 {
2726 log::debug!(
2727 "Skipping recently processed fill report: {}",
2728 fill_report.trade_id,
2729 );
2730 return None;
2731 }
2732
2733 self.exec_manager.observe_execution_report(report);
2734
2735 if let Some(client_order_id) = self.order_report_client_order_id(report) {
2736 order_ids.push(client_order_id);
2737 }
2738 }
2739 ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {}
2740 }
2741
2742 Some(order_ids)
2743 }
2744
2745 fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2746 match report {
2747 ExecutionReport::Order(order_report)
2748 | ExecutionReport::OrderWithFills(order_report, _) => {
2749 order_report.client_order_id.or_else(|| {
2750 self.kernel
2751 .cache
2752 .borrow()
2753 .client_order_id(&order_report.venue_order_id)
2754 .copied()
2755 })
2756 }
2757 ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2758 self.kernel
2759 .cache
2760 .borrow()
2761 .client_order_id(&fill_report.venue_order_id)
2762 .copied()
2763 }),
2764 _ => None,
2765 }
2766 }
2767
2768 fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2769 if matches!(
2770 cmd,
2771 TradingCommand::ModifyOrder(_)
2772 | TradingCommand::ModifyOrders(_)
2773 | TradingCommand::CancelOrder(_)
2774 | TradingCommand::CancelOrders(_)
2775 ) {
2776 let engine = self.kernel.exec_engine.borrow();
2777 let no_route = engine.find_client_for_command(cmd).is_none()
2778 && cmd.client_id().is_none_or(|client_id| {
2779 engine
2780 .config()
2781 .external_clients
2782 .as_ref()
2783 .is_none_or(|clients| !clients.contains(&client_id))
2784 });
2785
2786 if no_route {
2787 return;
2788 }
2789 }
2790
2791 let submission_client_id = if matches!(
2792 cmd,
2793 TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2794 ) {
2795 self.kernel
2796 .exec_engine
2797 .borrow()
2798 .find_client_for_command(cmd)
2799 .map(|client| client.client_id())
2800 } else {
2801 None
2802 };
2803
2804 match cmd {
2805 TradingCommand::SubmitOrder(submit) => {
2806 self.exec_manager.register_submission(
2807 &submit.order_init,
2808 submission_client_id.or(submit.client_id),
2809 );
2810 }
2811 TradingCommand::SubmitOrderList(submit) => {
2812 for order_init in &submit.order_inits {
2813 self.exec_manager
2814 .register_submission(order_init, submission_client_id.or(submit.client_id));
2815 }
2816 }
2817 TradingCommand::ModifyOrder(modify) => {
2818 self.exec_manager
2819 .register_inflight(modify.client_order_id, modify.command_id);
2820 }
2821 TradingCommand::ModifyOrders(modify) => {
2822 for child in &modify.modifies {
2823 self.exec_manager
2824 .register_inflight(child.client_order_id, child.command_id);
2825 }
2826 }
2827 TradingCommand::CancelOrder(cancel) => {
2828 self.exec_manager
2829 .register_inflight(cancel.client_order_id, cancel.command_id);
2830 }
2831 TradingCommand::CancelOrders(cancel) => {
2832 for child in &cancel.cancels {
2833 self.exec_manager
2834 .register_inflight(child.client_order_id, child.command_id);
2835 }
2836 }
2837 _ => {}
2838 }
2839 }
2840
2841 #[must_use]
2843 pub fn environment(&self) -> Environment {
2844 self.kernel.environment()
2845 }
2846
2847 #[must_use]
2849 pub const fn kernel(&self) -> &NautilusKernel {
2850 &self.kernel
2851 }
2852
2853 #[must_use]
2855 pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2856 &mut self.kernel
2857 }
2858
2859 #[must_use]
2861 pub fn trader_id(&self) -> TraderId {
2862 self.kernel.trader_id()
2863 }
2864
2865 #[must_use]
2867 pub const fn instance_id(&self) -> UUID4 {
2868 self.kernel.instance_id()
2869 }
2870
2871 #[must_use]
2873 pub fn state(&self) -> NodeState {
2874 self.handle.state()
2875 }
2876
2877 #[must_use]
2879 pub fn is_running(&self) -> bool {
2880 self.state().is_running()
2881 }
2882
2883 pub fn set_cache_database(
2893 &mut self,
2894 database: Box<dyn CacheDatabaseAdapter>,
2895 ) -> anyhow::Result<()> {
2896 if self.state() != NodeState::Idle {
2897 anyhow::bail!(
2898 "Cannot set cache database while node is running, set it before running the node"
2899 );
2900 }
2901
2902 self.kernel.cache().borrow_mut().set_database(database);
2903 Ok(())
2904 }
2905
2906 #[must_use]
2908 pub fn exec_manager(&self) -> &ExecutionManager {
2909 &self.exec_manager
2910 }
2911
2912 #[must_use]
2914 pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2915 &mut self.exec_manager
2916 }
2917
2918 pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2931 where
2932 T: DataActor + DataActorNative + Component + Actor + 'static,
2933 {
2934 if self.state() != NodeState::Idle {
2935 anyhow::bail!(
2936 "Cannot add actor while node is running, add actors before running the node"
2937 );
2938 }
2939
2940 self.kernel.trader.borrow_mut().add_actor(actor)
2941 }
2942
2943 pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2955 where
2956 F: FnOnce() -> anyhow::Result<T>,
2957 T: DataActor + DataActorNative + Component + Actor + 'static,
2958 {
2959 if self.state() != NodeState::Idle {
2960 anyhow::bail!(
2961 "Cannot add actor while node is running, add actors before running the node"
2962 );
2963 }
2964
2965 self.kernel
2966 .trader
2967 .borrow_mut()
2968 .add_actor_from_factory(factory)
2969 }
2970
2971 pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2987 where
2988 T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2989 {
2990 if self.state() != NodeState::Idle {
2991 anyhow::bail!(
2992 "Cannot add strategy while node is running, add strategies before running the node"
2993 );
2994 }
2995
2996 let strategy_id = self
2998 .kernel
2999 .trader
3000 .borrow()
3001 .prepare_strategy_for_registration(&mut strategy)?;
3002 let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
3003 let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
3004
3005 if !instrument_ids.is_empty() {
3006 self.register_external_order_claims(strategy_id, &instrument_ids)?;
3007 }
3008
3009 let mut exec_engine = match oms_type
3010 .map(|_| {
3011 self.kernel
3012 .exec_engine
3013 .try_borrow_mut()
3014 .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
3015 })
3016 .transpose()
3017 {
3018 Ok(exec_engine) => exec_engine,
3019 Err(e) => {
3020 if !instrument_ids.is_empty()
3021 && let Err(rollback_error) =
3022 self.rollback_external_order_claims(strategy_id, &instrument_ids)
3023 {
3024 anyhow::bail!(
3025 "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
3026 );
3027 }
3028
3029 return Err(e);
3030 }
3031 };
3032
3033 if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
3034 drop(exec_engine);
3035
3036 if !instrument_ids.is_empty()
3037 && let Err(rollback_error) =
3038 self.rollback_external_order_claims(strategy_id, &instrument_ids)
3039 {
3040 anyhow::bail!(
3041 "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
3042 );
3043 }
3044
3045 return Err(add_error);
3046 }
3047
3048 if let Some(exec_engine) = &mut exec_engine
3049 && let Some(oms_type) = oms_type
3050 {
3051 exec_engine.register_oms_type(strategy_id, oms_type);
3052 }
3053
3054 Ok(())
3055 }
3056
3057 pub fn register_external_order_claims(
3068 &self,
3069 strategy_id: StrategyId,
3070 instrument_ids: &[InstrumentId],
3071 ) -> anyhow::Result<()> {
3072 self.kernel
3073 .cache
3074 .try_borrow_mut()
3075 .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3076 .register_external_order_claims(strategy_id, instrument_ids)?;
3077
3078 if !instrument_ids.is_empty() {
3079 log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3080 }
3081
3082 Ok(())
3083 }
3084
3085 pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3095 self.kernel
3096 .cache
3097 .try_borrow_mut()
3098 .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3099 .set_external_order_claims(strategy_id, &[])?;
3100
3101 Ok(())
3102 }
3103
3104 pub(crate) fn rollback_external_order_claims(
3105 &self,
3106 strategy_id: StrategyId,
3107 instrument_ids: &[InstrumentId],
3108 ) -> anyhow::Result<()> {
3109 let mut cache = self
3110 .kernel
3111 .cache
3112 .try_borrow_mut()
3113 .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3114 let retained: Vec<_> = cache
3115 .external_order_claim_instrument_ids(Some(strategy_id))
3116 .into_iter()
3117 .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3118 .collect();
3119 cache.set_external_order_claims(strategy_id, &retained)
3120 }
3121
3122 pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3133 where
3134 T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3135 {
3136 if self.state() != NodeState::Idle {
3137 anyhow::bail!(
3138 "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3139 );
3140 }
3141
3142 self.kernel
3143 .trader
3144 .borrow_mut()
3145 .add_exec_algorithm(exec_algorithm)
3146 }
3147
3148 }
3152
3153#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3154enum SocketReconnectDispatchOutcome {
3155 Accepted,
3156 AlreadyReconnecting,
3157 Disconnected,
3158 Closed,
3159 Unsupported,
3160 InvalidTrader,
3161 UnknownClient,
3162 UnknownEndpoint,
3163 AmbiguousEndpoint,
3164}
3165
3166fn record_runner_dispatch(
3167 metrics: &RunnerMetrics,
3168 channel: SystemChannel,
3169 dispatch_start: dst::time::Instant,
3170 metrics_start: dst::time::Instant,
3171) {
3172 let dispatch_end = dst::time::Instant::now();
3173 metrics.record_dispatch(
3174 channel,
3175 dispatch_end.duration_since(dispatch_start),
3176 dispatch_end.duration_since(metrics_start),
3177 );
3178}
3179
3180fn record_runner_maintenance(
3181 metrics: &RunnerMetrics,
3182 work_start: dst::time::Instant,
3183 metrics_start: dst::time::Instant,
3184) {
3185 let work_end = dst::time::Instant::now();
3186 metrics.record_maintenance(
3187 work_end.duration_since(work_start),
3188 work_end.duration_since(metrics_start),
3189 );
3190}
3191
3192fn record_runner_external_msgbus(
3193 metrics: &RunnerMetrics,
3194 work_start: dst::time::Instant,
3195 metrics_start: dst::time::Instant,
3196) {
3197 let work_end = dst::time::Instant::now();
3198 metrics.record_external_msgbus(
3199 work_end.duration_since(work_start),
3200 work_end.duration_since(metrics_start),
3201 );
3202}
3203
3204async fn drive_with_external_ingress<F, T>(
3205 future: F,
3206 receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3207 pending: &mut VecDeque<BusMessage>,
3208) -> T
3209where
3210 F: std::future::Future<Output = T>,
3211{
3212 tokio::pin!(future);
3213 let capacity = receiver
3214 .as_ref()
3215 .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3216
3217 loop {
3218 tokio::select! {
3219 biased;
3220
3221 result = &mut future => return result,
3222 message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3223 let Some(message) = message else {
3224 log::info!("External message bus ingress closed");
3225 *receiver = None;
3226 continue;
3227 };
3228
3229 match msgbus::republish_pending_external_response(&message) {
3230 Ok(true) => {},
3231 Ok(false) | Err(_) => pending.push_back(message),
3232 }
3233 }
3234 }
3235 }
3236}
3237
3238async fn recv_external_msgbus_message(
3239 rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3240 pending: &mut VecDeque<BusMessage>,
3241) -> Option<BusMessage> {
3242 if let Some(message) = pending.pop_front() {
3243 return Some(message);
3244 }
3245
3246 match rx {
3247 Some(rx) => rx.recv().await,
3248 None => std::future::pending::<Option<BusMessage>>().await,
3249 }
3250}
3251
3252struct RunnerReceivers<'a> {
3253 time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3254 system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3255 system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3256 exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3257 exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3258 data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3259 data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3260}
3261
3262fn flush_pending_data(
3269 pending: &mut PendingEvents,
3270 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3271 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3272) {
3273 loop {
3274 let mut progressed = pending.drain_data();
3275
3276 while let Ok(evt) = data_evt_rx.try_recv() {
3277 AsyncRunner::dispatch_data_event(evt);
3278 progressed = true;
3279 }
3280
3281 while let Ok(cmd) = data_cmd_rx.try_recv() {
3282 AsyncRunner::handle_data_command(cmd);
3283 progressed = true;
3284 }
3285
3286 if !progressed {
3287 break;
3288 }
3289 }
3290}
3291
3292#[expect(
3298 clippy::too_many_arguments,
3299 reason = "all runner receivers are drained together"
3300)]
3301fn flush_all_pending(
3302 pending: &mut PendingEvents,
3303 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3304 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3305 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3306 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3307 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3308 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3309 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3310) {
3311 while let Ok(handler) = time_evt_rx.try_recv() {
3313 let _ = AsyncRunner::handle_time_event(handler);
3314 }
3315
3316 while let Ok(event) = system_evt_rx.try_recv() {
3317 pending.system_events.push(event);
3318 }
3319
3320 while let Ok(command) = system_cmd_rx.try_recv() {
3321 pending.system_commands.push(command);
3322 }
3323
3324 while let Ok(evt) = data_evt_rx.try_recv() {
3325 pending.data_evts.push(evt);
3326 }
3327
3328 while let Ok(cmd) = data_cmd_rx.try_recv() {
3329 pending.data_cmds.push(cmd);
3330 }
3331
3332 while let Ok(evt) = exec_evt_rx.try_recv() {
3333 pending.push_exec_event(evt);
3334 }
3335
3336 while let Ok(cmd) = exec_cmd_rx.try_recv() {
3337 pending.exec_cmds.push(cmd);
3338 }
3339
3340 pending.drain();
3341}
3342
3343#[expect(
3348 clippy::too_many_arguments,
3349 reason = "startup buffering owns one future plus the pending state and all runner receivers"
3350)]
3351async fn drive_with_event_buffering<F: std::future::Future>(
3352 future: F,
3353 pending: &mut PendingEvents,
3354 time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3355 system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3356 system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3357 exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3358 exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3359 data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3360 data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3361) -> F::Output {
3362 tokio::pin!(future);
3363
3364 loop {
3365 tokio::select! {
3366 biased;
3367
3368 result = &mut future => {
3369 break result;
3370 }
3371 Some(handler) = time_evt_rx.recv() => {
3372 let _ = AsyncRunner::handle_time_event(handler);
3373 }
3374 Some(event) = system_evt_rx.recv() => {
3375 pending.system_events.push(event);
3376 }
3377 Some(command) = system_cmd_rx.recv() => {
3378 pending.system_commands.push(command);
3379 }
3380 Some(evt) = exec_evt_rx.recv() => {
3381 pending.push_exec_event(evt);
3382 }
3383 Some(cmd) = exec_cmd_rx.recv() => {
3384 pending.exec_cmds.push(cmd);
3385 }
3386 Some(evt) = data_evt_rx.recv() => {
3387 pending.data_evts.push(evt);
3388 }
3389 Some(cmd) = data_cmd_rx.recv() => {
3390 pending.data_cmds.push(cmd);
3391 }
3392 }
3393 }
3394}
3395
3396#[derive(Default)]
3397struct PendingEvents {
3398 system_events: Vec<DispatchMessage<SystemEvent>>,
3399 system_commands: Vec<DispatchMessage<SystemCommand>>,
3400 data_evts: Vec<DispatchMessage<DataEvent>>,
3401 data_cmds: Vec<DispatchMessage<DataCommand>>,
3402 exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3403 order_evts: Vec<DispatchMessage<OrderEventAny>>,
3404 exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3405}
3406
3407impl PendingEvents {
3408 fn is_empty(&self) -> bool {
3409 self.system_events.is_empty()
3410 && self.system_commands.is_empty()
3411 && self.data_evts.is_empty()
3412 && self.data_cmds.is_empty()
3413 && self.exec_reports.is_empty()
3414 && self.order_evts.is_empty()
3415 && self.exec_cmds.is_empty()
3416 }
3417
3418 fn drain_data(&mut self) -> bool {
3422 let total = self.data_evts.len() + self.data_cmds.len();
3423
3424 if total > 0 {
3425 log::debug!(
3426 "Draining {total} data events/commands into cache \
3427 (data_evts={}, data_cmds={})",
3428 self.data_evts.len(),
3429 self.data_cmds.len(),
3430 );
3431 }
3432
3433 for evt in self.data_evts.drain(..) {
3434 AsyncRunner::dispatch_data_event(evt);
3435 }
3436
3437 for cmd in self.data_cmds.drain(..) {
3438 AsyncRunner::handle_data_command(cmd);
3439 }
3440
3441 total > 0
3442 }
3443
3444 fn drain(&mut self) {
3446 let total = self.data_evts.len()
3447 + self.data_cmds.len()
3448 + self.exec_reports.len()
3449 + self.order_evts.len()
3450 + self.exec_cmds.len();
3451
3452 if total > 0 {
3453 log::debug!(
3454 "Processing {total} events/commands queued during startup \
3455 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3456 self.data_evts.len(),
3457 self.data_cmds.len(),
3458 self.exec_reports.len(),
3459 self.order_evts.len(),
3460 self.exec_cmds.len()
3461 );
3462 }
3463
3464 for evt in self.data_evts.drain(..) {
3465 AsyncRunner::dispatch_data_event(evt);
3466 }
3467
3468 for cmd in self.data_cmds.drain(..) {
3469 AsyncRunner::handle_data_command(cmd);
3470 }
3471
3472 for report in self.exec_reports.drain(..) {
3473 report
3474 .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3475 }
3476
3477 for evt in self.order_evts.drain(..) {
3478 evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3479 }
3480
3481 for cmd in self.exec_cmds.drain(..) {
3482 AsyncRunner::handle_trading_command(cmd);
3483 }
3484 }
3485
3486 fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3487 let rooted = event.is_rooted();
3488 event.dispatch(|event| {
3489 let owner = std::thread::current().id();
3490
3491 let order = |event| {
3492 if rooted {
3493 DispatchMessage::new(event, owner)
3494 } else {
3495 DispatchMessage::from(event)
3496 }
3497 };
3498
3499 match event {
3502 ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {
3503 AsyncRunner::handle_exec_event(event);
3504 }
3505 ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3506 DispatchMessage::new(report, owner)
3507 } else {
3508 report.into()
3509 }),
3510 ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3511 ExecutionEvent::OrderSubmittedBatch(batch) => {
3512 for event in batch {
3513 self.order_evts.push(order(OrderEventAny::Submitted(event)));
3514 }
3515 }
3516 ExecutionEvent::OrderAcceptedBatch(batch) => {
3517 for event in batch {
3518 self.order_evts.push(order(OrderEventAny::Accepted(event)));
3519 }
3520 }
3521 ExecutionEvent::OrderCanceledBatch(batch) => {
3522 for event in batch {
3523 self.order_evts.push(order(OrderEventAny::Canceled(event)));
3524 }
3525 }
3526 }
3527 });
3528 }
3529
3530 fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3531 std::mem::take(&mut self.system_events)
3532 }
3533
3534 fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3535 std::mem::take(&mut self.system_commands)
3536 }
3537}
3538
3539struct ClientStatus {
3540 client: String,
3541 client_type: &'static str,
3542 connected: bool,
3543}
3544
3545fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3546 let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3547 builder.push_record(["Client", "Type", "Connected"]);
3548
3549 for row in rows {
3550 builder.push_record([
3551 row.client,
3552 row.client_type.to_string(),
3553 row.connected.to_string(),
3554 ]);
3555 }
3556
3557 builder.build().with(Style::rounded()).to_string()
3558}
3559
3560#[cfg(test)]
3561mod tests {
3562 mod report_acceptance;
3563 mod shutdown;
3564
3565 use std::{
3566 cell::{Cell, RefCell},
3567 fmt::Debug,
3568 rc::Rc,
3569 sync::{
3570 Arc,
3571 atomic::{AtomicBool, Ordering},
3572 },
3573 };
3574
3575 use bytes::Bytes;
3576 use indexmap::{IndexMap, IndexSet};
3577 use log::{Level, LevelFilter, Log, Metadata, Record};
3578 #[cfg(feature = "python")]
3579 use nautilus_common::runner::{
3580 SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3581 };
3582 use nautilus_common::{
3583 actor::{
3584 self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3585 },
3586 cache::Cache,
3587 clock::{Clock, VirtualClock},
3588 enums::SerializationEncoding,
3589 live::{
3590 runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3591 sender::DispatchSender,
3592 },
3593 logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3594 messages::{
3595 data::{SubscribeCommand, SubscribeQuotes},
3596 execution::{
3597 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3598 GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3599 TradingCommand,
3600 },
3601 system::{
3602 QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3603 },
3604 },
3605 msgbus::{
3606 self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3607 MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3608 MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3609 },
3610 nautilus_actor,
3611 runner::{SyncTradingCommandSender, TradingCommandSender},
3612 testing::wait_until_async,
3613 timer::{TimeEvent, TimeEventCallback},
3614 };
3615 use nautilus_core::{Params, UUID4, UnixNanos};
3616 use nautilus_execution::{
3617 engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3618 reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3619 };
3620 use nautilus_model::{
3621 accounts::{AccountAny, MarginAccount},
3622 data::QuoteTick,
3623 enums::{
3624 AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3625 TimeInForce,
3626 },
3627 events::{
3628 AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3629 order::spec::{
3630 OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3631 },
3632 },
3633 identifiers::{
3634 AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3635 TradeId, TraderId, Venue, VenueOrderId,
3636 },
3637 instruments::{
3638 Instrument, InstrumentAny,
3639 stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3640 },
3641 orders::{
3642 Order, OrderAny, OrderList, OrderTestBuilder,
3643 stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3644 },
3645 position::PositionReplayEvent,
3646 reports::{FillReport, PositionStatusReport},
3647 types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3648 };
3649 use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3650 use nautilus_testkit::{
3651 cache::TestCacheDatabaseControl,
3652 components::{StateActor, StateStrategy},
3653 };
3654 use nautilus_trading::{
3655 nautilus_strategy,
3656 strategy::{config::StrategyConfig, core::StrategyCore},
3657 };
3658 use parking_lot::Mutex;
3659 use rstest::*;
3660 use rust_decimal::Decimal;
3661 use rust_decimal_macros::dec;
3662 use ustr::Ustr;
3663
3664 use super::{
3665 reconciliation::{
3666 PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3667 request_position_fill_reports,
3668 },
3669 *,
3670 };
3671 use crate::{
3672 execution::{
3673 client::LiveExecutionClient,
3674 manager::{PositionFillReportQuery, ReportClientCoverage, SourcedOrderStatusReport},
3675 submission::{
3676 SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3677 },
3678 },
3679 socket::SocketControl,
3680 };
3681
3682 struct ExternalIngressLogCapture {
3683 messages: Mutex<Vec<String>>,
3684 }
3685
3686 static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3687 messages: Mutex::new(Vec::new()),
3688 };
3689
3690 #[derive(Debug)]
3691 enum FillReportClientOutcome {
3692 Reports(Vec<FillReport>),
3693 Failure,
3694 }
3695
3696 #[derive(Debug)]
3697 struct FillReportClient {
3698 client_id: ClientId,
3699 account_id: AccountId,
3700 venue: Venue,
3701 outcome: FillReportClientOutcome,
3702 commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3703 }
3704
3705 #[async_trait::async_trait(?Send)]
3706 impl ExecutionClient for FillReportClient {
3707 fn is_connected(&self) -> bool {
3708 true
3709 }
3710
3711 fn client_id(&self) -> ClientId {
3712 self.client_id
3713 }
3714
3715 fn account_id(&self) -> AccountId {
3716 self.account_id
3717 }
3718
3719 fn venue(&self) -> Venue {
3720 self.venue
3721 }
3722
3723 fn oms_type(&self) -> OmsType {
3724 OmsType::Netting
3725 }
3726
3727 fn get_account(&self) -> Option<AccountAny> {
3728 None
3729 }
3730
3731 fn generate_account_state(
3732 &self,
3733 _balances: Vec<AccountBalance>,
3734 _margins: Vec<MarginBalance>,
3735 _reported: bool,
3736 _ts_event: UnixNanos,
3737 _info: Option<Params>,
3738 ) -> anyhow::Result<()> {
3739 Ok(())
3740 }
3741
3742 fn start(&mut self) -> anyhow::Result<()> {
3743 Ok(())
3744 }
3745
3746 fn stop(&mut self) -> anyhow::Result<()> {
3747 Ok(())
3748 }
3749
3750 async fn generate_order_status_reports(
3751 &self,
3752 _cmd: &GenerateOrderStatusReports,
3753 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3754 match self.outcome {
3755 FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3756 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3757 }
3758 }
3759
3760 async fn generate_position_status_reports(
3761 &self,
3762 _cmd: &GeneratePositionStatusReports,
3763 ) -> anyhow::Result<Vec<PositionStatusReport>> {
3764 match self.outcome {
3765 FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3766 FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3767 }
3768 }
3769
3770 async fn generate_fill_reports(
3771 &self,
3772 cmd: GenerateFillReports,
3773 ) -> anyhow::Result<Vec<FillReport>> {
3774 self.commands.borrow_mut().push(cmd);
3775
3776 match &self.outcome {
3777 FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3778 FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3779 }
3780 }
3781 }
3782
3783 #[derive(Debug)]
3784 struct OrderReportClient {
3785 client_id: ClientId,
3786 report: OrderStatusReport,
3787 targeted: bool,
3788 queries: Rc<RefCell<Vec<&'static str>>>,
3789 }
3790
3791 #[async_trait::async_trait(?Send)]
3792 impl ExecutionClient for OrderReportClient {
3793 fn is_connected(&self) -> bool {
3794 true
3795 }
3796
3797 fn client_id(&self) -> ClientId {
3798 self.client_id
3799 }
3800
3801 fn account_id(&self) -> AccountId {
3802 self.report.account_id
3803 }
3804
3805 fn venue(&self) -> Venue {
3806 self.report.instrument_id.venue
3807 }
3808
3809 fn oms_type(&self) -> OmsType {
3810 OmsType::Netting
3811 }
3812
3813 fn get_account(&self) -> Option<AccountAny> {
3814 None
3815 }
3816
3817 fn generate_account_state(
3818 &self,
3819 _balances: Vec<AccountBalance>,
3820 _margins: Vec<MarginBalance>,
3821 _reported: bool,
3822 _ts_event: UnixNanos,
3823 _info: Option<Params>,
3824 ) -> anyhow::Result<()> {
3825 Ok(())
3826 }
3827
3828 fn start(&mut self) -> anyhow::Result<()> {
3829 Ok(())
3830 }
3831
3832 fn stop(&mut self) -> anyhow::Result<()> {
3833 Ok(())
3834 }
3835
3836 async fn generate_order_status_reports(
3837 &self,
3838 _command: &GenerateOrderStatusReports,
3839 ) -> anyhow::Result<Vec<OrderStatusReport>> {
3840 self.queries.borrow_mut().push("bulk");
3841 Ok(if self.targeted {
3842 Vec::new()
3843 } else {
3844 vec![self.report.clone()]
3845 })
3846 }
3847
3848 async fn generate_order_status_report(
3849 &self,
3850 _command: &GenerateOrderStatusReport,
3851 ) -> anyhow::Result<Option<OrderStatusReport>> {
3852 self.queries.borrow_mut().push("targeted");
3853 Ok(Some(self.report.clone()))
3854 }
3855 }
3856
3857 #[derive(Debug)]
3858 struct StartupSocketActor {
3859 core: DataActorCore,
3860 received: Rc<RefCell<Vec<SocketStateChanged>>>,
3861 }
3862
3863 impl StartupSocketActor {
3864 fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3865 Self {
3866 core: DataActorCore::new(DataActorConfig {
3867 actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3868 ..Default::default()
3869 }),
3870 received,
3871 }
3872 }
3873 }
3874
3875 impl DataActor for StartupSocketActor {
3876 fn on_start(&mut self) -> anyhow::Result<()> {
3877 self.subscribe_socket_state(None, None, None);
3878 Ok(())
3879 }
3880
3881 fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3882 self.received.borrow_mut().push(event.clone());
3883 Ok(())
3884 }
3885 }
3886
3887 nautilus_actor!(StartupSocketActor);
3888
3889 impl Log for ExternalIngressLogCapture {
3890 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3891 metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3892 }
3893
3894 fn log(&self, record: &Record<'_>) {
3895 if self.enabled(record.metadata()) {
3896 self.messages.lock().push(record.args().to_string());
3897 }
3898 }
3899
3900 fn flush(&self) {}
3901 }
3902
3903 #[derive(Debug)]
3904 struct FailingTimerActor {
3905 core: DataActorCore,
3906 received: Rc<RefCell<Vec<u64>>>,
3907 }
3908
3909 nautilus_actor!(FailingTimerActor);
3910
3911 impl DataActor for FailingTimerActor {
3912 fn on_start(&mut self) -> anyhow::Result<()> {
3913 for timestamp in [17, 23] {
3914 let received = self.received.clone();
3915
3916 let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3917 received.borrow_mut().push(timestamp);
3918
3919 if timestamp == 17 {
3920 crate::dispatch::tests::latch_callback_failure();
3921 }
3922 }));
3923
3924 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3925 TimeEvent::new(
3926 "callback-failure".into(),
3927 UUID4::new(),
3928 timestamp.into(),
3929 timestamp.into(),
3930 ),
3931 callback,
3932 ));
3933 }
3934
3935 Ok(())
3936 }
3937 }
3938
3939 #[rstest]
3940 #[tokio::test]
3941 async fn test_callback_failure_stops_later_live_events(
3942 #[values(false, true)] incomplete_submission: bool,
3943 ) {
3944 actor::clear_callbacks().unwrap();
3945
3946 let config = LiveNodeConfig {
3947 exec_engine: crate::config::LiveExecutionEngineConfig {
3948 reconciliation: false,
3949 submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3950 ..Default::default()
3951 },
3952 timeout_connection: Duration::ZERO,
3953 timeout_reconciliation: Duration::ZERO,
3954 timeout_portfolio: Duration::ZERO,
3955 timeout_disconnection: Duration::ZERO,
3956 delay_post_stop: Duration::ZERO,
3957 timeout_shutdown: Duration::ZERO,
3958 ..Default::default()
3959 };
3960
3961 let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3962
3963 if incomplete_submission {
3964 let order = OrderTestBuilder::new(OrderType::Market)
3965 .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3966 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3967 .quantity(Quantity::from("1"))
3968 .build();
3969 node.exec_manager
3970 .register_submission(order.init_event(), None);
3971 }
3972 let received = Rc::new(RefCell::new(Vec::new()));
3973 node.add_actor(FailingTimerActor {
3974 core: DataActorCore::new(DataActorConfig {
3975 actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3976 ..Default::default()
3977 }),
3978 received: received.clone(),
3979 })
3980 .unwrap();
3981
3982 let result = node.run_with_mode(NodeRunMode::Hosted).await;
3983
3984 let error = result.unwrap_err();
3985 assert!(error.to_string().contains("Callback delivery unwound"));
3986 if incomplete_submission {
3987 assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3988 }
3989 let state = node.state();
3990 let trader_stopped = node.kernel.trader.borrow().is_stopped();
3991 let failure = actor::callback_failure();
3992 node.dispose();
3993
3994 assert_eq!(
3995 error.downcast_ref::<CallbackDispatchError>(),
3996 Some(&CallbackDispatchError::DeliveryUnwound)
3997 );
3998 assert_eq!(*received.borrow(), [17]);
3999 assert_eq!(state, NodeState::Stopped);
4000 assert!(trader_stopped);
4001 assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
4002 assert_eq!(actor::callback_failure(), None);
4003 assert_eq!(actor::clear_callbacks(), Ok(()));
4004 }
4005
4006 #[rstest]
4007 fn test_render_client_statuses() {
4008 let rows = vec![
4009 ClientStatus {
4010 client: "BINANCE".to_string(),
4011 client_type: "Data",
4012 connected: true,
4013 },
4014 ClientStatus {
4015 client: "SIM".to_string(),
4016 client_type: "Execution",
4017 connected: false,
4018 },
4019 ];
4020
4021 let output = render_client_statuses(rows);
4022 let expected = "â•─────────┬───────────┬───────────╮\n\
4023│ Client │ Type │ Connected │\n\
4024├─────────┼───────────┼───────────┤\n\
4025│ BINANCE │ Data │ true │\n\
4026│ SIM │ Execution │ false │\n\
4027╰─────────┴───────────┴───────────╯";
4028
4029 assert_eq!(output, expected);
4030 }
4031
4032 #[rstest]
4033 fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
4034 log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
4035 log::set_max_level(LevelFilter::Error);
4036 EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
4037 let message = BusMessage::with_str_topic(
4038 "data.quotes.AUDUSD.SIM*",
4039 BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
4040 Bytes::new(),
4041 SerializationEncoding::Json,
4042 );
4043
4044 LiveNode::republish_external_msgbus_message(&message);
4045
4046 assert_eq!(
4047 *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
4048 vec![
4049 "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
4050 external message topic: Topic `value` contained invalid characters, was \
4051 data.quotes.AUDUSD.SIM*"
4052 .to_string()
4053 ],
4054 );
4055 }
4056
4057 #[rstest]
4058 #[tokio::test]
4059 async fn test_external_startup_prioritizes_completed_connection() {
4060 let message = BusMessage::with_str_topic(
4061 "data.trades.FIRST",
4062 BusPayloadType::TradeTick,
4063 bytes::Bytes::new(),
4064 SerializationEncoding::Json,
4065 );
4066 let (sender, receiver) = tokio::sync::mpsc::channel(1);
4067 sender.send(message.clone()).await.unwrap();
4068 let mut receiver = Some(receiver);
4069 let mut pending = VecDeque::new();
4070
4071 let result =
4072 drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4073 let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4074
4075 assert_eq!(result, 17);
4076 assert!(pending.is_empty());
4077 assert_eq!(
4078 serde_json::to_value(queued).unwrap(),
4079 serde_json::to_value(message).unwrap()
4080 );
4081 }
4082
4083 #[rstest]
4084 #[tokio::test]
4085 async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4086 use bytes::Bytes;
4087 use nautilus_common::messages::data::{
4088 CustomDataResponse, DataResponse, DataResponseEnvelope,
4089 };
4090 use nautilus_model::data::DataType;
4091
4092 let (egress, publications, _) = CapturingExternalEgress::new();
4093 let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4094 msgbus::set_message_bus(bus.clone());
4095 bus.borrow_mut()
4096 .set_external_egress_config(
4097 Box::new(egress),
4098 &MessageBusConfig {
4099 encoding: SerializationEncoding::MsgPack,
4100 external_streams: Some(vec!["external.test".into()]),
4101 ..Default::default()
4102 },
4103 )
4104 .unwrap();
4105
4106 bus.borrow_mut()
4107 .add_streaming_type(BusPayloadType::DataResponse);
4108 let id = UUID4::new();
4109 let response = DataResponse::Data(CustomDataResponse::new(
4110 id,
4111 "FEED".into(),
4112 None,
4113 DataType::new("Startup", None, None),
4114 Bytes::from_static(b"connected"),
4115 Some(101.into()),
4116 Some(103.into()),
4117 107.into(),
4118 None,
4119 ));
4120 msgbus::publish_any("responses.data.FEED".into(), &response);
4121 let message = BusMessage::with_str_topic(
4122 "responses.data.FEED",
4123 BusPayloadType::DataResponse,
4124 publications.borrow()[0].payload.clone(),
4125 SerializationEncoding::MsgPack,
4126 );
4127 let (done, connected) = tokio::sync::oneshot::channel();
4128 let done = Rc::new(RefCell::new(Some(done)));
4129 msgbus::register_response_handler(
4130 &id,
4131 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4132 let Some(DataResponse::Data(reply)) = envelope.response() else {
4133 panic!("expected custom reply");
4134 };
4135
4136 done.borrow_mut()
4137 .take()
4138 .unwrap()
4139 .send(reply.clone())
4140 .unwrap();
4141 }),
4142 );
4143
4144 msgbus::subscribe_any(
4145 message.topic.as_str().into(),
4146 ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4147 msgbus::send_response(reply.correlation_id(), reply);
4148 }),
4149 None,
4150 );
4151
4152 let unrelated = BusMessage::with_str_topic(
4153 "data.trades.FIRST",
4154 BusPayloadType::TradeTick,
4155 Bytes::new(),
4156 SerializationEncoding::Json,
4157 );
4158 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4159 sender.send(unrelated.clone()).await.unwrap();
4160 sender.send(message).await.unwrap();
4161 let mut receiver = Some(receiver);
4162 let mut pending = VecDeque::new();
4163 let actual = tokio::time::timeout(
4164 Duration::from_secs(1),
4165 drive_with_external_ingress(connected, &mut receiver, &mut pending),
4166 )
4167 .await
4168 .unwrap()
4169 .unwrap();
4170
4171 assert_eq!(actual.correlation_id, id);
4172 assert_eq!(actual.client_id, ClientId::from("FEED"));
4173 assert_eq!(actual.venue, None);
4174 assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4175 assert_eq!(
4176 actual.data.downcast_ref::<Bytes>(),
4177 Some(&Bytes::from_static(b"connected"))
4178 );
4179 assert_eq!(actual.start, Some(101.into()));
4180 assert_eq!(actual.end, Some(103.into()));
4181 assert_eq!(actual.ts_init, UnixNanos::from(107));
4182 assert_eq!(actual.params, None);
4183 assert_eq!(publications.borrow().len(), 1);
4184 assert_eq!(pending.len(), 1);
4185 assert_eq!(
4186 serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4187 serde_json::to_value(unrelated).unwrap()
4188 );
4189 assert!(receiver.as_ref().unwrap().is_empty());
4190 }
4191
4192 #[rstest]
4193 #[tokio::test]
4194 async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4195 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4196 let mut receiver = Some(receiver);
4197
4198 let message = |name: &str| {
4199 BusMessage::with_str_topic(
4200 name,
4201 BusPayloadType::TradeTick,
4202 bytes::Bytes::new(),
4203 SerializationEncoding::Json,
4204 )
4205 };
4206
4207 let first = message("data.trades.FIRST");
4208 let second = message("data.trades.SECOND");
4209 let third = message("data.trades.THIRD");
4210 sender.send(first.clone()).await.unwrap();
4211 sender.send(second.clone()).await.unwrap();
4212 let (done, connected) = tokio::sync::oneshot::channel();
4213 let mut pending = VecDeque::new();
4214 let mut connection = Box::pin(drive_with_external_ingress(
4215 connected,
4216 &mut receiver,
4217 &mut pending,
4218 ));
4219 assert!(futures_util::poll!(&mut connection).is_pending());
4220 sender.send(third.clone()).await.unwrap();
4221 assert!(futures_util::poll!(&mut connection).is_pending());
4222 done.send(()).unwrap();
4223 connection.await.unwrap();
4224
4225 assert_eq!(pending.len(), 2);
4226 assert_eq!(receiver.as_ref().unwrap().len(), 1);
4227
4228 for expected in [first, second, third] {
4229 let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4230 .await
4231 .unwrap();
4232 assert_eq!(
4233 serde_json::to_value(received).unwrap(),
4234 serde_json::to_value(expected).unwrap()
4235 );
4236 }
4237 }
4238
4239 #[rstest]
4240 #[tokio::test]
4241 async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4242 let (sender, receiver) = tokio::sync::mpsc::channel(2);
4243 drop(sender);
4244 let mut receiver = Some(receiver);
4245 let mut pending = VecDeque::new();
4246 let (done, connected) = tokio::sync::oneshot::channel();
4247 let mut connection = Box::pin(drive_with_external_ingress(
4248 connected,
4249 &mut receiver,
4250 &mut pending,
4251 ));
4252 assert!(futures_util::poll!(&mut connection).is_pending());
4253 done.send(()).unwrap();
4254 connection.await.unwrap();
4255
4256 assert!(receiver.is_none());
4257 assert!(pending.is_empty());
4258 }
4259
4260 #[rstest]
4261 #[case(None, 2)]
4262 #[case(Some(SystemChannel::DataEvents), 1)]
4263 #[case(Some(SystemChannel::ExecCommands), 1)]
4264 fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4265 #[case] channel: Option<SystemChannel>,
4266 #[case] expected_count: usize,
4267 ) {
4268 let config = LiveNodeConfig {
4269 trader_id: TraderId::from("QUEUE-001"),
4270 exec_engine: crate::config::LiveExecutionEngineConfig {
4271 reconciliation: false,
4272 ..Default::default()
4273 },
4274 ..Default::default()
4275 };
4276
4277 let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4278 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4279
4280 let handler = ShareableMessageHandler::from_typed({
4281 let received = received.clone();
4282 move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4283 });
4284
4285 msgbus::subscribe_any(
4286 MessagingSwitchboard::queue_state_changed_pattern(channel),
4287 handler,
4288 None,
4289 );
4290
4291 let transitions = [
4292 QueueStateTransition {
4293 channel: SystemChannel::DataEvents,
4294 condition: QueueCondition::Backlogged,
4295 state: QueueState::Triggered,
4296 queue_depth: 17,
4297 mean_dispatch_ns: 23,
4298 },
4299 QueueStateTransition {
4300 channel: SystemChannel::ExecCommands,
4301 condition: QueueCondition::Slow,
4302 state: QueueState::Triggered,
4303 queue_depth: 17,
4304 mean_dispatch_ns: 23,
4305 },
4306 ];
4307
4308 node.publish_queue_state_transitions(&transitions);
4309
4310 let events = received.borrow();
4311 assert_eq!(events.len(), expected_count);
4312
4313 let expected = transitions
4314 .into_iter()
4315 .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4316
4317 for (event, transition) in events.iter().zip(expected) {
4318 assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4319 assert_eq!(event.channel, transition.channel);
4320 assert_eq!(event.condition, transition.condition);
4321 assert_eq!(event.state, transition.state);
4322 assert_eq!(event.queue_depth, transition.queue_depth);
4323 assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4324 assert_ne!(event.event_id, UUID4::default());
4325 assert_ne!(event.ts_event, UnixNanos::default());
4326 assert_eq!(event.ts_init, event.ts_event);
4327 }
4328
4329 if channel.is_none() {
4330 assert_ne!(events[0].event_id, events[1].event_id);
4331 }
4332
4333 drop(events);
4334 msgbus::get_message_bus().borrow_mut().dispose();
4335 }
4336
4337 #[rstest]
4338 #[case(None, None)]
4339 #[case(Some(ClientId::from("BINANCE")), None)]
4340 #[case(None, Some("binance-futures-market-streams"))]
4341 #[case(
4342 Some(ClientId::from("BINANCE")),
4343 Some("binance-futures-market-streams")
4344 )]
4345 fn test_process_socket_state_change_reaches_typed_subscriber(
4346 #[case] client_id: Option<ClientId>,
4347 #[case] endpoint: Option<&str>,
4348 ) {
4349 let config = LiveNodeConfig {
4350 trader_id: TraderId::from("SOCKET-001"),
4351 exec_engine: crate::config::LiveExecutionEngineConfig {
4352 reconciliation: false,
4353 ..Default::default()
4354 },
4355 ..Default::default()
4356 };
4357
4358 let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4359 let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4360
4361 let handler = ShareableMessageHandler::from_typed({
4362 let received = received.clone();
4363 move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4364 });
4365
4366 msgbus::subscribe_any(
4367 MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4368 handler,
4369 None,
4370 );
4371
4372 let change = SocketStateChange::new(
4373 ClientId::from("BINANCE"),
4374 Some(Venue::from("BINANCE")),
4375 Ustr::from("binance-futures-market-streams"),
4376 SocketState::Disconnected,
4377 );
4378
4379 node.process_system_event(SystemEvent::SocketState(change));
4380
4381 let events = received.borrow();
4382 assert_eq!(events.len(), 1);
4383 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4384 assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4385 assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4386 assert_eq!(
4387 events[0].endpoint,
4388 Ustr::from("binance-futures-market-streams")
4389 );
4390 assert_eq!(events[0].state, SocketState::Disconnected);
4391 assert_ne!(events[0].event_id, UUID4::default());
4392 assert_ne!(events[0].ts_event, UnixNanos::default());
4393 assert_eq!(events[0].ts_init, events[0].ts_event);
4394 drop(events);
4395 msgbus::get_message_bus().borrow_mut().dispose();
4396 }
4397
4398 #[rstest]
4399 #[case::accepted(
4400 ReconnectRequestOutcome::Accepted,
4401 SocketReconnectDispatchOutcome::Accepted
4402 )]
4403 #[case::already_reconnecting(
4404 ReconnectRequestOutcome::AlreadyReconnecting,
4405 SocketReconnectDispatchOutcome::AlreadyReconnecting
4406 )]
4407 #[case::disconnected(
4408 ReconnectRequestOutcome::Disconnected,
4409 SocketReconnectDispatchOutcome::Disconnected
4410 )]
4411 #[case::closed(
4412 ReconnectRequestOutcome::Closed,
4413 SocketReconnectDispatchOutcome::Closed
4414 )]
4415 #[case::unsupported(
4416 ReconnectRequestOutcome::Unsupported,
4417 SocketReconnectDispatchOutcome::Unsupported
4418 )]
4419 fn test_request_socket_reconnect_maps_transport_outcome(
4420 #[case] transport: ReconnectRequestOutcome,
4421 #[case] expected: SocketReconnectDispatchOutcome,
4422 ) {
4423 let registry = SocketReconnectRegistry::default();
4424 let client_id = ClientId::from("TEST");
4425 let endpoint = Ustr::from("test-streams");
4426 let control = SocketControl::with_registry(client_id, None, endpoint, ®istry);
4427 let _sink = control.sink();
4428 control.register(move || transport);
4429
4430 let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4431
4432 assert_eq!(outcome, expected);
4433 }
4434
4435 #[rstest]
4436 #[case::client_not_found(
4437 SocketReconnectLookup::ClientNotFound,
4438 SocketReconnectDispatchOutcome::UnknownClient
4439 )]
4440 #[case::unsupported(
4441 SocketReconnectLookup::Unsupported,
4442 SocketReconnectDispatchOutcome::Unsupported
4443 )]
4444 #[case::endpoint_not_found(
4445 SocketReconnectLookup::EndpointNotFound,
4446 SocketReconnectDispatchOutcome::UnknownEndpoint
4447 )]
4448 #[case::ambiguous(
4449 SocketReconnectLookup::AmbiguousEndpoint,
4450 SocketReconnectDispatchOutcome::AmbiguousEndpoint
4451 )]
4452 fn test_request_socket_reconnect_maps_lookup_failure(
4453 #[case] lookup: SocketReconnectLookup,
4454 #[case] expected: SocketReconnectDispatchOutcome,
4455 ) {
4456 assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4457 }
4458
4459 #[rstest]
4460 fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4461 let trader_id = TraderId::from("SOCKET-001");
4462
4463 let config = LiveNodeConfig {
4464 trader_id,
4465 ..Default::default()
4466 };
4467
4468 let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4469 let client_id = ClientId::from("TEST");
4470 let endpoint = Ustr::from("test-streams");
4471 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4472 let sender = DispatchSender::new(tx);
4473 let events = sender.clone();
4474 msgbus::subscribe_any(
4475 MessagingSwitchboard::socket_state_changed_pattern(
4476 Some(client_id),
4477 Some(endpoint.as_str()),
4478 ),
4479 ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4480 events.send(17u32).unwrap();
4481 }),
4482 None,
4483 );
4484
4485 let control =
4486 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4487 let _sink = control.sink();
4488 control.register(move || {
4489 sender.send(23u32).unwrap();
4490 ReconnectRequestOutcome::Accepted
4491 });
4492
4493 let event = SystemEvent::SocketState(SocketStateChange::new(
4494 client_id,
4495 None,
4496 endpoint,
4497 SocketState::Connected,
4498 ));
4499 let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4500 trader_id,
4501 client_id,
4502 endpoint,
4503 31.into(),
4504 ));
4505
4506 if startup {
4507 let mut pending = PendingEvents::default();
4508 pending.system_events.push(event.into());
4509 pending.system_commands.push(command.into());
4510 node.process_system_events(pending.take_system_events());
4511 node.process_system_commands(pending.take_system_commands());
4512 assert!(pending.is_empty());
4513 } else {
4514 node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4515 node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4516 }
4517
4518 let event_child = rx.try_recv().unwrap();
4519 let command_child = rx.try_recv().unwrap();
4520 assert!(event_child.is_rooted());
4521 assert!(command_child.is_rooted());
4522 assert_eq!(event_child.dispatch(|value| value), 17);
4523 assert_eq!(command_child.dispatch(|value| value), 23);
4524 assert!(rx.is_empty());
4525 msgbus::get_message_bus().borrow_mut().dispose();
4526 }
4527
4528 #[rstest]
4529 fn test_process_socket_reconnect_routes_only_matching_trader() {
4530 let trader_id = TraderId::from("SOCKET-001");
4531
4532 let config = LiveNodeConfig {
4533 trader_id,
4534 exec_engine: crate::config::LiveExecutionEngineConfig {
4535 reconciliation: false,
4536 ..Default::default()
4537 },
4538 ..Default::default()
4539 };
4540
4541 let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4542 let client_id = ClientId::from("TEST");
4543 let endpoint = Ustr::from("test-streams");
4544 let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4545 let request_count = Arc::clone(&requests);
4546 let control =
4547 SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4548 let _sink = control.sink();
4549 control.register(move || {
4550 request_count.fetch_add(1, Ordering::SeqCst);
4551 ReconnectRequestOutcome::Accepted
4552 });
4553
4554 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4555 TraderId::from("OTHER-001"),
4556 client_id,
4557 endpoint,
4558 UnixNanos::default(),
4559 )));
4560 assert_eq!(requests.load(Ordering::SeqCst), 0);
4561
4562 node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4563 trader_id,
4564 client_id,
4565 endpoint,
4566 UnixNanos::default(),
4567 )));
4568 assert_eq!(requests.load(Ordering::SeqCst), 1);
4569 }
4570
4571 #[rstest]
4572 #[tokio::test]
4573 async fn test_start_publishes_socket_change_after_actor_subscribes() {
4574 let config = LiveNodeConfig {
4575 trader_id: TraderId::from("SOCKET-STARTUP-001"),
4576 exec_engine: crate::config::LiveExecutionEngineConfig {
4577 reconciliation: false,
4578 ..Default::default()
4579 },
4580 timeout_connection: Duration::ZERO,
4581 timeout_reconciliation: Duration::ZERO,
4582 timeout_portfolio: Duration::ZERO,
4583 timeout_disconnection: Duration::ZERO,
4584 delay_post_stop: Duration::ZERO,
4585 timeout_shutdown: Duration::ZERO,
4586 ..Default::default()
4587 };
4588
4589 let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4590 let received = Rc::new(RefCell::new(Vec::new()));
4591 node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4592 .unwrap();
4593 node.runner.as_ref().unwrap().bind_senders();
4594
4595 let change = SocketStateChange::new(
4596 ClientId::from("BINANCE"),
4597 Some(Venue::from("BINANCE")),
4598 Ustr::from("binance-futures-market-streams"),
4599 SocketState::Connected,
4600 );
4601 get_system_event_sender()
4602 .send(SystemEvent::SocketState(change))
4603 .unwrap();
4604
4605 node.start().await.unwrap();
4606
4607 {
4608 let events = received.borrow();
4609 assert_eq!(events.len(), 1);
4610 assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4611 assert_eq!(events[0].client_id, change.client_id);
4612 assert_eq!(events[0].venue, change.venue);
4613 assert_eq!(events[0].endpoint, change.endpoint);
4614 assert_eq!(events[0].state, change.state);
4615 assert_ne!(events[0].event_id, UUID4::default());
4616 assert_ne!(events[0].ts_event, UnixNanos::default());
4617 assert_eq!(events[0].ts_init, events[0].ts_event);
4618 }
4619
4620 node.stop().await.unwrap();
4621 node.dispose();
4622 }
4623
4624 #[rstest]
4625 #[tokio::test(flavor = "current_thread")]
4626 async fn test_run_publishes_queue_state_after_dispatch_sample() {
4627 let config = LiveNodeConfig {
4628 trader_id: TraderId::from("QUEUE-RUN-001"),
4629 queue_monitor: Some(crate::config::QueueMonitorConfig {
4630 queue_depth_trigger: usize::MAX,
4631 queue_depth_clear: 0,
4632 mean_dispatch_ns_trigger: 1,
4633 mean_dispatch_ns_clear: 0,
4634 }),
4635 exec_engine: crate::config::LiveExecutionEngineConfig {
4636 reconciliation: false,
4637 ..Default::default()
4638 },
4639 delay_post_stop: Duration::ZERO,
4640 ..Default::default()
4641 };
4642
4643 let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4644 let handle = node.handle();
4645 let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4646
4647 let handler = ShareableMessageHandler::from_typed({
4648 let received = received.clone();
4649 let stop_handle = handle.clone();
4650
4651 move |event: &QueueStateChanged| {
4652 received.borrow_mut().push(event.clone());
4653 stop_handle.stop();
4654 }
4655 });
4656
4657 msgbus::subscribe_any(
4658 MessagingSwitchboard::queue_state_changed_pattern(None),
4659 handler,
4660 None,
4661 );
4662 let drive_handle = handle.clone();
4663
4664 let result = tokio::time::timeout(Duration::from_secs(5), async {
4665 let run = node.run();
4666 tokio::pin!(run);
4667
4668 let drive = async move {
4669 wait_until_async(
4670 || async { drive_handle.is_running() },
4671 Duration::from_secs(2),
4672 )
4673 .await;
4674 get_data_event_sender().send(stub_data_event()).unwrap();
4675 };
4676
4677 let (run_result, ()) = tokio::join!(run, drive);
4678 run_result
4679 })
4680 .await;
4681
4682 assert!(
4683 result.is_ok(),
4684 "queue state event should arrive before timeout"
4685 );
4686 assert!(result.unwrap().is_ok(), "run() should succeed");
4687 let events = received.borrow();
4688 assert_eq!(events.len(), 1);
4689 assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4690 assert_eq!(events[0].channel, SystemChannel::DataEvents);
4691 assert_eq!(events[0].condition, QueueCondition::Slow);
4692 assert_eq!(events[0].state, QueueState::Triggered);
4693 assert_eq!(events[0].queue_depth, 0);
4694 assert!(events[0].mean_dispatch_ns > 0);
4695 assert_ne!(events[0].event_id, UUID4::default());
4696 assert_ne!(events[0].ts_event, UnixNanos::default());
4697 assert_eq!(events[0].ts_init, events[0].ts_event);
4698 drop(events);
4699 msgbus::get_message_bus().borrow_mut().dispose();
4700 }
4701
4702 #[rstest]
4703 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4704 async fn test_run_dispatch_progress_under_load(
4705 #[values(false, true)] execution: bool,
4706 #[values(false, true)] slow: bool,
4707 ) {
4708 let config = LiveNodeConfig {
4709 delay_post_stop: Duration::ZERO,
4710 ..Default::default()
4711 };
4712 let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4713 let handle = node.handle();
4714 let stop = handle.clone();
4715 let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4716 if slow {
4717 std::thread::sleep(Duration::from_millis(110));
4719 }
4720
4721 if event.ts_init == UnixNanos::from(8) {
4722 let snapshot = stop.metrics_snapshot();
4724 assert_eq!(snapshot.time_events.dispatched, 7);
4725
4726 if execution {
4727 assert!((1..16).contains(&snapshot.exec_events.dispatched));
4728 assert_eq!(snapshot.data_events.dispatched, 0);
4729 } else {
4730 assert!(snapshot.data_events.dispatched > 0);
4731 assert_eq!(snapshot.data_commands.dispatched, 1);
4732 }
4733 stop.stop();
4734 }
4735 }));
4736
4737 let drive = async {
4738 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4739
4740 for ts in (1..=16_u64).map(UnixNanos::from) {
4741 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4742 TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4743 callback.clone(),
4744 ));
4745 get_data_event_sender().send(stub_data_event()).unwrap();
4746
4747 if execution {
4748 get_exec_event_sender().send(stub_exec_event()).unwrap();
4749 }
4750 }
4751 nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4752 };
4753 tokio::time::timeout(Duration::from_secs(5), async {
4754 let (result, ()) = tokio::join!(node.run(), drive);
4755 result.unwrap();
4756 })
4757 .await
4758 .unwrap();
4759 assert!(handle.should_stop());
4760 }
4761
4762 #[rstest]
4763 fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4764 let config = LiveNodeConfig {
4765 exec_engine: crate::config::LiveExecutionEngineConfig {
4766 reconciliation: false,
4767 ..Default::default()
4768 },
4769 ..Default::default()
4770 };
4771
4772 let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4773 let event = stub_exec_event();
4774 let account_id = AccountId::from("TEST-001");
4775 let instrument_id = InstrumentId::from("TEST.VENUE");
4776 let trade_id = TradeId::from("T-001");
4777
4778 let close_ids = node.observe_exec_event_before_dispatch(&event);
4779 assert_eq!(close_ids, Some(Vec::new()));
4780 assert!(
4781 !node
4782 .exec_manager
4783 .is_fill_recently_processed(account_id, instrument_id, trade_id)
4784 );
4785
4786 node.exec_manager
4787 .mark_fill_processed(account_id, instrument_id, trade_id);
4788
4789 let close_ids = node.observe_exec_event_before_dispatch(&event);
4790 assert_eq!(close_ids, None);
4791 }
4792
4793 #[rstest]
4794 #[case(false, false, OrderStatus::Canceled, 1)]
4795 #[case(false, true, OrderStatus::Accepted, 0)]
4796 #[case(true, false, OrderStatus::Canceled, 1)]
4797 #[case(true, true, OrderStatus::Accepted, 0)]
4798 fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4799 #[case] with_fills: bool,
4800 #[case] superseded: bool,
4801 #[case] expected_status: OrderStatus,
4802 #[case] expected_query_count: usize,
4803 ) {
4804 let config = LiveNodeConfig {
4805 exec_engine: crate::config::LiveExecutionEngineConfig {
4806 reconciliation: true,
4807 open_check_threshold_ms: 5_000,
4808 single_order_query_delay_ms: 0,
4809 ..Default::default()
4810 },
4811 ..Default::default()
4812 };
4813
4814 let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4815 let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4816 let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4817 let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4818 let account_id = AccountId::from("TEST-001");
4819 let client_id = ClientId::from("TEST");
4820 let instrument = crypto_perpetual_ethusdt();
4821 let instrument_id = instrument.id();
4822 let account = AccountAny::Margin(MarginAccount::new(
4823 AccountState::new(
4824 account_id,
4825 AccountType::Margin,
4826 vec![AccountBalance::new(
4827 Money::from("1000000 USDT"),
4828 Money::from("0 USDT"),
4829 Money::from("1000000 USDT"),
4830 )],
4831 Vec::new(),
4832 true,
4833 UUID4::new(),
4834 UnixNanos::default(),
4835 UnixNanos::default(),
4836 Some(Currency::USDT()),
4837 ),
4838 true,
4839 ));
4840 node.kernel.cache.borrow_mut().add_account(account).unwrap();
4841 node.kernel
4842 .cache
4843 .borrow_mut()
4844 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4845 .unwrap();
4846 insert_accepted_limit_order_in_node(
4847 &node,
4848 account_id,
4849 client_id,
4850 instrument_id,
4851 client_order_id,
4852 old_venue_order_id,
4853 );
4854
4855 if superseded {
4856 let order = node
4857 .kernel
4858 .cache
4859 .borrow()
4860 .order_owned(&client_order_id)
4861 .unwrap();
4862 let pending_update = OrderPendingUpdateSpec::builder()
4863 .trader_id(order.trader_id())
4864 .strategy_id(order.strategy_id())
4865 .instrument_id(order.instrument_id())
4866 .client_order_id(client_order_id)
4867 .account_id(account_id)
4868 .venue_order_id(old_venue_order_id)
4869 .build();
4870 node.kernel
4871 .cache
4872 .borrow_mut()
4873 .update_order(&OrderEventAny::PendingUpdate(pending_update))
4874 .unwrap();
4875 let order = node
4876 .kernel
4877 .cache
4878 .borrow()
4879 .order_owned(&client_order_id)
4880 .unwrap();
4881 let updated = OrderUpdatedSpec::builder()
4882 .trader_id(order.trader_id())
4883 .strategy_id(order.strategy_id())
4884 .instrument_id(order.instrument_id())
4885 .client_order_id(client_order_id)
4886 .quantity(order.quantity())
4887 .venue_order_id(new_venue_order_id)
4888 .account_id(account_id)
4889 .build();
4890 node.kernel
4891 .cache
4892 .borrow_mut()
4893 .update_order(&OrderEventAny::Updated(updated))
4894 .unwrap();
4895 }
4896
4897 let report = OrderStatusReport::new(
4898 account_id,
4899 instrument_id,
4900 Some(client_order_id),
4901 old_venue_order_id,
4902 OrderSide::Buy.into(),
4903 OrderType::Limit,
4904 TimeInForce::Gtc,
4905 OrderStatus::Canceled,
4906 Quantity::from("10.0"),
4907 Quantity::from("0.0"),
4908 UnixNanos::from(1_000),
4909 UnixNanos::from(2_000),
4910 UnixNanos::from(3_000),
4911 None,
4912 );
4913
4914 let report = if with_fills {
4915 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4916 } else {
4917 ExecutionReport::Order(Box::new(report))
4918 };
4919
4920 let event = ExecutionEvent::Report(report);
4921
4922 node.process_exec_event(event);
4923
4924 let order = node
4925 .kernel
4926 .cache
4927 .borrow()
4928 .order_owned(&client_order_id)
4929 .unwrap();
4930
4931 let expected_venue_order_id = if superseded {
4932 new_venue_order_id
4933 } else {
4934 old_venue_order_id
4935 };
4936
4937 assert_eq!(order.status(), expected_status);
4938 assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4939
4940 if !superseded {
4941 let replacement = OrderTestBuilder::new(OrderType::Limit)
4942 .client_order_id(client_order_id)
4943 .instrument_id(instrument_id)
4944 .quantity(Quantity::from("20.0"))
4945 .price(Price::from("200.0"))
4946 .build();
4947 let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4948 node.kernel
4949 .cache
4950 .borrow_mut()
4951 .add_order(replacement, None, Some(client_id), true)
4952 .unwrap();
4953 let replacement = node
4954 .kernel
4955 .cache
4956 .borrow_mut()
4957 .update_order(&submitted)
4958 .unwrap();
4959 let accepted =
4960 TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4961 node.kernel
4962 .cache
4963 .borrow_mut()
4964 .update_order(&accepted)
4965 .unwrap();
4966 }
4967
4968 assert_eq!(
4969 node.exec_manager.check_open_order_queries().len(),
4970 expected_query_count,
4971 );
4972 }
4973
4974 #[rstest]
4975 fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4976 let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4977
4978 let OrderEventAny::Filled(fill) = &mut fill_event else {
4979 unreachable!();
4980 };
4981
4982 fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4983 fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4984 let fill = fill.clone();
4985 let report_event = fill_report_event(&fill);
4986 let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4987
4988 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4989 let marked_before_dispatch = is_recent_fill(&node, &fill);
4990
4991 node.dispatch_exec_event_and_commit_fill(event);
4992
4993 let marked_after_dispatch = is_recent_fill(&node, &fill);
4994 let later_report_is_eligible = node
4995 .observe_exec_event_before_dispatch(&report_event)
4996 .is_some();
4997 assert_eq!(
4998 (
4999 marked_before_dispatch,
5000 marked_after_dispatch,
5001 later_report_is_eligible,
5002 ),
5003 (false, false, true),
5004 );
5005 }
5006
5007 #[rstest]
5008 fn test_applied_direct_fill_commits_and_skips_later_report() {
5009 let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
5010
5011 let OrderEventAny::Filled(fill) = &fill_event else {
5012 unreachable!();
5013 };
5014
5015 let fill = fill.clone();
5016 let report_event = fill_report_event(&fill);
5017 let event = ExecutionEvent::Order(fill_event);
5018
5019 assert!(node.observe_exec_event_before_dispatch(&event).is_some());
5020 assert!(!is_recent_fill(&node, &fill));
5021
5022 node.dispatch_exec_event_and_commit_fill(event);
5023
5024 assert!(is_recent_fill(&node, &fill));
5025 assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
5026 }
5027
5028 #[rstest]
5029 fn test_canonical_duplicate_fill_counts_as_applied() {
5030 let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
5031
5032 let OrderEventAny::Filled(fill) = &fill_event else {
5033 unreachable!();
5034 };
5035
5036 let mut fill = fill.clone();
5037 node.kernel
5038 .cache
5039 .borrow_mut()
5040 .update_order(&fill_event)
5041 .unwrap();
5042 fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
5043
5044 node.exec_manager.commit_recent_fill_if_applied(&fill);
5045
5046 assert!(is_recent_fill(&node, &fill));
5047 }
5048
5049 #[rstest]
5050 #[case(false)]
5051 #[case(true)]
5052 fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
5053 let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
5054 "AppliedContinuousFillNode"
5055 } else {
5056 "RejectedContinuousFillNode"
5057 });
5058
5059 if !applied {
5060 let OrderEventAny::Filled(fill) = &mut fill_event else {
5061 unreachable!();
5062 };
5063
5064 fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5065 fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5066 }
5067
5068 let OrderEventAny::Filled(fill) = &fill_event else {
5069 unreachable!();
5070 };
5071
5072 let fill = fill.clone();
5073
5074 node.process_reconciliation_events(&[fill_event]);
5075
5076 assert_eq!(is_recent_fill(&node, &fill), applied);
5077 }
5078
5079 #[rstest]
5080 fn test_recent_fill_commit_requires_account_and_instrument_match() {
5081 let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5082 node.kernel
5083 .cache
5084 .borrow_mut()
5085 .update_order(&fill_event)
5086 .unwrap();
5087
5088 let OrderEventAny::Filled(fill) = fill_event else {
5089 unreachable!();
5090 };
5091
5092 let mut account_mismatch = fill.clone();
5093 account_mismatch.account_id = AccountId::from("OTHER-001");
5094 let mut instrument_mismatch = fill;
5095 instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5096
5097 node.exec_manager
5098 .commit_recent_fill_if_applied(&account_mismatch);
5099 node.exec_manager
5100 .commit_recent_fill_if_applied(&instrument_mismatch);
5101
5102 assert!(!is_recent_fill(&node, &account_mismatch));
5103 assert!(!is_recent_fill(&node, &instrument_mismatch));
5104 }
5105
5106 #[rstest]
5107 fn test_applied_inferred_fill_remains_recently_processed() {
5108 let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5109 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5110 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5111 let account_id = AccountId::from("TEST-001");
5112 let order = node
5113 .kernel
5114 .cache
5115 .borrow()
5116 .order_owned(&client_order_id)
5117 .unwrap();
5118 let report = OrderStatusReport::new(
5119 account_id,
5120 instrument.id(),
5121 Some(client_order_id),
5122 venue_order_id,
5123 OrderSide::Buy.into(),
5124 OrderType::Limit,
5125 TimeInForce::Gtc,
5126 OrderStatus::PartiallyFilled,
5127 Quantity::from("10.0"),
5128 Quantity::from("1.0"),
5129 UnixNanos::from(1_000),
5130 UnixNanos::from(1_000),
5131 UnixNanos::from(1_000),
5132 None,
5133 )
5134 .with_avg_px(dec!(100.0));
5135 let inferred = create_inferred_fill_for_qty(
5136 &order,
5137 &report,
5138 &account_id,
5139 &instrument,
5140 Quantity::from("1.0"),
5141 UnixNanos::from(1_000),
5142 None,
5143 )
5144 .unwrap();
5145
5146 let OrderEventAny::Filled(fill) = &inferred else {
5147 unreachable!();
5148 };
5149
5150 let fill = fill.clone();
5151
5152 node.process_reconciliation_events(&[inferred]);
5153
5154 assert!(fill.reconciliation);
5155 assert!(is_recent_fill(&node, &fill));
5156 }
5157
5158 #[rstest]
5159 fn test_drain_exec_events_before_reconciliation_applies_real_fill_first() {
5160 let (mut node, fill_event, instrument) = recent_fill_test_fixture("DeferredFillNode");
5161
5162 let OrderEventAny::Filled(fill) = fill_event else {
5163 unreachable!();
5164 };
5165
5166 let account_id = fill.account_id;
5167 let venue_order_id = fill.venue_order_id;
5168 let client_id = ClientId::from("TEST-EVIDENCE");
5169 node.kernel
5170 .exec_engine
5171 .borrow_mut()
5172 .register_client(Box::new(StubExecutionClient::new(
5173 client_id,
5174 account_id,
5175 instrument.id().venue,
5176 OmsType::Netting,
5177 None,
5178 )))
5179 .unwrap();
5180
5181 let report = FillReport::new(
5182 account_id,
5183 instrument.id(),
5184 venue_order_id,
5185 fill.trade_id,
5186 fill.order_side,
5187 fill.last_qty,
5188 fill.last_px,
5189 fill.commission.unwrap(),
5190 fill.liquidity_side,
5191 Some(fill.client_order_id),
5192 None,
5193 fill.ts_event,
5194 fill.ts_init,
5195 None,
5196 );
5197 let (exec_evt_tx, mut exec_evt_rx) =
5198 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
5199 exec_evt_tx
5200 .send(ExecutionEvent::Report(ExecutionReport::Fill(Box::new(report))).into())
5201 .unwrap();
5202
5203 node.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
5204
5205 assert!(exec_evt_rx.try_recv().is_err());
5206 let order = node
5207 .kernel
5208 .cache
5209 .borrow()
5210 .order_owned(&fill.client_order_id)
5211 .unwrap();
5212 assert_eq!(order.filled_qty(), fill.last_qty);
5213 assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5214
5215 let report = OrderStatusReport::new(
5216 account_id,
5217 instrument.id(),
5218 Some(fill.client_order_id),
5219 venue_order_id,
5220 OrderSide::Buy.into(),
5221 OrderType::Limit,
5222 TimeInForce::Gtc,
5223 OrderStatus::PartiallyFilled,
5224 Quantity::from("10.0"),
5225 fill.last_qty,
5226 UnixNanos::from(1_000),
5227 UnixNanos::from(1_000),
5228 UnixNanos::from(1_000),
5229 None,
5230 )
5231 .with_avg_px(dec!(100.0));
5232 let engine = node.kernel.exec_engine.borrow();
5233 let clients = engine.get_all_clients();
5234 let check = node
5235 .exec_manager
5236 .prepare_open_order_report_check(UUID4::new(), &clients);
5237 let reconciliation = node.exec_manager.reconcile_open_order_reports(
5238 &check,
5239 vec![SourcedOrderStatusReport { client_id, report }],
5240 &IndexSet::new(),
5241 &IndexSet::new(),
5242 &clients,
5243 );
5244 drop(engine);
5245 node.process_reconciliation_events(&reconciliation.events);
5246
5247 let order = node
5248 .kernel
5249 .cache
5250 .borrow()
5251 .order_owned(&fill.client_order_id)
5252 .unwrap();
5253 assert_eq!(order.filled_qty(), fill.last_qty);
5254 assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5255 }
5256
5257 #[rstest]
5258 #[tokio::test]
5259 async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5260 let client_id = ClientId::from("POSITION-FILLS");
5261 let account_id = AccountId::from("POSITION-FILLS-001");
5262 let instrument_id = crypto_perpetual_ethusdt().id();
5263 let commands = Rc::new(RefCell::new(Vec::new()));
5264
5265 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5266 client_id,
5267 account_id,
5268 venue: instrument_id.venue,
5269 outcome: FillReportClientOutcome::Failure,
5270 commands: commands.clone(),
5271 }));
5272
5273 let command = GenerateFillReports::new(
5274 UUID4::new(),
5275 UnixNanos::from(2_000),
5276 Some(instrument_id),
5277 None,
5278 Some(UnixNanos::from(1_000)),
5279 Some(UnixNanos::from(2_000)),
5280 None,
5281 None,
5282 );
5283
5284 let result = request_position_fill_reports(
5285 vec![client],
5286 vec![PositionFillReportQuery {
5287 key: (instrument_id, account_id),
5288 client_id,
5289 command: command.clone(),
5290 }],
5291 )
5292 .await
5293 .finish();
5294
5295 assert_eq!(*commands.borrow(), vec![command]);
5296 assert!(result.is_empty());
5297 }
5298
5299 #[rstest]
5300 #[tokio::test]
5301 async fn test_request_position_fill_reports_accepts_scoped_response() {
5302 let client_id = ClientId::from("POSITION-FILLS");
5303 let account_id = AccountId::from("POSITION-FILLS-001");
5304 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5305 let instrument_id = instrument.id();
5306
5307 let report_b = FillReport::new(
5308 account_id,
5309 instrument_id,
5310 VenueOrderId::from("V-POSITION-FILLS"),
5311 TradeId::from("T-POSITION-FILLS-B"),
5312 OrderSide::Buy,
5313 Quantity::from("1.0"),
5314 Price::from("100.0"),
5315 Money::zero(instrument.quote_currency()),
5316 LiquiditySide::Taker,
5317 Some(ClientOrderId::from("O-POSITION-FILLS")),
5318 None,
5319 UnixNanos::from(1_500),
5320 UnixNanos::from(2_000),
5321 None,
5322 );
5323 let mut report_a = report_b.clone();
5324 report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5325 let commands = Rc::new(RefCell::new(Vec::new()));
5326
5327 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5328 client_id,
5329 account_id,
5330 venue: instrument_id.venue,
5331 outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5332 commands,
5333 }));
5334
5335 let command = GenerateFillReports::new(
5336 UUID4::new(),
5337 UnixNanos::from(2_000),
5338 Some(instrument_id),
5339 None,
5340 Some(UnixNanos::from(1_000)),
5341 Some(UnixNanos::from(2_000)),
5342 None,
5343 None,
5344 );
5345 let key = (instrument_id, account_id);
5346
5347 let result = request_position_fill_reports(
5348 vec![client],
5349 vec![PositionFillReportQuery {
5350 key,
5351 client_id,
5352 command,
5353 }],
5354 )
5355 .await
5356 .finish();
5357
5358 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5359 assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5360 }
5361
5362 #[rstest]
5363 #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5364 #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5365 #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5366 #[tokio::test]
5367 async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5368 #[case] report_account: &str,
5369 #[case] report_instrument: &str,
5370 #[case] ts_event: u64,
5371 #[case] quantity: &str,
5372 ) {
5373 let client_id = ClientId::from("POSITION-FILLS");
5374 let account_id = AccountId::from("POSITION-FILLS-001");
5375 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5376 let instrument_id = instrument.id();
5377
5378 let report = FillReport::new(
5379 AccountId::from(report_account),
5380 InstrumentId::from(report_instrument),
5381 VenueOrderId::from("V-POSITION-FILLS"),
5382 TradeId::from("T-POSITION-FILLS"),
5383 OrderSide::Buy,
5384 Quantity::from(quantity),
5385 Price::from("100.0"),
5386 Money::zero(instrument.quote_currency()),
5387 LiquiditySide::Taker,
5388 Some(ClientOrderId::from("O-POSITION-FILLS")),
5389 None,
5390 UnixNanos::from(ts_event),
5391 UnixNanos::from(2_000),
5392 None,
5393 );
5394
5395 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5396 client_id,
5397 account_id,
5398 venue: instrument_id.venue,
5399 outcome: FillReportClientOutcome::Reports(vec![report]),
5400 commands: Rc::new(RefCell::new(Vec::new())),
5401 }));
5402
5403 let command = GenerateFillReports::new(
5404 UUID4::new(),
5405 UnixNanos::from(2_000),
5406 Some(instrument_id),
5407 None,
5408 Some(UnixNanos::from(1_000)),
5409 Some(UnixNanos::from(2_000)),
5410 None,
5411 None,
5412 );
5413 let key = (instrument_id, account_id);
5414
5415 let result = request_position_fill_reports(
5416 vec![client],
5417 vec![PositionFillReportQuery {
5418 key,
5419 client_id,
5420 command,
5421 }],
5422 )
5423 .await
5424 .finish();
5425
5426 assert!(result.is_empty());
5427 }
5428
5429 #[rstest]
5430 #[case(999)]
5431 #[case(2_001)]
5432 #[tokio::test]
5433 async fn test_request_position_fill_reports_filters_time_window_superset(
5434 #[case] ts_event: u64,
5435 ) {
5436 let client_id = ClientId::from("POSITION-FILLS");
5437 let account_id = AccountId::from("POSITION-FILLS-001");
5438 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5439 let instrument_id = instrument.id();
5440
5441 let report = FillReport::new(
5442 account_id,
5443 instrument_id,
5444 VenueOrderId::from("V-POSITION-FILLS"),
5445 TradeId::from("T-POSITION-FILLS"),
5446 OrderSide::Buy,
5447 Quantity::from("1.0"),
5448 Price::from("100.0"),
5449 Money::zero(instrument.quote_currency()),
5450 LiquiditySide::Taker,
5451 Some(ClientOrderId::from("O-POSITION-FILLS")),
5452 None,
5453 UnixNanos::from(ts_event),
5454 UnixNanos::from(2_000),
5455 None,
5456 );
5457
5458 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5459 client_id,
5460 account_id,
5461 venue: instrument_id.venue,
5462 outcome: FillReportClientOutcome::Reports(vec![report]),
5463 commands: Rc::new(RefCell::new(Vec::new())),
5464 }));
5465
5466 let command = GenerateFillReports::new(
5467 UUID4::new(),
5468 UnixNanos::from(2_000),
5469 Some(instrument_id),
5470 None,
5471 Some(UnixNanos::from(1_000)),
5472 Some(UnixNanos::from(2_000)),
5473 None,
5474 None,
5475 );
5476 let key = (instrument_id, account_id);
5477
5478 let result = request_position_fill_reports(
5479 vec![client],
5480 vec![PositionFillReportQuery {
5481 key,
5482 client_id,
5483 command,
5484 }],
5485 )
5486 .await
5487 .finish();
5488
5489 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5490 assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5491 }
5492
5493 #[rstest]
5494 #[case::query_error(false)]
5495 #[case::missing_client(true)]
5496 #[tokio::test]
5497 async fn test_position_fill_collection_discards_partial_failure(
5498 #[case] missing_client: bool,
5499 #[values(false, true)] failure_first: bool,
5500 position_fill_query: (FillReport, PositionFillReportQuery),
5501 ) {
5502 let (report, query) = position_fill_query;
5503 let failed_client_id = ClientId::from("FAILED-FILLS");
5504 let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5505 let mut healthy_report = report.clone();
5506 healthy_report.account_id = healthy_key.1;
5507 let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5508 let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5509 let empty_client_id = ClientId::from("EMPTY-FILLS");
5510 let mut clients = vec![
5511 LiveExecutionClient::new(Box::new(FillReportClient {
5512 client_id: query.client_id,
5513 account_id: query.key.1,
5514 venue: query.key.0.venue,
5515 outcome: FillReportClientOutcome::Reports(vec![report]),
5516 commands: Rc::new(RefCell::new(Vec::new())),
5517 })),
5518 LiveExecutionClient::new(Box::new(FillReportClient {
5519 client_id: healthy_client_id,
5520 account_id: healthy_key.1,
5521 venue: healthy_key.0.venue,
5522 outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5523 commands: Rc::new(RefCell::new(Vec::new())),
5524 })),
5525 LiveExecutionClient::new(Box::new(FillReportClient {
5526 client_id: empty_client_id,
5527 account_id: empty_key.1,
5528 venue: empty_key.0.venue,
5529 outcome: FillReportClientOutcome::Reports(Vec::new()),
5530 commands: Rc::new(RefCell::new(Vec::new())),
5531 })),
5532 ];
5533
5534 if !missing_client {
5535 clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5536 client_id: failed_client_id,
5537 account_id: query.key.1,
5538 venue: query.key.0.venue,
5539 outcome: FillReportClientOutcome::Failure,
5540 commands: Rc::new(RefCell::new(Vec::new())),
5541 })));
5542 }
5543
5544 let failed_query = PositionFillReportQuery {
5545 client_id: failed_client_id,
5546 key: query.key,
5547 command: query.command.clone(),
5548 };
5549
5550 let healthy_query = PositionFillReportQuery {
5551 key: healthy_key,
5552 client_id: healthy_client_id,
5553 command: query.command.clone(),
5554 };
5555
5556 let empty_query = PositionFillReportQuery {
5557 key: empty_key,
5558 client_id: empty_client_id,
5559 command: query.command.clone(),
5560 };
5561
5562 let mut queries = vec![query, failed_query];
5563
5564 if failure_first {
5565 queries.reverse();
5566 }
5567
5568 queries.insert(0, empty_query);
5569 queries.push(healthy_query);
5570
5571 let result = request_position_fill_reports(clients, queries)
5572 .await
5573 .finish();
5574
5575 assert_eq!(
5576 result.keys().copied().collect::<Vec<_>>(),
5577 vec![empty_key, healthy_key]
5578 );
5579 assert_eq!(
5580 result,
5581 IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5582 );
5583 }
5584
5585 #[rstest]
5586 #[case::same_economics("metadata", true)]
5587 #[case::venue_order("venue_order", false)]
5588 #[case::side("side", false)]
5589 #[case::quantity("quantity", false)]
5590 #[case::price("price", false)]
5591 #[case::commission("commission", false)]
5592 #[case::liquidity("liquidity", false)]
5593 #[case::average("average", false)]
5594 #[case::event_time("event_time", false)]
5595 #[case::client_order("client_order", false)]
5596 #[case::position("position", false)]
5597 #[tokio::test]
5598 async fn test_position_fill_collection_validates_duplicate_economics(
5599 #[case] changed: &str,
5600 #[case] accepted: bool,
5601 position_fill_query: (FillReport, PositionFillReportQuery),
5602 ) {
5603 let (report, query) = position_fill_query;
5604 let mut duplicate = report.clone();
5605
5606 match changed {
5607 "metadata" => {
5608 duplicate.report_id = UUID4::new();
5609 duplicate.ts_init = UnixNanos::from(3_000);
5610 }
5611 "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5612 "side" => duplicate.order_side = OrderSide::Sell,
5613 "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5614 "price" => duplicate.last_px = Price::from("101.0"),
5615 "commission" => duplicate.commission = Money::from("0.25 USDT"),
5616 "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5617 "average" => duplicate.avg_px = Some(dec!(100.5)),
5618 "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5619 "client_order" => duplicate.client_order_id = None,
5620 "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5621 _ => unreachable!(),
5622 }
5623
5624 let key = query.key;
5625
5626 let client = LiveExecutionClient::new(Box::new(FillReportClient {
5627 client_id: query.client_id,
5628 account_id: key.1,
5629 venue: key.0.venue,
5630 outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5631 commands: Rc::new(RefCell::new(Vec::new())),
5632 }));
5633
5634 let result = request_position_fill_reports(vec![client], vec![query])
5635 .await
5636 .finish();
5637
5638 if accepted {
5639 assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5640 assert_eq!(result, IndexMap::from([(key, vec![report])]));
5641 } else {
5642 assert!(result.is_empty());
5643 }
5644 }
5645
5646 #[fixture]
5647 fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5648 let account_id = AccountId::from("POSITION-FILLS-001");
5649 let instrument_id = crypto_perpetual_ethusdt().id();
5650
5651 let report = FillReport::new(
5652 account_id,
5653 instrument_id,
5654 VenueOrderId::from("V-POSITION-FILLS"),
5655 TradeId::from("T-POSITION-FILLS"),
5656 OrderSide::Buy,
5657 Quantity::from("1.0"),
5658 Price::from("100.0"),
5659 Money::from("0.10 USDT"),
5660 LiquiditySide::Taker,
5661 Some(ClientOrderId::from("O-POSITION-FILLS")),
5662 None,
5663 UnixNanos::from(1_500),
5664 UnixNanos::from(2_000),
5665 None,
5666 );
5667
5668 let query = PositionFillReportQuery {
5669 key: (instrument_id, account_id),
5670 client_id: ClientId::from("POSITION-FILLS"),
5671 command: GenerateFillReports::new(
5672 UUID4::new(),
5673 UnixNanos::from(2_000),
5674 Some(instrument_id),
5675 None,
5676 Some(UnixNanos::from(1_000)),
5677 Some(UnixNanos::from(2_000)),
5678 None,
5679 None,
5680 ),
5681 };
5682
5683 (report, query)
5684 }
5685
5686 #[rstest]
5687 #[case::failed_query(false)]
5688 #[case::local_activity(true)]
5689 fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5690 let (mut node, report, _) =
5691 position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5692 let active_key = (report.instrument_id, report.account_id);
5693 let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5694 let deferred_reports = vec![
5695 PositionStatusReport::new(
5696 deferred_key.1,
5697 deferred_key.0,
5698 PositionSide::Long,
5699 Quantity::from("2.0"),
5700 UnixNanos::from(1_000),
5701 UnixNanos::from(1_000),
5702 None,
5703 None,
5704 Some(dec!(100.0)),
5705 ),
5706 PositionStatusReport::new(
5707 deferred_key.1,
5708 deferred_key.0,
5709 PositionSide::Short,
5710 Quantity::from("1.0"),
5711 UnixNanos::from(1_000),
5712 UnixNanos::from(1_000),
5713 None,
5714 None,
5715 Some(dec!(100.0)),
5716 ),
5717 ];
5718 let mut check = node
5719 .exec_manager
5720 .prepare_position_report_check(UUID4::new(), &[]);
5721 check.client_coverage.clear();
5722 let events = node.exec_manager.reconcile_position_reports(
5723 &check,
5724 deferred_reports.clone(),
5725 &IndexSet::new(),
5726 &IndexSet::new(),
5727 );
5728 assert!(events.is_empty());
5729 assert_eq!(
5730 node.exec_manager.position_recon_retry_count(&deferred_key),
5731 1
5732 );
5733
5734 let mut position_result = position_report_result(&node, report);
5735 position_result.reports.extend(deferred_reports);
5736 position_result.check.client_coverage.insert(
5737 deferred_key,
5738 ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5739 );
5740 position_result
5741 .check
5742 .activity_revisions
5743 .insert(deferred_key, 0);
5744 position_result
5745 .queried_clients
5746 .insert(ClientId::from("SECOND"));
5747 let mut reports = IndexMap::from([(active_key, Vec::new())]);
5748 if local_activity {
5749 reports.insert(deferred_key, Vec::new());
5750 node.exec_manager
5751 .record_position_activity(deferred_key.0, deferred_key.1);
5752 }
5753
5754 node.handle_position_fill_report_result(PositionFillReportResult {
5755 position_result,
5756 reports,
5757 });
5758
5759 assert_eq!(
5760 node.exec_manager.position_recon_retry_count(&deferred_key),
5761 1
5762 );
5763 {
5764 let cache = node.kernel.cache.borrow();
5765 let positions =
5766 cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5767 assert_eq!(
5768 positions
5769 .iter()
5770 .map(|position| position.quantity)
5771 .collect::<Vec<_>>(),
5772 vec![Quantity::from("1.0"), Quantity::from("1.0")],
5773 );
5774 assert_eq!(
5775 cache
5776 .positions_open(
5777 None,
5778 Some(&deferred_key.0),
5779 None,
5780 Some(&deferred_key.1),
5781 None,
5782 )
5783 .len(),
5784 0
5785 );
5786 }
5787
5788 let mut fresh_check = node
5789 .exec_manager
5790 .prepare_position_report_check(UUID4::new(), &[]);
5791 node.exec_manager.plan_position_fill_reports(
5792 &mut fresh_check,
5793 &[],
5794 &IndexSet::new(),
5795 &IndexSet::new(),
5796 &[],
5797 );
5798
5799 assert_eq!(
5800 node.exec_manager.position_recon_retry_count(&deferred_key),
5801 0
5802 );
5803 }
5804
5805 #[rstest]
5806 fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5807 let (mut node, venue_report, fill_report) =
5808 position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5809 let key = (venue_report.instrument_id, venue_report.account_id);
5810 let revision = node.exec_manager.position_activity_revision(&key);
5811 let position_result = position_report_result(&node, venue_report);
5812
5813 node.handle_position_fill_report_result(PositionFillReportResult {
5814 position_result,
5815 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5816 });
5817
5818 let cache = node.kernel.cache.borrow();
5819 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5820 assert_eq!(
5821 positions
5822 .iter()
5823 .map(|position| position.quantity)
5824 .sum::<Quantity>(),
5825 Quantity::from("2.0")
5826 );
5827 assert_eq!(
5828 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5829 1
5830 );
5831 drop(positions);
5832 drop(cache);
5833 assert!(
5834 node.exec_manager
5835 .position_contains_fill_report(&fill_report)
5836 );
5837 assert_eq!(
5838 node.exec_manager.position_activity_revision(&key),
5839 revision + 1
5840 );
5841 }
5842
5843 #[rstest]
5844 fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5845 let (mut node, venue_report, mut first) =
5846 position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5847 first.commission = Money::from("0.13 USDT");
5848 let key = (first.instrument_id, first.account_id);
5849 let client_order_id = first.client_order_id.unwrap();
5850 let position_result = position_report_result(&node, venue_report.clone());
5851 node.handle_position_fill_report_result(PositionFillReportResult {
5852 position_result,
5853 reports: IndexMap::from([(key, vec![first.clone()])]),
5854 });
5855
5856 let mut second = first.clone();
5857 second.trade_id = TradeId::from("T-PREFIX-SECOND");
5858 second.last_qty = Quantity::from("0.75");
5859 second.commission = Money::from("0.29 USDT");
5860 let mut third = first.clone();
5861 third.trade_id = TradeId::from("T-PREFIX-THIRD");
5862 third.last_qty = Quantity::from("0.25");
5863 third.commission = Money::from("0.07 USDT");
5864 let revision = node.exec_manager.position_activity_revision(&key);
5865 let received = Rc::new(RefCell::new(Vec::new()));
5866 let cache = node.kernel.cache.clone();
5867 let callback_received = received.clone();
5868 let second_trade_id = second.trade_id;
5869
5870 let handler = TypedHandler::from(move |event: &OrderEventAny| {
5871 let OrderEventAny::Filled(fill) = event else {
5872 return;
5873 };
5874
5875 callback_received.borrow_mut().push(fill.trade_id);
5876 if fill.trade_id == second_trade_id {
5877 let mut cache = cache.borrow_mut();
5878 let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5879 let mut position = cache.position_mut(&position_id).unwrap();
5880 position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5881 }
5882 });
5883
5884 let strategy_id = node
5885 .kernel
5886 .cache
5887 .borrow()
5888 .order(&client_order_id)
5889 .unwrap()
5890 .strategy_id();
5891 let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5892 msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5893 let position_result = position_report_result(&node, venue_report);
5894 node.handle_position_fill_report_result(PositionFillReportResult {
5895 position_result,
5896 reports: IndexMap::from([(
5897 key,
5898 vec![first.clone(), first.clone(), second.clone(), third.clone()],
5899 )]),
5900 });
5901
5902 msgbus::unsubscribe_order_events(topic.into(), &handler);
5903 let cache = node.kernel.cache.borrow();
5904 let order = cache.order(&client_order_id).unwrap();
5905 let position_id = cache.position_id(&client_order_id).unwrap();
5906 let position = cache.position_ref(position_id).unwrap();
5907
5908 assert_eq!(*received.borrow(), vec![second.trade_id]);
5909 assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5910 assert_eq!(order.events().len(), 6);
5911 assert_eq!(
5912 order.trade_ids(),
5913 vec![
5914 &TradeId::from("T-POSITION-INITIAL"),
5915 &first.trade_id,
5916 &second.trade_id
5917 ]
5918 );
5919 assert_eq!(
5920 order.commissions().values().copied().collect::<Vec<_>>(),
5921 vec![Money::from("0.42 USDT")]
5922 );
5923 assert_eq!(position.quantity, Quantity::from("2.25"));
5924 assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5925 assert_eq!(position.replay_events.len(), 2);
5926 assert_eq!(
5927 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5928 1
5929 );
5930 assert_eq!(
5931 node.exec_manager.position_activity_revision(&key),
5932 revision + 1
5933 );
5934 assert!(node.exec_manager.position_contains_fill_report(&first));
5935 assert!(!node.exec_manager.position_contains_fill_report(&second));
5936 assert!(!node.exec_manager.position_contains_fill_report(&third));
5937 assert!(
5938 !node
5939 .exec_manager
5940 .is_fill_recently_processed(key.1, key.0, second.trade_id)
5941 );
5942 assert!(
5943 !node
5944 .exec_manager
5945 .is_fill_recently_processed(key.1, key.0, third.trade_id)
5946 );
5947 }
5948
5949 #[rstest]
5950 fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5951 let (mut node, venue_report, fill_report) =
5952 position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5953 let key = (venue_report.instrument_id, venue_report.account_id);
5954 let position_result = position_report_result(&node, venue_report);
5955
5956 node.handle_position_fill_report_result(PositionFillReportResult {
5957 position_result,
5958 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5959 });
5960
5961 let mut rest_report = fill_report;
5962 rest_report.ts_event = UnixNanos::from(2_000);
5963
5964 assert!(
5965 node.exec_manager
5966 .position_contains_fill_report(&rest_report)
5967 );
5968 }
5969
5970 #[rstest]
5971 fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5972 let (mut node, venue_report, fill_report) =
5973 position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5974 let key = (venue_report.instrument_id, venue_report.account_id);
5975 let client_order_id = fill_report.client_order_id.unwrap();
5976 apply_inferred_position_fill(&mut node, &fill_report);
5977
5978 let venue_report = PositionStatusReport::new(
5979 key.1,
5980 key.0,
5981 PositionSide::Long,
5982 Quantity::from("3.0"),
5983 venue_report.ts_last,
5984 venue_report.ts_init,
5985 None,
5986 None,
5987 venue_report.avg_px_open,
5988 );
5989 let mut later_fill_report = fill_report.clone();
5990 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5991 later_fill_report.ts_event = UnixNanos::from(1_001);
5992 let position_result = position_report_result(&node, venue_report);
5993
5994 node.handle_position_fill_report_result(PositionFillReportResult {
5995 position_result,
5996 reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5997 });
5998
5999 let cache = node.kernel.cache.borrow();
6000 let order = cache.order(&client_order_id).unwrap();
6001 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6002 assert_eq!(order.filled_qty(), Quantity::from("2.0"));
6003 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6004 assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
6005 assert_eq!(
6006 positions
6007 .iter()
6008 .map(|position| position.quantity)
6009 .sum::<Quantity>(),
6010 Quantity::from("3.0")
6011 );
6012 assert_eq!(
6013 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6014 2
6015 );
6016 }
6017
6018 #[rstest]
6019 fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
6020 let (mut node, _, _) =
6021 position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
6022 let account_id = AccountId::from("TEST-001");
6023 let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
6024 let key = (instrument.id(), account_id);
6025 let commission = Money::zero(instrument.quote_currency());
6026 node.kernel
6027 .cache
6028 .borrow_mut()
6029 .add_instrument(instrument.clone())
6030 .unwrap();
6031
6032 let order = OrderTestBuilder::new(OrderType::Market)
6033 .strategy_id(StrategyId::external())
6034 .instrument_id(instrument.id())
6035 .client_order_id(ClientOrderId::from("O-RECONCILED"))
6036 .quantity(Quantity::from("1.000000"))
6037 .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
6038 .build();
6039 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6040 let accepted =
6041 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
6042 node.kernel
6043 .cache
6044 .borrow_mut()
6045 .add_order(
6046 order.clone(),
6047 None,
6048 Some(ClientId::from("POSITION-FILLS")),
6049 false,
6050 )
6051 .unwrap();
6052 node.process_reconciliation_events(&[submitted, accepted]);
6053 let order = node
6054 .kernel
6055 .cache
6056 .borrow()
6057 .order_owned(&order.client_order_id())
6058 .unwrap();
6059 let opening_fill = TestOrderEventStubs::filled(
6060 &order,
6061 &instrument,
6062 Some(TradeId::from("T-RECONCILED")),
6063 None,
6064 Some(Price::from("100.00")),
6065 Some(Quantity::from("1.000000")),
6066 Some(LiquiditySide::Taker),
6067 Some(commission),
6068 Some(UnixNanos::from(2_000)),
6069 Some(account_id),
6070 );
6071 node.process_reconciliation_events(&[opening_fill]);
6072
6073 let held_fill_report = FillReport::new(
6075 account_id,
6076 instrument.id(),
6077 VenueOrderId::from("V-HELD"),
6078 TradeId::from("T-HELD"),
6079 OrderSide::Buy,
6080 Quantity::from("1.000000"),
6081 Price::from("100.00"),
6082 commission,
6083 LiquiditySide::Taker,
6084 None,
6085 None,
6086 UnixNanos::from(1_000),
6087 UnixNanos::from(1_000),
6088 None,
6089 );
6090 node.kernel
6091 .exec_engine
6092 .borrow_mut()
6093 .reconcile_fill_report(&held_fill_report);
6094
6095 let venue_report = PositionStatusReport::new(
6096 account_id,
6097 instrument.id(),
6098 PositionSide::Long,
6099 Quantity::from("2.000000"),
6100 UnixNanos::from(3_000),
6101 UnixNanos::from(3_000),
6102 None,
6103 None,
6104 Some(dec!(100.00)),
6105 );
6106 let position_result = position_report_result(&node, venue_report);
6107
6108 node.handle_position_fill_report_result(PositionFillReportResult {
6109 position_result,
6110 reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
6111 });
6112
6113 let cache = node.kernel.cache.borrow();
6114 let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
6115 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6116 assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
6117 assert_eq!(
6118 positions
6119 .iter()
6120 .map(|position| position.quantity)
6121 .sum::<Quantity>(),
6122 Quantity::from("2.000000")
6123 );
6124 assert!(
6125 positions
6126 .iter()
6127 .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
6128 );
6129 }
6130
6131 #[rstest]
6132 fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
6133 let (mut node, _, mut fill_report) =
6134 position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
6135 apply_inferred_position_fill(&mut node, &fill_report);
6136 let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
6137 fill_report.venue_position_id = Some(conflicting_position_id);
6138
6139 let error = node
6140 .exec_manager
6141 .prepare_position_fill_report(&mut fill_report, &[])
6142 .unwrap_err();
6143
6144 assert!(
6145 error.to_string().contains(&format!(
6146 "position ID {conflicting_position_id} conflicts with cached order position"
6147 )),
6148 "{error:#}"
6149 );
6150 }
6151
6152 #[rstest]
6153 #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
6154 #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
6155 fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
6156 #[case] authoritative_qty: &str,
6157 #[case] client_order_id: &str,
6158 ) {
6159 let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6160 "UnappliedPositionFillNode",
6161 Quantity::from(authoritative_qty),
6162 );
6163 fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6164 let reports = [fill_report.clone()];
6165
6166 let quantities = (0..2)
6167 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6168 .collect::<Vec<_>>();
6169
6170 assert_eq!(
6171 quantities,
6172 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6173 );
6174 assert!(
6175 !node
6176 .exec_manager
6177 .position_contains_fill_report(&fill_report)
6178 );
6179 }
6180
6181 #[rstest]
6182 fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6183 let (mut node, venue_report, fill_report) =
6184 position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6185 let mut later_fill_report = fill_report.clone();
6186 later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6187 later_fill_report.ts_event = UnixNanos::from(1_001);
6188 let reports = [fill_report.clone(), later_fill_report.clone()];
6189
6190 let quantities = (0..3)
6191 .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6192 .collect::<Vec<_>>();
6193
6194 assert_eq!(
6195 quantities,
6196 vec![
6197 Quantity::from("1.0"),
6198 Quantity::from("1.0"),
6199 Quantity::from("2.0"),
6200 ]
6201 );
6202 assert!(
6203 !node
6204 .exec_manager
6205 .position_contains_fill_report(&fill_report)
6206 );
6207 assert!(
6208 !node
6209 .exec_manager
6210 .position_contains_fill_report(&later_fill_report)
6211 );
6212 }
6213
6214 #[rstest]
6215 #[case::within_grace(60_000, true)]
6216 #[case::after_grace(0, false)]
6217 fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6218 #[case] position_check_threshold_ms: u32,
6219 #[case] expected_applied: bool,
6220 ) {
6221 let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6222 "TransientPositionFillNode",
6223 Quantity::from("1.0"),
6224 position_check_threshold_ms,
6225 );
6226 let mut conflicting_report = fill_report.clone();
6227 conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6228
6229 let quantities = [conflicting_report, fill_report.clone()]
6230 .into_iter()
6231 .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6232 .collect::<Vec<_>>();
6233
6234 assert_eq!(
6235 quantities,
6236 vec![Quantity::from("1.0"), Quantity::from("2.0")]
6237 );
6238 assert_eq!(
6239 node.exec_manager
6240 .position_contains_fill_report(&fill_report),
6241 expected_applied
6242 );
6243 }
6244
6245 #[rstest]
6246 fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6247 let (mut node, venue_report, fill_report) =
6248 position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6249 let key = (venue_report.instrument_id, venue_report.account_id);
6250 let first_result = position_report_result(&node, venue_report.clone());
6251
6252 node.handle_position_fill_report_result(PositionFillReportResult {
6253 position_result: first_result,
6254 reports: IndexMap::from([(key, vec![fill_report])]),
6255 });
6256
6257 let fresh_result = position_report_result(&node, venue_report);
6258 node.handle_position_fill_report_result(PositionFillReportResult {
6259 position_result: fresh_result,
6260 reports: IndexMap::from([(key, Vec::new())]),
6261 });
6262
6263 let cache = node.kernel.cache.borrow();
6264 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6265 assert_eq!(
6266 positions
6267 .iter()
6268 .map(|position| position.quantity)
6269 .sum::<Quantity>(),
6270 Quantity::from("2.0")
6271 );
6272 assert_eq!(
6273 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6274 2
6275 );
6276 }
6277
6278 #[rstest]
6279 #[case::below_limit(63)]
6280 #[case::at_limit(64)]
6281 #[case::above_limit(65)]
6282 fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6283 let (mut node, venue_report, fill_report) =
6284 position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6285 let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6286
6287 let venue_report = PositionStatusReport::new(
6288 venue_report.account_id,
6289 venue_report.instrument_id,
6290 PositionSide::Long,
6291 Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6292 venue_report.ts_last,
6293 venue_report.ts_init,
6294 None,
6295 None,
6296 venue_report.avg_px_open,
6297 );
6298 let key = (venue_report.instrument_id, venue_report.account_id);
6299 let client_order_id = fill_report.client_order_id.unwrap();
6300
6301 let reports = (0..count)
6302 .map(|index| {
6303 let mut report = fill_report.clone();
6304 report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6305 report.ts_event = UnixNanos::from(1_000 + index as u64);
6306 report
6307 })
6308 .collect::<Vec<_>>();
6309
6310 let position_result = position_report_result(&node, venue_report.clone());
6311
6312 node.handle_position_fill_report_result(PositionFillReportResult {
6313 position_result,
6314 reports: IndexMap::from([(key, reports.clone())]),
6315 });
6316
6317 {
6318 let cache = node.kernel.cache.borrow();
6319 let order = cache.order(&client_order_id).unwrap();
6320 let applied = count.min(64);
6321 assert_eq!(
6322 order.filled_qty().as_decimal(),
6323 dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6324 );
6325 assert_eq!(order.trade_ids().len(), applied + 1);
6326
6327 for (index, report) in reports.iter().enumerate() {
6328 assert_eq!(
6329 order.trade_ids().contains(&&report.trade_id),
6330 index < applied
6331 );
6332 }
6333
6334 assert_eq!(
6335 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6336 1
6337 );
6338 }
6339
6340 let position_result = position_report_result(&node, venue_report);
6342 node.handle_position_fill_report_result(PositionFillReportResult {
6343 position_result,
6344 reports: IndexMap::from([(key, reports)]),
6345 });
6346
6347 let cache = node.kernel.cache.borrow();
6348 let order = cache.order(&client_order_id).unwrap();
6349 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6350 assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6351 assert_eq!(order.trade_ids().len(), count + 1);
6352 assert_eq!(positions.len(), 1);
6353 assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6354 assert_eq!(positions[0].side, PositionSide::Long);
6355 assert_eq!(
6356 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6357 1
6358 );
6359 }
6360
6361 #[rstest]
6362 #[case::client_order(0)]
6363 #[case::venue_order(1)]
6364 #[case::side(2)]
6365 #[case::position(3)]
6366 fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6367 let (mut node, venue_report, valid_report) =
6368 position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6369 let key = (venue_report.instrument_id, venue_report.account_id);
6370 let client_order_id = valid_report.client_order_id.unwrap();
6371 let revision = node.exec_manager.position_activity_revision(&key);
6372 let position_result = position_report_result(&node, venue_report);
6373 let mut conflicting_report = valid_report.clone();
6374 conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6375
6376 match conflict {
6377 0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6378 1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6379 2 => conflicting_report.order_side = OrderSide::Sell,
6380 3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6381 _ => unreachable!(),
6382 }
6383
6384 node.handle_position_fill_report_result(PositionFillReportResult {
6385 position_result,
6386 reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6387 });
6388
6389 let cache = node.kernel.cache.borrow();
6390 let order = cache.order(&client_order_id).unwrap();
6391 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6392 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6393 assert_eq!(
6394 order.trade_ids(),
6395 vec![&TradeId::from("T-POSITION-INITIAL")]
6396 );
6397 assert_eq!(positions.len(), 1);
6398 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6399 assert_eq!(
6400 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6401 1
6402 );
6403 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6404 }
6405
6406 #[rstest]
6407 fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6408 let (mut node, venue_report, _) =
6409 position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6410 let key = (venue_report.instrument_id, venue_report.account_id);
6411 let position_result = position_report_result(&node, venue_report);
6412
6413 node.handle_position_fill_report_result(PositionFillReportResult {
6414 position_result,
6415 reports: IndexMap::new(),
6416 });
6417
6418 let cache = node.kernel.cache.borrow();
6419 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6420 assert_eq!(positions.len(), 1);
6421 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6422 assert_eq!(
6423 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6424 1
6425 );
6426 }
6427
6428 #[rstest]
6429 fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6430 let (mut node, mut venue_report, mut fill_report) =
6431 position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6432 node.kernel
6433 .exec_engine
6434 .borrow_mut()
6435 .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6436 let key = (venue_report.instrument_id, venue_report.account_id);
6437
6438 let position_id = {
6439 let cache = node.kernel.cache.borrow();
6440 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6441 assert_eq!(positions.len(), 1);
6442 positions[0].id
6443 };
6444
6445 venue_report.venue_position_id = Some(position_id);
6446 fill_report.client_order_id = None;
6447 fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6448 let position_result = position_report_result(&node, venue_report);
6449
6450 node.handle_position_fill_report_result(PositionFillReportResult {
6451 position_result,
6452 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6453 });
6454
6455 let cache = node.kernel.cache.borrow();
6456 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6457 assert_eq!(positions.len(), 1);
6458 assert_eq!(positions[0].id, position_id);
6459 assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6460 assert_eq!(
6461 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6462 2
6463 );
6464 drop(positions);
6465 drop(cache);
6466 assert!(
6467 !node
6468 .exec_manager
6469 .position_contains_fill_report(&fill_report)
6470 );
6471 }
6472
6473 #[rstest]
6474 fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6475 let (mut node, venue_report, fill_report) =
6476 position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6477 let key = (venue_report.instrument_id, venue_report.account_id);
6478 let client_order_id = fill_report.client_order_id.unwrap();
6479 let position_result = position_report_result(&node, venue_report);
6480 let revision = node.exec_manager.position_activity_revision(&key);
6481
6482 let order_report = OrderStatusReport::new(
6483 key.1,
6484 key.0,
6485 Some(client_order_id),
6486 fill_report.venue_order_id,
6487 OrderSide::Buy.into(),
6488 OrderType::Limit,
6489 TimeInForce::Gtc,
6490 OrderStatus::PartiallyFilled,
6491 Quantity::from("10.0"),
6492 Quantity::from("2.0"),
6493 UnixNanos::from(1_000),
6494 UnixNanos::from(1_000),
6495 UnixNanos::from(2_000),
6496 None,
6497 );
6498 let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6499 Box::new(order_report),
6500 vec![fill_report.clone()],
6501 ));
6502
6503 assert_eq!(
6504 node.observe_exec_event_before_dispatch(&event),
6505 Some(vec![client_order_id])
6506 );
6507
6508 assert_eq!(
6509 node.exec_manager.position_activity_revision(&key),
6510 revision + 1
6511 );
6512 assert!(
6513 !node
6514 .exec_manager
6515 .position_report_check_is_current(&position_result.check, &key)
6516 );
6517
6518 node.handle_position_fill_report_result(PositionFillReportResult {
6519 position_result,
6520 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6521 });
6522
6523 let cache = node.kernel.cache.borrow();
6524 let order = cache.order(&client_order_id).unwrap();
6525 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6526 assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6527 assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6528 assert_eq!(positions.len(), 1);
6529 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6530 assert_eq!(positions[0].side, PositionSide::Long);
6531 assert_eq!(
6532 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6533 1
6534 );
6535 }
6536
6537 #[rstest]
6538 fn test_position_fill_report_result_defers_after_local_position_activity() {
6539 let (mut node, venue_report, fill_report) =
6540 position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6541 let key = (venue_report.instrument_id, venue_report.account_id);
6542 let position_result = position_report_result(&node, venue_report);
6543 node.exec_manager.record_position_activity(key.0, key.1);
6544
6545 node.handle_position_fill_report_result(PositionFillReportResult {
6546 position_result,
6547 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6548 });
6549
6550 let cache = node.kernel.cache.borrow();
6551 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6552 assert_eq!(positions.len(), 1);
6553 assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6554 assert_eq!(
6555 cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6556 1
6557 );
6558 drop(positions);
6559 drop(cache);
6560 assert!(
6561 !node
6562 .exec_manager
6563 .position_contains_fill_report(&fill_report)
6564 );
6565 }
6566
6567 #[rstest]
6568 fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6569 let config = LiveNodeConfig {
6570 exec_engine: crate::config::LiveExecutionEngineConfig {
6571 reconciliation: true,
6572 open_check_threshold_ms: 5_000,
6573 single_order_query_delay_ms: 0,
6574 ..Default::default()
6575 },
6576 ..Default::default()
6577 };
6578
6579 let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6580 let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6581 let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6582 let instrument = crypto_perpetual_ethusdt();
6583 let instrument_id = instrument.id();
6584 let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6585 let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6586
6587 node.kernel
6588 .cache
6589 .borrow_mut()
6590 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6591 .unwrap();
6592 insert_accepted_limit_order_in_node(
6593 &node,
6594 account_id,
6595 client_id,
6596 instrument_id,
6597 client_order_id,
6598 venue_order_id,
6599 );
6600
6601 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6602
6603 let accepted = OrderAcceptedSpec::builder()
6604 .instrument_id(instrument_id)
6605 .client_order_id(client_order_id)
6606 .venue_order_id(venue_order_id)
6607 .account_id(account_id)
6608 .build();
6609 let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6610
6611 let close_ids = node.observe_exec_event_before_dispatch(&event);
6612
6613 assert_eq!(close_ids, Some(vec![client_order_id]));
6614 assert!(node.exec_manager.check_open_order_queries().is_empty());
6615 }
6616
6617 #[rstest]
6618 #[cfg_attr(
6619 not(all(feature = "simulation", madsim)),
6620 tokio::test(start_paused = true)
6621 )]
6622 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6623 async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6624 use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6625 use nautilus_model::{
6626 events::order::spec::OrderPendingCancelSpec, identifiers::ClientOrderId,
6627 };
6628
6629 let config = LiveNodeConfig {
6630 exec_engine: crate::config::LiveExecutionEngineConfig {
6631 reconciliation: true,
6632 inflight_check_threshold_ms: 100,
6633 inflight_check_retries: 1,
6634 ..Default::default()
6635 },
6636 ..Default::default()
6637 };
6638
6639 let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6640 let trader_id = TraderId::from("TESTER-001");
6641 let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6642 let account_id = AccountId::from("TEST-001");
6643 let instrument = crypto_perpetual_ethusdt();
6644 let instrument_id = instrument.id();
6645 node.kernel
6646 .exec_engine
6647 .borrow_mut()
6648 .register_client(Box::new(StubExecutionClient::new(
6649 ClientId::from("STUB"),
6650 account_id,
6651 instrument_id.venue,
6652 OmsType::Netting,
6653 None,
6654 )))
6655 .unwrap();
6656 node.kernel
6657 .exec_engine
6658 .borrow_mut()
6659 .set_default_client(ClientId::from("STUB"))
6660 .unwrap();
6661 let child_ids = [
6662 ClientOrderId::from("O-BATCH-CANCEL-1"),
6663 ClientOrderId::from("O-BATCH-CANCEL-2"),
6664 ];
6665 node.kernel
6666 .cache
6667 .borrow_mut()
6668 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6669 .unwrap();
6670
6671 for client_order_id in child_ids {
6672 let order = OrderTestBuilder::new(OrderType::Limit)
6673 .trader_id(trader_id)
6674 .strategy_id(strategy_id)
6675 .client_order_id(client_order_id)
6676 .instrument_id(instrument_id)
6677 .quantity(Quantity::from("10.0"))
6678 .price(Price::from("100.0"))
6679 .build();
6680 let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6681 let submitted = TestOrderEventStubs::submitted(&order, account_id);
6685 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6686 let pending_cancel = OrderEventAny::PendingCancel(
6687 OrderPendingCancelSpec::builder()
6688 .trader_id(trader_id)
6689 .strategy_id(strategy_id)
6690 .instrument_id(instrument_id)
6691 .client_order_id(client_order_id)
6692 .account_id(account_id)
6693 .venue_order_id(venue_order_id)
6694 .build(),
6695 );
6696 let mut cache = node.kernel.cache.borrow_mut();
6697 cache.add_order(order, None, None, false).unwrap();
6698 cache.update_order(&submitted).unwrap();
6699 cache.update_order(&accepted).unwrap();
6700 cache.update_order(&pending_cancel).unwrap();
6701 }
6702
6703 let cancels = child_ids
6704 .into_iter()
6705 .map(|client_order_id| {
6706 CancelOrder::new(
6707 trader_id,
6708 None,
6709 strategy_id,
6710 instrument_id,
6711 client_order_id,
6712 None,
6713 UUID4::new(),
6714 UnixNanos::default(),
6715 None,
6716 None,
6717 )
6718 })
6719 .collect();
6720
6721 let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6722 trader_id,
6723 None,
6724 strategy_id,
6725 instrument_id,
6726 cancels,
6727 UUID4::new(),
6728 UnixNanos::default(),
6729 None,
6730 None,
6731 ));
6732
6733 node.observe_exec_command_before_dispatch(&command);
6734 advance_clock(Duration::from_millis(101)).await;
6735 let result = node.exec_manager.check_inflight_orders();
6736 let timed_out_ids = result
6737 .events
6738 .iter()
6739 .map(OrderEventAny::client_order_id)
6740 .collect::<IndexSet<_>>();
6741
6742 assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6743 assert_eq!(result.events.len(), child_ids.len());
6744 assert!(
6745 result
6746 .events
6747 .iter()
6748 .all(|event| matches!(event, OrderEventAny::CancelRejected(_))),
6749 "batch-cancel children must time out as CancelRejected events",
6750 );
6751 }
6752
6753 #[rstest]
6754 #[cfg_attr(
6755 not(all(feature = "simulation", madsim)),
6756 tokio::test(start_paused = true)
6757 )]
6758 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6759 async fn test_risk_bound_command_does_not_register_inflight() {
6760 let config = LiveNodeConfig {
6761 exec_engine: crate::config::LiveExecutionEngineConfig {
6762 reconciliation: true,
6763 inflight_check_threshold_ms: 100,
6764 inflight_check_retries: 2,
6765 ..Default::default()
6766 },
6767 ..Default::default()
6768 };
6769
6770 let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6771 msgbus::register_trading_command_endpoint(
6772 MessagingSwitchboard::risk_engine_execute(),
6773 TypedIntoHandler::from(|_: TradingCommand| {}),
6774 );
6775 let instrument = crypto_perpetual_ethusdt();
6776 let instrument_id = instrument.id();
6777 let order = OrderTestBuilder::new(OrderType::Limit)
6778 .trader_id(node.trader_id())
6779 .strategy_id(StrategyId::from("S-RISK-DENIED"))
6780 .instrument_id(instrument_id)
6781 .side(OrderSide::Buy)
6782 .quantity(Quantity::from("1.000"))
6783 .price(Price::from("100.00"))
6784 .build();
6785 let client_order_id = order.client_order_id();
6786
6787 {
6788 let mut cache = node.kernel.cache.borrow_mut();
6789 cache
6790 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6791 .unwrap();
6792 cache.add_order(order.clone(), None, None, false).unwrap();
6793 }
6794
6795 let submit_order = SubmitOrder::new(
6796 order.trader_id(),
6797 None,
6798 order.strategy_id(),
6799 instrument_id,
6800 client_order_id,
6801 order.init_event().clone(),
6802 None,
6803 None,
6804 None,
6805 UUID4::new(),
6806 UnixNanos::default(),
6807 None,
6808 );
6809 node.process_exec_command(
6810 TradingCommandMessage::new(
6811 MessagingSwitchboard::risk_engine_execute(),
6812 TradingCommand::SubmitOrder(submit_order),
6813 )
6814 .into(),
6815 );
6816
6817 advance_clock(Duration::from_millis(101)).await;
6818 let result = node.exec_manager.check_inflight_orders();
6819 let status = node
6820 .kernel
6821 .cache
6822 .borrow()
6823 .order(&client_order_id)
6824 .unwrap()
6825 .status();
6826
6827 assert_eq!(status, OrderStatus::Initialized);
6828 assert_eq!(
6829 node.exec_manager.recon_check_retry_count(&client_order_id),
6830 0
6831 );
6832 assert!(result.events.is_empty());
6833 assert!(result.queries.is_empty());
6834 }
6835
6836 #[rstest]
6837 #[cfg_attr(
6838 not(all(feature = "simulation", madsim)),
6839 tokio::test(start_paused = true)
6840 )]
6841 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6842 async fn test_external_cancel_command_still_registers_inflight() {
6843 use nautilus_common::messages::execution::CancelOrder;
6844 let external = ClientId::from("EXTERNAL");
6845
6846 let config = LiveNodeConfig {
6847 exec_engine: crate::config::LiveExecutionEngineConfig {
6848 reconciliation: true,
6849 inflight_check_threshold_ms: 100,
6850 inflight_check_retries: 2,
6851 external_clients: Some(vec![external]),
6852 ..Default::default()
6853 },
6854 ..Default::default()
6855 };
6856
6857 let mut node = LiveNode::build("ExternalCancelNode".to_string(), Some(config)).unwrap();
6858 let instrument = crypto_perpetual_ethusdt();
6859 let mut order = OrderTestBuilder::new(OrderType::Limit)
6860 .trader_id(node.trader_id())
6861 .strategy_id(StrategyId::from("S-EXTERNAL"))
6862 .instrument_id(instrument.id())
6863 .quantity(Quantity::from("9.003"))
6864 .price(Price::from("113.47"))
6865 .build();
6866 let venue = VenueOrderId::from("V-EXTERNAL");
6867 order
6868 .apply(TestOrderEventStubs::accepted(
6869 &order,
6870 AccountId::from("SIM-001"),
6871 venue,
6872 ))
6873 .unwrap();
6874 let id = order.client_order_id();
6875 let command = TradingCommand::CancelOrder(CancelOrder::new(
6876 order.trader_id(),
6877 Some(external),
6878 order.strategy_id(),
6879 order.instrument_id(),
6880 id,
6881 Some(venue),
6882 UUID4::new(),
6883 17.into(),
6884 None,
6885 None,
6886 ));
6887 {
6888 let mut cache = node.kernel.cache.borrow_mut();
6889 cache.add_instrument(instrument.into()).unwrap();
6890 cache
6891 .add_order(order.clone(), None, Some(external), false)
6892 .unwrap();
6893 }
6894
6895 node.process_exec_command(
6896 TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
6897 );
6898 advance_clock(Duration::from_millis(101)).await;
6899 let result = node.exec_manager.check_inflight_orders();
6900
6901 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6902 panic!("expected external query");
6903 };
6904
6905 assert_eq!(
6906 (query.client_order_id, query.venue_order_id, query.client_id),
6907 (id, Some(venue), Some(external))
6908 );
6909 assert!(result.events.is_empty());
6910 assert_eq!(
6911 serde_json::to_value(node.kernel.cache.borrow().order(&id).unwrap().clone()).unwrap(),
6912 serde_json::to_value(&order).unwrap()
6913 );
6914 }
6915
6916 #[rstest]
6917 #[case::modify(false, false)]
6918 #[case::batch_modify(false, true)]
6919 #[case::cancel(true, false)]
6920 #[case::batch_cancel(true, true)]
6921 #[cfg_attr(
6922 not(all(feature = "simulation", madsim)),
6923 tokio::test(start_paused = true)
6924 )]
6925 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6926 async fn test_native_command_denial_does_not_leave_inflight_tracking(
6927 #[case] cancel: bool,
6928 #[case] batch: bool,
6929 #[values(false, true)] old_pending: bool,
6930 ) {
6931 use nautilus_common::messages::execution::{
6932 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
6933 };
6934 use nautilus_model::events::order::spec::{OrderModifyDeniedSpec, OrderPendingUpdateSpec};
6935
6936 let config = LiveNodeConfig {
6937 exec_engine: crate::config::LiveExecutionEngineConfig {
6938 reconciliation: true,
6939 inflight_check_threshold_ms: 100,
6940 inflight_check_retries: 2,
6941 ..Default::default()
6942 },
6943 ..Default::default()
6944 };
6945
6946 let mut node = LiveNode::build("LocalDenialNode".to_string(), Some(config)).unwrap();
6947 let instrument = crypto_perpetual_ethusdt();
6948 let mut order = OrderTestBuilder::new(OrderType::Limit)
6949 .trader_id(node.trader_id())
6950 .strategy_id(StrategyId::from("S-LOCAL-DENIED"))
6951 .instrument_id(instrument.id())
6952 .side(OrderSide::Buy)
6953 .quantity(Quantity::from("9.003"))
6954 .price(Price::from("113.47"))
6955 .build();
6956 let account_id = AccountId::from("SIM-001");
6957 let venue_id = VenueOrderId::from("V-LOCAL-DENIED");
6958 order
6959 .apply(TestOrderEventStubs::accepted(&order, account_id, venue_id))
6960 .unwrap();
6961 let id = order.client_order_id();
6962 let old_cause = UUID4::new();
6963
6964 if old_pending {
6965 let mut marker = OrderPendingUpdateSpec::builder()
6966 .trader_id(order.trader_id())
6967 .strategy_id(order.strategy_id())
6968 .instrument_id(order.instrument_id())
6969 .client_order_id(id)
6970 .venue_order_id(venue_id)
6971 .account_id(account_id)
6972 .build();
6973 marker.causation_id = Some(old_cause);
6974 order.apply(OrderEventAny::PendingUpdate(marker)).unwrap();
6975 node.exec_manager.register_inflight(id, old_cause);
6976 }
6977
6978 let expected_status = order.status();
6979 let events_before = order.event_count();
6980 {
6981 let mut cache = node.kernel.cache.borrow_mut();
6982 cache.add_instrument(instrument.into()).unwrap();
6983 cache.add_order(order.clone(), None, None, false).unwrap();
6984 }
6985
6986 let cause = UUID4::new();
6987
6988 let command = if cancel {
6989 let command = CancelOrder::new(
6990 order.trader_id(),
6991 None,
6992 order.strategy_id(),
6993 order.instrument_id(),
6994 id,
6995 Some(venue_id),
6996 cause,
6997 17.into(),
6998 None,
6999 None,
7000 );
7001
7002 if batch {
7003 TradingCommand::CancelOrders(BatchCancelOrders::new(
7004 order.trader_id(),
7005 None,
7006 order.strategy_id(),
7007 order.instrument_id(),
7008 vec![command],
7009 UUID4::new(),
7010 19.into(),
7011 None,
7012 None,
7013 ))
7014 } else {
7015 TradingCommand::CancelOrder(command)
7016 }
7017 } else {
7018 let command = ModifyOrder::new(
7019 order.trader_id(),
7020 None,
7021 order.strategy_id(),
7022 order.instrument_id(),
7023 id,
7024 Some(venue_id),
7025 Some(Quantity::from("8.001")),
7026 None,
7027 None,
7028 cause,
7029 17.into(),
7030 None,
7031 None,
7032 );
7033
7034 if batch {
7035 TradingCommand::ModifyOrders(BatchModifyOrders::new(
7036 order.trader_id(),
7037 None,
7038 order.strategy_id(),
7039 order.instrument_id(),
7040 vec![command],
7041 UUID4::new(),
7042 19.into(),
7043 None,
7044 None,
7045 ))
7046 } else {
7047 TradingCommand::ModifyOrder(command)
7048 }
7049 };
7050
7051 node.process_exec_command(
7052 TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
7053 );
7054 {
7055 let cache = node.kernel.cache.borrow();
7056 let cached = cache.order(&id).unwrap();
7057 match cached.last_event() {
7058 OrderEventAny::ModifyDenied(event) if !cancel => {
7059 assert_eq!(event.causation_id, Some(cause));
7060 }
7061 OrderEventAny::CancelDenied(event) if cancel => {
7062 assert_eq!(event.causation_id, Some(cause));
7063 }
7064 other => panic!("expected native command denial, was {other:?}"),
7065 }
7066
7067 assert_eq!(cached.status(), expected_status);
7068 assert_eq!(cached.quantity(), order.quantity());
7069 assert_eq!(cached.price(), order.price());
7070 assert_eq!(cached.filled_qty(), order.filled_qty());
7071 assert_eq!(cached.event_count(), events_before + 1);
7072 }
7073
7074 advance_clock(Duration::from_millis(101)).await;
7075 let result = node.exec_manager.check_inflight_orders();
7076 assert_eq!(result.queries.len(), usize::from(old_pending));
7077 assert!(result.events.is_empty());
7078
7079 if old_pending {
7080 let refusal = OrderModifyDeniedSpec::builder()
7081 .trader_id(order.trader_id())
7082 .strategy_id(order.strategy_id())
7083 .instrument_id(order.instrument_id())
7084 .client_order_id(id)
7085 .venue_order_id(venue_id)
7086 .account_id(account_id)
7087 .causation_id(old_cause)
7088 .build();
7089 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::ModifyDenied(refusal)));
7090 }
7091
7092 assert_eq!(node.exec_manager.recon_check_retry_count(&id), 0);
7093 }
7094
7095 #[rstest]
7096 #[cfg_attr(
7097 not(all(feature = "simulation", madsim)),
7098 tokio::test(start_paused = true)
7099 )]
7100 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7101 async fn test_risk_approved_command_registers_inflight() {
7102 let config = LiveNodeConfig {
7103 risk_engine: crate::config::LiveRiskEngineConfig {
7104 bypass: true,
7105 ..Default::default()
7106 },
7107 exec_engine: crate::config::LiveExecutionEngineConfig {
7108 reconciliation: true,
7109 inflight_check_threshold_ms: 100,
7110 inflight_check_retries: 2,
7111 ..Default::default()
7112 },
7113 ..Default::default()
7114 };
7115
7116 let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
7117 msgbus::register_trading_command_endpoint(
7118 MessagingSwitchboard::exec_engine_execute(),
7119 TypedIntoHandler::from(|_: TradingCommand| {}),
7120 );
7121 let instrument = crypto_perpetual_ethusdt();
7122 let instrument_id = instrument.id();
7123 let order = OrderTestBuilder::new(OrderType::Limit)
7124 .trader_id(node.trader_id())
7125 .strategy_id(StrategyId::from("S-RISK-APPROVED"))
7126 .instrument_id(instrument_id)
7127 .side(OrderSide::Buy)
7128 .quantity(Quantity::from("1.000"))
7129 .price(Price::from("100.00"))
7130 .build();
7131 let client_order_id = order.client_order_id();
7132
7133 {
7134 let mut cache = node.kernel.cache.borrow_mut();
7135 cache
7136 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7137 .unwrap();
7138 cache.add_order(order.clone(), None, None, false).unwrap();
7139 }
7140
7141 let submit_order = SubmitOrder::new(
7142 order.trader_id(),
7143 None,
7144 order.strategy_id(),
7145 instrument_id,
7146 client_order_id,
7147 order.init_event().clone(),
7148 None,
7149 None,
7150 None,
7151 UUID4::new(),
7152 UnixNanos::default(),
7153 None,
7154 );
7155 node.process_exec_command(
7156 TradingCommandMessage::new(
7157 MessagingSwitchboard::risk_engine_execute(),
7158 TradingCommand::SubmitOrder(submit_order),
7159 )
7160 .into(),
7161 );
7162
7163 advance_clock(Duration::from_millis(101)).await;
7164 let result = node.exec_manager.check_inflight_orders();
7165
7166 let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
7167 panic!("expected one query order command");
7168 };
7169
7170 assert_eq!(query.client_order_id, client_order_id);
7171 assert_eq!(
7172 node.exec_manager.recon_check_retry_count(&client_order_id),
7173 1
7174 );
7175 assert!(result.events.is_empty());
7176 }
7177
7178 #[rstest]
7179 fn test_live_node_builder_clock_factory_drives_kernel_clock() {
7180 let calls = Rc::new(Cell::new(0usize));
7181 let calls_in_factory = calls.clone();
7182 let sentinel = UnixNanos::from(123_456_789_u64);
7183
7184 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7185 .unwrap()
7186 .with_reconciliation(false)
7187 .with_clock_factory(move || {
7188 calls_in_factory.set(calls_in_factory.get() + 1);
7189 let mut clock = VirtualClock::new();
7190 clock.advance_time(sentinel, true);
7191 Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
7192 })
7193 .build()
7194 .unwrap();
7195
7196 assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
7197 assert_eq!(calls.get(), 1);
7198 }
7199
7200 #[derive(Debug)]
7201 struct ReplayKernelEventStore {
7202 fail_restore: bool,
7203 }
7204
7205 impl KernelEventStore for ReplayKernelEventStore {
7206 fn restore_parent_cache(
7207 &mut self,
7208 _instance_id: UUID4,
7209 _cache: &mut Cache,
7210 ) -> anyhow::Result<()> {
7211 if self.fail_restore {
7212 anyhow::bail!("replay restore failed");
7213 }
7214
7215 Ok(())
7216 }
7217
7218 fn open(
7219 &mut self,
7220 _instance_id: UUID4,
7221 _components: &RegisteredComponents,
7222 _environment: Environment,
7223 ) -> anyhow::Result<()> {
7224 Ok(())
7225 }
7226
7227 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
7228 None
7229 }
7230
7231 fn seal(&mut self, _ts_init: UnixNanos) {}
7232
7233 fn run_id(&self) -> Option<&str> {
7234 Some("replay-child")
7235 }
7236
7237 fn parent_run_id(&self) -> Option<&str> {
7238 Some("seed-run")
7239 }
7240
7241 fn is_event_store_replay_configured(&self) -> bool {
7242 true
7243 }
7244
7245 fn is_halted(&self) -> bool {
7246 false
7247 }
7248 }
7249
7250 #[derive(Debug)]
7251 struct TestStrategy {
7252 core: StrategyCore,
7253 }
7254
7255 impl TestStrategy {
7256 fn new(config: StrategyConfig) -> Self {
7257 Self {
7258 core: StrategyCore::new(config),
7259 }
7260 }
7261 }
7262
7263 impl DataActor for TestStrategy {}
7264
7265 nautilus_strategy!(TestStrategy, {
7266 fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
7267 self.core.config.external_order_instrument_ids.clone()
7268 }
7269 });
7270
7271 fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
7272 let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
7275 .unwrap()
7276 .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
7277 reconciliation: false,
7278 ..Default::default()
7279 })
7280 .with_load_state(true)
7281 .with_name("TestKernel")
7282 .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
7283 Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
7284 });
7285
7286 builder.build().unwrap()
7287 }
7288
7289 #[rstest]
7290 fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
7291 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7292 .unwrap()
7293 .with_reconciliation(false)
7294 .with_delay_post_stop_secs(0)
7295 .with_timeout_connection(1)
7296 .build()
7297 .unwrap();
7298 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7299 let strategy_id = StrategyId::from("CLAIMS-001");
7300
7301 node.add_strategy(TestStrategy::new(StrategyConfig {
7302 strategy_id: Some(strategy_id),
7303 external_order_instrument_ids: Some(vec![instrument_id]),
7304 ..Default::default()
7305 }))
7306 .unwrap();
7307
7308 assert_eq!(
7309 node.exec_manager.get_external_order_claim(&instrument_id),
7310 Some(strategy_id)
7311 );
7312
7313 {
7314 let exec_engine = node.kernel().exec_engine.borrow();
7315 assert_eq!(
7316 exec_engine.get_external_order_claim(&instrument_id),
7317 Some(strategy_id)
7318 );
7319 }
7320 }
7321
7322 #[rstest]
7323 fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
7324 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7325 .unwrap()
7326 .with_reconciliation(false)
7327 .build()
7328 .unwrap();
7329 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7330 let strategy_id = StrategyId::from("CLAIMS-001");
7331
7332 node.register_external_order_claims(strategy_id, &[instrument_id])
7333 .unwrap();
7334
7335 assert_eq!(
7336 node.exec_manager.get_external_order_claim(&instrument_id),
7337 Some(strategy_id)
7338 );
7339 assert_eq!(
7340 node.kernel
7341 .exec_engine
7342 .borrow()
7343 .get_external_order_claim(&instrument_id),
7344 Some(strategy_id)
7345 );
7346 }
7347
7348 #[rstest]
7349 #[tokio::test]
7350 async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
7351 let mut node = live_node_with_replay_store(false);
7352 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7353 let strategy_id = StrategyId::from("CLAIMS-001");
7354
7355 node.start().await.unwrap();
7356 assert_eq!(node.state(), NodeState::Running);
7357
7358 node.register_external_order_claims(strategy_id, &[instrument_id])
7359 .unwrap();
7360
7361 assert_eq!(
7362 node.exec_manager.get_external_order_claim(&instrument_id),
7363 Some(strategy_id)
7364 );
7365 assert_eq!(
7366 node.kernel
7367 .exec_engine
7368 .borrow()
7369 .get_external_order_claim(&instrument_id),
7370 Some(strategy_id)
7371 );
7372 }
7373
7374 #[rstest]
7375 fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
7376 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7377 .unwrap()
7378 .with_reconciliation(false)
7379 .build()
7380 .unwrap();
7381 let existing_instrument = InstrumentId::from("AUDUSD.SIM");
7382 let new_instruments = [
7383 InstrumentId::from("EURUSD.SIM"),
7384 InstrumentId::from("GBPUSD.SIM"),
7385 ];
7386 let existing_strategy_id = StrategyId::from("CLAIMS-001");
7387 let new_strategy_id = StrategyId::from("CLAIMS-002");
7388 node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
7389 .unwrap();
7390
7391 let result = node.register_external_order_claims(
7392 new_strategy_id,
7393 &[new_instruments[0], existing_instrument, new_instruments[1]],
7394 );
7395
7396 assert!(result.is_err());
7397 assert_eq!(
7398 node.exec_manager
7399 .get_external_order_claim(&existing_instrument),
7400 Some(existing_strategy_id)
7401 );
7402
7403 for instrument_id in new_instruments {
7404 assert_eq!(
7405 node.exec_manager.get_external_order_claim(&instrument_id),
7406 None
7407 );
7408 assert_eq!(
7409 node.kernel
7410 .exec_engine
7411 .borrow()
7412 .get_external_order_claim(&instrument_id),
7413 None
7414 );
7415 }
7416 }
7417
7418 #[rstest]
7419 fn test_deregister_external_order_claims_allows_successor_to_claim() {
7420 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7421 .unwrap()
7422 .with_reconciliation(false)
7423 .build()
7424 .unwrap();
7425 let instruments = [
7426 InstrumentId::from("AUDUSD.SIM"),
7427 InstrumentId::from("EURUSD.SIM"),
7428 ];
7429 let first_strategy_id = StrategyId::from("CLAIMS-001");
7430 let successor_strategy_id = StrategyId::from("CLAIMS-002");
7431 node.register_external_order_claims(first_strategy_id, &instruments)
7432 .unwrap();
7433
7434 node.deregister_external_order_claims(first_strategy_id)
7435 .unwrap();
7436 node.register_external_order_claims(successor_strategy_id, &instruments)
7437 .unwrap();
7438
7439 for instrument_id in instruments {
7440 assert_eq!(
7441 node.exec_manager.get_external_order_claim(&instrument_id),
7442 Some(successor_strategy_id)
7443 );
7444 assert_eq!(
7445 node.kernel
7446 .exec_engine
7447 .borrow()
7448 .get_external_order_claim(&instrument_id),
7449 Some(successor_strategy_id)
7450 );
7451 }
7452 }
7453
7454 #[rstest]
7455 fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7456 let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7457 .unwrap()
7458 .with_reconciliation(false)
7459 .build()
7460 .unwrap();
7461 let strategy_id = StrategyId::from("CLAIMS-001");
7462
7463 node.deregister_external_order_claims(strategy_id).unwrap();
7464 node.deregister_external_order_claims(strategy_id).unwrap();
7465 }
7466
7467 #[rstest]
7468 fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7469 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7470 .unwrap()
7471 .with_reconciliation(false)
7472 .with_delay_post_stop_secs(0)
7473 .with_timeout_connection(1)
7474 .build()
7475 .unwrap();
7476 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7477 let strategy_id = StrategyId::from("CLAIMS-001");
7478 let duplicate_strategy_id = StrategyId::from("OTHER-002");
7479
7480 node.add_strategy(TestStrategy::new(StrategyConfig {
7481 strategy_id: Some(strategy_id),
7482 external_order_instrument_ids: Some(vec![instrument_id]),
7483 ..Default::default()
7484 }))
7485 .unwrap();
7486
7487 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7488 strategy_id: Some(duplicate_strategy_id),
7489 external_order_instrument_ids: Some(vec![instrument_id]),
7490 ..Default::default()
7491 }));
7492
7493 assert!(result.is_err());
7494 assert!(
7495 result
7496 .unwrap_err()
7497 .to_string()
7498 .contains("already exists for CLAIMS-001")
7499 );
7500 assert_eq!(
7501 node.exec_manager.get_external_order_claim(&instrument_id),
7502 Some(strategy_id)
7503 );
7504
7505 {
7506 let exec_engine = node.kernel().exec_engine.borrow();
7507 assert_eq!(
7508 exec_engine.get_external_order_claim(&instrument_id),
7509 Some(strategy_id)
7510 );
7511 }
7512 }
7513
7514 #[rstest]
7515 fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7516 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7517 .unwrap()
7518 .with_reconciliation(false)
7519 .with_delay_post_stop_secs(0)
7520 .with_timeout_connection(1)
7521 .build()
7522 .unwrap();
7523 let instrument_id = InstrumentId::from("AUDUSD.SIM");
7524 let strategy_id = StrategyId::from("CLAIMS-001");
7525
7526 let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7527 strategy_id: Some(strategy_id),
7528 external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7529 ..Default::default()
7530 }));
7531
7532 assert!(result.is_err());
7533 assert!(
7534 result
7535 .unwrap_err()
7536 .to_string()
7537 .contains("appears more than once for CLAIMS-001")
7538 );
7539 assert_eq!(
7540 node.exec_manager.get_external_order_claim(&instrument_id),
7541 None
7542 );
7543
7544 {
7545 let exec_engine = node.kernel().exec_engine.borrow();
7546 assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7547 }
7548 }
7549
7550 #[rstest]
7551 fn test_add_strategy_failure_restores_external_order_claims() {
7552 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7553 .unwrap()
7554 .with_reconciliation(false)
7555 .with_delay_post_stop_secs(0)
7556 .with_timeout_connection(1)
7557 .build()
7558 .unwrap();
7559 let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7560 let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7561 let strategy_id = StrategyId::from("CLAIMS-001");
7562 node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7563 .unwrap();
7564
7565 let mut strategy = TestStrategy::new(StrategyConfig {
7566 strategy_id: Some(strategy_id),
7567 external_order_instrument_ids: Some(vec![configured_instrument_id]),
7568 ..Default::default()
7569 });
7570
7571 strategy
7572 .core
7573 .register(
7574 node.trader_id(),
7575 node.kernel.clock(),
7576 node.kernel.cache.clone(),
7577 node.kernel.portfolio.clone(),
7578 )
7579 .unwrap();
7580
7581 let result = node.add_strategy(strategy);
7582
7583 assert!(result.is_err());
7584 assert!(
7585 result
7586 .unwrap_err()
7587 .to_string()
7588 .contains("already registered with trader")
7589 );
7590 assert_eq!(
7591 node.exec_manager
7592 .get_external_order_claim(&existing_instrument_id),
7593 Some(strategy_id)
7594 );
7595 assert_eq!(
7596 node.kernel
7597 .exec_engine
7598 .borrow()
7599 .get_external_order_claim(&existing_instrument_id),
7600 Some(strategy_id)
7601 );
7602 assert_eq!(
7603 node.exec_manager
7604 .get_external_order_claim(&configured_instrument_id),
7605 None
7606 );
7607 assert_eq!(
7608 node.kernel
7609 .exec_engine
7610 .borrow()
7611 .get_external_order_claim(&configured_instrument_id),
7612 None
7613 );
7614 }
7615
7616 #[rstest]
7617 fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7618 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7619 .unwrap()
7620 .with_reconciliation(false)
7621 .build()
7622 .unwrap();
7623 let exec_engine = node.kernel.exec_engine.clone();
7624 let _engine_borrow = exec_engine.borrow_mut();
7625
7626 node.add_strategy(TestStrategy::new(StrategyConfig {
7627 strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7628 ..Default::default()
7629 }))
7630 .unwrap();
7631 }
7632
7633 #[rstest]
7634 fn test_add_strategy_registers_configured_hedging_oms_type() {
7635 let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7636 .unwrap()
7637 .with_reconciliation(false)
7638 .with_delay_post_stop_secs(0)
7639 .with_timeout_connection(1)
7640 .build()
7641 .unwrap();
7642 let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7643
7644 node.add_strategy(TestStrategy::new(StrategyConfig {
7645 strategy_id: Some(strategy_id),
7646 oms_type: Some(OmsType::Hedging),
7647 ..Default::default()
7648 }))
7649 .unwrap();
7650
7651 let instrument = crypto_perpetual_ethusdt();
7652 let instrument_id = instrument.id();
7653 let client_id = ClientId::from("STUB");
7654
7655 node.kernel
7656 .cache
7657 .borrow_mut()
7658 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7659 .unwrap();
7660 node.kernel
7661 .exec_engine
7662 .borrow_mut()
7663 .register_client(Box::new(StubExecutionClient::new(
7664 client_id,
7665 AccountId::from("TEST-ACCOUNT"),
7666 instrument_id.venue,
7667 OmsType::Netting,
7668 None,
7669 )))
7670 .unwrap();
7671
7672 let order = OrderTestBuilder::new(OrderType::Market)
7673 .trader_id(node.trader_id())
7674 .strategy_id(strategy_id)
7675 .instrument_id(instrument_id)
7676 .quantity(Quantity::from("1.000"))
7677 .build();
7678 let position_id = PositionId::new("CUSTOM-POSITION-001");
7679
7680 node.kernel
7681 .cache
7682 .borrow_mut()
7683 .add_order(order.clone(), Some(position_id), Some(client_id), true)
7684 .unwrap();
7685
7686 let submit_order = SubmitOrder::new(
7687 order.trader_id(),
7688 Some(client_id),
7689 strategy_id,
7690 instrument_id,
7691 order.client_order_id(),
7692 order.init_event().clone(),
7693 order.exec_algorithm_id(),
7694 Some(position_id),
7695 None,
7696 UUID4::new(),
7697 UnixNanos::default(),
7698 None,
7699 );
7700
7701 node.kernel
7702 .exec_engine
7703 .borrow()
7704 .execute(TradingCommand::SubmitOrder(submit_order));
7705
7706 let exec_engine = node.kernel.exec_engine.borrow();
7707 let cache = exec_engine.cache().borrow();
7708 let cached_order = cache
7709 .order(&order.client_order_id())
7710 .expect("Order should be cached");
7711
7712 assert_eq!(cached_order.status(), OrderStatus::Initialized);
7713 }
7714
7715 #[cfg(all(feature = "simulation", madsim))]
7716 async fn advance_clock(d: Duration) {
7717 madsim::time::advance(d);
7718 madsim::task::yield_now().await;
7719 }
7720
7721 #[cfg(not(all(feature = "simulation", madsim)))]
7722 async fn advance_clock(d: Duration) {
7723 tokio::time::advance(d).await;
7724 }
7725
7726 #[rstest]
7727 #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7728 #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7729 #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7730 #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7731 #[case::unapplied_cancel_default(
7732 SubmissionRecoveryPolicy::ResolveLocally,
7733 false,
7734 Some(OrderStatus::PendingCancel),
7735 false
7736 )]
7737 #[case::unapplied_modify_default(
7738 SubmissionRecoveryPolicy::ResolveLocally,
7739 false,
7740 Some(OrderStatus::PendingUpdate),
7741 false
7742 )]
7743 #[case::unapplied_cancel_diagnostics(
7744 SubmissionRecoveryPolicy::RetainUnresolved,
7745 false,
7746 Some(OrderStatus::PendingCancel),
7747 false
7748 )]
7749 #[case::unapplied_modify_diagnostics(
7750 SubmissionRecoveryPolicy::RetainUnresolved,
7751 false,
7752 Some(OrderStatus::PendingUpdate),
7753 false
7754 )]
7755 #[case::first_fill_cancel_default(
7756 SubmissionRecoveryPolicy::ResolveLocally,
7757 false,
7758 Some(OrderStatus::PendingCancel),
7759 true
7760 )]
7761 #[case::first_fill_modify_default(
7762 SubmissionRecoveryPolicy::ResolveLocally,
7763 false,
7764 Some(OrderStatus::PendingUpdate),
7765 true
7766 )]
7767 #[case::first_fill_cancel_diagnostics(
7768 SubmissionRecoveryPolicy::RetainUnresolved,
7769 false,
7770 Some(OrderStatus::PendingCancel),
7771 true
7772 )]
7773 #[case::first_fill_modify_diagnostics(
7774 SubmissionRecoveryPolicy::RetainUnresolved,
7775 false,
7776 Some(OrderStatus::PendingUpdate),
7777 true
7778 )]
7779 #[cfg_attr(
7780 not(all(feature = "simulation", madsim)),
7781 tokio::test(start_paused = true)
7782 )]
7783 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7784 async fn test_submission_registry_publishes_after_native_resolution(
7785 #[case] policy: SubmissionRecoveryPolicy,
7786 #[case] order_list: bool,
7787 #[case] pending_status: Option<OrderStatus>,
7788 #[case] first_fill: bool,
7789 ) {
7790 let config = LiveNodeConfig {
7791 exec_engine: crate::config::LiveExecutionEngineConfig {
7792 inflight_check_threshold_ms: 100,
7793 inflight_check_retries: 2,
7794 submission_recovery_policy: policy,
7795 ..Default::default()
7796 },
7797 ..Default::default()
7798 };
7799 let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7800 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7801 let client_id = ClientId::from("TEST-SUBMISSION");
7802 let account_id = AccountId::from("TEST-001");
7803 let strategy_id = StrategyId::from("SUBMISSION-001");
7804 let account = AccountAny::Margin(MarginAccount::new(
7805 AccountState::new(
7806 account_id,
7807 AccountType::Margin,
7808 vec![AccountBalance::new(
7809 Money::from("1000000 USDT"),
7810 Money::from("0 USDT"),
7811 Money::from("1000000 USDT"),
7812 )],
7813 Vec::new(),
7814 true,
7815 UUID4::new(),
7816 UnixNanos::default(),
7817 UnixNanos::default(),
7818 Some(Currency::USDT()),
7819 ),
7820 true,
7821 ));
7822 node.kernel.cache.borrow_mut().add_account(account).unwrap();
7823 node.kernel
7824 .cache
7825 .borrow_mut()
7826 .add_instrument(instrument.clone())
7827 .unwrap();
7828 let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7829 .map(|index| {
7830 OrderTestBuilder::new(OrderType::Limit)
7831 .trader_id(node.trader_id())
7832 .strategy_id(strategy_id)
7833 .client_order_id(ClientOrderId::from(
7834 format!("O-SUBMISSION-{index}").as_str(),
7835 ))
7836 .instrument_id(instrument.id())
7837 .quantity(Quantity::from("1.000"))
7838 .price(Price::from("100.00"))
7839 .build()
7840 })
7841 .collect();
7842 let command = if order_list {
7843 TradingCommand::SubmitOrderList(SubmitOrderList::new(
7844 node.trader_id(),
7845 Some(client_id),
7846 strategy_id,
7847 OrderList::new(
7848 OrderListId::from("OL-SUBMISSION"),
7849 instrument.id(),
7850 strategy_id,
7851 orders.iter().map(Order::client_order_id).collect(),
7852 UnixNanos::default(),
7853 ),
7854 orders
7855 .iter()
7856 .map(|order| order.init_event().clone())
7857 .collect(),
7858 None,
7859 None,
7860 None,
7861 UUID4::new(),
7862 UnixNanos::default(),
7863 None,
7864 ))
7865 } else {
7866 TradingCommand::SubmitOrder(SubmitOrder::from_order(
7867 &orders[0],
7868 node.trader_id(),
7869 Some(client_id),
7870 None,
7871 UUID4::new(),
7872 UnixNanos::default(),
7873 ))
7874 };
7875 node.observe_exec_command_before_dispatch(&command);
7876
7877 for order in &orders {
7878 let submitted = TestOrderEventStubs::submitted(order, account_id);
7879 node.kernel
7880 .cache
7881 .borrow_mut()
7882 .add_order(order.clone(), None, Some(client_id), false)
7883 .unwrap();
7884 node.process_exec_event(ExecutionEvent::Order(submitted));
7885 }
7886
7887 let expected_status = match pending_status {
7888 Some(status) if !first_fill => status,
7889 Some(_) => OrderStatus::Canceled,
7890 None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7891 None => OrderStatus::Rejected,
7892 };
7893 let diagnostics = Rc::new(RefCell::new(Vec::new()));
7894 let handler = ShareableMessageHandler::from_typed({
7895 let diagnostics = diagnostics.clone();
7896 let cache = node.kernel.cache.clone();
7897 move |diagnostic: &SubmissionRecoveryExhausted| {
7898 let cache = cache.borrow_mut();
7899 let order = cache.order(&diagnostic.client_order_id).unwrap();
7900 assert_eq!(order.status(), expected_status);
7901 assert!(diagnostic.ts_event >= order.ts_last());
7902 diagnostics.borrow_mut().push(diagnostic.clone());
7903 }
7904 });
7905 let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7906 msgbus::subscribe_any(topic.into(), handler.clone(), None);
7907 let queries = Rc::new(RefCell::new(Vec::new()));
7908 msgbus::register_trading_command_endpoint(
7909 MessagingSwitchboard::exec_engine_execute(),
7910 TypedIntoHandler::from({
7911 let queries = queries.clone();
7912 move |command: TradingCommand| queries.borrow_mut().push(command)
7913 }),
7914 );
7915
7916 let mut last_inflight_check = dst::time::Instant::now();
7917 let mut last_open_check = last_inflight_check;
7918 let mut last_position_check = last_inflight_check;
7919 let mut open_order_report_task = None;
7920 let mut targeted_order_report_task = None;
7921 let mut position_report_task = None;
7922
7923 for check in 1..=3 {
7924 advance_clock(Duration::from_millis(101)).await;
7925 node.run_reconciliation_checks(
7926 dst::time::Instant::now(),
7927 ReconciliationCheckIntervals {
7928 inflight: Duration::from_millis(100),
7929 open: Duration::ZERO,
7930 position: Duration::ZERO,
7931 },
7932 &mut ReconciliationCheckState {
7933 last_inflight_check: &mut last_inflight_check,
7934 last_open_check: &mut last_open_check,
7935 last_position_check: &mut last_position_check,
7936 open_order_report_task: &mut open_order_report_task,
7937 targeted_order_report_task: &mut targeted_order_report_task,
7938 position_report_task: &mut position_report_task,
7939 },
7940 );
7941
7942 if check == 1 {
7943 assert!(diagnostics.borrow().is_empty());
7944
7945 if let Some(pending_status) = pending_status {
7946 let order = &orders[0];
7947 let pending = if pending_status == OrderStatus::PendingCancel {
7948 OrderEventAny::PendingCancel(
7949 OrderPendingCancelSpec::builder()
7950 .trader_id(node.trader_id())
7951 .strategy_id(strategy_id)
7952 .instrument_id(instrument.id())
7953 .client_order_id(order.client_order_id())
7954 .account_id(account_id)
7955 .build(),
7956 )
7957 } else {
7958 OrderEventAny::PendingUpdate(
7959 OrderPendingUpdateSpec::builder()
7960 .trader_id(node.trader_id())
7961 .strategy_id(strategy_id)
7962 .instrument_id(instrument.id())
7963 .client_order_id(order.client_order_id())
7964 .account_id(account_id)
7965 .build(),
7966 )
7967 };
7968 node.process_exec_event(ExecutionEvent::Order(pending));
7969 node.exec_manager
7970 .register_inflight(order.client_order_id(), UUID4::new());
7971
7972 if first_fill {
7973 let cached = node
7974 .kernel
7975 .cache
7976 .borrow()
7977 .order(&order.client_order_id())
7978 .unwrap()
7979 .clone();
7980 let OrderEventAny::Filled(mut fill) =
7981 OrderFilledTestBuilder::new(&cached, &instrument)
7982 .last_qty(Quantity::from("0.400"))
7983 .last_px(Price::from("100.00"))
7984 .commission(Money::from("0 USDT"))
7985 .without_position_id()
7986 .build()
7987 else {
7988 unreachable!()
7989 };
7990 fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7991 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7992 } else {
7993 let report = OrderStatusReport::new(
7994 account_id,
7995 instrument.id(),
7996 Some(order.client_order_id()),
7997 VenueOrderId::from("V-SUBMISSION"),
7998 Some(order.order_side()),
7999 order.order_type(),
8000 order.time_in_force(),
8001 OrderStatus::Submitted,
8002 order.quantity(),
8003 Quantity::zero(3),
8004 UnixNanos::default(),
8005 UnixNanos::default(),
8006 UnixNanos::default(),
8007 None,
8008 )
8009 .with_price(Price::from("100.00"));
8010 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
8011 Box::new(report),
8012 )));
8013 }
8014 let cache = node.kernel.cache.borrow();
8015 let cached = cache.order(&order.client_order_id()).unwrap();
8016 assert_eq!(cached.status(), pending_status);
8017 assert_eq!(
8018 cached.filled_qty(),
8019 if first_fill {
8020 Quantity::from("0.400")
8021 } else {
8022 Quantity::zero(3)
8023 }
8024 );
8025 assert!(
8026 !cached
8027 .events()
8028 .iter()
8029 .any(|event| matches!(event, OrderEventAny::Accepted(_)))
8030 );
8031 }
8032 }
8033 }
8034
8035 assert_eq!(
8036 queries.borrow().len(),
8037 orders.len()
8038 + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
8039 );
8040
8041 for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
8042 assert!(matches!(command, TradingCommand::QueryOrder(query)
8043 if query.client_order_id == order.client_order_id()
8044 && query.client_id == Some(client_id)));
8045 assert_eq!(
8046 node.kernel
8047 .cache
8048 .borrow()
8049 .order(&order.client_order_id())
8050 .unwrap()
8051 .status(),
8052 if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
8053 pending_status.unwrap()
8054 } else {
8055 expected_status
8056 }
8057 );
8058 }
8059 let diagnostics = diagnostics.borrow();
8060 assert_eq!(
8061 diagnostics.len(),
8062 if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
8063 orders.len()
8064 } else {
8065 0
8066 }
8067 );
8068
8069 for (diagnostic, order) in diagnostics.iter().zip(&orders) {
8070 assert_eq!(diagnostic.trader_id, node.trader_id());
8071 assert_eq!(diagnostic.client_id, Some(client_id));
8072 assert_eq!(diagnostic.strategy_id, strategy_id);
8073 assert_eq!(diagnostic.instrument_id, instrument.id());
8074 assert_eq!(diagnostic.client_order_id, order.client_order_id());
8075 assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
8076 assert_eq!(diagnostic.retry_count, 2);
8077 }
8078 msgbus::unsubscribe_any(topic.into(), &handler);
8079 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8080 }
8081
8082 #[derive(Debug)]
8083 struct SubmissionCallbackStrategy {
8084 core: StrategyCore,
8085 cancel: bool,
8086 batch: bool,
8087 callback: &'static str,
8088 client_id: ClientId,
8089 }
8090
8091 impl DataActor for SubmissionCallbackStrategy {}
8092
8093 impl SubmissionCallbackStrategy {
8094 fn queue_command(&mut self, client_order_id: ClientOrderId) {
8095 match (self.cancel, self.batch) {
8096 (true, true) => {
8097 self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
8098 .unwrap();
8099 }
8100 (true, false) => {
8101 self.cancel_order(client_order_id, Some(self.client_id), None)
8102 .unwrap();
8103 }
8104 (false, true) => {
8105 self.modify_orders(
8106 vec![(client_order_id, None, Some(Price::from("101.00")), None)],
8107 Some(self.client_id),
8108 None,
8109 )
8110 .unwrap();
8111 }
8112 (false, false) => self
8113 .modify_order(
8114 client_order_id,
8115 None,
8116 Some(Price::from("101.00")),
8117 None,
8118 Some(self.client_id),
8119 None,
8120 )
8121 .unwrap(),
8122 }
8123 }
8124 }
8125
8126 nautilus_strategy!(SubmissionCallbackStrategy, {
8127 fn on_order_accepted(&mut self, event: OrderAccepted) {
8128 if self.callback == "accepted" {
8129 self.queue_command(event.client_order_id);
8130 }
8131 }
8132
8133 fn on_order_updated(&mut self, event: OrderUpdated) {
8134 if self.callback == "updated" {
8135 self.queue_command(event.client_order_id);
8136 }
8137 }
8138 });
8139
8140 #[rstest]
8141 #[case::native(false, "accepted")]
8142 #[case::report(true, "accepted")]
8143 #[case::updated(false, "updated")]
8144 #[cfg_attr(
8145 not(all(feature = "simulation", madsim)),
8146 tokio::test(start_paused = true)
8147 )]
8148 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8149 async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
8150 #[values(
8151 SubmissionRecoveryPolicy::ResolveLocally,
8152 SubmissionRecoveryPolicy::RetainUnresolved
8153 )]
8154 policy: SubmissionRecoveryPolicy,
8155 #[values(false, true)] cancel: bool,
8156 #[values(false, true)] batch: bool,
8157 #[case] report: bool,
8158 #[case] callback: &'static str,
8159 ) {
8160 let config = LiveNodeConfig {
8161 exec_engine: crate::config::LiveExecutionEngineConfig {
8162 inflight_check_threshold_ms: 100,
8163 inflight_check_retries: 2,
8164 submission_recovery_policy: policy,
8165 ..Default::default()
8166 },
8167 ..Default::default()
8168 };
8169 let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
8170 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8171 let account_id = AccountId::from("TEST-001");
8172 let strategy_id = StrategyId::from("CALLBACK-001");
8173 let client_id = ClientId::from("TEST-CALLBACK");
8174 let account = AccountAny::Margin(MarginAccount::new(
8175 AccountState::new(
8176 account_id,
8177 AccountType::Margin,
8178 vec![AccountBalance::new(
8179 Money::from("1000000 USDT"),
8180 Money::from("0 USDT"),
8181 Money::from("1000000 USDT"),
8182 )],
8183 Vec::new(),
8184 true,
8185 UUID4::new(),
8186 UnixNanos::default(),
8187 UnixNanos::default(),
8188 Some(Currency::USDT()),
8189 ),
8190 true,
8191 ));
8192 node.kernel.cache.borrow_mut().add_account(account).unwrap();
8193 node.kernel
8194 .cache
8195 .borrow_mut()
8196 .add_instrument(instrument.clone())
8197 .unwrap();
8198 node.kernel
8199 .exec_engine
8200 .borrow_mut()
8201 .register_client(Box::new(StubExecutionClient::new(
8202 client_id,
8203 account_id,
8204 instrument.id().venue,
8205 OmsType::Netting,
8206 None,
8207 )))
8208 .unwrap();
8209 node.add_strategy(SubmissionCallbackStrategy {
8210 core: StrategyCore::new(StrategyConfig {
8211 strategy_id: Some(strategy_id),
8212 ..Default::default()
8213 }),
8214 cancel,
8215 batch,
8216 callback,
8217 client_id,
8218 })
8219 .unwrap();
8220 node.kernel.trader.borrow_mut().initialize().unwrap();
8221 node.kernel.start_trader().unwrap();
8222 let runner = node.runner.take().unwrap();
8223 runner.bind_senders_for_node(node.handle.clone());
8224 let mut channels = runner.take_channels();
8225 let order = OrderTestBuilder::new(OrderType::Limit)
8226 .trader_id(node.trader_id())
8227 .strategy_id(strategy_id)
8228 .client_order_id(ClientOrderId::from("O-CALLBACK"))
8229 .instrument_id(instrument.id())
8230 .quantity(Quantity::from("1.000"))
8231 .price(Price::from("100.00"))
8232 .build();
8233 let client_order_id = order.client_order_id();
8234 let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
8235 &order,
8236 node.trader_id(),
8237 Some(client_id),
8238 None,
8239 UUID4::new(),
8240 UnixNanos::default(),
8241 ));
8242 node.observe_exec_command_before_dispatch(&submit);
8243 node.kernel
8244 .cache
8245 .borrow_mut()
8246 .add_order(order.clone(), None, Some(client_id), false)
8247 .unwrap();
8248 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
8249 &order, account_id,
8250 )));
8251 advance_clock(Duration::from_millis(101)).await;
8252 let first = node.exec_manager.check_inflight_orders();
8253 assert_eq!(first.queries.len(), 1);
8254 assert!(first.events.is_empty());
8255
8256 advance_clock(Duration::from_millis(101)).await;
8257 let accepted =
8258 TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
8259
8260 if callback == "updated" {
8261 let mut update = OrderUpdatedSpec::builder()
8262 .trader_id(node.trader_id())
8263 .strategy_id(strategy_id)
8264 .instrument_id(instrument.id())
8265 .client_order_id(client_order_id)
8266 .account_id(account_id)
8267 .quantity(order.quantity())
8268 .price(order.price().unwrap())
8269 .build();
8270 update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
8271 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
8272 } else if report {
8273 let accepted = OrderStatusReport::new(
8274 account_id,
8275 instrument.id(),
8276 Some(client_order_id),
8277 VenueOrderId::from("V-CALLBACK"),
8278 Some(order.order_side()),
8279 order.order_type(),
8280 order.time_in_force(),
8281 OrderStatus::Accepted,
8282 order.quantity(),
8283 Quantity::zero(3),
8284 UnixNanos::default(),
8285 UnixNanos::default(),
8286 UnixNanos::default(),
8287 None,
8288 )
8289 .with_price(Price::from("100.00"));
8290 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
8291 accepted,
8292 ))));
8293 } else {
8294 node.process_exec_event(ExecutionEvent::Order(accepted));
8295 }
8296
8297 let pending_status = if cancel {
8298 OrderStatus::PendingCancel
8299 } else {
8300 OrderStatus::PendingUpdate
8301 };
8302 assert_eq!(channels.exec_cmd_rx.len(), 1);
8303 assert_eq!(
8304 node.kernel
8305 .cache
8306 .borrow()
8307 .order(&client_order_id)
8308 .unwrap()
8309 .status(),
8310 pending_status
8311 );
8312
8313 for _ in 0..3 {
8315 advance_clock(Duration::from_millis(101)).await;
8316 let maintenance = node.exec_manager.check_inflight_orders();
8317 assert!(maintenance.events.is_empty());
8318 assert!(maintenance.queries.is_empty());
8319 assert!(
8320 node.exec_manager
8321 .take_submission_recovery_exhaustions()
8322 .is_empty()
8323 );
8324 }
8325
8326 let command = channels.exec_cmd_rx.try_recv().unwrap();
8327 node.process_exec_command(command);
8328 assert!(channels.exec_cmd_rx.is_empty());
8329 let immediate = node.exec_manager.check_inflight_orders();
8330 assert!(immediate.events.is_empty());
8331 assert!(immediate.queries.is_empty());
8332 advance_clock(Duration::from_millis(101)).await;
8333 let first_command_check = node.exec_manager.check_inflight_orders();
8334 assert!(first_command_check.events.is_empty());
8335 assert_eq!(first_command_check.queries.len(), 1);
8336 advance_clock(Duration::from_millis(101)).await;
8337 let exhausted = node.exec_manager.check_inflight_orders();
8338 assert!(exhausted.queries.is_empty());
8339 assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
8340 assert!(
8341 node.exec_manager
8342 .take_submission_recovery_exhaustions()
8343 .is_empty()
8344 );
8345 node.process_reconciliation_events(&exhausted.events);
8346
8347 let expected_status = match policy {
8348 SubmissionRecoveryPolicy::ResolveLocally => OrderStatus::Canceled,
8349 SubmissionRecoveryPolicy::RetainUnresolved if callback == "updated" => {
8350 OrderStatus::Submitted
8351 }
8352 SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Accepted,
8353 };
8354
8355 assert_eq!(
8356 node.kernel
8357 .cache
8358 .borrow()
8359 .order(&client_order_id)
8360 .unwrap()
8361 .status(),
8362 expected_status
8363 );
8364 }
8365
8366 #[cfg_attr(
8367 not(all(feature = "simulation", madsim)),
8368 tokio::test(start_paused = true)
8369 )]
8370 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8371 async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
8372 let last = dst::time::Instant::now();
8373 let interval = Duration::from_millis(100);
8374
8375 assert!(!reconciliation_check_due(last, last, Duration::ZERO));
8376 assert!(!reconciliation_check_due(last, last, interval));
8377
8378 advance_clock(Duration::from_millis(99)).await;
8379 let before_interval = dst::time::Instant::now();
8380 assert!(!reconciliation_check_due(before_interval, last, interval));
8381
8382 advance_clock(Duration::from_millis(1)).await;
8383 let at_interval = dst::time::Instant::now();
8384 assert!(reconciliation_check_due(at_interval, last, interval));
8385
8386 assert!(!reconciliation_check_due(last, at_interval, interval));
8387 }
8388
8389 #[cfg_attr(
8390 not(all(feature = "simulation", madsim)),
8391 tokio::test(start_paused = true)
8392 )]
8393 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8394 async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
8395 let config = LiveNodeConfig {
8396 exec_engine: crate::config::LiveExecutionEngineConfig {
8397 reconciliation: true,
8398 open_check_interval_secs: Some(1.0),
8399 position_check_interval_secs: Some(1.0),
8400 max_single_order_queries_per_cycle: 5,
8401 ..Default::default()
8402 },
8403 ..Default::default()
8404 };
8405
8406 let mut node =
8407 LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
8408 let client_id = ClientId::from("TEST-QUERY");
8409 let account_id = AccountId::from("TEST-QUERY-001");
8410
8411 let trading_commands = Rc::new(RefCell::new(Vec::new()));
8412 msgbus::register_trading_command_endpoint(
8413 MessagingSwitchboard::exec_engine_execute(),
8414 TypedIntoHandler::from({
8415 let trading_commands = trading_commands.clone();
8416 move |command: TradingCommand| {
8417 trading_commands.borrow_mut().push(command);
8418 }
8419 }),
8420 );
8421
8422 let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
8423 let instrument = crypto_perpetual_ethusdt();
8424 let instrument_id = instrument.id();
8425 let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
8426
8427 node.kernel
8428 .cache
8429 .borrow_mut()
8430 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
8431 .unwrap();
8432 insert_accepted_limit_order_in_node(
8433 &node,
8434 account_id,
8435 client_id,
8436 instrument_id,
8437 client_order_id,
8438 venue_order_id,
8439 );
8440
8441 let last = dst::time::Instant::now();
8442 advance_clock(Duration::from_nanos(1)).await;
8443 let now = dst::time::Instant::now();
8444 let mut last_inflight_check = last;
8445 let mut last_open_check = last;
8446 let mut last_position_check = last;
8447 let mut open_order_report_task = None;
8448 let mut targeted_order_report_task = None;
8449 let mut position_report_task = None;
8450
8451 node.run_reconciliation_checks(
8452 now,
8453 ReconciliationCheckIntervals {
8454 inflight: Duration::ZERO,
8455 open: Duration::from_nanos(1),
8456 position: Duration::ZERO,
8457 },
8458 &mut ReconciliationCheckState {
8459 last_inflight_check: &mut last_inflight_check,
8460 last_open_check: &mut last_open_check,
8461 last_position_check: &mut last_position_check,
8462 open_order_report_task: &mut open_order_report_task,
8463 targeted_order_report_task: &mut targeted_order_report_task,
8464 position_report_task: &mut position_report_task,
8465 },
8466 );
8467
8468 let commands = trading_commands.borrow();
8469
8470 assert!(commands.is_empty());
8471 assert!(open_order_report_task.is_none());
8472 assert!(targeted_order_report_task.is_none());
8473 assert!(position_report_task.is_none());
8474
8475 ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8476 }
8477
8478 #[rstest]
8479 #[tokio::test]
8480 async fn test_recurring_report_failure_preserves_client_coverage(
8481 #[values(false, true)] positions: bool,
8482 #[values(false, true)] failure_first: bool,
8483 ) {
8484 let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8485 let failed_id = ClientId::from("FAILED");
8486 let healthy_id = ClientId::from("HEALTHY");
8487
8488 for (client_id, outcome) in [
8489 (failed_id, FillReportClientOutcome::Failure),
8490 (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8491 ] {
8492 node.exec_clients
8493 .push(LiveExecutionClient::new(Box::new(FillReportClient {
8494 client_id,
8495 account_id: AccountId::from("TEST-001"),
8496 venue: Venue::from("BINANCE"),
8497 outcome,
8498 commands: Rc::new(RefCell::new(Vec::new())),
8499 })));
8500 }
8501
8502 if !failure_first {
8503 node.exec_clients.reverse();
8504 }
8505
8506 let last = dst::time::Instant::now();
8507 let now = last + Duration::from_secs(1);
8508 let mut last_inflight = last;
8509 let mut last_open = last;
8510 let mut last_position = last;
8511 let mut open_task = None;
8512 let mut targeted_task = None;
8513 let mut position_task = None;
8514
8515 node.run_reconciliation_checks(
8516 now,
8517 ReconciliationCheckIntervals {
8518 inflight: Duration::ZERO,
8519 open: if positions {
8520 Duration::ZERO
8521 } else {
8522 Duration::from_secs(1)
8523 },
8524 position: if positions {
8525 Duration::from_secs(1)
8526 } else {
8527 Duration::ZERO
8528 },
8529 },
8530 &mut ReconciliationCheckState {
8531 last_inflight_check: &mut last_inflight,
8532 last_open_check: &mut last_open,
8533 last_position_check: &mut last_position,
8534 open_order_report_task: &mut open_task,
8535 targeted_order_report_task: &mut targeted_task,
8536 position_report_task: &mut position_task,
8537 },
8538 );
8539
8540 let (queried_clients, failed_clients) = if positions {
8541 let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8542 position_task.unwrap().future.await
8543 else {
8544 panic!("position collection should complete despite one client failure");
8545 };
8546
8547 let result = result.finish();
8548 assert!(result.reports.is_empty());
8549 assert!(open_task.is_none());
8550 (result.queried_clients, result.failed_clients)
8551 } else {
8552 let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8553 panic!("order collection should complete despite one client failure");
8554 };
8555
8556 let result = result.finish();
8557
8558 assert!(result.reports.is_empty());
8559 assert!(position_task.is_none());
8560 (result.queried_clients, result.failed_clients)
8561 };
8562
8563 assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8564 assert_eq!(failed_clients, IndexSet::from([failed_id]));
8565 assert!(targeted_task.is_none());
8566 }
8567
8568 #[rstest]
8569 #[case::inflight(true, false)]
8570 #[case::open(false, false)]
8571 #[case::positions(false, true)]
8572 #[tokio::test]
8573 async fn test_reconciliation_shutdown_preserves_pending_checks(
8574 #[case] inflight: bool,
8575 #[case] positions: bool,
8576 ) {
8577 let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8578 node.handle.set_shutting_down();
8579 let last = dst::time::Instant::now();
8580 let now = last + Duration::from_secs(1);
8581 let mut last_inflight = last;
8582 let mut last_open = last;
8583 let mut last_position = last;
8584 let mut open_task = None;
8585 let mut targeted_task = None;
8586 let mut position_task = None;
8587
8588 node.run_reconciliation_checks(
8589 now,
8590 ReconciliationCheckIntervals {
8591 inflight: if inflight {
8592 Duration::from_secs(1)
8593 } else {
8594 Duration::ZERO
8595 },
8596 open: if !inflight && !positions {
8597 Duration::from_secs(1)
8598 } else {
8599 Duration::ZERO
8600 },
8601 position: if positions {
8602 Duration::from_secs(1)
8603 } else {
8604 Duration::ZERO
8605 },
8606 },
8607 &mut ReconciliationCheckState {
8608 last_inflight_check: &mut last_inflight,
8609 last_open_check: &mut last_open,
8610 last_position_check: &mut last_position,
8611 open_order_report_task: &mut open_task,
8612 targeted_order_report_task: &mut targeted_task,
8613 position_report_task: &mut position_task,
8614 },
8615 );
8616
8617 assert_eq!(
8618 (last_inflight, last_open, last_position),
8619 (last, last, last)
8620 );
8621 assert!(open_task.is_none());
8622 assert!(targeted_task.is_none());
8623 assert!(position_task.is_none());
8624 }
8625
8626 fn insert_accepted_limit_order_in_node(
8627 node: &LiveNode,
8628 account_id: AccountId,
8629 client_id: ClientId,
8630 instrument_id: InstrumentId,
8631 client_order_id: ClientOrderId,
8632 venue_order_id: VenueOrderId,
8633 ) {
8634 let order = OrderTestBuilder::new(OrderType::Limit)
8635 .client_order_id(client_order_id)
8636 .instrument_id(instrument_id)
8637 .quantity(Quantity::from("10.0"))
8638 .price(Price::from("100.0"))
8639 .build();
8640 let submitted = TestOrderEventStubs::submitted(&order, account_id);
8641 node.kernel
8642 .cache
8643 .borrow_mut()
8644 .add_order(order, None, Some(client_id), false)
8645 .unwrap();
8646 let order = node
8647 .kernel
8648 .cache
8649 .borrow_mut()
8650 .update_order(&submitted)
8651 .unwrap();
8652 let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8653 node.kernel
8654 .cache
8655 .borrow_mut()
8656 .update_order(&accepted)
8657 .unwrap();
8658 }
8659
8660 fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8661 let config = LiveNodeConfig {
8662 exec_engine: crate::config::LiveExecutionEngineConfig {
8663 reconciliation: true,
8664 ..Default::default()
8665 },
8666 ..Default::default()
8667 };
8668
8669 let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8670 let account_id = AccountId::from("TEST-001");
8671 let client_id = ClientId::from("TEST-RECENT-FILL");
8672 let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8673 let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8674 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8675 node.kernel
8676 .cache
8677 .borrow_mut()
8678 .add_instrument(instrument.clone())
8679 .unwrap();
8680 insert_accepted_limit_order_in_node(
8681 &node,
8682 account_id,
8683 client_id,
8684 instrument.id(),
8685 client_order_id,
8686 venue_order_id,
8687 );
8688 let order = node
8689 .kernel
8690 .cache
8691 .borrow()
8692 .order_owned(&client_order_id)
8693 .unwrap();
8694 let fill = TestOrderEventStubs::filled(
8695 &order,
8696 &instrument,
8697 Some(TradeId::from("T-RECENT-FILL")),
8698 None,
8699 Some(Price::from("100.0")),
8700 Some(Quantity::from("1.0")),
8701 Some(LiquiditySide::Taker),
8702 None,
8703 None,
8704 Some(account_id),
8705 );
8706
8707 (node, fill, instrument)
8708 }
8709
8710 fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8711 let client_order_id = fill_report.client_order_id.unwrap();
8712 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8713 let order = node
8714 .kernel
8715 .cache
8716 .borrow()
8717 .order_owned(&client_order_id)
8718 .unwrap();
8719 let order_report = OrderStatusReport::new(
8720 fill_report.account_id,
8721 fill_report.instrument_id,
8722 Some(client_order_id),
8723 fill_report.venue_order_id,
8724 OrderSide::Buy.into(),
8725 OrderType::Limit,
8726 TimeInForce::Gtc,
8727 OrderStatus::PartiallyFilled,
8728 Quantity::from("10.0"),
8729 Quantity::from("2.0"),
8730 UnixNanos::from(1_000),
8731 UnixNanos::from(1_000),
8732 UnixNanos::from(1_000),
8733 None,
8734 )
8735 .with_avg_px(dec!(100.0));
8736 let inferred = create_inferred_fill_for_qty(
8737 &order,
8738 &order_report,
8739 &fill_report.account_id,
8740 &instrument,
8741 Quantity::from("1.0"),
8742 UnixNanos::from(1_000),
8743 None,
8744 )
8745 .unwrap();
8746
8747 node.process_reconciliation_events(&[inferred]);
8748 }
8749
8750 #[rstest]
8751 #[case::rejected(false, "10.0", "10.0", "1.0")]
8752 #[case::dust(true, "9.004", "10.004", "10.004")]
8753 #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8754 fn test_position_check_overfill_raises_before_fill_when_client_allows(
8755 #[case] allow: bool,
8756 #[case] fill_qty: &str,
8757 #[case] expected_qty: &str,
8758 #[case] expected_filled: &str,
8759 ) {
8760 let (mut node, venue_report, mut fill_report) =
8761 position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8762 fill_report.commission = Money::from("1.25 USDT");
8763 let key = (fill_report.instrument_id, fill_report.account_id);
8764
8765 let client = StubExecutionClient::new(
8766 ClientId::from("POSITION-FILLS"),
8767 AccountId::from("TEST-001"),
8768 fill_report.instrument_id.venue,
8769 OmsType::Netting,
8770 None,
8771 );
8772
8773 let client = if allow {
8774 client.with_allows_reconciliation_overfill()
8775 } else {
8776 client
8777 };
8778
8779 node.kernel
8780 .exec_engine
8781 .borrow_mut()
8782 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8783 .unwrap();
8784
8785 let position_result = position_report_result(&node, venue_report);
8786 node.handle_position_fill_report_result(PositionFillReportResult {
8787 position_result,
8788 reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8789 });
8790
8791 let order = node
8792 .kernel
8793 .cache
8794 .borrow()
8795 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8796 .unwrap();
8797 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8798 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8799 let applied = order
8800 .trade_ids()
8801 .iter()
8802 .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8803 assert_eq!(applied, allow);
8804 let revision = node.exec_manager.position_activity_revision(&key);
8805
8806 if !allow {
8807 return;
8808 }
8809
8810 assert_eq!(revision, 2);
8811
8812 {
8813 let cache = node.kernel.cache.borrow();
8814 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8815 assert_eq!(positions.len(), 1);
8816 assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8817 assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8818 assert_eq!(positions[0].trade_ids.len(), 2);
8819 }
8820
8821 let replay_report = position_report_result(
8822 &node,
8823 PositionStatusReport::new(
8824 key.1,
8825 key.0,
8826 PositionSide::Long,
8827 Quantity::from(expected_qty),
8828 UnixNanos::from(2_000),
8829 UnixNanos::from(2_000),
8830 None,
8831 None,
8832 Some(dec!(100.0)),
8833 ),
8834 );
8835 node.handle_position_fill_report_result(PositionFillReportResult {
8836 position_result: replay_report,
8837 reports: IndexMap::from([(key, vec![fill_report])]),
8838 });
8839
8840 let order = node
8841 .kernel
8842 .cache
8843 .borrow()
8844 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8845 .unwrap();
8846 assert_eq!(order.quantity(), Quantity::from(expected_qty));
8847 assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8848 assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8849 }
8850
8851 #[rstest]
8852 fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8853 let (mut node, venue_report, mut overfill) =
8854 position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8855 overfill.commission = Money::from("0.20 USDT");
8856 let key = (overfill.instrument_id, overfill.account_id);
8857 let client = StubExecutionClient::new(
8858 ClientId::from("POSITION-FILLS"),
8859 AccountId::from("TEST-001"),
8860 overfill.instrument_id.venue,
8861 OmsType::Netting,
8862 None,
8863 )
8864 .with_allows_reconciliation_overfill();
8865 node.kernel
8866 .exec_engine
8867 .borrow_mut()
8868 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8869 .unwrap();
8870
8871 let mut fitting = overfill.clone();
8872 fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8873 fitting.last_qty = Quantity::from("8.0");
8874 fitting.commission = Money::from("0.80 USDT");
8875 let position_result = position_report_result(&node, venue_report);
8876 node.handle_position_fill_report_result(PositionFillReportResult {
8877 position_result,
8878 reports: IndexMap::from([(key, vec![fitting, overfill])]),
8879 });
8880
8881 let order = node
8882 .kernel
8883 .cache
8884 .borrow()
8885 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8886 .unwrap();
8887 assert_eq!(order.quantity(), Quantity::from("11.0"));
8888 assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8889 assert_eq!(order.trade_ids().len(), 3);
8890 assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8891 {
8892 let cache = node.kernel.cache.borrow();
8893 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8894 assert_eq!(positions.len(), 1);
8895 assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8896 assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8897 }
8898 }
8899
8900 #[rstest]
8901 fn test_position_check_second_overfill_does_not_raise_filled_order() {
8902 let (mut node, venue_report, mut first) =
8903 position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8904 first.commission = Money::from("1.20 USDT");
8905 let key = (first.instrument_id, first.account_id);
8906 let client = StubExecutionClient::new(
8907 ClientId::from("POSITION-FILLS"),
8908 AccountId::from("TEST-001"),
8909 first.instrument_id.venue,
8910 OmsType::Netting,
8911 None,
8912 )
8913 .with_allows_reconciliation_overfill();
8914 node.kernel
8915 .exec_engine
8916 .borrow_mut()
8917 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8918 .unwrap();
8919
8920 let mut second = first.clone();
8921 second.trade_id = TradeId::from("T-POSITION-LATER");
8922 second.last_qty = Quantity::from("1.0");
8923 second.commission = Money::from("0.10 USDT");
8924 second.ts_event = UnixNanos::from(1_001);
8925 let position_result = position_report_result(&node, venue_report);
8926 node.handle_position_fill_report_result(PositionFillReportResult {
8927 position_result,
8928 reports: IndexMap::from([(key, vec![first, second])]),
8929 });
8930
8931 let order = node
8932 .kernel
8933 .cache
8934 .borrow()
8935 .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8936 .unwrap();
8937 assert_eq!(order.quantity(), Quantity::from("13.0"));
8938 assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8939 assert_eq!(order.status(), OrderStatus::Filled);
8940 assert_eq!(order.trade_ids().len(), 2);
8941 assert!(
8942 order
8943 .trade_ids()
8944 .iter()
8945 .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8946 );
8947 {
8948 let cache = node.kernel.cache.borrow();
8949 let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8950 assert_eq!(positions.len(), 1);
8951 assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8952 assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8953 }
8954 }
8955
8956 #[rstest]
8957 fn test_allow_overfills_applies_fill_without_raising_quantity() {
8958 let config = LiveNodeConfig {
8959 exec_engine: crate::config::LiveExecutionEngineConfig {
8960 reconciliation: true,
8961 allow_overfills: true,
8962 ..Default::default()
8963 },
8964 ..Default::default()
8965 };
8966
8967 let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8968 let account_id = AccountId::from("TEST-001");
8969 let client_id = ClientId::from("POSITION-FILLS");
8970 let client_order_id = ClientOrderId::from("O-ALLOW");
8971 let venue_order_id = VenueOrderId::from("V-ALLOW");
8972 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8973 node.kernel
8974 .cache
8975 .borrow_mut()
8976 .add_instrument(instrument.clone())
8977 .unwrap();
8978 insert_accepted_limit_order_in_node(
8979 &node,
8980 account_id,
8981 client_id,
8982 instrument.id(),
8983 client_order_id,
8984 venue_order_id,
8985 );
8986 let client = StubExecutionClient::new(
8987 client_id,
8988 account_id,
8989 instrument.id().venue,
8990 OmsType::Netting,
8991 None,
8992 )
8993 .with_allows_reconciliation_overfill();
8994 node.kernel
8995 .exec_engine
8996 .borrow_mut()
8997 .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8998 .unwrap();
8999
9000 let fill = FillReport::new(
9001 account_id,
9002 instrument.id(),
9003 venue_order_id,
9004 TradeId::from("T-ALLOW"),
9005 OrderSide::Buy,
9006 Quantity::from("12.0"),
9007 Price::from("100.0"),
9008 Money::from("1.00 USDT"),
9009 LiquiditySide::Taker,
9010 Some(client_order_id),
9011 None,
9012 UnixNanos::from(1_000),
9013 UnixNanos::from(1_000),
9014 None,
9015 );
9016 node.kernel
9017 .exec_engine
9018 .borrow_mut()
9019 .reconcile_fill_report(&fill);
9020
9021 let order = node
9022 .kernel
9023 .cache
9024 .borrow()
9025 .order_owned(&client_order_id)
9026 .unwrap();
9027 assert_eq!(order.quantity(), Quantity::from("10.0"));
9028 assert_eq!(order.filled_qty(), Quantity::from("12.0"));
9029 assert_eq!(order.trade_ids().len(), 1);
9030 }
9031
9032 fn position_fill_test_fixture(
9033 name: &str,
9034 authoritative_qty: Quantity,
9035 ) -> (LiveNode, PositionStatusReport, FillReport) {
9036 position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
9037 }
9038
9039 fn position_fill_test_fixture_with_threshold(
9040 name: &str,
9041 authoritative_qty: Quantity,
9042 position_check_threshold_ms: u32,
9043 ) -> (LiveNode, PositionStatusReport, FillReport) {
9044 let config = LiveNodeConfig {
9045 exec_engine: crate::config::LiveExecutionEngineConfig {
9046 reconciliation: true,
9047 position_check_threshold_ms,
9048 ..Default::default()
9049 },
9050 ..Default::default()
9051 };
9052
9053 let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
9054 let account_id = AccountId::from("TEST-001");
9055 let client_id = ClientId::from("POSITION-FILLS");
9056 let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
9057 let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
9058 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
9059 let account = AccountAny::Margin(MarginAccount::new(
9060 AccountState::new(
9061 account_id,
9062 AccountType::Margin,
9063 vec![AccountBalance::new(
9064 Money::from("1000000 USDT"),
9065 Money::from("0 USDT"),
9066 Money::from("1000000 USDT"),
9067 )],
9068 Vec::new(),
9069 true,
9070 UUID4::new(),
9071 UnixNanos::default(),
9072 UnixNanos::default(),
9073 Some(Currency::USDT()),
9074 ),
9075 true,
9076 ));
9077 node.kernel.cache.borrow_mut().add_account(account).unwrap();
9078 node.kernel
9079 .cache
9080 .borrow_mut()
9081 .add_instrument(instrument.clone())
9082 .unwrap();
9083 insert_accepted_limit_order_in_node(
9084 &node,
9085 account_id,
9086 client_id,
9087 instrument.id(),
9088 client_order_id,
9089 venue_order_id,
9090 );
9091 let order = node
9092 .kernel
9093 .cache
9094 .borrow()
9095 .order_owned(&client_order_id)
9096 .unwrap();
9097 let mut initial_fill = TestOrderEventStubs::filled(
9098 &order,
9099 &instrument,
9100 Some(TradeId::from("T-POSITION-INITIAL")),
9101 None,
9102 Some(Price::from("100.0")),
9103 Some(Quantity::from("1.0")),
9104 Some(LiquiditySide::Taker),
9105 None,
9106 None,
9107 Some(account_id),
9108 );
9109
9110 let OrderEventAny::Filled(fill) = &mut initial_fill else {
9111 unreachable!();
9112 };
9113
9114 fill.commission = Some(Money::zero(instrument.quote_currency()));
9115 node.process_reconciliation_events(&[initial_fill]);
9116
9117 let ts_event = UnixNanos::from(1_000);
9118
9119 let fill_report = FillReport::new(
9120 account_id,
9121 instrument.id(),
9122 venue_order_id,
9123 TradeId::from("T-POSITION-AUTHORITATIVE"),
9124 OrderSide::Buy,
9125 authoritative_qty,
9126 Price::from("100.0"),
9127 Money::zero(instrument.quote_currency()),
9128 LiquiditySide::Taker,
9129 Some(client_order_id),
9130 None,
9131 ts_event,
9132 ts_event,
9133 None,
9134 );
9135
9136 let venue_report = PositionStatusReport::new(
9137 account_id,
9138 instrument.id(),
9139 PositionSide::Long,
9140 Quantity::from("2.0"),
9141 ts_event,
9142 ts_event,
9143 None,
9144 None,
9145 Some(dec!(100.0)),
9146 );
9147
9148 (node, venue_report, fill_report)
9149 }
9150
9151 fn position_report_result(
9152 node: &LiveNode,
9153 report: PositionStatusReport,
9154 ) -> PositionReportResult {
9155 let client_id = ClientId::from("POSITION-FILLS");
9156 let key = (report.instrument_id, report.account_id);
9157 let mut check = node
9158 .exec_manager
9159 .prepare_position_report_check(UUID4::new(), &[]);
9160 check.client_coverage.insert(
9161 key,
9162 ReportClientCoverage::Resolved(IndexSet::from([client_id])),
9163 );
9164
9165 PositionReportResult {
9166 check,
9167 reports: vec![report],
9168 queried_clients: IndexSet::from([client_id]),
9169 failed_clients: IndexSet::new(),
9170 }
9171 }
9172
9173 fn position_quantity_after_check(
9174 node: &mut LiveNode,
9175 venue_report: &PositionStatusReport,
9176 reports: &[FillReport],
9177 ) -> Quantity {
9178 let key = (venue_report.instrument_id, venue_report.account_id);
9179 let position_result = position_report_result(node, venue_report.clone());
9180 node.handle_position_fill_report_result(PositionFillReportResult {
9181 position_result,
9182 reports: IndexMap::from([(key, reports.to_vec())]),
9183 });
9184
9185 node.kernel
9186 .cache
9187 .borrow()
9188 .positions_open(None, Some(&key.0), None, Some(&key.1), None)
9189 .iter()
9190 .map(|position| position.quantity)
9191 .sum()
9192 }
9193
9194 fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
9195 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
9196 fill.account_id,
9197 fill.instrument_id,
9198 fill.venue_order_id,
9199 fill.trade_id,
9200 fill.order_side,
9201 fill.last_qty,
9202 fill.last_px,
9203 fill.commission
9204 .unwrap_or_else(|| Money::zero(fill.currency)),
9205 fill.liquidity_side,
9206 Some(fill.client_order_id),
9207 fill.position_id,
9208 fill.ts_event,
9209 fill.ts_init,
9210 None,
9211 ))))
9212 }
9213
9214 fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
9215 node.exec_manager.is_fill_recently_processed(
9216 fill.account_id,
9217 fill.instrument_id,
9218 fill.trade_id,
9219 )
9220 }
9221
9222 #[rstest]
9223 #[case(0, NodeState::Idle)]
9224 #[case(1, NodeState::Starting)]
9225 #[case(2, NodeState::Running)]
9226 #[case(3, NodeState::ShuttingDown)]
9227 #[case(4, NodeState::Stopped)]
9228 fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
9229 assert_eq!(NodeState::from_u8(value), expected);
9230 }
9231
9232 #[rstest]
9233 #[case(5)]
9234 #[case(255)]
9235 #[should_panic(expected = "Invalid NodeState value")]
9236 fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
9237 let _ = NodeState::from_u8(value);
9238 }
9239
9240 #[rstest]
9241 fn test_node_state_roundtrip() {
9242 for state in [
9243 NodeState::Idle,
9244 NodeState::Starting,
9245 NodeState::Running,
9246 NodeState::ShuttingDown,
9247 NodeState::Stopped,
9248 ] {
9249 assert_eq!(NodeState::from_u8(state.as_u8()), state);
9250 }
9251 }
9252
9253 #[rstest]
9254 fn test_node_state_is_running_only_for_running() {
9255 assert!(!NodeState::Idle.is_running());
9256 assert!(!NodeState::Starting.is_running());
9257 assert!(NodeState::Running.is_running());
9258 assert!(!NodeState::ShuttingDown.is_running());
9259 assert!(!NodeState::Stopped.is_running());
9260 }
9261
9262 #[rstest]
9263 #[tokio::test]
9264 async fn test_await_engines_connected_returns_stop_requested() {
9265 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9266 let handle = node.handle();
9267
9268 handle.stop();
9269
9270 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9271 let status = node.await_engines_connected(deadline).await;
9272
9273 assert_eq!(status, EngineConnectionStatus::StopRequested);
9274 assert!(handle.should_stop());
9275 }
9276
9277 #[rstest]
9278 #[tokio::test]
9279 async fn test_await_engines_connected_returns_shutdown_requested() {
9280 let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9281
9282 node.kernel().shutdown_flag().set(true);
9283
9284 let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9285 let status = node.await_engines_connected(deadline).await;
9286
9287 assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
9288 }
9289
9290 #[rstest]
9291 #[tokio::test]
9292 async fn test_start_stop_request_aborts_startup_without_running() {
9293 let config = LiveNodeConfig {
9294 exec_engine: crate::config::LiveExecutionEngineConfig {
9295 reconciliation: false,
9296 ..Default::default()
9297 },
9298 timeout_disconnection: Duration::from_millis(50),
9299 ..Default::default()
9300 };
9301
9302 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9303 let handle = node.handle();
9304
9305 handle.stop();
9306 node.start().await.unwrap();
9307
9308 assert_eq!(handle.state(), NodeState::Stopped);
9309 assert!(handle.should_stop());
9310 assert!(!handle.is_running());
9311 }
9312
9313 #[rstest]
9314 #[tokio::test(start_paused = true)]
9315 async fn test_stop_processes_residual_exec_event_during_grace_period() {
9316 let config = LiveNodeConfig {
9317 exec_engine: crate::config::LiveExecutionEngineConfig {
9318 reconciliation: false,
9319 ..Default::default()
9320 },
9321 timeout_connection: Duration::ZERO,
9322 timeout_reconciliation: Duration::ZERO,
9323 timeout_portfolio: Duration::ZERO,
9324 timeout_disconnection: Duration::ZERO,
9325 delay_post_stop: Duration::from_millis(20),
9326 timeout_shutdown: Duration::ZERO,
9327 ..Default::default()
9328 };
9329
9330 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9331 let order = OrderTestBuilder::new(OrderType::Market)
9332 .instrument_id(InstrumentId::from("EUR/USD.SIM"))
9333 .quantity(Quantity::from("2"))
9334 .build();
9335 let client_order_id = order.client_order_id();
9336 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
9337
9338 node.kernel
9339 .cache()
9340 .borrow_mut()
9341 .add_order(order, None, None, false)
9342 .unwrap();
9343
9344 node.start().await.unwrap();
9345 let exec_event_sender = get_exec_event_sender();
9346
9347 let send_residual = tokio::spawn(async move {
9348 tokio::time::sleep(Duration::from_millis(1)).await;
9349 exec_event_sender
9350 .send(ExecutionEvent::Order(submitted))
9351 .unwrap();
9352 });
9353
9354 node.stop().await.unwrap();
9355 send_residual.await.unwrap();
9356
9357 assert_eq!(
9358 node.kernel
9359 .cache()
9360 .borrow()
9361 .order(&client_order_id)
9362 .unwrap()
9363 .status(),
9364 OrderStatus::Submitted
9365 );
9366
9367 node.dispose();
9368 }
9369
9370 #[rstest]
9371 #[tokio::test]
9372 async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
9373 let actor_id = ActorId::from("LIVE-STATE-ACTOR");
9374 let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
9375 let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
9376 let strategy_load =
9377 IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
9378 let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
9379 let strategy_save =
9380 IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
9381 let (database, control) = TestCacheDatabaseControl::create();
9382 control.set_actor_state(actor_id, &actor_load);
9383 control.set_strategy_state(strategy_id, &strategy_load);
9384
9385 let config = LiveNodeConfig {
9386 load_state: true,
9387 save_state: true,
9388 exec_engine: crate::config::LiveExecutionEngineConfig {
9389 reconciliation: false,
9390 ..Default::default()
9391 },
9392 timeout_connection: Duration::ZERO,
9393 timeout_reconciliation: Duration::ZERO,
9394 timeout_portfolio: Duration::ZERO,
9395 timeout_disconnection: Duration::ZERO,
9396 delay_post_stop: Duration::ZERO,
9397 timeout_shutdown: Duration::ZERO,
9398 ..Default::default()
9399 };
9400
9401 let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
9402 node.set_cache_database(Box::new(database)).unwrap();
9403 node.add_actor(StateActor::new(
9404 actor_id,
9405 control.clone(),
9406 actor_save.clone(),
9407 ))
9408 .unwrap();
9409 node.add_strategy(StateStrategy::new(
9410 strategy_id,
9411 control.clone(),
9412 strategy_save.clone(),
9413 ))
9414 .unwrap();
9415
9416 node.start().await.unwrap();
9417 node.stop().await.unwrap();
9418 node.dispose();
9419
9420 assert_eq!(
9421 control.events(),
9422 vec![
9423 "actor.load:LIVE-STATE-ACTOR",
9424 "actor.on_load",
9425 "strategy.load:LIVE-STATE-STRATEGY-001",
9426 "strategy.on_load",
9427 "actor.on_start",
9428 "strategy.on_start",
9429 "actor.on_stop",
9430 "strategy.on_stop",
9431 "actor.on_save",
9432 "actor.update:LIVE-STATE-ACTOR",
9433 "strategy.on_save",
9434 "strategy.update:LIVE-STATE-STRATEGY-001",
9435 "database.close",
9436 ]
9437 );
9438 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
9439 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9440 assert_eq!(node.state(), NodeState::Stopped);
9441 }
9442
9443 #[rstest]
9444 #[tokio::test]
9445 async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9446 let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9447 let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9448 let (database, control) = TestCacheDatabaseControl::create();
9449
9450 let config = LiveNodeConfig {
9451 save_state: true,
9452 exec_engine: crate::config::LiveExecutionEngineConfig {
9453 reconciliation: false,
9454 ..Default::default()
9455 },
9456 timeout_connection: Duration::ZERO,
9457 timeout_reconciliation: Duration::ZERO,
9458 timeout_portfolio: Duration::ZERO,
9459 timeout_disconnection: Duration::ZERO,
9460 delay_post_stop: Duration::ZERO,
9461 timeout_shutdown: Duration::ZERO,
9462 ..Default::default()
9463 };
9464
9465 let mut node =
9466 LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9467 node.set_cache_database(Box::new(database)).unwrap();
9468 node.add_actor(
9469 StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9470 )
9471 .unwrap();
9472 node.add_strategy(
9473 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9474 )
9475 .unwrap();
9476
9477 node.start().await.unwrap();
9478 let error = node.stop().await.unwrap_err();
9479 node.dispose();
9480
9481 assert_eq!(
9482 error.to_string(),
9483 "failed while finalizing kernel shutdown: Failed to save component state: actor \
9484 LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9485 LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9486 );
9487 assert_eq!(
9488 control.events(),
9489 vec![
9490 "actor.on_start",
9491 "strategy.on_start",
9492 "actor.on_stop",
9493 "strategy.on_stop",
9494 "actor.on_save",
9495 "strategy.on_save",
9496 "database.close",
9497 ]
9498 );
9499 assert_eq!(node.state(), NodeState::Stopped);
9500 }
9501
9502 #[rstest]
9503 #[tokio::test]
9504 async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9505 let config = LiveNodeConfig {
9506 exec_engine: crate::config::LiveExecutionEngineConfig {
9507 reconciliation: false,
9508 ..Default::default()
9509 },
9510 timeout_connection: Duration::ZERO,
9511 timeout_reconciliation: Duration::ZERO,
9512 timeout_portfolio: Duration::ZERO,
9513 timeout_disconnection: Duration::ZERO,
9514 delay_post_stop: Duration::ZERO,
9515 timeout_shutdown: Duration::ZERO,
9516 ..Default::default()
9517 };
9518
9519 let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9520 let order = OrderTestBuilder::new(OrderType::Market)
9521 .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9522 .quantity(Quantity::from("3"))
9523 .build();
9524 let client_order_id = order.client_order_id();
9525 let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9526
9527 node.kernel
9528 .cache()
9529 .borrow_mut()
9530 .add_order(order, None, None, false)
9531 .unwrap();
9532
9533 node.start().await.unwrap();
9534 get_exec_event_sender()
9535 .send(ExecutionEvent::Order(submitted))
9536 .unwrap();
9537
9538 node.stop().await.unwrap();
9539
9540 assert_eq!(
9541 node.kernel
9542 .cache()
9543 .borrow()
9544 .order(&client_order_id)
9545 .unwrap()
9546 .status(),
9547 OrderStatus::Submitted
9548 );
9549
9550 node.dispose();
9551 }
9552
9553 #[rstest]
9554 #[case::manual(false)]
9555 #[case::integrated(true)]
9556 fn test_shutdown_drain_logs_progress(
9557 #[case] integrated: bool,
9558 #[values(0, 9_999, 10_000, 20_001)] pending: usize,
9559 ) {
9560 struct DrainLogCapture(Mutex<Vec<String>>);
9561
9562 impl Log for DrainLogCapture {
9563 fn enabled(&self, metadata: &Metadata<'_>) -> bool {
9564 metadata.level() == Level::Debug
9565 }
9566
9567 fn log(&self, record: &Record<'_>) {
9568 assert_eq!(record.level(), Level::Debug);
9569 self.0.lock().push(record.args().to_string());
9570 }
9571
9572 fn flush(&self) {}
9573 }
9574
9575 static LOGS: DrainLogCapture = DrainLogCapture(Mutex::new(Vec::new()));
9576 log::set_logger(&LOGS).unwrap();
9577 log::set_max_level(LevelFilter::Debug);
9578
9579 let mut runner = AsyncRunner::new();
9580 runner.bind_senders();
9581 let event = TimeEvent::new("drain".into(), UUID4::new(), 0.into(), 0.into());
9582
9583 for _ in 0..pending {
9584 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9585 event.clone(),
9586 TimeEventCallback::from(|_: TimeEvent| {}),
9587 ));
9588 }
9589
9590 if integrated {
9591 let mut channels = runner.take_channels();
9592 LiveNode::drain_channels(
9593 &mut channels.time_evt_rx,
9594 &mut channels.system_evt_rx,
9595 &mut channels.system_cmd_rx,
9596 &mut channels.exec_evt_rx,
9597 &mut channels.exec_cmd_rx,
9598 &mut channels.data_evt_rx,
9599 &mut channels.data_cmd_rx,
9600 );
9601 } else {
9602 assert_eq!(runner.poll_pending(|_| {}), pending);
9603 }
9604
9605 let mut expected = vec![format!(
9606 "Draining shutdown queues ({pending} events initially pending)"
9607 )];
9608 expected.extend(
9609 (10_000..=pending)
9610 .step_by(10_000)
9611 .map(|processed| format!("Processed {processed} events during shutdown drain")),
9612 );
9613 expected.push(format!(
9614 "Drained {pending} remaining events during shutdown"
9615 ));
9616 assert_eq!(*LOGS.0.lock(), expected);
9617 }
9618
9619 #[rstest]
9620 #[tokio::test]
9621 async fn test_start_event_store_replay_skips_live_connections() {
9622 let mut node = live_node_with_replay_store(false);
9623 let handle = node.handle();
9624
9625 node.start().await.unwrap();
9626
9627 assert_eq!(handle.state(), NodeState::Running);
9628 assert!(handle.is_running());
9629 assert!(node.kernel.is_event_store_replay());
9630 assert!(node.runner.is_some());
9631 }
9632
9633 #[rstest]
9634 #[tokio::test]
9635 async fn test_start_event_store_replay_preserves_stop_request() {
9636 let mut node = live_node_with_replay_store(false);
9637 let handle = node.handle();
9638 handle.stop();
9639
9640 node.start().await.unwrap();
9641
9642 assert_eq!(handle.state(), NodeState::Stopped);
9643 assert!(handle.should_stop());
9644 assert!(node.kernel.is_event_store_replay());
9645 assert!(node.runner.is_some());
9646 }
9647
9648 #[rstest]
9649 #[tokio::test]
9650 async fn test_start_event_store_replay_config_failure_aborts_startup() {
9651 let mut node = live_node_with_replay_store(true);
9652 let handle = node.handle();
9653
9654 node.start().await.unwrap();
9655
9656 assert_eq!(handle.state(), NodeState::Stopped);
9657 assert!(!handle.is_running());
9658 assert!(node.kernel.is_event_store_replay_configured());
9659 assert!(!node.kernel.is_event_store_replay());
9660 assert!(node.runner.is_some());
9661 }
9662
9663 #[rstest]
9664 #[tokio::test]
9665 async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9666 let mut node = live_node_with_replay_store(false);
9667 let handle = node.handle();
9668
9669 node.run().await.unwrap();
9670
9671 assert_eq!(handle.state(), NodeState::Running);
9672 assert!(handle.is_running());
9673 assert!(node.kernel.is_event_store_replay());
9674 assert!(node.runner.is_none());
9675 }
9676
9677 #[rstest]
9678 #[tokio::test]
9679 async fn test_run_event_store_replay_preserves_stop_request() {
9680 let mut node = live_node_with_replay_store(false);
9681 let handle = node.handle();
9682 handle.stop();
9683
9684 node.run().await.unwrap();
9685
9686 assert_eq!(handle.state(), NodeState::Stopped);
9687 assert!(handle.should_stop());
9688 assert!(node.kernel.is_event_store_replay());
9689 assert!(node.runner.is_none());
9690 }
9691
9692 #[rstest]
9693 #[tokio::test]
9694 async fn test_run_event_store_replay_config_failure_aborts_startup() {
9695 let mut node = live_node_with_replay_store(true);
9696 let handle = node.handle();
9697
9698 node.run().await.unwrap();
9699
9700 assert_eq!(handle.state(), NodeState::Stopped);
9701 assert!(!handle.is_running());
9702 assert!(node.kernel.is_event_store_replay_configured());
9703 assert!(!node.kernel.is_event_store_replay());
9704 assert!(node.runner.is_none());
9705 }
9706
9707 #[rstest]
9708 fn test_build_rejects_event_store_config_without_factory() {
9709 let config = LiveNodeConfig {
9710 event_store: Some(EventStoreConfig::default()),
9711 exec_engine: crate::config::LiveExecutionEngineConfig {
9712 reconciliation: false,
9713 ..Default::default()
9714 },
9715 ..Default::default()
9716 };
9717
9718 let err = LiveNodeBuilder::from_config(config)
9719 .expect("builder")
9720 .build()
9721 .expect_err("should reject event_store config without factory");
9722
9723 assert!(
9724 err.to_string().contains("with_event_store"),
9725 "error message should mention with_event_store, was: {err}"
9726 );
9727 }
9728
9729 #[rstest]
9730 fn test_direct_build_rejects_event_store_config() {
9731 let config = LiveNodeConfig {
9732 event_store: Some(EventStoreConfig::default()),
9733 exec_engine: crate::config::LiveExecutionEngineConfig {
9734 reconciliation: false,
9735 ..Default::default()
9736 },
9737 ..Default::default()
9738 };
9739
9740 let err = LiveNode::build("TestNode".to_string(), Some(config))
9741 .expect_err("LiveNode::build should reject event_store config");
9742
9743 assert!(
9744 err.to_string().contains("with_event_store"),
9745 "error message should mention with_event_store, was: {err}"
9746 );
9747 }
9748
9749 #[rstest]
9750 fn test_dispose_before_start_is_idempotent() {
9751 let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9752 node.add_strategy(TestStrategy::new(StrategyConfig {
9753 strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9754 ..Default::default()
9755 }))
9756 .unwrap();
9757
9758 node.dispose();
9759 node.dispose();
9760
9761 assert!(node.kernel.trader().borrow().is_disposed());
9762 assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9763 assert_eq!(node.state(), NodeState::Stopped);
9764 }
9765
9766 #[rstest]
9767 fn test_dispose_releases_retained_callback_roots(
9768 #[values(false, true)] fatal: bool,
9769 #[values(false, true)] external: bool,
9770 ) {
9771 actor::clear_callbacks().unwrap();
9772 let directory =
9773 std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9774
9775 let config = LiveNodeConfig {
9776 logging: LoggerConfig {
9777 fileout_level: LevelFilter::Info,
9778 file_config: Some(FileWriterConfig {
9779 directory: Some(directory.to_str().unwrap().to_string()),
9780 file_name: Some("disposal".to_string()),
9781 ..Default::default()
9782 }),
9783 ..Default::default()
9784 },
9785 ..Default::default()
9786 };
9787
9788 let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9789 let received = Rc::new(RefCell::new(Vec::new()));
9790 let observed = received.clone();
9791
9792 let retained = DispatchMessage::from(()).dispatch(|()| {
9793 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9794 TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9795 TimeEventCallback::RustLocal(Rc::new(move |_| {
9796 observed.borrow_mut().push("delivered");
9797 })),
9798 ));
9799
9800 external.then(|| DispatchMessage::new((), std::thread::current().id()))
9801 });
9802
9803 if let Some(retained) = &retained {
9804 assert!(retained.is_rooted());
9805 }
9806
9807 assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9808
9809 if fatal {
9810 crate::dispatch::tests::latch_callback_failure();
9811 }
9812
9813 node.dispose();
9814
9815 assert!(node.runner.is_none());
9816 assert!(received.borrow().is_empty());
9817 assert_eq!(Rc::strong_count(&received), 1);
9818 assert_eq!(
9819 actor::callback_failure(),
9820 (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9821 );
9822 assert_eq!(
9823 actor::clear_callbacks(),
9824 if external {
9825 Err(CallbackDispatchError::Active)
9826 } else {
9827 Ok(())
9828 }
9829 );
9830
9831 logging_sync_to_disk().unwrap();
9832 let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9833 drop(retained);
9834 node.dispose();
9835
9836 assert_eq!(actor::callback_failure(), None);
9837 assert_eq!(actor::clear_callbacks(), Ok(()));
9838 drop(node);
9839 std::fs::remove_dir_all(directory).unwrap();
9840
9841 let errors: Vec<_> = output
9842 .lines()
9843 .filter(|line| line.contains("[ERROR]"))
9844 .filter_map(|line| {
9845 line.split_once(".nautilus_live::node: ")
9846 .map(|(_, text)| text)
9847 })
9848 .collect();
9849
9850 let mut expected = Vec::new();
9851
9852 if fatal {
9853 expected.push(
9854 "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9855 );
9856 }
9857
9858 if external {
9859 expected.push(
9860 "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9861 );
9862 }
9863
9864 assert_eq!(errors, expected);
9865 }
9866
9867 #[tokio::test]
9868 async fn test_dispose_releases_stop_generated_callback_roots() {
9869 actor::clear_callbacks().unwrap();
9870
9871 let config = LiveNodeConfig {
9872 exec_engine: crate::config::LiveExecutionEngineConfig {
9873 reconciliation: false,
9874 ..Default::default()
9875 },
9876 timeout_connection: Duration::ZERO,
9877 timeout_reconciliation: Duration::ZERO,
9878 timeout_portfolio: Duration::ZERO,
9879 timeout_shutdown: Duration::ZERO,
9880 ..Default::default()
9881 };
9882
9883 let mut node =
9884 LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9885 let received = Rc::new(RefCell::new(Vec::new()));
9886 node.add_actor(StopCallbackActor {
9887 core: DataActorCore::new(DataActorConfig {
9888 actor_id: Some(ActorId::from("STOP-CALLBACK")),
9889 ..Default::default()
9890 }),
9891 received: received.clone(),
9892 })
9893 .unwrap();
9894
9895 node.start().await.unwrap();
9896
9897 assert!(node.kernel.trader().borrow().is_running());
9898 assert_eq!(actor::clear_callbacks(), Ok(()));
9899
9900 node.dispose();
9901
9902 assert_eq!(*received.borrow(), ["stop", "queued"]);
9903 assert_eq!(Rc::strong_count(&received), 1);
9904 assert!(node.runner.is_none());
9905 assert!(node.kernel.trader().borrow().is_disposed());
9906 assert_eq!(node.state(), NodeState::Stopped);
9907 assert_eq!(actor::clear_callbacks(), Ok(()));
9908 }
9909
9910 #[derive(Debug)]
9911 struct StopCallbackActor {
9912 core: DataActorCore,
9913 received: Rc<RefCell<Vec<&'static str>>>,
9914 }
9915
9916 nautilus_actor!(StopCallbackActor);
9917
9918 impl DataActor for StopCallbackActor {
9919 fn on_stop(&mut self) -> anyhow::Result<()> {
9920 self.received.borrow_mut().push("stop");
9921 let received = self.received.clone();
9922 DispatchMessage::from(()).dispatch(|()| {
9923 nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9924 TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9925 TimeEventCallback::RustLocal(Rc::new(move |_| {
9926 received.borrow_mut().push("delivered");
9927 })),
9928 ));
9929 });
9930
9931 let clear_result = actor::clear_callbacks();
9932 anyhow::ensure!(
9933 clear_result == Err(CallbackDispatchError::Active),
9934 "Expected active callback roots during stop, received {clear_result:?}"
9935 );
9936 self.received.borrow_mut().push("queued");
9937 Ok(())
9938 }
9939 }
9940
9941 #[rstest]
9942 fn test_handle_initial_state() {
9943 let handle = LiveNodeHandle::new();
9944
9945 assert_eq!(handle.state(), NodeState::Idle);
9946 assert!(!handle.should_stop());
9947 assert!(!handle.is_running());
9948 }
9949
9950 #[rstest]
9951 fn test_handle_initial_metrics_snapshot_is_zero() {
9952 let handle = LiveNodeHandle::new();
9953
9954 assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9955 }
9956
9957 #[rstest]
9958 fn test_record_runner_dispatch_updates_selected_channel() {
9959 let metrics = RunnerMetrics::default();
9960 let dispatch_start = dst::time::Instant::now();
9961 let metrics_start = dispatch_start
9962 .checked_sub(Duration::from_micros(1))
9963 .expect("test instant should support a one-microsecond lookback");
9964
9965 record_runner_dispatch(
9966 &metrics,
9967 SystemChannel::DataCommands,
9968 dispatch_start,
9969 metrics_start,
9970 );
9971 let snapshot = metrics.snapshot();
9972
9973 assert_eq!(snapshot.time_events.dispatched, 0);
9974 assert_eq!(snapshot.exec_events.dispatched, 0);
9975 assert_eq!(snapshot.exec_commands.dispatched, 0);
9976 assert_eq!(snapshot.data_events.dispatched, 0);
9977 assert_eq!(snapshot.data_commands.dispatched, 1);
9978 assert_eq!(
9979 snapshot.data_commands.last_dispatch_at_ns,
9980 snapshot.elapsed_ns
9981 );
9982 assert_eq!(
9983 snapshot.data_commands.dispatch_busy_ns,
9984 snapshot.dispatch_busy_ns
9985 );
9986 assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9987 }
9988
9989 #[rstest]
9990 fn test_handle_stop_sets_flag() {
9991 let handle = LiveNodeHandle::new();
9992
9993 handle.stop();
9994
9995 assert!(handle.should_stop());
9996 }
9997
9998 #[rstest]
9999 fn test_handle_stop_blocks_running_transition() {
10000 let handle = LiveNodeHandle::new();
10001 handle.set_starting();
10002 handle.stop();
10003
10004 let transition = handle.try_set_running();
10005
10006 assert_eq!(transition, RunningTransition::StopRequested);
10007 assert_eq!(handle.state(), NodeState::Starting);
10008 assert!(handle.should_stop());
10009 assert!(!handle.is_running());
10010 }
10011
10012 #[rstest]
10013 fn test_handle_stop_after_running_transition_remains_pending() {
10014 let handle = LiveNodeHandle::new();
10015 handle.set_starting();
10016
10017 let transition = handle.try_set_running();
10018 handle.stop();
10019
10020 assert_eq!(transition, RunningTransition::Entered);
10021 assert_eq!(handle.state(), NodeState::Running);
10022 assert!(handle.should_stop());
10023 assert!(handle.is_running());
10024 }
10025
10026 #[rstest]
10027 fn test_handle_node_state_transitions() {
10028 let handle = LiveNodeHandle::new();
10029 assert_eq!(handle.state(), NodeState::Idle);
10030
10031 handle.set_starting();
10032 assert_eq!(handle.state(), NodeState::Starting);
10033 assert!(!handle.is_running());
10034
10035 assert_eq!(handle.try_set_running(), RunningTransition::Entered);
10036 assert_eq!(handle.state(), NodeState::Running);
10037 assert!(handle.is_running());
10038
10039 handle.set_shutting_down();
10040 assert_eq!(handle.state(), NodeState::ShuttingDown);
10041 assert!(!handle.is_running());
10042
10043 handle.set_stopped();
10044 assert_eq!(handle.state(), NodeState::Stopped);
10045 assert!(!handle.is_running());
10046 }
10047
10048 #[rstest]
10049 fn test_handle_clone_shares_state_bidirectionally() {
10050 let handle1 = LiveNodeHandle::new();
10051 let handle2 = handle1.clone();
10052
10053 handle1.set_starting();
10054 let transition = handle2.try_set_running();
10055 handle1.stop();
10056
10057 assert_eq!(transition, RunningTransition::Entered);
10058 assert_eq!(handle1.state(), NodeState::Running);
10059 assert!(handle2.should_stop());
10060 }
10061
10062 #[rstest]
10063 fn test_handle_stop_flag_survives_non_running_state_changes() {
10064 let handle = LiveNodeHandle::new();
10065
10066 handle.set_starting();
10067 handle.stop();
10068 handle.set_shutting_down();
10069 handle.set_stopped();
10070
10071 assert_eq!(handle.state(), NodeState::Stopped);
10072 assert!(handle.should_stop());
10073 }
10074
10075 #[rstest]
10076 fn test_builder_creation() {
10077 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10078
10079 assert!(result.is_ok());
10080 }
10081
10082 #[rstest]
10083 fn test_stream_processor_receives_unregistered_typed_payload() {
10084 let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10085 .unwrap()
10086 .build()
10087 .unwrap();
10088 let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
10089 InstrumentId::from("AUD/USD.SIM"),
10090 Some(ClientId::from("EXTERNAL")),
10091 Some(Venue::from("SIM")),
10092 UUID4::from("00000000-0000-4000-8000-000000000001"),
10093 UnixNanos::from(1),
10094 None,
10095 None,
10096 ));
10097 let expected = serde_json::to_value(&command).unwrap();
10098 let received = Rc::new(RefCell::new(Vec::new()));
10099 let steps = Rc::new(RefCell::new(Vec::new()));
10100 let received_processor = received.clone();
10101 let first_steps = steps.clone();
10102 node.add_stream_processor(move |message| {
10103 let command = message
10104 .downcast_ref::<SubscribeCommand>()
10105 .expect("processor must receive the decoded concrete command");
10106 received_processor
10107 .borrow_mut()
10108 .push(serde_json::to_value(command).unwrap());
10109 first_steps.borrow_mut().push(1);
10110 });
10111
10112 let second_steps = steps.clone();
10113 node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
10114 let republished = Rc::new(RefCell::new(Vec::new()));
10115 let republished_handler = republished.clone();
10116 let subscriber_steps = steps.clone();
10117 msgbus::subscribe_any(
10118 "external.test".into(),
10119 ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
10120 republished_handler
10121 .borrow_mut()
10122 .push(serde_json::to_value(command).unwrap());
10123 subscriber_steps.borrow_mut().push(3);
10124 }),
10125 None,
10126 );
10127
10128 let message = BusMessage::with_str_topic(
10129 "external.test",
10130 BusPayloadType::SubscribeCommand,
10131 Bytes::from(serde_json::to_vec(&command).unwrap()),
10132 SerializationEncoding::Json,
10133 );
10134
10135 assert!(
10136 !msgbus::get_message_bus()
10137 .borrow()
10138 .is_streaming_type(BusPayloadType::SubscribeCommand)
10139 );
10140 node.process_external_msgbus_message(&message);
10141
10142 assert_eq!(*received.borrow(), vec![expected.clone()]);
10143 assert_eq!(*steps.borrow(), vec![1, 2]);
10144 assert!(republished.borrow().is_empty());
10145
10146 received.borrow_mut().clear();
10147 steps.borrow_mut().clear();
10148 msgbus::get_message_bus()
10149 .borrow_mut()
10150 .add_streaming_type(BusPayloadType::SubscribeCommand);
10151 node.process_external_msgbus_message(&message);
10152
10153 assert_eq!(*received.borrow(), vec![expected.clone()]);
10154 assert_eq!(*republished.borrow(), vec![expected]);
10155 assert_eq!(*steps.borrow(), vec![1, 2, 3]);
10156 msgbus::get_message_bus().borrow_mut().dispose();
10157 }
10158
10159 #[rstest]
10160 fn test_builder_rejects_backtest() {
10161 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
10162
10163 assert!(result.is_err());
10164 assert!(result.unwrap_err().to_string().contains("Backtest"));
10165 }
10166
10167 #[rstest]
10168 fn test_builder_accepts_live_environment() {
10169 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
10170
10171 assert!(result.is_ok());
10172 }
10173
10174 #[rstest]
10175 fn test_builder_accepts_sandbox_environment() {
10176 let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10177
10178 assert!(result.is_ok());
10179 }
10180
10181 #[rstest]
10182 fn test_builder_fluent_api_chaining() {
10183 let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
10184 .unwrap()
10185 .with_name("TestNode")
10186 .with_instance_id(UUID4::new())
10187 .with_load_state(false)
10188 .with_save_state(true)
10189 .with_timeout_connection(30)
10190 .with_timeout_reconciliation(60)
10191 .with_reconciliation(true)
10192 .with_reconciliation_lookback_mins(120)
10193 .with_timeout_portfolio(10)
10194 .with_timeout_disconnection_secs(5)
10195 .with_delay_post_stop_secs(3)
10196 .with_delay_shutdown_secs(10);
10197
10198 assert_eq!(builder.name(), "TestNode");
10199 }
10200
10201 #[rstest]
10202 fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
10203 let (external_egress, publications, closed) = CapturingExternalEgress::new();
10204
10205 let msgbus_config = MessageBusConfig {
10206 encoding: SerializationEncoding::Json,
10207 ..Default::default()
10208 };
10209
10210 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10211 .unwrap()
10212 .with_msgbus_config(msgbus_config)
10213 .with_external_msgbus_egress(Box::new(external_egress))
10214 .build()
10215 .expect("node builds with external message bus egress");
10216 let quote = QuoteTick::default();
10217
10218 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10219
10220 let publications = publications.borrow();
10221 assert_eq!(publications.len(), 1);
10222 assert_eq!(publications[0].topic, "data.quotes.TEST");
10223 assert_eq!(
10224 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10225 .expect("JSON payload must decode as QuoteTick"),
10226 quote
10227 );
10228 drop(publications);
10229
10230 msgbus::get_message_bus().borrow_mut().dispose();
10231 assert!(closed.get());
10232 drop(node);
10233 }
10234
10235 #[rstest]
10236 #[tokio::test(flavor = "current_thread")]
10237 async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
10238 let quote = QuoteTick::default();
10239 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10240 let publications = Arc::new(Mutex::new(Vec::new()));
10241 let closed = Arc::new(AtomicBool::new(false));
10242 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
10243
10244 let msgbus_config = MessageBusConfig {
10245 external_streams: Some(vec!["stream".to_string()]),
10246 ..Default::default()
10247 };
10248
10249 let config = LiveNodeConfig {
10250 environment: Environment::Sandbox,
10251 msgbus: Some(msgbus_config),
10252 exec_engine: crate::config::LiveExecutionEngineConfig {
10253 reconciliation: false,
10254 ..Default::default()
10255 },
10256 delay_post_stop: Duration::ZERO,
10257 timeout_connection: Duration::from_millis(500),
10258 timeout_disconnection: Duration::from_millis(500),
10259 ..Default::default()
10260 };
10261
10262 let mut node = LiveNodeBuilder::from_config(config)
10263 .unwrap()
10264 .with_external_msgbus_factory(Box::new(factory))
10265 .build()
10266 .expect("node builds with external message bus factory");
10267
10268 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10269 {
10270 let publications = publications.lock();
10271 assert_eq!(publications.len(), 1);
10272 assert_eq!(publications[0].topic, "data.quotes.TEST");
10273 assert_eq!(
10274 serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10275 .expect("JSON payload must decode as QuoteTick"),
10276 quote
10277 );
10278 }
10279
10280 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10281 let handle = node.handle();
10282
10283 let handler = TypedHandler::from({
10286 let received = received.clone();
10287 move |quote: &QuoteTick| {
10288 received.borrow_mut().push(*quote);
10289 }
10290 });
10291
10292 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10293 msgbus::get_message_bus()
10294 .borrow_mut()
10295 .add_streaming_type(BusPayloadType::QuoteTick);
10296
10297 let payload =
10298 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
10299 let message = BusMessage::with_str_topic(
10300 "data.quotes.TEST",
10301 BusPayloadType::QuoteTick,
10302 payload,
10303 SerializationEncoding::Json,
10304 );
10305
10306 tokio::time::timeout(Duration::from_secs(30), async {
10307 let run = node.run();
10308 tokio::pin!(run);
10309
10310 let drive = async {
10311 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10312
10313 tx.send(message)
10314 .await
10315 .expect("external ingress receiver should be open");
10316
10317 wait_until_async(
10318 || async { received.borrow().len() == 1 },
10319 Duration::from_secs(10),
10320 )
10321 .await;
10322 assert_eq!(*received.borrow(), vec![quote]);
10323 handle.stop();
10324 };
10325
10326 tokio::select! {
10327 biased;
10328
10329 () = drive => {}
10330 result = &mut run => {
10331 panic!("node stopped before factory ingress was republished: {result:?}");
10332 }
10333 }
10334
10335 run.await.expect("node should stop cleanly");
10336 })
10337 .await
10338 .expect("live node should republish factory ingress and stop before timeout");
10339
10340 assert_eq!(handle.state(), NodeState::Stopped);
10341 assert!(closed.load(Ordering::Relaxed));
10342 msgbus::get_message_bus().borrow_mut().dispose();
10343 }
10344
10345 #[rstest]
10346 #[tokio::test(flavor = "current_thread")]
10347 async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
10348 let quote = QuoteTick::default();
10349 let publications = Arc::new(Mutex::new(Vec::new()));
10350 let closed = Arc::new(AtomicBool::new(false));
10351 let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
10352
10353 let config = LiveNodeConfig {
10354 environment: Environment::Sandbox,
10355 msgbus: Some(MessageBusConfig::default()),
10356 exec_engine: crate::config::LiveExecutionEngineConfig {
10357 reconciliation: false,
10358 ..Default::default()
10359 },
10360 delay_post_stop: Duration::ZERO,
10361 timeout_connection: Duration::from_millis(500),
10362 timeout_disconnection: Duration::from_millis(500),
10363 ..Default::default()
10364 };
10365
10366 let mut node = LiveNodeBuilder::from_config(config)
10367 .unwrap()
10368 .with_external_msgbus_factory(Box::new(factory))
10369 .build()
10370 .expect("node builds with egress-only message bus factory");
10371 let handle = node.handle();
10372
10373 msgbus::publish_quote("data.quotes.TEST".into(), "e);
10374 {
10375 let publications = publications.lock();
10376 assert_eq!(publications.len(), 1);
10377 assert_eq!(publications[0].topic, "data.quotes.TEST");
10378 }
10379
10380 tokio::time::timeout(Duration::from_secs(30), async {
10381 let run = node.run();
10382 tokio::pin!(run);
10383
10384 let drive = async {
10385 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10386 handle.stop();
10387 };
10388
10389 tokio::select! {
10390 biased;
10391
10392 () = drive => {}
10393 result = &mut run => {
10394 panic!("node stopped before egress-only factory run was observed: {result:?}");
10395 }
10396 }
10397
10398 run.await.expect("node should stop cleanly");
10399 })
10400 .await
10401 .expect("live node should run without external ingress before timeout");
10402
10403 assert_eq!(handle.state(), NodeState::Stopped);
10404 msgbus::get_message_bus().borrow_mut().dispose();
10405 assert!(closed.load(Ordering::Relaxed));
10406 }
10407
10408 #[rstest]
10409 fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
10410 let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
10411
10412 let egress_factory = CapturingBackingFactory::new(
10413 Arc::new(Mutex::new(Vec::new())),
10414 Arc::new(AtomicBool::new(false)),
10415 None,
10416 );
10417 let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10418 .unwrap()
10419 .with_external_msgbus_factory(Box::new(egress_factory))
10420 .with_external_msgbus_egress(Box::new(external_egress))
10421 .build()
10422 .expect_err("builder should reject factory plus injected egress");
10423
10424 assert!(
10425 egress_error
10426 .to_string()
10427 .contains("cannot be combined with injected egress or ingress")
10428 );
10429
10430 let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10431
10432 let ingress_factory = CapturingBackingFactory::new(
10433 Arc::new(Mutex::new(Vec::new())),
10434 Arc::new(AtomicBool::new(false)),
10435 None,
10436 );
10437 let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
10438 let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10439 .unwrap()
10440 .with_external_msgbus_factory(Box::new(ingress_factory))
10441 .with_external_ingress(Box::new(ingress))
10442 .build()
10443 .expect_err("builder should reject factory plus injected ingress");
10444
10445 assert!(
10446 ingress_error
10447 .to_string()
10448 .contains("cannot be combined with injected egress or ingress")
10449 );
10450 }
10451
10452 #[rstest]
10453 #[tokio::test(flavor = "current_thread")]
10454 async fn test_run_republishes_external_ingress_on_local_msgbus() {
10455 let quote = QuoteTick::default();
10456 let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10457 let payload =
10458 Bytes::from(serde_json::to_vec("e).expect("QuoteTick should serialize as JSON"));
10459 let message = BusMessage::with_str_topic(
10460 "data.quotes.TEST",
10461 BusPayloadType::QuoteTick,
10462 payload,
10463 SerializationEncoding::Json,
10464 );
10465 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10466 let closed = Rc::new(Cell::new(false));
10467 let ingress = CapturingExternalIngress::new(rx, closed.clone());
10468
10469 let config = LiveNodeConfig {
10470 environment: Environment::Sandbox,
10471 exec_engine: crate::config::LiveExecutionEngineConfig {
10472 reconciliation: false,
10473 ..Default::default()
10474 },
10475 delay_post_stop: Duration::ZERO,
10476 timeout_connection: Duration::from_millis(500),
10477 timeout_disconnection: Duration::from_millis(500),
10478 ..Default::default()
10479 };
10480
10481 let mut node = LiveNodeBuilder::from_config(config)
10482 .unwrap()
10483 .with_external_ingress(Box::new(ingress))
10484 .build()
10485 .expect("node builds with external message bus ingress");
10486 let handle = node.handle();
10487
10488 let handler = TypedHandler::from({
10489 let received = received.clone();
10490 move |quote: &QuoteTick| {
10491 received.borrow_mut().push(*quote);
10492 }
10493 });
10494
10495 msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10496 msgbus::get_message_bus()
10497 .borrow_mut()
10498 .add_streaming_type(BusPayloadType::QuoteTick);
10499
10500 tokio::time::timeout(Duration::from_secs(30), async {
10501 let run = node.run();
10502 tokio::pin!(run);
10503
10504 let drive = async {
10505 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10506
10507 tx.send(message)
10508 .await
10509 .expect("external ingress receiver should be open");
10510
10511 wait_until_async(
10512 || async { received.borrow().len() == 1 },
10513 Duration::from_secs(10),
10514 )
10515 .await;
10516 assert_eq!(*received.borrow(), vec![quote]);
10517 handle.stop();
10518 };
10519
10520 tokio::select! {
10521 biased;
10522
10523 () = drive => {}
10524 result = &mut run => {
10525 panic!("node stopped before external message was republished: {result:?}");
10526 }
10527 }
10528
10529 run.await.expect("node should stop cleanly");
10530 })
10531 .await
10532 .expect("live node should republish ingress and stop before timeout");
10533
10534 assert_eq!(handle.state(), NodeState::Stopped);
10535 assert!(closed.get());
10536 msgbus::get_message_bus().borrow_mut().dispose();
10537 }
10538
10539 #[rstest]
10540 #[tokio::test(flavor = "current_thread")]
10541 async fn test_run_closes_external_ingress_when_receiver_closes() {
10542 let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10543 let closed = Rc::new(Cell::new(false));
10544 let ingress = CapturingExternalIngress::new(rx, closed.clone());
10545
10546 let config = LiveNodeConfig {
10547 environment: Environment::Sandbox,
10548 exec_engine: crate::config::LiveExecutionEngineConfig {
10549 reconciliation: false,
10550 ..Default::default()
10551 },
10552 delay_post_stop: Duration::ZERO,
10553 timeout_connection: Duration::from_millis(500),
10554 timeout_disconnection: Duration::from_millis(500),
10555 ..Default::default()
10556 };
10557
10558 let mut node = LiveNodeBuilder::from_config(config)
10559 .unwrap()
10560 .with_external_ingress(Box::new(ingress))
10561 .build()
10562 .expect("node builds with external message bus ingress");
10563 let handle = node.handle();
10564
10565 tokio::time::timeout(Duration::from_secs(30), async {
10566 let run = node.run();
10567 tokio::pin!(run);
10568
10569 let drive = async {
10570 wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10571
10572 drop(tx);
10573
10574 wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10575 assert!(
10576 handle.is_running(),
10577 "node should keep running after ingress closes"
10578 );
10579 handle.stop();
10580 };
10581
10582 tokio::select! {
10583 biased;
10584
10585 () = drive => {}
10586 result = &mut run => {
10587 panic!("node stopped before ingress close was observed: {result:?}");
10588 }
10589 }
10590
10591 run.await.expect("node should stop cleanly");
10592 })
10593 .await
10594 .expect("live node should close ingress and stop before timeout");
10595
10596 assert_eq!(handle.state(), NodeState::Stopped);
10597 }
10598
10599 #[rstest]
10600 #[tokio::test(flavor = "current_thread")]
10601 async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10602 let closed = Rc::new(Cell::new(false));
10603 let ingress = FailingExternalIngress::new(closed.clone());
10604
10605 let config = LiveNodeConfig {
10606 environment: Environment::Sandbox,
10607 exec_engine: crate::config::LiveExecutionEngineConfig {
10608 reconciliation: false,
10609 ..Default::default()
10610 },
10611 delay_post_stop: Duration::ZERO,
10612 timeout_connection: Duration::from_millis(500),
10613 timeout_disconnection: Duration::from_millis(500),
10614 ..Default::default()
10615 };
10616
10617 let mut node = LiveNodeBuilder::from_config(config)
10618 .unwrap()
10619 .with_external_ingress(Box::new(ingress))
10620 .build()
10621 .expect("node builds with external message bus ingress");
10622 let handle = node.handle();
10623
10624 let err = node.run().await.expect_err("run should fail");
10625
10626 assert!(
10627 err.to_string()
10628 .contains("external ingress receiver unavailable")
10629 );
10630 assert_eq!(handle.state(), NodeState::Stopped);
10631 assert!(closed.get());
10632 }
10633
10634 #[rstest]
10635 fn test_node_build_rejects_existing_node_preserving_account_delivery(
10636 #[values(false, true)] first_builder: bool,
10637 #[values(false, true)] second_builder: bool,
10638 ) {
10639 let config = LiveNodeConfig {
10640 trader_id: TraderId::from("PROBE-001"),
10641 ..Default::default()
10642 };
10643
10644 let mut node = if first_builder {
10645 LiveNodeBuilder::from_config(config)
10646 .unwrap()
10647 .build()
10648 .unwrap()
10649 } else {
10650 LiveNode::build("Original".to_string(), Some(config)).unwrap()
10651 };
10652
10653 let bus = msgbus::get_message_bus();
10654
10655 let ExecutionEvent::Account(before) = stub_account_event() else {
10656 unreachable!()
10657 };
10658
10659 get_exec_event_sender()
10660 .send(ExecutionEvent::Account(before.clone()))
10661 .unwrap();
10662 assert_eq!(node.drain_runner_pending(), 1);
10663 assert_eq!(
10664 node.kernel
10665 .cache
10666 .borrow()
10667 .account(&before.account_id)
10668 .unwrap()
10669 .last_event(),
10670 Some(before)
10671 );
10672
10673 let result = if second_builder {
10674 LiveNodeBuilder::from_config(LiveNodeConfig::default())
10675 .unwrap()
10676 .build()
10677 } else {
10678 LiveNode::build("Extra".to_string(), None)
10679 };
10680
10681 assert_eq!(
10682 result.unwrap_err().to_string(),
10683 "A LiveNode already exists or is being built on this thread; drop it before building another"
10684 );
10685
10686 let ExecutionEvent::Account(after) = stub_account_event() else {
10687 unreachable!()
10688 };
10689
10690 get_exec_event_sender()
10691 .send(ExecutionEvent::Account(after.clone()))
10692 .unwrap();
10693 assert_eq!(node.drain_runner_pending(), 1);
10694 assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10695 assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10696 assert_eq!(
10697 node.kernel
10698 .cache
10699 .borrow()
10700 .account(&after.account_id)
10701 .unwrap()
10702 .last_event(),
10703 Some(after)
10704 );
10705 }
10706
10707 #[rstest]
10708 fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10709 let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10710 .unwrap()
10711 .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10712 let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10713 assert!(
10714 error
10715 .to_string()
10716 .contains("already exists or is being built")
10717 );
10718 anyhow::bail!("Event store construction failed")
10719 })
10720 .build();
10721
10722 assert_eq!(
10723 result.unwrap_err().to_string(),
10724 "Event store construction failed"
10725 );
10726
10727 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10728 assert_eq!(node.state(), NodeState::Idle);
10729 }
10730
10731 #[rstest]
10732 fn test_node_build_allows_separate_threads() {
10733 let node = LiveNode::build("First".to_string(), None).unwrap();
10734
10735 let other_state = std::thread::spawn(|| {
10736 let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10737 other.state()
10738 })
10739 .join()
10740 .unwrap();
10741
10742 assert_eq!(node.state(), NodeState::Idle);
10743 assert_eq!(other_state, NodeState::Idle);
10744 }
10745
10746 #[rstest]
10747 fn test_node_build_releases_thread_after_drop() {
10748 let mut node = LiveNode::build("First".to_string(), None).unwrap();
10749 node.dispose();
10750 assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10751 drop(node);
10752
10753 let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10754 assert_eq!(replacement.state(), NodeState::Idle);
10755 }
10756
10757 #[rstest]
10758 fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10759 let config = LiveNodeConfig {
10760 event_store: Some(EventStoreConfig::default()),
10761 ..Default::default()
10762 };
10763
10764 let result = if builder {
10765 LiveNodeBuilder::from_config(config).unwrap().build()
10766 } else {
10767 LiveNode::build("Failure".to_string(), Some(config))
10768 };
10769
10770 assert!(result.unwrap_err().to_string().contains("factory"));
10771
10772 let node = LiveNode::build("Retry".to_string(), None).unwrap();
10773 assert_eq!(node.state(), NodeState::Idle);
10774 }
10775
10776 #[cfg(feature = "python")]
10777 #[rstest]
10778 fn test_node_build_and_initial_state() {
10779 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10780 .unwrap()
10781 .with_name("TestNode")
10782 .build()
10783 .unwrap();
10784
10785 assert_eq!(node.state(), NodeState::Idle);
10786 assert!(!node.is_running());
10787 assert_eq!(node.environment(), Environment::Sandbox);
10788 assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10789 }
10790
10791 #[cfg(feature = "python")]
10792 #[rstest]
10793 fn test_node_build_replaces_stale_runner_senders() {
10794 replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10795 replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10796
10797 let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10798 .unwrap()
10799 .with_name("FirstNode")
10800 .build()
10801 .unwrap();
10802
10803 assert_eq!(first.state(), NodeState::Idle);
10804 drop(first);
10805
10806 let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10807 .unwrap()
10808 .with_name("SecondNode")
10809 .build()
10810 .unwrap();
10811
10812 assert_eq!(second.state(), NodeState::Idle);
10813 assert!(!second.is_running());
10814 }
10815
10816 #[cfg(feature = "python")]
10817 #[rstest]
10818 fn test_node_handle_reflects_node_state() {
10819 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10820 .unwrap()
10821 .with_name("TestNode")
10822 .build()
10823 .unwrap();
10824
10825 let handle = node.handle();
10826
10827 assert_eq!(handle.state(), NodeState::Idle);
10828 assert!(!handle.is_running());
10829 }
10830
10831 #[rstest]
10832 fn test_pending_drain_data_returns_false_when_empty() {
10833 let mut pending = PendingEvents::default();
10834
10835 assert!(!pending.drain_data());
10836 }
10837
10838 #[rstest]
10839 fn test_pending_drain_data_returns_true_when_non_empty() {
10840 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10841
10842 let mut pending = PendingEvents::default();
10843 pending.data_evts.push(
10844 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10845 .into(),
10846 );
10847
10848 assert!(pending.drain_data());
10849 assert!(pending.data_evts.is_empty());
10850 }
10851
10852 fn stub_data_event() -> DataEvent {
10853 use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10854
10855 DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10856 }
10857
10858 fn stub_data_command() -> DataCommand {
10859 use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10860 use nautilus_core::{UUID4, UnixNanos};
10861 use nautilus_model::identifiers::Venue;
10862
10863 DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10864 None,
10865 Venue::from("TEST"),
10866 UUID4::new(),
10867 UnixNanos::default(),
10868 None,
10869 None,
10870 )))
10871 }
10872
10873 fn stub_system_command() -> SystemCommand {
10874 SystemCommand::ReconnectSocket(ReconnectSocket::new(
10875 TraderId::from("TRADER-001"),
10876 ClientId::from("POLYMARKET"),
10877 Ustr::from("polymarket-market-streams"),
10878 UnixNanos::default(),
10879 ))
10880 }
10881
10882 #[rstest]
10883 fn test_flush_pending_data_drains_events_and_commands() {
10884 let (evt_tx, mut evt_rx) =
10885 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10886 let (cmd_tx, mut cmd_rx) =
10887 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10888
10889 let mut pending = PendingEvents::default();
10890
10891 pending.data_evts.push((stub_data_event()).into());
10893 pending.data_cmds.push(stub_data_command().into());
10894
10895 evt_tx.send((stub_data_event()).into()).unwrap();
10897 cmd_tx.send(stub_data_command().into()).unwrap();
10898
10899 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10900
10901 assert!(pending.data_evts.is_empty());
10902 assert!(pending.data_cmds.is_empty());
10903 assert!(evt_rx.try_recv().is_err());
10904 assert!(cmd_rx.try_recv().is_err());
10905 }
10906
10907 #[rstest]
10908 fn test_flush_pending_data_drains_mixed_sources() {
10909 let (evt_tx, mut evt_rx) =
10910 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10911 let (cmd_tx, mut cmd_rx) =
10912 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10913
10914 let mut pending = PendingEvents::default();
10915
10916 pending.data_evts.push((stub_data_event()).into());
10918 cmd_tx.send(stub_data_command().into()).unwrap();
10919
10920 evt_tx.send((stub_data_event()).into()).unwrap();
10922 evt_tx.send((stub_data_event()).into()).unwrap();
10923 cmd_tx.send(stub_data_command().into()).unwrap();
10924
10925 flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10926
10927 assert!(pending.data_evts.is_empty());
10928 assert!(pending.data_cmds.is_empty());
10929 assert!(evt_rx.try_recv().is_err());
10930 assert!(cmd_rx.try_recv().is_err());
10931 }
10932
10933 #[rstest]
10934 fn test_pending_system_events_stay_separate_from_data() {
10935 let mut pending = PendingEvents::default();
10936
10937 let change = SocketStateChange::new(
10938 ClientId::from("BINANCE"),
10939 Some(Venue::from("BINANCE")),
10940 ustr::Ustr::from("binance-futures-market-streams"),
10941 SocketState::Connected,
10942 );
10943
10944 pending
10945 .system_events
10946 .push(SystemEvent::SocketState(change).into());
10947 let system_events = pending
10948 .take_system_events()
10949 .into_iter()
10950 .map(|message| message.dispatch(|value| value))
10951 .collect::<Vec<_>>();
10952
10953 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10954 assert!(pending.is_empty());
10955 }
10956
10957 #[rstest]
10958 fn test_pending_system_commands_stay_separate_from_data() {
10959 let mut pending = PendingEvents::default();
10960 let command = stub_system_command();
10961
10962 pending.system_commands.push(command.into());
10963 let system_commands = pending
10964 .take_system_commands()
10965 .into_iter()
10966 .map(|message| message.dispatch(|value| value))
10967 .collect::<Vec<_>>();
10968
10969 assert_eq!(system_commands, vec![command]);
10970 assert!(pending.is_empty());
10971 }
10972
10973 fn stub_time_event_handler() -> TimeEventMessage {
10974 use std::rc::Rc;
10975
10976 use nautilus_common::{
10977 runner::TimeEventMessage,
10978 timer::{TimeEvent, TimeEventCallback},
10979 };
10980 use nautilus_core::{UUID4, UnixNanos};
10981 use ustr::Ustr;
10982
10983 TimeEventMessage::new(
10984 TimeEvent::new(
10985 Ustr::from("test-timer"),
10986 UUID4::new(),
10987 UnixNanos::default(),
10988 UnixNanos::default(),
10989 ),
10990 TimeEventCallback::RustLocal(Rc::new(|_| {})),
10991 )
10992 }
10993
10994 fn stub_trading_command_message() -> TradingCommandMessage {
10995 use nautilus_common::messages::execution::query::QueryAccount;
10996 use nautilus_core::{UUID4, UnixNanos};
10997 use nautilus_model::identifiers::AccountId;
10998
10999 TradingCommandMessage::new(
11000 MessagingSwitchboard::exec_engine_execute(),
11001 TradingCommand::QueryAccount(QueryAccount::new(
11002 TraderId::from("TESTER-001"),
11003 None,
11004 AccountId::from("TEST-001"),
11005 UUID4::new(),
11006 UnixNanos::default(),
11007 None,
11008 None, )),
11010 )
11011 }
11012
11013 fn stub_exec_event() -> ExecutionEvent {
11014 use nautilus_model::{
11015 enums::{LiquiditySide, OrderSide},
11016 identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
11017 reports::FillReport,
11018 types::{Money, Price, Quantity},
11019 };
11020
11021 ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
11022 AccountId::from("TEST-001"),
11023 InstrumentId::from("TEST.VENUE"),
11024 VenueOrderId::from("V-001"),
11025 TradeId::from("T-001"),
11026 OrderSide::Buy,
11027 Quantity::from("1.0"),
11028 Price::from("100.0"),
11029 Money::from("0.01 USD"),
11030 LiquiditySide::Maker,
11031 None,
11032 None,
11033 nautilus_core::UnixNanos::default(),
11034 nautilus_core::UnixNanos::default(),
11035 None,
11036 ))))
11037 }
11038
11039 #[rstest]
11040 fn test_flush_all_pending_drains_buffered_channels() {
11041 let (time_tx, mut time_rx) =
11042 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11043 let (system_evt_tx, mut system_evt_rx) =
11044 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11045 let (system_cmd_tx, mut system_cmd_rx) =
11046 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11047 let (data_evt_tx, mut data_evt_rx) =
11048 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11049 let (data_cmd_tx, mut data_cmd_rx) =
11050 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11051 let (exec_evt_tx, mut exec_evt_rx) =
11052 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11053 let (exec_cmd_tx, mut exec_cmd_rx) =
11054 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11055
11056 let mut pending = PendingEvents::default();
11057
11058 pending.data_evts.push((stub_data_event()).into());
11060 pending.data_cmds.push(stub_data_command().into());
11061
11062 time_tx.send((stub_time_event_handler()).into()).unwrap();
11064
11065 let change = SocketStateChange::new(
11066 ClientId::from("BINANCE"),
11067 Some(Venue::from("BINANCE")),
11068 Ustr::from("binance-futures-market-streams"),
11069 SocketState::Connected,
11070 );
11071 system_evt_tx
11072 .send((SystemEvent::SocketState(change)).into())
11073 .unwrap();
11074 system_cmd_tx.send((stub_system_command()).into()).unwrap();
11075 data_evt_tx.send((stub_data_event()).into()).unwrap();
11076 data_cmd_tx.send(stub_data_command().into()).unwrap();
11077 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11078 exec_cmd_tx
11079 .send(stub_trading_command_message().into())
11080 .unwrap();
11081
11082 flush_all_pending(
11083 &mut pending,
11084 &mut time_rx,
11085 &mut system_evt_rx,
11086 &mut system_cmd_rx,
11087 &mut exec_evt_rx,
11088 &mut exec_cmd_rx,
11089 &mut data_evt_rx,
11090 &mut data_cmd_rx,
11091 );
11092
11093 let system_events = pending
11094 .take_system_events()
11095 .into_iter()
11096 .map(|message| message.dispatch(|value| value))
11097 .collect::<Vec<_>>();
11098 let system_commands = pending
11099 .take_system_commands()
11100 .into_iter()
11101 .map(|message| message.dispatch(|value| value))
11102 .collect::<Vec<_>>();
11103 assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
11104 assert_eq!(system_commands, vec![stub_system_command()]);
11105 assert!(pending.data_evts.is_empty());
11106 assert!(pending.data_cmds.is_empty());
11107 assert!(pending.exec_reports.is_empty());
11108 assert!(pending.exec_cmds.is_empty());
11109 assert!(pending.order_evts.is_empty());
11110 assert!(time_rx.try_recv().is_err());
11111 assert!(system_evt_rx.try_recv().is_err());
11112 assert!(system_cmd_rx.try_recv().is_err());
11113 assert!(data_evt_rx.try_recv().is_err());
11114 assert!(data_cmd_rx.try_recv().is_err());
11115 assert!(exec_evt_rx.try_recv().is_err());
11116 assert!(exec_cmd_rx.try_recv().is_err());
11117 }
11118
11119 fn stub_order_event() -> ExecutionEvent {
11120 use nautilus_model::events::order::spec::OrderSubmittedSpec;
11121
11122 ExecutionEvent::Order(OrderEventAny::Submitted(
11123 OrderSubmittedSpec::builder().build(),
11124 ))
11125 }
11126
11127 fn stub_account_event() -> ExecutionEvent {
11128 use nautilus_core::{UUID4, UnixNanos};
11129 use nautilus_model::{
11130 enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
11131 };
11132
11133 ExecutionEvent::Account(AccountState::new(
11134 AccountId::from("TEST-001"),
11135 AccountType::Cash,
11136 vec![],
11137 vec![],
11138 true,
11139 UUID4::new(),
11140 UnixNanos::default(),
11141 UnixNanos::default(),
11142 None,
11143 ))
11144 }
11145
11146 #[rstest]
11147 fn test_flush_all_pending_routes_order_event_to_order_evts() {
11148 let (_time_tx, mut time_rx) =
11149 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11150 let (_system_evt_tx, mut system_evt_rx) =
11151 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11152 let (_system_cmd_tx, mut system_cmd_rx) =
11153 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11154 let (_data_evt_tx, mut data_evt_rx) =
11155 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11156 let (_data_cmd_tx, mut data_cmd_rx) =
11157 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11158 let (exec_evt_tx, mut exec_evt_rx) =
11159 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11160 let (_exec_cmd_tx, mut exec_cmd_rx) =
11161 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11162
11163 let mut pending = PendingEvents::default();
11164
11165 exec_evt_tx.send((stub_order_event()).into()).unwrap();
11166 exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11167
11168 flush_all_pending(
11169 &mut pending,
11170 &mut time_rx,
11171 &mut system_evt_rx,
11172 &mut system_cmd_rx,
11173 &mut exec_evt_rx,
11174 &mut exec_cmd_rx,
11175 &mut data_evt_rx,
11176 &mut data_cmd_rx,
11177 );
11178
11179 assert!(pending.order_evts.is_empty());
11181 assert!(pending.exec_reports.is_empty());
11182 assert!(exec_evt_rx.try_recv().is_err());
11183 }
11184
11185 #[rstest]
11186 fn test_flush_all_pending_routes_account_event_immediately() {
11187 let (_time_tx, mut time_rx) =
11188 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11189 let (_system_evt_tx, mut system_evt_rx) =
11190 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11191 let (_system_cmd_tx, mut system_cmd_rx) =
11192 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11193 let (_data_evt_tx, mut data_evt_rx) =
11194 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11195 let (_data_cmd_tx, mut data_cmd_rx) =
11196 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11197 let (exec_evt_tx, mut exec_evt_rx) =
11198 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11199 let (_exec_cmd_tx, mut exec_cmd_rx) =
11200 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11201
11202 let mut pending = PendingEvents::default();
11203
11204 exec_evt_tx.send((stub_account_event()).into()).unwrap();
11205
11206 flush_all_pending(
11207 &mut pending,
11208 &mut time_rx,
11209 &mut system_evt_rx,
11210 &mut system_cmd_rx,
11211 &mut exec_evt_rx,
11212 &mut exec_cmd_rx,
11213 &mut data_evt_rx,
11214 &mut data_cmd_rx,
11215 );
11216
11217 assert!(pending.exec_reports.is_empty());
11219 assert!(pending.order_evts.is_empty());
11220 assert!(pending.exec_cmds.is_empty());
11221 assert!(exec_evt_rx.try_recv().is_err());
11222 }
11223
11224 #[rstest]
11225 fn test_pending_is_empty_when_default() {
11226 let pending = PendingEvents::default();
11227
11228 assert!(pending.is_empty());
11229 }
11230
11231 #[rstest]
11232 fn test_pending_is_empty_false_with_data_evt() {
11233 let mut pending = PendingEvents::default();
11234 pending.data_evts.push((stub_data_event()).into());
11235
11236 assert!(!pending.is_empty());
11237 }
11238
11239 #[rstest]
11240 fn test_pending_is_empty_false_with_data_cmd() {
11241 let mut pending = PendingEvents::default();
11242 pending.data_cmds.push(stub_data_command().into());
11243
11244 assert!(!pending.is_empty());
11245 }
11246
11247 #[rstest]
11248 fn test_pending_is_empty_false_with_exec_cmd() {
11249 let mut pending = PendingEvents::default();
11250 pending
11251 .exec_cmds
11252 .push(stub_trading_command_message().into());
11253
11254 assert!(!pending.is_empty());
11255 }
11256
11257 #[rstest]
11258 fn test_pending_drain_preserves_trading_command_target() {
11259 std::thread::spawn(|| {
11260 msgbus::get_message_bus().borrow_mut().dispose();
11261 let risk_commands = Rc::new(RefCell::new(Vec::new()));
11262 let exec_commands = Rc::new(RefCell::new(Vec::new()));
11263
11264 let risk_commands_handler = risk_commands.clone();
11265 msgbus::register_trading_command_endpoint(
11266 MessagingSwitchboard::risk_engine_execute(),
11267 TypedIntoHandler::from(move |command: TradingCommand| {
11268 risk_commands_handler.borrow_mut().push(command);
11269 }),
11270 );
11271
11272 let exec_commands_handler = exec_commands.clone();
11273 msgbus::register_trading_command_endpoint(
11274 MessagingSwitchboard::exec_engine_execute(),
11275 TypedIntoHandler::from(move |command: TradingCommand| {
11276 exec_commands_handler.borrow_mut().push(command);
11277 }),
11278 );
11279
11280 let mut pending = PendingEvents::default();
11281 pending.exec_cmds.push(
11282 TradingCommandMessage::new(
11283 MessagingSwitchboard::risk_engine_execute(),
11284 TradingCommand::QueryAccount(QueryAccount::new(
11285 TraderId::from("TESTER-001"),
11286 None,
11287 AccountId::from("TEST-001"),
11288 UUID4::new(),
11289 UnixNanos::default(),
11290 None,
11291 None,
11292 )),
11293 )
11294 .into(),
11295 );
11296
11297 pending.drain();
11298
11299 assert!(pending.is_empty());
11300 assert_eq!(risk_commands.borrow().len(), 1);
11301 assert!(matches!(
11302 &risk_commands.borrow()[0],
11303 TradingCommand::QueryAccount(_)
11304 ));
11305 assert_eq!(exec_commands.borrow().as_slice(), &[]);
11306 })
11307 .join()
11308 .unwrap();
11309 }
11310
11311 #[rstest]
11312 fn test_pending_is_empty_false_with_exec_report() {
11313 let mut pending = PendingEvents::default();
11314
11315 if let ExecutionEvent::Report(report) = stub_exec_event() {
11316 pending.exec_reports.push((report).into());
11317 }
11318
11319 assert!(!pending.is_empty());
11320 }
11321
11322 #[rstest]
11323 fn test_pending_is_empty_false_with_order_evt() {
11324 let mut pending = PendingEvents::default();
11325
11326 if let ExecutionEvent::Order(order_evt) = stub_order_event() {
11327 pending.order_evts.push((order_evt).into());
11328 }
11329
11330 assert!(!pending.is_empty());
11331 }
11332
11333 fn stub_submitted_batch_event() -> ExecutionEvent {
11334 use nautilus_model::{
11335 events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
11336 identifiers::ClientOrderId,
11337 };
11338
11339 let events = vec![
11340 OrderSubmittedSpec::builder()
11341 .client_order_id(ClientOrderId::from("O-001"))
11342 .build(),
11343 OrderSubmittedSpec::builder()
11344 .client_order_id(ClientOrderId::from("O-002"))
11345 .build(),
11346 ];
11347
11348 ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
11349 }
11350
11351 fn stub_canceled_batch_event() -> ExecutionEvent {
11352 use nautilus_model::{
11353 events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
11354 identifiers::ClientOrderId,
11355 };
11356
11357 let events = vec![
11358 OrderCanceledSpec::builder()
11359 .client_order_id(ClientOrderId::from("O-001"))
11360 .build(),
11361 OrderCanceledSpec::builder()
11362 .client_order_id(ClientOrderId::from("O-002"))
11363 .build(),
11364 ];
11365
11366 ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
11367 }
11368
11369 #[rstest]
11370 fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
11371 let (_time_tx, mut time_rx) =
11372 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11373 let (_system_evt_tx, mut system_evt_rx) =
11374 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11375 let (_system_cmd_tx, mut system_cmd_rx) =
11376 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11377 let (_data_evt_tx, mut data_evt_rx) =
11378 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11379 let (_data_cmd_tx, mut data_cmd_rx) =
11380 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11381 let (exec_evt_tx, mut exec_evt_rx) =
11382 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11383 let (_exec_cmd_tx, mut exec_cmd_rx) =
11384 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11385
11386 let mut pending = PendingEvents::default();
11387
11388 exec_evt_tx
11389 .send((stub_submitted_batch_event()).into())
11390 .unwrap();
11391
11392 flush_all_pending(
11393 &mut pending,
11394 &mut time_rx,
11395 &mut system_evt_rx,
11396 &mut system_cmd_rx,
11397 &mut exec_evt_rx,
11398 &mut exec_cmd_rx,
11399 &mut data_evt_rx,
11400 &mut data_cmd_rx,
11401 );
11402
11403 assert!(pending.order_evts.is_empty());
11405 assert!(exec_evt_rx.try_recv().is_err());
11406 }
11407
11408 #[rstest]
11409 fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
11410 let (_time_tx, mut time_rx) =
11411 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11412 let (_system_evt_tx, mut system_evt_rx) =
11413 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11414 let (_system_cmd_tx, mut system_cmd_rx) =
11415 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11416 let (_data_evt_tx, mut data_evt_rx) =
11417 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11418 let (_data_cmd_tx, mut data_cmd_rx) =
11419 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11420 let (exec_evt_tx, mut exec_evt_rx) =
11421 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11422 let (_exec_cmd_tx, mut exec_cmd_rx) =
11423 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11424
11425 let mut pending = PendingEvents::default();
11426
11427 exec_evt_tx
11428 .send((stub_canceled_batch_event()).into())
11429 .unwrap();
11430
11431 flush_all_pending(
11432 &mut pending,
11433 &mut time_rx,
11434 &mut system_evt_rx,
11435 &mut system_cmd_rx,
11436 &mut exec_evt_rx,
11437 &mut exec_cmd_rx,
11438 &mut data_evt_rx,
11439 &mut data_cmd_rx,
11440 );
11441
11442 assert!(pending.order_evts.is_empty());
11444 assert!(exec_evt_rx.try_recv().is_err());
11445 }
11446
11447 #[rstest]
11448 fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
11449 use nautilus_model::identifiers::ClientOrderId;
11450
11451 let (exec_evt_tx, mut exec_evt_rx) =
11452 tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11453
11454 exec_evt_tx
11455 .send((stub_canceled_batch_event()).into())
11456 .unwrap();
11457
11458 let mut pending = PendingEvents::default();
11459
11460 while let Ok(evt) = exec_evt_rx.try_recv() {
11461 pending.push_exec_event(evt);
11462 }
11463
11464 let events: Vec<_> = pending
11465 .order_evts
11466 .drain(..)
11467 .map(|event| event.dispatch(|event| event))
11468 .collect();
11469 assert_eq!(events.len(), 2);
11470 assert!(
11471 matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
11472 );
11473 assert!(
11474 matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
11475 );
11476 }
11477
11478 #[rstest]
11479 #[cfg_attr(
11480 not(all(feature = "simulation", madsim)),
11481 tokio::test(start_paused = true)
11482 )]
11483 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11484 async fn test_submission_confirmation_prevents_duplicate_tracking(
11485 #[values(false, true)] report_confirmation: bool,
11486 #[values(false, true)] exhausted: bool,
11487 ) {
11488 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11489 let client_order_id = order.client_order_id();
11490 let account_id = AccountId::from("TEST-001");
11491 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11492 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11493 OrderPendingUpdateSpec::builder()
11494 .trader_id(order.trader_id())
11495 .strategy_id(order.strategy_id())
11496 .instrument_id(order.instrument_id())
11497 .client_order_id(client_order_id)
11498 .account_id(account_id)
11499 .build(),
11500 )));
11501 advance_clock(Duration::from_millis(101)).await;
11502 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11503
11504 if exhausted {
11505 advance_clock(Duration::from_millis(101)).await;
11506 let result = node.exec_manager.check_inflight_orders();
11507 assert!(result.events.is_empty());
11508 assert!(result.queries.is_empty());
11509 assert_eq!(
11510 node.exec_manager
11511 .take_submission_recovery_exhaustions()
11512 .len(),
11513 1
11514 );
11515 }
11516
11517 if report_confirmation {
11518 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11519 OrderStatusReport::new(
11520 account_id,
11521 order.instrument_id(),
11522 Some(client_order_id),
11523 venue_order_id,
11524 Some(order.order_side()),
11525 order.order_type(),
11526 order.time_in_force(),
11527 OrderStatus::Submitted,
11528 order.quantity(),
11529 Quantity::zero(3),
11530 UnixNanos::default(),
11531 UnixNanos::default(),
11532 UnixNanos::default(),
11533 None,
11534 )
11535 .with_price(order.price().unwrap()),
11536 ))));
11537 node.exec_manager
11538 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11539 advance_clock(Duration::from_millis(101)).await;
11540 let result = node.exec_manager.check_inflight_orders();
11541 assert!(result.queries.is_empty());
11542 assert!(result.events.is_empty());
11543 }
11544
11545 let mut update = OrderUpdatedSpec::builder()
11546 .trader_id(order.trader_id())
11547 .strategy_id(order.strategy_id())
11548 .instrument_id(order.instrument_id())
11549 .client_order_id(client_order_id)
11550 .account_id(account_id)
11551 .quantity(order.quantity())
11552 .price(order.price().unwrap())
11553 .build();
11554
11555 if !report_confirmation {
11556 update.venue_order_id = Some(venue_order_id);
11557 }
11558 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11559 assert_eq!(
11560 node.kernel
11561 .cache
11562 .borrow()
11563 .order(&client_order_id)
11564 .unwrap()
11565 .status(),
11566 OrderStatus::Submitted
11567 );
11568
11569 for _ in 0..3 {
11570 node.exec_manager
11571 .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11572 advance_clock(Duration::from_millis(101)).await;
11573 let result = node.exec_manager.check_inflight_orders();
11574 assert!(result.queries.is_empty());
11575 assert!(result.events.is_empty());
11576 assert!(
11577 node.exec_manager
11578 .take_submission_recovery_exhaustions()
11579 .is_empty()
11580 );
11581 }
11582 }
11583
11584 #[rstest]
11585 #[cfg_attr(
11586 not(all(feature = "simulation", madsim)),
11587 tokio::test(start_paused = true)
11588 )]
11589 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11590 async fn test_submission_report_confirmation_preserves_later_command_budget(
11591 #[values(false, true)] cancel: bool,
11592 #[values(false, true)] later_fill: bool,
11593 #[values(false, true)] repeated_report: bool,
11594 #[values(
11595 SubmissionRecoveryPolicy::ResolveLocally,
11596 SubmissionRecoveryPolicy::RetainUnresolved
11597 )]
11598 policy: SubmissionRecoveryPolicy,
11599 ) {
11600 let (mut node, order, instrument) = venue_evidence_node(policy);
11601 let client_order_id = order.client_order_id();
11602 let account_id = AccountId::from("TEST-001");
11603 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11604 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11605 OrderPendingUpdateSpec::builder()
11606 .trader_id(order.trader_id())
11607 .strategy_id(order.strategy_id())
11608 .instrument_id(order.instrument_id())
11609 .client_order_id(client_order_id)
11610 .account_id(account_id)
11611 .build(),
11612 )));
11613 node.exec_manager
11614 .register_inflight(client_order_id, UUID4::new());
11615 advance_clock(Duration::from_millis(101)).await;
11616 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11617 let report = ExecutionReport::Order(Box::new(
11618 OrderStatusReport::new(
11619 account_id,
11620 order.instrument_id(),
11621 Some(client_order_id),
11622 venue_order_id,
11623 Some(order.order_side()),
11624 order.order_type(),
11625 order.time_in_force(),
11626 OrderStatus::Submitted,
11627 order.quantity(),
11628 Quantity::zero(3),
11629 UnixNanos::default(),
11630 UnixNanos::default(),
11631 UnixNanos::default(),
11632 None,
11633 )
11634 .with_price(order.price().unwrap()),
11635 ));
11636 node.process_exec_event(ExecutionEvent::Report(report.clone()));
11637 advance_clock(Duration::from_millis(101)).await;
11638 let result = node.exec_manager.check_inflight_orders();
11639 assert!(result.events.is_empty());
11640 assert!(result.queries.is_empty());
11641
11642 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11643 OrderUpdatedSpec::builder()
11644 .trader_id(order.trader_id())
11645 .strategy_id(order.strategy_id())
11646 .instrument_id(order.instrument_id())
11647 .client_order_id(client_order_id)
11648 .account_id(account_id)
11649 .quantity(order.quantity())
11650 .price(order.price().unwrap())
11651 .build(),
11652 )));
11653 let pending = if cancel {
11654 OrderEventAny::PendingCancel(
11655 OrderPendingCancelSpec::builder()
11656 .trader_id(order.trader_id())
11657 .strategy_id(order.strategy_id())
11658 .instrument_id(order.instrument_id())
11659 .client_order_id(client_order_id)
11660 .account_id(account_id)
11661 .build(),
11662 )
11663 } else {
11664 OrderEventAny::PendingUpdate(
11665 OrderPendingUpdateSpec::builder()
11666 .trader_id(order.trader_id())
11667 .strategy_id(order.strategy_id())
11668 .instrument_id(order.instrument_id())
11669 .client_order_id(client_order_id)
11670 .account_id(account_id)
11671 .build(),
11672 )
11673 };
11674 node.process_exec_event(ExecutionEvent::Order(pending));
11675 node.exec_manager
11676 .register_inflight(client_order_id, UUID4::new());
11677
11678 if repeated_report {
11679 node.process_exec_event(ExecutionEvent::Report(report));
11680 }
11681 advance_clock(Duration::from_millis(50)).await;
11682
11683 if later_fill {
11684 let cached = node
11685 .kernel
11686 .cache
11687 .borrow()
11688 .order_owned(&client_order_id)
11689 .unwrap();
11690 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11691 .last_qty(Quantity::from("0.400"))
11692 .last_px(order.price().unwrap())
11693 .commission(Money::from("0 USDT"))
11694 .without_position_id()
11695 .build()
11696 else {
11697 unreachable!()
11698 };
11699 fill.venue_order_id = venue_order_id;
11700 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11701 }
11702 let early = node.exec_manager.check_inflight_orders();
11703 assert!(early.events.is_empty());
11704 assert!(early.queries.is_empty());
11705 advance_clock(Duration::from_millis(51)).await;
11706 let first = node.exec_manager.check_inflight_orders();
11707 assert_eq!(first.queries.len(), usize::from(!repeated_report));
11708 assert!(first.events.is_empty());
11709 advance_clock(Duration::from_millis(101)).await;
11710 let second = node.exec_manager.check_inflight_orders();
11711 assert!(second.queries.is_empty());
11712 if repeated_report {
11713 assert!(second.events.is_empty());
11714 } else {
11715 assert_command_timeout_event(&second.events, client_order_id, policy, cancel);
11716 }
11717 assert!(
11718 node.exec_manager
11719 .take_submission_recovery_exhaustions()
11720 .is_empty()
11721 );
11722 }
11723
11724 #[rstest]
11725 #[cfg_attr(
11726 not(all(feature = "simulation", madsim)),
11727 tokio::test(start_paused = true)
11728 )]
11729 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11730 async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11731 #[values(false, true)] report_confirmation: bool,
11732 #[values(false, true)] cancel: bool,
11733 #[values(false, true)] batch: bool,
11734 #[values(false, true)] pending_before_dispatch: bool,
11735 #[values(false, true)] missing_order_check: bool,
11736 #[values(
11737 SubmissionRecoveryPolicy::ResolveLocally,
11738 SubmissionRecoveryPolicy::RetainUnresolved
11739 )]
11740 policy: SubmissionRecoveryPolicy,
11741 ) {
11742 let (mut node, order, instrument) = venue_evidence_node_with_config(
11743 crate::config::LiveExecutionEngineConfig {
11744 reconciliation: true,
11745 inflight_check_threshold_ms: 100,
11746 inflight_check_retries: 2,
11747 open_check_threshold_ms: 5_000,
11748 open_check_open_only: !missing_order_check,
11749 open_check_lookback_mins: None,
11750 open_check_missing_retries: 5,
11751 single_order_query_delay_ms: 0,
11752 submission_recovery_policy: policy,
11753 ..Default::default()
11754 },
11755 OrderType::Limit,
11756 true,
11757 );
11758 let client_order_id = order.client_order_id();
11759 let client_id = ClientId::from("TEST-EVIDENCE");
11760 let account_id = AccountId::from("TEST-001");
11761 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11762 node.kernel
11763 .exec_engine
11764 .borrow_mut()
11765 .register_client(Box::new(StubExecutionClient::new(
11766 client_id,
11767 account_id,
11768 instrument.id().venue,
11769 OmsType::Netting,
11770 None,
11771 )))
11772 .unwrap();
11773
11774 advance_clock(Duration::from_millis(101)).await;
11776 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11777
11778 if report_confirmation {
11779 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11780 OrderPendingUpdateSpec::builder()
11781 .trader_id(order.trader_id())
11782 .strategy_id(order.strategy_id())
11783 .instrument_id(order.instrument_id())
11784 .client_order_id(client_order_id)
11785 .account_id(account_id)
11786 .build(),
11787 )));
11788 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11789 OrderStatusReport::new(
11790 account_id,
11791 order.instrument_id(),
11792 Some(client_order_id),
11793 venue_order_id,
11794 Some(order.order_side()),
11795 order.order_type(),
11796 order.time_in_force(),
11797 OrderStatus::Submitted,
11798 order.quantity(),
11799 Quantity::zero(3),
11800 UnixNanos::default(),
11801 UnixNanos::default(),
11802 UnixNanos::default(),
11803 None,
11804 )
11805 .with_price(order.price().unwrap()),
11806 ))));
11807 }
11808
11809 let mut update = OrderUpdatedSpec::builder()
11810 .trader_id(order.trader_id())
11811 .strategy_id(order.strategy_id())
11812 .instrument_id(order.instrument_id())
11813 .client_order_id(client_order_id)
11814 .account_id(account_id)
11815 .quantity(order.quantity())
11816 .price(order.price().unwrap())
11817 .build();
11818
11819 if !report_confirmation {
11820 update.venue_order_id = Some(venue_order_id);
11821 }
11822 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11823 assert_eq!(
11824 node.kernel
11825 .cache
11826 .borrow()
11827 .order(&client_order_id)
11828 .unwrap()
11829 .status(),
11830 OrderStatus::Submitted,
11831 );
11832 advance_clock(Duration::from_millis(101)).await;
11833 let confirmed = node.exec_manager.check_inflight_orders();
11834 assert!(confirmed.queries.is_empty());
11835 assert!(confirmed.events.is_empty());
11836
11837 let pending = if cancel {
11838 OrderEventAny::PendingCancel(
11839 OrderPendingCancelSpec::builder()
11840 .trader_id(order.trader_id())
11841 .strategy_id(order.strategy_id())
11842 .instrument_id(order.instrument_id())
11843 .client_order_id(client_order_id)
11844 .account_id(account_id)
11845 .build(),
11846 )
11847 } else {
11848 OrderEventAny::PendingUpdate(
11849 OrderPendingUpdateSpec::builder()
11850 .trader_id(order.trader_id())
11851 .strategy_id(order.strategy_id())
11852 .instrument_id(order.instrument_id())
11853 .client_order_id(client_order_id)
11854 .account_id(account_id)
11855 .build(),
11856 )
11857 };
11858 let command = venue_evidence_command(&order, cancel, batch);
11859
11860 if pending_before_dispatch {
11861 node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11862 }
11863 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11864 MessagingSwitchboard::exec_engine_execute(),
11865 command,
11866 )));
11867 advance_clock(Duration::from_millis(50)).await;
11868
11869 if missing_order_check {
11870 let engine = node.kernel.exec_engine.borrow();
11871 let clients = engine.get_all_clients();
11872 let check = node
11873 .exec_manager
11874 .prepare_open_order_report_check(UUID4::new(), &clients);
11875 assert_eq!(check.filtered_orders.len(), 1);
11876 let missing = node.exec_manager.reconcile_open_order_reports(
11877 &check,
11878 Vec::new(),
11879 &IndexSet::from([client_id]),
11880 &IndexSet::new(),
11881 &clients,
11882 );
11883 assert!(missing.events.is_empty());
11884 assert!(missing.targeted_queries.is_empty());
11885 assert_eq!(
11886 node.exec_manager.recon_check_retry_count(&client_order_id),
11887 1
11888 );
11889 }
11890 let early = node.exec_manager.check_inflight_orders();
11891 assert!(early.queries.is_empty());
11892 assert!(early.events.is_empty());
11893
11894 if !pending_before_dispatch {
11895 assert_eq!(
11898 node.kernel
11899 .cache
11900 .borrow()
11901 .order(&client_order_id)
11902 .unwrap()
11903 .status(),
11904 OrderStatus::Submitted,
11905 );
11906 node.process_exec_event(ExecutionEvent::Order(pending));
11907 }
11908
11909 if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11910 node.exec_manager
11911 .register_submission(order.init_event(), Some(client_id));
11912 }
11913 advance_clock(Duration::from_millis(51)).await;
11914 let first = node.exec_manager.check_inflight_orders();
11915 assert_eq!(first.queries.len(), 1);
11916 assert!(
11917 matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11918 if query.client_order_id == client_order_id)
11919 );
11920 assert!(first.events.is_empty());
11921 advance_clock(Duration::from_millis(101)).await;
11922 let exhausted = node.exec_manager.check_inflight_orders();
11923 assert!(exhausted.queries.is_empty());
11924 assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
11925 assert!(
11926 node.exec_manager
11927 .take_submission_recovery_exhaustions()
11928 .is_empty()
11929 );
11930 node.process_reconciliation_events(&exhausted.events);
11931
11932 let expected_status = match policy {
11933 SubmissionRecoveryPolicy::ResolveLocally => OrderStatus::Canceled,
11934 SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
11935 };
11936
11937 assert_eq!(
11938 node.kernel
11939 .cache
11940 .borrow()
11941 .order(&client_order_id)
11942 .unwrap()
11943 .status(),
11944 expected_status,
11945 );
11946 }
11947
11948 #[rstest]
11949 #[cfg_attr(
11950 not(all(feature = "simulation", madsim)),
11951 tokio::test(start_paused = true)
11952 )]
11953 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11954 async fn test_submission_command_report_cleanup_matches_default(
11955 #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11956 #[values(
11957 SubmissionRecoveryPolicy::ResolveLocally,
11958 SubmissionRecoveryPolicy::RetainUnresolved
11959 )]
11960 policy: SubmissionRecoveryPolicy,
11961 ) {
11962 let (mut node, order, instrument) = venue_evidence_node(policy);
11963 let client_order_id = order.client_order_id();
11964 let client_id = ClientId::from("TEST-EVIDENCE");
11965 let account_id = AccountId::from("TEST-001");
11966 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11967 node.kernel
11968 .exec_engine
11969 .borrow_mut()
11970 .register_client(Box::new(StubExecutionClient::new(
11971 client_id,
11972 account_id,
11973 instrument.id().venue,
11974 OmsType::Netting,
11975 None,
11976 )))
11977 .unwrap();
11978 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11979 &order,
11980 account_id,
11981 venue_order_id,
11982 )));
11983 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11984 MessagingSwitchboard::exec_engine_execute(),
11985 venue_evidence_command(&order, true, false),
11986 )));
11987 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11988 OrderPendingCancelSpec::builder()
11989 .trader_id(order.trader_id())
11990 .strategy_id(order.strategy_id())
11991 .instrument_id(order.instrument_id())
11992 .client_order_id(client_order_id)
11993 .account_id(account_id)
11994 .venue_order_id(venue_order_id)
11995 .build(),
11996 )));
11997 advance_clock(Duration::from_millis(101)).await;
11998 let first = node.exec_manager.check_inflight_orders();
11999 assert_eq!(first.queries.len(), 1);
12000 assert!(first.events.is_empty());
12001
12002 let filled_qty = if report_status == OrderStatus::PartiallyFilled {
12003 Quantity::from("0.400")
12004 } else {
12005 Quantity::zero(3)
12006 };
12007
12008 let report = OrderStatusReport::new(
12009 account_id,
12010 order.instrument_id(),
12011 Some(client_order_id),
12012 venue_order_id,
12013 Some(order.order_side()),
12014 order.order_type(),
12015 order.time_in_force(),
12016 report_status,
12017 order.quantity(),
12018 filled_qty,
12019 UnixNanos::default(),
12020 UnixNanos::default(),
12021 UnixNanos::default(),
12022 None,
12023 )
12024 .with_price(order.price().unwrap())
12025 .with_avg_px(dec!(100.00));
12026 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12027 report,
12028 ))));
12029 advance_clock(Duration::from_millis(101)).await;
12030 let exhausted = node.exec_manager.check_inflight_orders();
12031
12032 assert!(exhausted.queries.is_empty());
12033 assert!(exhausted.events.is_empty());
12034 assert!(
12035 node.exec_manager
12036 .take_submission_recovery_exhaustions()
12037 .is_empty()
12038 );
12039 let cache = node.kernel.cache.borrow();
12040 let cached = cache.order_ref(&client_order_id).unwrap();
12041 assert_eq!(cached.status(), OrderStatus::PendingCancel);
12042 assert_eq!(cached.filled_qty(), filled_qty);
12043 }
12044
12045 #[rstest]
12046 #[case::accepted_modify(false, "accepted")]
12047 #[case::accepted_cancel(true, "accepted")]
12048 #[case::accepted_batch_modify(false, "accepted_batch")]
12049 #[case::accepted_batch_cancel(true, "accepted_batch")]
12050 #[case::updated_cancel(true, "updated")]
12051 #[case::triggered_cancel(true, "triggered")]
12052 #[cfg_attr(
12053 not(all(feature = "simulation", madsim)),
12054 tokio::test(start_paused = true)
12055 )]
12056 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12057 async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
12058 #[case] cancel: bool,
12059 #[case] evidence: &str,
12060 #[values(false, true)] batch: bool,
12061 #[values(false, true)] report_first: bool,
12062 #[values(
12063 SubmissionRecoveryPolicy::ResolveLocally,
12064 SubmissionRecoveryPolicy::RetainUnresolved
12065 )]
12066 policy: SubmissionRecoveryPolicy,
12067 ) {
12068 use nautilus_model::events::order::spec::OrderTriggeredSpec;
12069
12070 let triggered = evidence == "triggered";
12071
12072 let order_type = if triggered {
12073 OrderType::StopLimit
12074 } else {
12075 OrderType::Limit
12076 };
12077 let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
12078 let client_order_id = order.client_order_id();
12079 let account_id = AccountId::from("TEST-001");
12080 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12081 node.kernel
12082 .exec_engine
12083 .borrow_mut()
12084 .register_client(Box::new(StubExecutionClient::new(
12085 ClientId::from("TEST-EVIDENCE"),
12086 account_id,
12087 instrument.id().venue,
12088 OmsType::Netting,
12089 None,
12090 )))
12091 .unwrap();
12092 advance_clock(Duration::from_millis(101)).await;
12093 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12094
12095 if triggered || report_first {
12096 let previous_pending = if cancel {
12098 OrderEventAny::PendingUpdate(
12099 OrderPendingUpdateSpec::builder()
12100 .trader_id(order.trader_id())
12101 .strategy_id(order.strategy_id())
12102 .instrument_id(order.instrument_id())
12103 .client_order_id(client_order_id)
12104 .account_id(account_id)
12105 .build(),
12106 )
12107 } else {
12108 OrderEventAny::PendingCancel(
12109 OrderPendingCancelSpec::builder()
12110 .trader_id(order.trader_id())
12111 .strategy_id(order.strategy_id())
12112 .instrument_id(order.instrument_id())
12113 .client_order_id(client_order_id)
12114 .account_id(account_id)
12115 .build(),
12116 )
12117 };
12118 node.process_exec_event(ExecutionEvent::Order(previous_pending));
12119 }
12120 advance_clock(Duration::from_millis(10)).await;
12121 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12122 MessagingSwitchboard::exec_engine_execute(),
12123 venue_evidence_command(&order, cancel, batch),
12124 )));
12125
12126 if report_first {
12127 let report = OrderStatusReport::new(
12128 account_id,
12129 instrument.id(),
12130 Some(client_order_id),
12131 venue_order_id,
12132 Some(order.order_side()),
12133 order.order_type(),
12134 order.time_in_force(),
12135 OrderStatus::Submitted,
12136 order.quantity(),
12137 Quantity::zero(3),
12138 UnixNanos::default(),
12139 UnixNanos::default(),
12140 UnixNanos::default(),
12141 None,
12142 )
12143 .with_price(order.price().unwrap());
12144 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12145 report,
12146 ))));
12147 assert!(
12148 node.kernel
12149 .cache
12150 .borrow()
12151 .order(&client_order_id)
12152 .unwrap()
12153 .venue_order_id()
12154 .is_none()
12155 );
12156 }
12157 advance_clock(Duration::from_millis(40)).await;
12158 let acknowledgement = if triggered {
12159 OrderEventAny::Triggered(
12160 OrderTriggeredSpec::builder()
12161 .trader_id(order.trader_id())
12162 .strategy_id(order.strategy_id())
12163 .instrument_id(order.instrument_id())
12164 .client_order_id(client_order_id)
12165 .account_id(account_id)
12166 .venue_order_id(venue_order_id)
12167 .build(),
12168 )
12169 } else if evidence == "updated" {
12170 OrderEventAny::Updated(
12171 OrderUpdatedSpec::builder()
12172 .trader_id(order.trader_id())
12173 .strategy_id(order.strategy_id())
12174 .instrument_id(order.instrument_id())
12175 .client_order_id(client_order_id)
12176 .account_id(account_id)
12177 .venue_order_id(venue_order_id)
12178 .quantity(order.quantity())
12179 .price(order.price().unwrap())
12180 .build(),
12181 )
12182 } else {
12183 TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
12184 };
12185
12186 if evidence == "accepted_batch" {
12187 let OrderEventAny::Accepted(accepted) = acknowledgement else {
12188 unreachable!()
12189 };
12190 node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
12191 vec![accepted],
12192 )));
12193 } else {
12194 node.process_exec_event(ExecutionEvent::Order(acknowledgement));
12195 }
12196 assert_eq!(
12197 node.kernel
12198 .cache
12199 .borrow()
12200 .order(&client_order_id)
12201 .unwrap()
12202 .status(),
12203 match evidence {
12204 "triggered" => OrderStatus::Triggered,
12205 "updated" => OrderStatus::Submitted,
12206 _ => OrderStatus::Accepted,
12207 }
12208 );
12209 advance_clock(Duration::from_millis(20)).await;
12210 let pending = if cancel {
12211 OrderEventAny::PendingCancel(
12212 OrderPendingCancelSpec::builder()
12213 .trader_id(order.trader_id())
12214 .strategy_id(order.strategy_id())
12215 .instrument_id(order.instrument_id())
12216 .client_order_id(client_order_id)
12217 .account_id(account_id)
12218 .build(),
12219 )
12220 } else {
12221 OrderEventAny::PendingUpdate(
12222 OrderPendingUpdateSpec::builder()
12223 .trader_id(order.trader_id())
12224 .strategy_id(order.strategy_id())
12225 .instrument_id(order.instrument_id())
12226 .client_order_id(client_order_id)
12227 .account_id(account_id)
12228 .build(),
12229 )
12230 };
12231 node.process_exec_event(ExecutionEvent::Order(pending));
12232 let early = node.exec_manager.check_inflight_orders();
12233 assert!(early.queries.is_empty());
12234 assert!(early.events.is_empty());
12235 advance_clock(Duration::from_millis(41)).await;
12236 let first = node.exec_manager.check_inflight_orders();
12237 let recover_command =
12239 triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
12240 assert_eq!(first.queries.len(), usize::from(recover_command));
12241 assert!(first.events.is_empty());
12242 advance_clock(Duration::from_millis(101)).await;
12243 let exhausted = node.exec_manager.check_inflight_orders();
12244 assert!(exhausted.queries.is_empty());
12245 assert!(
12246 node.exec_manager
12247 .take_submission_recovery_exhaustions()
12248 .is_empty()
12249 );
12250
12251 if recover_command {
12252 assert!(
12253 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12254 if event.client_order_id == client_order_id)
12255 );
12256 } else {
12257 assert!(exhausted.events.is_empty());
12258 }
12259 }
12260
12261 #[rstest]
12262 #[case::submission_only(None, false)]
12263 #[case::modify(Some(false), false)]
12264 #[case::cancel(Some(true), false)]
12265 #[case::batch_modify(Some(false), true)]
12266 #[case::batch_cancel(Some(true), true)]
12267 #[cfg_attr(
12268 not(all(feature = "simulation", madsim)),
12269 tokio::test(start_paused = true)
12270 )]
12271 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12272 async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
12273 #[case] cancel: Option<bool>,
12274 #[case] batch: bool,
12275 #[values(false, true)] missing_order_check: bool,
12276 #[values(
12277 SubmissionRecoveryPolicy::ResolveLocally,
12278 SubmissionRecoveryPolicy::RetainUnresolved
12279 )]
12280 policy: SubmissionRecoveryPolicy,
12281 ) {
12282 let (mut node, order, instrument) = venue_evidence_node_with_config(
12283 crate::config::LiveExecutionEngineConfig {
12284 reconciliation: true,
12285 inflight_check_threshold_ms: 100,
12286 inflight_check_retries: 2,
12287 open_check_threshold_ms: 1,
12288 open_check_open_only: false,
12289 open_check_lookback_mins: None,
12290 open_check_missing_retries: 5,
12291 single_order_query_delay_ms: 0,
12292 submission_recovery_policy: policy,
12293 ..Default::default()
12294 },
12295 OrderType::Limit,
12296 false,
12297 );
12298 let client_order_id = order.client_order_id();
12299 let client_id = ClientId::from("TEST-EVIDENCE");
12300 let account_id = AccountId::from("TEST-001");
12301 node.kernel
12302 .exec_engine
12303 .borrow_mut()
12304 .register_client(Box::new(StubExecutionClient::new(
12305 client_id,
12306 account_id,
12307 instrument.id().venue,
12308 OmsType::Netting,
12309 None,
12310 )))
12311 .unwrap();
12312 let queries = Rc::new(RefCell::new(Vec::new()));
12313 let client = OrderReportClient {
12314 client_id,
12315 report: OrderStatusReport::new(
12316 account_id,
12317 instrument.id(),
12318 Some(client_order_id),
12319 VenueOrderId::from("V-VENUE-EVIDENCE"),
12320 Some(order.order_side()),
12321 order.order_type(),
12322 order.time_in_force(),
12323 OrderStatus::Submitted,
12324 order.quantity(),
12325 Quantity::zero(3),
12326 UnixNanos::default(),
12327 UnixNanos::default(),
12328 UnixNanos::default(),
12329 None,
12330 )
12331 .with_price(order.price().unwrap()),
12332 targeted: false,
12333 queries: queries.clone(),
12334 };
12335 advance_clock(Duration::from_millis(2)).await;
12336 let events = node.exec_manager.check_open_orders(&[&client]).await;
12337 assert!(events.is_empty());
12338 assert_eq!(*queries.borrow(), vec!["bulk"]);
12339 assert_eq!(
12340 node.kernel
12341 .cache
12342 .borrow()
12343 .order(&client_order_id)
12344 .unwrap()
12345 .status(),
12346 OrderStatus::Submitted
12347 );
12348 advance_clock(Duration::from_millis(2)).await;
12349
12350 if missing_order_check {
12351 let engine = node.kernel.exec_engine.borrow();
12352 let clients = engine.get_all_clients();
12353 let check = node
12354 .exec_manager
12355 .prepare_open_order_report_check(UUID4::new(), &clients);
12356 assert_eq!(check.filtered_orders.len(), 1);
12357 let missing = node.exec_manager.reconcile_open_order_reports(
12358 &check,
12359 Vec::new(),
12360 &IndexSet::from([client_id]),
12361 &IndexSet::new(),
12362 &clients,
12363 );
12364 assert!(missing.events.is_empty());
12365 assert!(missing.targeted_queries.is_empty());
12366 assert_eq!(
12367 node.exec_manager.recon_check_retry_count(&client_order_id),
12368 1
12369 );
12370 }
12371 node.exec_manager
12372 .register_submission(order.init_event(), Some(client_id));
12373 advance_clock(Duration::from_millis(101)).await;
12374 let duplicate = node.exec_manager.check_inflight_orders();
12375 assert!(duplicate.events.is_empty());
12376 assert_eq!(
12377 duplicate.queries.len(),
12378 usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
12379 );
12380
12381 if let Some(cancel) = cancel {
12382 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12383 MessagingSwitchboard::exec_engine_execute(),
12384 venue_evidence_command(&order, cancel, batch),
12385 )));
12386 let pending = if cancel {
12387 OrderEventAny::PendingCancel(
12388 OrderPendingCancelSpec::builder()
12389 .trader_id(order.trader_id())
12390 .strategy_id(order.strategy_id())
12391 .instrument_id(order.instrument_id())
12392 .client_order_id(client_order_id)
12393 .account_id(account_id)
12394 .build(),
12395 )
12396 } else {
12397 OrderEventAny::PendingUpdate(
12398 OrderPendingUpdateSpec::builder()
12399 .trader_id(order.trader_id())
12400 .strategy_id(order.strategy_id())
12401 .instrument_id(order.instrument_id())
12402 .client_order_id(client_order_id)
12403 .account_id(account_id)
12404 .build(),
12405 )
12406 };
12407 node.process_exec_event(ExecutionEvent::Order(pending));
12408 }
12409 advance_clock(Duration::from_millis(101)).await;
12410 let first = node.exec_manager.check_inflight_orders();
12411 assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
12412
12413 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12414 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12415 if event.client_order_id == client_order_id));
12416 } else {
12417 assert!(first.events.is_empty());
12418 }
12419 node.process_reconciliation_events(&first.events);
12420 advance_clock(Duration::from_millis(101)).await;
12421 let exhausted = node.exec_manager.check_inflight_orders();
12422 assert!(exhausted.queries.is_empty());
12423
12424 if let Some(cancel) = cancel {
12425 assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
12426 } else {
12427 assert!(exhausted.events.is_empty());
12428 }
12429 assert!(
12430 node.exec_manager
12431 .take_submission_recovery_exhaustions()
12432 .is_empty()
12433 );
12434 }
12435
12436 fn assert_command_timeout_event(
12437 events: &[OrderEventAny],
12438 client_order_id: ClientOrderId,
12439 policy: SubmissionRecoveryPolicy,
12440 cancel: bool,
12441 ) {
12442 let [event] = events else {
12443 panic!("Expected one command timeout event, was {events:?}");
12444 };
12445
12446 let is_expected = match (policy, cancel) {
12447 (SubmissionRecoveryPolicy::ResolveLocally, _) => {
12448 matches!(event, OrderEventAny::Canceled(_))
12449 }
12450 (SubmissionRecoveryPolicy::RetainUnresolved, true) => {
12451 matches!(event, OrderEventAny::CancelRejected(_))
12452 }
12453 (SubmissionRecoveryPolicy::RetainUnresolved, false) => {
12454 matches!(event, OrderEventAny::ModifyRejected(_))
12455 }
12456 };
12457
12458 assert!(is_expected, "Unexpected command timeout event {event:?}");
12459 assert_eq!(event.client_order_id(), client_order_id);
12460 }
12461
12462 #[rstest]
12463 #[case::submission_only(None, false)]
12464 #[case::modify(Some(false), false)]
12465 #[case::cancel(Some(true), false)]
12466 #[case::batch_modify(Some(false), true)]
12467 #[case::batch_cancel(Some(true), true)]
12468 #[cfg_attr(
12469 not(all(feature = "simulation", madsim)),
12470 tokio::test(start_paused = true)
12471 )]
12472 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12473 async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
12474 #[case] cancel: Option<bool>,
12475 #[case] batch: bool,
12476 #[values(false, true)] targeted: bool,
12477 #[values(
12478 SubmissionRecoveryPolicy::ResolveLocally,
12479 SubmissionRecoveryPolicy::RetainUnresolved
12480 )]
12481 policy: SubmissionRecoveryPolicy,
12482 ) {
12483 let (mut node, order, instrument) = venue_evidence_node_with_config(
12484 crate::config::LiveExecutionEngineConfig {
12485 reconciliation: true,
12486 inflight_check_threshold_ms: 100,
12487 inflight_check_retries: 2,
12488 open_check_threshold_ms: 1,
12489 open_check_open_only: false,
12490 open_check_lookback_mins: None,
12491 open_check_missing_retries: 1,
12492 single_order_query_delay_ms: 0,
12493 submission_recovery_policy: policy,
12494 ..Default::default()
12495 },
12496 OrderType::Limit,
12497 true,
12498 );
12499 let client_order_id = order.client_order_id();
12500 let client_id = ClientId::from("TEST-EVIDENCE");
12501 let account_id = AccountId::from("TEST-001");
12502 node.kernel
12503 .exec_engine
12504 .borrow_mut()
12505 .register_client(Box::new(StubExecutionClient::new(
12506 client_id,
12507 account_id,
12508 instrument.id().venue,
12509 OmsType::Netting,
12510 None,
12511 )))
12512 .unwrap();
12513 advance_clock(Duration::from_millis(101)).await;
12514 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12515 advance_clock(Duration::from_millis(10)).await;
12516
12517 if let Some(cancel) = cancel {
12518 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12519 MessagingSwitchboard::exec_engine_execute(),
12520 venue_evidence_command(&order, cancel, batch),
12521 )));
12522 let pending = if cancel {
12523 OrderEventAny::PendingCancel(
12524 OrderPendingCancelSpec::builder()
12525 .trader_id(order.trader_id())
12526 .strategy_id(order.strategy_id())
12527 .instrument_id(order.instrument_id())
12528 .client_order_id(client_order_id)
12529 .account_id(account_id)
12530 .build(),
12531 )
12532 } else {
12533 OrderEventAny::PendingUpdate(
12534 OrderPendingUpdateSpec::builder()
12535 .trader_id(order.trader_id())
12536 .strategy_id(order.strategy_id())
12537 .instrument_id(order.instrument_id())
12538 .client_order_id(client_order_id)
12539 .account_id(account_id)
12540 .build(),
12541 )
12542 };
12543 node.process_exec_event(ExecutionEvent::Order(pending));
12544 }
12545 let queries = Rc::new(RefCell::new(Vec::new()));
12546 let client = OrderReportClient {
12547 client_id,
12548 report: OrderStatusReport::new(
12549 account_id,
12550 order.instrument_id(),
12551 Some(client_order_id),
12552 VenueOrderId::from("V-VENUE-EVIDENCE"),
12553 Some(order.order_side()),
12554 order.order_type(),
12555 order.time_in_force(),
12556 OrderStatus::Submitted,
12557 order.quantity(),
12558 Quantity::zero(3),
12559 UnixNanos::default(),
12560 UnixNanos::default(),
12561 UnixNanos::default(),
12562 None,
12563 )
12564 .with_price(order.price().unwrap()),
12565 targeted,
12566 queries: queries.clone(),
12567 };
12568 advance_clock(Duration::from_millis(40)).await;
12569 let events = node.exec_manager.check_open_orders(&[&client]).await;
12570 assert!(events.is_empty());
12571 assert_eq!(
12572 *queries.borrow(),
12573 if targeted {
12574 vec!["bulk", "targeted"]
12575 } else {
12576 vec!["bulk"]
12577 }
12578 );
12579 node.process_reconciliation_events(&events);
12580 advance_clock(Duration::from_millis(61)).await;
12581 let first = node.exec_manager.check_inflight_orders();
12582 let recover_command =
12583 cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12584 assert_eq!(first.queries.len(), usize::from(recover_command));
12585
12586 if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12587 assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12588 if event.client_order_id == client_order_id));
12589 } else {
12590 assert!(first.events.is_empty());
12591 }
12592 node.process_reconciliation_events(&first.events);
12593 advance_clock(Duration::from_millis(101)).await;
12594 let exhausted = node.exec_manager.check_inflight_orders();
12595 assert!(exhausted.queries.is_empty());
12596
12597 if recover_command {
12598 assert!(
12599 matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12600 if event.client_order_id == client_order_id)
12601 );
12602 } else {
12603 assert!(exhausted.events.is_empty());
12604 }
12605 assert!(
12606 node.exec_manager
12607 .take_submission_recovery_exhaustions()
12608 .is_empty()
12609 );
12610 }
12611
12612 #[rstest]
12613 #[case::local_update_before_pending("updated")]
12614 #[case::modify_rejected_after_pending("modify_rejected")]
12615 #[case::cancel_rejected_after_pending("cancel_rejected")]
12616 #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12617 #[cfg_attr(
12618 not(all(feature = "simulation", madsim)),
12619 tokio::test(start_paused = true)
12620 )]
12621 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12622 async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12623 #[case] completion: &str,
12624 #[values(
12625 SubmissionRecoveryPolicy::ResolveLocally,
12626 SubmissionRecoveryPolicy::RetainUnresolved
12627 )]
12628 policy: SubmissionRecoveryPolicy,
12629 ) {
12630 use nautilus_model::events::order::spec::{
12631 OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12632 };
12633
12634 let (mut node, order, instrument) = venue_evidence_node(policy);
12635 let client_order_id = order.client_order_id();
12636 let client_id = ClientId::from("TEST-EVIDENCE");
12637 let account_id = AccountId::from("TEST-001");
12638 let cancel = completion == "cancel_rejected";
12639 let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12640 node.kernel
12641 .exec_engine
12642 .borrow_mut()
12643 .register_client(Box::new(StubExecutionClient::new(
12644 client_id,
12645 account_id,
12646 instrument.id().venue,
12647 OmsType::Netting,
12648 None,
12649 )))
12650 .unwrap();
12651 node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12652 MessagingSwitchboard::exec_engine_execute(),
12653 venue_evidence_command(&order, cancel, false),
12654 )));
12655 advance_clock(Duration::from_millis(10)).await;
12656
12657 if completion != "updated" {
12658 let pending = if cancel {
12659 OrderEventAny::PendingCancel(
12660 OrderPendingCancelSpec::builder()
12661 .trader_id(order.trader_id())
12662 .strategy_id(order.strategy_id())
12663 .instrument_id(order.instrument_id())
12664 .client_order_id(client_order_id)
12665 .account_id(account_id)
12666 .build(),
12667 )
12668 } else {
12669 OrderEventAny::PendingUpdate(
12670 OrderPendingUpdateSpec::builder()
12671 .trader_id(order.trader_id())
12672 .strategy_id(order.strategy_id())
12673 .instrument_id(order.instrument_id())
12674 .client_order_id(client_order_id)
12675 .account_id(account_id)
12676 .build(),
12677 )
12678 };
12679 node.process_exec_event(ExecutionEvent::Order(pending));
12680 }
12681
12682 if later_cancel_pending {
12683 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12685 OrderPendingCancelSpec::builder()
12686 .trader_id(order.trader_id())
12687 .strategy_id(order.strategy_id())
12688 .instrument_id(order.instrument_id())
12689 .client_order_id(client_order_id)
12690 .account_id(account_id)
12691 .build(),
12692 )));
12693 }
12694 advance_clock(Duration::from_millis(10)).await;
12695 let completed = match completion {
12696 "updated" => OrderEventAny::Updated(
12697 OrderUpdatedSpec::builder()
12698 .trader_id(order.trader_id())
12699 .strategy_id(order.strategy_id())
12700 .instrument_id(order.instrument_id())
12701 .client_order_id(client_order_id)
12702 .account_id(account_id)
12703 .quantity(order.quantity())
12704 .price(Price::from("101.00"))
12705 .build(),
12706 ),
12707 "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12708 OrderEventAny::ModifyRejected(
12709 OrderModifyRejectedSpec::builder()
12710 .trader_id(order.trader_id())
12711 .strategy_id(order.strategy_id())
12712 .instrument_id(order.instrument_id())
12713 .client_order_id(client_order_id)
12714 .account_id(account_id)
12715 .build(),
12716 )
12717 }
12718 "cancel_rejected" => OrderEventAny::CancelRejected(
12719 OrderCancelRejectedSpec::builder()
12720 .trader_id(order.trader_id())
12721 .strategy_id(order.strategy_id())
12722 .instrument_id(order.instrument_id())
12723 .client_order_id(client_order_id)
12724 .account_id(account_id)
12725 .build(),
12726 ),
12727 _ => unreachable!(),
12728 };
12729 node.process_exec_event(ExecutionEvent::Order(completed));
12730 assert_eq!(
12731 node.kernel
12732 .cache
12733 .borrow()
12734 .order(&client_order_id)
12735 .unwrap()
12736 .status(),
12737 if later_cancel_pending {
12738 OrderStatus::PendingCancel
12739 } else {
12740 OrderStatus::Submitted
12741 }
12742 );
12743
12744 advance_clock(Duration::from_millis(10)).await;
12746
12747 if !later_cancel_pending {
12748 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12749 OrderPendingUpdateSpec::builder()
12750 .trader_id(order.trader_id())
12751 .strategy_id(order.strategy_id())
12752 .instrument_id(order.instrument_id())
12753 .client_order_id(client_order_id)
12754 .account_id(account_id)
12755 .build(),
12756 )));
12757 }
12758 advance_clock(Duration::from_millis(10)).await;
12759 let cached = node
12760 .kernel
12761 .cache
12762 .borrow()
12763 .order_owned(&client_order_id)
12764 .unwrap();
12765 let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12766 .last_qty(Quantity::from("0.400"))
12767 .last_px(order.price().unwrap())
12768 .commission(Money::from("0 USDT"))
12769 .without_position_id()
12770 .build()
12771 else {
12772 unreachable!()
12773 };
12774 fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12775 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12776
12777 for _ in 0..3 {
12778 advance_clock(Duration::from_millis(101)).await;
12779 let result = node.exec_manager.check_inflight_orders();
12780 assert!(result.queries.is_empty());
12781 assert!(result.events.is_empty());
12782 assert!(
12783 node.exec_manager
12784 .take_submission_recovery_exhaustions()
12785 .is_empty()
12786 );
12787 }
12788 assert_eq!(
12789 node.kernel
12790 .cache
12791 .borrow()
12792 .order(&client_order_id)
12793 .unwrap()
12794 .status(),
12795 if later_cancel_pending {
12796 OrderStatus::PendingCancel
12797 } else {
12798 OrderStatus::PendingUpdate
12799 }
12800 );
12801 }
12802
12803 #[rstest]
12804 #[case::wrong_account("account")]
12805 #[case::wrong_instrument("instrument")]
12806 #[case::conflicting_report_venue("report_venue")]
12807 #[case::unapplied_venue_update("update_venue")]
12808 #[case::local_update("local_update")]
12809 #[cfg_attr(
12810 not(all(feature = "simulation", madsim)),
12811 tokio::test(start_paused = true)
12812 )]
12813 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12814 async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12815 let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12816 let client_order_id = order.client_order_id();
12817 let account_id = AccountId::from("TEST-001");
12818 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12819
12820 if invalid.ends_with("venue") {
12821 insert_accepted_limit_order_in_node(
12822 &node,
12823 account_id,
12824 ClientId::from("TEST-EVIDENCE"),
12825 order.instrument_id(),
12826 ClientOrderId::from("O-OTHER"),
12827 venue_order_id,
12828 );
12829 }
12830 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12831 OrderPendingCancelSpec::builder()
12832 .trader_id(order.trader_id())
12833 .strategy_id(order.strategy_id())
12834 .instrument_id(order.instrument_id())
12835 .client_order_id(client_order_id)
12836 .account_id(account_id)
12837 .build(),
12838 )));
12839 advance_clock(Duration::from_millis(101)).await;
12840 assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12841
12842 if invalid == "update_venue" || invalid == "local_update" {
12843 let mut updated = OrderUpdatedSpec::builder()
12844 .trader_id(order.trader_id())
12845 .strategy_id(order.strategy_id())
12846 .instrument_id(order.instrument_id())
12847 .client_order_id(client_order_id)
12848 .account_id(account_id)
12849 .quantity(order.quantity())
12850 .price(order.price().unwrap())
12851 .build();
12852
12853 if invalid == "update_venue" {
12854 updated.venue_order_id = Some(venue_order_id);
12855 }
12856 node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12857 } else {
12858 let report = OrderStatusReport::new(
12859 if invalid == "account" {
12860 AccountId::from("OTHER-001")
12861 } else {
12862 account_id
12863 },
12864 if invalid == "instrument" {
12865 InstrumentId::from("OTHER.BINANCE")
12866 } else {
12867 order.instrument_id()
12868 },
12869 Some(client_order_id),
12870 venue_order_id,
12871 Some(order.order_side()),
12872 order.order_type(),
12873 order.time_in_force(),
12874 OrderStatus::Submitted,
12875 order.quantity(),
12876 Quantity::zero(3),
12877 UnixNanos::default(),
12878 UnixNanos::default(),
12879 UnixNanos::default(),
12880 None,
12881 )
12882 .with_price(order.price().unwrap());
12883 node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12884 report,
12885 ))));
12886 }
12887 assert!(
12888 node.kernel
12889 .cache
12890 .borrow()
12891 .order(&client_order_id)
12892 .unwrap()
12893 .venue_order_id()
12894 .is_none()
12895 );
12896 advance_clock(Duration::from_millis(101)).await;
12897 let result = node.exec_manager.check_inflight_orders();
12898 assert!(result.events.is_empty());
12899 assert!(result.queries.is_empty());
12900 let cached = node
12901 .kernel
12902 .cache
12903 .borrow()
12904 .order_owned(&client_order_id)
12905 .unwrap();
12906 assert_eq!(cached.status(), OrderStatus::PendingCancel);
12907 let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12908 assert_eq!(diagnostics.len(), 1);
12909 assert_eq!(diagnostics[0].client_order_id, client_order_id);
12910 }
12911
12912 fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12913 use nautilus_common::messages::execution::{
12914 BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12915 };
12916
12917 let client_id = ClientId::from("TEST-EVIDENCE");
12918 let client_order_id = order.client_order_id();
12919 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12920
12921 if cancel {
12922 let cancel = CancelOrder::new(
12923 order.trader_id(),
12924 Some(client_id),
12925 order.strategy_id(),
12926 order.instrument_id(),
12927 client_order_id,
12928 Some(venue_order_id),
12929 UUID4::new(),
12930 UnixNanos::default(),
12931 None,
12932 None,
12933 );
12934
12935 if batch {
12936 TradingCommand::CancelOrders(BatchCancelOrders::new(
12937 order.trader_id(),
12938 Some(client_id),
12939 order.strategy_id(),
12940 order.instrument_id(),
12941 vec![cancel],
12942 UUID4::new(),
12943 UnixNanos::default(),
12944 None,
12945 None,
12946 ))
12947 } else {
12948 TradingCommand::CancelOrder(cancel)
12949 }
12950 } else {
12951 let modify = ModifyOrder::new(
12952 order.trader_id(),
12953 Some(client_id),
12954 order.strategy_id(),
12955 order.instrument_id(),
12956 client_order_id,
12957 Some(venue_order_id),
12958 None,
12959 Some(Price::from("101.00")),
12960 None,
12961 UUID4::new(),
12962 UnixNanos::default(),
12963 None,
12964 None,
12965 );
12966
12967 if batch {
12968 TradingCommand::ModifyOrders(BatchModifyOrders::new(
12969 order.trader_id(),
12970 Some(client_id),
12971 order.strategy_id(),
12972 order.instrument_id(),
12973 vec![modify],
12974 UUID4::new(),
12975 UnixNanos::default(),
12976 None,
12977 None,
12978 ))
12979 } else {
12980 TradingCommand::ModifyOrder(modify)
12981 }
12982 }
12983 }
12984
12985 fn venue_evidence_node(
12986 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12987 ) -> (LiveNode, OrderAny, InstrumentAny) {
12988 venue_evidence_node_with_order_type(policy, OrderType::Limit)
12989 }
12990
12991 fn venue_evidence_node_with_order_type(
12992 policy: crate::execution::submission::SubmissionRecoveryPolicy,
12993 order_type: OrderType,
12994 ) -> (LiveNode, OrderAny, InstrumentAny) {
12995 venue_evidence_node_with_config(
12996 crate::config::LiveExecutionEngineConfig {
12997 reconciliation: true,
12998 inflight_check_threshold_ms: 100,
12999 inflight_check_retries: 2,
13000 open_check_threshold_ms: 5_000,
13001 single_order_query_delay_ms: 0,
13002 submission_recovery_policy: policy,
13003 ..Default::default()
13004 },
13005 order_type,
13006 true,
13007 )
13008 }
13009
13010 fn venue_evidence_node_with_config(
13011 exec_engine: crate::config::LiveExecutionEngineConfig,
13012 order_type: OrderType,
13013 register_submission: bool,
13014 ) -> (LiveNode, OrderAny, InstrumentAny) {
13015 let config = LiveNodeConfig {
13016 exec_engine,
13017 ..Default::default()
13018 };
13019 let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
13020 let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
13021 let account_id = AccountId::from("TEST-001");
13022 let client_id = ClientId::from("TEST-EVIDENCE");
13023 let account = AccountAny::Margin(MarginAccount::new(
13024 AccountState::new(
13025 account_id,
13026 AccountType::Margin,
13027 vec![AccountBalance::new(
13028 Money::from("1000000 USDT"),
13029 Money::from("0 USDT"),
13030 Money::from("1000000 USDT"),
13031 )],
13032 Vec::new(),
13033 true,
13034 UUID4::new(),
13035 UnixNanos::default(),
13036 UnixNanos::default(),
13037 Some(Currency::USDT()),
13038 ),
13039 true,
13040 ));
13041 node.kernel.cache.borrow_mut().add_account(account).unwrap();
13042 node.kernel
13043 .cache
13044 .borrow_mut()
13045 .add_instrument(instrument.clone())
13046 .unwrap();
13047 let mut builder = OrderTestBuilder::new(order_type);
13048 builder
13049 .trader_id(node.trader_id())
13050 .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
13051 .instrument_id(instrument.id())
13052 .quantity(Quantity::from("1.000"))
13053 .price(Price::from("100.00"));
13054
13055 if order_type == OrderType::StopLimit {
13056 builder.trigger_price(Price::from("110.00"));
13057 }
13058 let order = builder.build();
13059 let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
13060 &order,
13061 node.trader_id(),
13062 Some(client_id),
13063 None,
13064 UUID4::new(),
13065 UnixNanos::default(),
13066 ));
13067
13068 if register_submission {
13069 node.observe_exec_command_before_dispatch(&command);
13070 }
13071 node.kernel
13072 .cache
13073 .borrow_mut()
13074 .add_order(order.clone(), None, Some(client_id), false)
13075 .unwrap();
13076 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
13077 &order, account_id,
13078 )));
13079 (node, order, instrument)
13080 }
13081
13082 #[rstest]
13083 #[case::partial("0.400", OrderStatus::PartiallyFilled)]
13084 #[case::full("1.000", OrderStatus::Filled)]
13085 #[tokio::test(start_paused = true)]
13086 async fn test_retained_submission_resolves_late_fill(
13087 #[case] quantity: &str,
13088 #[case] expected_status: OrderStatus,
13089 ) {
13090 let (mut node, order, instrument) =
13091 venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
13092 let client_order_id = order.client_order_id();
13093
13094 for _ in 0..2 {
13095 advance_clock(Duration::from_millis(101)).await;
13096 assert!(node.exec_manager.check_inflight_orders().events.is_empty());
13097 }
13098
13099 assert_eq!(
13100 node.exec_manager
13101 .take_submission_recovery_exhaustions()
13102 .len(),
13103 1
13104 );
13105 let fill = TestOrderEventStubs::filled(
13106 &order,
13107 &instrument,
13108 Some(TradeId::from("T-LATE-RETAINED")),
13109 None,
13110 Some(Price::from("100.00")),
13111 Some(Quantity::from(quantity)),
13112 Some(LiquiditySide::Taker),
13113 None,
13114 None,
13115 Some(AccountId::from("TEST-001")),
13116 );
13117 node.process_exec_event(ExecutionEvent::Order(fill));
13118 advance_clock(Duration::from_millis(101)).await;
13119 let result = node.exec_manager.check_inflight_orders();
13120 let cache = node.kernel.cache.borrow();
13121 let cached = cache.order_ref(&client_order_id).unwrap();
13122 assert_eq!(cached.status(), expected_status);
13123 assert_eq!(cached.filled_qty(), Quantity::from(quantity));
13124 assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
13125 assert!(result.events.is_empty());
13126 assert!(result.queries.is_empty());
13127 assert!(
13128 node.exec_manager
13129 .take_submission_recovery_exhaustions()
13130 .is_empty()
13131 );
13132 }
13133
13134 #[rstest]
13135 #[case::submitted_report("order")]
13136 #[case::submitted_report_with_fills("order_with_fills")]
13137 #[case::venue_update("updated")]
13138 #[cfg_attr(
13139 not(all(feature = "simulation", madsim)),
13140 tokio::test(start_paused = true)
13141 )]
13142 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13143 async fn test_venue_evidence_stops_submission_timeout(
13144 #[case] evidence: &str,
13145 #[values(
13146 OrderStatus::Submitted,
13147 OrderStatus::PendingCancel,
13148 OrderStatus::PendingUpdate
13149 )]
13150 status: OrderStatus,
13151 #[values(
13152 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13153 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13154 )]
13155 policy: crate::execution::submission::SubmissionRecoveryPolicy,
13156 ) {
13157 let (mut node, order, _) = venue_evidence_node(policy);
13158 let client_order_id = order.client_order_id();
13159 let account_id = AccountId::from("TEST-001");
13160 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13161 let pending = match status {
13162 OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
13163 OrderPendingCancelSpec::builder()
13164 .trader_id(order.trader_id())
13165 .strategy_id(order.strategy_id())
13166 .instrument_id(order.instrument_id())
13167 .client_order_id(client_order_id)
13168 .account_id(account_id)
13169 .build(),
13170 )),
13171 OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
13172 OrderPendingUpdateSpec::builder()
13173 .trader_id(order.trader_id())
13174 .strategy_id(order.strategy_id())
13175 .instrument_id(order.instrument_id())
13176 .client_order_id(client_order_id)
13177 .account_id(account_id)
13178 .build(),
13179 )),
13180 _ => None,
13181 };
13182
13183 if let Some(pending) = pending {
13184 node.process_exec_event(ExecutionEvent::Order(pending));
13185 }
13186 advance_clock(Duration::from_millis(101)).await;
13187 let first = node.exec_manager.check_inflight_orders();
13188 assert_eq!(first.queries.len(), 1);
13189 assert!(first.events.is_empty());
13190 advance_clock(Duration::from_millis(94)).await;
13191
13192 let event = if evidence == "updated" {
13193 ExecutionEvent::Order(OrderEventAny::Updated(
13194 OrderUpdatedSpec::builder()
13195 .trader_id(order.trader_id())
13196 .strategy_id(order.strategy_id())
13197 .instrument_id(order.instrument_id())
13198 .client_order_id(client_order_id)
13199 .account_id(account_id)
13200 .venue_order_id(venue_order_id)
13201 .quantity(order.quantity())
13202 .price(Price::from("101.00"))
13203 .build(),
13204 ))
13205 } else {
13206 let report = OrderStatusReport::new(
13207 account_id,
13208 order.instrument_id(),
13209 Some(client_order_id),
13210 venue_order_id,
13211 Some(order.order_side()),
13212 order.order_type(),
13213 order.time_in_force(),
13214 OrderStatus::Submitted,
13215 order.quantity(),
13216 Quantity::zero(3),
13217 UnixNanos::default(),
13218 UnixNanos::default(),
13219 UnixNanos::default(),
13220 None,
13221 )
13222 .with_price(Price::from("100.00"));
13223 ExecutionEvent::Report(if evidence == "order" {
13224 ExecutionReport::Order(Box::new(report))
13225 } else {
13226 ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
13227 })
13228 };
13229 node.process_exec_event(event);
13230
13231 let expected_status = if evidence == "updated" && status != OrderStatus::PendingCancel {
13232 OrderStatus::Submitted
13233 } else if status == OrderStatus::Submitted {
13234 OrderStatus::Accepted
13235 } else {
13236 status
13237 };
13238 assert_eq!(
13239 node.kernel
13240 .cache
13241 .borrow()
13242 .order(&client_order_id)
13243 .unwrap()
13244 .status(),
13245 expected_status,
13246 );
13247
13248 for _ in 0..3 {
13249 advance_clock(Duration::from_millis(101)).await;
13250 let result = node.exec_manager.check_inflight_orders();
13251 assert!(result.queries.is_empty());
13252 assert!(result.events.is_empty());
13253 }
13254 assert_eq!(
13255 node.exec_manager.recon_check_retry_count(&client_order_id),
13256 0
13257 );
13258 assert!(node.exec_manager.check_open_order_queries().is_empty());
13259 advance_clock(Duration::from_secs(5)).await;
13260 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13261 }
13262
13263 #[rstest]
13264 #[case::fill("fill", false)]
13265 #[case::terminal_order("order", true)]
13266 #[case::terminal_order_with_fills("order_with_fills", true)]
13267 #[case::working_order_for_closed_cache("order", false)]
13268 #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
13269 #[cfg_attr(
13270 not(all(feature = "simulation", madsim)),
13271 tokio::test(start_paused = true)
13272 )]
13273 #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13274 async fn test_report_cleanup_preserves_original_settling_predicate(
13275 #[case] report_kind: &str,
13276 #[case] terminal_report: bool,
13277 #[values(false, true)] explicit_client_id: bool,
13278 #[values(
13279 crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13280 crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13281 )]
13282 policy: crate::execution::submission::SubmissionRecoveryPolicy,
13283 ) {
13284 let (mut node, order, instrument) = venue_evidence_node(policy);
13285 let client_order_id = order.client_order_id();
13286 let account_id = AccountId::from("TEST-001");
13287 let client_id = ClientId::from("TEST-EVIDENCE");
13288 let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13289 node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
13290 &order,
13291 account_id,
13292 venue_order_id,
13293 )));
13294 let accepted = node
13295 .kernel
13296 .cache
13297 .borrow()
13298 .order_owned(&client_order_id)
13299 .unwrap();
13300 let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
13301 .last_qty(accepted.quantity())
13302 .last_px(Price::from("100.00"))
13303 .commission(Money::from("0 USDT"))
13304 .without_position_id()
13305 .build()
13306 else {
13307 unreachable!()
13308 };
13309 let mut fill_report = FillReport::new(
13310 account_id,
13311 instrument.id(),
13312 venue_order_id,
13313 fill.trade_id,
13314 fill.order_side,
13315 fill.last_qty,
13316 fill.last_px,
13317 fill.commission.unwrap(),
13318 fill.liquidity_side,
13319 Some(client_order_id),
13320 None,
13321 fill.ts_event,
13322 fill.ts_init,
13323 None,
13324 );
13325
13326 if !explicit_client_id {
13327 fill_report.client_order_id = None;
13328 }
13329 let event = if report_kind == "fill" {
13330 ExecutionReport::Fill(Box::new(fill_report))
13331 } else {
13332 if !terminal_report {
13333 node.kernel
13334 .cache
13335 .borrow_mut()
13336 .update_order(&OrderEventAny::Filled(fill))
13337 .unwrap();
13338 }
13339 let report = OrderStatusReport::new(
13340 account_id,
13341 instrument.id(),
13342 explicit_client_id.then_some(client_order_id),
13343 venue_order_id,
13344 Some(order.order_side()),
13345 order.order_type(),
13346 order.time_in_force(),
13347 if terminal_report {
13348 OrderStatus::Filled
13349 } else {
13350 OrderStatus::Accepted
13351 },
13352 order.quantity(),
13353 order.quantity(),
13354 UnixNanos::default(),
13355 UnixNanos::default(),
13356 UnixNanos::default(),
13357 None,
13358 )
13359 .with_price(Price::from("100.00"))
13360 .with_avg_px(dec!(100.00));
13361
13362 if report_kind == "order" {
13363 ExecutionReport::Order(Box::new(report))
13364 } else {
13365 ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
13366 }
13367 };
13368 node.process_exec_event(ExecutionEvent::Report(event));
13369 assert_eq!(
13370 node.kernel
13371 .cache
13372 .borrow()
13373 .order(&client_order_id)
13374 .unwrap()
13375 .status(),
13376 OrderStatus::Filled
13377 );
13378
13379 node.kernel
13381 .cache
13382 .borrow_mut()
13383 .add_order(order.clone(), None, Some(client_id), true)
13384 .unwrap();
13385 node.kernel
13386 .cache
13387 .borrow_mut()
13388 .update_order(&TestOrderEventStubs::submitted(&order, account_id))
13389 .unwrap();
13390 node.kernel
13391 .cache
13392 .borrow_mut()
13393 .update_order(&TestOrderEventStubs::accepted(
13394 &order,
13395 account_id,
13396 venue_order_id,
13397 ))
13398 .unwrap();
13399 let should_clear = terminal_report && explicit_client_id;
13400 assert_eq!(
13401 node.exec_manager.check_open_order_queries().len(),
13402 usize::from(should_clear)
13403 );
13404
13405 if !should_clear {
13406 advance_clock(Duration::from_secs(5)).await;
13407 assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13408 }
13409 }
13410
13411 #[rstest]
13412 #[case(stub_order_event, 1)]
13413 #[case(stub_submitted_batch_event, 2)]
13414 #[case(stub_accepted_batch_event, 2)]
13415 #[case(stub_canceled_batch_event, 2)]
13416 fn test_pending_events_preserve_roots_when_splitting_batches(
13417 #[values(false, true)] rooted: bool,
13418 #[case] make_event: fn() -> ExecutionEvent,
13419 #[case] orders: usize,
13420 ) {
13421 let runner = AsyncRunner::new();
13422 runner.bind_senders();
13423 let mut channels = runner.take_channels();
13424
13425 let send = move || {
13426 get_data_event_sender().send(stub_data_event()).unwrap();
13427 get_exec_event_sender().send(stub_exec_event()).unwrap();
13428 get_exec_event_sender().send(make_event()).unwrap();
13429 };
13430
13431 if rooted {
13432 msgbus::register_trading_command_endpoint(
13433 MessagingSwitchboard::risk_engine_execute(),
13434 TypedIntoHandler::from(move |_| send()),
13435 );
13436 TradingCommandSender::execute(
13437 &SyncTradingCommandSender,
13438 TradingCommandMessage::new(
13439 MessagingSwitchboard::risk_engine_execute(),
13440 TradingCommand::QueryAccount(QueryAccount::new(
13441 "TRADER-001".into(),
13442 None,
13443 "SIM-001".into(),
13444 UUID4::new(),
13445 37.into(),
13446 None,
13447 None,
13448 )),
13449 ),
13450 );
13451 nautilus_common::runner::drain_trading_cmd_queue();
13452 } else {
13453 send();
13454 }
13455
13456 let mut pending = PendingEvents::default();
13457 pending
13458 .data_evts
13459 .push(channels.data_evt_rx.try_recv().unwrap());
13460 while let Ok(event) = channels.exec_evt_rx.try_recv() {
13461 pending.push_exec_event(event);
13462 }
13463
13464 assert_eq!(pending.data_evts.len(), 1);
13465 assert_eq!(pending.exec_reports.len(), 1);
13466 assert_eq!(pending.order_evts.len(), orders);
13467 assert_eq!(pending.data_evts[0].is_rooted(), rooted);
13468 assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
13469 assert!(
13470 pending
13471 .order_evts
13472 .iter()
13473 .all(|event| event.is_rooted() == rooted)
13474 );
13475
13476 let results = std::thread::spawn(move || {
13477 let mut results = Vec::new();
13478
13479 for event in pending.data_evts {
13480 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13481 || event.dispatch(drop),
13482 )));
13483 }
13484
13485 for event in pending.exec_reports {
13486 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13487 || event.dispatch(drop),
13488 )));
13489 }
13490
13491 for event in pending.order_evts {
13492 results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13493 || event.dispatch(drop),
13494 )));
13495 }
13496
13497 results
13498 })
13499 .join()
13500 .unwrap();
13501
13502 assert_eq!(results.len(), orders + 2);
13503
13504 for result in results {
13505 if rooted {
13506 let error = result.unwrap_err();
13507 assert!(
13508 error
13509 .downcast_ref::<String>()
13510 .unwrap()
13511 .contains("command context dispatched outside its owner thread")
13512 );
13513 } else {
13514 result.unwrap();
13515 }
13516 }
13517 }
13518
13519 fn stub_accepted_batch_event() -> ExecutionEvent {
13520 ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
13521 OrderAcceptedSpec::builder()
13522 .client_order_id(ClientOrderId::from("O-017"))
13523 .build(),
13524 OrderAcceptedSpec::builder()
13525 .client_order_id(ClientOrderId::from("O-023"))
13526 .build(),
13527 ]))
13528 }
13529
13530 #[derive(Debug)]
13531 struct CapturedEgressMessage {
13532 topic: String,
13533 payload: Bytes,
13534 }
13535
13536 type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13537 type SharedClosed = Rc<Cell<bool>>;
13538
13539 #[derive(Debug)]
13540 struct CapturingExternalIngress {
13541 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13542 closed: SharedClosed,
13543 }
13544
13545 impl CapturingExternalIngress {
13546 fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13547 Self {
13548 rx: Some(rx),
13549 closed,
13550 }
13551 }
13552 }
13553
13554 impl MessageBusExternalIngress for CapturingExternalIngress {
13555 fn is_closed(&self) -> bool {
13556 self.closed.get()
13557 }
13558
13559 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13560 self.rx
13561 .take()
13562 .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13563 }
13564
13565 fn close(&mut self) {
13566 self.closed.set(true);
13567 }
13568 }
13569
13570 #[derive(Debug)]
13571 struct FailingExternalIngress {
13572 closed: SharedClosed,
13573 }
13574
13575 impl FailingExternalIngress {
13576 fn new(closed: SharedClosed) -> Self {
13577 Self { closed }
13578 }
13579 }
13580
13581 impl MessageBusExternalIngress for FailingExternalIngress {
13582 fn is_closed(&self) -> bool {
13583 self.closed.get()
13584 }
13585
13586 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13587 anyhow::bail!("external ingress receiver unavailable")
13588 }
13589
13590 fn close(&mut self) {
13591 self.closed.set(true);
13592 }
13593 }
13594
13595 struct CapturingExternalEgress {
13596 publications: CapturedEgressMessages,
13597 closed: SharedClosed,
13598 }
13599
13600 impl CapturingExternalEgress {
13601 fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13602 let publications = Rc::new(RefCell::new(Vec::new()));
13603 let closed = Rc::new(Cell::new(false));
13604 (
13605 Self {
13606 publications: publications.clone(),
13607 closed: closed.clone(),
13608 },
13609 publications,
13610 closed,
13611 )
13612 }
13613 }
13614
13615 impl MessageBusExternalEgress for CapturingExternalEgress {
13616 fn is_closed(&self) -> bool {
13617 self.closed.get()
13618 }
13619
13620 fn publish(&self, message: BusMessage) {
13621 self.publications.borrow_mut().push(CapturedEgressMessage {
13622 topic: message.topic.to_string(),
13623 payload: message.payload,
13624 });
13625 }
13626
13627 fn close(&mut self) {
13628 self.closed.set(true);
13629 }
13630 }
13631
13632 struct CapturingBackingFactory {
13633 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13634 closed: Arc<AtomicBool>,
13635 rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13636 }
13637
13638 impl CapturingBackingFactory {
13639 fn new(
13640 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13641 closed: Arc<AtomicBool>,
13642 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13643 ) -> Self {
13644 Self {
13645 publications,
13646 closed,
13647 rx: Mutex::new(rx),
13648 }
13649 }
13650 }
13651
13652 impl Debug for CapturingBackingFactory {
13653 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13654 f.debug_struct(stringify!(CapturingBackingFactory))
13655 .finish_non_exhaustive()
13656 }
13657 }
13658
13659 impl MessageBusBackingFactory for CapturingBackingFactory {
13660 fn create(
13661 &self,
13662 _trader_id: TraderId,
13663 _instance_id: UUID4,
13664 _config: MessageBusConfig,
13665 ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13666 let rx = self.rx.lock().take();
13667 Ok(Box::new(CapturingBacking {
13668 publications: self.publications.clone(),
13669 closed: self.closed.clone(),
13670 rx,
13671 }))
13672 }
13673 }
13674
13675 struct CapturingBacking {
13676 publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13677 closed: Arc<AtomicBool>,
13678 rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13679 }
13680
13681 impl MessageBusBacking for CapturingBacking {
13682 fn is_closed(&self) -> bool {
13683 self.closed.load(Ordering::Relaxed)
13684 }
13685
13686 fn publish(&self, message: BusMessage) {
13687 self.publications.lock().push(CapturedEgressMessage {
13688 topic: message.topic.to_string(),
13689 payload: message.payload,
13690 });
13691 }
13692
13693 fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13694 self.rx
13695 .take()
13696 .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13697 }
13698
13699 fn close(&mut self) {
13700 self.closed.store(true, Ordering::Relaxed);
13701 }
13702 }
13703}