Skip to main content

nautilus_live/node/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Live trading node built on a single-threaded tokio event loop.
17//!
18//! The node owns system lifecycle and the event loop. Its reconciliation module schedules checks,
19//! manages report futures and deadlines, and dispatches results. The execution manager owns
20//! reconciliation state, discrepancy decisions, and individual reconciliation operations.
21//!
22//! `LiveNode::run()` drives the system through a `tokio::select!` loop that
23//! multiplexes data events, execution events, trading commands, timers, and
24//! periodic maintenance tasks (reconciliation, purge, prune, audit).
25//!
26//! # Threading model
27//!
28//! The core types (`ExecutionManager`, `ExecutionEngine`, `Cache`) use
29//! `Rc<RefCell<..>>` and are `!Send`. All access happens on the same thread.
30//! Pending report futures can retain client borrows while other select branches run.
31//! The client facade defers instrument updates until those borrows are released;
32//! completion and cancellation paths flush the deferred updates. Single-threaded
33//! execution does not by itself prevent conflicting borrows or reentrant callbacks.
34//!
35//! # Startup sequencing
36//!
37//! Startup connects clients in two phases so that instruments are in the
38//! cache before execution clients read them:
39//!
40//! 1. Connect data clients (instruments arrive as buffered `DataEvent`s).
41//! 2. Flush all pending data events and commands into the cache via
42//!    `flush_pending_data`, which loops `try_recv` on the channel receivers
43//!    until no items remain.
44//! 3. Connect execution clients (`load_instruments_from_cache` now finds
45//!    populated instruments).
46//! 4. Drain remaining events, then run reconciliation.
47//!
48//! Both `run()` (integrated event loop) and `start()` (manual lifecycle)
49//! follow this sequence.
50//!
51//! # Reconciliation
52//!
53//! Continuous inflight, open-order, and position checks run on independent intervals. The
54//! shared maintenance timer in the select loop dispatches reconciliation at
55//! the minimum enabled interval. Each dispatch the handler checks which
56//! sub-checks are due based on elapsed nanoseconds and schedules their work.
57//! Continuous checks do not await venue HTTP in the select loop: open-order
58//! and position checks poll bulk venue report futures from the loop.
59//!
60//! # Maintenance dispatcher
61//!
62//! Six periodic tasks share a single coarse `maintenance_timer`:
63//!
64//! - reconciliation (inflight, open, position sub-checks)
65//! - purge closed orders
66//! - purge closed positions
67//! - purge account events
68//! - own-books audit
69//! - recent-fills cache prune
70//!
71//! The runner wakes one timer per loop iteration regardless of how many
72//! maintenance tasks are configured. Each task tracks its own
73//! `next_fire: Instant` and the dispatcher fires the bodies whose deadline
74//! has passed, rescheduling `next = now + interval` (equivalent to
75//! `MissedTickBehavior::Delay`). Disabled tasks anchor on a far-future
76//! `next` that never trips.
77//!
78//! The 100ms timer cadence is the effective floor for any maintenance
79//! interval. Configured intervals below 100ms (the config types allow
80//! `inflight_check_interval_ms` and `own_books_audit_interval_secs` smaller)
81//! become eligible on the next maintenance tick. Event processing and runtime
82//! scheduling can delay dispatch further; the timer does not guarantee a maximum delay.
83
84use std::{
85    any::Any,
86    cell::RefCell,
87    collections::VecDeque,
88    fmt::Debug,
89    rc::{Rc, Weak},
90    time::Duration,
91};
92
93use anyhow::Context;
94use nautilus_common::{
95    actor::{self, Actor, DataActor, DataActorNative},
96    cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
97    clients::ExecutionClient,
98    component::Component,
99    enums::{Environment, LogColor},
100    live::{dispatch::DispatchMessage, dst},
101    log_info,
102    messages::{
103        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
104        data::DataCommand,
105        execution::TradingCommand,
106        system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
107    },
108    msgbus::{self, BusMessage, MessagingSwitchboard},
109    runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
110};
111use nautilus_core::{
112    UUID4,
113    datetime::{mins_to_secs, secs_to_nanos_unchecked},
114};
115#[cfg(test)]
116use nautilus_model::reports::OrderStatusReport;
117use nautilus_model::{
118    events::OrderEventAny,
119    identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
120    orders::Order,
121};
122use nautilus_network::mode::ReconnectRequestOutcome;
123#[cfg(feature = "python")]
124use nautilus_system::trader::Trader;
125use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
126use nautilus_trading::{
127    ExecutionAlgorithm, ExecutionAlgorithmNative,
128    strategy::{Strategy, StrategyNative},
129};
130use tabled::{builder::Builder, settings::Style};
131
132use crate::{
133    dispatch::drain_callbacks,
134    execution::{
135        client::LiveExecutionClient,
136        manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
137    },
138    runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
139    socket::{SocketReconnectLookup, SocketReconnectRegistry},
140};
141
142pub mod builder;
143pub mod config;
144
145#[cfg(feature = "plugin")]
146pub mod plugin;
147
148mod metrics;
149mod queue;
150mod reconciliation;
151mod state;
152
153use builder::ExternalMessageBusIngress;
154pub use builder::LiveNodeBuilder;
155use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
156pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
157use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
158use queue::{QueueMonitor, QueueStateTransition};
159use reconciliation::{
160    OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
161    PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
162    ReportTaskOutcome, TargetedOrderReportTask,
163};
164use state::{EngineConnectionStatus, RunningTransition};
165pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
166
167/// Dispatches the run loop performs before yielding to the executor.
168///
169/// A saturated channel keeps every select branch ready, so the loop would otherwise never return
170/// `Pending`. Under a host event loop that starves the adapter I/O tasks feeding those channels,
171/// which shows up as lapsed heartbeats and reconnects rather than as backpressure.
172const DISPATCHES_PER_YIELD: usize = 64;
173
174thread_local! {
175    static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
176}
177
178type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
179
180struct StreamProcessor(Box<StreamProcessorCallback>);
181
182impl Debug for StreamProcessor {
183    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
184        f.debug_struct(stringify!(StreamProcessor)).finish()
185    }
186}
187
188/// High-level abstraction for a live Nautilus system node.
189///
190/// Provides a simplified interface for running live systems
191/// with automatic client management and lifecycle handling.
192///
193/// Only one live node may exist on a thread at a time. Drop the node before building another,
194/// including after disposal, because retained nodes can still access thread-local messaging.
195/// Concurrent nodes in one process remain unsupported, even on separate threads.
196#[derive(Debug)]
197pub struct LiveNode {
198    kernel: NautilusKernel,
199    runner: Option<AsyncRunner>,
200    config: LiveNodeConfig,
201    handle: LiveNodeHandle,
202    exec_manager: ExecutionManager,
203    exec_clients: Vec<LiveExecutionClient>,
204    socket_registry: SocketReconnectRegistry,
205    cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
206    external_msgbus: Option<ExternalMessageBusIngress>,
207    stream_processors: Vec<StreamProcessor>,
208    shutdown_deadline: Option<dst::time::Instant>,
209    #[cfg(feature = "plugin")]
210    plugins: plugin::NodePlugins,
211    _thread_owner: Rc<()>,
212}
213
214impl LiveNode {
215    /// Creates a new `LiveNode` from builder components.
216    ///
217    /// This is an internal constructor used by `LiveNodeBuilder`.
218    #[must_use]
219    #[allow(
220        clippy::too_many_arguments,
221        reason = "builder components have distinct lifecycle roles"
222    )]
223    pub(crate) fn new_from_builder(
224        kernel: NautilusKernel,
225        runner: AsyncRunner,
226        config: LiveNodeConfig,
227        handle: LiveNodeHandle,
228        mut exec_manager: ExecutionManager,
229        exec_clients: Vec<LiveExecutionClient>,
230        socket_registry: SocketReconnectRegistry,
231        cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
232        external_msgbus: Option<ExternalMessageBusIngress>,
233        thread_owner: Rc<()>,
234    ) -> Self {
235        for client in &exec_clients {
236            exec_manager.register_submission_retention(client);
237        }
238
239        Self {
240            kernel,
241            runner: Some(runner),
242            config,
243            handle,
244            exec_manager,
245            exec_clients,
246            socket_registry,
247            cache_database_factory,
248            external_msgbus,
249            stream_processors: Vec::new(),
250            shutdown_deadline: None,
251            #[cfg(feature = "plugin")]
252            plugins: plugin::NodePlugins,
253            _thread_owner: thread_owner,
254        }
255    }
256
257    /// Creates a new [`LiveNodeBuilder`] for fluent configuration.
258    ///
259    /// # Errors
260    ///
261    /// Returns an error if the environment is invalid for live trading.
262    pub fn builder(
263        trader_id: TraderId,
264        environment: Environment,
265    ) -> anyhow::Result<LiveNodeBuilder> {
266        LiveNodeBuilder::new(trader_id, environment)
267    }
268
269    /// Creates a new [`LiveNode`] directly from a kernel name and optional configuration.
270    ///
271    /// This is a convenience method for creating a live node with a pre-configured
272    /// kernel configuration, bypassing the builder pattern. If no config is provided,
273    /// a default configuration will be used.
274    ///
275    /// # Errors
276    ///
277    /// Returns an error if kernel construction fails or another live node exists or is being
278    /// built on this thread.
279    pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
280        let thread_owner = Self::acquire_thread()?;
281        let config = config.unwrap_or_default();
282        validate_live_environment(config.environment())?;
283
284        config.validate_runtime_support()?;
285
286        if config.event_store.is_some() {
287            anyhow::bail!(
288                "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
289                 use LiveNodeBuilder::with_event_store(...) instead"
290            );
291        }
292
293        let handle = LiveNodeHandle::new();
294        let runner = AsyncRunner::new();
295        runner.bind_senders_for_node(handle.clone());
296
297        let kernel = NautilusKernel::new(name, config.clone())?;
298        #[cfg(feature = "python")]
299        if let Some(controller) = config.controller.as_ref() {
300            Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
301        }
302
303        #[cfg(not(feature = "python"))]
304        if let Some(controller) = config.controller.as_ref() {
305            anyhow::bail!(
306                "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
307                controller.controller_path
308            );
309        }
310
311        let exec_manager_config =
312            ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
313
314        let exec_manager = ExecutionManager::new(
315            kernel.clock.clone(),
316            kernel.cache.clone(),
317            exec_manager_config,
318        )?;
319
320        let node = Self {
321            kernel,
322            runner: Some(runner),
323            config,
324            handle,
325            exec_manager,
326            exec_clients: Vec::new(),
327            socket_registry: SocketReconnectRegistry::default(),
328            cache_database_factory: None,
329            external_msgbus: None,
330            stream_processors: Vec::new(),
331            shutdown_deadline: None,
332            #[cfg(feature = "plugin")]
333            plugins: plugin::NodePlugins,
334            _thread_owner: thread_owner,
335        };
336
337        node.load_configured_plugins()?;
338
339        log::info!("LiveNode built successfully with kernel config");
340
341        Ok(node)
342    }
343
344    fn acquire_thread() -> anyhow::Result<Rc<()>> {
345        NODE_THREAD_OWNER.with(|slot| {
346            let mut owner = slot.borrow_mut();
347            anyhow::ensure!(
348                owner.upgrade().is_none(),
349                "A LiveNode already exists or is being built on this thread; drop it before building another"
350            );
351            let token = Rc::new(());
352            *owner = Rc::downgrade(&token);
353            Ok(token)
354        })
355    }
356
357    /// Loads and registers plug-ins declared on the node config.
358    ///
359    /// # Errors
360    ///
361    /// Returns an error if plug-ins are configured without host-side support.
362    pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
363        if self.config.plugins.is_empty() {
364            return Ok(());
365        }
366
367        anyhow::bail!(
368            "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
369        )
370    }
371
372    /// Loads and registers one plug-in instance.
373    ///
374    /// # Errors
375    ///
376    /// Returns an error if called. Dynamic plug-in hosting lives in the host-side integration.
377    #[expect(
378        clippy::needless_pass_by_value,
379        reason = "signature mirrors the host-enabled API"
380    )]
381    pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
382        #[cfg(feature = "plugin")]
383        config.validate_runtime_support(self.config.plugins.len())?;
384        #[cfg(not(feature = "plugin"))]
385        let _ = config;
386
387        anyhow::bail!(
388            "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
389        )
390    }
391
392    /// Returns a thread-safe handle to control this node.
393    #[must_use]
394    pub fn handle(&self) -> LiveNodeHandle {
395        self.handle.clone()
396    }
397
398    /// Adds a callback for supported typed external messages.
399    ///
400    /// While the node's run loop services external ingress, processors run in registration order
401    /// before normal inbound streaming filters for supported typed JSON or MessagePack payloads.
402    /// Unsupported encodings do not reach processors. Rust callbacks receive the decoded
403    /// concrete value as [`Any`], allowing them to downcast to the concrete payload type.
404    ///
405    /// Python callbacks receive the serialized payload shape with an added `payload_type` field.
406    /// Native data replies use the shared response representation keyed by response kind.
407    /// Callback exceptions are logged, stop remaining processors, and skip internal republishing.
408    /// The Python wrapper returns an error if the node is no longer available for mutation.
409    ///
410    /// External egress is suppressed while processors run, so synchronous publications remain
411    /// local. During data-client connection, streaming-registered `DataResponse` messages with a
412    /// pending correlation handler bypass processors and can overtake messages buffered until
413    /// normal dispatch starts.
414    pub fn add_stream_processor<F>(&mut self, callback: F)
415    where
416        F: Fn(&dyn Any) + 'static,
417    {
418        self.add_stream_processor_with_mapping(move |message, _| {
419            callback(message);
420            Ok(())
421        });
422    }
423
424    pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
425    where
426        F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
427    {
428        self.stream_processors
429            .push(StreamProcessor(Box::new(callback)));
430    }
431
432    /// Starts the live node without entering a select loop.
433    ///
434    /// Connects clients, runs reconciliation, and starts the trader, but does
435    /// not consume the runner or drive channel receivers, so channel traffic arriving after
436    /// startup is never serviced. This is a building block for tests and embedding, not a
437    /// lifecycle: use [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) to run a node.
438    ///
439    /// # Errors
440    ///
441    /// Returns an error if startup fails.
442    pub async fn start(&mut self) -> anyhow::Result<()> {
443        if self.state().is_running() {
444            anyhow::bail!("Already running");
445        }
446
447        if self.external_msgbus.is_some() {
448            log::warn!(
449                "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
450            );
451        }
452
453        self.prepare_cache().await?;
454
455        if let Some(runner) = self.runner.as_ref() {
456            runner.bind_senders_for_node(self.handle.clone());
457        }
458
459        self.handle.set_starting();
460
461        self.kernel.reset_shutdown_flag();
462        self.kernel.start_async().await;
463
464        if self.kernel.is_event_store_replay() {
465            log::info!(
466                "Event-store replay loaded; skipping live client connection and reconciliation",
467            );
468
469            if !self.finish_startup_replay().await? {
470                return Ok(());
471            }
472
473            return Ok(());
474        }
475
476        if self.kernel.is_event_store_replay_configured() {
477            self.abort_startup("Event-store replay did not start")
478                .await?;
479            return Ok(());
480        }
481
482        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
483
484        // Connect data clients first and flush instrument events into cache
485        if let Err(e) = self.connect_data_phase(connection_deadline).await {
486            return self
487                .abort_startup_with_error("Data client connection timed out", e)
488                .await;
489        }
490
491        let (startup_system_events, startup_system_commands) =
492            if let Some(runner) = self.runner.as_mut() {
493                runner.flush_pending_data();
494                (
495                    runner.drain_pending_system_events(),
496                    runner.drain_pending_system_commands(),
497                )
498            } else {
499                (Vec::new(), Vec::new())
500            };
501
502        if let Err(e) = self.connect_exec_clients(connection_deadline).await {
503            return self
504                .abort_startup_with_error("Execution client connection timed out", e)
505                .await;
506        }
507
508        if let Some(reason) = self.startup_abort_reason() {
509            self.abort_startup(reason).await?;
510            return Ok(());
511        }
512
513        match self.await_engines_connected(connection_deadline).await {
514            EngineConnectionStatus::Connected => {}
515            EngineConnectionStatus::TimedOut => {
516                return self
517                    .abort_startup_with_error(
518                        "Engine readiness timed out",
519                        anyhow::anyhow!("readiness timeout while waiting for engine connections"),
520                    )
521                    .await;
522            }
523            EngineConnectionStatus::StopRequested => {
524                self.abort_startup("Stop signal received during startup")
525                    .await?;
526                return Ok(());
527            }
528            EngineConnectionStatus::ShutdownRequested => {
529                self.abort_startup("Shutdown signal received during startup")
530                    .await?;
531                return Ok(());
532            }
533        }
534
535        if let Err(e) = self.perform_startup_reconciliation().await {
536            if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
537                anyhow::bail!(
538                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
539                );
540            }
541
542            return Err(e);
543        }
544
545        if let Some(reason) = self.startup_abort_reason() {
546            self.abort_startup(reason).await?;
547            return Ok(());
548        }
549
550        if let Err(e) = self.kernel.start_trader() {
551            return self.abort_after_trader_start_failure(e).await;
552        }
553
554        #[cfg(feature = "plugin")]
555        if let Err(e) = self.plugins.start_controllers() {
556            return self.abort_after_trader_start_failure(e).await;
557        }
558
559        self.process_system_events(startup_system_events);
560        self.process_system_commands(startup_system_commands);
561
562        if !self.finish_startup_trader(None).await? {
563            return Ok(());
564        }
565
566        Ok(())
567    }
568
569    /// Stop the live node.
570    ///
571    /// This method stops the trader, waits for the configured grace period to allow
572    /// residual events to be processed, then finalizes the shutdown sequence.
573    ///
574    /// # Errors
575    ///
576    /// Returns an error if shutdown fails or retained submissions remain unresolved at the end
577    /// of `delay_post_stop`. Teardown still completes; subsequent events cannot clear this result.
578    pub async fn stop(&mut self) -> anyhow::Result<()> {
579        if !self.state().is_running() {
580            anyhow::bail!("Not running");
581        }
582
583        self.handle.set_shutting_down();
584
585        #[cfg(feature = "plugin")]
586        let controller_stop_result = self.plugins.stop_controllers();
587        #[cfg(not(feature = "plugin"))]
588        let controller_stop_result: anyhow::Result<()> = Ok(());
589
590        self.kernel.stop_trader();
591        let delay = self.kernel.delay_post_stop();
592        log::info!("Awaiting residual events ({delay:?})...");
593
594        let residual_events = self.process_runner_for(delay).await;
595        if residual_events > 0 {
596            log::debug!("Processed {residual_events} residual events during shutdown");
597        }
598
599        let stop_result = self.finalize_stop().await;
600        let drained_events = self.drain_runner_pending();
601        if drained_events > 0 {
602            log::info!("Drained {drained_events} remaining events during shutdown");
603        }
604
605        match (controller_stop_result, stop_result) {
606            (Ok(()), Ok(())) => Ok(()),
607            (Err(controller_err), Ok(())) => Err(controller_err),
608            (Ok(()), Err(stop_err)) => Err(stop_err),
609            (Err(controller_err), Err(stop_err)) => {
610                log::error!("Error stopping plug-in controllers: {controller_err}");
611                Err(stop_err)
612            }
613        }
614    }
615
616    /// Disposes the live node kernel and releases resources.
617    ///
618    /// Discards any retained runner messages and attempts callback cleanup. Logs latched callback
619    /// failures and cleanup rejection; externally retained work can prevent clearing.
620    pub fn dispose(&mut self) {
621        self.close_external_ingress();
622        self.handle.set_stopped();
623        self.kernel.dispose();
624        drop(self.runner.take());
625
626        if let Some(e) = actor::callback_failure() {
627            log::error!("Callback dispatch failed before disposal cleanup: {e}");
628        }
629
630        if let Err(e) = actor::clear_callbacks() {
631            log::error!("Failed to clear callback dispatch during disposal: {e}");
632        }
633    }
634
635    async fn process_runner_for(&mut self, duration: Duration) -> usize {
636        let Some(mut runner) = self.runner.take() else {
637            dst::time::sleep(duration).await;
638            return 0;
639        };
640
641        runner.bind_senders_for_node(self.handle.clone());
642        let deadline = dst::time::Instant::now() + duration;
643        let mut processed = 0;
644
645        loop {
646            tokio::select! {
647                biased;
648
649                () = dst::time::sleep_until(deadline) => break,
650                event = runner.recv() => {
651                    let Some(event) = event else {
652                        dst::time::sleep_until(deadline).await;
653                        break;
654                    };
655
656                    self.process_runner_event(event);
657                    processed += 1;
658                }
659            }
660        }
661
662        self.runner = Some(runner);
663        processed
664    }
665
666    fn drain_runner_pending(&mut self) -> usize {
667        let Some(mut runner) = self.runner.take() else {
668            return 0;
669        };
670
671        let processed = runner.poll_pending(|event| self.process_runner_event(event));
672        self.runner = Some(runner);
673        processed
674    }
675
676    fn process_runner_event(&mut self, event: PendingRunnerEvent) {
677        match event {
678            PendingRunnerEvent::TimeEvent(message) => {
679                let _ = AsyncRunner::handle_time_event(message);
680            }
681            PendingRunnerEvent::SystemEvent(event) => {
682                event.dispatch(|event| self.process_system_event(event));
683            }
684            PendingRunnerEvent::SystemCommand(command) => {
685                command.dispatch(|command| self.process_system_command(command));
686            }
687            PendingRunnerEvent::ExecEvent(event) => {
688                event.dispatch(|event| self.process_exec_event(event));
689            }
690            PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
691            PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
692            PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
693        }
694    }
695
696    fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
697        for event in events {
698            event.dispatch(|event| self.process_system_event(event));
699        }
700    }
701
702    fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
703        for command in commands {
704            command.dispatch(|command| self.process_system_command(command));
705        }
706    }
707
708    fn process_system_command(&self, command: SystemCommand) {
709        match command {
710            SystemCommand::ReconnectSocket(command) => {
711                self.process_socket_reconnect(command);
712            }
713        }
714    }
715
716    fn process_socket_reconnect(&self, command: ReconnectSocket) {
717        let outcome = if command.trader_id == self.config.trader_id {
718            Self::request_socket_reconnect(
719                self.socket_registry
720                    .get(command.client_id, command.endpoint),
721            )
722        } else {
723            SocketReconnectDispatchOutcome::InvalidTrader
724        };
725
726        if outcome == SocketReconnectDispatchOutcome::Accepted {
727            log::info!(
728                "Requested socket reconnect for client {} endpoint {}",
729                command.client_id,
730                command.endpoint
731            );
732        } else {
733            log::warn!(
734                "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
735                command.client_id,
736                command.endpoint
737            );
738        }
739    }
740
741    fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
742        match lookup {
743            SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
744                ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
745                ReconnectRequestOutcome::AlreadyReconnecting => {
746                    SocketReconnectDispatchOutcome::AlreadyReconnecting
747                }
748                ReconnectRequestOutcome::Disconnected => {
749                    SocketReconnectDispatchOutcome::Disconnected
750                }
751                ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
752                ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
753            },
754            SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
755            SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
756            SocketReconnectLookup::EndpointNotFound => {
757                SocketReconnectDispatchOutcome::UnknownEndpoint
758            }
759            SocketReconnectLookup::AmbiguousEndpoint => {
760                SocketReconnectDispatchOutcome::AmbiguousEndpoint
761            }
762        }
763    }
764
765    fn process_system_event(&self, event: SystemEvent) {
766        match event {
767            SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
768        }
769    }
770
771    fn publish_socket_state_change(&self, change: SocketStateChange) {
772        let timestamp = self.kernel.generate_timestamp_ns();
773
774        let event = SocketStateChanged::new(
775            self.config.trader_id,
776            change.client_id,
777            change.venue,
778            change.endpoint,
779            change.state,
780            UUID4::new(),
781            timestamp,
782            timestamp,
783        );
784
785        msgbus::publish_any(
786            MessagingSwitchboard::socket_state_changed_topic(
787                event.client_id,
788                event.endpoint.as_str(),
789            ),
790            event.as_any(),
791        );
792    }
793
794    /// Awaits engine clients to connect with timeout.
795    ///
796    /// Returns the final connection wait status.
797    async fn await_engines_connected(
798        &self,
799        deadline: dst::time::Instant,
800    ) -> EngineConnectionStatus {
801        log::info!(
802            "Awaiting engine connections ({:?} timeout)...",
803            self.config.timeout_connection
804        );
805
806        let interval = Duration::from_millis(100);
807
808        loop {
809            if self.handle.should_stop() {
810                log::warn!("Stop signal received, aborting connection wait");
811                return EngineConnectionStatus::StopRequested;
812            }
813
814            if self.kernel.is_shutdown_requested() {
815                log::warn!("Shutdown signal received, aborting connection wait");
816                return EngineConnectionStatus::ShutdownRequested;
817            }
818
819            if self.kernel.check_engines_connected() {
820                log::info!("All engine clients connected");
821                return EngineConnectionStatus::Connected;
822            }
823
824            let now = dst::time::Instant::now();
825            if now >= deadline {
826                break;
827            }
828
829            dst::time::sleep(interval.min(deadline - now)).await;
830        }
831
832        self.log_connection_status();
833        EngineConnectionStatus::TimedOut
834    }
835
836    /// Awaits engine clients to disconnect with timeout.
837    ///
838    /// Returns an error with client status on timeout.
839    async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
840        log::info!(
841            "Awaiting engine disconnections ({:?} timeout)...",
842            self.config.timeout_disconnection
843        );
844
845        let timeout = self.config.timeout_disconnection;
846        let interval = Duration::from_millis(100);
847
848        loop {
849            if self.kernel.check_engines_disconnected() {
850                log::info!("All engine clients disconnected");
851                return Ok(());
852            }
853
854            let now = dst::time::Instant::now();
855            if now >= deadline {
856                break;
857            }
858
859            dst::time::sleep(interval.min(deadline - now)).await;
860        }
861
862        log::error!(
863            "Timed out ({:?}) waiting for engines to disconnect\n\
864             DataEngine.check_disconnected() == {}\n\
865             ExecEngine.check_disconnected() == {}",
866            timeout,
867            self.kernel.data_engine().check_disconnected(),
868            self.kernel.exec_engine().borrow().check_disconnected(),
869        );
870        anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
871    }
872
873    fn log_connection_status(&self) {
874        let data_status = self.kernel.data_client_connection_status();
875        let exec_status = self.kernel.exec_client_connection_status();
876
877        let mut rows: Vec<ClientStatus> = Vec::new();
878
879        for (client_id, connected) in data_status {
880            rows.push(ClientStatus {
881                client: client_id.to_string(),
882                client_type: "Data",
883                connected,
884            });
885        }
886
887        for (client_id, connected) in exec_status {
888            rows.push(ClientStatus {
889                client: client_id.to_string(),
890                client_type: "Execution",
891                connected,
892            });
893        }
894
895        let table = render_client_statuses(rows);
896
897        log::warn!(
898            "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
899             DataEngine.check_connected() == {}\n\
900             ExecEngine.check_connected() == {}",
901            self.config.timeout_connection,
902            self.kernel.data_engine().check_connected(),
903            self.kernel.exec_engine().borrow().check_connected(),
904        );
905    }
906
907    /// Performs startup reconciliation to align internal state with venue state.
908    ///
909    /// This method queries each execution client for mass status (orders, fills, positions)
910    /// and reconciles any discrepancies with the local cache state.
911    ///
912    /// # Errors
913    ///
914    /// Returns an error if reconciliation fails or times out.
915    #[expect(clippy::await_holding_refcell_ref)] // Single-threaded runtime, intentional design
916    async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
917        if !self.config.exec_engine.reconciliation {
918            log::info!("Startup reconciliation disabled");
919            self.kernel
920                .portfolio
921                .borrow_mut()
922                .initialize_wallet_orders()?;
923            return Ok(());
924        }
925
926        log_info!(
927            "Starting execution state reconciliation...",
928            color = LogColor::Blue
929        );
930
931        let lookback_mins = self
932            .config
933            .exec_engine
934            .reconciliation_lookback_mins
935            .map(u64::from);
936
937        let timeout = self.config.timeout_reconciliation;
938        let start = dst::time::Instant::now();
939        let client_ids = self.kernel.exec_engine.borrow().client_ids();
940
941        for client_id in client_ids {
942            let elapsed = start.elapsed();
943            if elapsed >= timeout {
944                anyhow::bail!("Startup reconciliation timeout reached");
945            }
946
947            let remaining = timeout
948                .checked_sub(elapsed)
949                .expect("elapsed checked against reconciliation timeout");
950
951            log_info!(
952                "Requesting mass status from {}...",
953                client_id,
954                color = LogColor::Blue
955            );
956
957            let mass_status_result = dst::time::timeout(remaining, async {
958                self.kernel
959                    .exec_engine
960                    .borrow_mut()
961                    .generate_mass_status(&client_id, lookback_mins)
962                    .await
963            })
964            .await
965            .map_err(|_| {
966                anyhow::anyhow!(
967                    "Startup reconciliation timeout reached while requesting mass status from {client_id}"
968                )
969            })?;
970
971            match mass_status_result {
972                Ok(Some(mass_status)) => {
973                    log_info!(
974                        "Reconciling ExecutionMassStatus for {}",
975                        client_id,
976                        color = LogColor::Blue
977                    );
978
979                    let result = self
980                        .exec_manager
981                        .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
982
983                    anyhow::ensure!(
984                        self.kernel
985                            .exec_engine
986                            .borrow()
987                            .get_client(&client_id)
988                            .is_some(),
989                        "Execution client {client_id} disappeared during startup reconciliation",
990                    );
991
992                    anyhow::ensure!(
993                        result.unresolved_positions.is_empty(),
994                        "Unresolved positions during startup reconciliation for {client_id}: {}",
995                        result.unresolved_positions.join("; "),
996                    );
997
998                    if result.events.is_empty() {
999                        log_info!(
1000                            "Reconciliation for {} succeeded",
1001                            client_id,
1002                            color = LogColor::Blue
1003                        );
1004                    } else {
1005                        log::info!(
1006                            color = LogColor::Blue as u8;
1007                            "Reconciliation for {} processed {} events",
1008                            client_id,
1009                            result.events.len()
1010                        );
1011                    }
1012
1013                    // Register external orders with execution clients for tracking
1014                    if !result.external_orders.is_empty() {
1015                        let exec_engine = self.kernel.exec_engine.borrow();
1016
1017                        let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1018                            anyhow::anyhow!(
1019                                "Execution client {client_id} disappeared during startup reconciliation"
1020                            )
1021                        })?;
1022
1023                        for external in result.external_orders {
1024                            source_client.register_external_order(
1025                                external.client_order_id,
1026                                external.venue_order_id,
1027                                external.instrument_id,
1028                                external.strategy_id,
1029                                external.ts_init,
1030                            );
1031                        }
1032                    }
1033                }
1034                Ok(None) => {
1035                    log::warn!(
1036                        "No mass status available from {client_id} \
1037                         (likely adapter error when generating reports)"
1038                    );
1039                }
1040                Err(e) => {
1041                    return Err(e).context(format!("Failed to get mass status from {client_id}"));
1042                }
1043            }
1044        }
1045
1046        self.kernel.portfolio.borrow_mut().initialize_orders();
1047        self.kernel.portfolio.borrow_mut().initialize_positions();
1048        self.kernel
1049            .portfolio
1050            .borrow_mut()
1051            .initialize_wallet_orders()?;
1052
1053        let elapsed_secs = start.elapsed().as_secs_f64();
1054        log_info!(
1055            "Startup reconciliation completed in {:.2}s",
1056            elapsed_secs,
1057            color = LogColor::Blue
1058        );
1059
1060        Ok(())
1061    }
1062
1063    /// Run the live node with automatic shutdown handling.
1064    ///
1065    /// This method starts the node, runs indefinitely, and handles graceful shutdown
1066    /// on interrupt signals.
1067    ///
1068    /// # Thread Safety
1069    ///
1070    /// The event loop runs directly on the current thread (not spawned) because the
1071    /// msgbus uses thread-local storage. Endpoints registered by the kernel are only
1072    /// accessible from the same thread.
1073    ///
1074    /// # Shutdown Sequence
1075    ///
1076    /// 1. Signal received (SIGINT, SIGTERM, or handle stop).
1077    /// 2. Trader components stopped (triggers order cancellations, etc.).
1078    /// 3. Event loop continues processing residual events for the configured grace period.
1079    /// 4. Kernel finalized, clients disconnected, remaining events drained.
1080    ///
1081    /// # Errors
1082    ///
1083    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1084    /// with retained submissions unresolved at the end of `delay_post_stop`.
1085    pub async fn run(&mut self) -> anyhow::Result<()> {
1086        self.run_with_mode(NodeRunMode::Owned).await
1087    }
1088
1089    /// Run the live node under the given mode.
1090    ///
1091    /// [`NodeRunMode::Hosted`] leaves signal handling to the host application. Every other
1092    /// responsibility, including maintenance, reconciliation, external ingress, and the shutdown
1093    /// sequence, is identical across modes so that hosted and owned nodes cannot diverge.
1094    ///
1095    /// # Errors
1096    ///
1097    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1098    /// with retained submissions unresolved at the end of `delay_post_stop`.
1099    pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1100        if self.state().is_running() {
1101            anyhow::bail!("Already running");
1102        }
1103
1104        if self.runner.is_none() {
1105            anyhow::bail!("Runner already consumed - run() called twice");
1106        }
1107
1108        self.prepare_cache().await?;
1109
1110        let Some(runner) = self.runner.take() else {
1111            anyhow::bail!("Runner already consumed - run() called twice");
1112        };
1113
1114        runner.bind_senders_for_node(self.handle.clone());
1115
1116        let AsyncRunnerChannels {
1117            mut time_evt_rx,
1118            mut system_evt_rx,
1119            mut system_cmd_rx,
1120            mut exec_evt_rx,
1121            mut exec_cmd_rx,
1122            mut data_evt_rx,
1123            mut data_cmd_rx,
1124        } = runner.take_channels();
1125
1126        log::info!("Event loop starting");
1127
1128        self.handle.set_starting();
1129        self.kernel.reset_shutdown_flag();
1130        self.kernel.start_async().await;
1131
1132        if self.kernel.is_event_store_replay() {
1133            log::info!(
1134                "Event-store replay loaded; skipping live client connection and reconciliation",
1135            );
1136
1137            if !self.finish_startup_replay().await? {
1138                return Ok(());
1139            }
1140
1141            return Ok(());
1142        }
1143
1144        if self.kernel.is_event_store_replay_configured() {
1145            self.abort_startup("Event-store replay did not start")
1146                .await?;
1147            return Ok(());
1148        }
1149
1150        let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1151            Ok(rx) => rx,
1152            Err(e) => {
1153                let result = self
1154                    .abort_startup("External message bus ingress failed to start")
1155                    .await;
1156                Self::drain_channels(
1157                    &mut time_evt_rx,
1158                    &mut system_evt_rx,
1159                    &mut system_cmd_rx,
1160                    &mut exec_evt_rx,
1161                    &mut exec_cmd_rx,
1162                    &mut data_evt_rx,
1163                    &mut data_cmd_rx,
1164                );
1165                log::info!("Event loop stopped");
1166
1167                if let Err(finalize_err) = result {
1168                    anyhow::bail!(
1169                        "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1170                    );
1171                }
1172
1173                return Err(e);
1174            }
1175        };
1176
1177        let stop_handle = self.handle.clone();
1178        let mut pending = PendingEvents::default();
1179        let mut startup_system_events = Vec::new();
1180        let mut startup_system_commands = Vec::new();
1181        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1182
1183        // Startup phase 1: Connect data clients and drain instrument events into cache.
1184        // This ensures the cache is populated before execution clients connect.
1185        let mut pending_external = VecDeque::new();
1186        let data_connect_result = drive_with_event_buffering(
1187            drive_with_external_ingress(
1188                self.connect_data_phase(connection_deadline),
1189                &mut external_msgbus_rx,
1190                &mut pending_external,
1191            ),
1192            &mut pending,
1193            &mut time_evt_rx,
1194            &mut system_evt_rx,
1195            &mut system_cmd_rx,
1196            &mut exec_evt_rx,
1197            &mut exec_cmd_rx,
1198            &mut data_evt_rx,
1199            &mut data_cmd_rx,
1200        )
1201        .await;
1202
1203        self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1204
1205        if let Err(e) = data_connect_result {
1206            flush_all_pending(
1207                &mut pending,
1208                &mut time_evt_rx,
1209                &mut system_evt_rx,
1210                &mut system_cmd_rx,
1211                &mut exec_evt_rx,
1212                &mut exec_cmd_rx,
1213                &mut data_evt_rx,
1214                &mut data_cmd_rx,
1215            );
1216            let result = self
1217                .abort_startup_with_error("Data client connection timed out", e)
1218                .await;
1219            Self::drain_channels(
1220                &mut time_evt_rx,
1221                &mut system_evt_rx,
1222                &mut system_cmd_rx,
1223                &mut exec_evt_rx,
1224                &mut exec_cmd_rx,
1225                &mut data_evt_rx,
1226                &mut data_cmd_rx,
1227            );
1228            log::info!("Event loop stopped");
1229            return result;
1230        }
1231
1232        // Flush any data events still queued in the channel receivers that the
1233        // select loop did not capture before the connect future resolved, then
1234        // drain everything into cache.
1235        flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1236        startup_system_events.extend(pending.take_system_events());
1237        startup_system_commands.extend(pending.take_system_commands());
1238        debug_assert!(
1239            pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1240            "data must be drained into cache before exec clients connect",
1241        );
1242
1243        // Startup phase 2: Connect execution clients (instruments now in cache)
1244        let engine_connection_result = drive_with_event_buffering(
1245            self.connect_exec_phase(connection_deadline),
1246            &mut pending,
1247            &mut time_evt_rx,
1248            &mut system_evt_rx,
1249            &mut system_cmd_rx,
1250            &mut exec_evt_rx,
1251            &mut exec_cmd_rx,
1252            &mut data_evt_rx,
1253            &mut data_cmd_rx,
1254        )
1255        .await;
1256
1257        // Flush channel receivers and drain all remaining pending events
1258        flush_all_pending(
1259            &mut pending,
1260            &mut time_evt_rx,
1261            &mut system_evt_rx,
1262            &mut system_cmd_rx,
1263            &mut exec_evt_rx,
1264            &mut exec_cmd_rx,
1265            &mut data_evt_rx,
1266            &mut data_cmd_rx,
1267        );
1268        startup_system_events.extend(pending.take_system_events());
1269        startup_system_commands.extend(pending.take_system_commands());
1270        debug_assert!(
1271            pending.is_empty(),
1272            "all startup events must be processed before reconciliation",
1273        );
1274
1275        let engine_connection_status = match engine_connection_result {
1276            Ok(status) => status,
1277            Err(e) => {
1278                let result = self
1279                    .abort_startup_with_error("Execution client connection timed out", e)
1280                    .await;
1281                Self::drain_channels(
1282                    &mut time_evt_rx,
1283                    &mut system_evt_rx,
1284                    &mut system_cmd_rx,
1285                    &mut exec_evt_rx,
1286                    &mut exec_cmd_rx,
1287                    &mut data_evt_rx,
1288                    &mut data_cmd_rx,
1289                );
1290                log::info!("Event loop stopped");
1291                return result;
1292            }
1293        };
1294
1295        if engine_connection_status == EngineConnectionStatus::TimedOut {
1296            let result = self
1297                .abort_startup_with_error(
1298                    "Engine readiness timed out",
1299                    anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1300                )
1301                .await;
1302            Self::drain_channels(
1303                &mut time_evt_rx,
1304                &mut system_evt_rx,
1305                &mut system_cmd_rx,
1306                &mut exec_evt_rx,
1307                &mut exec_cmd_rx,
1308                &mut data_evt_rx,
1309                &mut data_cmd_rx,
1310            );
1311            log::info!("Event loop stopped");
1312            return result;
1313        }
1314
1315        if let Some(reason) = engine_connection_status
1316            .abort_reason()
1317            .or_else(|| self.startup_abort_reason())
1318        {
1319            self.abort_startup(reason).await?;
1320            Self::drain_channels(
1321                &mut time_evt_rx,
1322                &mut system_evt_rx,
1323                &mut system_cmd_rx,
1324                &mut exec_evt_rx,
1325                &mut exec_cmd_rx,
1326                &mut data_evt_rx,
1327                &mut data_cmd_rx,
1328            );
1329            log::info!("Event loop stopped");
1330            return Ok(());
1331        }
1332
1333        debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1334
1335        // Run reconciliation now that instruments are in cache and start trader
1336        if let Err(e) = self.perform_startup_reconciliation().await {
1337            let result = self.abort_startup("Startup reconciliation failed").await;
1338            Self::drain_channels(
1339                &mut time_evt_rx,
1340                &mut system_evt_rx,
1341                &mut system_cmd_rx,
1342                &mut exec_evt_rx,
1343                &mut exec_cmd_rx,
1344                &mut data_evt_rx,
1345                &mut data_cmd_rx,
1346            );
1347            log::info!("Event loop stopped");
1348
1349            if let Err(finalize_err) = result {
1350                anyhow::bail!(
1351                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1352                );
1353            }
1354
1355            return Err(e);
1356        }
1357
1358        if let Some(reason) = self.startup_abort_reason() {
1359            let result = self.abort_startup(reason).await;
1360            Self::drain_channels(
1361                &mut time_evt_rx,
1362                &mut system_evt_rx,
1363                &mut system_cmd_rx,
1364                &mut exec_evt_rx,
1365                &mut exec_cmd_rx,
1366                &mut data_evt_rx,
1367                &mut data_cmd_rx,
1368            );
1369            log::info!("Event loop stopped");
1370            return result;
1371        }
1372
1373        if let Err(e) = self.kernel.start_trader() {
1374            let result = self.abort_after_trader_start_failure(e).await;
1375            Self::drain_channels(
1376                &mut time_evt_rx,
1377                &mut system_evt_rx,
1378                &mut system_cmd_rx,
1379                &mut exec_evt_rx,
1380                &mut exec_cmd_rx,
1381                &mut data_evt_rx,
1382                &mut data_cmd_rx,
1383            );
1384            log::info!("Event loop stopped");
1385            return result;
1386        }
1387
1388        #[cfg(feature = "plugin")]
1389        if let Err(e) = self.plugins.start_controllers() {
1390            let result = self.abort_after_trader_start_failure(e).await;
1391            Self::drain_channels(
1392                &mut time_evt_rx,
1393                &mut system_evt_rx,
1394                &mut system_cmd_rx,
1395                &mut exec_evt_rx,
1396                &mut exec_cmd_rx,
1397                &mut data_evt_rx,
1398                &mut data_cmd_rx,
1399            );
1400            log::info!("Event loop stopped");
1401            return result;
1402        }
1403
1404        self.process_system_events(startup_system_events);
1405        self.process_system_commands(startup_system_commands);
1406
1407        let finish_result = {
1408            let mut receivers = RunnerReceivers {
1409                time_evt: &mut time_evt_rx,
1410                system_evt: &mut system_evt_rx,
1411                system_cmd: &mut system_cmd_rx,
1412                data_evt: &mut data_evt_rx,
1413                data_cmd: &mut data_cmd_rx,
1414                exec_evt: &mut exec_evt_rx,
1415                exec_cmd: &mut exec_cmd_rx,
1416            };
1417
1418            self.finish_startup_trader(Some(&mut receivers)).await
1419        };
1420
1421        match finish_result {
1422            Ok(true) => {}
1423            result => {
1424                log::info!("Event loop stopped");
1425                return result.map(|_| ());
1426            }
1427        }
1428
1429        let exec_config = &self.config.exec_engine;
1430        let inflight_interval =
1431            Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1432
1433        let open_interval = exec_config
1434            .open_check_interval_secs
1435            .filter(|&s| s > 0.0)
1436            .map_or(Duration::ZERO, |secs| {
1437                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1438            });
1439
1440        let position_interval = exec_config
1441            .position_check_interval_secs
1442            .filter(|&s| s > 0.0)
1443            .map_or(Duration::ZERO, |secs| {
1444                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1445            });
1446
1447        let has_clients = !self
1448            .kernel
1449            .exec_engine
1450            .borrow()
1451            .get_all_clients()
1452            .is_empty();
1453        let recon_enabled = has_clients
1454            && (!inflight_interval.is_zero()
1455                || !open_interval.is_zero()
1456                || !position_interval.is_zero());
1457
1458        let recon_min_interval = if recon_enabled {
1459            let mut intervals = Vec::new();
1460
1461            if !inflight_interval.is_zero() {
1462                intervals.push(inflight_interval);
1463            }
1464
1465            if !open_interval.is_zero() {
1466                intervals.push(open_interval);
1467            }
1468
1469            if !position_interval.is_zero() {
1470                intervals.push(position_interval);
1471            }
1472
1473            intervals
1474                .into_iter()
1475                .min()
1476                .unwrap_or(Duration::from_secs(1))
1477        } else {
1478            Duration::from_secs(1) // Unused, timer won't fire
1479        };
1480
1481        // `reconciliation_startup_delay_secs` is a post-reconciliation grace period:
1482        // startup reconciliation has already completed above, and this delay offsets
1483        // the first periodic tick to let the system stabilize before continuous checks
1484        // begin.
1485        let startup_delay = if self.config.exec_engine.reconciliation {
1486            Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1487        } else {
1488            Duration::ZERO
1489        };
1490
1491        let recon_start = dst::time::Instant::now() + startup_delay;
1492
1493        let mut last_inflight_check = dst::time::Instant::now();
1494        let mut last_open_check = last_inflight_check;
1495        let mut last_position_check = last_inflight_check;
1496
1497        // Per-task `(interval, next_fire)` schedules dispatched by the
1498        // shared `maintenance_timer` below. See module docs for rationale.
1499        let far_future = Duration::from_hours(24 * 365 * 100);
1500
1501        let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1502            let dur = opt_dur.unwrap_or(far_future);
1503            (dur, recon_start + dur)
1504        };
1505
1506        let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1507            Some(recon_min_interval)
1508        } else {
1509            None
1510        });
1511
1512        let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1513            exec_config
1514                .purge_closed_orders_interval_mins
1515                .filter(|&m| m > 0)
1516                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1517        );
1518
1519        let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1520            exec_config
1521                .purge_closed_positions_interval_mins
1522                .filter(|&m| m > 0)
1523                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1524        );
1525
1526        let (purge_account_interval, mut purge_account_next) = make_schedule(
1527            exec_config
1528                .purge_account_events_interval_mins
1529                .filter(|&m| m > 0)
1530                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1531        );
1532
1533        let (own_books_interval, mut own_books_next) = make_schedule(
1534            exec_config
1535                .own_books_audit_interval_secs
1536                .filter(|&s| s > 0.0)
1537                .map(Duration::from_secs_f64),
1538        );
1539
1540        let (prune_fills_interval, mut prune_fills_next) =
1541            make_schedule(Some(Duration::from_mins(1)));
1542
1543        let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1544        maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1545
1546        // Stop-check timer is not subject to the reconciliation startup delay,
1547        // so shutdown signals remain responsive from the moment the node reaches
1548        // `Running`. Set `MissedTickBehavior::Skip` so backlog ticks do not fire
1549        // a burst after the select arm was suspended by other branches.
1550        let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1551        stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1552
1553        // Running phase: runs until shutdown deadline expires
1554        let mut residual_events = 0usize;
1555        let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1556        let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1557        let mut position_report_task: Option<PositionReportTask> = None;
1558
1559        // A hosted node never installs signal handlers, so these futures stay pending and their
1560        // listeners are never registered. Both arms resolve to the same type as the real listeners.
1561        let owns_signals = mode.owns_signals();
1562
1563        let ctrl_c = async move {
1564            if owns_signals {
1565                dst::signal::ctrl_c().await
1566            } else {
1567                std::future::pending::<std::io::Result<()>>().await
1568            }
1569        };
1570
1571        let terminate = async move {
1572            if owns_signals {
1573                dst::signal::terminate().await
1574            } else {
1575                std::future::pending::<std::io::Result<()>>().await
1576            }
1577        };
1578
1579        tokio::pin!(ctrl_c);
1580        tokio::pin!(terminate);
1581
1582        let metrics = self.handle.metrics.clone();
1583        let metrics_start = dst::time::Instant::now();
1584        metrics.reset();
1585
1586        let mut queue_monitor = self
1587            .config
1588            .queue_monitor
1589            .as_ref()
1590            .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1591        let mut dispatches_since_yield = 0usize;
1592        let mut skip_time = false;
1593        let mut skip_data = false;
1594
1595        let dispatch_result = loop {
1596            let callbacks_pending = match drain_callbacks().await {
1597                Ok(pending) => pending,
1598                Err(e) => {
1599                    if self.state() == NodeState::Running {
1600                        self.initiate_shutdown();
1601                    }
1602
1603                    log::warn!(
1604                        "Skipping residual events and final buffered dispatch after callback failure"
1605                    );
1606
1607                    break Err(e);
1608                }
1609            };
1610
1611            let shutdown_deadline = self.shutdown_deadline;
1612            let is_shutting_down = self.state() == NodeState::ShuttingDown;
1613            let is_running = self.state() == NodeState::Running;
1614            let commands_empty = data_cmd_rx.is_empty();
1615            let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1616            let poll_data = !skip_data || commands_empty;
1617
1618            tokio::select! {
1619                biased;
1620
1621                // Signal branches first so they are always checked
1622                result = &mut ctrl_c, if is_running => {
1623                    match result {
1624                        Ok(()) => log::info!("Received SIGINT, shutting down"),
1625                        Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1626                    }
1627                    self.initiate_shutdown();
1628                }
1629                result = &mut terminate, if is_running => {
1630                    match result {
1631                        Ok(()) => log::info!("Received SIGTERM, shutting down"),
1632                        Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1633                    }
1634                    self.initiate_shutdown();
1635                }
1636                _ = stop_check_timer.tick(), if is_running => {
1637                    if stop_handle.should_stop() {
1638                        log::info!("Received stop signal from handle");
1639                        self.initiate_shutdown();
1640                    } else if self.kernel.is_shutdown_requested() {
1641                        log::info!("Received ShutdownSystem command, shutting down");
1642                        self.initiate_shutdown();
1643                    }
1644                }
1645                () = async {
1646                    match shutdown_deadline {
1647                        Some(deadline) => dst::time::sleep_until(deadline).await,
1648                        None => std::future::pending::<()>().await,
1649                    }
1650                }, if self.state() == NodeState::ShuttingDown => {
1651                    break Ok(());
1652                }
1653                () = std::future::ready(()), if callbacks_pending => {},
1654                result = async {
1655                    match open_order_report_task.as_mut() {
1656                        Some(task) => task.future.as_mut().await,
1657                        None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1658                    }
1659                }, if open_order_report_task.is_some() => {
1660                    let maintenance_start = dst::time::Instant::now();
1661
1662                    drop(open_order_report_task.take());
1663
1664                    match result {
1665                        ReportTaskOutcome::Completed(result) => {
1666                            let result = result.finish();
1667                            let client_refs = self
1668                                .exec_clients
1669                                .iter()
1670                                .map(|client| client as &dyn ExecutionClient)
1671                                .collect::<Vec<_>>();
1672                            let reconciliation = self.exec_manager.reconcile_open_order_reports(
1673                                &result.check,
1674                                result.reports,
1675                                &result.queried_clients,
1676                                &result.failed_clients,
1677                                &client_refs,
1678                            );
1679                            self.process_reconciliation_events(&reconciliation.events);
1680                            if !reconciliation.targeted_queries.is_empty() {
1681                                if is_shutting_down {
1682                                    let planned_client_order_ids = reconciliation
1683                                        .targeted_queries
1684                                        .iter()
1685                                        .map(TargetedOrderQuery::client_order_id)
1686                                        .collect::<Vec<_>>();
1687                                    self.cleanup_cancelled_report_tasks(
1688                                        &planned_client_order_ids,
1689                                    );
1690                                } else {
1691                                    targeted_order_report_task = Some(
1692                                        self.start_targeted_order_report_check(
1693                                            reconciliation.targeted_queries,
1694                                        ),
1695                                    );
1696                                }
1697                            }
1698                        }
1699                        ReportTaskOutcome::TimedOut => {
1700                            self.cleanup_cancelled_report_tasks(&[]);
1701                            log::warn!(
1702                                "Open-order report collection expired after {:?}",
1703                                self.config.timeout_reconciliation,
1704                            );
1705                        }
1706                    }
1707                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1708                }
1709                result = async {
1710                    match targeted_order_report_task.as_mut() {
1711                        Some(task) => task.future.as_mut().await,
1712                        None => std::future::pending::<ReportTaskOutcome<()>>().await,
1713                    }
1714                }, if targeted_order_report_task.is_some() => {
1715                    let maintenance_start = dst::time::Instant::now();
1716
1717                    if let Some(task) = targeted_order_report_task.take() {
1718                        self.complete_targeted_order_report_check(task, &result);
1719                    }
1720                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1721                }
1722                result = async {
1723                    match position_report_task.as_mut() {
1724                        Some(task) => task.future.as_mut().await,
1725                        None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1726                    }
1727                }, if position_report_task.is_some() => {
1728                    let maintenance_start = dst::time::Instant::now();
1729
1730                    drop(position_report_task.take());
1731
1732                    match result {
1733                        ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1734                            if is_shutting_down {
1735                                self.cleanup_cancelled_report_tasks(&[]);
1736                            } else {
1737                                let result = result.finish();
1738                                position_report_task = self.handle_position_report_result(result);
1739                            }
1740                        }
1741                        ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1742                            let reports = collection.finish();
1743                            self.handle_position_fill_report_result(PositionFillReportResult {
1744                                position_result,
1745                                reports,
1746                            });
1747                        }
1748                        ReportTaskOutcome::TimedOut => {
1749                            self.cleanup_cancelled_report_tasks(&[]);
1750                            log::warn!(
1751                                "Position report collection expired after {:?}",
1752                                self.config.timeout_reconciliation,
1753                            );
1754                        }
1755                    }
1756                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1757                }
1758
1759                // Maintenance dispatcher (before event processing to avoid
1760                // starvation). See module docs for design rationale.
1761                _ = maintenance_timer.tick(), if is_running => {
1762                    let maintenance_start = dst::time::Instant::now();
1763                    metrics.publish_queue_depths(
1764                        RunnerChannelQueueDepths::from_receivers(
1765                            &time_evt_rx,
1766                            &exec_evt_rx,
1767                            &exec_cmd_rx,
1768                            &data_evt_rx,
1769                            &data_cmd_rx,
1770                        ),
1771                        metrics_start.elapsed(),
1772                    );
1773
1774                    if let Some(queue_monitor) = queue_monitor.as_mut() {
1775                        let transitions = queue_monitor.evaluate(metrics.snapshot());
1776                        self.publish_queue_state_transitions(&transitions);
1777                    }
1778
1779                    let mut now = dst::time::Instant::now();
1780
1781                    if recon_enabled && now >= recon_next {
1782                        let recon_intervals = ReconciliationCheckIntervals {
1783                            inflight: inflight_interval,
1784                            open: open_interval,
1785                            position: position_interval,
1786                        };
1787                        let mut recon_state = ReconciliationCheckState {
1788                            last_inflight_check: &mut last_inflight_check,
1789                            last_open_check: &mut last_open_check,
1790                            last_position_check: &mut last_position_check,
1791                            open_order_report_task: &mut open_order_report_task,
1792                            targeted_order_report_task: &mut targeted_order_report_task,
1793                            position_report_task: &mut position_report_task,
1794                        };
1795
1796                        self.run_reconciliation_checks(
1797                            now,
1798                            recon_intervals,
1799                            &mut recon_state,
1800                        );
1801
1802                        now = dst::time::Instant::now();
1803                        recon_next = now + recon_interval;
1804                    }
1805
1806                    if now >= purge_orders_next {
1807                        self.exec_manager.purge_closed_orders();
1808                        purge_orders_next = now + purge_orders_interval;
1809                    }
1810
1811                    if now >= purge_positions_next {
1812                        self.exec_manager.purge_closed_positions();
1813                        purge_positions_next = now + purge_positions_interval;
1814                    }
1815
1816                    if now >= purge_account_next {
1817                        self.exec_manager.purge_account_events();
1818                        purge_account_next = now + purge_account_interval;
1819                    }
1820
1821                    if now >= own_books_next {
1822                        self.kernel.cache().borrow_mut().audit_own_order_books();
1823                        own_books_next = now + own_books_interval;
1824                    }
1825
1826                    if now >= prune_fills_next {
1827                        self.exec_manager.prune_recent_fills_cache(60.0);
1828                        self.exec_manager.prune_processed_fills();
1829                        self.exec_manager.prune_order_local_activity();
1830                        prune_fills_next = now + prune_fills_interval;
1831                    }
1832
1833                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1834                }
1835
1836                // Event processing branches. Exec commands and events are
1837                // ordered ahead of data events so a strategy action (cancel,
1838                // submit, etc.) is not delayed behind a market data backlog
1839                // when the biased select polls receivers each iteration.
1840                Some(handler) = time_evt_rx.recv(), if poll_time => {
1841                    skip_time = true;
1842                    let dispatch_start = dst::time::Instant::now();
1843                    let dispatched = AsyncRunner::handle_time_event(handler);
1844
1845                    if dispatched && is_shutting_down {
1846                        log::debug!("Residual time event");
1847                        residual_events += 1;
1848                    }
1849
1850                    if dispatched {
1851                        record_runner_dispatch(
1852                            &metrics,
1853                            SystemChannel::TimeEvents,
1854                            dispatch_start,
1855                            metrics_start,
1856                        );
1857                    }
1858                }
1859                Some(event) = system_evt_rx.recv() => {
1860                    skip_time = false;
1861
1862                    if is_shutting_down {
1863                        log::debug!("Residual system event: {event}");
1864                        residual_events += 1;
1865                    }
1866                    event.dispatch(|event| self.process_system_event(event));
1867                }
1868                Some(command) = system_cmd_rx.recv() => {
1869                    skip_time = false;
1870
1871                    if is_shutting_down {
1872                        log::debug!("Residual system command: {command}");
1873                        residual_events += 1;
1874                    }
1875                    command.dispatch(|command| self.process_system_command(command));
1876                }
1877                Some(evt) = exec_evt_rx.recv() => {
1878                    skip_time = false;
1879                    let dispatch_start = dst::time::Instant::now();
1880
1881                    if is_shutting_down {
1882                        log::debug!("Residual exec event: {evt}");
1883                        residual_events += 1;
1884                    }
1885
1886                    evt.dispatch(|evt| self.process_exec_event(evt));
1887                    record_runner_dispatch(
1888                        &metrics,
1889                        SystemChannel::ExecEvents,
1890                        dispatch_start,
1891                        metrics_start,
1892                    );
1893                }
1894                Some(cmd) = exec_cmd_rx.recv() => {
1895                    skip_time = false;
1896                    let dispatch_start = dst::time::Instant::now();
1897
1898                    if is_shutting_down {
1899                        log::debug!("Residual exec command: {cmd}");
1900                        residual_events += 1;
1901                    }
1902
1903                    self.process_exec_command(cmd);
1904                    record_runner_dispatch(
1905                        &metrics,
1906                        SystemChannel::ExecCommands,
1907                        dispatch_start,
1908                        metrics_start,
1909                    );
1910                }
1911                message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1912                    skip_time = false;
1913                    let external_msgbus_start = dst::time::Instant::now();
1914
1915                    match message {
1916                        Some(message) => {
1917                            if is_shutting_down {
1918                                log::debug!("Residual external message bus message: {message}");
1919                                residual_events += 1;
1920                            }
1921                            self.process_external_msgbus_message(&message);
1922                        }
1923                        None => {
1924                            log::info!("External message bus ingress closed");
1925                            external_msgbus_rx = None;
1926                            self.close_external_ingress();
1927                        }
1928                    }
1929
1930                    record_runner_external_msgbus(
1931                        &metrics,
1932                        external_msgbus_start,
1933                        metrics_start,
1934                    );
1935                }
1936                Some(evt) = data_evt_rx.recv(), if poll_data => {
1937                    skip_time = false;
1938                    skip_data = true;
1939                    let dispatch_start = dst::time::Instant::now();
1940
1941                    if is_shutting_down {
1942                        log::debug!("Residual data event: {evt:?}");
1943                        residual_events += 1;
1944                    }
1945                    AsyncRunner::dispatch_data_event(evt);
1946                    record_runner_dispatch(
1947                        &metrics,
1948                        SystemChannel::DataEvents,
1949                        dispatch_start,
1950                        metrics_start,
1951                    );
1952                }
1953                Some(cmd) = data_cmd_rx.recv() => {
1954                    skip_time = false;
1955                    skip_data = false;
1956                    let dispatch_start = dst::time::Instant::now();
1957
1958                    if is_shutting_down {
1959                        log::debug!("Residual data command: {cmd:?}");
1960                        residual_events += 1;
1961                    }
1962                    AsyncRunner::handle_data_command(cmd);
1963                    record_runner_dispatch(
1964                        &metrics,
1965                        SystemChannel::DataCommands,
1966                        dispatch_start,
1967                        metrics_start,
1968                    );
1969                }
1970            }
1971
1972            dispatches_since_yield += 1;
1973            if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1974                dispatches_since_yield = 0;
1975                tokio::task::yield_now().await;
1976            }
1977        };
1978
1979        if residual_events > 0 {
1980            log::debug!("Processed {residual_events} residual events during shutdown");
1981        }
1982
1983        self.cancel_report_tasks(
1984            &mut open_order_report_task,
1985            &mut targeted_order_report_task,
1986            &mut position_report_task,
1987            dispatch_result.is_ok(),
1988        );
1989
1990        drop(external_msgbus_rx.take());
1991        let _ = self.kernel.cache().borrow().check_residuals();
1992
1993        let stop_result = self.finalize_stop().await;
1994
1995        if let Err(e) = dispatch_result {
1996            if let Err(stop_err) = stop_result {
1997                log::error!("Failed to finalize node after callback failure: {stop_err}");
1998                // Python converts the outer display message, so preserve both diagnostics there.
1999                let diagnostic = format!("{e}; {stop_err}");
2000                return Err(anyhow::Error::new(e).context(diagnostic));
2001            }
2002
2003            return Err(e.into());
2004        }
2005
2006        // Handle events that arrived during finalize_stop
2007        Self::drain_channels(
2008            &mut time_evt_rx,
2009            &mut system_evt_rx,
2010            &mut system_cmd_rx,
2011            &mut exec_evt_rx,
2012            &mut exec_cmd_rx,
2013            &mut data_evt_rx,
2014            &mut data_cmd_rx,
2015        );
2016
2017        log::info!("Event loop stopped");
2018
2019        stop_result
2020    }
2021
2022    fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2023        for transition in transitions {
2024            let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2025            let timestamp = self.kernel.generate_timestamp_ns();
2026
2027            let event = QueueStateChanged::new(
2028                self.config.trader_id,
2029                transition.channel,
2030                transition.condition,
2031                transition.state,
2032                transition.queue_depth,
2033                transition.mean_dispatch_ns,
2034                UUID4::new(),
2035                timestamp,
2036                timestamp,
2037            );
2038
2039            msgbus::publish_any(topic, event.as_any());
2040        }
2041    }
2042
2043    #[expect(
2044        clippy::await_holding_refcell_ref,
2045        reason = "cache loading is serialized before the single-threaded live node starts"
2046    )]
2047    async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2048        self.install_cache_database().await?;
2049
2050        let cache = self.kernel.cache();
2051        if !cache.borrow().has_backing() {
2052            return Ok(());
2053        }
2054
2055        if self
2056            .config
2057            .cache
2058            .as_ref()
2059            .is_some_and(|config| config.flush_on_start)
2060        {
2061            cache
2062                .borrow_mut()
2063                .flush_db()
2064                .context("Failed to flush persistent cache")?;
2065            return Ok(());
2066        }
2067
2068        if self.config.exec_engine.load_cache {
2069            self.kernel
2070                .exec_engine()
2071                .borrow_mut()
2072                .load_cache()
2073                .await
2074                .context("Failed to load persistent cache")?;
2075        }
2076
2077        Ok(())
2078    }
2079
2080    /// Returns whether a cache database backing is configured but not yet installed.
2081    #[must_use]
2082    pub const fn has_pending_cache_database(&self) -> bool {
2083        self.cache_database_factory.is_some()
2084    }
2085
2086    /// Constructs the configured cache database backing and installs it on the kernel cache.
2087    ///
2088    /// Construction is deferred to startup so the adapter is built inside the async runtime rather
2089    /// than by blocking the synchronous builder, and so the connection opens only when the node runs.
2090    async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2091        let Some(factory) = self.cache_database_factory.as_ref() else {
2092            return Ok(());
2093        };
2094
2095        let config = self.config.cache.clone().unwrap_or_default();
2096
2097        // Cleared only after a successful install, so a failed startup can be retried rather than
2098        // silently starting without the backing the caller asked for.
2099        let database = factory
2100            .create(self.config.trader_id, self.kernel.instance_id, config)
2101            .await
2102            .context("failed to create cache database backing")?;
2103        self.kernel.cache().borrow_mut().set_database(database);
2104        self.cache_database_factory = None;
2105
2106        Ok(())
2107    }
2108
2109    fn take_external_ingress_receiver(
2110        &mut self,
2111    ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2112        let Some(external_ingress) = self.external_msgbus.as_mut() else {
2113            return Ok(None);
2114        };
2115
2116        let receiver = external_ingress.take_receiver()?;
2117        log::info!("External message bus ingress started");
2118        Ok(Some(receiver))
2119    }
2120
2121    fn republish_external_msgbus_message(message: &BusMessage) {
2122        if let Err(e) = msgbus::republish_external_message(message) {
2123            log::error!(
2124                "Failed to republish external message bus topic '{}': {e:#}",
2125                message.topic
2126            );
2127        }
2128    }
2129
2130    fn process_external_msgbus_message(&self, message: &BusMessage) {
2131        if self.stream_processors.is_empty() {
2132            Self::republish_external_msgbus_message(message);
2133            return;
2134        }
2135
2136        let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2137            for processor in &self.stream_processors {
2138                (processor.0)(value, mapping)?;
2139            }
2140
2141            Ok(())
2142        };
2143
2144        if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2145            log::error!(
2146                "Failed to process external message bus topic '{}': {e:#}",
2147                message.topic
2148            );
2149        }
2150    }
2151
2152    fn finish_external_ingress_connection(
2153        &mut self,
2154        receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2155    ) {
2156        if receiver.is_none() {
2157            self.close_external_ingress();
2158        }
2159    }
2160
2161    fn close_external_ingress(&mut self) {
2162        if let Some(external_ingress) = self.external_msgbus.as_mut()
2163            && !external_ingress.is_closed()
2164        {
2165            external_ingress.close();
2166        }
2167    }
2168
2169    fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2170        if events.is_empty() {
2171            self.publish_submission_recovery_exhaustions();
2172            return;
2173        }
2174
2175        log::info!(
2176            "Processing {} reconciliation event{}",
2177            events.len(),
2178            if events.len() == 1 { "" } else { "s" }
2179        );
2180
2181        for event in events {
2182            self.exec_manager
2183                .record_local_activity(event.client_order_id());
2184            if let OrderEventAny::Filled(fill) = event {
2185                self.exec_manager
2186                    .record_position_activity(fill.instrument_id, fill.account_id);
2187            }
2188
2189            self.kernel.exec_engine.borrow_mut().process(event);
2190            self.exec_manager
2191                .confirm_submission_outcome(&event.client_order_id());
2192            if let OrderEventAny::Filled(fill) = event {
2193                self.exec_manager.commit_recent_fill_if_applied(fill);
2194            }
2195        }
2196
2197        self.publish_submission_recovery_exhaustions();
2198    }
2199
2200    fn publish_submission_recovery_exhaustions(&mut self) {
2201        for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2202            log::warn!(
2203                "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2204                diagnostic.client_order_id,
2205                diagnostic.source,
2206                diagnostic.retry_count,
2207            );
2208            msgbus::publish_any(
2209                MessagingSwitchboard::submission_recovery_exhausted_topic(),
2210                &diagnostic,
2211            );
2212        }
2213    }
2214
2215    fn process_exec_event(&mut self, event: ExecutionEvent) {
2216        let clear_closed_orders = matches!(
2217            &event,
2218            ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2219        );
2220        let closed_report_id = match &event {
2221            ExecutionEvent::Report(
2222                ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2223            ) if report.order_status.is_closed() => report.client_order_id,
2224            _ => None,
2225        };
2226        let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2227            return;
2228        };
2229
2230        self.dispatch_exec_event_and_commit_fill(event);
2231
2232        for client_order_id in &order_ids {
2233            self.exec_manager
2234                .confirm_submission_outcome(client_order_id);
2235            let is_closed = self
2236                .kernel
2237                .cache()
2238                .borrow()
2239                .order(client_order_id)
2240                .is_some_and(|order| order.is_closed());
2241
2242            if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2243                self.exec_manager
2244                    .clear_recon_tracking(client_order_id, true);
2245            }
2246        }
2247    }
2248
2249    fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2250        message.dispatch_trading(|message| {
2251            if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2252                self.observe_exec_command_before_dispatch(message.command());
2253            }
2254        });
2255    }
2256
2257    /// Dispatches a normal-ingress execution event, then commits a direct
2258    /// `OrderFilled` to the recent-fills dedup cache only once it is present on
2259    /// its canonical order.
2260    ///
2261    /// The fill candidate is captured from `&evt` BEFORE the value is moved into
2262    /// [`AsyncRunner::handle_exec_event`]; the gated commit runs AFTER dispatch,
2263    /// so a fill the execution engine rejects (unknown order, invalid
2264    /// transition) is never marked and its later `Fill` report stays eligible.
2265    fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2266        let recent_fill_candidate = match &evt {
2267            ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2268            _ => None,
2269        };
2270
2271        AsyncRunner::handle_exec_event(evt);
2272
2273        if let Some(fill) = &recent_fill_candidate {
2274            self.exec_manager.commit_recent_fill_if_applied(fill);
2275        }
2276    }
2277
2278    async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2279        // A zero remaining budget still admits an immediately-ready connect (an
2280        // empty/ready client set completes on the first poll); a pending connect
2281        // fails closed on that same poll. This keeps a zero `timeout_connection`
2282        // - a supported "do not wait, but allow ready work" configuration -
2283        // working, while still bounding a hung connect.
2284        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2285        dst::time::timeout(remaining, self.kernel.connect_data_clients())
2286            .await
2287            .map_err(|_| anyhow::anyhow!("data-connect timeout"))
2288    }
2289
2290    async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2291        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2292        dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2293            .await
2294            .map_err(|_| anyhow::anyhow!("exec-connect timeout"))
2295    }
2296
2297    /// Connects execution clients and checks all engines are connected.
2298    ///
2299    /// Returns the final connection wait status.
2300    /// Must be called after data clients are connected and instrument events drained.
2301    async fn connect_exec_phase(
2302        &mut self,
2303        deadline: dst::time::Instant,
2304    ) -> anyhow::Result<EngineConnectionStatus> {
2305        self.connect_exec_clients(deadline).await?;
2306        Ok(self.await_engines_connected(deadline).await)
2307    }
2308
2309    fn startup_abort_reason(&self) -> Option<&'static str> {
2310        if self.handle.should_stop() {
2311            Some("Stop signal received during startup")
2312        } else if self.kernel.is_shutdown_requested() {
2313            Some("Shutdown signal received during startup")
2314        } else {
2315            None
2316        }
2317    }
2318
2319    async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2320        match self.handle.try_set_running() {
2321            RunningTransition::Entered => Ok(true),
2322            RunningTransition::StopRequested => {
2323                self.abort_startup("Stop signal received during startup")
2324                    .await?;
2325                Ok(false)
2326            }
2327            RunningTransition::Invalid(control) => {
2328                self.abort_startup_with_error(
2329                    "Invalid lifecycle state during startup",
2330                    anyhow::anyhow!(
2331                        "Invalid LiveNode control state {control:#04x} while entering Running"
2332                    ),
2333                )
2334                .await?;
2335                Ok(false)
2336            }
2337        }
2338    }
2339
2340    async fn finish_startup_trader(
2341        &mut self,
2342        receivers: Option<&mut RunnerReceivers<'_>>,
2343    ) -> anyhow::Result<bool> {
2344        match self.handle.try_set_running() {
2345            RunningTransition::Entered => Ok(true),
2346            RunningTransition::StopRequested => {
2347                self.abort_started_trader("Stop signal received during startup", receivers)
2348                    .await?;
2349                Ok(false)
2350            }
2351            RunningTransition::Invalid(control) => {
2352                let state_err = anyhow::anyhow!(
2353                    "Invalid LiveNode control state {control:#04x} while entering Running"
2354                );
2355
2356                match self
2357                    .abort_started_trader("Invalid lifecycle state during startup", receivers)
2358                    .await
2359                {
2360                    Ok(()) => Err(state_err),
2361                    Err(finalize_err) => {
2362                        anyhow::bail!(
2363                            "{state_err}; failed to finalize startup abort: {finalize_err}"
2364                        )
2365                    }
2366                }
2367            }
2368        }
2369    }
2370
2371    async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2372        log::info!("{reason}, aborting startup");
2373        self.handle.set_shutting_down();
2374        self.finalize_stop().await
2375    }
2376
2377    async fn abort_startup_with_error(
2378        &mut self,
2379        reason: &str,
2380        startup_err: anyhow::Error,
2381    ) -> anyhow::Result<()> {
2382        match self.abort_startup(reason).await {
2383            Ok(()) => Err(startup_err),
2384            Err(finalize_err) => {
2385                anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2386            }
2387        }
2388    }
2389
2390    async fn abort_started_trader(
2391        &mut self,
2392        reason: &str,
2393        mut receivers: Option<&mut RunnerReceivers<'_>>,
2394    ) -> anyhow::Result<()> {
2395        log::info!("{reason}, aborting startup");
2396        self.handle.set_shutting_down();
2397
2398        #[cfg(feature = "plugin")]
2399        let controller_stop_result = self.plugins.stop_controllers();
2400        #[cfg(not(feature = "plugin"))]
2401        let controller_stop_result: anyhow::Result<()> = Ok(());
2402
2403        let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2404        let delay = self.kernel.delay_post_stop();
2405        log::info!("Awaiting residual events ({delay:?})...");
2406
2407        let residual_events = match receivers.as_mut() {
2408            Some(receivers) => self.process_receivers_for(delay, receivers).await,
2409            None => self.process_runner_for(delay).await,
2410        };
2411
2412        if residual_events > 0 {
2413            log::debug!("Processed {residual_events} residual events during shutdown");
2414        }
2415
2416        let finalize_result = self.finalize_stop().await;
2417
2418        if let Some(receivers) = receivers {
2419            Self::drain_channels(
2420                receivers.time_evt,
2421                receivers.system_evt,
2422                receivers.system_cmd,
2423                receivers.exec_evt,
2424                receivers.exec_cmd,
2425                receivers.data_evt,
2426                receivers.data_cmd,
2427            );
2428        } else {
2429            let drained_events = self.drain_runner_pending();
2430            if drained_events > 0 {
2431                log::info!("Drained {drained_events} remaining events during shutdown");
2432            }
2433        }
2434
2435        let mut errors = Vec::new();
2436
2437        if let Err(e) = controller_stop_result {
2438            errors.push(format!("Failed to stop plug-in controllers: {e}"));
2439        }
2440
2441        if let Err(e) = trader_stop_result {
2442            errors.push(format!("Failed to stop trader: {e}"));
2443        }
2444
2445        if let Err(e) = finalize_result {
2446            errors.push(format!("Failed to finalize startup abort: {e}"));
2447        }
2448
2449        if errors.is_empty() {
2450            Ok(())
2451        } else {
2452            anyhow::bail!("{}", errors.join("; "))
2453        }
2454    }
2455
2456    async fn process_receivers_for(
2457        &mut self,
2458        duration: Duration,
2459        receivers: &mut RunnerReceivers<'_>,
2460    ) -> usize {
2461        let deadline = dst::time::Instant::now() + duration;
2462        let mut processed = 0;
2463
2464        loop {
2465            tokio::select! {
2466                biased;
2467
2468                () = dst::time::sleep_until(deadline) => break,
2469                Some(message) = receivers.time_evt.recv() => {
2470                    let _ = AsyncRunner::handle_time_event(message);
2471                    processed += 1;
2472                }
2473                Some(event) = receivers.system_evt.recv() => {
2474                    event.dispatch(|event| self.process_system_event(event));
2475                    processed += 1;
2476                }
2477                Some(command) = receivers.system_cmd.recv() => {
2478                    command.dispatch(|command| self.process_system_command(command));
2479                    processed += 1;
2480                }
2481                Some(event) = receivers.exec_evt.recv() => {
2482                    event.dispatch(|event| self.process_exec_event(event));
2483                    processed += 1;
2484                }
2485                Some(command) = receivers.exec_cmd.recv() => {
2486                    self.process_exec_command(command);
2487                    processed += 1;
2488                }
2489                Some(event) = receivers.data_evt.recv() => {
2490                    AsyncRunner::dispatch_data_event(event);
2491                    processed += 1;
2492                }
2493                Some(command) = receivers.data_cmd.recv() => {
2494                    AsyncRunner::handle_data_command(command);
2495                    processed += 1;
2496                }
2497            }
2498        }
2499
2500        processed
2501    }
2502
2503    async fn abort_after_trader_start_failure(
2504        &mut self,
2505        start_err: anyhow::Error,
2506    ) -> anyhow::Result<()> {
2507        log::info!("Trader startup failed, aborting startup");
2508        self.handle.set_shutting_down();
2509        let stop_result = self.kernel.stop_trader_after_start_failure();
2510        let finalize_result = self.finalize_stop().await;
2511
2512        match (stop_result, finalize_result) {
2513            (Ok(()), Ok(())) => Err(start_err),
2514            (Err(stop_err), Ok(())) => anyhow::bail!(
2515                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2516                 {stop_err}"
2517            ),
2518            (Ok(()), Err(finalize_err)) => anyhow::bail!(
2519                "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2520                 {finalize_err}"
2521            ),
2522            (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2523                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2524                 {stop_err}; failed to finalize startup abort: {finalize_err}"
2525            ),
2526        }
2527    }
2528
2529    fn initiate_shutdown(&mut self) {
2530        #[cfg(feature = "plugin")]
2531        if let Err(e) = self.plugins.stop_controllers() {
2532            log::error!("Error stopping plug-in controllers: {e}");
2533        }
2534
2535        self.kernel.stop_trader();
2536        let delay = self.kernel.delay_post_stop();
2537        log::info!("Awaiting residual events ({delay:?})...");
2538
2539        self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2540        self.handle.set_shutting_down();
2541    }
2542
2543    async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2544        // Capture recovery at the grace-period boundary, before disconnection can queue more
2545        // evidence. A later drain must not turn incomplete recovery into a successful shutdown.
2546        let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2547        self.close_external_ingress();
2548
2549        let timeout = self.config.timeout_disconnection;
2550        let deadline = dst::time::Instant::now() + timeout;
2551
2552        let mut errors = Vec::new();
2553        let (report_result, disconnect_result) = futures_util::join!(
2554            dst::time::timeout(
2555                timeout,
2556                Self::finish_report_tasks(&self.exec_clients, &mut errors)
2557            ),
2558            dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2559        );
2560
2561        if report_result.is_err() {
2562            errors.push("report collection shutdown timeout".to_string());
2563        }
2564
2565        let disconnect_result = match disconnect_result {
2566            Ok(result) => result,
2567            Err(_) => Err(anyhow::anyhow!(
2568                "disconnect timeout while disconnecting clients"
2569            )),
2570        };
2571
2572        if let Err(ref e) = disconnect_result {
2573            log::error!("Error disconnecting clients: {e}");
2574        }
2575
2576        let readiness_result = self.await_engines_disconnected(deadline).await;
2577        let kernel_result = self.kernel.finalize_stop().await;
2578
2579        self.handle.set_stopped();
2580
2581        if !unresolved_submissions.is_empty() {
2582            errors.push(format!(
2583                "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2584            ));
2585        }
2586
2587        if let Err(e) = disconnect_result {
2588            errors.push(e.to_string());
2589        }
2590
2591        if let Err(e) = readiness_result {
2592            errors.push(format!("failed while awaiting engine disconnection: {e}"));
2593        }
2594
2595        if let Err(e) = kernel_result {
2596            errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2597        }
2598
2599        if errors.is_empty() {
2600            Ok(())
2601        } else {
2602            anyhow::bail!("{}", errors.join("; "))
2603        }
2604    }
2605
2606    fn drain_channels(
2607        time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2608        system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2609        system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2610        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2611        exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2612            DispatchMessage<TradingCommandMessage>,
2613        >,
2614        data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2615        data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2616    ) {
2617        let mut drained = 0;
2618
2619        while let Ok(handler) = time_evt_rx.try_recv() {
2620            let _ = AsyncRunner::handle_time_event(handler);
2621            drained += 1;
2622        }
2623
2624        while system_evt_rx.try_recv().is_ok() {
2625            drained += 1;
2626        }
2627
2628        while system_cmd_rx.try_recv().is_ok() {
2629            drained += 1;
2630        }
2631
2632        while let Ok(evt) = data_evt_rx.try_recv() {
2633            AsyncRunner::dispatch_data_event(evt);
2634            drained += 1;
2635        }
2636
2637        while let Ok(cmd) = data_cmd_rx.try_recv() {
2638            AsyncRunner::handle_data_command(cmd);
2639            drained += 1;
2640        }
2641
2642        while let Ok(evt) = exec_evt_rx.try_recv() {
2643            AsyncRunner::dispatch_exec_event(evt);
2644            drained += 1;
2645        }
2646
2647        while let Ok(cmd) = exec_cmd_rx.try_recv() {
2648            AsyncRunner::handle_trading_command(cmd);
2649            drained += 1;
2650        }
2651
2652        if drained > 0 {
2653            log::info!("Drained {drained} remaining events during shutdown");
2654        }
2655    }
2656
2657    fn observe_exec_event_before_dispatch(
2658        &mut self,
2659        evt: &ExecutionEvent,
2660    ) -> Option<Vec<ClientOrderId>> {
2661        let mut order_ids = Vec::new();
2662
2663        match evt {
2664            ExecutionEvent::Order(order_evt) => {
2665                self.exec_manager.observe_order_event(order_evt);
2666                order_ids.push(order_evt.client_order_id());
2667            }
2668            ExecutionEvent::OrderSubmittedBatch(batch) => {
2669                for submitted in &batch.events {
2670                    self.exec_manager
2671                        .record_local_activity(submitted.client_order_id);
2672                }
2673            }
2674            ExecutionEvent::OrderAcceptedBatch(batch) => {
2675                for accepted in &batch.events {
2676                    self.exec_manager
2677                        .observe_order_event(&OrderEventAny::Accepted(*accepted));
2678                    order_ids.push(accepted.client_order_id);
2679                }
2680            }
2681            ExecutionEvent::OrderCanceledBatch(batch) => {
2682                for canceled in &batch.events {
2683                    self.exec_manager
2684                        .observe_order_event(&OrderEventAny::Canceled(*canceled));
2685                    order_ids.push(canceled.client_order_id);
2686                }
2687            }
2688            ExecutionEvent::Report(report) => {
2689                if let ExecutionReport::Fill(fill_report) = report
2690                    && self.exec_manager.is_fill_recently_processed(
2691                        fill_report.account_id,
2692                        fill_report.instrument_id,
2693                        fill_report.trade_id,
2694                    )
2695                {
2696                    log::debug!(
2697                        "Skipping recently processed fill report: {}",
2698                        fill_report.trade_id,
2699                    );
2700                    return None;
2701                }
2702
2703                self.exec_manager.observe_execution_report(report);
2704
2705                if let Some(client_order_id) = self.order_report_client_order_id(report) {
2706                    order_ids.push(client_order_id);
2707                }
2708            }
2709            ExecutionEvent::Account(_) => {}
2710        }
2711
2712        Some(order_ids)
2713    }
2714
2715    fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2716        match report {
2717            ExecutionReport::Order(order_report)
2718            | ExecutionReport::OrderWithFills(order_report, _) => {
2719                order_report.client_order_id.or_else(|| {
2720                    self.kernel
2721                        .cache
2722                        .borrow()
2723                        .client_order_id(&order_report.venue_order_id)
2724                        .copied()
2725                })
2726            }
2727            ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2728                self.kernel
2729                    .cache
2730                    .borrow()
2731                    .client_order_id(&fill_report.venue_order_id)
2732                    .copied()
2733            }),
2734            _ => None,
2735        }
2736    }
2737
2738    fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2739        let submission_client_id = if matches!(
2740            cmd,
2741            TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2742        ) {
2743            self.kernel
2744                .exec_engine
2745                .borrow()
2746                .find_client_for_command(cmd)
2747                .map(|client| client.client_id())
2748        } else {
2749            None
2750        };
2751
2752        match cmd {
2753            TradingCommand::SubmitOrder(submit) => {
2754                self.exec_manager.register_submission(
2755                    &submit.order_init,
2756                    submission_client_id.or(submit.client_id),
2757                );
2758            }
2759            TradingCommand::SubmitOrderList(submit) => {
2760                for order_init in &submit.order_inits {
2761                    self.exec_manager
2762                        .register_submission(order_init, submission_client_id.or(submit.client_id));
2763                }
2764            }
2765            TradingCommand::ModifyOrder(modify) => {
2766                self.exec_manager.register_inflight(modify.client_order_id);
2767            }
2768            TradingCommand::ModifyOrders(modify) => {
2769                for child in &modify.modifies {
2770                    self.exec_manager.register_inflight(child.client_order_id);
2771                }
2772            }
2773            TradingCommand::CancelOrder(cancel) => {
2774                self.exec_manager.register_inflight(cancel.client_order_id);
2775            }
2776            TradingCommand::CancelOrders(cancel) => {
2777                for child in &cancel.cancels {
2778                    self.exec_manager.register_inflight(child.client_order_id);
2779                }
2780            }
2781            _ => {}
2782        }
2783    }
2784
2785    /// Gets the node's environment.
2786    #[must_use]
2787    pub fn environment(&self) -> Environment {
2788        self.kernel.environment()
2789    }
2790
2791    /// Gets a reference to the underlying kernel.
2792    #[must_use]
2793    pub const fn kernel(&self) -> &NautilusKernel {
2794        &self.kernel
2795    }
2796
2797    /// Gets an exclusive reference to the underlying kernel.
2798    #[must_use]
2799    pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2800        &mut self.kernel
2801    }
2802
2803    /// Gets the node's trader ID.
2804    #[must_use]
2805    pub fn trader_id(&self) -> TraderId {
2806        self.kernel.trader_id()
2807    }
2808
2809    /// Gets the node's instance ID.
2810    #[must_use]
2811    pub const fn instance_id(&self) -> UUID4 {
2812        self.kernel.instance_id()
2813    }
2814
2815    /// Returns the current node state.
2816    #[must_use]
2817    pub fn state(&self) -> NodeState {
2818        self.handle.state()
2819    }
2820
2821    /// Checks if the live node is currently running.
2822    #[must_use]
2823    pub fn is_running(&self) -> bool {
2824        self.state().is_running()
2825    }
2826
2827    /// Sets the cache database adapter for persistence.
2828    ///
2829    /// This allows setting a database adapter (e.g., PostgreSQL, Redis) after the node
2830    /// is built but before it starts running. The database adapter is used to persist
2831    /// cache data for recovery and state management.
2832    ///
2833    /// # Errors
2834    ///
2835    /// Returns an error if the node is already running.
2836    pub fn set_cache_database(
2837        &mut self,
2838        database: Box<dyn CacheDatabaseAdapter>,
2839    ) -> anyhow::Result<()> {
2840        if self.state() != NodeState::Idle {
2841            anyhow::bail!(
2842                "Cannot set cache database while node is running, set it before running the node"
2843            );
2844        }
2845
2846        self.kernel.cache().borrow_mut().set_database(database);
2847        Ok(())
2848    }
2849
2850    /// Returns the execution manager.
2851    #[must_use]
2852    pub fn exec_manager(&self) -> &ExecutionManager {
2853        &self.exec_manager
2854    }
2855
2856    /// Returns a mutable reference to the execution manager.
2857    #[must_use]
2858    pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2859        &mut self.exec_manager
2860    }
2861
2862    /// Adds an actor to the trader.
2863    ///
2864    /// This method provides a high-level interface for adding actors to the underlying
2865    /// trader without requiring direct access to the kernel. Actors should be added
2866    /// after the node is built but before starting the node.
2867    ///
2868    /// # Errors
2869    ///
2870    /// Returns an error if:
2871    /// - The trader is not in a valid state for adding components.
2872    /// - An actor with the same ID is already registered.
2873    /// - The node is currently running.
2874    pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2875    where
2876        T: DataActor + DataActorNative + Component + Actor + 'static,
2877    {
2878        if self.state() != NodeState::Idle {
2879            anyhow::bail!(
2880                "Cannot add actor while node is running, add actors before running the node"
2881            );
2882        }
2883
2884        self.kernel.trader.borrow_mut().add_actor(actor)
2885    }
2886
2887    /// Adds an actor to the live node using a factory function.
2888    ///
2889    /// The factory function is called at registration time to create the actor,
2890    /// avoiding cloning issues with non-cloneable actor types.
2891    ///
2892    /// # Errors
2893    ///
2894    /// Returns an error if:
2895    /// - The node is currently running.
2896    /// - The factory function fails to create the actor.
2897    /// - The underlying trader registration fails.
2898    pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2899    where
2900        F: FnOnce() -> anyhow::Result<T>,
2901        T: DataActor + DataActorNative + Component + Actor + 'static,
2902    {
2903        if self.state() != NodeState::Idle {
2904            anyhow::bail!(
2905                "Cannot add actor while node is running, add actors before running the node"
2906            );
2907        }
2908
2909        self.kernel
2910            .trader
2911            .borrow_mut()
2912            .add_actor_from_factory(factory)
2913    }
2914
2915    /// Adds a strategy to the trader.
2916    ///
2917    /// Strategies are registered in both the component registry (for lifecycle management)
2918    /// and the actor registry (for data callbacks via msgbus).
2919    ///
2920    /// # Errors
2921    ///
2922    /// Returns an error if:
2923    /// - The node is currently running.
2924    /// - A strategy with the same ID is already registered.
2925    /// - The configured external order instrument IDs repeat an instrument or the cache already
2926    ///   contains a requested claim.
2927    /// - The strategy configures one or more external order instrument IDs and the cache is already
2928    ///   borrowed.
2929    /// - The strategy configures an OMS type override and the execution engine is already borrowed.
2930    pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2931    where
2932        T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2933    {
2934        if self.state() != NodeState::Idle {
2935            anyhow::bail!(
2936                "Cannot add strategy while node is running, add strategies before running the node"
2937            );
2938        }
2939
2940        // Capture strategy-owned values before adding the strategy, which moves it
2941        let strategy_id = self
2942            .kernel
2943            .trader
2944            .borrow()
2945            .prepare_strategy_for_registration(&mut strategy)?;
2946        let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
2947        let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
2948
2949        if !instrument_ids.is_empty() {
2950            self.register_external_order_claims(strategy_id, &instrument_ids)?;
2951        }
2952
2953        let mut exec_engine = match oms_type
2954            .map(|_| {
2955                self.kernel
2956                    .exec_engine
2957                    .try_borrow_mut()
2958                    .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
2959            })
2960            .transpose()
2961        {
2962            Ok(exec_engine) => exec_engine,
2963            Err(e) => {
2964                if !instrument_ids.is_empty()
2965                    && let Err(rollback_error) =
2966                        self.rollback_external_order_claims(strategy_id, &instrument_ids)
2967                {
2968                    anyhow::bail!(
2969                        "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
2970                    );
2971                }
2972
2973                return Err(e);
2974            }
2975        };
2976
2977        if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
2978            drop(exec_engine);
2979
2980            if !instrument_ids.is_empty()
2981                && let Err(rollback_error) =
2982                    self.rollback_external_order_claims(strategy_id, &instrument_ids)
2983            {
2984                anyhow::bail!(
2985                    "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
2986                );
2987            }
2988
2989            return Err(add_error);
2990        }
2991
2992        if let Some(exec_engine) = &mut exec_engine
2993            && let Some(oms_type) = oms_type
2994        {
2995            exec_engine.register_oms_type(strategy_id, oms_type);
2996        }
2997
2998        Ok(())
2999    }
3000
3001    /// Registers external order claims in the shared cache.
3002    ///
3003    /// It can be called while the node is idle, after manual [`start`](Self::start) returns, or
3004    /// after the node stops. A running strategy can update its own claims through
3005    /// `Strategy::set_external_order_instrument_ids`.
3006    ///
3007    /// # Errors
3008    ///
3009    /// Returns an error if the cache is already borrowed, the request repeats an instrument, or any
3010    /// requested instrument already has a claim. The cache remains unchanged.
3011    pub fn register_external_order_claims(
3012        &self,
3013        strategy_id: StrategyId,
3014        instrument_ids: &[InstrumentId],
3015    ) -> anyhow::Result<()> {
3016        self.kernel
3017            .cache
3018            .try_borrow_mut()
3019            .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3020            .register_external_order_claims(strategy_id, instrument_ids)?;
3021
3022        if !instrument_ids.is_empty() {
3023            log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3024        }
3025
3026        Ok(())
3027    }
3028
3029    /// Deregisters all external order claims owned by `strategy_id` from the shared cache.
3030    ///
3031    /// The operation is synchronous and can be called while the node is idle, after manual
3032    /// [`start`](Self::start) returns, or after the node stops. It cannot be called while
3033    /// [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) owns the node.
3034    ///
3035    /// # Errors
3036    ///
3037    /// Returns an error if the cache is already borrowed.
3038    pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3039        self.kernel
3040            .cache
3041            .try_borrow_mut()
3042            .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3043            .set_external_order_claims(strategy_id, &[])?;
3044
3045        Ok(())
3046    }
3047
3048    pub(crate) fn rollback_external_order_claims(
3049        &self,
3050        strategy_id: StrategyId,
3051        instrument_ids: &[InstrumentId],
3052    ) -> anyhow::Result<()> {
3053        let mut cache = self
3054            .kernel
3055            .cache
3056            .try_borrow_mut()
3057            .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3058        let retained: Vec<_> = cache
3059            .external_order_claim_instrument_ids(Some(strategy_id))
3060            .into_iter()
3061            .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3062            .collect();
3063        cache.set_external_order_claims(strategy_id, &retained)
3064    }
3065
3066    /// Adds an execution algorithm to the trader.
3067    ///
3068    /// Execution algorithms are registered in both the component registry (for lifecycle
3069    /// management) and the actor registry (for data callbacks via msgbus).
3070    ///
3071    /// # Errors
3072    ///
3073    /// Returns an error if:
3074    /// - The node is currently running.
3075    /// - An execution algorithm with the same ID is already registered.
3076    pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3077    where
3078        T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3079    {
3080        if self.state() != NodeState::Idle {
3081            anyhow::bail!(
3082                "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3083            );
3084        }
3085
3086        self.kernel
3087            .trader
3088            .borrow_mut()
3089            .add_exec_algorithm(exec_algorithm)
3090    }
3091
3092    // Runs reconciliation sub-checks, each gated by its own interval.
3093    // Continuous checks only schedule venue work; they do not await venue I/O
3094    // in the event loop.
3095}
3096
3097#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3098enum SocketReconnectDispatchOutcome {
3099    Accepted,
3100    AlreadyReconnecting,
3101    Disconnected,
3102    Closed,
3103    Unsupported,
3104    InvalidTrader,
3105    UnknownClient,
3106    UnknownEndpoint,
3107    AmbiguousEndpoint,
3108}
3109
3110fn record_runner_dispatch(
3111    metrics: &RunnerMetrics,
3112    channel: SystemChannel,
3113    dispatch_start: dst::time::Instant,
3114    metrics_start: dst::time::Instant,
3115) {
3116    let dispatch_end = dst::time::Instant::now();
3117    metrics.record_dispatch(
3118        channel,
3119        dispatch_end.duration_since(dispatch_start),
3120        dispatch_end.duration_since(metrics_start),
3121    );
3122}
3123
3124fn record_runner_maintenance(
3125    metrics: &RunnerMetrics,
3126    work_start: dst::time::Instant,
3127    metrics_start: dst::time::Instant,
3128) {
3129    let work_end = dst::time::Instant::now();
3130    metrics.record_maintenance(
3131        work_end.duration_since(work_start),
3132        work_end.duration_since(metrics_start),
3133    );
3134}
3135
3136fn record_runner_external_msgbus(
3137    metrics: &RunnerMetrics,
3138    work_start: dst::time::Instant,
3139    metrics_start: dst::time::Instant,
3140) {
3141    let work_end = dst::time::Instant::now();
3142    metrics.record_external_msgbus(
3143        work_end.duration_since(work_start),
3144        work_end.duration_since(metrics_start),
3145    );
3146}
3147
3148async fn drive_with_external_ingress<F, T>(
3149    future: F,
3150    receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3151    pending: &mut VecDeque<BusMessage>,
3152) -> T
3153where
3154    F: std::future::Future<Output = T>,
3155{
3156    tokio::pin!(future);
3157    let capacity = receiver
3158        .as_ref()
3159        .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3160
3161    loop {
3162        tokio::select! {
3163            biased;
3164
3165            result = &mut future => return result,
3166            message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3167                let Some(message) = message else {
3168                    log::info!("External message bus ingress closed");
3169                    *receiver = None;
3170                    continue;
3171                };
3172
3173                match msgbus::republish_pending_external_response(&message) {
3174                    Ok(true) => {},
3175                    Ok(false) | Err(_) => pending.push_back(message),
3176                }
3177            }
3178        }
3179    }
3180}
3181
3182async fn recv_external_msgbus_message(
3183    rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3184    pending: &mut VecDeque<BusMessage>,
3185) -> Option<BusMessage> {
3186    if let Some(message) = pending.pop_front() {
3187        return Some(message);
3188    }
3189
3190    match rx {
3191        Some(rx) => rx.recv().await,
3192        None => std::future::pending::<Option<BusMessage>>().await,
3193    }
3194}
3195
3196struct RunnerReceivers<'a> {
3197    time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3198    system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3199    system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3200    exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3201    exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3202    data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3203    data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3204}
3205
3206/// Flushes data events and commands from both `pending` and the channel receivers
3207/// into the cache, looping until no progress is made.
3208///
3209/// This closes the gap where `drive_with_event_buffering` exits as soon as its
3210/// driven future resolves (biased select), leaving items in the channel receivers
3211/// that were not captured into `pending`.
3212fn flush_pending_data(
3213    pending: &mut PendingEvents,
3214    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3215    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3216) {
3217    loop {
3218        let mut progressed = pending.drain_data();
3219
3220        while let Ok(evt) = data_evt_rx.try_recv() {
3221            AsyncRunner::dispatch_data_event(evt);
3222            progressed = true;
3223        }
3224
3225        while let Ok(cmd) = data_cmd_rx.try_recv() {
3226            AsyncRunner::handle_data_command(cmd);
3227            progressed = true;
3228        }
3229
3230        if !progressed {
3231            break;
3232        }
3233    }
3234}
3235
3236/// Flushes all channel receivers into `pending`, then drains everything.
3237///
3238/// Unlike [`flush_pending_data`] this is a single pass, not a drain-until-quiet
3239/// loop. Sufficient for phase 2 where the goal is to capture items the biased
3240/// select did not poll before the connect future resolved.
3241#[expect(
3242    clippy::too_many_arguments,
3243    reason = "all runner receivers are drained together"
3244)]
3245fn flush_all_pending(
3246    pending: &mut PendingEvents,
3247    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3248    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3249    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3250    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3251    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3252    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3253    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3254) {
3255    // Flush channel receivers into pending
3256    while let Ok(handler) = time_evt_rx.try_recv() {
3257        let _ = AsyncRunner::handle_time_event(handler);
3258    }
3259
3260    while let Ok(event) = system_evt_rx.try_recv() {
3261        pending.system_events.push(event);
3262    }
3263
3264    while let Ok(command) = system_cmd_rx.try_recv() {
3265        pending.system_commands.push(command);
3266    }
3267
3268    while let Ok(evt) = data_evt_rx.try_recv() {
3269        pending.data_evts.push(evt);
3270    }
3271
3272    while let Ok(cmd) = data_cmd_rx.try_recv() {
3273        pending.data_cmds.push(cmd);
3274    }
3275
3276    while let Ok(evt) = exec_evt_rx.try_recv() {
3277        pending.push_exec_event(evt);
3278    }
3279
3280    while let Ok(cmd) = exec_cmd_rx.try_recv() {
3281        pending.exec_cmds.push(cmd);
3282    }
3283
3284    pending.drain();
3285}
3286
3287/// Drives a future to completion while buffering channel events.
3288///
3289/// Time events are handled immediately. Account events are forwarded directly.
3290/// All other events are buffered in `pending` for later processing.
3291#[expect(
3292    clippy::too_many_arguments,
3293    reason = "startup buffering owns one future plus the pending state and all runner receivers"
3294)]
3295async fn drive_with_event_buffering<F: std::future::Future>(
3296    future: F,
3297    pending: &mut PendingEvents,
3298    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3299    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3300    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3301    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3302    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3303    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3304    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3305) -> F::Output {
3306    tokio::pin!(future);
3307
3308    loop {
3309        tokio::select! {
3310            biased;
3311
3312            result = &mut future => {
3313                break result;
3314            }
3315            Some(handler) = time_evt_rx.recv() => {
3316                let _ = AsyncRunner::handle_time_event(handler);
3317            }
3318            Some(event) = system_evt_rx.recv() => {
3319                pending.system_events.push(event);
3320            }
3321            Some(command) = system_cmd_rx.recv() => {
3322                pending.system_commands.push(command);
3323            }
3324            Some(evt) = exec_evt_rx.recv() => {
3325                pending.push_exec_event(evt);
3326            }
3327            Some(cmd) = exec_cmd_rx.recv() => {
3328                pending.exec_cmds.push(cmd);
3329            }
3330            Some(evt) = data_evt_rx.recv() => {
3331                pending.data_evts.push(evt);
3332            }
3333            Some(cmd) = data_cmd_rx.recv() => {
3334                pending.data_cmds.push(cmd);
3335            }
3336        }
3337    }
3338}
3339
3340#[derive(Default)]
3341struct PendingEvents {
3342    system_events: Vec<DispatchMessage<SystemEvent>>,
3343    system_commands: Vec<DispatchMessage<SystemCommand>>,
3344    data_evts: Vec<DispatchMessage<DataEvent>>,
3345    data_cmds: Vec<DispatchMessage<DataCommand>>,
3346    exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3347    order_evts: Vec<DispatchMessage<OrderEventAny>>,
3348    exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3349}
3350
3351impl PendingEvents {
3352    fn is_empty(&self) -> bool {
3353        self.system_events.is_empty()
3354            && self.system_commands.is_empty()
3355            && self.data_evts.is_empty()
3356            && self.data_cmds.is_empty()
3357            && self.exec_reports.is_empty()
3358            && self.order_evts.is_empty()
3359            && self.exec_cmds.is_empty()
3360    }
3361
3362    /// Drains only data events and commands into the cache.
3363    ///
3364    /// Returns `true` if any events or commands were drained.
3365    fn drain_data(&mut self) -> bool {
3366        let total = self.data_evts.len() + self.data_cmds.len();
3367
3368        if total > 0 {
3369            log::debug!(
3370                "Draining {total} data events/commands into cache \
3371                 (data_evts={}, data_cmds={})",
3372                self.data_evts.len(),
3373                self.data_cmds.len(),
3374            );
3375        }
3376
3377        for evt in self.data_evts.drain(..) {
3378            AsyncRunner::dispatch_data_event(evt);
3379        }
3380
3381        for cmd in self.data_cmds.drain(..) {
3382            AsyncRunner::handle_data_command(cmd);
3383        }
3384
3385        total > 0
3386    }
3387
3388    /// Drains all remaining pending events.
3389    fn drain(&mut self) {
3390        let total = self.data_evts.len()
3391            + self.data_cmds.len()
3392            + self.exec_reports.len()
3393            + self.order_evts.len()
3394            + self.exec_cmds.len();
3395
3396        if total > 0 {
3397            log::debug!(
3398                "Processing {total} events/commands queued during startup \
3399                 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3400                self.data_evts.len(),
3401                self.data_cmds.len(),
3402                self.exec_reports.len(),
3403                self.order_evts.len(),
3404                self.exec_cmds.len()
3405            );
3406        }
3407
3408        for evt in self.data_evts.drain(..) {
3409            AsyncRunner::dispatch_data_event(evt);
3410        }
3411
3412        for cmd in self.data_cmds.drain(..) {
3413            AsyncRunner::handle_data_command(cmd);
3414        }
3415
3416        for report in self.exec_reports.drain(..) {
3417            report
3418                .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3419        }
3420
3421        for evt in self.order_evts.drain(..) {
3422            evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3423        }
3424
3425        for cmd in self.exec_cmds.drain(..) {
3426            AsyncRunner::handle_trading_command(cmd);
3427        }
3428    }
3429
3430    fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3431        let rooted = event.is_rooted();
3432        event.dispatch(|event| {
3433            let owner = std::thread::current().id();
3434
3435            let order = |event| {
3436                if rooted {
3437                    DispatchMessage::new(event, owner)
3438                } else {
3439                    DispatchMessage::from(event)
3440                }
3441            };
3442
3443            // Account events are safe to process immediately. Reports and orders need
3444            // ExecEngine access, which may conflict with the driven startup future's borrow.
3445            match event {
3446                ExecutionEvent::Account(_) => AsyncRunner::handle_exec_event(event),
3447                ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3448                    DispatchMessage::new(report, owner)
3449                } else {
3450                    report.into()
3451                }),
3452                ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3453                ExecutionEvent::OrderSubmittedBatch(batch) => {
3454                    for event in batch {
3455                        self.order_evts.push(order(OrderEventAny::Submitted(event)));
3456                    }
3457                }
3458                ExecutionEvent::OrderAcceptedBatch(batch) => {
3459                    for event in batch {
3460                        self.order_evts.push(order(OrderEventAny::Accepted(event)));
3461                    }
3462                }
3463                ExecutionEvent::OrderCanceledBatch(batch) => {
3464                    for event in batch {
3465                        self.order_evts.push(order(OrderEventAny::Canceled(event)));
3466                    }
3467                }
3468            }
3469        });
3470    }
3471
3472    fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3473        std::mem::take(&mut self.system_events)
3474    }
3475
3476    fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3477        std::mem::take(&mut self.system_commands)
3478    }
3479}
3480
3481struct ClientStatus {
3482    client: String,
3483    client_type: &'static str,
3484    connected: bool,
3485}
3486
3487fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3488    let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3489    builder.push_record(["Client", "Type", "Connected"]);
3490
3491    for row in rows {
3492        builder.push_record([
3493            row.client,
3494            row.client_type.to_string(),
3495            row.connected.to_string(),
3496        ]);
3497    }
3498
3499    builder.build().with(Style::rounded()).to_string()
3500}
3501
3502#[cfg(test)]
3503mod tests {
3504    mod report_acceptance;
3505    mod shutdown;
3506
3507    use std::{
3508        cell::{Cell, RefCell},
3509        fmt::Debug,
3510        rc::Rc,
3511        sync::{
3512            Arc,
3513            atomic::{AtomicBool, Ordering},
3514        },
3515    };
3516
3517    use bytes::Bytes;
3518    use indexmap::{IndexMap, IndexSet};
3519    use log::{Level, LevelFilter, Log, Metadata, Record};
3520    #[cfg(feature = "python")]
3521    use nautilus_common::runner::{
3522        SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3523    };
3524    use nautilus_common::{
3525        actor::{
3526            self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3527        },
3528        cache::Cache,
3529        clock::{Clock, VirtualClock},
3530        enums::SerializationEncoding,
3531        live::{
3532            runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3533            sender::DispatchSender,
3534        },
3535        logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3536        messages::{
3537            data::{SubscribeCommand, SubscribeQuotes},
3538            execution::{
3539                GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3540                GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3541                TradingCommand,
3542            },
3543            system::{
3544                QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3545            },
3546        },
3547        msgbus::{
3548            self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3549            MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3550            MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3551        },
3552        nautilus_actor,
3553        runner::{SyncTradingCommandSender, TradingCommandSender},
3554        testing::wait_until_async,
3555        timer::{TimeEvent, TimeEventCallback},
3556    };
3557    use nautilus_core::{Params, UUID4, UnixNanos};
3558    use nautilus_execution::{
3559        engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3560        reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3561    };
3562    use nautilus_model::{
3563        accounts::{AccountAny, MarginAccount},
3564        data::QuoteTick,
3565        enums::{
3566            AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3567            TimeInForce,
3568        },
3569        events::{
3570            AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3571            order::spec::{
3572                OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3573            },
3574        },
3575        identifiers::{
3576            AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3577            TradeId, TraderId, Venue, VenueOrderId,
3578        },
3579        instruments::{
3580            Instrument, InstrumentAny,
3581            stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3582        },
3583        orders::{
3584            Order, OrderAny, OrderList, OrderTestBuilder,
3585            stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3586        },
3587        position::PositionReplayEvent,
3588        reports::{FillReport, PositionStatusReport},
3589        types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3590    };
3591    use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3592    use nautilus_testkit::{
3593        cache::TestCacheDatabaseControl,
3594        components::{StateActor, StateStrategy},
3595    };
3596    use nautilus_trading::{
3597        nautilus_strategy,
3598        strategy::{config::StrategyConfig, core::StrategyCore},
3599    };
3600    use parking_lot::Mutex;
3601    use rstest::*;
3602    use rust_decimal::Decimal;
3603    use rust_decimal_macros::dec;
3604    use ustr::Ustr;
3605
3606    use super::{
3607        reconciliation::{
3608            PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3609            request_position_fill_reports,
3610        },
3611        *,
3612    };
3613    use crate::{
3614        execution::{
3615            client::LiveExecutionClient,
3616            manager::{PositionFillReportQuery, ReportClientCoverage},
3617            submission::{
3618                SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3619            },
3620        },
3621        socket::SocketControl,
3622    };
3623
3624    struct ExternalIngressLogCapture {
3625        messages: Mutex<Vec<String>>,
3626    }
3627
3628    static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3629        messages: Mutex::new(Vec::new()),
3630    };
3631
3632    #[derive(Debug)]
3633    enum FillReportClientOutcome {
3634        Reports(Vec<FillReport>),
3635        Failure,
3636    }
3637
3638    #[derive(Debug)]
3639    struct FillReportClient {
3640        client_id: ClientId,
3641        account_id: AccountId,
3642        venue: Venue,
3643        outcome: FillReportClientOutcome,
3644        commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3645    }
3646
3647    #[async_trait::async_trait(?Send)]
3648    impl ExecutionClient for FillReportClient {
3649        fn is_connected(&self) -> bool {
3650            true
3651        }
3652
3653        fn client_id(&self) -> ClientId {
3654            self.client_id
3655        }
3656
3657        fn account_id(&self) -> AccountId {
3658            self.account_id
3659        }
3660
3661        fn venue(&self) -> Venue {
3662            self.venue
3663        }
3664
3665        fn oms_type(&self) -> OmsType {
3666            OmsType::Netting
3667        }
3668
3669        fn get_account(&self) -> Option<AccountAny> {
3670            None
3671        }
3672
3673        fn generate_account_state(
3674            &self,
3675            _balances: Vec<AccountBalance>,
3676            _margins: Vec<MarginBalance>,
3677            _reported: bool,
3678            _ts_event: UnixNanos,
3679            _info: Option<Params>,
3680        ) -> anyhow::Result<()> {
3681            Ok(())
3682        }
3683
3684        fn start(&mut self) -> anyhow::Result<()> {
3685            Ok(())
3686        }
3687
3688        fn stop(&mut self) -> anyhow::Result<()> {
3689            Ok(())
3690        }
3691
3692        async fn generate_order_status_reports(
3693            &self,
3694            _cmd: &GenerateOrderStatusReports,
3695        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3696            match self.outcome {
3697                FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3698                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3699            }
3700        }
3701
3702        async fn generate_position_status_reports(
3703            &self,
3704            _cmd: &GeneratePositionStatusReports,
3705        ) -> anyhow::Result<Vec<PositionStatusReport>> {
3706            match self.outcome {
3707                FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3708                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3709            }
3710        }
3711
3712        async fn generate_fill_reports(
3713            &self,
3714            cmd: GenerateFillReports,
3715        ) -> anyhow::Result<Vec<FillReport>> {
3716            self.commands.borrow_mut().push(cmd);
3717
3718            match &self.outcome {
3719                FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3720                FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3721            }
3722        }
3723    }
3724
3725    #[derive(Debug)]
3726    struct OrderReportClient {
3727        client_id: ClientId,
3728        report: OrderStatusReport,
3729        targeted: bool,
3730        queries: Rc<RefCell<Vec<&'static str>>>,
3731    }
3732
3733    #[async_trait::async_trait(?Send)]
3734    impl ExecutionClient for OrderReportClient {
3735        fn is_connected(&self) -> bool {
3736            true
3737        }
3738
3739        fn client_id(&self) -> ClientId {
3740            self.client_id
3741        }
3742
3743        fn account_id(&self) -> AccountId {
3744            self.report.account_id
3745        }
3746
3747        fn venue(&self) -> Venue {
3748            self.report.instrument_id.venue
3749        }
3750
3751        fn oms_type(&self) -> OmsType {
3752            OmsType::Netting
3753        }
3754
3755        fn get_account(&self) -> Option<AccountAny> {
3756            None
3757        }
3758
3759        fn generate_account_state(
3760            &self,
3761            _balances: Vec<AccountBalance>,
3762            _margins: Vec<MarginBalance>,
3763            _reported: bool,
3764            _ts_event: UnixNanos,
3765            _info: Option<Params>,
3766        ) -> anyhow::Result<()> {
3767            Ok(())
3768        }
3769
3770        fn start(&mut self) -> anyhow::Result<()> {
3771            Ok(())
3772        }
3773
3774        fn stop(&mut self) -> anyhow::Result<()> {
3775            Ok(())
3776        }
3777
3778        async fn generate_order_status_reports(
3779            &self,
3780            _command: &GenerateOrderStatusReports,
3781        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3782            self.queries.borrow_mut().push("bulk");
3783            Ok(if self.targeted {
3784                Vec::new()
3785            } else {
3786                vec![self.report.clone()]
3787            })
3788        }
3789
3790        async fn generate_order_status_report(
3791            &self,
3792            _command: &GenerateOrderStatusReport,
3793        ) -> anyhow::Result<Option<OrderStatusReport>> {
3794            self.queries.borrow_mut().push("targeted");
3795            Ok(Some(self.report.clone()))
3796        }
3797    }
3798
3799    #[derive(Debug)]
3800    struct StartupSocketActor {
3801        core: DataActorCore,
3802        received: Rc<RefCell<Vec<SocketStateChanged>>>,
3803    }
3804
3805    impl StartupSocketActor {
3806        fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3807            Self {
3808                core: DataActorCore::new(DataActorConfig {
3809                    actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3810                    ..Default::default()
3811                }),
3812                received,
3813            }
3814        }
3815    }
3816
3817    impl DataActor for StartupSocketActor {
3818        fn on_start(&mut self) -> anyhow::Result<()> {
3819            self.subscribe_socket_state(None, None, None);
3820            Ok(())
3821        }
3822
3823        fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3824            self.received.borrow_mut().push(event.clone());
3825            Ok(())
3826        }
3827    }
3828
3829    nautilus_actor!(StartupSocketActor);
3830
3831    impl Log for ExternalIngressLogCapture {
3832        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3833            metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3834        }
3835
3836        fn log(&self, record: &Record<'_>) {
3837            if self.enabled(record.metadata()) {
3838                self.messages.lock().push(record.args().to_string());
3839            }
3840        }
3841
3842        fn flush(&self) {}
3843    }
3844
3845    #[derive(Debug)]
3846    struct FailingTimerActor {
3847        core: DataActorCore,
3848        received: Rc<RefCell<Vec<u64>>>,
3849    }
3850
3851    nautilus_actor!(FailingTimerActor);
3852
3853    impl DataActor for FailingTimerActor {
3854        fn on_start(&mut self) -> anyhow::Result<()> {
3855            for timestamp in [17, 23] {
3856                let received = self.received.clone();
3857
3858                let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3859                    received.borrow_mut().push(timestamp);
3860
3861                    if timestamp == 17 {
3862                        crate::dispatch::tests::latch_callback_failure();
3863                    }
3864                }));
3865
3866                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3867                    TimeEvent::new(
3868                        "callback-failure".into(),
3869                        UUID4::new(),
3870                        timestamp.into(),
3871                        timestamp.into(),
3872                    ),
3873                    callback,
3874                ));
3875            }
3876
3877            Ok(())
3878        }
3879    }
3880
3881    #[rstest]
3882    #[tokio::test]
3883    async fn test_callback_failure_stops_later_live_events(
3884        #[values(false, true)] incomplete_submission: bool,
3885    ) {
3886        actor::clear_callbacks().unwrap();
3887
3888        let config = LiveNodeConfig {
3889            exec_engine: crate::config::LiveExecutionEngineConfig {
3890                reconciliation: false,
3891                submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3892                ..Default::default()
3893            },
3894            timeout_connection: Duration::ZERO,
3895            timeout_reconciliation: Duration::ZERO,
3896            timeout_portfolio: Duration::ZERO,
3897            timeout_disconnection: Duration::ZERO,
3898            delay_post_stop: Duration::ZERO,
3899            timeout_shutdown: Duration::ZERO,
3900            ..Default::default()
3901        };
3902
3903        let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3904
3905        if incomplete_submission {
3906            let order = OrderTestBuilder::new(OrderType::Market)
3907                .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3908                .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3909                .quantity(Quantity::from("1"))
3910                .build();
3911            node.exec_manager
3912                .register_submission(order.init_event(), None);
3913        }
3914        let received = Rc::new(RefCell::new(Vec::new()));
3915        node.add_actor(FailingTimerActor {
3916            core: DataActorCore::new(DataActorConfig {
3917                actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3918                ..Default::default()
3919            }),
3920            received: received.clone(),
3921        })
3922        .unwrap();
3923
3924        let result = node.run_with_mode(NodeRunMode::Hosted).await;
3925
3926        let error = result.unwrap_err();
3927        assert!(error.to_string().contains("Callback delivery unwound"));
3928        if incomplete_submission {
3929            assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3930        }
3931        let state = node.state();
3932        let trader_stopped = node.kernel.trader.borrow().is_stopped();
3933        let failure = actor::callback_failure();
3934        node.dispose();
3935
3936        assert_eq!(
3937            error.downcast_ref::<CallbackDispatchError>(),
3938            Some(&CallbackDispatchError::DeliveryUnwound)
3939        );
3940        assert_eq!(*received.borrow(), [17]);
3941        assert_eq!(state, NodeState::Stopped);
3942        assert!(trader_stopped);
3943        assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
3944        assert_eq!(actor::callback_failure(), None);
3945        assert_eq!(actor::clear_callbacks(), Ok(()));
3946    }
3947
3948    #[rstest]
3949    fn test_render_client_statuses() {
3950        let rows = vec![
3951            ClientStatus {
3952                client: "BINANCE".to_string(),
3953                client_type: "Data",
3954                connected: true,
3955            },
3956            ClientStatus {
3957                client: "SIM".to_string(),
3958                client_type: "Execution",
3959                connected: false,
3960            },
3961        ];
3962
3963        let output = render_client_statuses(rows);
3964        let expected = "╭─────────┬───────────┬───────────╮\n\
3965│ Client  │ Type      │ Connected │\n\
3966├─────────┼───────────┼───────────┤\n\
3967│ BINANCE │ Data      │ true      │\n\
3968│ SIM     │ Execution │ false     │\n\
3969╰─────────┴───────────┴───────────╯";
3970
3971        assert_eq!(output, expected);
3972    }
3973
3974    #[rstest]
3975    fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
3976        log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
3977        log::set_max_level(LevelFilter::Error);
3978        EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
3979        let message = BusMessage::with_str_topic(
3980            "data.quotes.AUDUSD.SIM*",
3981            BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
3982            Bytes::new(),
3983            SerializationEncoding::Json,
3984        );
3985
3986        LiveNode::republish_external_msgbus_message(&message);
3987
3988        assert_eq!(
3989            *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
3990            vec![
3991                "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
3992                 external message topic: Topic `value` contained invalid characters, was \
3993                 data.quotes.AUDUSD.SIM*"
3994                    .to_string()
3995            ],
3996        );
3997    }
3998
3999    #[rstest]
4000    #[tokio::test]
4001    async fn test_external_startup_prioritizes_completed_connection() {
4002        let message = BusMessage::with_str_topic(
4003            "data.trades.FIRST",
4004            BusPayloadType::TradeTick,
4005            bytes::Bytes::new(),
4006            SerializationEncoding::Json,
4007        );
4008        let (sender, receiver) = tokio::sync::mpsc::channel(1);
4009        sender.send(message.clone()).await.unwrap();
4010        let mut receiver = Some(receiver);
4011        let mut pending = VecDeque::new();
4012
4013        let result =
4014            drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4015        let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4016
4017        assert_eq!(result, 17);
4018        assert!(pending.is_empty());
4019        assert_eq!(
4020            serde_json::to_value(queued).unwrap(),
4021            serde_json::to_value(message).unwrap()
4022        );
4023    }
4024
4025    #[rstest]
4026    #[tokio::test]
4027    async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4028        use bytes::Bytes;
4029        use nautilus_common::messages::data::{
4030            CustomDataResponse, DataResponse, DataResponseEnvelope,
4031        };
4032        use nautilus_model::data::DataType;
4033
4034        let (egress, publications, _) = CapturingExternalEgress::new();
4035        let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4036        msgbus::set_message_bus(bus.clone());
4037        bus.borrow_mut()
4038            .set_external_egress_config(
4039                Box::new(egress),
4040                &MessageBusConfig {
4041                    encoding: SerializationEncoding::MsgPack,
4042                    external_streams: Some(vec!["external.test".into()]),
4043                    ..Default::default()
4044                },
4045            )
4046            .unwrap();
4047
4048        bus.borrow_mut()
4049            .add_streaming_type(BusPayloadType::DataResponse);
4050        let id = UUID4::new();
4051        let response = DataResponse::Data(CustomDataResponse::new(
4052            id,
4053            "FEED".into(),
4054            None,
4055            DataType::new("Startup", None, None),
4056            Bytes::from_static(b"connected"),
4057            Some(101.into()),
4058            Some(103.into()),
4059            107.into(),
4060            None,
4061        ));
4062        msgbus::publish_any("responses.data.FEED".into(), &response);
4063        let message = BusMessage::with_str_topic(
4064            "responses.data.FEED",
4065            BusPayloadType::DataResponse,
4066            publications.borrow()[0].payload.clone(),
4067            SerializationEncoding::MsgPack,
4068        );
4069        let (done, connected) = tokio::sync::oneshot::channel();
4070        let done = Rc::new(RefCell::new(Some(done)));
4071        msgbus::register_response_handler(
4072            &id,
4073            TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4074                let Some(DataResponse::Data(reply)) = envelope.response() else {
4075                    panic!("expected custom reply");
4076                };
4077
4078                done.borrow_mut()
4079                    .take()
4080                    .unwrap()
4081                    .send(reply.clone())
4082                    .unwrap();
4083            }),
4084        );
4085
4086        msgbus::subscribe_any(
4087            message.topic.as_str().into(),
4088            ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4089                msgbus::send_response(reply.correlation_id(), reply);
4090            }),
4091            None,
4092        );
4093
4094        let unrelated = BusMessage::with_str_topic(
4095            "data.trades.FIRST",
4096            BusPayloadType::TradeTick,
4097            Bytes::new(),
4098            SerializationEncoding::Json,
4099        );
4100        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4101        sender.send(unrelated.clone()).await.unwrap();
4102        sender.send(message).await.unwrap();
4103        let mut receiver = Some(receiver);
4104        let mut pending = VecDeque::new();
4105        let actual = tokio::time::timeout(
4106            Duration::from_secs(1),
4107            drive_with_external_ingress(connected, &mut receiver, &mut pending),
4108        )
4109        .await
4110        .unwrap()
4111        .unwrap();
4112
4113        assert_eq!(actual.correlation_id, id);
4114        assert_eq!(actual.client_id, ClientId::from("FEED"));
4115        assert_eq!(actual.venue, None);
4116        assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4117        assert_eq!(
4118            actual.data.downcast_ref::<Bytes>(),
4119            Some(&Bytes::from_static(b"connected"))
4120        );
4121        assert_eq!(actual.start, Some(101.into()));
4122        assert_eq!(actual.end, Some(103.into()));
4123        assert_eq!(actual.ts_init, UnixNanos::from(107));
4124        assert_eq!(actual.params, None);
4125        assert_eq!(publications.borrow().len(), 1);
4126        assert_eq!(pending.len(), 1);
4127        assert_eq!(
4128            serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4129            serde_json::to_value(unrelated).unwrap()
4130        );
4131        assert!(receiver.as_ref().unwrap().is_empty());
4132    }
4133
4134    #[rstest]
4135    #[tokio::test]
4136    async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4137        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4138        let mut receiver = Some(receiver);
4139
4140        let message = |name: &str| {
4141            BusMessage::with_str_topic(
4142                name,
4143                BusPayloadType::TradeTick,
4144                bytes::Bytes::new(),
4145                SerializationEncoding::Json,
4146            )
4147        };
4148
4149        let first = message("data.trades.FIRST");
4150        let second = message("data.trades.SECOND");
4151        let third = message("data.trades.THIRD");
4152        sender.send(first.clone()).await.unwrap();
4153        sender.send(second.clone()).await.unwrap();
4154        let (done, connected) = tokio::sync::oneshot::channel();
4155        let mut pending = VecDeque::new();
4156        let mut connection = Box::pin(drive_with_external_ingress(
4157            connected,
4158            &mut receiver,
4159            &mut pending,
4160        ));
4161        assert!(futures_util::poll!(&mut connection).is_pending());
4162        sender.send(third.clone()).await.unwrap();
4163        assert!(futures_util::poll!(&mut connection).is_pending());
4164        done.send(()).unwrap();
4165        connection.await.unwrap();
4166
4167        assert_eq!(pending.len(), 2);
4168        assert_eq!(receiver.as_ref().unwrap().len(), 1);
4169
4170        for expected in [first, second, third] {
4171            let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4172                .await
4173                .unwrap();
4174            assert_eq!(
4175                serde_json::to_value(received).unwrap(),
4176                serde_json::to_value(expected).unwrap()
4177            );
4178        }
4179    }
4180
4181    #[rstest]
4182    #[tokio::test]
4183    async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4184        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4185        drop(sender);
4186        let mut receiver = Some(receiver);
4187        let mut pending = VecDeque::new();
4188        let (done, connected) = tokio::sync::oneshot::channel();
4189        let mut connection = Box::pin(drive_with_external_ingress(
4190            connected,
4191            &mut receiver,
4192            &mut pending,
4193        ));
4194        assert!(futures_util::poll!(&mut connection).is_pending());
4195        done.send(()).unwrap();
4196        connection.await.unwrap();
4197
4198        assert!(receiver.is_none());
4199        assert!(pending.is_empty());
4200    }
4201
4202    #[rstest]
4203    #[case(None, 2)]
4204    #[case(Some(SystemChannel::DataEvents), 1)]
4205    #[case(Some(SystemChannel::ExecCommands), 1)]
4206    fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4207        #[case] channel: Option<SystemChannel>,
4208        #[case] expected_count: usize,
4209    ) {
4210        let config = LiveNodeConfig {
4211            trader_id: TraderId::from("QUEUE-001"),
4212            exec_engine: crate::config::LiveExecutionEngineConfig {
4213                reconciliation: false,
4214                ..Default::default()
4215            },
4216            ..Default::default()
4217        };
4218
4219        let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4220        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4221
4222        let handler = ShareableMessageHandler::from_typed({
4223            let received = received.clone();
4224            move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4225        });
4226
4227        msgbus::subscribe_any(
4228            MessagingSwitchboard::queue_state_changed_pattern(channel),
4229            handler,
4230            None,
4231        );
4232
4233        let transitions = [
4234            QueueStateTransition {
4235                channel: SystemChannel::DataEvents,
4236                condition: QueueCondition::Backlogged,
4237                state: QueueState::Triggered,
4238                queue_depth: 17,
4239                mean_dispatch_ns: 23,
4240            },
4241            QueueStateTransition {
4242                channel: SystemChannel::ExecCommands,
4243                condition: QueueCondition::Slow,
4244                state: QueueState::Triggered,
4245                queue_depth: 17,
4246                mean_dispatch_ns: 23,
4247            },
4248        ];
4249
4250        node.publish_queue_state_transitions(&transitions);
4251
4252        let events = received.borrow();
4253        assert_eq!(events.len(), expected_count);
4254
4255        let expected = transitions
4256            .into_iter()
4257            .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4258
4259        for (event, transition) in events.iter().zip(expected) {
4260            assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4261            assert_eq!(event.channel, transition.channel);
4262            assert_eq!(event.condition, transition.condition);
4263            assert_eq!(event.state, transition.state);
4264            assert_eq!(event.queue_depth, transition.queue_depth);
4265            assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4266            assert_ne!(event.event_id, UUID4::default());
4267            assert_ne!(event.ts_event, UnixNanos::default());
4268            assert_eq!(event.ts_init, event.ts_event);
4269        }
4270
4271        if channel.is_none() {
4272            assert_ne!(events[0].event_id, events[1].event_id);
4273        }
4274
4275        drop(events);
4276        msgbus::get_message_bus().borrow_mut().dispose();
4277    }
4278
4279    #[rstest]
4280    #[case(None, None)]
4281    #[case(Some(ClientId::from("BINANCE")), None)]
4282    #[case(None, Some("binance-futures-market-streams"))]
4283    #[case(
4284        Some(ClientId::from("BINANCE")),
4285        Some("binance-futures-market-streams")
4286    )]
4287    fn test_process_socket_state_change_reaches_typed_subscriber(
4288        #[case] client_id: Option<ClientId>,
4289        #[case] endpoint: Option<&str>,
4290    ) {
4291        let config = LiveNodeConfig {
4292            trader_id: TraderId::from("SOCKET-001"),
4293            exec_engine: crate::config::LiveExecutionEngineConfig {
4294                reconciliation: false,
4295                ..Default::default()
4296            },
4297            ..Default::default()
4298        };
4299
4300        let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4301        let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4302
4303        let handler = ShareableMessageHandler::from_typed({
4304            let received = received.clone();
4305            move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4306        });
4307
4308        msgbus::subscribe_any(
4309            MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4310            handler,
4311            None,
4312        );
4313
4314        let change = SocketStateChange::new(
4315            ClientId::from("BINANCE"),
4316            Some(Venue::from("BINANCE")),
4317            Ustr::from("binance-futures-market-streams"),
4318            SocketState::Disconnected,
4319        );
4320
4321        node.process_system_event(SystemEvent::SocketState(change));
4322
4323        let events = received.borrow();
4324        assert_eq!(events.len(), 1);
4325        assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4326        assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4327        assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4328        assert_eq!(
4329            events[0].endpoint,
4330            Ustr::from("binance-futures-market-streams")
4331        );
4332        assert_eq!(events[0].state, SocketState::Disconnected);
4333        assert_ne!(events[0].event_id, UUID4::default());
4334        assert_ne!(events[0].ts_event, UnixNanos::default());
4335        assert_eq!(events[0].ts_init, events[0].ts_event);
4336        drop(events);
4337        msgbus::get_message_bus().borrow_mut().dispose();
4338    }
4339
4340    #[rstest]
4341    #[case::accepted(
4342        ReconnectRequestOutcome::Accepted,
4343        SocketReconnectDispatchOutcome::Accepted
4344    )]
4345    #[case::already_reconnecting(
4346        ReconnectRequestOutcome::AlreadyReconnecting,
4347        SocketReconnectDispatchOutcome::AlreadyReconnecting
4348    )]
4349    #[case::disconnected(
4350        ReconnectRequestOutcome::Disconnected,
4351        SocketReconnectDispatchOutcome::Disconnected
4352    )]
4353    #[case::closed(
4354        ReconnectRequestOutcome::Closed,
4355        SocketReconnectDispatchOutcome::Closed
4356    )]
4357    #[case::unsupported(
4358        ReconnectRequestOutcome::Unsupported,
4359        SocketReconnectDispatchOutcome::Unsupported
4360    )]
4361    fn test_request_socket_reconnect_maps_transport_outcome(
4362        #[case] transport: ReconnectRequestOutcome,
4363        #[case] expected: SocketReconnectDispatchOutcome,
4364    ) {
4365        let registry = SocketReconnectRegistry::default();
4366        let client_id = ClientId::from("TEST");
4367        let endpoint = Ustr::from("test-streams");
4368        let control = SocketControl::with_registry(client_id, None, endpoint, &registry);
4369        let _sink = control.sink();
4370        control.register(move || transport);
4371
4372        let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4373
4374        assert_eq!(outcome, expected);
4375    }
4376
4377    #[rstest]
4378    #[case::client_not_found(
4379        SocketReconnectLookup::ClientNotFound,
4380        SocketReconnectDispatchOutcome::UnknownClient
4381    )]
4382    #[case::unsupported(
4383        SocketReconnectLookup::Unsupported,
4384        SocketReconnectDispatchOutcome::Unsupported
4385    )]
4386    #[case::endpoint_not_found(
4387        SocketReconnectLookup::EndpointNotFound,
4388        SocketReconnectDispatchOutcome::UnknownEndpoint
4389    )]
4390    #[case::ambiguous(
4391        SocketReconnectLookup::AmbiguousEndpoint,
4392        SocketReconnectDispatchOutcome::AmbiguousEndpoint
4393    )]
4394    fn test_request_socket_reconnect_maps_lookup_failure(
4395        #[case] lookup: SocketReconnectLookup,
4396        #[case] expected: SocketReconnectDispatchOutcome,
4397    ) {
4398        assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4399    }
4400
4401    #[rstest]
4402    fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4403        let trader_id = TraderId::from("SOCKET-001");
4404
4405        let config = LiveNodeConfig {
4406            trader_id,
4407            ..Default::default()
4408        };
4409
4410        let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4411        let client_id = ClientId::from("TEST");
4412        let endpoint = Ustr::from("test-streams");
4413        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4414        let sender = DispatchSender::new(tx);
4415        let events = sender.clone();
4416        msgbus::subscribe_any(
4417            MessagingSwitchboard::socket_state_changed_pattern(
4418                Some(client_id),
4419                Some(endpoint.as_str()),
4420            ),
4421            ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4422                events.send(17u32).unwrap();
4423            }),
4424            None,
4425        );
4426
4427        let control =
4428            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4429        let _sink = control.sink();
4430        control.register(move || {
4431            sender.send(23u32).unwrap();
4432            ReconnectRequestOutcome::Accepted
4433        });
4434
4435        let event = SystemEvent::SocketState(SocketStateChange::new(
4436            client_id,
4437            None,
4438            endpoint,
4439            SocketState::Connected,
4440        ));
4441        let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4442            trader_id,
4443            client_id,
4444            endpoint,
4445            31.into(),
4446        ));
4447
4448        if startup {
4449            let mut pending = PendingEvents::default();
4450            pending.system_events.push(event.into());
4451            pending.system_commands.push(command.into());
4452            node.process_system_events(pending.take_system_events());
4453            node.process_system_commands(pending.take_system_commands());
4454            assert!(pending.is_empty());
4455        } else {
4456            node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4457            node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4458        }
4459
4460        let event_child = rx.try_recv().unwrap();
4461        let command_child = rx.try_recv().unwrap();
4462        assert!(event_child.is_rooted());
4463        assert!(command_child.is_rooted());
4464        assert_eq!(event_child.dispatch(|value| value), 17);
4465        assert_eq!(command_child.dispatch(|value| value), 23);
4466        assert!(rx.is_empty());
4467        msgbus::get_message_bus().borrow_mut().dispose();
4468    }
4469
4470    #[rstest]
4471    fn test_process_socket_reconnect_routes_only_matching_trader() {
4472        let trader_id = TraderId::from("SOCKET-001");
4473
4474        let config = LiveNodeConfig {
4475            trader_id,
4476            exec_engine: crate::config::LiveExecutionEngineConfig {
4477                reconciliation: false,
4478                ..Default::default()
4479            },
4480            ..Default::default()
4481        };
4482
4483        let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4484        let client_id = ClientId::from("TEST");
4485        let endpoint = Ustr::from("test-streams");
4486        let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4487        let request_count = Arc::clone(&requests);
4488        let control =
4489            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4490        let _sink = control.sink();
4491        control.register(move || {
4492            request_count.fetch_add(1, Ordering::SeqCst);
4493            ReconnectRequestOutcome::Accepted
4494        });
4495
4496        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4497            TraderId::from("OTHER-001"),
4498            client_id,
4499            endpoint,
4500            UnixNanos::default(),
4501        )));
4502        assert_eq!(requests.load(Ordering::SeqCst), 0);
4503
4504        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4505            trader_id,
4506            client_id,
4507            endpoint,
4508            UnixNanos::default(),
4509        )));
4510        assert_eq!(requests.load(Ordering::SeqCst), 1);
4511    }
4512
4513    #[rstest]
4514    #[tokio::test]
4515    async fn test_start_publishes_socket_change_after_actor_subscribes() {
4516        let config = LiveNodeConfig {
4517            trader_id: TraderId::from("SOCKET-STARTUP-001"),
4518            exec_engine: crate::config::LiveExecutionEngineConfig {
4519                reconciliation: false,
4520                ..Default::default()
4521            },
4522            timeout_connection: Duration::ZERO,
4523            timeout_reconciliation: Duration::ZERO,
4524            timeout_portfolio: Duration::ZERO,
4525            timeout_disconnection: Duration::ZERO,
4526            delay_post_stop: Duration::ZERO,
4527            timeout_shutdown: Duration::ZERO,
4528            ..Default::default()
4529        };
4530
4531        let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4532        let received = Rc::new(RefCell::new(Vec::new()));
4533        node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4534            .unwrap();
4535        node.runner.as_ref().unwrap().bind_senders();
4536
4537        let change = SocketStateChange::new(
4538            ClientId::from("BINANCE"),
4539            Some(Venue::from("BINANCE")),
4540            Ustr::from("binance-futures-market-streams"),
4541            SocketState::Connected,
4542        );
4543        get_system_event_sender()
4544            .send(SystemEvent::SocketState(change))
4545            .unwrap();
4546
4547        node.start().await.unwrap();
4548
4549        {
4550            let events = received.borrow();
4551            assert_eq!(events.len(), 1);
4552            assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4553            assert_eq!(events[0].client_id, change.client_id);
4554            assert_eq!(events[0].venue, change.venue);
4555            assert_eq!(events[0].endpoint, change.endpoint);
4556            assert_eq!(events[0].state, change.state);
4557            assert_ne!(events[0].event_id, UUID4::default());
4558            assert_ne!(events[0].ts_event, UnixNanos::default());
4559            assert_eq!(events[0].ts_init, events[0].ts_event);
4560        }
4561
4562        node.stop().await.unwrap();
4563        node.dispose();
4564    }
4565
4566    #[rstest]
4567    #[tokio::test(flavor = "current_thread")]
4568    async fn test_run_publishes_queue_state_after_dispatch_sample() {
4569        let config = LiveNodeConfig {
4570            trader_id: TraderId::from("QUEUE-RUN-001"),
4571            queue_monitor: Some(crate::config::QueueMonitorConfig {
4572                queue_depth_trigger: usize::MAX,
4573                queue_depth_clear: 0,
4574                mean_dispatch_ns_trigger: 1,
4575                mean_dispatch_ns_clear: 0,
4576            }),
4577            exec_engine: crate::config::LiveExecutionEngineConfig {
4578                reconciliation: false,
4579                ..Default::default()
4580            },
4581            delay_post_stop: Duration::ZERO,
4582            ..Default::default()
4583        };
4584
4585        let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4586        let handle = node.handle();
4587        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4588
4589        let handler = ShareableMessageHandler::from_typed({
4590            let received = received.clone();
4591            let stop_handle = handle.clone();
4592
4593            move |event: &QueueStateChanged| {
4594                received.borrow_mut().push(event.clone());
4595                stop_handle.stop();
4596            }
4597        });
4598
4599        msgbus::subscribe_any(
4600            MessagingSwitchboard::queue_state_changed_pattern(None),
4601            handler,
4602            None,
4603        );
4604        let drive_handle = handle.clone();
4605
4606        let result = tokio::time::timeout(Duration::from_secs(5), async {
4607            let run = node.run();
4608            tokio::pin!(run);
4609
4610            let drive = async move {
4611                wait_until_async(
4612                    || async { drive_handle.is_running() },
4613                    Duration::from_secs(2),
4614                )
4615                .await;
4616                get_data_event_sender().send(stub_data_event()).unwrap();
4617            };
4618
4619            let (run_result, ()) = tokio::join!(run, drive);
4620            run_result
4621        })
4622        .await;
4623
4624        assert!(
4625            result.is_ok(),
4626            "queue state event should arrive before timeout"
4627        );
4628        assert!(result.unwrap().is_ok(), "run() should succeed");
4629        let events = received.borrow();
4630        assert_eq!(events.len(), 1);
4631        assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4632        assert_eq!(events[0].channel, SystemChannel::DataEvents);
4633        assert_eq!(events[0].condition, QueueCondition::Slow);
4634        assert_eq!(events[0].state, QueueState::Triggered);
4635        assert_eq!(events[0].queue_depth, 0);
4636        assert!(events[0].mean_dispatch_ns > 0);
4637        assert_ne!(events[0].event_id, UUID4::default());
4638        assert_ne!(events[0].ts_event, UnixNanos::default());
4639        assert_eq!(events[0].ts_init, events[0].ts_event);
4640        drop(events);
4641        msgbus::get_message_bus().borrow_mut().dispose();
4642    }
4643
4644    #[rstest]
4645    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4646    async fn test_run_dispatch_progress_under_load(
4647        #[values(false, true)] execution: bool,
4648        #[values(false, true)] slow: bool,
4649    ) {
4650        let config = LiveNodeConfig {
4651            delay_post_stop: Duration::ZERO,
4652            ..Default::default()
4653        };
4654        let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4655        let handle = node.handle();
4656        let stop = handle.clone();
4657        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4658            if slow {
4659                // Make housekeeping ready after each timer callback
4660                std::thread::sleep(Duration::from_millis(110));
4661            }
4662
4663            if event.ts_init == UnixNanos::from(8) {
4664                // Check progress before shutdown can drain the queues
4665                let snapshot = stop.metrics_snapshot();
4666                assert_eq!(snapshot.time_events.dispatched, 7);
4667
4668                if execution {
4669                    assert!((1..16).contains(&snapshot.exec_events.dispatched));
4670                    assert_eq!(snapshot.data_events.dispatched, 0);
4671                } else {
4672                    assert!(snapshot.data_events.dispatched > 0);
4673                    assert_eq!(snapshot.data_commands.dispatched, 1);
4674                }
4675                stop.stop();
4676            }
4677        }));
4678
4679        let drive = async {
4680            wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4681
4682            for ts in (1..=16_u64).map(UnixNanos::from) {
4683                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4684                    TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4685                    callback.clone(),
4686                ));
4687                get_data_event_sender().send(stub_data_event()).unwrap();
4688
4689                if execution {
4690                    get_exec_event_sender().send(stub_exec_event()).unwrap();
4691                }
4692            }
4693            nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4694        };
4695        tokio::time::timeout(Duration::from_secs(5), async {
4696            let (result, ()) = tokio::join!(node.run(), drive);
4697            result.unwrap();
4698        })
4699        .await
4700        .unwrap();
4701        assert!(handle.should_stop());
4702    }
4703
4704    #[rstest]
4705    fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4706        let config = LiveNodeConfig {
4707            exec_engine: crate::config::LiveExecutionEngineConfig {
4708                reconciliation: false,
4709                ..Default::default()
4710            },
4711            ..Default::default()
4712        };
4713
4714        let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4715        let event = stub_exec_event();
4716        let account_id = AccountId::from("TEST-001");
4717        let instrument_id = InstrumentId::from("TEST.VENUE");
4718        let trade_id = TradeId::from("T-001");
4719
4720        let close_ids = node.observe_exec_event_before_dispatch(&event);
4721        assert_eq!(close_ids, Some(Vec::new()));
4722        assert!(
4723            !node
4724                .exec_manager
4725                .is_fill_recently_processed(account_id, instrument_id, trade_id)
4726        );
4727
4728        node.exec_manager
4729            .mark_fill_processed(account_id, instrument_id, trade_id);
4730
4731        let close_ids = node.observe_exec_event_before_dispatch(&event);
4732        assert_eq!(close_ids, None);
4733    }
4734
4735    #[rstest]
4736    #[case(false, false, OrderStatus::Canceled, 1)]
4737    #[case(false, true, OrderStatus::Accepted, 0)]
4738    #[case(true, false, OrderStatus::Canceled, 1)]
4739    #[case(true, true, OrderStatus::Accepted, 0)]
4740    fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4741        #[case] with_fills: bool,
4742        #[case] superseded: bool,
4743        #[case] expected_status: OrderStatus,
4744        #[case] expected_query_count: usize,
4745    ) {
4746        let config = LiveNodeConfig {
4747            exec_engine: crate::config::LiveExecutionEngineConfig {
4748                reconciliation: true,
4749                open_check_threshold_ms: 5_000,
4750                single_order_query_delay_ms: 0,
4751                ..Default::default()
4752            },
4753            ..Default::default()
4754        };
4755
4756        let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4757        let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4758        let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4759        let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4760        let account_id = AccountId::from("TEST-001");
4761        let client_id = ClientId::from("TEST");
4762        let instrument = crypto_perpetual_ethusdt();
4763        let instrument_id = instrument.id();
4764        let account = AccountAny::Margin(MarginAccount::new(
4765            AccountState::new(
4766                account_id,
4767                AccountType::Margin,
4768                vec![AccountBalance::new(
4769                    Money::from("1000000 USDT"),
4770                    Money::from("0 USDT"),
4771                    Money::from("1000000 USDT"),
4772                )],
4773                Vec::new(),
4774                true,
4775                UUID4::new(),
4776                UnixNanos::default(),
4777                UnixNanos::default(),
4778                Some(Currency::USDT()),
4779            ),
4780            true,
4781        ));
4782        node.kernel.cache.borrow_mut().add_account(account).unwrap();
4783        node.kernel
4784            .cache
4785            .borrow_mut()
4786            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4787            .unwrap();
4788        insert_accepted_limit_order_in_node(
4789            &node,
4790            account_id,
4791            client_id,
4792            instrument_id,
4793            client_order_id,
4794            old_venue_order_id,
4795        );
4796
4797        if superseded {
4798            let order = node
4799                .kernel
4800                .cache
4801                .borrow()
4802                .order_owned(&client_order_id)
4803                .unwrap();
4804            let pending_update = OrderPendingUpdateSpec::builder()
4805                .trader_id(order.trader_id())
4806                .strategy_id(order.strategy_id())
4807                .instrument_id(order.instrument_id())
4808                .client_order_id(client_order_id)
4809                .account_id(account_id)
4810                .venue_order_id(old_venue_order_id)
4811                .build();
4812            node.kernel
4813                .cache
4814                .borrow_mut()
4815                .update_order(&OrderEventAny::PendingUpdate(pending_update))
4816                .unwrap();
4817            let order = node
4818                .kernel
4819                .cache
4820                .borrow()
4821                .order_owned(&client_order_id)
4822                .unwrap();
4823            let updated = OrderUpdatedSpec::builder()
4824                .trader_id(order.trader_id())
4825                .strategy_id(order.strategy_id())
4826                .instrument_id(order.instrument_id())
4827                .client_order_id(client_order_id)
4828                .quantity(order.quantity())
4829                .venue_order_id(new_venue_order_id)
4830                .account_id(account_id)
4831                .build();
4832            node.kernel
4833                .cache
4834                .borrow_mut()
4835                .update_order(&OrderEventAny::Updated(updated))
4836                .unwrap();
4837        }
4838
4839        let report = OrderStatusReport::new(
4840            account_id,
4841            instrument_id,
4842            Some(client_order_id),
4843            old_venue_order_id,
4844            OrderSide::Buy.into(),
4845            OrderType::Limit,
4846            TimeInForce::Gtc,
4847            OrderStatus::Canceled,
4848            Quantity::from("10.0"),
4849            Quantity::from("0.0"),
4850            UnixNanos::from(1_000),
4851            UnixNanos::from(2_000),
4852            UnixNanos::from(3_000),
4853            None,
4854        );
4855
4856        let report = if with_fills {
4857            ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4858        } else {
4859            ExecutionReport::Order(Box::new(report))
4860        };
4861
4862        let event = ExecutionEvent::Report(report);
4863
4864        node.process_exec_event(event);
4865
4866        let order = node
4867            .kernel
4868            .cache
4869            .borrow()
4870            .order_owned(&client_order_id)
4871            .unwrap();
4872
4873        let expected_venue_order_id = if superseded {
4874            new_venue_order_id
4875        } else {
4876            old_venue_order_id
4877        };
4878
4879        assert_eq!(order.status(), expected_status);
4880        assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4881
4882        if !superseded {
4883            let replacement = OrderTestBuilder::new(OrderType::Limit)
4884                .client_order_id(client_order_id)
4885                .instrument_id(instrument_id)
4886                .quantity(Quantity::from("20.0"))
4887                .price(Price::from("200.0"))
4888                .build();
4889            let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4890            node.kernel
4891                .cache
4892                .borrow_mut()
4893                .add_order(replacement, None, Some(client_id), true)
4894                .unwrap();
4895            let replacement = node
4896                .kernel
4897                .cache
4898                .borrow_mut()
4899                .update_order(&submitted)
4900                .unwrap();
4901            let accepted =
4902                TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4903            node.kernel
4904                .cache
4905                .borrow_mut()
4906                .update_order(&accepted)
4907                .unwrap();
4908        }
4909
4910        assert_eq!(
4911            node.exec_manager.check_open_order_queries().len(),
4912            expected_query_count,
4913        );
4914    }
4915
4916    #[rstest]
4917    fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4918        let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4919
4920        let OrderEventAny::Filled(fill) = &mut fill_event else {
4921            unreachable!();
4922        };
4923
4924        fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4925        fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4926        let fill = fill.clone();
4927        let report_event = fill_report_event(&fill);
4928        let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4929
4930        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4931        let marked_before_dispatch = is_recent_fill(&node, &fill);
4932
4933        node.dispatch_exec_event_and_commit_fill(event);
4934
4935        let marked_after_dispatch = is_recent_fill(&node, &fill);
4936        let later_report_is_eligible = node
4937            .observe_exec_event_before_dispatch(&report_event)
4938            .is_some();
4939        assert_eq!(
4940            (
4941                marked_before_dispatch,
4942                marked_after_dispatch,
4943                later_report_is_eligible,
4944            ),
4945            (false, false, true),
4946        );
4947    }
4948
4949    #[rstest]
4950    fn test_applied_direct_fill_commits_and_skips_later_report() {
4951        let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
4952
4953        let OrderEventAny::Filled(fill) = &fill_event else {
4954            unreachable!();
4955        };
4956
4957        let fill = fill.clone();
4958        let report_event = fill_report_event(&fill);
4959        let event = ExecutionEvent::Order(fill_event);
4960
4961        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4962        assert!(!is_recent_fill(&node, &fill));
4963
4964        node.dispatch_exec_event_and_commit_fill(event);
4965
4966        assert!(is_recent_fill(&node, &fill));
4967        assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
4968    }
4969
4970    #[rstest]
4971    fn test_canonical_duplicate_fill_counts_as_applied() {
4972        let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
4973
4974        let OrderEventAny::Filled(fill) = &fill_event else {
4975            unreachable!();
4976        };
4977
4978        let mut fill = fill.clone();
4979        node.kernel
4980            .cache
4981            .borrow_mut()
4982            .update_order(&fill_event)
4983            .unwrap();
4984        fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
4985
4986        node.exec_manager.commit_recent_fill_if_applied(&fill);
4987
4988        assert!(is_recent_fill(&node, &fill));
4989    }
4990
4991    #[rstest]
4992    #[case(false)]
4993    #[case(true)]
4994    fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
4995        let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
4996            "AppliedContinuousFillNode"
4997        } else {
4998            "RejectedContinuousFillNode"
4999        });
5000
5001        if !applied {
5002            let OrderEventAny::Filled(fill) = &mut fill_event else {
5003                unreachable!();
5004            };
5005
5006            fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5007            fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5008        }
5009
5010        let OrderEventAny::Filled(fill) = &fill_event else {
5011            unreachable!();
5012        };
5013
5014        let fill = fill.clone();
5015
5016        node.process_reconciliation_events(&[fill_event]);
5017
5018        assert_eq!(is_recent_fill(&node, &fill), applied);
5019    }
5020
5021    #[rstest]
5022    fn test_recent_fill_commit_requires_account_and_instrument_match() {
5023        let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5024        node.kernel
5025            .cache
5026            .borrow_mut()
5027            .update_order(&fill_event)
5028            .unwrap();
5029
5030        let OrderEventAny::Filled(fill) = fill_event else {
5031            unreachable!();
5032        };
5033
5034        let mut account_mismatch = fill.clone();
5035        account_mismatch.account_id = AccountId::from("OTHER-001");
5036        let mut instrument_mismatch = fill;
5037        instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5038
5039        node.exec_manager
5040            .commit_recent_fill_if_applied(&account_mismatch);
5041        node.exec_manager
5042            .commit_recent_fill_if_applied(&instrument_mismatch);
5043
5044        assert!(!is_recent_fill(&node, &account_mismatch));
5045        assert!(!is_recent_fill(&node, &instrument_mismatch));
5046    }
5047
5048    #[rstest]
5049    fn test_applied_inferred_fill_remains_recently_processed() {
5050        let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5051        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5052        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5053        let account_id = AccountId::from("TEST-001");
5054        let order = node
5055            .kernel
5056            .cache
5057            .borrow()
5058            .order_owned(&client_order_id)
5059            .unwrap();
5060        let report = OrderStatusReport::new(
5061            account_id,
5062            instrument.id(),
5063            Some(client_order_id),
5064            venue_order_id,
5065            OrderSide::Buy.into(),
5066            OrderType::Limit,
5067            TimeInForce::Gtc,
5068            OrderStatus::PartiallyFilled,
5069            Quantity::from("10.0"),
5070            Quantity::from("1.0"),
5071            UnixNanos::from(1_000),
5072            UnixNanos::from(1_000),
5073            UnixNanos::from(1_000),
5074            None,
5075        )
5076        .with_avg_px(dec!(100.0));
5077        let inferred = create_inferred_fill_for_qty(
5078            &order,
5079            &report,
5080            &account_id,
5081            &instrument,
5082            Quantity::from("1.0"),
5083            UnixNanos::from(1_000),
5084            None,
5085        )
5086        .unwrap();
5087
5088        let OrderEventAny::Filled(fill) = &inferred else {
5089            unreachable!();
5090        };
5091
5092        let fill = fill.clone();
5093
5094        node.process_reconciliation_events(&[inferred]);
5095
5096        assert!(fill.reconciliation);
5097        assert!(is_recent_fill(&node, &fill));
5098    }
5099
5100    #[rstest]
5101    #[tokio::test]
5102    async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5103        let client_id = ClientId::from("POSITION-FILLS");
5104        let account_id = AccountId::from("POSITION-FILLS-001");
5105        let instrument_id = crypto_perpetual_ethusdt().id();
5106        let commands = Rc::new(RefCell::new(Vec::new()));
5107
5108        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5109            client_id,
5110            account_id,
5111            venue: instrument_id.venue,
5112            outcome: FillReportClientOutcome::Failure,
5113            commands: commands.clone(),
5114        }));
5115
5116        let command = GenerateFillReports::new(
5117            UUID4::new(),
5118            UnixNanos::from(2_000),
5119            Some(instrument_id),
5120            None,
5121            Some(UnixNanos::from(1_000)),
5122            Some(UnixNanos::from(2_000)),
5123            None,
5124            None,
5125        );
5126
5127        let result = request_position_fill_reports(
5128            vec![client],
5129            vec![PositionFillReportQuery {
5130                key: (instrument_id, account_id),
5131                client_id,
5132                command: command.clone(),
5133            }],
5134        )
5135        .await
5136        .finish();
5137
5138        assert_eq!(*commands.borrow(), vec![command]);
5139        assert!(result.is_empty());
5140    }
5141
5142    #[rstest]
5143    #[tokio::test]
5144    async fn test_request_position_fill_reports_accepts_scoped_response() {
5145        let client_id = ClientId::from("POSITION-FILLS");
5146        let account_id = AccountId::from("POSITION-FILLS-001");
5147        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5148        let instrument_id = instrument.id();
5149
5150        let report_b = FillReport::new(
5151            account_id,
5152            instrument_id,
5153            VenueOrderId::from("V-POSITION-FILLS"),
5154            TradeId::from("T-POSITION-FILLS-B"),
5155            OrderSide::Buy,
5156            Quantity::from("1.0"),
5157            Price::from("100.0"),
5158            Money::zero(instrument.quote_currency()),
5159            LiquiditySide::Taker,
5160            Some(ClientOrderId::from("O-POSITION-FILLS")),
5161            None,
5162            UnixNanos::from(1_500),
5163            UnixNanos::from(2_000),
5164            None,
5165        );
5166        let mut report_a = report_b.clone();
5167        report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5168        let commands = Rc::new(RefCell::new(Vec::new()));
5169
5170        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5171            client_id,
5172            account_id,
5173            venue: instrument_id.venue,
5174            outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5175            commands,
5176        }));
5177
5178        let command = GenerateFillReports::new(
5179            UUID4::new(),
5180            UnixNanos::from(2_000),
5181            Some(instrument_id),
5182            None,
5183            Some(UnixNanos::from(1_000)),
5184            Some(UnixNanos::from(2_000)),
5185            None,
5186            None,
5187        );
5188        let key = (instrument_id, account_id);
5189
5190        let result = request_position_fill_reports(
5191            vec![client],
5192            vec![PositionFillReportQuery {
5193                key,
5194                client_id,
5195                command,
5196            }],
5197        )
5198        .await
5199        .finish();
5200
5201        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5202        assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5203    }
5204
5205    #[rstest]
5206    #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5207    #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5208    #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5209    #[tokio::test]
5210    async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5211        #[case] report_account: &str,
5212        #[case] report_instrument: &str,
5213        #[case] ts_event: u64,
5214        #[case] quantity: &str,
5215    ) {
5216        let client_id = ClientId::from("POSITION-FILLS");
5217        let account_id = AccountId::from("POSITION-FILLS-001");
5218        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5219        let instrument_id = instrument.id();
5220
5221        let report = FillReport::new(
5222            AccountId::from(report_account),
5223            InstrumentId::from(report_instrument),
5224            VenueOrderId::from("V-POSITION-FILLS"),
5225            TradeId::from("T-POSITION-FILLS"),
5226            OrderSide::Buy,
5227            Quantity::from(quantity),
5228            Price::from("100.0"),
5229            Money::zero(instrument.quote_currency()),
5230            LiquiditySide::Taker,
5231            Some(ClientOrderId::from("O-POSITION-FILLS")),
5232            None,
5233            UnixNanos::from(ts_event),
5234            UnixNanos::from(2_000),
5235            None,
5236        );
5237
5238        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5239            client_id,
5240            account_id,
5241            venue: instrument_id.venue,
5242            outcome: FillReportClientOutcome::Reports(vec![report]),
5243            commands: Rc::new(RefCell::new(Vec::new())),
5244        }));
5245
5246        let command = GenerateFillReports::new(
5247            UUID4::new(),
5248            UnixNanos::from(2_000),
5249            Some(instrument_id),
5250            None,
5251            Some(UnixNanos::from(1_000)),
5252            Some(UnixNanos::from(2_000)),
5253            None,
5254            None,
5255        );
5256        let key = (instrument_id, account_id);
5257
5258        let result = request_position_fill_reports(
5259            vec![client],
5260            vec![PositionFillReportQuery {
5261                key,
5262                client_id,
5263                command,
5264            }],
5265        )
5266        .await
5267        .finish();
5268
5269        assert!(result.is_empty());
5270    }
5271
5272    #[rstest]
5273    #[case(999)]
5274    #[case(2_001)]
5275    #[tokio::test]
5276    async fn test_request_position_fill_reports_filters_time_window_superset(
5277        #[case] ts_event: u64,
5278    ) {
5279        let client_id = ClientId::from("POSITION-FILLS");
5280        let account_id = AccountId::from("POSITION-FILLS-001");
5281        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5282        let instrument_id = instrument.id();
5283
5284        let report = FillReport::new(
5285            account_id,
5286            instrument_id,
5287            VenueOrderId::from("V-POSITION-FILLS"),
5288            TradeId::from("T-POSITION-FILLS"),
5289            OrderSide::Buy,
5290            Quantity::from("1.0"),
5291            Price::from("100.0"),
5292            Money::zero(instrument.quote_currency()),
5293            LiquiditySide::Taker,
5294            Some(ClientOrderId::from("O-POSITION-FILLS")),
5295            None,
5296            UnixNanos::from(ts_event),
5297            UnixNanos::from(2_000),
5298            None,
5299        );
5300
5301        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5302            client_id,
5303            account_id,
5304            venue: instrument_id.venue,
5305            outcome: FillReportClientOutcome::Reports(vec![report]),
5306            commands: Rc::new(RefCell::new(Vec::new())),
5307        }));
5308
5309        let command = GenerateFillReports::new(
5310            UUID4::new(),
5311            UnixNanos::from(2_000),
5312            Some(instrument_id),
5313            None,
5314            Some(UnixNanos::from(1_000)),
5315            Some(UnixNanos::from(2_000)),
5316            None,
5317            None,
5318        );
5319        let key = (instrument_id, account_id);
5320
5321        let result = request_position_fill_reports(
5322            vec![client],
5323            vec![PositionFillReportQuery {
5324                key,
5325                client_id,
5326                command,
5327            }],
5328        )
5329        .await
5330        .finish();
5331
5332        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5333        assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5334    }
5335
5336    #[rstest]
5337    #[case::query_error(false)]
5338    #[case::missing_client(true)]
5339    #[tokio::test]
5340    async fn test_position_fill_collection_discards_partial_failure(
5341        #[case] missing_client: bool,
5342        #[values(false, true)] failure_first: bool,
5343        position_fill_query: (FillReport, PositionFillReportQuery),
5344    ) {
5345        let (report, query) = position_fill_query;
5346        let failed_client_id = ClientId::from("FAILED-FILLS");
5347        let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5348        let mut healthy_report = report.clone();
5349        healthy_report.account_id = healthy_key.1;
5350        let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5351        let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5352        let empty_client_id = ClientId::from("EMPTY-FILLS");
5353        let mut clients = vec![
5354            LiveExecutionClient::new(Box::new(FillReportClient {
5355                client_id: query.client_id,
5356                account_id: query.key.1,
5357                venue: query.key.0.venue,
5358                outcome: FillReportClientOutcome::Reports(vec![report]),
5359                commands: Rc::new(RefCell::new(Vec::new())),
5360            })),
5361            LiveExecutionClient::new(Box::new(FillReportClient {
5362                client_id: healthy_client_id,
5363                account_id: healthy_key.1,
5364                venue: healthy_key.0.venue,
5365                outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5366                commands: Rc::new(RefCell::new(Vec::new())),
5367            })),
5368            LiveExecutionClient::new(Box::new(FillReportClient {
5369                client_id: empty_client_id,
5370                account_id: empty_key.1,
5371                venue: empty_key.0.venue,
5372                outcome: FillReportClientOutcome::Reports(Vec::new()),
5373                commands: Rc::new(RefCell::new(Vec::new())),
5374            })),
5375        ];
5376
5377        if !missing_client {
5378            clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5379                client_id: failed_client_id,
5380                account_id: query.key.1,
5381                venue: query.key.0.venue,
5382                outcome: FillReportClientOutcome::Failure,
5383                commands: Rc::new(RefCell::new(Vec::new())),
5384            })));
5385        }
5386
5387        let failed_query = PositionFillReportQuery {
5388            client_id: failed_client_id,
5389            key: query.key,
5390            command: query.command.clone(),
5391        };
5392
5393        let healthy_query = PositionFillReportQuery {
5394            key: healthy_key,
5395            client_id: healthy_client_id,
5396            command: query.command.clone(),
5397        };
5398
5399        let empty_query = PositionFillReportQuery {
5400            key: empty_key,
5401            client_id: empty_client_id,
5402            command: query.command.clone(),
5403        };
5404
5405        let mut queries = vec![query, failed_query];
5406
5407        if failure_first {
5408            queries.reverse();
5409        }
5410
5411        queries.insert(0, empty_query);
5412        queries.push(healthy_query);
5413
5414        let result = request_position_fill_reports(clients, queries)
5415            .await
5416            .finish();
5417
5418        assert_eq!(
5419            result.keys().copied().collect::<Vec<_>>(),
5420            vec![empty_key, healthy_key]
5421        );
5422        assert_eq!(
5423            result,
5424            IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5425        );
5426    }
5427
5428    #[rstest]
5429    #[case::same_economics("metadata", true)]
5430    #[case::venue_order("venue_order", false)]
5431    #[case::side("side", false)]
5432    #[case::quantity("quantity", false)]
5433    #[case::price("price", false)]
5434    #[case::commission("commission", false)]
5435    #[case::liquidity("liquidity", false)]
5436    #[case::average("average", false)]
5437    #[case::event_time("event_time", false)]
5438    #[case::client_order("client_order", false)]
5439    #[case::position("position", false)]
5440    #[tokio::test]
5441    async fn test_position_fill_collection_validates_duplicate_economics(
5442        #[case] changed: &str,
5443        #[case] accepted: bool,
5444        position_fill_query: (FillReport, PositionFillReportQuery),
5445    ) {
5446        let (report, query) = position_fill_query;
5447        let mut duplicate = report.clone();
5448
5449        match changed {
5450            "metadata" => {
5451                duplicate.report_id = UUID4::new();
5452                duplicate.ts_init = UnixNanos::from(3_000);
5453            }
5454            "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5455            "side" => duplicate.order_side = OrderSide::Sell,
5456            "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5457            "price" => duplicate.last_px = Price::from("101.0"),
5458            "commission" => duplicate.commission = Money::from("0.25 USDT"),
5459            "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5460            "average" => duplicate.avg_px = Some(dec!(100.5)),
5461            "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5462            "client_order" => duplicate.client_order_id = None,
5463            "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5464            _ => unreachable!(),
5465        }
5466
5467        let key = query.key;
5468
5469        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5470            client_id: query.client_id,
5471            account_id: key.1,
5472            venue: key.0.venue,
5473            outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5474            commands: Rc::new(RefCell::new(Vec::new())),
5475        }));
5476
5477        let result = request_position_fill_reports(vec![client], vec![query])
5478            .await
5479            .finish();
5480
5481        if accepted {
5482            assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5483            assert_eq!(result, IndexMap::from([(key, vec![report])]));
5484        } else {
5485            assert!(result.is_empty());
5486        }
5487    }
5488
5489    #[fixture]
5490    fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5491        let account_id = AccountId::from("POSITION-FILLS-001");
5492        let instrument_id = crypto_perpetual_ethusdt().id();
5493
5494        let report = FillReport::new(
5495            account_id,
5496            instrument_id,
5497            VenueOrderId::from("V-POSITION-FILLS"),
5498            TradeId::from("T-POSITION-FILLS"),
5499            OrderSide::Buy,
5500            Quantity::from("1.0"),
5501            Price::from("100.0"),
5502            Money::from("0.10 USDT"),
5503            LiquiditySide::Taker,
5504            Some(ClientOrderId::from("O-POSITION-FILLS")),
5505            None,
5506            UnixNanos::from(1_500),
5507            UnixNanos::from(2_000),
5508            None,
5509        );
5510
5511        let query = PositionFillReportQuery {
5512            key: (instrument_id, account_id),
5513            client_id: ClientId::from("POSITION-FILLS"),
5514            command: GenerateFillReports::new(
5515                UUID4::new(),
5516                UnixNanos::from(2_000),
5517                Some(instrument_id),
5518                None,
5519                Some(UnixNanos::from(1_000)),
5520                Some(UnixNanos::from(2_000)),
5521                None,
5522                None,
5523            ),
5524        };
5525
5526        (report, query)
5527    }
5528
5529    #[rstest]
5530    #[case::failed_query(false)]
5531    #[case::local_activity(true)]
5532    fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5533        let (mut node, report, _) =
5534            position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5535        let active_key = (report.instrument_id, report.account_id);
5536        let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5537        let deferred_reports = vec![
5538            PositionStatusReport::new(
5539                deferred_key.1,
5540                deferred_key.0,
5541                PositionSide::Long,
5542                Quantity::from("2.0"),
5543                UnixNanos::from(1_000),
5544                UnixNanos::from(1_000),
5545                None,
5546                None,
5547                Some(dec!(100.0)),
5548            ),
5549            PositionStatusReport::new(
5550                deferred_key.1,
5551                deferred_key.0,
5552                PositionSide::Short,
5553                Quantity::from("1.0"),
5554                UnixNanos::from(1_000),
5555                UnixNanos::from(1_000),
5556                None,
5557                None,
5558                Some(dec!(100.0)),
5559            ),
5560        ];
5561        let mut check = node
5562            .exec_manager
5563            .prepare_position_report_check(UUID4::new(), &[]);
5564        check.client_coverage.clear();
5565        let events = node.exec_manager.reconcile_position_reports(
5566            &check,
5567            deferred_reports.clone(),
5568            &IndexSet::new(),
5569            &IndexSet::new(),
5570        );
5571        assert!(events.is_empty());
5572        assert_eq!(
5573            node.exec_manager.position_recon_retry_count(&deferred_key),
5574            1
5575        );
5576
5577        let mut position_result = position_report_result(&node, report);
5578        position_result.reports.extend(deferred_reports);
5579        position_result.check.client_coverage.insert(
5580            deferred_key,
5581            ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5582        );
5583        position_result
5584            .check
5585            .activity_revisions
5586            .insert(deferred_key, 0);
5587        position_result
5588            .queried_clients
5589            .insert(ClientId::from("SECOND"));
5590        let mut reports = IndexMap::from([(active_key, Vec::new())]);
5591        if local_activity {
5592            reports.insert(deferred_key, Vec::new());
5593            node.exec_manager
5594                .record_position_activity(deferred_key.0, deferred_key.1);
5595        }
5596
5597        node.handle_position_fill_report_result(PositionFillReportResult {
5598            position_result,
5599            reports,
5600        });
5601
5602        assert_eq!(
5603            node.exec_manager.position_recon_retry_count(&deferred_key),
5604            1
5605        );
5606        {
5607            let cache = node.kernel.cache.borrow();
5608            let positions =
5609                cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5610            assert_eq!(
5611                positions
5612                    .iter()
5613                    .map(|position| position.quantity)
5614                    .collect::<Vec<_>>(),
5615                vec![Quantity::from("1.0"), Quantity::from("1.0")],
5616            );
5617            assert_eq!(
5618                cache
5619                    .positions_open(
5620                        None,
5621                        Some(&deferred_key.0),
5622                        None,
5623                        Some(&deferred_key.1),
5624                        None,
5625                    )
5626                    .len(),
5627                0
5628            );
5629        }
5630
5631        let mut fresh_check = node
5632            .exec_manager
5633            .prepare_position_report_check(UUID4::new(), &[]);
5634        node.exec_manager.plan_position_fill_reports(
5635            &mut fresh_check,
5636            &[],
5637            &IndexSet::new(),
5638            &IndexSet::new(),
5639            &[],
5640        );
5641
5642        assert_eq!(
5643            node.exec_manager.position_recon_retry_count(&deferred_key),
5644            0
5645        );
5646    }
5647
5648    #[rstest]
5649    fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5650        let (mut node, venue_report, fill_report) =
5651            position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5652        let key = (venue_report.instrument_id, venue_report.account_id);
5653        let revision = node.exec_manager.position_activity_revision(&key);
5654        let position_result = position_report_result(&node, venue_report);
5655
5656        node.handle_position_fill_report_result(PositionFillReportResult {
5657            position_result,
5658            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5659        });
5660
5661        let cache = node.kernel.cache.borrow();
5662        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5663        assert_eq!(
5664            positions
5665                .iter()
5666                .map(|position| position.quantity)
5667                .sum::<Quantity>(),
5668            Quantity::from("2.0")
5669        );
5670        assert_eq!(
5671            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5672            1
5673        );
5674        drop(positions);
5675        drop(cache);
5676        assert!(
5677            node.exec_manager
5678                .position_contains_fill_report(&fill_report)
5679        );
5680        assert_eq!(
5681            node.exec_manager.position_activity_revision(&key),
5682            revision + 1
5683        );
5684    }
5685
5686    #[rstest]
5687    fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5688        let (mut node, venue_report, mut first) =
5689            position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5690        first.commission = Money::from("0.13 USDT");
5691        let key = (first.instrument_id, first.account_id);
5692        let client_order_id = first.client_order_id.unwrap();
5693        let position_result = position_report_result(&node, venue_report.clone());
5694        node.handle_position_fill_report_result(PositionFillReportResult {
5695            position_result,
5696            reports: IndexMap::from([(key, vec![first.clone()])]),
5697        });
5698
5699        let mut second = first.clone();
5700        second.trade_id = TradeId::from("T-PREFIX-SECOND");
5701        second.last_qty = Quantity::from("0.75");
5702        second.commission = Money::from("0.29 USDT");
5703        let mut third = first.clone();
5704        third.trade_id = TradeId::from("T-PREFIX-THIRD");
5705        third.last_qty = Quantity::from("0.25");
5706        third.commission = Money::from("0.07 USDT");
5707        let revision = node.exec_manager.position_activity_revision(&key);
5708        let received = Rc::new(RefCell::new(Vec::new()));
5709        let cache = node.kernel.cache.clone();
5710        let callback_received = received.clone();
5711        let second_trade_id = second.trade_id;
5712
5713        let handler = TypedHandler::from(move |event: &OrderEventAny| {
5714            let OrderEventAny::Filled(fill) = event else {
5715                return;
5716            };
5717
5718            callback_received.borrow_mut().push(fill.trade_id);
5719            if fill.trade_id == second_trade_id {
5720                let mut cache = cache.borrow_mut();
5721                let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5722                let mut position = cache.position_mut(&position_id).unwrap();
5723                position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5724            }
5725        });
5726
5727        let strategy_id = node
5728            .kernel
5729            .cache
5730            .borrow()
5731            .order(&client_order_id)
5732            .unwrap()
5733            .strategy_id();
5734        let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5735        msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5736        let position_result = position_report_result(&node, venue_report);
5737        node.handle_position_fill_report_result(PositionFillReportResult {
5738            position_result,
5739            reports: IndexMap::from([(
5740                key,
5741                vec![first.clone(), first.clone(), second.clone(), third.clone()],
5742            )]),
5743        });
5744
5745        msgbus::unsubscribe_order_events(topic.into(), &handler);
5746        let cache = node.kernel.cache.borrow();
5747        let order = cache.order(&client_order_id).unwrap();
5748        let position_id = cache.position_id(&client_order_id).unwrap();
5749        let position = cache.position_ref(position_id).unwrap();
5750
5751        assert_eq!(*received.borrow(), vec![second.trade_id]);
5752        assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5753        assert_eq!(order.events().len(), 6);
5754        assert_eq!(
5755            order.trade_ids(),
5756            vec![
5757                &TradeId::from("T-POSITION-INITIAL"),
5758                &first.trade_id,
5759                &second.trade_id
5760            ]
5761        );
5762        assert_eq!(
5763            order.commissions().values().copied().collect::<Vec<_>>(),
5764            vec![Money::from("0.42 USDT")]
5765        );
5766        assert_eq!(position.quantity, Quantity::from("2.25"));
5767        assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5768        assert_eq!(position.replay_events.len(), 2);
5769        assert_eq!(
5770            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5771            1
5772        );
5773        assert_eq!(
5774            node.exec_manager.position_activity_revision(&key),
5775            revision + 1
5776        );
5777        assert!(node.exec_manager.position_contains_fill_report(&first));
5778        assert!(!node.exec_manager.position_contains_fill_report(&second));
5779        assert!(!node.exec_manager.position_contains_fill_report(&third));
5780        assert!(
5781            !node
5782                .exec_manager
5783                .is_fill_recently_processed(key.1, key.0, second.trade_id)
5784        );
5785        assert!(
5786            !node
5787                .exec_manager
5788                .is_fill_recently_processed(key.1, key.0, third.trade_id)
5789        );
5790    }
5791
5792    #[rstest]
5793    fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5794        let (mut node, venue_report, fill_report) =
5795            position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5796        let key = (venue_report.instrument_id, venue_report.account_id);
5797        let position_result = position_report_result(&node, venue_report);
5798
5799        node.handle_position_fill_report_result(PositionFillReportResult {
5800            position_result,
5801            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5802        });
5803
5804        let mut rest_report = fill_report;
5805        rest_report.ts_event = UnixNanos::from(2_000);
5806
5807        assert!(
5808            node.exec_manager
5809                .position_contains_fill_report(&rest_report)
5810        );
5811    }
5812
5813    #[rstest]
5814    fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5815        let (mut node, venue_report, fill_report) =
5816            position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5817        let key = (venue_report.instrument_id, venue_report.account_id);
5818        let client_order_id = fill_report.client_order_id.unwrap();
5819        apply_inferred_position_fill(&mut node, &fill_report);
5820
5821        let venue_report = PositionStatusReport::new(
5822            key.1,
5823            key.0,
5824            PositionSide::Long,
5825            Quantity::from("3.0"),
5826            venue_report.ts_last,
5827            venue_report.ts_init,
5828            None,
5829            None,
5830            venue_report.avg_px_open,
5831        );
5832        let mut later_fill_report = fill_report.clone();
5833        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5834        later_fill_report.ts_event = UnixNanos::from(1_001);
5835        let position_result = position_report_result(&node, venue_report);
5836
5837        node.handle_position_fill_report_result(PositionFillReportResult {
5838            position_result,
5839            reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5840        });
5841
5842        let cache = node.kernel.cache.borrow();
5843        let order = cache.order(&client_order_id).unwrap();
5844        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5845        assert_eq!(order.filled_qty(), Quantity::from("2.0"));
5846        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
5847        assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
5848        assert_eq!(
5849            positions
5850                .iter()
5851                .map(|position| position.quantity)
5852                .sum::<Quantity>(),
5853            Quantity::from("3.0")
5854        );
5855        assert_eq!(
5856            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5857            2
5858        );
5859    }
5860
5861    #[rstest]
5862    fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
5863        let (mut node, _, _) =
5864            position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
5865        let account_id = AccountId::from("TEST-001");
5866        let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
5867        let key = (instrument.id(), account_id);
5868        let commission = Money::zero(instrument.quote_currency());
5869        node.kernel
5870            .cache
5871            .borrow_mut()
5872            .add_instrument(instrument.clone())
5873            .unwrap();
5874
5875        let order = OrderTestBuilder::new(OrderType::Market)
5876            .strategy_id(StrategyId::external())
5877            .instrument_id(instrument.id())
5878            .client_order_id(ClientOrderId::from("O-RECONCILED"))
5879            .quantity(Quantity::from("1.000000"))
5880            .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
5881            .build();
5882        let submitted = TestOrderEventStubs::submitted(&order, account_id);
5883        let accepted =
5884            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
5885        node.kernel
5886            .cache
5887            .borrow_mut()
5888            .add_order(
5889                order.clone(),
5890                None,
5891                Some(ClientId::from("POSITION-FILLS")),
5892                false,
5893            )
5894            .unwrap();
5895        node.process_reconciliation_events(&[submitted, accepted]);
5896        let order = node
5897            .kernel
5898            .cache
5899            .borrow()
5900            .order_owned(&order.client_order_id())
5901            .unwrap();
5902        let opening_fill = TestOrderEventStubs::filled(
5903            &order,
5904            &instrument,
5905            Some(TradeId::from("T-RECONCILED")),
5906            None,
5907            Some(Price::from("100.00")),
5908            Some(Quantity::from("1.000000")),
5909            Some(LiquiditySide::Taker),
5910            Some(commission),
5911            Some(UnixNanos::from(2_000)),
5912            Some(account_id),
5913        );
5914        node.process_reconciliation_events(&[opening_fill]);
5915
5916        // A real fill between the position report query and reconciliation stays off the position
5917        let held_fill_report = FillReport::new(
5918            account_id,
5919            instrument.id(),
5920            VenueOrderId::from("V-HELD"),
5921            TradeId::from("T-HELD"),
5922            OrderSide::Buy,
5923            Quantity::from("1.000000"),
5924            Price::from("100.00"),
5925            commission,
5926            LiquiditySide::Taker,
5927            None,
5928            None,
5929            UnixNanos::from(1_000),
5930            UnixNanos::from(1_000),
5931            None,
5932        );
5933        node.kernel
5934            .exec_engine
5935            .borrow_mut()
5936            .reconcile_fill_report(&held_fill_report);
5937
5938        let venue_report = PositionStatusReport::new(
5939            account_id,
5940            instrument.id(),
5941            PositionSide::Long,
5942            Quantity::from("2.000000"),
5943            UnixNanos::from(3_000),
5944            UnixNanos::from(3_000),
5945            None,
5946            None,
5947            Some(dec!(100.00)),
5948        );
5949        let position_result = position_report_result(&node, venue_report);
5950
5951        node.handle_position_fill_report_result(PositionFillReportResult {
5952            position_result,
5953            reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
5954        });
5955
5956        let cache = node.kernel.cache.borrow();
5957        let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
5958        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5959        assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
5960        assert_eq!(
5961            positions
5962                .iter()
5963                .map(|position| position.quantity)
5964                .sum::<Quantity>(),
5965            Quantity::from("2.000000")
5966        );
5967        assert!(
5968            positions
5969                .iter()
5970                .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
5971        );
5972    }
5973
5974    #[rstest]
5975    fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
5976        let (mut node, _, mut fill_report) =
5977            position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
5978        apply_inferred_position_fill(&mut node, &fill_report);
5979        let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
5980        fill_report.venue_position_id = Some(conflicting_position_id);
5981
5982        let error = node
5983            .exec_manager
5984            .prepare_position_fill_report(&mut fill_report, &[])
5985            .unwrap_err();
5986
5987        assert!(
5988            error.to_string().contains(&format!(
5989                "position ID {conflicting_position_id} conflicts with cached order position"
5990            )),
5991            "{error:#}"
5992        );
5993    }
5994
5995    #[rstest]
5996    #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
5997    #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
5998    fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
5999        #[case] authoritative_qty: &str,
6000        #[case] client_order_id: &str,
6001    ) {
6002        let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6003            "UnappliedPositionFillNode",
6004            Quantity::from(authoritative_qty),
6005        );
6006        fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6007        let reports = [fill_report.clone()];
6008
6009        let quantities = (0..2)
6010            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6011            .collect::<Vec<_>>();
6012
6013        assert_eq!(
6014            quantities,
6015            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6016        );
6017        assert!(
6018            !node
6019                .exec_manager
6020                .position_contains_fill_report(&fill_report)
6021        );
6022    }
6023
6024    #[rstest]
6025    fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6026        let (mut node, venue_report, fill_report) =
6027            position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6028        let mut later_fill_report = fill_report.clone();
6029        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6030        later_fill_report.ts_event = UnixNanos::from(1_001);
6031        let reports = [fill_report.clone(), later_fill_report.clone()];
6032
6033        let quantities = (0..3)
6034            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6035            .collect::<Vec<_>>();
6036
6037        assert_eq!(
6038            quantities,
6039            vec![
6040                Quantity::from("1.0"),
6041                Quantity::from("1.0"),
6042                Quantity::from("2.0"),
6043            ]
6044        );
6045        assert!(
6046            !node
6047                .exec_manager
6048                .position_contains_fill_report(&fill_report)
6049        );
6050        assert!(
6051            !node
6052                .exec_manager
6053                .position_contains_fill_report(&later_fill_report)
6054        );
6055    }
6056
6057    #[rstest]
6058    #[case::within_grace(60_000, true)]
6059    #[case::after_grace(0, false)]
6060    fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6061        #[case] position_check_threshold_ms: u32,
6062        #[case] expected_applied: bool,
6063    ) {
6064        let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6065            "TransientPositionFillNode",
6066            Quantity::from("1.0"),
6067            position_check_threshold_ms,
6068        );
6069        let mut conflicting_report = fill_report.clone();
6070        conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6071
6072        let quantities = [conflicting_report, fill_report.clone()]
6073            .into_iter()
6074            .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6075            .collect::<Vec<_>>();
6076
6077        assert_eq!(
6078            quantities,
6079            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6080        );
6081        assert_eq!(
6082            node.exec_manager
6083                .position_contains_fill_report(&fill_report),
6084            expected_applied
6085        );
6086    }
6087
6088    #[rstest]
6089    fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6090        let (mut node, venue_report, fill_report) =
6091            position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6092        let key = (venue_report.instrument_id, venue_report.account_id);
6093        let first_result = position_report_result(&node, venue_report.clone());
6094
6095        node.handle_position_fill_report_result(PositionFillReportResult {
6096            position_result: first_result,
6097            reports: IndexMap::from([(key, vec![fill_report])]),
6098        });
6099
6100        let fresh_result = position_report_result(&node, venue_report);
6101        node.handle_position_fill_report_result(PositionFillReportResult {
6102            position_result: fresh_result,
6103            reports: IndexMap::from([(key, Vec::new())]),
6104        });
6105
6106        let cache = node.kernel.cache.borrow();
6107        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6108        assert_eq!(
6109            positions
6110                .iter()
6111                .map(|position| position.quantity)
6112                .sum::<Quantity>(),
6113            Quantity::from("2.0")
6114        );
6115        assert_eq!(
6116            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6117            2
6118        );
6119    }
6120
6121    #[rstest]
6122    #[case::below_limit(63)]
6123    #[case::at_limit(64)]
6124    #[case::above_limit(65)]
6125    fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6126        let (mut node, venue_report, fill_report) =
6127            position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6128        let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6129
6130        let venue_report = PositionStatusReport::new(
6131            venue_report.account_id,
6132            venue_report.instrument_id,
6133            PositionSide::Long,
6134            Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6135            venue_report.ts_last,
6136            venue_report.ts_init,
6137            None,
6138            None,
6139            venue_report.avg_px_open,
6140        );
6141        let key = (venue_report.instrument_id, venue_report.account_id);
6142        let client_order_id = fill_report.client_order_id.unwrap();
6143
6144        let reports = (0..count)
6145            .map(|index| {
6146                let mut report = fill_report.clone();
6147                report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6148                report.ts_event = UnixNanos::from(1_000 + index as u64);
6149                report
6150            })
6151            .collect::<Vec<_>>();
6152
6153        let position_result = position_report_result(&node, venue_report.clone());
6154
6155        node.handle_position_fill_report_result(PositionFillReportResult {
6156            position_result,
6157            reports: IndexMap::from([(key, reports.clone())]),
6158        });
6159
6160        {
6161            let cache = node.kernel.cache.borrow();
6162            let order = cache.order(&client_order_id).unwrap();
6163            let applied = count.min(64);
6164            assert_eq!(
6165                order.filled_qty().as_decimal(),
6166                dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6167            );
6168            assert_eq!(order.trade_ids().len(), applied + 1);
6169
6170            for (index, report) in reports.iter().enumerate() {
6171                assert_eq!(
6172                    order.trade_ids().contains(&&report.trade_id),
6173                    index < applied
6174                );
6175            }
6176
6177            assert_eq!(
6178                cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6179                1
6180            );
6181        }
6182
6183        // A fresh cycle replays the full response; only the deferred fill may change exposure
6184        let position_result = position_report_result(&node, venue_report);
6185        node.handle_position_fill_report_result(PositionFillReportResult {
6186            position_result,
6187            reports: IndexMap::from([(key, reports)]),
6188        });
6189
6190        let cache = node.kernel.cache.borrow();
6191        let order = cache.order(&client_order_id).unwrap();
6192        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6193        assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6194        assert_eq!(order.trade_ids().len(), count + 1);
6195        assert_eq!(positions.len(), 1);
6196        assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6197        assert_eq!(positions[0].side, PositionSide::Long);
6198        assert_eq!(
6199            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6200            1
6201        );
6202    }
6203
6204    #[rstest]
6205    #[case::client_order(0)]
6206    #[case::venue_order(1)]
6207    #[case::side(2)]
6208    #[case::position(3)]
6209    fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6210        let (mut node, venue_report, valid_report) =
6211            position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6212        let key = (venue_report.instrument_id, venue_report.account_id);
6213        let client_order_id = valid_report.client_order_id.unwrap();
6214        let revision = node.exec_manager.position_activity_revision(&key);
6215        let position_result = position_report_result(&node, venue_report);
6216        let mut conflicting_report = valid_report.clone();
6217        conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6218
6219        match conflict {
6220            0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6221            1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6222            2 => conflicting_report.order_side = OrderSide::Sell,
6223            3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6224            _ => unreachable!(),
6225        }
6226
6227        node.handle_position_fill_report_result(PositionFillReportResult {
6228            position_result,
6229            reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6230        });
6231
6232        let cache = node.kernel.cache.borrow();
6233        let order = cache.order(&client_order_id).unwrap();
6234        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6235        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6236        assert_eq!(
6237            order.trade_ids(),
6238            vec![&TradeId::from("T-POSITION-INITIAL")]
6239        );
6240        assert_eq!(positions.len(), 1);
6241        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6242        assert_eq!(
6243            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6244            1
6245        );
6246        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6247    }
6248
6249    #[rstest]
6250    fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6251        let (mut node, venue_report, _) =
6252            position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6253        let key = (venue_report.instrument_id, venue_report.account_id);
6254        let position_result = position_report_result(&node, venue_report);
6255
6256        node.handle_position_fill_report_result(PositionFillReportResult {
6257            position_result,
6258            reports: IndexMap::new(),
6259        });
6260
6261        let cache = node.kernel.cache.borrow();
6262        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6263        assert_eq!(positions.len(), 1);
6264        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6265        assert_eq!(
6266            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6267            1
6268        );
6269    }
6270
6271    #[rstest]
6272    fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6273        let (mut node, mut venue_report, mut fill_report) =
6274            position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6275        node.kernel
6276            .exec_engine
6277            .borrow_mut()
6278            .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6279        let key = (venue_report.instrument_id, venue_report.account_id);
6280
6281        let position_id = {
6282            let cache = node.kernel.cache.borrow();
6283            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6284            assert_eq!(positions.len(), 1);
6285            positions[0].id
6286        };
6287
6288        venue_report.venue_position_id = Some(position_id);
6289        fill_report.client_order_id = None;
6290        fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6291        let position_result = position_report_result(&node, venue_report);
6292
6293        node.handle_position_fill_report_result(PositionFillReportResult {
6294            position_result,
6295            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6296        });
6297
6298        let cache = node.kernel.cache.borrow();
6299        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6300        assert_eq!(positions.len(), 1);
6301        assert_eq!(positions[0].id, position_id);
6302        assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6303        assert_eq!(
6304            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6305            2
6306        );
6307        drop(positions);
6308        drop(cache);
6309        assert!(
6310            !node
6311                .exec_manager
6312                .position_contains_fill_report(&fill_report)
6313        );
6314    }
6315
6316    #[rstest]
6317    fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6318        let (mut node, venue_report, fill_report) =
6319            position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6320        let key = (venue_report.instrument_id, venue_report.account_id);
6321        let client_order_id = fill_report.client_order_id.unwrap();
6322        let position_result = position_report_result(&node, venue_report);
6323        let revision = node.exec_manager.position_activity_revision(&key);
6324
6325        let order_report = OrderStatusReport::new(
6326            key.1,
6327            key.0,
6328            Some(client_order_id),
6329            fill_report.venue_order_id,
6330            OrderSide::Buy.into(),
6331            OrderType::Limit,
6332            TimeInForce::Gtc,
6333            OrderStatus::PartiallyFilled,
6334            Quantity::from("10.0"),
6335            Quantity::from("2.0"),
6336            UnixNanos::from(1_000),
6337            UnixNanos::from(1_000),
6338            UnixNanos::from(2_000),
6339            None,
6340        );
6341        let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6342            Box::new(order_report),
6343            vec![fill_report.clone()],
6344        ));
6345
6346        assert_eq!(
6347            node.observe_exec_event_before_dispatch(&event),
6348            Some(vec![client_order_id])
6349        );
6350
6351        assert_eq!(
6352            node.exec_manager.position_activity_revision(&key),
6353            revision + 1
6354        );
6355        assert!(
6356            !node
6357                .exec_manager
6358                .position_report_check_is_current(&position_result.check, &key)
6359        );
6360
6361        node.handle_position_fill_report_result(PositionFillReportResult {
6362            position_result,
6363            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6364        });
6365
6366        let cache = node.kernel.cache.borrow();
6367        let order = cache.order(&client_order_id).unwrap();
6368        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6369        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6370        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6371        assert_eq!(positions.len(), 1);
6372        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6373        assert_eq!(positions[0].side, PositionSide::Long);
6374        assert_eq!(
6375            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6376            1
6377        );
6378    }
6379
6380    #[rstest]
6381    fn test_position_fill_report_result_defers_after_local_position_activity() {
6382        let (mut node, venue_report, fill_report) =
6383            position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6384        let key = (venue_report.instrument_id, venue_report.account_id);
6385        let position_result = position_report_result(&node, venue_report);
6386        node.exec_manager.record_position_activity(key.0, key.1);
6387
6388        node.handle_position_fill_report_result(PositionFillReportResult {
6389            position_result,
6390            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6391        });
6392
6393        let cache = node.kernel.cache.borrow();
6394        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6395        assert_eq!(positions.len(), 1);
6396        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6397        assert_eq!(
6398            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6399            1
6400        );
6401        drop(positions);
6402        drop(cache);
6403        assert!(
6404            !node
6405                .exec_manager
6406                .position_contains_fill_report(&fill_report)
6407        );
6408    }
6409
6410    #[rstest]
6411    fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6412        let config = LiveNodeConfig {
6413            exec_engine: crate::config::LiveExecutionEngineConfig {
6414                reconciliation: true,
6415                open_check_threshold_ms: 5_000,
6416                single_order_query_delay_ms: 0,
6417                ..Default::default()
6418            },
6419            ..Default::default()
6420        };
6421
6422        let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6423        let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6424        let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6425        let instrument = crypto_perpetual_ethusdt();
6426        let instrument_id = instrument.id();
6427        let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6428        let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6429
6430        node.kernel
6431            .cache
6432            .borrow_mut()
6433            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6434            .unwrap();
6435        insert_accepted_limit_order_in_node(
6436            &node,
6437            account_id,
6438            client_id,
6439            instrument_id,
6440            client_order_id,
6441            venue_order_id,
6442        );
6443
6444        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6445
6446        let accepted = OrderAcceptedSpec::builder()
6447            .instrument_id(instrument_id)
6448            .client_order_id(client_order_id)
6449            .venue_order_id(venue_order_id)
6450            .account_id(account_id)
6451            .build();
6452        let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6453
6454        let close_ids = node.observe_exec_event_before_dispatch(&event);
6455
6456        assert_eq!(close_ids, Some(vec![client_order_id]));
6457        assert!(node.exec_manager.check_open_order_queries().is_empty());
6458    }
6459
6460    #[rstest]
6461    #[cfg_attr(
6462        not(all(feature = "simulation", madsim)),
6463        tokio::test(start_paused = true)
6464    )]
6465    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6466    async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6467        use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6468        use nautilus_model::{events::OrderPendingCancel, identifiers::ClientOrderId};
6469
6470        let config = LiveNodeConfig {
6471            exec_engine: crate::config::LiveExecutionEngineConfig {
6472                reconciliation: true,
6473                inflight_check_threshold_ms: 100,
6474                inflight_check_retries: 1,
6475                ..Default::default()
6476            },
6477            ..Default::default()
6478        };
6479
6480        let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6481        let trader_id = TraderId::from("TESTER-001");
6482        let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6483        let account_id = AccountId::from("TEST-001");
6484        let instrument = crypto_perpetual_ethusdt();
6485        let instrument_id = instrument.id();
6486        let child_ids = [
6487            ClientOrderId::from("O-BATCH-CANCEL-1"),
6488            ClientOrderId::from("O-BATCH-CANCEL-2"),
6489        ];
6490        node.kernel
6491            .cache
6492            .borrow_mut()
6493            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6494            .unwrap();
6495
6496        for client_order_id in child_ids {
6497            let order = OrderTestBuilder::new(OrderType::Limit)
6498                .trader_id(trader_id)
6499                .strategy_id(strategy_id)
6500                .client_order_id(client_order_id)
6501                .instrument_id(instrument_id)
6502                .quantity(Quantity::from("10.0"))
6503                .price(Price::from("100.0"))
6504                .build();
6505            let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6506            // Model the production batch-cancel path: each child is accepted, then
6507            // moved to PendingCancel, so an inflight timeout must emit a Canceled
6508            // event (not a Submitted-order rejection).
6509            let submitted = TestOrderEventStubs::submitted(&order, account_id);
6510            let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6511            let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
6512                trader_id,
6513                strategy_id,
6514                instrument_id,
6515                client_order_id,
6516                Some(account_id),
6517                UUID4::new(),
6518                UnixNanos::default(),
6519                UnixNanos::default(),
6520                false,
6521                Some(venue_order_id),
6522            ));
6523            let mut cache = node.kernel.cache.borrow_mut();
6524            cache.add_order(order, None, None, false).unwrap();
6525            cache.update_order(&submitted).unwrap();
6526            cache.update_order(&accepted).unwrap();
6527            cache.update_order(&pending_cancel).unwrap();
6528        }
6529
6530        let cancels = child_ids
6531            .into_iter()
6532            .map(|client_order_id| {
6533                CancelOrder::new(
6534                    trader_id,
6535                    None,
6536                    strategy_id,
6537                    instrument_id,
6538                    client_order_id,
6539                    None,
6540                    UUID4::new(),
6541                    UnixNanos::default(),
6542                    None,
6543                    None,
6544                )
6545            })
6546            .collect();
6547
6548        let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6549            trader_id,
6550            None,
6551            strategy_id,
6552            instrument_id,
6553            cancels,
6554            UUID4::new(),
6555            UnixNanos::default(),
6556            None,
6557            None,
6558        ));
6559
6560        node.observe_exec_command_before_dispatch(&command);
6561        advance_clock(Duration::from_millis(101)).await;
6562        let result = node.exec_manager.check_inflight_orders();
6563        let timed_out_ids = result
6564            .events
6565            .iter()
6566            .map(OrderEventAny::client_order_id)
6567            .collect::<IndexSet<_>>();
6568
6569        assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6570        assert_eq!(result.events.len(), child_ids.len());
6571        assert!(
6572            result
6573                .events
6574                .iter()
6575                .all(|event| matches!(event, OrderEventAny::Canceled(_))),
6576            "batch-cancel children must time out as Canceled events",
6577        );
6578    }
6579
6580    #[rstest]
6581    #[cfg_attr(
6582        not(all(feature = "simulation", madsim)),
6583        tokio::test(start_paused = true)
6584    )]
6585    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6586    async fn test_risk_bound_command_does_not_register_inflight() {
6587        let config = LiveNodeConfig {
6588            exec_engine: crate::config::LiveExecutionEngineConfig {
6589                reconciliation: true,
6590                inflight_check_threshold_ms: 100,
6591                inflight_check_retries: 2,
6592                ..Default::default()
6593            },
6594            ..Default::default()
6595        };
6596
6597        let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6598        msgbus::register_trading_command_endpoint(
6599            MessagingSwitchboard::risk_engine_execute(),
6600            TypedIntoHandler::from(|_: TradingCommand| {}),
6601        );
6602        let instrument = crypto_perpetual_ethusdt();
6603        let instrument_id = instrument.id();
6604        let order = OrderTestBuilder::new(OrderType::Limit)
6605            .trader_id(node.trader_id())
6606            .strategy_id(StrategyId::from("S-RISK-DENIED"))
6607            .instrument_id(instrument_id)
6608            .side(OrderSide::Buy)
6609            .quantity(Quantity::from("1.000"))
6610            .price(Price::from("100.00"))
6611            .build();
6612        let client_order_id = order.client_order_id();
6613
6614        {
6615            let mut cache = node.kernel.cache.borrow_mut();
6616            cache
6617                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6618                .unwrap();
6619            cache.add_order(order.clone(), None, None, false).unwrap();
6620        }
6621
6622        let submit_order = SubmitOrder::new(
6623            order.trader_id(),
6624            None,
6625            order.strategy_id(),
6626            instrument_id,
6627            client_order_id,
6628            order.init_event().clone(),
6629            None,
6630            None,
6631            None,
6632            UUID4::new(),
6633            UnixNanos::default(),
6634            None,
6635        );
6636        node.process_exec_command(
6637            TradingCommandMessage::new(
6638                MessagingSwitchboard::risk_engine_execute(),
6639                TradingCommand::SubmitOrder(submit_order),
6640            )
6641            .into(),
6642        );
6643
6644        advance_clock(Duration::from_millis(101)).await;
6645        let result = node.exec_manager.check_inflight_orders();
6646        let status = node
6647            .kernel
6648            .cache
6649            .borrow()
6650            .order(&client_order_id)
6651            .unwrap()
6652            .status();
6653
6654        assert_eq!(status, OrderStatus::Initialized);
6655        assert_eq!(
6656            node.exec_manager.recon_check_retry_count(&client_order_id),
6657            0
6658        );
6659        assert!(result.events.is_empty());
6660        assert!(result.queries.is_empty());
6661    }
6662
6663    #[rstest]
6664    #[cfg_attr(
6665        not(all(feature = "simulation", madsim)),
6666        tokio::test(start_paused = true)
6667    )]
6668    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6669    async fn test_risk_approved_command_registers_inflight() {
6670        let config = LiveNodeConfig {
6671            risk_engine: crate::config::LiveRiskEngineConfig {
6672                bypass: true,
6673                ..Default::default()
6674            },
6675            exec_engine: crate::config::LiveExecutionEngineConfig {
6676                reconciliation: true,
6677                inflight_check_threshold_ms: 100,
6678                inflight_check_retries: 2,
6679                ..Default::default()
6680            },
6681            ..Default::default()
6682        };
6683
6684        let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
6685        msgbus::register_trading_command_endpoint(
6686            MessagingSwitchboard::exec_engine_execute(),
6687            TypedIntoHandler::from(|_: TradingCommand| {}),
6688        );
6689        let instrument = crypto_perpetual_ethusdt();
6690        let instrument_id = instrument.id();
6691        let order = OrderTestBuilder::new(OrderType::Limit)
6692            .trader_id(node.trader_id())
6693            .strategy_id(StrategyId::from("S-RISK-APPROVED"))
6694            .instrument_id(instrument_id)
6695            .side(OrderSide::Buy)
6696            .quantity(Quantity::from("1.000"))
6697            .price(Price::from("100.00"))
6698            .build();
6699        let client_order_id = order.client_order_id();
6700
6701        {
6702            let mut cache = node.kernel.cache.borrow_mut();
6703            cache
6704                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6705                .unwrap();
6706            cache.add_order(order.clone(), None, None, false).unwrap();
6707        }
6708
6709        let submit_order = SubmitOrder::new(
6710            order.trader_id(),
6711            None,
6712            order.strategy_id(),
6713            instrument_id,
6714            client_order_id,
6715            order.init_event().clone(),
6716            None,
6717            None,
6718            None,
6719            UUID4::new(),
6720            UnixNanos::default(),
6721            None,
6722        );
6723        node.process_exec_command(
6724            TradingCommandMessage::new(
6725                MessagingSwitchboard::risk_engine_execute(),
6726                TradingCommand::SubmitOrder(submit_order),
6727            )
6728            .into(),
6729        );
6730
6731        advance_clock(Duration::from_millis(101)).await;
6732        let result = node.exec_manager.check_inflight_orders();
6733
6734        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6735            panic!("expected one query order command");
6736        };
6737
6738        assert_eq!(query.client_order_id, client_order_id);
6739        assert_eq!(
6740            node.exec_manager.recon_check_retry_count(&client_order_id),
6741            1
6742        );
6743        assert!(result.events.is_empty());
6744    }
6745
6746    #[rstest]
6747    fn test_live_node_builder_clock_factory_drives_kernel_clock() {
6748        let calls = Rc::new(Cell::new(0usize));
6749        let calls_in_factory = calls.clone();
6750        let sentinel = UnixNanos::from(123_456_789_u64);
6751
6752        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6753            .unwrap()
6754            .with_reconciliation(false)
6755            .with_clock_factory(move || {
6756                calls_in_factory.set(calls_in_factory.get() + 1);
6757                let mut clock = VirtualClock::new();
6758                clock.advance_time(sentinel, true);
6759                Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
6760            })
6761            .build()
6762            .unwrap();
6763
6764        assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
6765        assert_eq!(calls.get(), 1);
6766    }
6767
6768    #[derive(Debug)]
6769    struct ReplayKernelEventStore {
6770        fail_restore: bool,
6771    }
6772
6773    impl KernelEventStore for ReplayKernelEventStore {
6774        fn restore_parent_cache(
6775            &mut self,
6776            _instance_id: UUID4,
6777            _cache: &mut Cache,
6778        ) -> anyhow::Result<()> {
6779            if self.fail_restore {
6780                anyhow::bail!("replay restore failed");
6781            }
6782
6783            Ok(())
6784        }
6785
6786        fn open(
6787            &mut self,
6788            _instance_id: UUID4,
6789            _components: &RegisteredComponents,
6790            _environment: Environment,
6791        ) -> anyhow::Result<()> {
6792            Ok(())
6793        }
6794
6795        fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
6796            None
6797        }
6798
6799        fn seal(&mut self, _ts_init: UnixNanos) {}
6800
6801        fn run_id(&self) -> Option<&str> {
6802            Some("replay-child")
6803        }
6804
6805        fn parent_run_id(&self) -> Option<&str> {
6806            Some("seed-run")
6807        }
6808
6809        fn is_event_store_replay_configured(&self) -> bool {
6810            true
6811        }
6812
6813        fn is_halted(&self) -> bool {
6814            false
6815        }
6816    }
6817
6818    #[derive(Debug)]
6819    struct TestStrategy {
6820        core: StrategyCore,
6821    }
6822
6823    impl TestStrategy {
6824        fn new(config: StrategyConfig) -> Self {
6825            Self {
6826                core: StrategyCore::new(config),
6827            }
6828        }
6829    }
6830
6831    impl DataActor for TestStrategy {}
6832
6833    nautilus_strategy!(TestStrategy, {
6834        fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
6835            self.core.config.external_order_instrument_ids.clone()
6836        }
6837    });
6838
6839    fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
6840        // load_state must be true: the kernel rejects event-store replay otherwise,
6841        // and LiveNodeConfig defaults it to false.
6842        let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
6843            .unwrap()
6844            .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
6845                reconciliation: false,
6846                ..Default::default()
6847            })
6848            .with_load_state(true)
6849            .with_name("TestKernel")
6850            .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
6851                Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
6852            });
6853
6854        builder.build().unwrap()
6855    }
6856
6857    #[rstest]
6858    fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
6859        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6860            .unwrap()
6861            .with_reconciliation(false)
6862            .with_delay_post_stop_secs(0)
6863            .with_timeout_connection(1)
6864            .build()
6865            .unwrap();
6866        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6867        let strategy_id = StrategyId::from("CLAIMS-001");
6868
6869        node.add_strategy(TestStrategy::new(StrategyConfig {
6870            strategy_id: Some(strategy_id),
6871            external_order_instrument_ids: Some(vec![instrument_id]),
6872            ..Default::default()
6873        }))
6874        .unwrap();
6875
6876        assert_eq!(
6877            node.exec_manager.get_external_order_claim(&instrument_id),
6878            Some(strategy_id)
6879        );
6880
6881        {
6882            let exec_engine = node.kernel().exec_engine.borrow();
6883            assert_eq!(
6884                exec_engine.get_external_order_claim(&instrument_id),
6885                Some(strategy_id)
6886            );
6887        }
6888    }
6889
6890    #[rstest]
6891    fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
6892        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6893            .unwrap()
6894            .with_reconciliation(false)
6895            .build()
6896            .unwrap();
6897        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6898        let strategy_id = StrategyId::from("CLAIMS-001");
6899
6900        node.register_external_order_claims(strategy_id, &[instrument_id])
6901            .unwrap();
6902
6903        assert_eq!(
6904            node.exec_manager.get_external_order_claim(&instrument_id),
6905            Some(strategy_id)
6906        );
6907        assert_eq!(
6908            node.kernel
6909                .exec_engine
6910                .borrow()
6911                .get_external_order_claim(&instrument_id),
6912            Some(strategy_id)
6913        );
6914    }
6915
6916    #[rstest]
6917    #[tokio::test]
6918    async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
6919        let mut node = live_node_with_replay_store(false);
6920        let instrument_id = InstrumentId::from("AUDUSD.SIM");
6921        let strategy_id = StrategyId::from("CLAIMS-001");
6922
6923        node.start().await.unwrap();
6924        assert_eq!(node.state(), NodeState::Running);
6925
6926        node.register_external_order_claims(strategy_id, &[instrument_id])
6927            .unwrap();
6928
6929        assert_eq!(
6930            node.exec_manager.get_external_order_claim(&instrument_id),
6931            Some(strategy_id)
6932        );
6933        assert_eq!(
6934            node.kernel
6935                .exec_engine
6936                .borrow()
6937                .get_external_order_claim(&instrument_id),
6938            Some(strategy_id)
6939        );
6940    }
6941
6942    #[rstest]
6943    fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
6944        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6945            .unwrap()
6946            .with_reconciliation(false)
6947            .build()
6948            .unwrap();
6949        let existing_instrument = InstrumentId::from("AUDUSD.SIM");
6950        let new_instruments = [
6951            InstrumentId::from("EURUSD.SIM"),
6952            InstrumentId::from("GBPUSD.SIM"),
6953        ];
6954        let existing_strategy_id = StrategyId::from("CLAIMS-001");
6955        let new_strategy_id = StrategyId::from("CLAIMS-002");
6956        node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
6957            .unwrap();
6958
6959        let result = node.register_external_order_claims(
6960            new_strategy_id,
6961            &[new_instruments[0], existing_instrument, new_instruments[1]],
6962        );
6963
6964        assert!(result.is_err());
6965        assert_eq!(
6966            node.exec_manager
6967                .get_external_order_claim(&existing_instrument),
6968            Some(existing_strategy_id)
6969        );
6970
6971        for instrument_id in new_instruments {
6972            assert_eq!(
6973                node.exec_manager.get_external_order_claim(&instrument_id),
6974                None
6975            );
6976            assert_eq!(
6977                node.kernel
6978                    .exec_engine
6979                    .borrow()
6980                    .get_external_order_claim(&instrument_id),
6981                None
6982            );
6983        }
6984    }
6985
6986    #[rstest]
6987    fn test_deregister_external_order_claims_allows_successor_to_claim() {
6988        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
6989            .unwrap()
6990            .with_reconciliation(false)
6991            .build()
6992            .unwrap();
6993        let instruments = [
6994            InstrumentId::from("AUDUSD.SIM"),
6995            InstrumentId::from("EURUSD.SIM"),
6996        ];
6997        let first_strategy_id = StrategyId::from("CLAIMS-001");
6998        let successor_strategy_id = StrategyId::from("CLAIMS-002");
6999        node.register_external_order_claims(first_strategy_id, &instruments)
7000            .unwrap();
7001
7002        node.deregister_external_order_claims(first_strategy_id)
7003            .unwrap();
7004        node.register_external_order_claims(successor_strategy_id, &instruments)
7005            .unwrap();
7006
7007        for instrument_id in instruments {
7008            assert_eq!(
7009                node.exec_manager.get_external_order_claim(&instrument_id),
7010                Some(successor_strategy_id)
7011            );
7012            assert_eq!(
7013                node.kernel
7014                    .exec_engine
7015                    .borrow()
7016                    .get_external_order_claim(&instrument_id),
7017                Some(successor_strategy_id)
7018            );
7019        }
7020    }
7021
7022    #[rstest]
7023    fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7024        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7025            .unwrap()
7026            .with_reconciliation(false)
7027            .build()
7028            .unwrap();
7029        let strategy_id = StrategyId::from("CLAIMS-001");
7030
7031        node.deregister_external_order_claims(strategy_id).unwrap();
7032        node.deregister_external_order_claims(strategy_id).unwrap();
7033    }
7034
7035    #[rstest]
7036    fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7037        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7038            .unwrap()
7039            .with_reconciliation(false)
7040            .with_delay_post_stop_secs(0)
7041            .with_timeout_connection(1)
7042            .build()
7043            .unwrap();
7044        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7045        let strategy_id = StrategyId::from("CLAIMS-001");
7046        let duplicate_strategy_id = StrategyId::from("OTHER-002");
7047
7048        node.add_strategy(TestStrategy::new(StrategyConfig {
7049            strategy_id: Some(strategy_id),
7050            external_order_instrument_ids: Some(vec![instrument_id]),
7051            ..Default::default()
7052        }))
7053        .unwrap();
7054
7055        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7056            strategy_id: Some(duplicate_strategy_id),
7057            external_order_instrument_ids: Some(vec![instrument_id]),
7058            ..Default::default()
7059        }));
7060
7061        assert!(result.is_err());
7062        assert!(
7063            result
7064                .unwrap_err()
7065                .to_string()
7066                .contains("already exists for CLAIMS-001")
7067        );
7068        assert_eq!(
7069            node.exec_manager.get_external_order_claim(&instrument_id),
7070            Some(strategy_id)
7071        );
7072
7073        {
7074            let exec_engine = node.kernel().exec_engine.borrow();
7075            assert_eq!(
7076                exec_engine.get_external_order_claim(&instrument_id),
7077                Some(strategy_id)
7078            );
7079        }
7080    }
7081
7082    #[rstest]
7083    fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7084        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7085            .unwrap()
7086            .with_reconciliation(false)
7087            .with_delay_post_stop_secs(0)
7088            .with_timeout_connection(1)
7089            .build()
7090            .unwrap();
7091        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7092        let strategy_id = StrategyId::from("CLAIMS-001");
7093
7094        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7095            strategy_id: Some(strategy_id),
7096            external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7097            ..Default::default()
7098        }));
7099
7100        assert!(result.is_err());
7101        assert!(
7102            result
7103                .unwrap_err()
7104                .to_string()
7105                .contains("appears more than once for CLAIMS-001")
7106        );
7107        assert_eq!(
7108            node.exec_manager.get_external_order_claim(&instrument_id),
7109            None
7110        );
7111
7112        {
7113            let exec_engine = node.kernel().exec_engine.borrow();
7114            assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7115        }
7116    }
7117
7118    #[rstest]
7119    fn test_add_strategy_failure_restores_external_order_claims() {
7120        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7121            .unwrap()
7122            .with_reconciliation(false)
7123            .with_delay_post_stop_secs(0)
7124            .with_timeout_connection(1)
7125            .build()
7126            .unwrap();
7127        let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7128        let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7129        let strategy_id = StrategyId::from("CLAIMS-001");
7130        node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7131            .unwrap();
7132
7133        let mut strategy = TestStrategy::new(StrategyConfig {
7134            strategy_id: Some(strategy_id),
7135            external_order_instrument_ids: Some(vec![configured_instrument_id]),
7136            ..Default::default()
7137        });
7138
7139        strategy
7140            .core
7141            .register(
7142                node.trader_id(),
7143                node.kernel.clock(),
7144                node.kernel.cache.clone(),
7145                node.kernel.portfolio.clone(),
7146            )
7147            .unwrap();
7148
7149        let result = node.add_strategy(strategy);
7150
7151        assert!(result.is_err());
7152        assert!(
7153            result
7154                .unwrap_err()
7155                .to_string()
7156                .contains("already registered with trader")
7157        );
7158        assert_eq!(
7159            node.exec_manager
7160                .get_external_order_claim(&existing_instrument_id),
7161            Some(strategy_id)
7162        );
7163        assert_eq!(
7164            node.kernel
7165                .exec_engine
7166                .borrow()
7167                .get_external_order_claim(&existing_instrument_id),
7168            Some(strategy_id)
7169        );
7170        assert_eq!(
7171            node.exec_manager
7172                .get_external_order_claim(&configured_instrument_id),
7173            None
7174        );
7175        assert_eq!(
7176            node.kernel
7177                .exec_engine
7178                .borrow()
7179                .get_external_order_claim(&configured_instrument_id),
7180            None
7181        );
7182    }
7183
7184    #[rstest]
7185    fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7186        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7187            .unwrap()
7188            .with_reconciliation(false)
7189            .build()
7190            .unwrap();
7191        let exec_engine = node.kernel.exec_engine.clone();
7192        let _engine_borrow = exec_engine.borrow_mut();
7193
7194        node.add_strategy(TestStrategy::new(StrategyConfig {
7195            strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7196            ..Default::default()
7197        }))
7198        .unwrap();
7199    }
7200
7201    #[rstest]
7202    fn test_add_strategy_registers_configured_hedging_oms_type() {
7203        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7204            .unwrap()
7205            .with_reconciliation(false)
7206            .with_delay_post_stop_secs(0)
7207            .with_timeout_connection(1)
7208            .build()
7209            .unwrap();
7210        let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7211
7212        node.add_strategy(TestStrategy::new(StrategyConfig {
7213            strategy_id: Some(strategy_id),
7214            oms_type: Some(OmsType::Hedging),
7215            ..Default::default()
7216        }))
7217        .unwrap();
7218
7219        let instrument = crypto_perpetual_ethusdt();
7220        let instrument_id = instrument.id();
7221        let client_id = ClientId::from("STUB");
7222
7223        node.kernel
7224            .cache
7225            .borrow_mut()
7226            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7227            .unwrap();
7228        node.kernel
7229            .exec_engine
7230            .borrow_mut()
7231            .register_client(Box::new(StubExecutionClient::new(
7232                client_id,
7233                AccountId::from("TEST-ACCOUNT"),
7234                instrument_id.venue,
7235                OmsType::Netting,
7236                None,
7237            )))
7238            .unwrap();
7239
7240        let order = OrderTestBuilder::new(OrderType::Market)
7241            .trader_id(node.trader_id())
7242            .strategy_id(strategy_id)
7243            .instrument_id(instrument_id)
7244            .quantity(Quantity::from("1.000"))
7245            .build();
7246        let position_id = PositionId::new("CUSTOM-POSITION-001");
7247
7248        node.kernel
7249            .cache
7250            .borrow_mut()
7251            .add_order(order.clone(), Some(position_id), Some(client_id), true)
7252            .unwrap();
7253
7254        let submit_order = SubmitOrder::new(
7255            order.trader_id(),
7256            Some(client_id),
7257            strategy_id,
7258            instrument_id,
7259            order.client_order_id(),
7260            order.init_event().clone(),
7261            order.exec_algorithm_id(),
7262            Some(position_id),
7263            None,
7264            UUID4::new(),
7265            UnixNanos::default(),
7266            None,
7267        );
7268
7269        node.kernel
7270            .exec_engine
7271            .borrow()
7272            .execute(TradingCommand::SubmitOrder(submit_order));
7273
7274        let exec_engine = node.kernel.exec_engine.borrow();
7275        let cache = exec_engine.cache().borrow();
7276        let cached_order = cache
7277            .order(&order.client_order_id())
7278            .expect("Order should be cached");
7279
7280        assert_eq!(cached_order.status(), OrderStatus::Initialized);
7281    }
7282
7283    #[cfg(all(feature = "simulation", madsim))]
7284    async fn advance_clock(d: Duration) {
7285        madsim::time::advance(d);
7286        madsim::task::yield_now().await;
7287    }
7288
7289    #[cfg(not(all(feature = "simulation", madsim)))]
7290    async fn advance_clock(d: Duration) {
7291        tokio::time::advance(d).await;
7292    }
7293
7294    #[rstest]
7295    #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7296    #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7297    #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7298    #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7299    #[case::unapplied_cancel_default(
7300        SubmissionRecoveryPolicy::ResolveLocally,
7301        false,
7302        Some(OrderStatus::PendingCancel),
7303        false
7304    )]
7305    #[case::unapplied_modify_default(
7306        SubmissionRecoveryPolicy::ResolveLocally,
7307        false,
7308        Some(OrderStatus::PendingUpdate),
7309        false
7310    )]
7311    #[case::unapplied_cancel_diagnostics(
7312        SubmissionRecoveryPolicy::RetainUnresolved,
7313        false,
7314        Some(OrderStatus::PendingCancel),
7315        false
7316    )]
7317    #[case::unapplied_modify_diagnostics(
7318        SubmissionRecoveryPolicy::RetainUnresolved,
7319        false,
7320        Some(OrderStatus::PendingUpdate),
7321        false
7322    )]
7323    #[case::first_fill_cancel_default(
7324        SubmissionRecoveryPolicy::ResolveLocally,
7325        false,
7326        Some(OrderStatus::PendingCancel),
7327        true
7328    )]
7329    #[case::first_fill_modify_default(
7330        SubmissionRecoveryPolicy::ResolveLocally,
7331        false,
7332        Some(OrderStatus::PendingUpdate),
7333        true
7334    )]
7335    #[case::first_fill_cancel_diagnostics(
7336        SubmissionRecoveryPolicy::RetainUnresolved,
7337        false,
7338        Some(OrderStatus::PendingCancel),
7339        true
7340    )]
7341    #[case::first_fill_modify_diagnostics(
7342        SubmissionRecoveryPolicy::RetainUnresolved,
7343        false,
7344        Some(OrderStatus::PendingUpdate),
7345        true
7346    )]
7347    #[cfg_attr(
7348        not(all(feature = "simulation", madsim)),
7349        tokio::test(start_paused = true)
7350    )]
7351    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7352    async fn test_submission_registry_publishes_after_native_resolution(
7353        #[case] policy: SubmissionRecoveryPolicy,
7354        #[case] order_list: bool,
7355        #[case] pending_status: Option<OrderStatus>,
7356        #[case] first_fill: bool,
7357    ) {
7358        let config = LiveNodeConfig {
7359            exec_engine: crate::config::LiveExecutionEngineConfig {
7360                inflight_check_threshold_ms: 100,
7361                inflight_check_retries: 2,
7362                submission_recovery_policy: policy,
7363                ..Default::default()
7364            },
7365            ..Default::default()
7366        };
7367        let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7368        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7369        let client_id = ClientId::from("TEST-SUBMISSION");
7370        let account_id = AccountId::from("TEST-001");
7371        let strategy_id = StrategyId::from("SUBMISSION-001");
7372        let account = AccountAny::Margin(MarginAccount::new(
7373            AccountState::new(
7374                account_id,
7375                AccountType::Margin,
7376                vec![AccountBalance::new(
7377                    Money::from("1000000 USDT"),
7378                    Money::from("0 USDT"),
7379                    Money::from("1000000 USDT"),
7380                )],
7381                Vec::new(),
7382                true,
7383                UUID4::new(),
7384                UnixNanos::default(),
7385                UnixNanos::default(),
7386                Some(Currency::USDT()),
7387            ),
7388            true,
7389        ));
7390        node.kernel.cache.borrow_mut().add_account(account).unwrap();
7391        node.kernel
7392            .cache
7393            .borrow_mut()
7394            .add_instrument(instrument.clone())
7395            .unwrap();
7396        let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7397            .map(|index| {
7398                OrderTestBuilder::new(OrderType::Limit)
7399                    .trader_id(node.trader_id())
7400                    .strategy_id(strategy_id)
7401                    .client_order_id(ClientOrderId::from(
7402                        format!("O-SUBMISSION-{index}").as_str(),
7403                    ))
7404                    .instrument_id(instrument.id())
7405                    .quantity(Quantity::from("1.000"))
7406                    .price(Price::from("100.00"))
7407                    .build()
7408            })
7409            .collect();
7410        let command = if order_list {
7411            TradingCommand::SubmitOrderList(SubmitOrderList::new(
7412                node.trader_id(),
7413                Some(client_id),
7414                strategy_id,
7415                OrderList::new(
7416                    OrderListId::from("OL-SUBMISSION"),
7417                    instrument.id(),
7418                    strategy_id,
7419                    orders.iter().map(Order::client_order_id).collect(),
7420                    UnixNanos::default(),
7421                ),
7422                orders
7423                    .iter()
7424                    .map(|order| order.init_event().clone())
7425                    .collect(),
7426                None,
7427                None,
7428                None,
7429                UUID4::new(),
7430                UnixNanos::default(),
7431                None,
7432            ))
7433        } else {
7434            TradingCommand::SubmitOrder(SubmitOrder::from_order(
7435                &orders[0],
7436                node.trader_id(),
7437                Some(client_id),
7438                None,
7439                UUID4::new(),
7440                UnixNanos::default(),
7441            ))
7442        };
7443        node.observe_exec_command_before_dispatch(&command);
7444
7445        for order in &orders {
7446            let submitted = TestOrderEventStubs::submitted(order, account_id);
7447            node.kernel
7448                .cache
7449                .borrow_mut()
7450                .add_order(order.clone(), None, Some(client_id), false)
7451                .unwrap();
7452            node.process_exec_event(ExecutionEvent::Order(submitted));
7453        }
7454
7455        let expected_status = match pending_status {
7456            Some(status) if !first_fill => status,
7457            Some(_) => OrderStatus::Canceled,
7458            None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7459            None => OrderStatus::Rejected,
7460        };
7461        let diagnostics = Rc::new(RefCell::new(Vec::new()));
7462        let handler = ShareableMessageHandler::from_typed({
7463            let diagnostics = diagnostics.clone();
7464            let cache = node.kernel.cache.clone();
7465            move |diagnostic: &SubmissionRecoveryExhausted| {
7466                let cache = cache.borrow_mut();
7467                let order = cache.order(&diagnostic.client_order_id).unwrap();
7468                assert_eq!(order.status(), expected_status);
7469                assert!(diagnostic.ts_event >= order.ts_last());
7470                diagnostics.borrow_mut().push(diagnostic.clone());
7471            }
7472        });
7473        let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7474        msgbus::subscribe_any(topic.into(), handler.clone(), None);
7475        let queries = Rc::new(RefCell::new(Vec::new()));
7476        msgbus::register_trading_command_endpoint(
7477            MessagingSwitchboard::exec_engine_execute(),
7478            TypedIntoHandler::from({
7479                let queries = queries.clone();
7480                move |command: TradingCommand| queries.borrow_mut().push(command)
7481            }),
7482        );
7483
7484        let mut last_inflight_check = dst::time::Instant::now();
7485        let mut last_open_check = last_inflight_check;
7486        let mut last_position_check = last_inflight_check;
7487        let mut open_order_report_task = None;
7488        let mut targeted_order_report_task = None;
7489        let mut position_report_task = None;
7490
7491        for check in 1..=3 {
7492            advance_clock(Duration::from_millis(101)).await;
7493            node.run_reconciliation_checks(
7494                dst::time::Instant::now(),
7495                ReconciliationCheckIntervals {
7496                    inflight: Duration::from_millis(100),
7497                    open: Duration::ZERO,
7498                    position: Duration::ZERO,
7499                },
7500                &mut ReconciliationCheckState {
7501                    last_inflight_check: &mut last_inflight_check,
7502                    last_open_check: &mut last_open_check,
7503                    last_position_check: &mut last_position_check,
7504                    open_order_report_task: &mut open_order_report_task,
7505                    targeted_order_report_task: &mut targeted_order_report_task,
7506                    position_report_task: &mut position_report_task,
7507                },
7508            );
7509
7510            if check == 1 {
7511                assert!(diagnostics.borrow().is_empty());
7512
7513                if let Some(pending_status) = pending_status {
7514                    let order = &orders[0];
7515                    let pending = if pending_status == OrderStatus::PendingCancel {
7516                        OrderEventAny::PendingCancel(
7517                            OrderPendingCancelSpec::builder()
7518                                .trader_id(node.trader_id())
7519                                .strategy_id(strategy_id)
7520                                .instrument_id(instrument.id())
7521                                .client_order_id(order.client_order_id())
7522                                .account_id(account_id)
7523                                .build(),
7524                        )
7525                    } else {
7526                        OrderEventAny::PendingUpdate(
7527                            OrderPendingUpdateSpec::builder()
7528                                .trader_id(node.trader_id())
7529                                .strategy_id(strategy_id)
7530                                .instrument_id(instrument.id())
7531                                .client_order_id(order.client_order_id())
7532                                .account_id(account_id)
7533                                .build(),
7534                        )
7535                    };
7536                    node.process_exec_event(ExecutionEvent::Order(pending));
7537                    node.exec_manager.register_inflight(order.client_order_id());
7538                    if first_fill {
7539                        let cached = node
7540                            .kernel
7541                            .cache
7542                            .borrow()
7543                            .order(&order.client_order_id())
7544                            .unwrap()
7545                            .clone();
7546                        let OrderEventAny::Filled(mut fill) =
7547                            OrderFilledTestBuilder::new(&cached, &instrument)
7548                                .last_qty(Quantity::from("0.400"))
7549                                .last_px(Price::from("100.00"))
7550                                .commission(Money::from("0 USDT"))
7551                                .without_position_id()
7552                                .build()
7553                        else {
7554                            unreachable!()
7555                        };
7556                        fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7557                        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7558                    } else {
7559                        let report = OrderStatusReport::new(
7560                            account_id,
7561                            instrument.id(),
7562                            Some(order.client_order_id()),
7563                            VenueOrderId::from("V-SUBMISSION"),
7564                            Some(order.order_side()),
7565                            order.order_type(),
7566                            order.time_in_force(),
7567                            OrderStatus::Submitted,
7568                            order.quantity(),
7569                            Quantity::zero(3),
7570                            UnixNanos::default(),
7571                            UnixNanos::default(),
7572                            UnixNanos::default(),
7573                            None,
7574                        )
7575                        .with_price(Price::from("100.00"));
7576                        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
7577                            Box::new(report),
7578                        )));
7579                    }
7580                    let cache = node.kernel.cache.borrow();
7581                    let cached = cache.order(&order.client_order_id()).unwrap();
7582                    assert_eq!(cached.status(), pending_status);
7583                    assert_eq!(
7584                        cached.filled_qty(),
7585                        if first_fill {
7586                            Quantity::from("0.400")
7587                        } else {
7588                            Quantity::zero(3)
7589                        }
7590                    );
7591                    assert!(
7592                        !cached
7593                            .events()
7594                            .iter()
7595                            .any(|event| matches!(event, OrderEventAny::Accepted(_)))
7596                    );
7597                }
7598            }
7599        }
7600
7601        assert_eq!(
7602            queries.borrow().len(),
7603            orders.len()
7604                + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
7605        );
7606
7607        for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
7608            assert!(matches!(command, TradingCommand::QueryOrder(query)
7609                if query.client_order_id == order.client_order_id()
7610                && query.client_id == Some(client_id)));
7611            assert_eq!(
7612                node.kernel
7613                    .cache
7614                    .borrow()
7615                    .order(&order.client_order_id())
7616                    .unwrap()
7617                    .status(),
7618                if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
7619                    pending_status.unwrap()
7620                } else {
7621                    expected_status
7622                }
7623            );
7624        }
7625        let diagnostics = diagnostics.borrow();
7626        assert_eq!(
7627            diagnostics.len(),
7628            if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
7629                orders.len()
7630            } else {
7631                0
7632            }
7633        );
7634
7635        for (diagnostic, order) in diagnostics.iter().zip(&orders) {
7636            assert_eq!(diagnostic.trader_id, node.trader_id());
7637            assert_eq!(diagnostic.client_id, Some(client_id));
7638            assert_eq!(diagnostic.strategy_id, strategy_id);
7639            assert_eq!(diagnostic.instrument_id, instrument.id());
7640            assert_eq!(diagnostic.client_order_id, order.client_order_id());
7641            assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
7642            assert_eq!(diagnostic.retry_count, 2);
7643        }
7644        msgbus::unsubscribe_any(topic.into(), &handler);
7645        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
7646    }
7647
7648    #[derive(Debug)]
7649    struct SubmissionCallbackStrategy {
7650        core: StrategyCore,
7651        cancel: bool,
7652        batch: bool,
7653        callback: &'static str,
7654        client_id: ClientId,
7655    }
7656
7657    impl DataActor for SubmissionCallbackStrategy {}
7658
7659    impl SubmissionCallbackStrategy {
7660        fn queue_command(&mut self, client_order_id: ClientOrderId) {
7661            match (self.cancel, self.batch) {
7662                (true, true) => {
7663                    self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
7664                        .unwrap();
7665                }
7666                (true, false) => {
7667                    self.cancel_order(client_order_id, Some(self.client_id), None)
7668                        .unwrap();
7669                }
7670                (false, true) => {
7671                    self.modify_orders(
7672                        vec![(client_order_id, None, Some(Price::from("101.00")), None)],
7673                        Some(self.client_id),
7674                        None,
7675                    )
7676                    .unwrap();
7677                }
7678                (false, false) => self
7679                    .modify_order(
7680                        client_order_id,
7681                        None,
7682                        Some(Price::from("101.00")),
7683                        None,
7684                        Some(self.client_id),
7685                        None,
7686                    )
7687                    .unwrap(),
7688            }
7689        }
7690    }
7691
7692    nautilus_strategy!(SubmissionCallbackStrategy, {
7693        fn on_order_accepted(&mut self, event: OrderAccepted) {
7694            if self.callback == "accepted" {
7695                self.queue_command(event.client_order_id);
7696            }
7697        }
7698
7699        fn on_order_updated(&mut self, event: OrderUpdated) {
7700            if self.callback == "updated" {
7701                self.queue_command(event.client_order_id);
7702            }
7703        }
7704    });
7705
7706    #[rstest]
7707    #[case::native(false, "accepted")]
7708    #[case::report(true, "accepted")]
7709    #[case::updated(false, "updated")]
7710    #[cfg_attr(
7711        not(all(feature = "simulation", madsim)),
7712        tokio::test(start_paused = true)
7713    )]
7714    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7715    async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
7716        #[values(
7717            SubmissionRecoveryPolicy::ResolveLocally,
7718            SubmissionRecoveryPolicy::RetainUnresolved
7719        )]
7720        policy: SubmissionRecoveryPolicy,
7721        #[values(false, true)] cancel: bool,
7722        #[values(false, true)] batch: bool,
7723        #[case] report: bool,
7724        #[case] callback: &'static str,
7725    ) {
7726        let config = LiveNodeConfig {
7727            exec_engine: crate::config::LiveExecutionEngineConfig {
7728                inflight_check_threshold_ms: 100,
7729                inflight_check_retries: 2,
7730                submission_recovery_policy: policy,
7731                ..Default::default()
7732            },
7733            ..Default::default()
7734        };
7735        let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
7736        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7737        let account_id = AccountId::from("TEST-001");
7738        let strategy_id = StrategyId::from("CALLBACK-001");
7739        let client_id = ClientId::from("TEST-CALLBACK");
7740        let account = AccountAny::Margin(MarginAccount::new(
7741            AccountState::new(
7742                account_id,
7743                AccountType::Margin,
7744                vec![AccountBalance::new(
7745                    Money::from("1000000 USDT"),
7746                    Money::from("0 USDT"),
7747                    Money::from("1000000 USDT"),
7748                )],
7749                Vec::new(),
7750                true,
7751                UUID4::new(),
7752                UnixNanos::default(),
7753                UnixNanos::default(),
7754                Some(Currency::USDT()),
7755            ),
7756            true,
7757        ));
7758        node.kernel.cache.borrow_mut().add_account(account).unwrap();
7759        node.kernel
7760            .cache
7761            .borrow_mut()
7762            .add_instrument(instrument.clone())
7763            .unwrap();
7764        node.kernel
7765            .exec_engine
7766            .borrow_mut()
7767            .register_client(Box::new(StubExecutionClient::new(
7768                client_id,
7769                account_id,
7770                instrument.id().venue,
7771                OmsType::Netting,
7772                None,
7773            )))
7774            .unwrap();
7775        node.add_strategy(SubmissionCallbackStrategy {
7776            core: StrategyCore::new(StrategyConfig {
7777                strategy_id: Some(strategy_id),
7778                ..Default::default()
7779            }),
7780            cancel,
7781            batch,
7782            callback,
7783            client_id,
7784        })
7785        .unwrap();
7786        node.kernel.trader.borrow_mut().initialize().unwrap();
7787        node.kernel.start_trader().unwrap();
7788        let runner = node.runner.take().unwrap();
7789        runner.bind_senders_for_node(node.handle.clone());
7790        let mut channels = runner.take_channels();
7791        let order = OrderTestBuilder::new(OrderType::Limit)
7792            .trader_id(node.trader_id())
7793            .strategy_id(strategy_id)
7794            .client_order_id(ClientOrderId::from("O-CALLBACK"))
7795            .instrument_id(instrument.id())
7796            .quantity(Quantity::from("1.000"))
7797            .price(Price::from("100.00"))
7798            .build();
7799        let client_order_id = order.client_order_id();
7800        let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
7801            &order,
7802            node.trader_id(),
7803            Some(client_id),
7804            None,
7805            UUID4::new(),
7806            UnixNanos::default(),
7807        ));
7808        node.observe_exec_command_before_dispatch(&submit);
7809        node.kernel
7810            .cache
7811            .borrow_mut()
7812            .add_order(order.clone(), None, Some(client_id), false)
7813            .unwrap();
7814        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
7815            &order, account_id,
7816        )));
7817        advance_clock(Duration::from_millis(101)).await;
7818        let first = node.exec_manager.check_inflight_orders();
7819        assert_eq!(first.queries.len(), 1);
7820        assert!(first.events.is_empty());
7821
7822        advance_clock(Duration::from_millis(101)).await;
7823        let accepted =
7824            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
7825
7826        if callback == "updated" {
7827            let mut update = OrderUpdatedSpec::builder()
7828                .trader_id(node.trader_id())
7829                .strategy_id(strategy_id)
7830                .instrument_id(instrument.id())
7831                .client_order_id(client_order_id)
7832                .account_id(account_id)
7833                .quantity(order.quantity())
7834                .price(order.price().unwrap())
7835                .build();
7836            update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
7837            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
7838        } else if report {
7839            let accepted = OrderStatusReport::new(
7840                account_id,
7841                instrument.id(),
7842                Some(client_order_id),
7843                VenueOrderId::from("V-CALLBACK"),
7844                Some(order.order_side()),
7845                order.order_type(),
7846                order.time_in_force(),
7847                OrderStatus::Accepted,
7848                order.quantity(),
7849                Quantity::zero(3),
7850                UnixNanos::default(),
7851                UnixNanos::default(),
7852                UnixNanos::default(),
7853                None,
7854            )
7855            .with_price(Price::from("100.00"));
7856            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
7857                accepted,
7858            ))));
7859        } else {
7860            node.process_exec_event(ExecutionEvent::Order(accepted));
7861        }
7862
7863        let pending_status = if cancel {
7864            OrderStatus::PendingCancel
7865        } else {
7866            OrderStatus::PendingUpdate
7867        };
7868        assert_eq!(channels.exec_cmd_rx.len(), 1);
7869        assert_eq!(
7870            node.kernel
7871                .cache
7872                .borrow()
7873                .order(&client_order_id)
7874                .unwrap()
7875                .status(),
7876            pending_status
7877        );
7878
7879        // Maintenance may win the runner selection before the queued command dispatches
7880        for _ in 0..3 {
7881            advance_clock(Duration::from_millis(101)).await;
7882            let maintenance = node.exec_manager.check_inflight_orders();
7883            assert!(maintenance.events.is_empty());
7884            assert!(maintenance.queries.is_empty());
7885            assert!(
7886                node.exec_manager
7887                    .take_submission_recovery_exhaustions()
7888                    .is_empty()
7889            );
7890        }
7891
7892        let command = channels.exec_cmd_rx.try_recv().unwrap();
7893        node.process_exec_command(command);
7894        assert!(channels.exec_cmd_rx.is_empty());
7895        let immediate = node.exec_manager.check_inflight_orders();
7896        assert!(immediate.events.is_empty());
7897        assert!(immediate.queries.is_empty());
7898        advance_clock(Duration::from_millis(101)).await;
7899        let first_command_check = node.exec_manager.check_inflight_orders();
7900        assert!(first_command_check.events.is_empty());
7901        assert_eq!(first_command_check.queries.len(), 1);
7902        advance_clock(Duration::from_millis(101)).await;
7903        let exhausted = node.exec_manager.check_inflight_orders();
7904        assert!(exhausted.queries.is_empty());
7905        assert!(
7906            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
7907            if event.client_order_id == client_order_id)
7908        );
7909        assert!(
7910            node.exec_manager
7911                .take_submission_recovery_exhaustions()
7912                .is_empty()
7913        );
7914        node.process_reconciliation_events(&exhausted.events);
7915        assert_eq!(
7916            node.kernel
7917                .cache
7918                .borrow()
7919                .order(&client_order_id)
7920                .unwrap()
7921                .status(),
7922            OrderStatus::Canceled
7923        );
7924    }
7925
7926    #[cfg_attr(
7927        not(all(feature = "simulation", madsim)),
7928        tokio::test(start_paused = true)
7929    )]
7930    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7931    async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
7932        let last = dst::time::Instant::now();
7933        let interval = Duration::from_millis(100);
7934
7935        assert!(!reconciliation_check_due(last, last, Duration::ZERO));
7936        assert!(!reconciliation_check_due(last, last, interval));
7937
7938        advance_clock(Duration::from_millis(99)).await;
7939        let before_interval = dst::time::Instant::now();
7940        assert!(!reconciliation_check_due(before_interval, last, interval));
7941
7942        advance_clock(Duration::from_millis(1)).await;
7943        let at_interval = dst::time::Instant::now();
7944        assert!(reconciliation_check_due(at_interval, last, interval));
7945
7946        assert!(!reconciliation_check_due(last, at_interval, interval));
7947    }
7948
7949    #[cfg_attr(
7950        not(all(feature = "simulation", madsim)),
7951        tokio::test(start_paused = true)
7952    )]
7953    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7954    async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
7955        let config = LiveNodeConfig {
7956            exec_engine: crate::config::LiveExecutionEngineConfig {
7957                reconciliation: true,
7958                open_check_interval_secs: Some(1.0),
7959                position_check_interval_secs: Some(1.0),
7960                max_single_order_queries_per_cycle: 5,
7961                ..Default::default()
7962            },
7963            ..Default::default()
7964        };
7965
7966        let mut node =
7967            LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
7968        let client_id = ClientId::from("TEST-QUERY");
7969        let account_id = AccountId::from("TEST-QUERY-001");
7970
7971        let trading_commands = Rc::new(RefCell::new(Vec::new()));
7972        msgbus::register_trading_command_endpoint(
7973            MessagingSwitchboard::exec_engine_execute(),
7974            TypedIntoHandler::from({
7975                let trading_commands = trading_commands.clone();
7976                move |command: TradingCommand| {
7977                    trading_commands.borrow_mut().push(command);
7978                }
7979            }),
7980        );
7981
7982        let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
7983        let instrument = crypto_perpetual_ethusdt();
7984        let instrument_id = instrument.id();
7985        let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
7986
7987        node.kernel
7988            .cache
7989            .borrow_mut()
7990            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7991            .unwrap();
7992        insert_accepted_limit_order_in_node(
7993            &node,
7994            account_id,
7995            client_id,
7996            instrument_id,
7997            client_order_id,
7998            venue_order_id,
7999        );
8000
8001        let last = dst::time::Instant::now();
8002        advance_clock(Duration::from_nanos(1)).await;
8003        let now = dst::time::Instant::now();
8004        let mut last_inflight_check = last;
8005        let mut last_open_check = last;
8006        let mut last_position_check = last;
8007        let mut open_order_report_task = None;
8008        let mut targeted_order_report_task = None;
8009        let mut position_report_task = None;
8010
8011        node.run_reconciliation_checks(
8012            now,
8013            ReconciliationCheckIntervals {
8014                inflight: Duration::ZERO,
8015                open: Duration::from_nanos(1),
8016                position: Duration::ZERO,
8017            },
8018            &mut ReconciliationCheckState {
8019                last_inflight_check: &mut last_inflight_check,
8020                last_open_check: &mut last_open_check,
8021                last_position_check: &mut last_position_check,
8022                open_order_report_task: &mut open_order_report_task,
8023                targeted_order_report_task: &mut targeted_order_report_task,
8024                position_report_task: &mut position_report_task,
8025            },
8026        );
8027
8028        let commands = trading_commands.borrow();
8029
8030        assert!(commands.is_empty());
8031        assert!(open_order_report_task.is_none());
8032        assert!(targeted_order_report_task.is_none());
8033        assert!(position_report_task.is_none());
8034
8035        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8036    }
8037
8038    #[rstest]
8039    #[tokio::test]
8040    async fn test_recurring_report_failure_preserves_client_coverage(
8041        #[values(false, true)] positions: bool,
8042        #[values(false, true)] failure_first: bool,
8043    ) {
8044        let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8045        let failed_id = ClientId::from("FAILED");
8046        let healthy_id = ClientId::from("HEALTHY");
8047
8048        for (client_id, outcome) in [
8049            (failed_id, FillReportClientOutcome::Failure),
8050            (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8051        ] {
8052            node.exec_clients
8053                .push(LiveExecutionClient::new(Box::new(FillReportClient {
8054                    client_id,
8055                    account_id: AccountId::from("TEST-001"),
8056                    venue: Venue::from("BINANCE"),
8057                    outcome,
8058                    commands: Rc::new(RefCell::new(Vec::new())),
8059                })));
8060        }
8061
8062        if !failure_first {
8063            node.exec_clients.reverse();
8064        }
8065
8066        let last = dst::time::Instant::now();
8067        let now = last + Duration::from_secs(1);
8068        let mut last_inflight = last;
8069        let mut last_open = last;
8070        let mut last_position = last;
8071        let mut open_task = None;
8072        let mut targeted_task = None;
8073        let mut position_task = None;
8074
8075        node.run_reconciliation_checks(
8076            now,
8077            ReconciliationCheckIntervals {
8078                inflight: Duration::ZERO,
8079                open: if positions {
8080                    Duration::ZERO
8081                } else {
8082                    Duration::from_secs(1)
8083                },
8084                position: if positions {
8085                    Duration::from_secs(1)
8086                } else {
8087                    Duration::ZERO
8088                },
8089            },
8090            &mut ReconciliationCheckState {
8091                last_inflight_check: &mut last_inflight,
8092                last_open_check: &mut last_open,
8093                last_position_check: &mut last_position,
8094                open_order_report_task: &mut open_task,
8095                targeted_order_report_task: &mut targeted_task,
8096                position_report_task: &mut position_task,
8097            },
8098        );
8099
8100        let (queried_clients, failed_clients) = if positions {
8101            let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8102                position_task.unwrap().future.await
8103            else {
8104                panic!("position collection should complete despite one client failure");
8105            };
8106
8107            let result = result.finish();
8108            assert!(result.reports.is_empty());
8109            assert!(open_task.is_none());
8110            (result.queried_clients, result.failed_clients)
8111        } else {
8112            let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8113                panic!("order collection should complete despite one client failure");
8114            };
8115
8116            let result = result.finish();
8117
8118            assert!(result.reports.is_empty());
8119            assert!(position_task.is_none());
8120            (result.queried_clients, result.failed_clients)
8121        };
8122
8123        assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8124        assert_eq!(failed_clients, IndexSet::from([failed_id]));
8125        assert!(targeted_task.is_none());
8126    }
8127
8128    #[rstest]
8129    #[case::inflight(true, false)]
8130    #[case::open(false, false)]
8131    #[case::positions(false, true)]
8132    #[tokio::test]
8133    async fn test_reconciliation_shutdown_preserves_pending_checks(
8134        #[case] inflight: bool,
8135        #[case] positions: bool,
8136    ) {
8137        let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8138        node.handle.set_shutting_down();
8139        let last = dst::time::Instant::now();
8140        let now = last + Duration::from_secs(1);
8141        let mut last_inflight = last;
8142        let mut last_open = last;
8143        let mut last_position = last;
8144        let mut open_task = None;
8145        let mut targeted_task = None;
8146        let mut position_task = None;
8147
8148        node.run_reconciliation_checks(
8149            now,
8150            ReconciliationCheckIntervals {
8151                inflight: if inflight {
8152                    Duration::from_secs(1)
8153                } else {
8154                    Duration::ZERO
8155                },
8156                open: if !inflight && !positions {
8157                    Duration::from_secs(1)
8158                } else {
8159                    Duration::ZERO
8160                },
8161                position: if positions {
8162                    Duration::from_secs(1)
8163                } else {
8164                    Duration::ZERO
8165                },
8166            },
8167            &mut ReconciliationCheckState {
8168                last_inflight_check: &mut last_inflight,
8169                last_open_check: &mut last_open,
8170                last_position_check: &mut last_position,
8171                open_order_report_task: &mut open_task,
8172                targeted_order_report_task: &mut targeted_task,
8173                position_report_task: &mut position_task,
8174            },
8175        );
8176
8177        assert_eq!(
8178            (last_inflight, last_open, last_position),
8179            (last, last, last)
8180        );
8181        assert!(open_task.is_none());
8182        assert!(targeted_task.is_none());
8183        assert!(position_task.is_none());
8184    }
8185
8186    fn insert_accepted_limit_order_in_node(
8187        node: &LiveNode,
8188        account_id: AccountId,
8189        client_id: ClientId,
8190        instrument_id: InstrumentId,
8191        client_order_id: ClientOrderId,
8192        venue_order_id: VenueOrderId,
8193    ) {
8194        let order = OrderTestBuilder::new(OrderType::Limit)
8195            .client_order_id(client_order_id)
8196            .instrument_id(instrument_id)
8197            .quantity(Quantity::from("10.0"))
8198            .price(Price::from("100.0"))
8199            .build();
8200        let submitted = TestOrderEventStubs::submitted(&order, account_id);
8201        node.kernel
8202            .cache
8203            .borrow_mut()
8204            .add_order(order, None, Some(client_id), false)
8205            .unwrap();
8206        let order = node
8207            .kernel
8208            .cache
8209            .borrow_mut()
8210            .update_order(&submitted)
8211            .unwrap();
8212        let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8213        node.kernel
8214            .cache
8215            .borrow_mut()
8216            .update_order(&accepted)
8217            .unwrap();
8218    }
8219
8220    fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8221        let config = LiveNodeConfig {
8222            exec_engine: crate::config::LiveExecutionEngineConfig {
8223                reconciliation: true,
8224                ..Default::default()
8225            },
8226            ..Default::default()
8227        };
8228
8229        let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8230        let account_id = AccountId::from("TEST-001");
8231        let client_id = ClientId::from("TEST-RECENT-FILL");
8232        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8233        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8234        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8235        node.kernel
8236            .cache
8237            .borrow_mut()
8238            .add_instrument(instrument.clone())
8239            .unwrap();
8240        insert_accepted_limit_order_in_node(
8241            &node,
8242            account_id,
8243            client_id,
8244            instrument.id(),
8245            client_order_id,
8246            venue_order_id,
8247        );
8248        let order = node
8249            .kernel
8250            .cache
8251            .borrow()
8252            .order_owned(&client_order_id)
8253            .unwrap();
8254        let fill = TestOrderEventStubs::filled(
8255            &order,
8256            &instrument,
8257            Some(TradeId::from("T-RECENT-FILL")),
8258            None,
8259            Some(Price::from("100.0")),
8260            Some(Quantity::from("1.0")),
8261            Some(LiquiditySide::Taker),
8262            None,
8263            None,
8264            Some(account_id),
8265        );
8266
8267        (node, fill, instrument)
8268    }
8269
8270    fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8271        let client_order_id = fill_report.client_order_id.unwrap();
8272        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8273        let order = node
8274            .kernel
8275            .cache
8276            .borrow()
8277            .order_owned(&client_order_id)
8278            .unwrap();
8279        let order_report = OrderStatusReport::new(
8280            fill_report.account_id,
8281            fill_report.instrument_id,
8282            Some(client_order_id),
8283            fill_report.venue_order_id,
8284            OrderSide::Buy.into(),
8285            OrderType::Limit,
8286            TimeInForce::Gtc,
8287            OrderStatus::PartiallyFilled,
8288            Quantity::from("10.0"),
8289            Quantity::from("2.0"),
8290            UnixNanos::from(1_000),
8291            UnixNanos::from(1_000),
8292            UnixNanos::from(1_000),
8293            None,
8294        )
8295        .with_avg_px(dec!(100.0));
8296        let inferred = create_inferred_fill_for_qty(
8297            &order,
8298            &order_report,
8299            &fill_report.account_id,
8300            &instrument,
8301            Quantity::from("1.0"),
8302            UnixNanos::from(1_000),
8303            None,
8304        )
8305        .unwrap();
8306
8307        node.process_reconciliation_events(&[inferred]);
8308    }
8309
8310    #[rstest]
8311    #[case::rejected(false, "10.0", "10.0", "1.0")]
8312    #[case::dust(true, "9.004", "10.004", "10.004")]
8313    #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8314    fn test_position_check_overfill_raises_before_fill_when_client_allows(
8315        #[case] allow: bool,
8316        #[case] fill_qty: &str,
8317        #[case] expected_qty: &str,
8318        #[case] expected_filled: &str,
8319    ) {
8320        let (mut node, venue_report, mut fill_report) =
8321            position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8322        fill_report.commission = Money::from("1.25 USDT");
8323        let key = (fill_report.instrument_id, fill_report.account_id);
8324
8325        let client = StubExecutionClient::new(
8326            ClientId::from("POSITION-FILLS"),
8327            AccountId::from("TEST-001"),
8328            fill_report.instrument_id.venue,
8329            OmsType::Netting,
8330            None,
8331        );
8332
8333        let client = if allow {
8334            client.with_allows_reconciliation_overfill()
8335        } else {
8336            client
8337        };
8338
8339        node.kernel
8340            .exec_engine
8341            .borrow_mut()
8342            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8343            .unwrap();
8344
8345        let position_result = position_report_result(&node, venue_report);
8346        node.handle_position_fill_report_result(PositionFillReportResult {
8347            position_result,
8348            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8349        });
8350
8351        let order = node
8352            .kernel
8353            .cache
8354            .borrow()
8355            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8356            .unwrap();
8357        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8358        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8359        let applied = order
8360            .trade_ids()
8361            .iter()
8362            .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8363        assert_eq!(applied, allow);
8364        let revision = node.exec_manager.position_activity_revision(&key);
8365
8366        if !allow {
8367            return;
8368        }
8369
8370        assert_eq!(revision, 2);
8371
8372        {
8373            let cache = node.kernel.cache.borrow();
8374            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8375            assert_eq!(positions.len(), 1);
8376            assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8377            assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8378            assert_eq!(positions[0].trade_ids.len(), 2);
8379        }
8380
8381        let replay_report = position_report_result(
8382            &node,
8383            PositionStatusReport::new(
8384                key.1,
8385                key.0,
8386                PositionSide::Long,
8387                Quantity::from(expected_qty),
8388                UnixNanos::from(2_000),
8389                UnixNanos::from(2_000),
8390                None,
8391                None,
8392                Some(dec!(100.0)),
8393            ),
8394        );
8395        node.handle_position_fill_report_result(PositionFillReportResult {
8396            position_result: replay_report,
8397            reports: IndexMap::from([(key, vec![fill_report])]),
8398        });
8399
8400        let order = node
8401            .kernel
8402            .cache
8403            .borrow()
8404            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8405            .unwrap();
8406        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8407        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8408        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8409    }
8410
8411    #[rstest]
8412    fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8413        let (mut node, venue_report, mut overfill) =
8414            position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8415        overfill.commission = Money::from("0.20 USDT");
8416        let key = (overfill.instrument_id, overfill.account_id);
8417        let client = StubExecutionClient::new(
8418            ClientId::from("POSITION-FILLS"),
8419            AccountId::from("TEST-001"),
8420            overfill.instrument_id.venue,
8421            OmsType::Netting,
8422            None,
8423        )
8424        .with_allows_reconciliation_overfill();
8425        node.kernel
8426            .exec_engine
8427            .borrow_mut()
8428            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8429            .unwrap();
8430
8431        let mut fitting = overfill.clone();
8432        fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8433        fitting.last_qty = Quantity::from("8.0");
8434        fitting.commission = Money::from("0.80 USDT");
8435        let position_result = position_report_result(&node, venue_report);
8436        node.handle_position_fill_report_result(PositionFillReportResult {
8437            position_result,
8438            reports: IndexMap::from([(key, vec![fitting, overfill])]),
8439        });
8440
8441        let order = node
8442            .kernel
8443            .cache
8444            .borrow()
8445            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8446            .unwrap();
8447        assert_eq!(order.quantity(), Quantity::from("11.0"));
8448        assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8449        assert_eq!(order.trade_ids().len(), 3);
8450        assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8451        {
8452            let cache = node.kernel.cache.borrow();
8453            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8454            assert_eq!(positions.len(), 1);
8455            assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8456            assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8457        }
8458    }
8459
8460    #[rstest]
8461    fn test_position_check_second_overfill_does_not_raise_filled_order() {
8462        let (mut node, venue_report, mut first) =
8463            position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8464        first.commission = Money::from("1.20 USDT");
8465        let key = (first.instrument_id, first.account_id);
8466        let client = StubExecutionClient::new(
8467            ClientId::from("POSITION-FILLS"),
8468            AccountId::from("TEST-001"),
8469            first.instrument_id.venue,
8470            OmsType::Netting,
8471            None,
8472        )
8473        .with_allows_reconciliation_overfill();
8474        node.kernel
8475            .exec_engine
8476            .borrow_mut()
8477            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8478            .unwrap();
8479
8480        let mut second = first.clone();
8481        second.trade_id = TradeId::from("T-POSITION-LATER");
8482        second.last_qty = Quantity::from("1.0");
8483        second.commission = Money::from("0.10 USDT");
8484        second.ts_event = UnixNanos::from(1_001);
8485        let position_result = position_report_result(&node, venue_report);
8486        node.handle_position_fill_report_result(PositionFillReportResult {
8487            position_result,
8488            reports: IndexMap::from([(key, vec![first, second])]),
8489        });
8490
8491        let order = node
8492            .kernel
8493            .cache
8494            .borrow()
8495            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8496            .unwrap();
8497        assert_eq!(order.quantity(), Quantity::from("13.0"));
8498        assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8499        assert_eq!(order.status(), OrderStatus::Filled);
8500        assert_eq!(order.trade_ids().len(), 2);
8501        assert!(
8502            order
8503                .trade_ids()
8504                .iter()
8505                .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8506        );
8507        {
8508            let cache = node.kernel.cache.borrow();
8509            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8510            assert_eq!(positions.len(), 1);
8511            assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8512            assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8513        }
8514    }
8515
8516    #[rstest]
8517    fn test_allow_overfills_applies_fill_without_raising_quantity() {
8518        let config = LiveNodeConfig {
8519            exec_engine: crate::config::LiveExecutionEngineConfig {
8520                reconciliation: true,
8521                allow_overfills: true,
8522                ..Default::default()
8523            },
8524            ..Default::default()
8525        };
8526
8527        let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8528        let account_id = AccountId::from("TEST-001");
8529        let client_id = ClientId::from("POSITION-FILLS");
8530        let client_order_id = ClientOrderId::from("O-ALLOW");
8531        let venue_order_id = VenueOrderId::from("V-ALLOW");
8532        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8533        node.kernel
8534            .cache
8535            .borrow_mut()
8536            .add_instrument(instrument.clone())
8537            .unwrap();
8538        insert_accepted_limit_order_in_node(
8539            &node,
8540            account_id,
8541            client_id,
8542            instrument.id(),
8543            client_order_id,
8544            venue_order_id,
8545        );
8546        let client = StubExecutionClient::new(
8547            client_id,
8548            account_id,
8549            instrument.id().venue,
8550            OmsType::Netting,
8551            None,
8552        )
8553        .with_allows_reconciliation_overfill();
8554        node.kernel
8555            .exec_engine
8556            .borrow_mut()
8557            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8558            .unwrap();
8559
8560        let fill = FillReport::new(
8561            account_id,
8562            instrument.id(),
8563            venue_order_id,
8564            TradeId::from("T-ALLOW"),
8565            OrderSide::Buy,
8566            Quantity::from("12.0"),
8567            Price::from("100.0"),
8568            Money::from("1.00 USDT"),
8569            LiquiditySide::Taker,
8570            Some(client_order_id),
8571            None,
8572            UnixNanos::from(1_000),
8573            UnixNanos::from(1_000),
8574            None,
8575        );
8576        node.kernel
8577            .exec_engine
8578            .borrow_mut()
8579            .reconcile_fill_report(&fill);
8580
8581        let order = node
8582            .kernel
8583            .cache
8584            .borrow()
8585            .order_owned(&client_order_id)
8586            .unwrap();
8587        assert_eq!(order.quantity(), Quantity::from("10.0"));
8588        assert_eq!(order.filled_qty(), Quantity::from("12.0"));
8589        assert_eq!(order.trade_ids().len(), 1);
8590    }
8591
8592    fn position_fill_test_fixture(
8593        name: &str,
8594        authoritative_qty: Quantity,
8595    ) -> (LiveNode, PositionStatusReport, FillReport) {
8596        position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
8597    }
8598
8599    fn position_fill_test_fixture_with_threshold(
8600        name: &str,
8601        authoritative_qty: Quantity,
8602        position_check_threshold_ms: u32,
8603    ) -> (LiveNode, PositionStatusReport, FillReport) {
8604        let config = LiveNodeConfig {
8605            exec_engine: crate::config::LiveExecutionEngineConfig {
8606                reconciliation: true,
8607                position_check_threshold_ms,
8608                ..Default::default()
8609            },
8610            ..Default::default()
8611        };
8612
8613        let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8614        let account_id = AccountId::from("TEST-001");
8615        let client_id = ClientId::from("POSITION-FILLS");
8616        let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
8617        let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
8618        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8619        let account = AccountAny::Margin(MarginAccount::new(
8620            AccountState::new(
8621                account_id,
8622                AccountType::Margin,
8623                vec![AccountBalance::new(
8624                    Money::from("1000000 USDT"),
8625                    Money::from("0 USDT"),
8626                    Money::from("1000000 USDT"),
8627                )],
8628                Vec::new(),
8629                true,
8630                UUID4::new(),
8631                UnixNanos::default(),
8632                UnixNanos::default(),
8633                Some(Currency::USDT()),
8634            ),
8635            true,
8636        ));
8637        node.kernel.cache.borrow_mut().add_account(account).unwrap();
8638        node.kernel
8639            .cache
8640            .borrow_mut()
8641            .add_instrument(instrument.clone())
8642            .unwrap();
8643        insert_accepted_limit_order_in_node(
8644            &node,
8645            account_id,
8646            client_id,
8647            instrument.id(),
8648            client_order_id,
8649            venue_order_id,
8650        );
8651        let order = node
8652            .kernel
8653            .cache
8654            .borrow()
8655            .order_owned(&client_order_id)
8656            .unwrap();
8657        let mut initial_fill = TestOrderEventStubs::filled(
8658            &order,
8659            &instrument,
8660            Some(TradeId::from("T-POSITION-INITIAL")),
8661            None,
8662            Some(Price::from("100.0")),
8663            Some(Quantity::from("1.0")),
8664            Some(LiquiditySide::Taker),
8665            None,
8666            None,
8667            Some(account_id),
8668        );
8669
8670        let OrderEventAny::Filled(fill) = &mut initial_fill else {
8671            unreachable!();
8672        };
8673
8674        fill.commission = Some(Money::zero(instrument.quote_currency()));
8675        node.process_reconciliation_events(&[initial_fill]);
8676
8677        let ts_event = UnixNanos::from(1_000);
8678
8679        let fill_report = FillReport::new(
8680            account_id,
8681            instrument.id(),
8682            venue_order_id,
8683            TradeId::from("T-POSITION-AUTHORITATIVE"),
8684            OrderSide::Buy,
8685            authoritative_qty,
8686            Price::from("100.0"),
8687            Money::zero(instrument.quote_currency()),
8688            LiquiditySide::Taker,
8689            Some(client_order_id),
8690            None,
8691            ts_event,
8692            ts_event,
8693            None,
8694        );
8695
8696        let venue_report = PositionStatusReport::new(
8697            account_id,
8698            instrument.id(),
8699            PositionSide::Long,
8700            Quantity::from("2.0"),
8701            ts_event,
8702            ts_event,
8703            None,
8704            None,
8705            Some(dec!(100.0)),
8706        );
8707
8708        (node, venue_report, fill_report)
8709    }
8710
8711    fn position_report_result(
8712        node: &LiveNode,
8713        report: PositionStatusReport,
8714    ) -> PositionReportResult {
8715        let client_id = ClientId::from("POSITION-FILLS");
8716        let key = (report.instrument_id, report.account_id);
8717        let mut check = node
8718            .exec_manager
8719            .prepare_position_report_check(UUID4::new(), &[]);
8720        check.client_coverage.insert(
8721            key,
8722            ReportClientCoverage::Resolved(IndexSet::from([client_id])),
8723        );
8724
8725        PositionReportResult {
8726            check,
8727            reports: vec![report],
8728            queried_clients: IndexSet::from([client_id]),
8729            failed_clients: IndexSet::new(),
8730        }
8731    }
8732
8733    fn position_quantity_after_check(
8734        node: &mut LiveNode,
8735        venue_report: &PositionStatusReport,
8736        reports: &[FillReport],
8737    ) -> Quantity {
8738        let key = (venue_report.instrument_id, venue_report.account_id);
8739        let position_result = position_report_result(node, venue_report.clone());
8740        node.handle_position_fill_report_result(PositionFillReportResult {
8741            position_result,
8742            reports: IndexMap::from([(key, reports.to_vec())]),
8743        });
8744
8745        node.kernel
8746            .cache
8747            .borrow()
8748            .positions_open(None, Some(&key.0), None, Some(&key.1), None)
8749            .iter()
8750            .map(|position| position.quantity)
8751            .sum()
8752    }
8753
8754    fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
8755        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
8756            fill.account_id,
8757            fill.instrument_id,
8758            fill.venue_order_id,
8759            fill.trade_id,
8760            fill.order_side,
8761            fill.last_qty,
8762            fill.last_px,
8763            fill.commission
8764                .unwrap_or_else(|| Money::zero(fill.currency)),
8765            fill.liquidity_side,
8766            Some(fill.client_order_id),
8767            fill.position_id,
8768            fill.ts_event,
8769            fill.ts_init,
8770            None,
8771        ))))
8772    }
8773
8774    fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
8775        node.exec_manager.is_fill_recently_processed(
8776            fill.account_id,
8777            fill.instrument_id,
8778            fill.trade_id,
8779        )
8780    }
8781
8782    #[rstest]
8783    #[case(0, NodeState::Idle)]
8784    #[case(1, NodeState::Starting)]
8785    #[case(2, NodeState::Running)]
8786    #[case(3, NodeState::ShuttingDown)]
8787    #[case(4, NodeState::Stopped)]
8788    fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
8789        assert_eq!(NodeState::from_u8(value), expected);
8790    }
8791
8792    #[rstest]
8793    #[case(5)]
8794    #[case(255)]
8795    #[should_panic(expected = "Invalid NodeState value")]
8796    fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
8797        let _ = NodeState::from_u8(value);
8798    }
8799
8800    #[rstest]
8801    fn test_node_state_roundtrip() {
8802        for state in [
8803            NodeState::Idle,
8804            NodeState::Starting,
8805            NodeState::Running,
8806            NodeState::ShuttingDown,
8807            NodeState::Stopped,
8808        ] {
8809            assert_eq!(NodeState::from_u8(state.as_u8()), state);
8810        }
8811    }
8812
8813    #[rstest]
8814    fn test_node_state_is_running_only_for_running() {
8815        assert!(!NodeState::Idle.is_running());
8816        assert!(!NodeState::Starting.is_running());
8817        assert!(NodeState::Running.is_running());
8818        assert!(!NodeState::ShuttingDown.is_running());
8819        assert!(!NodeState::Stopped.is_running());
8820    }
8821
8822    #[rstest]
8823    #[tokio::test]
8824    async fn test_await_engines_connected_returns_stop_requested() {
8825        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8826        let handle = node.handle();
8827
8828        handle.stop();
8829
8830        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8831        let status = node.await_engines_connected(deadline).await;
8832
8833        assert_eq!(status, EngineConnectionStatus::StopRequested);
8834        assert!(handle.should_stop());
8835    }
8836
8837    #[rstest]
8838    #[tokio::test]
8839    async fn test_await_engines_connected_returns_shutdown_requested() {
8840        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
8841
8842        node.kernel().shutdown_flag().set(true);
8843
8844        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
8845        let status = node.await_engines_connected(deadline).await;
8846
8847        assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
8848    }
8849
8850    #[rstest]
8851    #[tokio::test]
8852    async fn test_start_stop_request_aborts_startup_without_running() {
8853        let config = LiveNodeConfig {
8854            exec_engine: crate::config::LiveExecutionEngineConfig {
8855                reconciliation: false,
8856                ..Default::default()
8857            },
8858            timeout_disconnection: Duration::from_millis(50),
8859            ..Default::default()
8860        };
8861
8862        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8863        let handle = node.handle();
8864
8865        handle.stop();
8866        node.start().await.unwrap();
8867
8868        assert_eq!(handle.state(), NodeState::Stopped);
8869        assert!(handle.should_stop());
8870        assert!(!handle.is_running());
8871    }
8872
8873    #[rstest]
8874    #[tokio::test(start_paused = true)]
8875    async fn test_stop_processes_residual_exec_event_during_grace_period() {
8876        let config = LiveNodeConfig {
8877            exec_engine: crate::config::LiveExecutionEngineConfig {
8878                reconciliation: false,
8879                ..Default::default()
8880            },
8881            timeout_connection: Duration::ZERO,
8882            timeout_reconciliation: Duration::ZERO,
8883            timeout_portfolio: Duration::ZERO,
8884            timeout_disconnection: Duration::ZERO,
8885            delay_post_stop: Duration::from_millis(20),
8886            timeout_shutdown: Duration::ZERO,
8887            ..Default::default()
8888        };
8889
8890        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
8891        let order = OrderTestBuilder::new(OrderType::Market)
8892            .instrument_id(InstrumentId::from("EUR/USD.SIM"))
8893            .quantity(Quantity::from("2"))
8894            .build();
8895        let client_order_id = order.client_order_id();
8896        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
8897
8898        node.kernel
8899            .cache()
8900            .borrow_mut()
8901            .add_order(order, None, None, false)
8902            .unwrap();
8903
8904        node.start().await.unwrap();
8905        let exec_event_sender = get_exec_event_sender();
8906
8907        let send_residual = tokio::spawn(async move {
8908            tokio::time::sleep(Duration::from_millis(1)).await;
8909            exec_event_sender
8910                .send(ExecutionEvent::Order(submitted))
8911                .unwrap();
8912        });
8913
8914        node.stop().await.unwrap();
8915        send_residual.await.unwrap();
8916
8917        assert_eq!(
8918            node.kernel
8919                .cache()
8920                .borrow()
8921                .order(&client_order_id)
8922                .unwrap()
8923                .status(),
8924            OrderStatus::Submitted
8925        );
8926
8927        node.dispose();
8928    }
8929
8930    #[rstest]
8931    #[tokio::test]
8932    async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
8933        let actor_id = ActorId::from("LIVE-STATE-ACTOR");
8934        let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
8935        let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
8936        let strategy_load =
8937            IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
8938        let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
8939        let strategy_save =
8940            IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
8941        let (database, control) = TestCacheDatabaseControl::create();
8942        control.set_actor_state(actor_id, &actor_load);
8943        control.set_strategy_state(strategy_id, &strategy_load);
8944
8945        let config = LiveNodeConfig {
8946            load_state: true,
8947            save_state: true,
8948            exec_engine: crate::config::LiveExecutionEngineConfig {
8949                reconciliation: false,
8950                ..Default::default()
8951            },
8952            timeout_connection: Duration::ZERO,
8953            timeout_reconciliation: Duration::ZERO,
8954            timeout_portfolio: Duration::ZERO,
8955            timeout_disconnection: Duration::ZERO,
8956            delay_post_stop: Duration::ZERO,
8957            timeout_shutdown: Duration::ZERO,
8958            ..Default::default()
8959        };
8960
8961        let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
8962        node.set_cache_database(Box::new(database)).unwrap();
8963        node.add_actor(StateActor::new(
8964            actor_id,
8965            control.clone(),
8966            actor_save.clone(),
8967        ))
8968        .unwrap();
8969        node.add_strategy(StateStrategy::new(
8970            strategy_id,
8971            control.clone(),
8972            strategy_save.clone(),
8973        ))
8974        .unwrap();
8975
8976        node.start().await.unwrap();
8977        node.stop().await.unwrap();
8978        node.dispose();
8979
8980        assert_eq!(
8981            control.events(),
8982            vec![
8983                "actor.load:LIVE-STATE-ACTOR",
8984                "actor.on_load",
8985                "strategy.load:LIVE-STATE-STRATEGY-001",
8986                "strategy.on_load",
8987                "actor.on_start",
8988                "strategy.on_start",
8989                "actor.on_stop",
8990                "strategy.on_stop",
8991                "actor.on_save",
8992                "actor.update:LIVE-STATE-ACTOR",
8993                "strategy.on_save",
8994                "strategy.update:LIVE-STATE-STRATEGY-001",
8995                "database.close",
8996            ]
8997        );
8998        assert_eq!(control.actor_state(&actor_id), Some(actor_save));
8999        assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9000        assert_eq!(node.state(), NodeState::Stopped);
9001    }
9002
9003    #[rstest]
9004    #[tokio::test]
9005    async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9006        let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9007        let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9008        let (database, control) = TestCacheDatabaseControl::create();
9009
9010        let config = LiveNodeConfig {
9011            save_state: true,
9012            exec_engine: crate::config::LiveExecutionEngineConfig {
9013                reconciliation: false,
9014                ..Default::default()
9015            },
9016            timeout_connection: Duration::ZERO,
9017            timeout_reconciliation: Duration::ZERO,
9018            timeout_portfolio: Duration::ZERO,
9019            timeout_disconnection: Duration::ZERO,
9020            delay_post_stop: Duration::ZERO,
9021            timeout_shutdown: Duration::ZERO,
9022            ..Default::default()
9023        };
9024
9025        let mut node =
9026            LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9027        node.set_cache_database(Box::new(database)).unwrap();
9028        node.add_actor(
9029            StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9030        )
9031        .unwrap();
9032        node.add_strategy(
9033            StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9034        )
9035        .unwrap();
9036
9037        node.start().await.unwrap();
9038        let error = node.stop().await.unwrap_err();
9039        node.dispose();
9040
9041        assert_eq!(
9042            error.to_string(),
9043            "failed while finalizing kernel shutdown: Failed to save component state: actor \
9044             LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9045             LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9046        );
9047        assert_eq!(
9048            control.events(),
9049            vec![
9050                "actor.on_start",
9051                "strategy.on_start",
9052                "actor.on_stop",
9053                "strategy.on_stop",
9054                "actor.on_save",
9055                "strategy.on_save",
9056                "database.close",
9057            ]
9058        );
9059        assert_eq!(node.state(), NodeState::Stopped);
9060    }
9061
9062    #[rstest]
9063    #[tokio::test]
9064    async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9065        let config = LiveNodeConfig {
9066            exec_engine: crate::config::LiveExecutionEngineConfig {
9067                reconciliation: false,
9068                ..Default::default()
9069            },
9070            timeout_connection: Duration::ZERO,
9071            timeout_reconciliation: Duration::ZERO,
9072            timeout_portfolio: Duration::ZERO,
9073            timeout_disconnection: Duration::ZERO,
9074            delay_post_stop: Duration::ZERO,
9075            timeout_shutdown: Duration::ZERO,
9076            ..Default::default()
9077        };
9078
9079        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9080        let order = OrderTestBuilder::new(OrderType::Market)
9081            .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9082            .quantity(Quantity::from("3"))
9083            .build();
9084        let client_order_id = order.client_order_id();
9085        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9086
9087        node.kernel
9088            .cache()
9089            .borrow_mut()
9090            .add_order(order, None, None, false)
9091            .unwrap();
9092
9093        node.start().await.unwrap();
9094        get_exec_event_sender()
9095            .send(ExecutionEvent::Order(submitted))
9096            .unwrap();
9097
9098        node.stop().await.unwrap();
9099
9100        assert_eq!(
9101            node.kernel
9102                .cache()
9103                .borrow()
9104                .order(&client_order_id)
9105                .unwrap()
9106                .status(),
9107            OrderStatus::Submitted
9108        );
9109
9110        node.dispose();
9111    }
9112
9113    #[rstest]
9114    #[tokio::test]
9115    async fn test_start_event_store_replay_skips_live_connections() {
9116        let mut node = live_node_with_replay_store(false);
9117        let handle = node.handle();
9118
9119        node.start().await.unwrap();
9120
9121        assert_eq!(handle.state(), NodeState::Running);
9122        assert!(handle.is_running());
9123        assert!(node.kernel.is_event_store_replay());
9124        assert!(node.runner.is_some());
9125    }
9126
9127    #[rstest]
9128    #[tokio::test]
9129    async fn test_start_event_store_replay_preserves_stop_request() {
9130        let mut node = live_node_with_replay_store(false);
9131        let handle = node.handle();
9132        handle.stop();
9133
9134        node.start().await.unwrap();
9135
9136        assert_eq!(handle.state(), NodeState::Stopped);
9137        assert!(handle.should_stop());
9138        assert!(node.kernel.is_event_store_replay());
9139        assert!(node.runner.is_some());
9140    }
9141
9142    #[rstest]
9143    #[tokio::test]
9144    async fn test_start_event_store_replay_config_failure_aborts_startup() {
9145        let mut node = live_node_with_replay_store(true);
9146        let handle = node.handle();
9147
9148        node.start().await.unwrap();
9149
9150        assert_eq!(handle.state(), NodeState::Stopped);
9151        assert!(!handle.is_running());
9152        assert!(node.kernel.is_event_store_replay_configured());
9153        assert!(!node.kernel.is_event_store_replay());
9154        assert!(node.runner.is_some());
9155    }
9156
9157    #[rstest]
9158    #[tokio::test]
9159    async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9160        let mut node = live_node_with_replay_store(false);
9161        let handle = node.handle();
9162
9163        node.run().await.unwrap();
9164
9165        assert_eq!(handle.state(), NodeState::Running);
9166        assert!(handle.is_running());
9167        assert!(node.kernel.is_event_store_replay());
9168        assert!(node.runner.is_none());
9169    }
9170
9171    #[rstest]
9172    #[tokio::test]
9173    async fn test_run_event_store_replay_preserves_stop_request() {
9174        let mut node = live_node_with_replay_store(false);
9175        let handle = node.handle();
9176        handle.stop();
9177
9178        node.run().await.unwrap();
9179
9180        assert_eq!(handle.state(), NodeState::Stopped);
9181        assert!(handle.should_stop());
9182        assert!(node.kernel.is_event_store_replay());
9183        assert!(node.runner.is_none());
9184    }
9185
9186    #[rstest]
9187    #[tokio::test]
9188    async fn test_run_event_store_replay_config_failure_aborts_startup() {
9189        let mut node = live_node_with_replay_store(true);
9190        let handle = node.handle();
9191
9192        node.run().await.unwrap();
9193
9194        assert_eq!(handle.state(), NodeState::Stopped);
9195        assert!(!handle.is_running());
9196        assert!(node.kernel.is_event_store_replay_configured());
9197        assert!(!node.kernel.is_event_store_replay());
9198        assert!(node.runner.is_none());
9199    }
9200
9201    #[rstest]
9202    fn test_build_rejects_event_store_config_without_factory() {
9203        let config = LiveNodeConfig {
9204            event_store: Some(EventStoreConfig::default()),
9205            exec_engine: crate::config::LiveExecutionEngineConfig {
9206                reconciliation: false,
9207                ..Default::default()
9208            },
9209            ..Default::default()
9210        };
9211
9212        let err = LiveNodeBuilder::from_config(config)
9213            .expect("builder")
9214            .build()
9215            .expect_err("should reject event_store config without factory");
9216
9217        assert!(
9218            err.to_string().contains("with_event_store"),
9219            "error message should mention with_event_store, was: {err}"
9220        );
9221    }
9222
9223    #[rstest]
9224    fn test_direct_build_rejects_event_store_config() {
9225        let config = LiveNodeConfig {
9226            event_store: Some(EventStoreConfig::default()),
9227            exec_engine: crate::config::LiveExecutionEngineConfig {
9228                reconciliation: false,
9229                ..Default::default()
9230            },
9231            ..Default::default()
9232        };
9233
9234        let err = LiveNode::build("TestNode".to_string(), Some(config))
9235            .expect_err("LiveNode::build should reject event_store config");
9236
9237        assert!(
9238            err.to_string().contains("with_event_store"),
9239            "error message should mention with_event_store, was: {err}"
9240        );
9241    }
9242
9243    #[rstest]
9244    fn test_dispose_before_start_is_idempotent() {
9245        let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9246        node.add_strategy(TestStrategy::new(StrategyConfig {
9247            strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9248            ..Default::default()
9249        }))
9250        .unwrap();
9251
9252        node.dispose();
9253        node.dispose();
9254
9255        assert!(node.kernel.trader().borrow().is_disposed());
9256        assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9257        assert_eq!(node.state(), NodeState::Stopped);
9258    }
9259
9260    #[rstest]
9261    fn test_dispose_releases_retained_callback_roots(
9262        #[values(false, true)] fatal: bool,
9263        #[values(false, true)] external: bool,
9264    ) {
9265        actor::clear_callbacks().unwrap();
9266        let directory =
9267            std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9268
9269        let config = LiveNodeConfig {
9270            logging: LoggerConfig {
9271                fileout_level: LevelFilter::Info,
9272                file_config: Some(FileWriterConfig {
9273                    directory: Some(directory.to_str().unwrap().to_string()),
9274                    file_name: Some("disposal".to_string()),
9275                    ..Default::default()
9276                }),
9277                ..Default::default()
9278            },
9279            ..Default::default()
9280        };
9281
9282        let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9283        let received = Rc::new(RefCell::new(Vec::new()));
9284        let observed = received.clone();
9285
9286        let retained = DispatchMessage::from(()).dispatch(|()| {
9287            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9288                TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9289                TimeEventCallback::RustLocal(Rc::new(move |_| {
9290                    observed.borrow_mut().push("delivered");
9291                })),
9292            ));
9293
9294            external.then(|| DispatchMessage::new((), std::thread::current().id()))
9295        });
9296
9297        if let Some(retained) = &retained {
9298            assert!(retained.is_rooted());
9299        }
9300
9301        assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9302
9303        if fatal {
9304            crate::dispatch::tests::latch_callback_failure();
9305        }
9306
9307        node.dispose();
9308
9309        assert!(node.runner.is_none());
9310        assert!(received.borrow().is_empty());
9311        assert_eq!(Rc::strong_count(&received), 1);
9312        assert_eq!(
9313            actor::callback_failure(),
9314            (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9315        );
9316        assert_eq!(
9317            actor::clear_callbacks(),
9318            if external {
9319                Err(CallbackDispatchError::Active)
9320            } else {
9321                Ok(())
9322            }
9323        );
9324
9325        logging_sync_to_disk().unwrap();
9326        let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9327        drop(retained);
9328        node.dispose();
9329
9330        assert_eq!(actor::callback_failure(), None);
9331        assert_eq!(actor::clear_callbacks(), Ok(()));
9332        drop(node);
9333        std::fs::remove_dir_all(directory).unwrap();
9334
9335        let errors: Vec<_> = output
9336            .lines()
9337            .filter(|line| line.contains("[ERROR]"))
9338            .filter_map(|line| {
9339                line.split_once(".nautilus_live::node: ")
9340                    .map(|(_, text)| text)
9341            })
9342            .collect();
9343
9344        let mut expected = Vec::new();
9345
9346        if fatal {
9347            expected.push(
9348                "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9349            );
9350        }
9351
9352        if external {
9353            expected.push(
9354                "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9355            );
9356        }
9357
9358        assert_eq!(errors, expected);
9359    }
9360
9361    #[tokio::test]
9362    async fn test_dispose_releases_stop_generated_callback_roots() {
9363        actor::clear_callbacks().unwrap();
9364
9365        let config = LiveNodeConfig {
9366            exec_engine: crate::config::LiveExecutionEngineConfig {
9367                reconciliation: false,
9368                ..Default::default()
9369            },
9370            timeout_connection: Duration::ZERO,
9371            timeout_reconciliation: Duration::ZERO,
9372            timeout_portfolio: Duration::ZERO,
9373            timeout_shutdown: Duration::ZERO,
9374            ..Default::default()
9375        };
9376
9377        let mut node =
9378            LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9379        let received = Rc::new(RefCell::new(Vec::new()));
9380        node.add_actor(StopCallbackActor {
9381            core: DataActorCore::new(DataActorConfig {
9382                actor_id: Some(ActorId::from("STOP-CALLBACK")),
9383                ..Default::default()
9384            }),
9385            received: received.clone(),
9386        })
9387        .unwrap();
9388
9389        node.start().await.unwrap();
9390
9391        assert!(node.kernel.trader().borrow().is_running());
9392        assert_eq!(actor::clear_callbacks(), Ok(()));
9393
9394        node.dispose();
9395
9396        assert_eq!(*received.borrow(), ["stop", "queued"]);
9397        assert_eq!(Rc::strong_count(&received), 1);
9398        assert!(node.runner.is_none());
9399        assert!(node.kernel.trader().borrow().is_disposed());
9400        assert_eq!(node.state(), NodeState::Stopped);
9401        assert_eq!(actor::clear_callbacks(), Ok(()));
9402    }
9403
9404    #[derive(Debug)]
9405    struct StopCallbackActor {
9406        core: DataActorCore,
9407        received: Rc<RefCell<Vec<&'static str>>>,
9408    }
9409
9410    nautilus_actor!(StopCallbackActor);
9411
9412    impl DataActor for StopCallbackActor {
9413        fn on_stop(&mut self) -> anyhow::Result<()> {
9414            self.received.borrow_mut().push("stop");
9415            let received = self.received.clone();
9416            DispatchMessage::from(()).dispatch(|()| {
9417                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9418                    TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9419                    TimeEventCallback::RustLocal(Rc::new(move |_| {
9420                        received.borrow_mut().push("delivered");
9421                    })),
9422                ));
9423            });
9424
9425            let clear_result = actor::clear_callbacks();
9426            anyhow::ensure!(
9427                clear_result == Err(CallbackDispatchError::Active),
9428                "Expected active callback roots during stop, received {clear_result:?}"
9429            );
9430            self.received.borrow_mut().push("queued");
9431            Ok(())
9432        }
9433    }
9434
9435    #[rstest]
9436    fn test_handle_initial_state() {
9437        let handle = LiveNodeHandle::new();
9438
9439        assert_eq!(handle.state(), NodeState::Idle);
9440        assert!(!handle.should_stop());
9441        assert!(!handle.is_running());
9442    }
9443
9444    #[rstest]
9445    fn test_handle_initial_metrics_snapshot_is_zero() {
9446        let handle = LiveNodeHandle::new();
9447
9448        assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9449    }
9450
9451    #[rstest]
9452    fn test_record_runner_dispatch_updates_selected_channel() {
9453        let metrics = RunnerMetrics::default();
9454        let dispatch_start = dst::time::Instant::now();
9455        let metrics_start = dispatch_start
9456            .checked_sub(Duration::from_micros(1))
9457            .expect("test instant should support a one-microsecond lookback");
9458
9459        record_runner_dispatch(
9460            &metrics,
9461            SystemChannel::DataCommands,
9462            dispatch_start,
9463            metrics_start,
9464        );
9465        let snapshot = metrics.snapshot();
9466
9467        assert_eq!(snapshot.time_events.dispatched, 0);
9468        assert_eq!(snapshot.exec_events.dispatched, 0);
9469        assert_eq!(snapshot.exec_commands.dispatched, 0);
9470        assert_eq!(snapshot.data_events.dispatched, 0);
9471        assert_eq!(snapshot.data_commands.dispatched, 1);
9472        assert_eq!(
9473            snapshot.data_commands.last_dispatch_at_ns,
9474            snapshot.elapsed_ns
9475        );
9476        assert_eq!(
9477            snapshot.data_commands.dispatch_busy_ns,
9478            snapshot.dispatch_busy_ns
9479        );
9480        assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9481    }
9482
9483    #[rstest]
9484    fn test_handle_stop_sets_flag() {
9485        let handle = LiveNodeHandle::new();
9486
9487        handle.stop();
9488
9489        assert!(handle.should_stop());
9490    }
9491
9492    #[rstest]
9493    fn test_handle_stop_blocks_running_transition() {
9494        let handle = LiveNodeHandle::new();
9495        handle.set_starting();
9496        handle.stop();
9497
9498        let transition = handle.try_set_running();
9499
9500        assert_eq!(transition, RunningTransition::StopRequested);
9501        assert_eq!(handle.state(), NodeState::Starting);
9502        assert!(handle.should_stop());
9503        assert!(!handle.is_running());
9504    }
9505
9506    #[rstest]
9507    fn test_handle_stop_after_running_transition_remains_pending() {
9508        let handle = LiveNodeHandle::new();
9509        handle.set_starting();
9510
9511        let transition = handle.try_set_running();
9512        handle.stop();
9513
9514        assert_eq!(transition, RunningTransition::Entered);
9515        assert_eq!(handle.state(), NodeState::Running);
9516        assert!(handle.should_stop());
9517        assert!(handle.is_running());
9518    }
9519
9520    #[rstest]
9521    fn test_handle_node_state_transitions() {
9522        let handle = LiveNodeHandle::new();
9523        assert_eq!(handle.state(), NodeState::Idle);
9524
9525        handle.set_starting();
9526        assert_eq!(handle.state(), NodeState::Starting);
9527        assert!(!handle.is_running());
9528
9529        assert_eq!(handle.try_set_running(), RunningTransition::Entered);
9530        assert_eq!(handle.state(), NodeState::Running);
9531        assert!(handle.is_running());
9532
9533        handle.set_shutting_down();
9534        assert_eq!(handle.state(), NodeState::ShuttingDown);
9535        assert!(!handle.is_running());
9536
9537        handle.set_stopped();
9538        assert_eq!(handle.state(), NodeState::Stopped);
9539        assert!(!handle.is_running());
9540    }
9541
9542    #[rstest]
9543    fn test_handle_clone_shares_state_bidirectionally() {
9544        let handle1 = LiveNodeHandle::new();
9545        let handle2 = handle1.clone();
9546
9547        handle1.set_starting();
9548        let transition = handle2.try_set_running();
9549        handle1.stop();
9550
9551        assert_eq!(transition, RunningTransition::Entered);
9552        assert_eq!(handle1.state(), NodeState::Running);
9553        assert!(handle2.should_stop());
9554    }
9555
9556    #[rstest]
9557    fn test_handle_stop_flag_survives_non_running_state_changes() {
9558        let handle = LiveNodeHandle::new();
9559
9560        handle.set_starting();
9561        handle.stop();
9562        handle.set_shutting_down();
9563        handle.set_stopped();
9564
9565        assert_eq!(handle.state(), NodeState::Stopped);
9566        assert!(handle.should_stop());
9567    }
9568
9569    #[rstest]
9570    fn test_builder_creation() {
9571        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9572
9573        assert!(result.is_ok());
9574    }
9575
9576    #[rstest]
9577    fn test_stream_processor_receives_unregistered_typed_payload() {
9578        let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9579            .unwrap()
9580            .build()
9581            .unwrap();
9582        let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
9583            InstrumentId::from("AUD/USD.SIM"),
9584            Some(ClientId::from("EXTERNAL")),
9585            Some(Venue::from("SIM")),
9586            UUID4::from("00000000-0000-4000-8000-000000000001"),
9587            UnixNanos::from(1),
9588            None,
9589            None,
9590        ));
9591        let expected = serde_json::to_value(&command).unwrap();
9592        let received = Rc::new(RefCell::new(Vec::new()));
9593        let steps = Rc::new(RefCell::new(Vec::new()));
9594        let received_processor = received.clone();
9595        let first_steps = steps.clone();
9596        node.add_stream_processor(move |message| {
9597            let command = message
9598                .downcast_ref::<SubscribeCommand>()
9599                .expect("processor must receive the decoded concrete command");
9600            received_processor
9601                .borrow_mut()
9602                .push(serde_json::to_value(command).unwrap());
9603            first_steps.borrow_mut().push(1);
9604        });
9605
9606        let second_steps = steps.clone();
9607        node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
9608        let republished = Rc::new(RefCell::new(Vec::new()));
9609        let republished_handler = republished.clone();
9610        let subscriber_steps = steps.clone();
9611        msgbus::subscribe_any(
9612            "external.test".into(),
9613            ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
9614                republished_handler
9615                    .borrow_mut()
9616                    .push(serde_json::to_value(command).unwrap());
9617                subscriber_steps.borrow_mut().push(3);
9618            }),
9619            None,
9620        );
9621
9622        let message = BusMessage::with_str_topic(
9623            "external.test",
9624            BusPayloadType::SubscribeCommand,
9625            Bytes::from(serde_json::to_vec(&command).unwrap()),
9626            SerializationEncoding::Json,
9627        );
9628
9629        assert!(
9630            !msgbus::get_message_bus()
9631                .borrow()
9632                .is_streaming_type(BusPayloadType::SubscribeCommand)
9633        );
9634        node.process_external_msgbus_message(&message);
9635
9636        assert_eq!(*received.borrow(), vec![expected.clone()]);
9637        assert_eq!(*steps.borrow(), vec![1, 2]);
9638        assert!(republished.borrow().is_empty());
9639
9640        received.borrow_mut().clear();
9641        steps.borrow_mut().clear();
9642        msgbus::get_message_bus()
9643            .borrow_mut()
9644            .add_streaming_type(BusPayloadType::SubscribeCommand);
9645        node.process_external_msgbus_message(&message);
9646
9647        assert_eq!(*received.borrow(), vec![expected.clone()]);
9648        assert_eq!(*republished.borrow(), vec![expected]);
9649        assert_eq!(*steps.borrow(), vec![1, 2, 3]);
9650        msgbus::get_message_bus().borrow_mut().dispose();
9651    }
9652
9653    #[rstest]
9654    fn test_builder_rejects_backtest() {
9655        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
9656
9657        assert!(result.is_err());
9658        assert!(result.unwrap_err().to_string().contains("Backtest"));
9659    }
9660
9661    #[rstest]
9662    fn test_builder_accepts_live_environment() {
9663        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
9664
9665        assert!(result.is_ok());
9666    }
9667
9668    #[rstest]
9669    fn test_builder_accepts_sandbox_environment() {
9670        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
9671
9672        assert!(result.is_ok());
9673    }
9674
9675    #[rstest]
9676    fn test_builder_fluent_api_chaining() {
9677        let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
9678            .unwrap()
9679            .with_name("TestNode")
9680            .with_instance_id(UUID4::new())
9681            .with_load_state(false)
9682            .with_save_state(true)
9683            .with_timeout_connection(30)
9684            .with_timeout_reconciliation(60)
9685            .with_reconciliation(true)
9686            .with_reconciliation_lookback_mins(120)
9687            .with_timeout_portfolio(10)
9688            .with_timeout_disconnection_secs(5)
9689            .with_delay_post_stop_secs(3)
9690            .with_delay_shutdown_secs(10);
9691
9692        assert_eq!(builder.name(), "TestNode");
9693    }
9694
9695    #[rstest]
9696    fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
9697        let (external_egress, publications, closed) = CapturingExternalEgress::new();
9698
9699        let msgbus_config = MessageBusConfig {
9700            encoding: SerializationEncoding::Json,
9701            ..Default::default()
9702        };
9703
9704        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9705            .unwrap()
9706            .with_msgbus_config(msgbus_config)
9707            .with_external_msgbus_egress(Box::new(external_egress))
9708            .build()
9709            .expect("node builds with external message bus egress");
9710        let quote = QuoteTick::default();
9711
9712        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9713
9714        let publications = publications.borrow();
9715        assert_eq!(publications.len(), 1);
9716        assert_eq!(publications[0].topic, "data.quotes.TEST");
9717        assert_eq!(
9718            serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9719                .expect("JSON payload must decode as QuoteTick"),
9720            quote
9721        );
9722        drop(publications);
9723
9724        msgbus::get_message_bus().borrow_mut().dispose();
9725        assert!(closed.get());
9726        drop(node);
9727    }
9728
9729    #[rstest]
9730    #[tokio::test(flavor = "current_thread")]
9731    async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
9732        let quote = QuoteTick::default();
9733        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9734        let publications = Arc::new(Mutex::new(Vec::new()));
9735        let closed = Arc::new(AtomicBool::new(false));
9736        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
9737
9738        let msgbus_config = MessageBusConfig {
9739            external_streams: Some(vec!["stream".to_string()]),
9740            ..Default::default()
9741        };
9742
9743        let config = LiveNodeConfig {
9744            environment: Environment::Sandbox,
9745            msgbus: Some(msgbus_config),
9746            exec_engine: crate::config::LiveExecutionEngineConfig {
9747                reconciliation: false,
9748                ..Default::default()
9749            },
9750            delay_post_stop: Duration::ZERO,
9751            timeout_connection: Duration::from_millis(500),
9752            timeout_disconnection: Duration::from_millis(500),
9753            ..Default::default()
9754        };
9755
9756        let mut node = LiveNodeBuilder::from_config(config)
9757            .unwrap()
9758            .with_external_msgbus_factory(Box::new(factory))
9759            .build()
9760            .expect("node builds with external message bus factory");
9761
9762        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9763        {
9764            let publications = publications.lock();
9765            assert_eq!(publications.len(), 1);
9766            assert_eq!(publications[0].topic, "data.quotes.TEST");
9767            assert_eq!(
9768                serde_json::from_slice::<QuoteTick>(&publications[0].payload)
9769                    .expect("JSON payload must decode as QuoteTick"),
9770                quote
9771            );
9772        }
9773
9774        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9775        let handle = node.handle();
9776
9777        // Stopping from `drive` rather than here, so the node cannot finish `run` before the
9778        // republished quote is observed.
9779        let handler = TypedHandler::from({
9780            let received = received.clone();
9781            move |quote: &QuoteTick| {
9782                received.borrow_mut().push(*quote);
9783            }
9784        });
9785
9786        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9787        msgbus::get_message_bus()
9788            .borrow_mut()
9789            .add_streaming_type(BusPayloadType::QuoteTick);
9790
9791        let payload =
9792            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
9793        let message = BusMessage::with_str_topic(
9794            "data.quotes.TEST",
9795            BusPayloadType::QuoteTick,
9796            payload,
9797            SerializationEncoding::Json,
9798        );
9799
9800        tokio::time::timeout(Duration::from_secs(30), async {
9801            let run = node.run();
9802            tokio::pin!(run);
9803
9804            let drive = async {
9805                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9806
9807                tx.send(message)
9808                    .await
9809                    .expect("external ingress receiver should be open");
9810
9811                wait_until_async(
9812                    || async { received.borrow().len() == 1 },
9813                    Duration::from_secs(10),
9814                )
9815                .await;
9816                assert_eq!(*received.borrow(), vec![quote]);
9817                handle.stop();
9818            };
9819
9820            tokio::select! {
9821                biased;
9822
9823                () = drive => {}
9824                result = &mut run => {
9825                    panic!("node stopped before factory ingress was republished: {result:?}");
9826                }
9827            }
9828
9829            run.await.expect("node should stop cleanly");
9830        })
9831        .await
9832        .expect("live node should republish factory ingress and stop before timeout");
9833
9834        assert_eq!(handle.state(), NodeState::Stopped);
9835        assert!(closed.load(Ordering::Relaxed));
9836        msgbus::get_message_bus().borrow_mut().dispose();
9837    }
9838
9839    #[rstest]
9840    #[tokio::test(flavor = "current_thread")]
9841    async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
9842        let quote = QuoteTick::default();
9843        let publications = Arc::new(Mutex::new(Vec::new()));
9844        let closed = Arc::new(AtomicBool::new(false));
9845        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
9846
9847        let config = LiveNodeConfig {
9848            environment: Environment::Sandbox,
9849            msgbus: Some(MessageBusConfig::default()),
9850            exec_engine: crate::config::LiveExecutionEngineConfig {
9851                reconciliation: false,
9852                ..Default::default()
9853            },
9854            delay_post_stop: Duration::ZERO,
9855            timeout_connection: Duration::from_millis(500),
9856            timeout_disconnection: Duration::from_millis(500),
9857            ..Default::default()
9858        };
9859
9860        let mut node = LiveNodeBuilder::from_config(config)
9861            .unwrap()
9862            .with_external_msgbus_factory(Box::new(factory))
9863            .build()
9864            .expect("node builds with egress-only message bus factory");
9865        let handle = node.handle();
9866
9867        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
9868        {
9869            let publications = publications.lock();
9870            assert_eq!(publications.len(), 1);
9871            assert_eq!(publications[0].topic, "data.quotes.TEST");
9872        }
9873
9874        tokio::time::timeout(Duration::from_secs(30), async {
9875            let run = node.run();
9876            tokio::pin!(run);
9877
9878            let drive = async {
9879                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
9880                handle.stop();
9881            };
9882
9883            tokio::select! {
9884                biased;
9885
9886                () = drive => {}
9887                result = &mut run => {
9888                    panic!("node stopped before egress-only factory run was observed: {result:?}");
9889                }
9890            }
9891
9892            run.await.expect("node should stop cleanly");
9893        })
9894        .await
9895        .expect("live node should run without external ingress before timeout");
9896
9897        assert_eq!(handle.state(), NodeState::Stopped);
9898        msgbus::get_message_bus().borrow_mut().dispose();
9899        assert!(closed.load(Ordering::Relaxed));
9900    }
9901
9902    #[rstest]
9903    fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
9904        let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
9905
9906        let egress_factory = CapturingBackingFactory::new(
9907            Arc::new(Mutex::new(Vec::new())),
9908            Arc::new(AtomicBool::new(false)),
9909            None,
9910        );
9911        let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9912            .unwrap()
9913            .with_external_msgbus_factory(Box::new(egress_factory))
9914            .with_external_msgbus_egress(Box::new(external_egress))
9915            .build()
9916            .expect_err("builder should reject factory plus injected egress");
9917
9918        assert!(
9919            egress_error
9920                .to_string()
9921                .contains("cannot be combined with injected egress or ingress")
9922        );
9923
9924        let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9925
9926        let ingress_factory = CapturingBackingFactory::new(
9927            Arc::new(Mutex::new(Vec::new())),
9928            Arc::new(AtomicBool::new(false)),
9929            None,
9930        );
9931        let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
9932        let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
9933            .unwrap()
9934            .with_external_msgbus_factory(Box::new(ingress_factory))
9935            .with_external_ingress(Box::new(ingress))
9936            .build()
9937            .expect_err("builder should reject factory plus injected ingress");
9938
9939        assert!(
9940            ingress_error
9941                .to_string()
9942                .contains("cannot be combined with injected egress or ingress")
9943        );
9944    }
9945
9946    #[rstest]
9947    #[tokio::test(flavor = "current_thread")]
9948    async fn test_run_republishes_external_ingress_on_local_msgbus() {
9949        let quote = QuoteTick::default();
9950        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
9951        let payload =
9952            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
9953        let message = BusMessage::with_str_topic(
9954            "data.quotes.TEST",
9955            BusPayloadType::QuoteTick,
9956            payload,
9957            SerializationEncoding::Json,
9958        );
9959        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
9960        let closed = Rc::new(Cell::new(false));
9961        let ingress = CapturingExternalIngress::new(rx, closed.clone());
9962
9963        let config = LiveNodeConfig {
9964            environment: Environment::Sandbox,
9965            exec_engine: crate::config::LiveExecutionEngineConfig {
9966                reconciliation: false,
9967                ..Default::default()
9968            },
9969            delay_post_stop: Duration::ZERO,
9970            timeout_connection: Duration::from_millis(500),
9971            timeout_disconnection: Duration::from_millis(500),
9972            ..Default::default()
9973        };
9974
9975        let mut node = LiveNodeBuilder::from_config(config)
9976            .unwrap()
9977            .with_external_ingress(Box::new(ingress))
9978            .build()
9979            .expect("node builds with external message bus ingress");
9980        let handle = node.handle();
9981
9982        let handler = TypedHandler::from({
9983            let received = received.clone();
9984            move |quote: &QuoteTick| {
9985                received.borrow_mut().push(*quote);
9986            }
9987        });
9988
9989        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
9990        msgbus::get_message_bus()
9991            .borrow_mut()
9992            .add_streaming_type(BusPayloadType::QuoteTick);
9993
9994        tokio::time::timeout(Duration::from_secs(30), async {
9995            let run = node.run();
9996            tokio::pin!(run);
9997
9998            let drive = async {
9999                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10000
10001                tx.send(message)
10002                    .await
10003                    .expect("external ingress receiver should be open");
10004
10005                wait_until_async(
10006                    || async { received.borrow().len() == 1 },
10007                    Duration::from_secs(10),
10008                )
10009                .await;
10010                assert_eq!(*received.borrow(), vec![quote]);
10011                handle.stop();
10012            };
10013
10014            tokio::select! {
10015                biased;
10016
10017                () = drive => {}
10018                result = &mut run => {
10019                    panic!("node stopped before external message was republished: {result:?}");
10020                }
10021            }
10022
10023            run.await.expect("node should stop cleanly");
10024        })
10025        .await
10026        .expect("live node should republish ingress and stop before timeout");
10027
10028        assert_eq!(handle.state(), NodeState::Stopped);
10029        assert!(closed.get());
10030        msgbus::get_message_bus().borrow_mut().dispose();
10031    }
10032
10033    #[rstest]
10034    #[tokio::test(flavor = "current_thread")]
10035    async fn test_run_closes_external_ingress_when_receiver_closes() {
10036        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10037        let closed = Rc::new(Cell::new(false));
10038        let ingress = CapturingExternalIngress::new(rx, closed.clone());
10039
10040        let config = LiveNodeConfig {
10041            environment: Environment::Sandbox,
10042            exec_engine: crate::config::LiveExecutionEngineConfig {
10043                reconciliation: false,
10044                ..Default::default()
10045            },
10046            delay_post_stop: Duration::ZERO,
10047            timeout_connection: Duration::from_millis(500),
10048            timeout_disconnection: Duration::from_millis(500),
10049            ..Default::default()
10050        };
10051
10052        let mut node = LiveNodeBuilder::from_config(config)
10053            .unwrap()
10054            .with_external_ingress(Box::new(ingress))
10055            .build()
10056            .expect("node builds with external message bus ingress");
10057        let handle = node.handle();
10058
10059        tokio::time::timeout(Duration::from_secs(30), async {
10060            let run = node.run();
10061            tokio::pin!(run);
10062
10063            let drive = async {
10064                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10065
10066                drop(tx);
10067
10068                wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10069                assert!(
10070                    handle.is_running(),
10071                    "node should keep running after ingress closes"
10072                );
10073                handle.stop();
10074            };
10075
10076            tokio::select! {
10077                biased;
10078
10079                () = drive => {}
10080                result = &mut run => {
10081                    panic!("node stopped before ingress close was observed: {result:?}");
10082                }
10083            }
10084
10085            run.await.expect("node should stop cleanly");
10086        })
10087        .await
10088        .expect("live node should close ingress and stop before timeout");
10089
10090        assert_eq!(handle.state(), NodeState::Stopped);
10091    }
10092
10093    #[rstest]
10094    #[tokio::test(flavor = "current_thread")]
10095    async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10096        let closed = Rc::new(Cell::new(false));
10097        let ingress = FailingExternalIngress::new(closed.clone());
10098
10099        let config = LiveNodeConfig {
10100            environment: Environment::Sandbox,
10101            exec_engine: crate::config::LiveExecutionEngineConfig {
10102                reconciliation: false,
10103                ..Default::default()
10104            },
10105            delay_post_stop: Duration::ZERO,
10106            timeout_connection: Duration::from_millis(500),
10107            timeout_disconnection: Duration::from_millis(500),
10108            ..Default::default()
10109        };
10110
10111        let mut node = LiveNodeBuilder::from_config(config)
10112            .unwrap()
10113            .with_external_ingress(Box::new(ingress))
10114            .build()
10115            .expect("node builds with external message bus ingress");
10116        let handle = node.handle();
10117
10118        let err = node.run().await.expect_err("run should fail");
10119
10120        assert!(
10121            err.to_string()
10122                .contains("external ingress receiver unavailable")
10123        );
10124        assert_eq!(handle.state(), NodeState::Stopped);
10125        assert!(closed.get());
10126    }
10127
10128    #[rstest]
10129    fn test_node_build_rejects_existing_node_preserving_account_delivery(
10130        #[values(false, true)] first_builder: bool,
10131        #[values(false, true)] second_builder: bool,
10132    ) {
10133        let config = LiveNodeConfig {
10134            trader_id: TraderId::from("PROBE-001"),
10135            ..Default::default()
10136        };
10137
10138        let mut node = if first_builder {
10139            LiveNodeBuilder::from_config(config)
10140                .unwrap()
10141                .build()
10142                .unwrap()
10143        } else {
10144            LiveNode::build("Original".to_string(), Some(config)).unwrap()
10145        };
10146
10147        let bus = msgbus::get_message_bus();
10148
10149        let ExecutionEvent::Account(before) = stub_account_event() else {
10150            unreachable!()
10151        };
10152
10153        get_exec_event_sender()
10154            .send(ExecutionEvent::Account(before.clone()))
10155            .unwrap();
10156        assert_eq!(node.drain_runner_pending(), 1);
10157        assert_eq!(
10158            node.kernel
10159                .cache
10160                .borrow()
10161                .account(&before.account_id)
10162                .unwrap()
10163                .last_event(),
10164            Some(before)
10165        );
10166
10167        let result = if second_builder {
10168            LiveNodeBuilder::from_config(LiveNodeConfig::default())
10169                .unwrap()
10170                .build()
10171        } else {
10172            LiveNode::build("Extra".to_string(), None)
10173        };
10174
10175        assert_eq!(
10176            result.unwrap_err().to_string(),
10177            "A LiveNode already exists or is being built on this thread; drop it before building another"
10178        );
10179
10180        let ExecutionEvent::Account(after) = stub_account_event() else {
10181            unreachable!()
10182        };
10183
10184        get_exec_event_sender()
10185            .send(ExecutionEvent::Account(after.clone()))
10186            .unwrap();
10187        assert_eq!(node.drain_runner_pending(), 1);
10188        assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10189        assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10190        assert_eq!(
10191            node.kernel
10192                .cache
10193                .borrow()
10194                .account(&after.account_id)
10195                .unwrap()
10196                .last_event(),
10197            Some(after)
10198        );
10199    }
10200
10201    #[rstest]
10202    fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10203        let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10204            .unwrap()
10205            .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10206                let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10207                assert!(
10208                    error
10209                        .to_string()
10210                        .contains("already exists or is being built")
10211                );
10212                anyhow::bail!("Event store construction failed")
10213            })
10214            .build();
10215
10216        assert_eq!(
10217            result.unwrap_err().to_string(),
10218            "Event store construction failed"
10219        );
10220
10221        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10222        assert_eq!(node.state(), NodeState::Idle);
10223    }
10224
10225    #[rstest]
10226    fn test_node_build_allows_separate_threads() {
10227        let node = LiveNode::build("First".to_string(), None).unwrap();
10228
10229        let other_state = std::thread::spawn(|| {
10230            let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10231            other.state()
10232        })
10233        .join()
10234        .unwrap();
10235
10236        assert_eq!(node.state(), NodeState::Idle);
10237        assert_eq!(other_state, NodeState::Idle);
10238    }
10239
10240    #[rstest]
10241    fn test_node_build_releases_thread_after_drop() {
10242        let mut node = LiveNode::build("First".to_string(), None).unwrap();
10243        node.dispose();
10244        assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10245        drop(node);
10246
10247        let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10248        assert_eq!(replacement.state(), NodeState::Idle);
10249    }
10250
10251    #[rstest]
10252    fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10253        let config = LiveNodeConfig {
10254            event_store: Some(EventStoreConfig::default()),
10255            ..Default::default()
10256        };
10257
10258        let result = if builder {
10259            LiveNodeBuilder::from_config(config).unwrap().build()
10260        } else {
10261            LiveNode::build("Failure".to_string(), Some(config))
10262        };
10263
10264        assert!(result.unwrap_err().to_string().contains("factory"));
10265
10266        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10267        assert_eq!(node.state(), NodeState::Idle);
10268    }
10269
10270    #[cfg(feature = "python")]
10271    #[rstest]
10272    fn test_node_build_and_initial_state() {
10273        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10274            .unwrap()
10275            .with_name("TestNode")
10276            .build()
10277            .unwrap();
10278
10279        assert_eq!(node.state(), NodeState::Idle);
10280        assert!(!node.is_running());
10281        assert_eq!(node.environment(), Environment::Sandbox);
10282        assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10283    }
10284
10285    #[cfg(feature = "python")]
10286    #[rstest]
10287    fn test_node_build_replaces_stale_runner_senders() {
10288        replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10289        replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10290
10291        let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10292            .unwrap()
10293            .with_name("FirstNode")
10294            .build()
10295            .unwrap();
10296
10297        assert_eq!(first.state(), NodeState::Idle);
10298        drop(first);
10299
10300        let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10301            .unwrap()
10302            .with_name("SecondNode")
10303            .build()
10304            .unwrap();
10305
10306        assert_eq!(second.state(), NodeState::Idle);
10307        assert!(!second.is_running());
10308    }
10309
10310    #[cfg(feature = "python")]
10311    #[rstest]
10312    fn test_node_handle_reflects_node_state() {
10313        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10314            .unwrap()
10315            .with_name("TestNode")
10316            .build()
10317            .unwrap();
10318
10319        let handle = node.handle();
10320
10321        assert_eq!(handle.state(), NodeState::Idle);
10322        assert!(!handle.is_running());
10323    }
10324
10325    #[rstest]
10326    fn test_pending_drain_data_returns_false_when_empty() {
10327        let mut pending = PendingEvents::default();
10328
10329        assert!(!pending.drain_data());
10330    }
10331
10332    #[rstest]
10333    fn test_pending_drain_data_returns_true_when_non_empty() {
10334        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10335
10336        let mut pending = PendingEvents::default();
10337        pending.data_evts.push(
10338            DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10339                .into(),
10340        );
10341
10342        assert!(pending.drain_data());
10343        assert!(pending.data_evts.is_empty());
10344    }
10345
10346    fn stub_data_event() -> DataEvent {
10347        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10348
10349        DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10350    }
10351
10352    fn stub_data_command() -> DataCommand {
10353        use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10354        use nautilus_core::{UUID4, UnixNanos};
10355        use nautilus_model::identifiers::Venue;
10356
10357        DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10358            None,
10359            Venue::from("TEST"),
10360            UUID4::new(),
10361            UnixNanos::default(),
10362            None,
10363            None,
10364        )))
10365    }
10366
10367    fn stub_system_command() -> SystemCommand {
10368        SystemCommand::ReconnectSocket(ReconnectSocket::new(
10369            TraderId::from("TRADER-001"),
10370            ClientId::from("POLYMARKET"),
10371            Ustr::from("polymarket-market-streams"),
10372            UnixNanos::default(),
10373        ))
10374    }
10375
10376    #[rstest]
10377    fn test_flush_pending_data_drains_events_and_commands() {
10378        let (evt_tx, mut evt_rx) =
10379            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10380        let (cmd_tx, mut cmd_rx) =
10381            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10382
10383        let mut pending = PendingEvents::default();
10384
10385        // Pre-load pending (items captured by the select loop)
10386        pending.data_evts.push((stub_data_event()).into());
10387        pending.data_cmds.push(stub_data_command().into());
10388
10389        // Pre-load channels (items missed by the select loop)
10390        evt_tx.send((stub_data_event()).into()).unwrap();
10391        cmd_tx.send(stub_data_command().into()).unwrap();
10392
10393        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10394
10395        assert!(pending.data_evts.is_empty());
10396        assert!(pending.data_cmds.is_empty());
10397        assert!(evt_rx.try_recv().is_err());
10398        assert!(cmd_rx.try_recv().is_err());
10399    }
10400
10401    #[rstest]
10402    fn test_flush_pending_data_drains_mixed_sources() {
10403        let (evt_tx, mut evt_rx) =
10404            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10405        let (cmd_tx, mut cmd_rx) =
10406            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10407
10408        let mut pending = PendingEvents::default();
10409
10410        // First pass: pending has an event, channel has a command
10411        pending.data_evts.push((stub_data_event()).into());
10412        cmd_tx.send(stub_data_command().into()).unwrap();
10413
10414        // Second pass: channel has items that simulate arrival during first drain
10415        evt_tx.send((stub_data_event()).into()).unwrap();
10416        evt_tx.send((stub_data_event()).into()).unwrap();
10417        cmd_tx.send(stub_data_command().into()).unwrap();
10418
10419        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10420
10421        assert!(pending.data_evts.is_empty());
10422        assert!(pending.data_cmds.is_empty());
10423        assert!(evt_rx.try_recv().is_err());
10424        assert!(cmd_rx.try_recv().is_err());
10425    }
10426
10427    #[rstest]
10428    fn test_pending_system_events_stay_separate_from_data() {
10429        let mut pending = PendingEvents::default();
10430
10431        let change = SocketStateChange::new(
10432            ClientId::from("BINANCE"),
10433            Some(Venue::from("BINANCE")),
10434            ustr::Ustr::from("binance-futures-market-streams"),
10435            SocketState::Connected,
10436        );
10437
10438        pending
10439            .system_events
10440            .push(SystemEvent::SocketState(change).into());
10441        let system_events = pending
10442            .take_system_events()
10443            .into_iter()
10444            .map(|message| message.dispatch(|value| value))
10445            .collect::<Vec<_>>();
10446
10447        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10448        assert!(pending.is_empty());
10449    }
10450
10451    #[rstest]
10452    fn test_pending_system_commands_stay_separate_from_data() {
10453        let mut pending = PendingEvents::default();
10454        let command = stub_system_command();
10455
10456        pending.system_commands.push(command.into());
10457        let system_commands = pending
10458            .take_system_commands()
10459            .into_iter()
10460            .map(|message| message.dispatch(|value| value))
10461            .collect::<Vec<_>>();
10462
10463        assert_eq!(system_commands, vec![command]);
10464        assert!(pending.is_empty());
10465    }
10466
10467    fn stub_time_event_handler() -> TimeEventMessage {
10468        use std::rc::Rc;
10469
10470        use nautilus_common::{
10471            runner::TimeEventMessage,
10472            timer::{TimeEvent, TimeEventCallback},
10473        };
10474        use nautilus_core::{UUID4, UnixNanos};
10475        use ustr::Ustr;
10476
10477        TimeEventMessage::new(
10478            TimeEvent::new(
10479                Ustr::from("test-timer"),
10480                UUID4::new(),
10481                UnixNanos::default(),
10482                UnixNanos::default(),
10483            ),
10484            TimeEventCallback::RustLocal(Rc::new(|_| {})),
10485        )
10486    }
10487
10488    fn stub_trading_command_message() -> TradingCommandMessage {
10489        use nautilus_common::messages::execution::query::QueryAccount;
10490        use nautilus_core::{UUID4, UnixNanos};
10491        use nautilus_model::identifiers::AccountId;
10492
10493        TradingCommandMessage::new(
10494            MessagingSwitchboard::exec_engine_execute(),
10495            TradingCommand::QueryAccount(QueryAccount::new(
10496                TraderId::from("TESTER-001"),
10497                None,
10498                AccountId::from("TEST-001"),
10499                UUID4::new(),
10500                UnixNanos::default(),
10501                None,
10502                None, // correlation_id
10503            )),
10504        )
10505    }
10506
10507    fn stub_exec_event() -> ExecutionEvent {
10508        use nautilus_model::{
10509            enums::{LiquiditySide, OrderSide},
10510            identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
10511            reports::FillReport,
10512            types::{Money, Price, Quantity},
10513        };
10514
10515        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
10516            AccountId::from("TEST-001"),
10517            InstrumentId::from("TEST.VENUE"),
10518            VenueOrderId::from("V-001"),
10519            TradeId::from("T-001"),
10520            OrderSide::Buy,
10521            Quantity::from("1.0"),
10522            Price::from("100.0"),
10523            Money::from("0.01 USD"),
10524            LiquiditySide::Maker,
10525            None,
10526            None,
10527            nautilus_core::UnixNanos::default(),
10528            nautilus_core::UnixNanos::default(),
10529            None,
10530        ))))
10531    }
10532
10533    #[rstest]
10534    fn test_flush_all_pending_drains_buffered_channels() {
10535        let (time_tx, mut time_rx) =
10536            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10537        let (system_evt_tx, mut system_evt_rx) =
10538            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10539        let (system_cmd_tx, mut system_cmd_rx) =
10540            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10541        let (data_evt_tx, mut data_evt_rx) =
10542            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10543        let (data_cmd_tx, mut data_cmd_rx) =
10544            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10545        let (exec_evt_tx, mut exec_evt_rx) =
10546            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10547        let (exec_cmd_tx, mut exec_cmd_rx) =
10548            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10549
10550        let mut pending = PendingEvents::default();
10551
10552        // Pre-load pending with data items
10553        pending.data_evts.push((stub_data_event()).into());
10554        pending.data_cmds.push(stub_data_command().into());
10555
10556        // Pre-load all channel types
10557        time_tx.send((stub_time_event_handler()).into()).unwrap();
10558
10559        let change = SocketStateChange::new(
10560            ClientId::from("BINANCE"),
10561            Some(Venue::from("BINANCE")),
10562            Ustr::from("binance-futures-market-streams"),
10563            SocketState::Connected,
10564        );
10565        system_evt_tx
10566            .send((SystemEvent::SocketState(change)).into())
10567            .unwrap();
10568        system_cmd_tx.send((stub_system_command()).into()).unwrap();
10569        data_evt_tx.send((stub_data_event()).into()).unwrap();
10570        data_cmd_tx.send(stub_data_command().into()).unwrap();
10571        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10572        exec_cmd_tx
10573            .send(stub_trading_command_message().into())
10574            .unwrap();
10575
10576        flush_all_pending(
10577            &mut pending,
10578            &mut time_rx,
10579            &mut system_evt_rx,
10580            &mut system_cmd_rx,
10581            &mut exec_evt_rx,
10582            &mut exec_cmd_rx,
10583            &mut data_evt_rx,
10584            &mut data_cmd_rx,
10585        );
10586
10587        let system_events = pending
10588            .take_system_events()
10589            .into_iter()
10590            .map(|message| message.dispatch(|value| value))
10591            .collect::<Vec<_>>();
10592        let system_commands = pending
10593            .take_system_commands()
10594            .into_iter()
10595            .map(|message| message.dispatch(|value| value))
10596            .collect::<Vec<_>>();
10597        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10598        assert_eq!(system_commands, vec![stub_system_command()]);
10599        assert!(pending.data_evts.is_empty());
10600        assert!(pending.data_cmds.is_empty());
10601        assert!(pending.exec_reports.is_empty());
10602        assert!(pending.exec_cmds.is_empty());
10603        assert!(pending.order_evts.is_empty());
10604        assert!(time_rx.try_recv().is_err());
10605        assert!(system_evt_rx.try_recv().is_err());
10606        assert!(system_cmd_rx.try_recv().is_err());
10607        assert!(data_evt_rx.try_recv().is_err());
10608        assert!(data_cmd_rx.try_recv().is_err());
10609        assert!(exec_evt_rx.try_recv().is_err());
10610        assert!(exec_cmd_rx.try_recv().is_err());
10611    }
10612
10613    fn stub_order_event() -> ExecutionEvent {
10614        use nautilus_model::events::order::spec::OrderSubmittedSpec;
10615
10616        ExecutionEvent::Order(OrderEventAny::Submitted(
10617            OrderSubmittedSpec::builder().build(),
10618        ))
10619    }
10620
10621    fn stub_account_event() -> ExecutionEvent {
10622        use nautilus_core::{UUID4, UnixNanos};
10623        use nautilus_model::{
10624            enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
10625        };
10626
10627        ExecutionEvent::Account(AccountState::new(
10628            AccountId::from("TEST-001"),
10629            AccountType::Cash,
10630            vec![],
10631            vec![],
10632            true,
10633            UUID4::new(),
10634            UnixNanos::default(),
10635            UnixNanos::default(),
10636            None,
10637        ))
10638    }
10639
10640    #[rstest]
10641    fn test_flush_all_pending_routes_order_event_to_order_evts() {
10642        let (_time_tx, mut time_rx) =
10643            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10644        let (_system_evt_tx, mut system_evt_rx) =
10645            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10646        let (_system_cmd_tx, mut system_cmd_rx) =
10647            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10648        let (_data_evt_tx, mut data_evt_rx) =
10649            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10650        let (_data_cmd_tx, mut data_cmd_rx) =
10651            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10652        let (exec_evt_tx, mut exec_evt_rx) =
10653            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10654        let (_exec_cmd_tx, mut exec_cmd_rx) =
10655            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10656
10657        let mut pending = PendingEvents::default();
10658
10659        exec_evt_tx.send((stub_order_event()).into()).unwrap();
10660        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
10661
10662        flush_all_pending(
10663            &mut pending,
10664            &mut time_rx,
10665            &mut system_evt_rx,
10666            &mut system_cmd_rx,
10667            &mut exec_evt_rx,
10668            &mut exec_cmd_rx,
10669            &mut data_evt_rx,
10670            &mut data_cmd_rx,
10671        );
10672
10673        // Both order and report events are drained by pending.drain()
10674        assert!(pending.order_evts.is_empty());
10675        assert!(pending.exec_reports.is_empty());
10676        assert!(exec_evt_rx.try_recv().is_err());
10677    }
10678
10679    #[rstest]
10680    fn test_flush_all_pending_routes_account_event_immediately() {
10681        let (_time_tx, mut time_rx) =
10682            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10683        let (_system_evt_tx, mut system_evt_rx) =
10684            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10685        let (_system_cmd_tx, mut system_cmd_rx) =
10686            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10687        let (_data_evt_tx, mut data_evt_rx) =
10688            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10689        let (_data_cmd_tx, mut data_cmd_rx) =
10690            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10691        let (exec_evt_tx, mut exec_evt_rx) =
10692            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10693        let (_exec_cmd_tx, mut exec_cmd_rx) =
10694            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10695
10696        let mut pending = PendingEvents::default();
10697
10698        exec_evt_tx.send((stub_account_event()).into()).unwrap();
10699
10700        flush_all_pending(
10701            &mut pending,
10702            &mut time_rx,
10703            &mut system_evt_rx,
10704            &mut system_cmd_rx,
10705            &mut exec_evt_rx,
10706            &mut exec_cmd_rx,
10707            &mut data_evt_rx,
10708            &mut data_cmd_rx,
10709        );
10710
10711        // Account events are forwarded immediately, never buffered in pending
10712        assert!(pending.exec_reports.is_empty());
10713        assert!(pending.order_evts.is_empty());
10714        assert!(pending.exec_cmds.is_empty());
10715        assert!(exec_evt_rx.try_recv().is_err());
10716    }
10717
10718    #[rstest]
10719    fn test_pending_is_empty_when_default() {
10720        let pending = PendingEvents::default();
10721
10722        assert!(pending.is_empty());
10723    }
10724
10725    #[rstest]
10726    fn test_pending_is_empty_false_with_data_evt() {
10727        let mut pending = PendingEvents::default();
10728        pending.data_evts.push((stub_data_event()).into());
10729
10730        assert!(!pending.is_empty());
10731    }
10732
10733    #[rstest]
10734    fn test_pending_is_empty_false_with_data_cmd() {
10735        let mut pending = PendingEvents::default();
10736        pending.data_cmds.push(stub_data_command().into());
10737
10738        assert!(!pending.is_empty());
10739    }
10740
10741    #[rstest]
10742    fn test_pending_is_empty_false_with_exec_cmd() {
10743        let mut pending = PendingEvents::default();
10744        pending
10745            .exec_cmds
10746            .push(stub_trading_command_message().into());
10747
10748        assert!(!pending.is_empty());
10749    }
10750
10751    #[rstest]
10752    fn test_pending_drain_preserves_trading_command_target() {
10753        std::thread::spawn(|| {
10754            msgbus::get_message_bus().borrow_mut().dispose();
10755            let risk_commands = Rc::new(RefCell::new(Vec::new()));
10756            let exec_commands = Rc::new(RefCell::new(Vec::new()));
10757
10758            let risk_commands_handler = risk_commands.clone();
10759            msgbus::register_trading_command_endpoint(
10760                MessagingSwitchboard::risk_engine_execute(),
10761                TypedIntoHandler::from(move |command: TradingCommand| {
10762                    risk_commands_handler.borrow_mut().push(command);
10763                }),
10764            );
10765
10766            let exec_commands_handler = exec_commands.clone();
10767            msgbus::register_trading_command_endpoint(
10768                MessagingSwitchboard::exec_engine_execute(),
10769                TypedIntoHandler::from(move |command: TradingCommand| {
10770                    exec_commands_handler.borrow_mut().push(command);
10771                }),
10772            );
10773
10774            let mut pending = PendingEvents::default();
10775            pending.exec_cmds.push(
10776                TradingCommandMessage::new(
10777                    MessagingSwitchboard::risk_engine_execute(),
10778                    TradingCommand::QueryAccount(QueryAccount::new(
10779                        TraderId::from("TESTER-001"),
10780                        None,
10781                        AccountId::from("TEST-001"),
10782                        UUID4::new(),
10783                        UnixNanos::default(),
10784                        None,
10785                        None,
10786                    )),
10787                )
10788                .into(),
10789            );
10790
10791            pending.drain();
10792
10793            assert!(pending.is_empty());
10794            assert_eq!(risk_commands.borrow().len(), 1);
10795            assert!(matches!(
10796                &risk_commands.borrow()[0],
10797                TradingCommand::QueryAccount(_)
10798            ));
10799            assert_eq!(exec_commands.borrow().as_slice(), &[]);
10800        })
10801        .join()
10802        .unwrap();
10803    }
10804
10805    #[rstest]
10806    fn test_pending_is_empty_false_with_exec_report() {
10807        let mut pending = PendingEvents::default();
10808
10809        if let ExecutionEvent::Report(report) = stub_exec_event() {
10810            pending.exec_reports.push((report).into());
10811        }
10812
10813        assert!(!pending.is_empty());
10814    }
10815
10816    #[rstest]
10817    fn test_pending_is_empty_false_with_order_evt() {
10818        let mut pending = PendingEvents::default();
10819
10820        if let ExecutionEvent::Order(order_evt) = stub_order_event() {
10821            pending.order_evts.push((order_evt).into());
10822        }
10823
10824        assert!(!pending.is_empty());
10825    }
10826
10827    fn stub_submitted_batch_event() -> ExecutionEvent {
10828        use nautilus_model::{
10829            events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
10830            identifiers::ClientOrderId,
10831        };
10832
10833        let events = vec![
10834            OrderSubmittedSpec::builder()
10835                .client_order_id(ClientOrderId::from("O-001"))
10836                .build(),
10837            OrderSubmittedSpec::builder()
10838                .client_order_id(ClientOrderId::from("O-002"))
10839                .build(),
10840        ];
10841
10842        ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
10843    }
10844
10845    fn stub_canceled_batch_event() -> ExecutionEvent {
10846        use nautilus_model::{
10847            events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
10848            identifiers::ClientOrderId,
10849        };
10850
10851        let events = vec![
10852            OrderCanceledSpec::builder()
10853                .client_order_id(ClientOrderId::from("O-001"))
10854                .build(),
10855            OrderCanceledSpec::builder()
10856                .client_order_id(ClientOrderId::from("O-002"))
10857                .build(),
10858        ];
10859
10860        ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
10861    }
10862
10863    #[rstest]
10864    fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
10865        let (_time_tx, mut time_rx) =
10866            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10867        let (_system_evt_tx, mut system_evt_rx) =
10868            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10869        let (_system_cmd_tx, mut system_cmd_rx) =
10870            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10871        let (_data_evt_tx, mut data_evt_rx) =
10872            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10873        let (_data_cmd_tx, mut data_cmd_rx) =
10874            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10875        let (exec_evt_tx, mut exec_evt_rx) =
10876            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10877        let (_exec_cmd_tx, mut exec_cmd_rx) =
10878            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10879
10880        let mut pending = PendingEvents::default();
10881
10882        exec_evt_tx
10883            .send((stub_submitted_batch_event()).into())
10884            .unwrap();
10885
10886        flush_all_pending(
10887            &mut pending,
10888            &mut time_rx,
10889            &mut system_evt_rx,
10890            &mut system_cmd_rx,
10891            &mut exec_evt_rx,
10892            &mut exec_cmd_rx,
10893            &mut data_evt_rx,
10894            &mut data_cmd_rx,
10895        );
10896
10897        // Batch should be unpacked into individual Submitted events then drained
10898        assert!(pending.order_evts.is_empty());
10899        assert!(exec_evt_rx.try_recv().is_err());
10900    }
10901
10902    #[rstest]
10903    fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
10904        let (_time_tx, mut time_rx) =
10905            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
10906        let (_system_evt_tx, mut system_evt_rx) =
10907            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
10908        let (_system_cmd_tx, mut system_cmd_rx) =
10909            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
10910        let (_data_evt_tx, mut data_evt_rx) =
10911            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10912        let (_data_cmd_tx, mut data_cmd_rx) =
10913            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10914        let (exec_evt_tx, mut exec_evt_rx) =
10915            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10916        let (_exec_cmd_tx, mut exec_cmd_rx) =
10917            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
10918
10919        let mut pending = PendingEvents::default();
10920
10921        exec_evt_tx
10922            .send((stub_canceled_batch_event()).into())
10923            .unwrap();
10924
10925        flush_all_pending(
10926            &mut pending,
10927            &mut time_rx,
10928            &mut system_evt_rx,
10929            &mut system_cmd_rx,
10930            &mut exec_evt_rx,
10931            &mut exec_cmd_rx,
10932            &mut data_evt_rx,
10933            &mut data_cmd_rx,
10934        );
10935
10936        // Batch should be unpacked into individual Canceled events then drained
10937        assert!(pending.order_evts.is_empty());
10938        assert!(exec_evt_rx.try_recv().is_err());
10939    }
10940
10941    #[rstest]
10942    fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
10943        use nautilus_model::identifiers::ClientOrderId;
10944
10945        let (exec_evt_tx, mut exec_evt_rx) =
10946            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
10947
10948        exec_evt_tx
10949            .send((stub_canceled_batch_event()).into())
10950            .unwrap();
10951
10952        let mut pending = PendingEvents::default();
10953
10954        while let Ok(evt) = exec_evt_rx.try_recv() {
10955            pending.push_exec_event(evt);
10956        }
10957
10958        let events: Vec<_> = pending
10959            .order_evts
10960            .drain(..)
10961            .map(|event| event.dispatch(|event| event))
10962            .collect();
10963        assert_eq!(events.len(), 2);
10964        assert!(
10965            matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
10966        );
10967        assert!(
10968            matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
10969        );
10970    }
10971
10972    #[rstest]
10973    #[cfg_attr(
10974        not(all(feature = "simulation", madsim)),
10975        tokio::test(start_paused = true)
10976    )]
10977    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
10978    async fn test_submission_confirmation_prevents_duplicate_tracking(
10979        #[values(false, true)] report_confirmation: bool,
10980        #[values(false, true)] exhausted: bool,
10981    ) {
10982        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
10983        let client_order_id = order.client_order_id();
10984        let account_id = AccountId::from("TEST-001");
10985        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
10986        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
10987            OrderPendingUpdateSpec::builder()
10988                .trader_id(order.trader_id())
10989                .strategy_id(order.strategy_id())
10990                .instrument_id(order.instrument_id())
10991                .client_order_id(client_order_id)
10992                .account_id(account_id)
10993                .build(),
10994        )));
10995        advance_clock(Duration::from_millis(101)).await;
10996        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
10997
10998        if exhausted {
10999            advance_clock(Duration::from_millis(101)).await;
11000            let result = node.exec_manager.check_inflight_orders();
11001            assert!(result.events.is_empty());
11002            assert!(result.queries.is_empty());
11003            assert_eq!(
11004                node.exec_manager
11005                    .take_submission_recovery_exhaustions()
11006                    .len(),
11007                1
11008            );
11009        }
11010
11011        if report_confirmation {
11012            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11013                OrderStatusReport::new(
11014                    account_id,
11015                    order.instrument_id(),
11016                    Some(client_order_id),
11017                    venue_order_id,
11018                    Some(order.order_side()),
11019                    order.order_type(),
11020                    order.time_in_force(),
11021                    OrderStatus::Submitted,
11022                    order.quantity(),
11023                    Quantity::zero(3),
11024                    UnixNanos::default(),
11025                    UnixNanos::default(),
11026                    UnixNanos::default(),
11027                    None,
11028                )
11029                .with_price(order.price().unwrap()),
11030            ))));
11031            node.exec_manager
11032                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11033            advance_clock(Duration::from_millis(101)).await;
11034            let result = node.exec_manager.check_inflight_orders();
11035            assert!(result.queries.is_empty());
11036            assert!(result.events.is_empty());
11037        }
11038
11039        let mut update = OrderUpdatedSpec::builder()
11040            .trader_id(order.trader_id())
11041            .strategy_id(order.strategy_id())
11042            .instrument_id(order.instrument_id())
11043            .client_order_id(client_order_id)
11044            .account_id(account_id)
11045            .quantity(order.quantity())
11046            .price(order.price().unwrap())
11047            .build();
11048
11049        if !report_confirmation {
11050            update.venue_order_id = Some(venue_order_id);
11051        }
11052        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11053        assert_eq!(
11054            node.kernel
11055                .cache
11056                .borrow()
11057                .order(&client_order_id)
11058                .unwrap()
11059                .status(),
11060            OrderStatus::Submitted
11061        );
11062
11063        for _ in 0..3 {
11064            node.exec_manager
11065                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11066            advance_clock(Duration::from_millis(101)).await;
11067            let result = node.exec_manager.check_inflight_orders();
11068            assert!(result.queries.is_empty());
11069            assert!(result.events.is_empty());
11070            assert!(
11071                node.exec_manager
11072                    .take_submission_recovery_exhaustions()
11073                    .is_empty()
11074            );
11075        }
11076    }
11077
11078    #[rstest]
11079    #[cfg_attr(
11080        not(all(feature = "simulation", madsim)),
11081        tokio::test(start_paused = true)
11082    )]
11083    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11084    async fn test_submission_report_confirmation_preserves_later_command_budget(
11085        #[values(false, true)] cancel: bool,
11086        #[values(false, true)] later_fill: bool,
11087        #[values(false, true)] repeated_report: bool,
11088        #[values(
11089            SubmissionRecoveryPolicy::ResolveLocally,
11090            SubmissionRecoveryPolicy::RetainUnresolved
11091        )]
11092        policy: SubmissionRecoveryPolicy,
11093    ) {
11094        let (mut node, order, instrument) = venue_evidence_node(policy);
11095        let client_order_id = order.client_order_id();
11096        let account_id = AccountId::from("TEST-001");
11097        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11098        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11099            OrderPendingUpdateSpec::builder()
11100                .trader_id(order.trader_id())
11101                .strategy_id(order.strategy_id())
11102                .instrument_id(order.instrument_id())
11103                .client_order_id(client_order_id)
11104                .account_id(account_id)
11105                .build(),
11106        )));
11107        node.exec_manager.register_inflight(client_order_id);
11108        advance_clock(Duration::from_millis(101)).await;
11109        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11110        let report = ExecutionReport::Order(Box::new(
11111            OrderStatusReport::new(
11112                account_id,
11113                order.instrument_id(),
11114                Some(client_order_id),
11115                venue_order_id,
11116                Some(order.order_side()),
11117                order.order_type(),
11118                order.time_in_force(),
11119                OrderStatus::Submitted,
11120                order.quantity(),
11121                Quantity::zero(3),
11122                UnixNanos::default(),
11123                UnixNanos::default(),
11124                UnixNanos::default(),
11125                None,
11126            )
11127            .with_price(order.price().unwrap()),
11128        ));
11129        node.process_exec_event(ExecutionEvent::Report(report.clone()));
11130        advance_clock(Duration::from_millis(101)).await;
11131        let result = node.exec_manager.check_inflight_orders();
11132        assert!(result.events.is_empty());
11133        assert!(result.queries.is_empty());
11134
11135        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11136            OrderUpdatedSpec::builder()
11137                .trader_id(order.trader_id())
11138                .strategy_id(order.strategy_id())
11139                .instrument_id(order.instrument_id())
11140                .client_order_id(client_order_id)
11141                .account_id(account_id)
11142                .quantity(order.quantity())
11143                .price(order.price().unwrap())
11144                .build(),
11145        )));
11146        let pending = if cancel {
11147            OrderEventAny::PendingCancel(
11148                OrderPendingCancelSpec::builder()
11149                    .trader_id(order.trader_id())
11150                    .strategy_id(order.strategy_id())
11151                    .instrument_id(order.instrument_id())
11152                    .client_order_id(client_order_id)
11153                    .account_id(account_id)
11154                    .build(),
11155            )
11156        } else {
11157            OrderEventAny::PendingUpdate(
11158                OrderPendingUpdateSpec::builder()
11159                    .trader_id(order.trader_id())
11160                    .strategy_id(order.strategy_id())
11161                    .instrument_id(order.instrument_id())
11162                    .client_order_id(client_order_id)
11163                    .account_id(account_id)
11164                    .build(),
11165            )
11166        };
11167        node.process_exec_event(ExecutionEvent::Order(pending));
11168        node.exec_manager.register_inflight(client_order_id);
11169
11170        if repeated_report {
11171            node.process_exec_event(ExecutionEvent::Report(report));
11172        }
11173        advance_clock(Duration::from_millis(50)).await;
11174
11175        if later_fill {
11176            let cached = node
11177                .kernel
11178                .cache
11179                .borrow()
11180                .order_owned(&client_order_id)
11181                .unwrap();
11182            let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11183                .last_qty(Quantity::from("0.400"))
11184                .last_px(order.price().unwrap())
11185                .commission(Money::from("0 USDT"))
11186                .without_position_id()
11187                .build()
11188            else {
11189                unreachable!()
11190            };
11191            fill.venue_order_id = venue_order_id;
11192            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11193        }
11194        let early = node.exec_manager.check_inflight_orders();
11195        assert!(early.events.is_empty());
11196        assert!(early.queries.is_empty());
11197        advance_clock(Duration::from_millis(51)).await;
11198        let first = node.exec_manager.check_inflight_orders();
11199        assert_eq!(first.queries.len(), usize::from(!repeated_report));
11200        assert!(first.events.is_empty());
11201        advance_clock(Duration::from_millis(101)).await;
11202        let second = node.exec_manager.check_inflight_orders();
11203        assert!(second.queries.is_empty());
11204        if repeated_report {
11205            assert!(second.events.is_empty());
11206        } else {
11207            assert!(
11208                matches!(&second.events[..], [OrderEventAny::Canceled(event)] if event.client_order_id == client_order_id)
11209            );
11210        }
11211        assert!(
11212            node.exec_manager
11213                .take_submission_recovery_exhaustions()
11214                .is_empty()
11215        );
11216    }
11217
11218    #[rstest]
11219    #[cfg_attr(
11220        not(all(feature = "simulation", madsim)),
11221        tokio::test(start_paused = true)
11222    )]
11223    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11224    async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11225        #[values(false, true)] report_confirmation: bool,
11226        #[values(false, true)] cancel: bool,
11227        #[values(false, true)] batch: bool,
11228        #[values(false, true)] pending_before_dispatch: bool,
11229        #[values(false, true)] missing_order_check: bool,
11230        #[values(
11231            SubmissionRecoveryPolicy::ResolveLocally,
11232            SubmissionRecoveryPolicy::RetainUnresolved
11233        )]
11234        policy: SubmissionRecoveryPolicy,
11235    ) {
11236        let (mut node, order, instrument) = venue_evidence_node_with_config(
11237            crate::config::LiveExecutionEngineConfig {
11238                reconciliation: true,
11239                inflight_check_threshold_ms: 100,
11240                inflight_check_retries: 2,
11241                open_check_threshold_ms: 5_000,
11242                open_check_open_only: !missing_order_check,
11243                open_check_lookback_mins: None,
11244                open_check_missing_retries: 5,
11245                single_order_query_delay_ms: 0,
11246                submission_recovery_policy: policy,
11247                ..Default::default()
11248            },
11249            OrderType::Limit,
11250            true,
11251        );
11252        let client_order_id = order.client_order_id();
11253        let client_id = ClientId::from("TEST-EVIDENCE");
11254        let account_id = AccountId::from("TEST-001");
11255        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11256        node.kernel
11257            .exec_engine
11258            .borrow_mut()
11259            .register_client(Box::new(StubExecutionClient::new(
11260                client_id,
11261                account_id,
11262                instrument.id().venue,
11263                OmsType::Netting,
11264                None,
11265            )))
11266            .unwrap();
11267
11268        // Consume part of the original submission budget before venue confirmation
11269        advance_clock(Duration::from_millis(101)).await;
11270        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11271
11272        if report_confirmation {
11273            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11274                OrderPendingUpdateSpec::builder()
11275                    .trader_id(order.trader_id())
11276                    .strategy_id(order.strategy_id())
11277                    .instrument_id(order.instrument_id())
11278                    .client_order_id(client_order_id)
11279                    .account_id(account_id)
11280                    .build(),
11281            )));
11282            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11283                OrderStatusReport::new(
11284                    account_id,
11285                    order.instrument_id(),
11286                    Some(client_order_id),
11287                    venue_order_id,
11288                    Some(order.order_side()),
11289                    order.order_type(),
11290                    order.time_in_force(),
11291                    OrderStatus::Submitted,
11292                    order.quantity(),
11293                    Quantity::zero(3),
11294                    UnixNanos::default(),
11295                    UnixNanos::default(),
11296                    UnixNanos::default(),
11297                    None,
11298                )
11299                .with_price(order.price().unwrap()),
11300            ))));
11301        }
11302
11303        let mut update = OrderUpdatedSpec::builder()
11304            .trader_id(order.trader_id())
11305            .strategy_id(order.strategy_id())
11306            .instrument_id(order.instrument_id())
11307            .client_order_id(client_order_id)
11308            .account_id(account_id)
11309            .quantity(order.quantity())
11310            .price(order.price().unwrap())
11311            .build();
11312
11313        if !report_confirmation {
11314            update.venue_order_id = Some(venue_order_id);
11315        }
11316        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11317        assert_eq!(
11318            node.kernel
11319                .cache
11320                .borrow()
11321                .order(&client_order_id)
11322                .unwrap()
11323                .status(),
11324            OrderStatus::Submitted,
11325        );
11326        advance_clock(Duration::from_millis(101)).await;
11327        let confirmed = node.exec_manager.check_inflight_orders();
11328        assert!(confirmed.queries.is_empty());
11329        assert!(confirmed.events.is_empty());
11330
11331        let pending = if cancel {
11332            OrderEventAny::PendingCancel(
11333                OrderPendingCancelSpec::builder()
11334                    .trader_id(order.trader_id())
11335                    .strategy_id(order.strategy_id())
11336                    .instrument_id(order.instrument_id())
11337                    .client_order_id(client_order_id)
11338                    .account_id(account_id)
11339                    .build(),
11340            )
11341        } else {
11342            OrderEventAny::PendingUpdate(
11343                OrderPendingUpdateSpec::builder()
11344                    .trader_id(order.trader_id())
11345                    .strategy_id(order.strategy_id())
11346                    .instrument_id(order.instrument_id())
11347                    .client_order_id(client_order_id)
11348                    .account_id(account_id)
11349                    .build(),
11350            )
11351        };
11352        let command = venue_evidence_command(&order, cancel, batch);
11353
11354        if pending_before_dispatch {
11355            node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11356        }
11357        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11358            MessagingSwitchboard::exec_engine_execute(),
11359            command,
11360        )));
11361        advance_clock(Duration::from_millis(50)).await;
11362
11363        if missing_order_check {
11364            let engine = node.kernel.exec_engine.borrow();
11365            let clients = engine.get_all_clients();
11366            let check = node
11367                .exec_manager
11368                .prepare_open_order_report_check(UUID4::new(), &clients);
11369            assert_eq!(check.filtered_orders.len(), 1);
11370            let missing = node.exec_manager.reconcile_open_order_reports(
11371                &check,
11372                Vec::new(),
11373                &IndexSet::from([client_id]),
11374                &IndexSet::new(),
11375                &clients,
11376            );
11377            assert!(missing.events.is_empty());
11378            assert!(missing.targeted_queries.is_empty());
11379            assert_eq!(
11380                node.exec_manager.recon_check_retry_count(&client_order_id),
11381                1
11382            );
11383        }
11384        let early = node.exec_manager.check_inflight_orders();
11385        assert!(early.queries.is_empty());
11386        assert!(early.events.is_empty());
11387
11388        if !pending_before_dispatch {
11389            // The queued pending event must not be needed to register the command,
11390            // nor shift its timeout start from dispatch to event application.
11391            assert_eq!(
11392                node.kernel
11393                    .cache
11394                    .borrow()
11395                    .order(&client_order_id)
11396                    .unwrap()
11397                    .status(),
11398                OrderStatus::Submitted,
11399            );
11400            node.process_exec_event(ExecutionEvent::Order(pending));
11401        }
11402
11403        if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11404            node.exec_manager
11405                .register_submission(order.init_event(), Some(client_id));
11406        }
11407        advance_clock(Duration::from_millis(51)).await;
11408        let first = node.exec_manager.check_inflight_orders();
11409        assert_eq!(first.queries.len(), 1);
11410        assert!(
11411            matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11412            if query.client_order_id == client_order_id)
11413        );
11414        assert!(first.events.is_empty());
11415        advance_clock(Duration::from_millis(101)).await;
11416        let exhausted = node.exec_manager.check_inflight_orders();
11417        assert!(exhausted.queries.is_empty());
11418        assert!(
11419            matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11420                if event.client_order_id == client_order_id)
11421        );
11422        assert!(
11423            node.exec_manager
11424                .take_submission_recovery_exhaustions()
11425                .is_empty()
11426        );
11427        node.process_reconciliation_events(&exhausted.events);
11428        assert_eq!(
11429            node.kernel
11430                .cache
11431                .borrow()
11432                .order(&client_order_id)
11433                .unwrap()
11434                .status(),
11435            OrderStatus::Canceled,
11436        );
11437    }
11438
11439    #[rstest]
11440    #[cfg_attr(
11441        not(all(feature = "simulation", madsim)),
11442        tokio::test(start_paused = true)
11443    )]
11444    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11445    async fn test_submission_command_report_cleanup_matches_default(
11446        #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11447        #[values(
11448            SubmissionRecoveryPolicy::ResolveLocally,
11449            SubmissionRecoveryPolicy::RetainUnresolved
11450        )]
11451        policy: SubmissionRecoveryPolicy,
11452    ) {
11453        let (mut node, order, instrument) = venue_evidence_node(policy);
11454        let client_order_id = order.client_order_id();
11455        let client_id = ClientId::from("TEST-EVIDENCE");
11456        let account_id = AccountId::from("TEST-001");
11457        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11458        node.kernel
11459            .exec_engine
11460            .borrow_mut()
11461            .register_client(Box::new(StubExecutionClient::new(
11462                client_id,
11463                account_id,
11464                instrument.id().venue,
11465                OmsType::Netting,
11466                None,
11467            )))
11468            .unwrap();
11469        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11470            &order,
11471            account_id,
11472            venue_order_id,
11473        )));
11474        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11475            MessagingSwitchboard::exec_engine_execute(),
11476            venue_evidence_command(&order, true, false),
11477        )));
11478        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11479            OrderPendingCancelSpec::builder()
11480                .trader_id(order.trader_id())
11481                .strategy_id(order.strategy_id())
11482                .instrument_id(order.instrument_id())
11483                .client_order_id(client_order_id)
11484                .account_id(account_id)
11485                .venue_order_id(venue_order_id)
11486                .build(),
11487        )));
11488        advance_clock(Duration::from_millis(101)).await;
11489        let first = node.exec_manager.check_inflight_orders();
11490        assert_eq!(first.queries.len(), 1);
11491        assert!(first.events.is_empty());
11492
11493        let filled_qty = if report_status == OrderStatus::PartiallyFilled {
11494            Quantity::from("0.400")
11495        } else {
11496            Quantity::zero(3)
11497        };
11498
11499        let report = OrderStatusReport::new(
11500            account_id,
11501            order.instrument_id(),
11502            Some(client_order_id),
11503            venue_order_id,
11504            Some(order.order_side()),
11505            order.order_type(),
11506            order.time_in_force(),
11507            report_status,
11508            order.quantity(),
11509            filled_qty,
11510            UnixNanos::default(),
11511            UnixNanos::default(),
11512            UnixNanos::default(),
11513            None,
11514        )
11515        .with_price(order.price().unwrap())
11516        .with_avg_px(dec!(100.00));
11517        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11518            report,
11519        ))));
11520        advance_clock(Duration::from_millis(101)).await;
11521        let exhausted = node.exec_manager.check_inflight_orders();
11522
11523        assert!(exhausted.queries.is_empty());
11524        assert!(exhausted.events.is_empty());
11525        assert!(
11526            node.exec_manager
11527                .take_submission_recovery_exhaustions()
11528                .is_empty()
11529        );
11530        let cache = node.kernel.cache.borrow();
11531        let cached = cache.order_ref(&client_order_id).unwrap();
11532        assert_eq!(cached.status(), OrderStatus::PendingCancel);
11533        assert_eq!(cached.filled_qty(), filled_qty);
11534    }
11535
11536    #[rstest]
11537    #[case::accepted_modify(false, "accepted")]
11538    #[case::accepted_cancel(true, "accepted")]
11539    #[case::accepted_batch_modify(false, "accepted_batch")]
11540    #[case::accepted_batch_cancel(true, "accepted_batch")]
11541    #[case::updated_cancel(true, "updated")]
11542    #[case::triggered_cancel(true, "triggered")]
11543    #[cfg_attr(
11544        not(all(feature = "simulation", madsim)),
11545        tokio::test(start_paused = true)
11546    )]
11547    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11548    async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
11549        #[case] cancel: bool,
11550        #[case] evidence: &str,
11551        #[values(false, true)] batch: bool,
11552        #[values(false, true)] report_first: bool,
11553        #[values(
11554            SubmissionRecoveryPolicy::ResolveLocally,
11555            SubmissionRecoveryPolicy::RetainUnresolved
11556        )]
11557        policy: SubmissionRecoveryPolicy,
11558    ) {
11559        use nautilus_model::events::order::spec::OrderTriggeredSpec;
11560
11561        let triggered = evidence == "triggered";
11562
11563        let order_type = if triggered {
11564            OrderType::StopLimit
11565        } else {
11566            OrderType::Limit
11567        };
11568        let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
11569        let client_order_id = order.client_order_id();
11570        let account_id = AccountId::from("TEST-001");
11571        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11572        node.kernel
11573            .exec_engine
11574            .borrow_mut()
11575            .register_client(Box::new(StubExecutionClient::new(
11576                ClientId::from("TEST-EVIDENCE"),
11577                account_id,
11578                instrument.id().venue,
11579                OmsType::Netting,
11580                None,
11581            )))
11582            .unwrap();
11583        advance_clock(Duration::from_millis(101)).await;
11584        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11585
11586        if triggered || report_first {
11587            // Evidence can overtake a different, queued command intent
11588            let previous_pending = if cancel {
11589                OrderEventAny::PendingUpdate(
11590                    OrderPendingUpdateSpec::builder()
11591                        .trader_id(order.trader_id())
11592                        .strategy_id(order.strategy_id())
11593                        .instrument_id(order.instrument_id())
11594                        .client_order_id(client_order_id)
11595                        .account_id(account_id)
11596                        .build(),
11597                )
11598            } else {
11599                OrderEventAny::PendingCancel(
11600                    OrderPendingCancelSpec::builder()
11601                        .trader_id(order.trader_id())
11602                        .strategy_id(order.strategy_id())
11603                        .instrument_id(order.instrument_id())
11604                        .client_order_id(client_order_id)
11605                        .account_id(account_id)
11606                        .build(),
11607                )
11608            };
11609            node.process_exec_event(ExecutionEvent::Order(previous_pending));
11610        }
11611        advance_clock(Duration::from_millis(10)).await;
11612        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11613            MessagingSwitchboard::exec_engine_execute(),
11614            venue_evidence_command(&order, cancel, batch),
11615        )));
11616
11617        if report_first {
11618            let report = OrderStatusReport::new(
11619                account_id,
11620                instrument.id(),
11621                Some(client_order_id),
11622                venue_order_id,
11623                Some(order.order_side()),
11624                order.order_type(),
11625                order.time_in_force(),
11626                OrderStatus::Submitted,
11627                order.quantity(),
11628                Quantity::zero(3),
11629                UnixNanos::default(),
11630                UnixNanos::default(),
11631                UnixNanos::default(),
11632                None,
11633            )
11634            .with_price(order.price().unwrap());
11635            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11636                report,
11637            ))));
11638            assert!(
11639                node.kernel
11640                    .cache
11641                    .borrow()
11642                    .order(&client_order_id)
11643                    .unwrap()
11644                    .venue_order_id()
11645                    .is_none()
11646            );
11647        }
11648        advance_clock(Duration::from_millis(40)).await;
11649        let acknowledgement = if triggered {
11650            OrderEventAny::Triggered(
11651                OrderTriggeredSpec::builder()
11652                    .trader_id(order.trader_id())
11653                    .strategy_id(order.strategy_id())
11654                    .instrument_id(order.instrument_id())
11655                    .client_order_id(client_order_id)
11656                    .account_id(account_id)
11657                    .venue_order_id(venue_order_id)
11658                    .build(),
11659            )
11660        } else if evidence == "updated" {
11661            OrderEventAny::Updated(
11662                OrderUpdatedSpec::builder()
11663                    .trader_id(order.trader_id())
11664                    .strategy_id(order.strategy_id())
11665                    .instrument_id(order.instrument_id())
11666                    .client_order_id(client_order_id)
11667                    .account_id(account_id)
11668                    .venue_order_id(venue_order_id)
11669                    .quantity(order.quantity())
11670                    .price(order.price().unwrap())
11671                    .build(),
11672            )
11673        } else {
11674            TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
11675        };
11676
11677        if evidence == "accepted_batch" {
11678            let OrderEventAny::Accepted(accepted) = acknowledgement else {
11679                unreachable!()
11680            };
11681            node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
11682                vec![accepted],
11683            )));
11684        } else {
11685            node.process_exec_event(ExecutionEvent::Order(acknowledgement));
11686        }
11687        assert_eq!(
11688            node.kernel
11689                .cache
11690                .borrow()
11691                .order(&client_order_id)
11692                .unwrap()
11693                .status(),
11694            match evidence {
11695                "triggered" => OrderStatus::Triggered,
11696                "updated" => OrderStatus::Submitted,
11697                _ => OrderStatus::Accepted,
11698            }
11699        );
11700        advance_clock(Duration::from_millis(20)).await;
11701        let pending = if cancel {
11702            OrderEventAny::PendingCancel(
11703                OrderPendingCancelSpec::builder()
11704                    .trader_id(order.trader_id())
11705                    .strategy_id(order.strategy_id())
11706                    .instrument_id(order.instrument_id())
11707                    .client_order_id(client_order_id)
11708                    .account_id(account_id)
11709                    .build(),
11710            )
11711        } else {
11712            OrderEventAny::PendingUpdate(
11713                OrderPendingUpdateSpec::builder()
11714                    .trader_id(order.trader_id())
11715                    .strategy_id(order.strategy_id())
11716                    .instrument_id(order.instrument_id())
11717                    .client_order_id(client_order_id)
11718                    .account_id(account_id)
11719                    .build(),
11720            )
11721        };
11722        node.process_exec_event(ExecutionEvent::Order(pending));
11723        let early = node.exec_manager.check_inflight_orders();
11724        assert!(early.queries.is_empty());
11725        assert!(early.events.is_empty());
11726        advance_clock(Duration::from_millis(41)).await;
11727        let first = node.exec_manager.check_inflight_orders();
11728        // The default does not clear inflight tracking on Triggered events.
11729        let recover_command =
11730            triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
11731        assert_eq!(first.queries.len(), usize::from(recover_command));
11732        assert!(first.events.is_empty());
11733        advance_clock(Duration::from_millis(101)).await;
11734        let exhausted = node.exec_manager.check_inflight_orders();
11735        assert!(exhausted.queries.is_empty());
11736        assert!(
11737            node.exec_manager
11738                .take_submission_recovery_exhaustions()
11739                .is_empty()
11740        );
11741
11742        if recover_command {
11743            assert!(
11744                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11745                if event.client_order_id == client_order_id)
11746            );
11747        } else {
11748            assert!(exhausted.events.is_empty());
11749        }
11750    }
11751
11752    #[rstest]
11753    #[case::submission_only(None, false)]
11754    #[case::modify(Some(false), false)]
11755    #[case::cancel(Some(true), false)]
11756    #[case::batch_modify(Some(false), true)]
11757    #[case::batch_cancel(Some(true), true)]
11758    #[cfg_attr(
11759        not(all(feature = "simulation", madsim)),
11760        tokio::test(start_paused = true)
11761    )]
11762    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11763    async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
11764        #[case] cancel: Option<bool>,
11765        #[case] batch: bool,
11766        #[values(false, true)] missing_order_check: bool,
11767        #[values(
11768            SubmissionRecoveryPolicy::ResolveLocally,
11769            SubmissionRecoveryPolicy::RetainUnresolved
11770        )]
11771        policy: SubmissionRecoveryPolicy,
11772    ) {
11773        let (mut node, order, instrument) = venue_evidence_node_with_config(
11774            crate::config::LiveExecutionEngineConfig {
11775                reconciliation: true,
11776                inflight_check_threshold_ms: 100,
11777                inflight_check_retries: 2,
11778                open_check_threshold_ms: 1,
11779                open_check_open_only: false,
11780                open_check_lookback_mins: None,
11781                open_check_missing_retries: 5,
11782                single_order_query_delay_ms: 0,
11783                submission_recovery_policy: policy,
11784                ..Default::default()
11785            },
11786            OrderType::Limit,
11787            false,
11788        );
11789        let client_order_id = order.client_order_id();
11790        let client_id = ClientId::from("TEST-EVIDENCE");
11791        let account_id = AccountId::from("TEST-001");
11792        node.kernel
11793            .exec_engine
11794            .borrow_mut()
11795            .register_client(Box::new(StubExecutionClient::new(
11796                client_id,
11797                account_id,
11798                instrument.id().venue,
11799                OmsType::Netting,
11800                None,
11801            )))
11802            .unwrap();
11803        let queries = Rc::new(RefCell::new(Vec::new()));
11804        let client = OrderReportClient {
11805            client_id,
11806            report: OrderStatusReport::new(
11807                account_id,
11808                instrument.id(),
11809                Some(client_order_id),
11810                VenueOrderId::from("V-VENUE-EVIDENCE"),
11811                Some(order.order_side()),
11812                order.order_type(),
11813                order.time_in_force(),
11814                OrderStatus::Submitted,
11815                order.quantity(),
11816                Quantity::zero(3),
11817                UnixNanos::default(),
11818                UnixNanos::default(),
11819                UnixNanos::default(),
11820                None,
11821            )
11822            .with_price(order.price().unwrap()),
11823            targeted: false,
11824            queries: queries.clone(),
11825        };
11826        advance_clock(Duration::from_millis(2)).await;
11827        let events = node.exec_manager.check_open_orders(&[&client]).await;
11828        assert!(events.is_empty());
11829        assert_eq!(*queries.borrow(), vec!["bulk"]);
11830        assert_eq!(
11831            node.kernel
11832                .cache
11833                .borrow()
11834                .order(&client_order_id)
11835                .unwrap()
11836                .status(),
11837            OrderStatus::Submitted
11838        );
11839        advance_clock(Duration::from_millis(2)).await;
11840
11841        if missing_order_check {
11842            let engine = node.kernel.exec_engine.borrow();
11843            let clients = engine.get_all_clients();
11844            let check = node
11845                .exec_manager
11846                .prepare_open_order_report_check(UUID4::new(), &clients);
11847            assert_eq!(check.filtered_orders.len(), 1);
11848            let missing = node.exec_manager.reconcile_open_order_reports(
11849                &check,
11850                Vec::new(),
11851                &IndexSet::from([client_id]),
11852                &IndexSet::new(),
11853                &clients,
11854            );
11855            assert!(missing.events.is_empty());
11856            assert!(missing.targeted_queries.is_empty());
11857            assert_eq!(
11858                node.exec_manager.recon_check_retry_count(&client_order_id),
11859                1
11860            );
11861        }
11862        node.exec_manager
11863            .register_submission(order.init_event(), Some(client_id));
11864        advance_clock(Duration::from_millis(101)).await;
11865        let duplicate = node.exec_manager.check_inflight_orders();
11866        assert!(duplicate.events.is_empty());
11867        assert_eq!(
11868            duplicate.queries.len(),
11869            usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
11870        );
11871
11872        if let Some(cancel) = cancel {
11873            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11874                MessagingSwitchboard::exec_engine_execute(),
11875                venue_evidence_command(&order, cancel, batch),
11876            )));
11877            let pending = if cancel {
11878                OrderEventAny::PendingCancel(
11879                    OrderPendingCancelSpec::builder()
11880                        .trader_id(order.trader_id())
11881                        .strategy_id(order.strategy_id())
11882                        .instrument_id(order.instrument_id())
11883                        .client_order_id(client_order_id)
11884                        .account_id(account_id)
11885                        .build(),
11886                )
11887            } else {
11888                OrderEventAny::PendingUpdate(
11889                    OrderPendingUpdateSpec::builder()
11890                        .trader_id(order.trader_id())
11891                        .strategy_id(order.strategy_id())
11892                        .instrument_id(order.instrument_id())
11893                        .client_order_id(client_order_id)
11894                        .account_id(account_id)
11895                        .build(),
11896                )
11897            };
11898            node.process_exec_event(ExecutionEvent::Order(pending));
11899        }
11900        advance_clock(Duration::from_millis(101)).await;
11901        let first = node.exec_manager.check_inflight_orders();
11902        assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
11903
11904        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
11905            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
11906                if event.client_order_id == client_order_id));
11907        } else {
11908            assert!(first.events.is_empty());
11909        }
11910        node.process_reconciliation_events(&first.events);
11911        advance_clock(Duration::from_millis(101)).await;
11912        let exhausted = node.exec_manager.check_inflight_orders();
11913        assert!(exhausted.queries.is_empty());
11914
11915        if cancel.is_some() {
11916            assert!(
11917                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
11918                if event.client_order_id == client_order_id)
11919            );
11920        } else {
11921            assert!(exhausted.events.is_empty());
11922        }
11923        assert!(
11924            node.exec_manager
11925                .take_submission_recovery_exhaustions()
11926                .is_empty()
11927        );
11928    }
11929
11930    #[rstest]
11931    #[case::submission_only(None, false)]
11932    #[case::modify(Some(false), false)]
11933    #[case::cancel(Some(true), false)]
11934    #[case::batch_modify(Some(false), true)]
11935    #[case::batch_cancel(Some(true), true)]
11936    #[cfg_attr(
11937        not(all(feature = "simulation", madsim)),
11938        tokio::test(start_paused = true)
11939    )]
11940    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11941    async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
11942        #[case] cancel: Option<bool>,
11943        #[case] batch: bool,
11944        #[values(false, true)] targeted: bool,
11945        #[values(
11946            SubmissionRecoveryPolicy::ResolveLocally,
11947            SubmissionRecoveryPolicy::RetainUnresolved
11948        )]
11949        policy: SubmissionRecoveryPolicy,
11950    ) {
11951        let (mut node, order, instrument) = venue_evidence_node_with_config(
11952            crate::config::LiveExecutionEngineConfig {
11953                reconciliation: true,
11954                inflight_check_threshold_ms: 100,
11955                inflight_check_retries: 2,
11956                open_check_threshold_ms: 1,
11957                open_check_open_only: false,
11958                open_check_lookback_mins: None,
11959                open_check_missing_retries: 1,
11960                single_order_query_delay_ms: 0,
11961                submission_recovery_policy: policy,
11962                ..Default::default()
11963            },
11964            OrderType::Limit,
11965            true,
11966        );
11967        let client_order_id = order.client_order_id();
11968        let client_id = ClientId::from("TEST-EVIDENCE");
11969        let account_id = AccountId::from("TEST-001");
11970        node.kernel
11971            .exec_engine
11972            .borrow_mut()
11973            .register_client(Box::new(StubExecutionClient::new(
11974                client_id,
11975                account_id,
11976                instrument.id().venue,
11977                OmsType::Netting,
11978                None,
11979            )))
11980            .unwrap();
11981        advance_clock(Duration::from_millis(101)).await;
11982        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11983        advance_clock(Duration::from_millis(10)).await;
11984
11985        if let Some(cancel) = cancel {
11986            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11987                MessagingSwitchboard::exec_engine_execute(),
11988                venue_evidence_command(&order, cancel, batch),
11989            )));
11990            let pending = if cancel {
11991                OrderEventAny::PendingCancel(
11992                    OrderPendingCancelSpec::builder()
11993                        .trader_id(order.trader_id())
11994                        .strategy_id(order.strategy_id())
11995                        .instrument_id(order.instrument_id())
11996                        .client_order_id(client_order_id)
11997                        .account_id(account_id)
11998                        .build(),
11999                )
12000            } else {
12001                OrderEventAny::PendingUpdate(
12002                    OrderPendingUpdateSpec::builder()
12003                        .trader_id(order.trader_id())
12004                        .strategy_id(order.strategy_id())
12005                        .instrument_id(order.instrument_id())
12006                        .client_order_id(client_order_id)
12007                        .account_id(account_id)
12008                        .build(),
12009                )
12010            };
12011            node.process_exec_event(ExecutionEvent::Order(pending));
12012        }
12013        let queries = Rc::new(RefCell::new(Vec::new()));
12014        let client = OrderReportClient {
12015            client_id,
12016            report: OrderStatusReport::new(
12017                account_id,
12018                order.instrument_id(),
12019                Some(client_order_id),
12020                VenueOrderId::from("V-VENUE-EVIDENCE"),
12021                Some(order.order_side()),
12022                order.order_type(),
12023                order.time_in_force(),
12024                OrderStatus::Submitted,
12025                order.quantity(),
12026                Quantity::zero(3),
12027                UnixNanos::default(),
12028                UnixNanos::default(),
12029                UnixNanos::default(),
12030                None,
12031            )
12032            .with_price(order.price().unwrap()),
12033            targeted,
12034            queries: queries.clone(),
12035        };
12036        advance_clock(Duration::from_millis(40)).await;
12037        let events = node.exec_manager.check_open_orders(&[&client]).await;
12038        assert!(events.is_empty());
12039        assert_eq!(
12040            *queries.borrow(),
12041            if targeted {
12042                vec!["bulk", "targeted"]
12043            } else {
12044                vec!["bulk"]
12045            }
12046        );
12047        node.process_reconciliation_events(&events);
12048        advance_clock(Duration::from_millis(61)).await;
12049        let first = node.exec_manager.check_inflight_orders();
12050        let recover_command =
12051            cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12052        assert_eq!(first.queries.len(), usize::from(recover_command));
12053
12054        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12055            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12056                if event.client_order_id == client_order_id));
12057        } else {
12058            assert!(first.events.is_empty());
12059        }
12060        node.process_reconciliation_events(&first.events);
12061        advance_clock(Duration::from_millis(101)).await;
12062        let exhausted = node.exec_manager.check_inflight_orders();
12063        assert!(exhausted.queries.is_empty());
12064
12065        if recover_command {
12066            assert!(
12067                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12068                if event.client_order_id == client_order_id)
12069            );
12070        } else {
12071            assert!(exhausted.events.is_empty());
12072        }
12073        assert!(
12074            node.exec_manager
12075                .take_submission_recovery_exhaustions()
12076                .is_empty()
12077        );
12078    }
12079
12080    #[rstest]
12081    #[case::local_update_before_pending("updated")]
12082    #[case::modify_rejected_after_pending("modify_rejected")]
12083    #[case::cancel_rejected_after_pending("cancel_rejected")]
12084    #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12085    #[cfg_attr(
12086        not(all(feature = "simulation", madsim)),
12087        tokio::test(start_paused = true)
12088    )]
12089    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12090    async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12091        #[case] completion: &str,
12092        #[values(
12093            SubmissionRecoveryPolicy::ResolveLocally,
12094            SubmissionRecoveryPolicy::RetainUnresolved
12095        )]
12096        policy: SubmissionRecoveryPolicy,
12097    ) {
12098        use nautilus_model::events::order::spec::{
12099            OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12100        };
12101
12102        let (mut node, order, instrument) = venue_evidence_node(policy);
12103        let client_order_id = order.client_order_id();
12104        let client_id = ClientId::from("TEST-EVIDENCE");
12105        let account_id = AccountId::from("TEST-001");
12106        let cancel = completion == "cancel_rejected";
12107        let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12108        node.kernel
12109            .exec_engine
12110            .borrow_mut()
12111            .register_client(Box::new(StubExecutionClient::new(
12112                client_id,
12113                account_id,
12114                instrument.id().venue,
12115                OmsType::Netting,
12116                None,
12117            )))
12118            .unwrap();
12119        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12120            MessagingSwitchboard::exec_engine_execute(),
12121            venue_evidence_command(&order, cancel, false),
12122        )));
12123        advance_clock(Duration::from_millis(10)).await;
12124
12125        if completion != "updated" {
12126            let pending = if cancel {
12127                OrderEventAny::PendingCancel(
12128                    OrderPendingCancelSpec::builder()
12129                        .trader_id(order.trader_id())
12130                        .strategy_id(order.strategy_id())
12131                        .instrument_id(order.instrument_id())
12132                        .client_order_id(client_order_id)
12133                        .account_id(account_id)
12134                        .build(),
12135                )
12136            } else {
12137                OrderEventAny::PendingUpdate(
12138                    OrderPendingUpdateSpec::builder()
12139                        .trader_id(order.trader_id())
12140                        .strategy_id(order.strategy_id())
12141                        .instrument_id(order.instrument_id())
12142                        .client_order_id(client_order_id)
12143                        .account_id(account_id)
12144                        .build(),
12145                )
12146            };
12147            node.process_exec_event(ExecutionEvent::Order(pending));
12148        }
12149
12150        if later_cancel_pending {
12151            // The cancel is pending in native state, but has not dispatched
12152            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12153                OrderPendingCancelSpec::builder()
12154                    .trader_id(order.trader_id())
12155                    .strategy_id(order.strategy_id())
12156                    .instrument_id(order.instrument_id())
12157                    .client_order_id(client_order_id)
12158                    .account_id(account_id)
12159                    .build(),
12160            )));
12161        }
12162        advance_clock(Duration::from_millis(10)).await;
12163        let completed = match completion {
12164            "updated" => OrderEventAny::Updated(
12165                OrderUpdatedSpec::builder()
12166                    .trader_id(order.trader_id())
12167                    .strategy_id(order.strategy_id())
12168                    .instrument_id(order.instrument_id())
12169                    .client_order_id(client_order_id)
12170                    .account_id(account_id)
12171                    .quantity(order.quantity())
12172                    .price(Price::from("101.00"))
12173                    .build(),
12174            ),
12175            "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12176                OrderEventAny::ModifyRejected(
12177                    OrderModifyRejectedSpec::builder()
12178                        .trader_id(order.trader_id())
12179                        .strategy_id(order.strategy_id())
12180                        .instrument_id(order.instrument_id())
12181                        .client_order_id(client_order_id)
12182                        .account_id(account_id)
12183                        .build(),
12184                )
12185            }
12186            "cancel_rejected" => OrderEventAny::CancelRejected(
12187                OrderCancelRejectedSpec::builder()
12188                    .trader_id(order.trader_id())
12189                    .strategy_id(order.strategy_id())
12190                    .instrument_id(order.instrument_id())
12191                    .client_order_id(client_order_id)
12192                    .account_id(account_id)
12193                    .build(),
12194            ),
12195            _ => unreachable!(),
12196        };
12197        node.process_exec_event(ExecutionEvent::Order(completed));
12198        assert_eq!(
12199            node.kernel
12200                .cache
12201                .borrow()
12202                .order(&client_order_id)
12203                .unwrap()
12204                .status(),
12205            if later_cancel_pending {
12206                OrderStatus::PendingCancel
12207            } else {
12208                OrderStatus::Submitted
12209            }
12210        );
12211
12212        // A newer pending event is queued, but that command has not dispatched
12213        advance_clock(Duration::from_millis(10)).await;
12214
12215        if !later_cancel_pending {
12216            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12217                OrderPendingUpdateSpec::builder()
12218                    .trader_id(order.trader_id())
12219                    .strategy_id(order.strategy_id())
12220                    .instrument_id(order.instrument_id())
12221                    .client_order_id(client_order_id)
12222                    .account_id(account_id)
12223                    .build(),
12224            )));
12225        }
12226        advance_clock(Duration::from_millis(10)).await;
12227        let cached = node
12228            .kernel
12229            .cache
12230            .borrow()
12231            .order_owned(&client_order_id)
12232            .unwrap();
12233        let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12234            .last_qty(Quantity::from("0.400"))
12235            .last_px(order.price().unwrap())
12236            .commission(Money::from("0 USDT"))
12237            .without_position_id()
12238            .build()
12239        else {
12240            unreachable!()
12241        };
12242        fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12243        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12244
12245        for _ in 0..3 {
12246            advance_clock(Duration::from_millis(101)).await;
12247            let result = node.exec_manager.check_inflight_orders();
12248            assert!(result.queries.is_empty());
12249            assert!(result.events.is_empty());
12250            assert!(
12251                node.exec_manager
12252                    .take_submission_recovery_exhaustions()
12253                    .is_empty()
12254            );
12255        }
12256        assert_eq!(
12257            node.kernel
12258                .cache
12259                .borrow()
12260                .order(&client_order_id)
12261                .unwrap()
12262                .status(),
12263            if later_cancel_pending {
12264                OrderStatus::PendingCancel
12265            } else {
12266                OrderStatus::PendingUpdate
12267            }
12268        );
12269    }
12270
12271    #[rstest]
12272    #[case::wrong_account("account")]
12273    #[case::wrong_instrument("instrument")]
12274    #[case::conflicting_report_venue("report_venue")]
12275    #[case::unapplied_venue_update("update_venue")]
12276    #[case::local_update("local_update")]
12277    #[cfg_attr(
12278        not(all(feature = "simulation", madsim)),
12279        tokio::test(start_paused = true)
12280    )]
12281    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12282    async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12283        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12284        let client_order_id = order.client_order_id();
12285        let account_id = AccountId::from("TEST-001");
12286        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12287
12288        if invalid.ends_with("venue") {
12289            insert_accepted_limit_order_in_node(
12290                &node,
12291                account_id,
12292                ClientId::from("TEST-EVIDENCE"),
12293                order.instrument_id(),
12294                ClientOrderId::from("O-OTHER"),
12295                venue_order_id,
12296            );
12297        }
12298        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12299            OrderPendingCancelSpec::builder()
12300                .trader_id(order.trader_id())
12301                .strategy_id(order.strategy_id())
12302                .instrument_id(order.instrument_id())
12303                .client_order_id(client_order_id)
12304                .account_id(account_id)
12305                .build(),
12306        )));
12307        advance_clock(Duration::from_millis(101)).await;
12308        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12309
12310        if invalid == "update_venue" || invalid == "local_update" {
12311            let mut updated = OrderUpdatedSpec::builder()
12312                .trader_id(order.trader_id())
12313                .strategy_id(order.strategy_id())
12314                .instrument_id(order.instrument_id())
12315                .client_order_id(client_order_id)
12316                .account_id(account_id)
12317                .quantity(order.quantity())
12318                .price(order.price().unwrap())
12319                .build();
12320
12321            if invalid == "update_venue" {
12322                updated.venue_order_id = Some(venue_order_id);
12323            }
12324            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12325        } else {
12326            let report = OrderStatusReport::new(
12327                if invalid == "account" {
12328                    AccountId::from("OTHER-001")
12329                } else {
12330                    account_id
12331                },
12332                if invalid == "instrument" {
12333                    InstrumentId::from("OTHER.BINANCE")
12334                } else {
12335                    order.instrument_id()
12336                },
12337                Some(client_order_id),
12338                venue_order_id,
12339                Some(order.order_side()),
12340                order.order_type(),
12341                order.time_in_force(),
12342                OrderStatus::Submitted,
12343                order.quantity(),
12344                Quantity::zero(3),
12345                UnixNanos::default(),
12346                UnixNanos::default(),
12347                UnixNanos::default(),
12348                None,
12349            )
12350            .with_price(order.price().unwrap());
12351            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12352                report,
12353            ))));
12354        }
12355        assert!(
12356            node.kernel
12357                .cache
12358                .borrow()
12359                .order(&client_order_id)
12360                .unwrap()
12361                .venue_order_id()
12362                .is_none()
12363        );
12364        advance_clock(Duration::from_millis(101)).await;
12365        let result = node.exec_manager.check_inflight_orders();
12366        assert!(result.events.is_empty());
12367        assert!(result.queries.is_empty());
12368        let cached = node
12369            .kernel
12370            .cache
12371            .borrow()
12372            .order_owned(&client_order_id)
12373            .unwrap();
12374        assert_eq!(
12375            cached.status(),
12376            if invalid == "local_update" {
12377                OrderStatus::Submitted
12378            } else {
12379                OrderStatus::PendingCancel
12380            }
12381        );
12382        let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12383        assert_eq!(diagnostics.len(), 1);
12384        assert_eq!(diagnostics[0].client_order_id, client_order_id);
12385    }
12386
12387    fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12388        use nautilus_common::messages::execution::{
12389            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12390        };
12391
12392        let client_id = ClientId::from("TEST-EVIDENCE");
12393        let client_order_id = order.client_order_id();
12394        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12395
12396        if cancel {
12397            let cancel = CancelOrder::new(
12398                order.trader_id(),
12399                Some(client_id),
12400                order.strategy_id(),
12401                order.instrument_id(),
12402                client_order_id,
12403                Some(venue_order_id),
12404                UUID4::new(),
12405                UnixNanos::default(),
12406                None,
12407                None,
12408            );
12409
12410            if batch {
12411                TradingCommand::CancelOrders(BatchCancelOrders::new(
12412                    order.trader_id(),
12413                    Some(client_id),
12414                    order.strategy_id(),
12415                    order.instrument_id(),
12416                    vec![cancel],
12417                    UUID4::new(),
12418                    UnixNanos::default(),
12419                    None,
12420                    None,
12421                ))
12422            } else {
12423                TradingCommand::CancelOrder(cancel)
12424            }
12425        } else {
12426            let modify = ModifyOrder::new(
12427                order.trader_id(),
12428                Some(client_id),
12429                order.strategy_id(),
12430                order.instrument_id(),
12431                client_order_id,
12432                Some(venue_order_id),
12433                None,
12434                Some(Price::from("101.00")),
12435                None,
12436                UUID4::new(),
12437                UnixNanos::default(),
12438                None,
12439                None,
12440            );
12441
12442            if batch {
12443                TradingCommand::ModifyOrders(BatchModifyOrders::new(
12444                    order.trader_id(),
12445                    Some(client_id),
12446                    order.strategy_id(),
12447                    order.instrument_id(),
12448                    vec![modify],
12449                    UUID4::new(),
12450                    UnixNanos::default(),
12451                    None,
12452                    None,
12453                ))
12454            } else {
12455                TradingCommand::ModifyOrder(modify)
12456            }
12457        }
12458    }
12459
12460    fn venue_evidence_node(
12461        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12462    ) -> (LiveNode, OrderAny, InstrumentAny) {
12463        venue_evidence_node_with_order_type(policy, OrderType::Limit)
12464    }
12465
12466    fn venue_evidence_node_with_order_type(
12467        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12468        order_type: OrderType,
12469    ) -> (LiveNode, OrderAny, InstrumentAny) {
12470        venue_evidence_node_with_config(
12471            crate::config::LiveExecutionEngineConfig {
12472                reconciliation: true,
12473                inflight_check_threshold_ms: 100,
12474                inflight_check_retries: 2,
12475                open_check_threshold_ms: 5_000,
12476                single_order_query_delay_ms: 0,
12477                submission_recovery_policy: policy,
12478                ..Default::default()
12479            },
12480            order_type,
12481            true,
12482        )
12483    }
12484
12485    fn venue_evidence_node_with_config(
12486        exec_engine: crate::config::LiveExecutionEngineConfig,
12487        order_type: OrderType,
12488        register_submission: bool,
12489    ) -> (LiveNode, OrderAny, InstrumentAny) {
12490        let config = LiveNodeConfig {
12491            exec_engine,
12492            ..Default::default()
12493        };
12494        let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
12495        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
12496        let account_id = AccountId::from("TEST-001");
12497        let client_id = ClientId::from("TEST-EVIDENCE");
12498        let account = AccountAny::Margin(MarginAccount::new(
12499            AccountState::new(
12500                account_id,
12501                AccountType::Margin,
12502                vec![AccountBalance::new(
12503                    Money::from("1000000 USDT"),
12504                    Money::from("0 USDT"),
12505                    Money::from("1000000 USDT"),
12506                )],
12507                Vec::new(),
12508                true,
12509                UUID4::new(),
12510                UnixNanos::default(),
12511                UnixNanos::default(),
12512                Some(Currency::USDT()),
12513            ),
12514            true,
12515        ));
12516        node.kernel.cache.borrow_mut().add_account(account).unwrap();
12517        node.kernel
12518            .cache
12519            .borrow_mut()
12520            .add_instrument(instrument.clone())
12521            .unwrap();
12522        let mut builder = OrderTestBuilder::new(order_type);
12523        builder
12524            .trader_id(node.trader_id())
12525            .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
12526            .instrument_id(instrument.id())
12527            .quantity(Quantity::from("1.000"))
12528            .price(Price::from("100.00"));
12529
12530        if order_type == OrderType::StopLimit {
12531            builder.trigger_price(Price::from("110.00"));
12532        }
12533        let order = builder.build();
12534        let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
12535            &order,
12536            node.trader_id(),
12537            Some(client_id),
12538            None,
12539            UUID4::new(),
12540            UnixNanos::default(),
12541        ));
12542
12543        if register_submission {
12544            node.observe_exec_command_before_dispatch(&command);
12545        }
12546        node.kernel
12547            .cache
12548            .borrow_mut()
12549            .add_order(order.clone(), None, Some(client_id), false)
12550            .unwrap();
12551        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
12552            &order, account_id,
12553        )));
12554        (node, order, instrument)
12555    }
12556
12557    #[rstest]
12558    #[case::partial("0.400", OrderStatus::PartiallyFilled)]
12559    #[case::full("1.000", OrderStatus::Filled)]
12560    #[tokio::test(start_paused = true)]
12561    async fn test_retained_submission_resolves_late_fill(
12562        #[case] quantity: &str,
12563        #[case] expected_status: OrderStatus,
12564    ) {
12565        let (mut node, order, instrument) =
12566            venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12567        let client_order_id = order.client_order_id();
12568
12569        for _ in 0..2 {
12570            advance_clock(Duration::from_millis(101)).await;
12571            assert!(node.exec_manager.check_inflight_orders().events.is_empty());
12572        }
12573
12574        assert_eq!(
12575            node.exec_manager
12576                .take_submission_recovery_exhaustions()
12577                .len(),
12578            1
12579        );
12580        let fill = TestOrderEventStubs::filled(
12581            &order,
12582            &instrument,
12583            Some(TradeId::from("T-LATE-RETAINED")),
12584            None,
12585            Some(Price::from("100.00")),
12586            Some(Quantity::from(quantity)),
12587            Some(LiquiditySide::Taker),
12588            None,
12589            None,
12590            Some(AccountId::from("TEST-001")),
12591        );
12592        node.process_exec_event(ExecutionEvent::Order(fill));
12593        advance_clock(Duration::from_millis(101)).await;
12594        let result = node.exec_manager.check_inflight_orders();
12595        let cache = node.kernel.cache.borrow();
12596        let cached = cache.order_ref(&client_order_id).unwrap();
12597        assert_eq!(cached.status(), expected_status);
12598        assert_eq!(cached.filled_qty(), Quantity::from(quantity));
12599        assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
12600        assert!(result.events.is_empty());
12601        assert!(result.queries.is_empty());
12602        assert!(
12603            node.exec_manager
12604                .take_submission_recovery_exhaustions()
12605                .is_empty()
12606        );
12607    }
12608
12609    #[rstest]
12610    #[case::submitted_report("order")]
12611    #[case::submitted_report_with_fills("order_with_fills")]
12612    #[case::venue_update("updated")]
12613    #[cfg_attr(
12614        not(all(feature = "simulation", madsim)),
12615        tokio::test(start_paused = true)
12616    )]
12617    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12618    async fn test_venue_evidence_stops_submission_timeout(
12619        #[case] evidence: &str,
12620        #[values(
12621            OrderStatus::Submitted,
12622            OrderStatus::PendingCancel,
12623            OrderStatus::PendingUpdate
12624        )]
12625        status: OrderStatus,
12626        #[values(
12627            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12628            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12629        )]
12630        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12631    ) {
12632        let (mut node, order, _) = venue_evidence_node(policy);
12633        let client_order_id = order.client_order_id();
12634        let account_id = AccountId::from("TEST-001");
12635        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12636        let pending = match status {
12637            OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
12638                OrderPendingCancelSpec::builder()
12639                    .trader_id(order.trader_id())
12640                    .strategy_id(order.strategy_id())
12641                    .instrument_id(order.instrument_id())
12642                    .client_order_id(client_order_id)
12643                    .account_id(account_id)
12644                    .build(),
12645            )),
12646            OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
12647                OrderPendingUpdateSpec::builder()
12648                    .trader_id(order.trader_id())
12649                    .strategy_id(order.strategy_id())
12650                    .instrument_id(order.instrument_id())
12651                    .client_order_id(client_order_id)
12652                    .account_id(account_id)
12653                    .build(),
12654            )),
12655            _ => None,
12656        };
12657
12658        if let Some(pending) = pending {
12659            node.process_exec_event(ExecutionEvent::Order(pending));
12660        }
12661        advance_clock(Duration::from_millis(101)).await;
12662        let first = node.exec_manager.check_inflight_orders();
12663        assert_eq!(first.queries.len(), 1);
12664        assert!(first.events.is_empty());
12665        advance_clock(Duration::from_millis(94)).await;
12666
12667        let event = if evidence == "updated" {
12668            ExecutionEvent::Order(OrderEventAny::Updated(
12669                OrderUpdatedSpec::builder()
12670                    .trader_id(order.trader_id())
12671                    .strategy_id(order.strategy_id())
12672                    .instrument_id(order.instrument_id())
12673                    .client_order_id(client_order_id)
12674                    .account_id(account_id)
12675                    .venue_order_id(venue_order_id)
12676                    .quantity(order.quantity())
12677                    .price(Price::from("101.00"))
12678                    .build(),
12679            ))
12680        } else {
12681            let report = OrderStatusReport::new(
12682                account_id,
12683                order.instrument_id(),
12684                Some(client_order_id),
12685                venue_order_id,
12686                Some(order.order_side()),
12687                order.order_type(),
12688                order.time_in_force(),
12689                OrderStatus::Submitted,
12690                order.quantity(),
12691                Quantity::zero(3),
12692                UnixNanos::default(),
12693                UnixNanos::default(),
12694                UnixNanos::default(),
12695                None,
12696            )
12697            .with_price(Price::from("100.00"));
12698            ExecutionEvent::Report(if evidence == "order" {
12699                ExecutionReport::Order(Box::new(report))
12700            } else {
12701                ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
12702            })
12703        };
12704        node.process_exec_event(event);
12705        let expected_status = if evidence == "updated" {
12706            OrderStatus::Submitted
12707        } else if status == OrderStatus::Submitted {
12708            OrderStatus::Accepted
12709        } else {
12710            status
12711        };
12712        assert_eq!(
12713            node.kernel
12714                .cache
12715                .borrow()
12716                .order(&client_order_id)
12717                .unwrap()
12718                .status(),
12719            expected_status,
12720        );
12721
12722        for _ in 0..3 {
12723            advance_clock(Duration::from_millis(101)).await;
12724            let result = node.exec_manager.check_inflight_orders();
12725            assert!(result.queries.is_empty());
12726            assert!(result.events.is_empty());
12727        }
12728        assert_eq!(
12729            node.exec_manager.recon_check_retry_count(&client_order_id),
12730            0
12731        );
12732        assert!(node.exec_manager.check_open_order_queries().is_empty());
12733        advance_clock(Duration::from_secs(5)).await;
12734        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12735    }
12736
12737    #[rstest]
12738    #[case::fill("fill", false)]
12739    #[case::terminal_order("order", true)]
12740    #[case::terminal_order_with_fills("order_with_fills", true)]
12741    #[case::working_order_for_closed_cache("order", false)]
12742    #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
12743    #[cfg_attr(
12744        not(all(feature = "simulation", madsim)),
12745        tokio::test(start_paused = true)
12746    )]
12747    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12748    async fn test_report_cleanup_preserves_original_settling_predicate(
12749        #[case] report_kind: &str,
12750        #[case] terminal_report: bool,
12751        #[values(false, true)] explicit_client_id: bool,
12752        #[values(
12753            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
12754            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
12755        )]
12756        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12757    ) {
12758        let (mut node, order, instrument) = venue_evidence_node(policy);
12759        let client_order_id = order.client_order_id();
12760        let account_id = AccountId::from("TEST-001");
12761        let client_id = ClientId::from("TEST-EVIDENCE");
12762        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12763        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
12764            &order,
12765            account_id,
12766            venue_order_id,
12767        )));
12768        let accepted = node
12769            .kernel
12770            .cache
12771            .borrow()
12772            .order_owned(&client_order_id)
12773            .unwrap();
12774        let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
12775            .last_qty(accepted.quantity())
12776            .last_px(Price::from("100.00"))
12777            .commission(Money::from("0 USDT"))
12778            .without_position_id()
12779            .build()
12780        else {
12781            unreachable!()
12782        };
12783        let mut fill_report = FillReport::new(
12784            account_id,
12785            instrument.id(),
12786            venue_order_id,
12787            fill.trade_id,
12788            fill.order_side,
12789            fill.last_qty,
12790            fill.last_px,
12791            fill.commission.unwrap(),
12792            fill.liquidity_side,
12793            Some(client_order_id),
12794            None,
12795            fill.ts_event,
12796            fill.ts_init,
12797            None,
12798        );
12799
12800        if !explicit_client_id {
12801            fill_report.client_order_id = None;
12802        }
12803        let event = if report_kind == "fill" {
12804            ExecutionReport::Fill(Box::new(fill_report))
12805        } else {
12806            if !terminal_report {
12807                node.kernel
12808                    .cache
12809                    .borrow_mut()
12810                    .update_order(&OrderEventAny::Filled(fill))
12811                    .unwrap();
12812            }
12813            let report = OrderStatusReport::new(
12814                account_id,
12815                instrument.id(),
12816                explicit_client_id.then_some(client_order_id),
12817                venue_order_id,
12818                Some(order.order_side()),
12819                order.order_type(),
12820                order.time_in_force(),
12821                if terminal_report {
12822                    OrderStatus::Filled
12823                } else {
12824                    OrderStatus::Accepted
12825                },
12826                order.quantity(),
12827                order.quantity(),
12828                UnixNanos::default(),
12829                UnixNanos::default(),
12830                UnixNanos::default(),
12831                None,
12832            )
12833            .with_price(Price::from("100.00"))
12834            .with_avg_px(dec!(100.00));
12835
12836            if report_kind == "order" {
12837                ExecutionReport::Order(Box::new(report))
12838            } else {
12839                ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
12840            }
12841        };
12842        node.process_exec_event(ExecutionEvent::Report(event));
12843        assert_eq!(
12844            node.kernel
12845                .cache
12846                .borrow()
12847                .order(&client_order_id)
12848                .unwrap()
12849                .status(),
12850            OrderStatus::Filled
12851        );
12852
12853        // Replace only cached state, without an observer stamp, to expose the existing grace gate
12854        node.kernel
12855            .cache
12856            .borrow_mut()
12857            .add_order(order.clone(), None, Some(client_id), true)
12858            .unwrap();
12859        node.kernel
12860            .cache
12861            .borrow_mut()
12862            .update_order(&TestOrderEventStubs::submitted(&order, account_id))
12863            .unwrap();
12864        node.kernel
12865            .cache
12866            .borrow_mut()
12867            .update_order(&TestOrderEventStubs::accepted(
12868                &order,
12869                account_id,
12870                venue_order_id,
12871            ))
12872            .unwrap();
12873        let should_clear = terminal_report && explicit_client_id;
12874        assert_eq!(
12875            node.exec_manager.check_open_order_queries().len(),
12876            usize::from(should_clear)
12877        );
12878
12879        if !should_clear {
12880            advance_clock(Duration::from_secs(5)).await;
12881            assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
12882        }
12883    }
12884
12885    #[rstest]
12886    #[case(stub_order_event, 1)]
12887    #[case(stub_submitted_batch_event, 2)]
12888    #[case(stub_accepted_batch_event, 2)]
12889    #[case(stub_canceled_batch_event, 2)]
12890    fn test_pending_events_preserve_roots_when_splitting_batches(
12891        #[values(false, true)] rooted: bool,
12892        #[case] make_event: fn() -> ExecutionEvent,
12893        #[case] orders: usize,
12894    ) {
12895        let runner = AsyncRunner::new();
12896        runner.bind_senders();
12897        let mut channels = runner.take_channels();
12898
12899        let send = move || {
12900            get_data_event_sender().send(stub_data_event()).unwrap();
12901            get_exec_event_sender().send(stub_exec_event()).unwrap();
12902            get_exec_event_sender().send(make_event()).unwrap();
12903        };
12904
12905        if rooted {
12906            msgbus::register_trading_command_endpoint(
12907                MessagingSwitchboard::risk_engine_execute(),
12908                TypedIntoHandler::from(move |_| send()),
12909            );
12910            TradingCommandSender::execute(
12911                &SyncTradingCommandSender,
12912                TradingCommandMessage::new(
12913                    MessagingSwitchboard::risk_engine_execute(),
12914                    TradingCommand::QueryAccount(QueryAccount::new(
12915                        "TRADER-001".into(),
12916                        None,
12917                        "SIM-001".into(),
12918                        UUID4::new(),
12919                        37.into(),
12920                        None,
12921                        None,
12922                    )),
12923                ),
12924            );
12925            nautilus_common::runner::drain_trading_cmd_queue();
12926        } else {
12927            send();
12928        }
12929
12930        let mut pending = PendingEvents::default();
12931        pending
12932            .data_evts
12933            .push(channels.data_evt_rx.try_recv().unwrap());
12934        while let Ok(event) = channels.exec_evt_rx.try_recv() {
12935            pending.push_exec_event(event);
12936        }
12937
12938        assert_eq!(pending.data_evts.len(), 1);
12939        assert_eq!(pending.exec_reports.len(), 1);
12940        assert_eq!(pending.order_evts.len(), orders);
12941        assert_eq!(pending.data_evts[0].is_rooted(), rooted);
12942        assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
12943        assert!(
12944            pending
12945                .order_evts
12946                .iter()
12947                .all(|event| event.is_rooted() == rooted)
12948        );
12949
12950        let results = std::thread::spawn(move || {
12951            let mut results = Vec::new();
12952
12953            for event in pending.data_evts {
12954                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12955                    || event.dispatch(drop),
12956                )));
12957            }
12958
12959            for event in pending.exec_reports {
12960                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12961                    || event.dispatch(drop),
12962                )));
12963            }
12964
12965            for event in pending.order_evts {
12966                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
12967                    || event.dispatch(drop),
12968                )));
12969            }
12970
12971            results
12972        })
12973        .join()
12974        .unwrap();
12975
12976        assert_eq!(results.len(), orders + 2);
12977
12978        for result in results {
12979            if rooted {
12980                let error = result.unwrap_err();
12981                assert!(
12982                    error
12983                        .downcast_ref::<String>()
12984                        .unwrap()
12985                        .contains("command context dispatched outside its owner thread")
12986                );
12987            } else {
12988                result.unwrap();
12989            }
12990        }
12991    }
12992
12993    fn stub_accepted_batch_event() -> ExecutionEvent {
12994        ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
12995            OrderAcceptedSpec::builder()
12996                .client_order_id(ClientOrderId::from("O-017"))
12997                .build(),
12998            OrderAcceptedSpec::builder()
12999                .client_order_id(ClientOrderId::from("O-023"))
13000                .build(),
13001        ]))
13002    }
13003
13004    #[derive(Debug)]
13005    struct CapturedEgressMessage {
13006        topic: String,
13007        payload: Bytes,
13008    }
13009
13010    type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13011    type SharedClosed = Rc<Cell<bool>>;
13012
13013    #[derive(Debug)]
13014    struct CapturingExternalIngress {
13015        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13016        closed: SharedClosed,
13017    }
13018
13019    impl CapturingExternalIngress {
13020        fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13021            Self {
13022                rx: Some(rx),
13023                closed,
13024            }
13025        }
13026    }
13027
13028    impl MessageBusExternalIngress for CapturingExternalIngress {
13029        fn is_closed(&self) -> bool {
13030            self.closed.get()
13031        }
13032
13033        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13034            self.rx
13035                .take()
13036                .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13037        }
13038
13039        fn close(&mut self) {
13040            self.closed.set(true);
13041        }
13042    }
13043
13044    #[derive(Debug)]
13045    struct FailingExternalIngress {
13046        closed: SharedClosed,
13047    }
13048
13049    impl FailingExternalIngress {
13050        fn new(closed: SharedClosed) -> Self {
13051            Self { closed }
13052        }
13053    }
13054
13055    impl MessageBusExternalIngress for FailingExternalIngress {
13056        fn is_closed(&self) -> bool {
13057            self.closed.get()
13058        }
13059
13060        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13061            anyhow::bail!("external ingress receiver unavailable")
13062        }
13063
13064        fn close(&mut self) {
13065            self.closed.set(true);
13066        }
13067    }
13068
13069    struct CapturingExternalEgress {
13070        publications: CapturedEgressMessages,
13071        closed: SharedClosed,
13072    }
13073
13074    impl CapturingExternalEgress {
13075        fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13076            let publications = Rc::new(RefCell::new(Vec::new()));
13077            let closed = Rc::new(Cell::new(false));
13078            (
13079                Self {
13080                    publications: publications.clone(),
13081                    closed: closed.clone(),
13082                },
13083                publications,
13084                closed,
13085            )
13086        }
13087    }
13088
13089    impl MessageBusExternalEgress for CapturingExternalEgress {
13090        fn is_closed(&self) -> bool {
13091            self.closed.get()
13092        }
13093
13094        fn publish(&self, message: BusMessage) {
13095            self.publications.borrow_mut().push(CapturedEgressMessage {
13096                topic: message.topic.to_string(),
13097                payload: message.payload,
13098            });
13099        }
13100
13101        fn close(&mut self) {
13102            self.closed.set(true);
13103        }
13104    }
13105
13106    struct CapturingBackingFactory {
13107        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13108        closed: Arc<AtomicBool>,
13109        rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13110    }
13111
13112    impl CapturingBackingFactory {
13113        fn new(
13114            publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13115            closed: Arc<AtomicBool>,
13116            rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13117        ) -> Self {
13118            Self {
13119                publications,
13120                closed,
13121                rx: Mutex::new(rx),
13122            }
13123        }
13124    }
13125
13126    impl Debug for CapturingBackingFactory {
13127        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13128            f.debug_struct(stringify!(CapturingBackingFactory))
13129                .finish_non_exhaustive()
13130        }
13131    }
13132
13133    impl MessageBusBackingFactory for CapturingBackingFactory {
13134        fn create(
13135            &self,
13136            _trader_id: TraderId,
13137            _instance_id: UUID4,
13138            _config: MessageBusConfig,
13139        ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13140            let rx = self.rx.lock().take();
13141            Ok(Box::new(CapturingBacking {
13142                publications: self.publications.clone(),
13143                closed: self.closed.clone(),
13144                rx,
13145            }))
13146        }
13147    }
13148
13149    struct CapturingBacking {
13150        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13151        closed: Arc<AtomicBool>,
13152        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13153    }
13154
13155    impl MessageBusBacking for CapturingBacking {
13156        fn is_closed(&self) -> bool {
13157            self.closed.load(Ordering::Relaxed)
13158        }
13159
13160        fn publish(&self, message: BusMessage) {
13161            self.publications.lock().push(CapturedEgressMessage {
13162                topic: message.topic.to_string(),
13163                payload: message.payload,
13164            });
13165        }
13166
13167        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13168            self.rx
13169                .take()
13170                .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13171        }
13172
13173        fn close(&mut self) {
13174            self.closed.store(true, Ordering::Relaxed);
13175        }
13176    }
13177}