Skip to main content

nautilus_live/node/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Live trading node built on a single-threaded tokio event loop.
17//!
18//! The node owns system lifecycle and the event loop. Its reconciliation module schedules checks,
19//! manages report futures and deadlines, and dispatches results. The execution manager owns
20//! reconciliation state, discrepancy decisions, and individual reconciliation operations.
21//!
22//! `LiveNode::run()` drives the system through a `tokio::select!` loop that
23//! multiplexes data events, execution events, trading commands, timers, and
24//! periodic maintenance tasks (reconciliation, purge, prune, audit).
25//!
26//! # Threading model
27//!
28//! The core types (`ExecutionManager`, `ExecutionEngine`, `Cache`) use
29//! `Rc<RefCell<..>>` and are `!Send`. All access happens on the same thread.
30//! Pending report futures can retain client borrows while other select branches run.
31//! The client facade defers instrument updates until those borrows are released;
32//! completion and cancellation paths flush the deferred updates. Single-threaded
33//! execution does not by itself prevent conflicting borrows or reentrant callbacks.
34//!
35//! # Startup sequencing
36//!
37//! Startup connects clients in two phases so that instruments are in the
38//! cache before execution clients read them:
39//!
40//! 1. Connect data clients (instruments arrive as buffered `DataEvent`s).
41//! 2. Flush all pending data events and commands into the cache via
42//!    `flush_pending_data`, which loops `try_recv` on the channel receivers
43//!    until no items remain.
44//! 3. Connect execution clients (`load_instruments_from_cache` now finds
45//!    populated instruments).
46//! 4. Drain remaining events, then run reconciliation.
47//!
48//! Both `run()` (integrated event loop) and `start()` (manual lifecycle)
49//! follow this sequence.
50//! Startup aborts if a data or execution client returns a connection error.
51//!
52//! # Reconciliation
53//!
54//! Continuous inflight, open-order, and position checks run on independent intervals. The
55//! shared maintenance timer in the select loop dispatches reconciliation at
56//! the minimum enabled interval. Each dispatch the handler checks which
57//! sub-checks are due based on elapsed nanoseconds and schedules their work.
58//! Continuous checks do not await venue HTTP in the select loop: open-order
59//! and position checks poll bulk venue report futures from the loop.
60//!
61//! # Maintenance dispatcher
62//!
63//! Six periodic tasks share a single coarse `maintenance_timer`:
64//!
65//! - reconciliation (inflight, open, position sub-checks)
66//! - purge closed orders
67//! - purge closed positions
68//! - purge account events
69//! - own-books audit
70//! - recent-fills cache prune
71//!
72//! The runner wakes one timer per loop iteration regardless of how many
73//! maintenance tasks are configured. Each task tracks its own
74//! `next_fire: Instant` and the dispatcher fires the bodies whose deadline
75//! has passed, rescheduling `next = now + interval` (equivalent to
76//! `MissedTickBehavior::Delay`). Disabled tasks anchor on a far-future
77//! `next` that never trips.
78//!
79//! The 100ms timer cadence is the effective floor for any maintenance
80//! interval. Configured intervals below 100ms (the config types allow
81//! `inflight_check_interval_ms` and `own_books_audit_interval_secs` smaller)
82//! become eligible on the next maintenance tick. Event processing and runtime
83//! scheduling can delay dispatch further; the timer does not guarantee a maximum delay.
84
85use std::{
86    any::Any,
87    cell::RefCell,
88    collections::VecDeque,
89    fmt::Debug,
90    rc::{Rc, Weak},
91    time::Duration,
92};
93
94use anyhow::Context;
95use nautilus_common::{
96    actor::{self, Actor, DataActor, DataActorNative},
97    cache::database::{CacheDatabaseAdapter, CacheDatabaseFactory},
98    clients::ExecutionClient,
99    component::Component,
100    enums::{Environment, LogColor},
101    live::{dispatch::DispatchMessage, dst},
102    log_info,
103    messages::{
104        DataEvent, ExecutionEvent, ExecutionReport, SystemCommand, SystemEvent,
105        data::DataCommand,
106        execution::TradingCommand,
107        system::{QueueStateChanged, ReconnectSocket, SocketStateChange, SocketStateChanged},
108    },
109    msgbus::{self, BusMessage, MessagingSwitchboard},
110    runner::{SystemChannel, TimeEventMessage, TradingCommandMessage},
111};
112use nautilus_core::{
113    UUID4,
114    datetime::{mins_to_secs, secs_to_nanos_unchecked},
115};
116use nautilus_execution::client::ExecutionClientAdapter;
117#[cfg(test)]
118use nautilus_model::reports::OrderStatusReport;
119use nautilus_model::{
120    events::OrderEventAny,
121    identifiers::{ClientOrderId, InstrumentId, StrategyId, TraderId},
122    orders::Order,
123};
124use nautilus_network::mode::ReconnectRequestOutcome;
125#[cfg(feature = "python")]
126use nautilus_system::trader::Trader;
127use nautilus_system::{config::NautilusKernelConfig, kernel::NautilusKernel};
128use nautilus_trading::{
129    ExecutionAlgorithm, ExecutionAlgorithmNative,
130    strategy::{Strategy, StrategyNative},
131};
132use tabled::{builder::Builder, settings::Style};
133
134use crate::{
135    dispatch::drain_callbacks,
136    execution::{
137        client::LiveExecutionClient,
138        manager::{ExecutionManager, ExecutionManagerConfig, TargetedOrderQuery},
139    },
140    runner::{AsyncRunner, AsyncRunnerChannels, PendingRunnerEvent},
141    socket::{SocketReconnectLookup, SocketReconnectRegistry},
142};
143
144pub mod builder;
145pub mod config;
146
147mod metrics;
148mod queue;
149mod reconciliation;
150mod state;
151
152#[cfg(feature = "plugin")]
153pub mod plugin;
154
155use builder::ExternalMessageBusIngress;
156pub use builder::LiveNodeBuilder;
157use config::{LiveNodeConfig, PluginConfig, validate_live_environment};
158pub use metrics::{RunnerChannelMetricsSnapshot, RunnerMetricsDelta, RunnerMetricsSnapshot};
159use metrics::{RunnerChannelQueueDepths, RunnerMetrics};
160use queue::{QueueMonitor, QueueStateTransition};
161use reconciliation::{
162    OpenOrderReportCollection, OpenOrderReportTask, PositionFillReportResult, PositionReportTask,
163    PositionReportTaskResult, ReconciliationCheckIntervals, ReconciliationCheckState,
164    ReportTaskOutcome, TargetedOrderReportTask,
165};
166use state::{EngineConnectionStatus, RunningTransition};
167pub use state::{LiveNodeHandle, NodeRunMode, NodeState};
168
169/// Dispatches the run loop performs before yielding to the executor.
170///
171/// A saturated channel keeps every select branch ready, so the loop would otherwise never return
172/// `Pending`. Under a host event loop that starves the adapter I/O tasks feeding those channels,
173/// which shows up as lapsed heartbeats and reconnects rather than as backpressure.
174const DISPATCHES_PER_YIELD: usize = 64;
175
176thread_local! {
177    static NODE_THREAD_OWNER: RefCell<Weak<()>> = const { RefCell::new(Weak::new()) };
178}
179
180type StreamProcessorCallback = dyn Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static;
181
182struct StreamProcessor(Box<StreamProcessorCallback>);
183
184impl Debug for StreamProcessor {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        f.debug_struct(stringify!(StreamProcessor)).finish()
187    }
188}
189
190/// High-level abstraction for a live Nautilus system node.
191///
192/// Provides a simplified interface for running live systems
193/// with automatic client management and lifecycle handling.
194///
195/// Only one live node may exist on a thread at a time. Drop the node before building another,
196/// including after disposal, because retained nodes can still access thread-local messaging.
197/// Concurrent nodes in one process remain unsupported, even on separate threads.
198#[derive(Debug)]
199pub struct LiveNode {
200    kernel: NautilusKernel,
201    runner: Option<AsyncRunner>,
202    config: LiveNodeConfig,
203    handle: LiveNodeHandle,
204    exec_manager: ExecutionManager,
205    exec_clients: Vec<LiveExecutionClient>,
206    socket_registry: SocketReconnectRegistry,
207    cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
208    external_msgbus: Option<ExternalMessageBusIngress>,
209    stream_processors: Vec<StreamProcessor>,
210    shutdown_deadline: Option<dst::time::Instant>,
211    #[cfg(feature = "plugin")]
212    plugins: plugin::NodePlugins,
213    _thread_owner: Rc<()>,
214}
215
216impl LiveNode {
217    /// Creates a new `LiveNode` from builder components.
218    ///
219    /// This is an internal constructor used by `LiveNodeBuilder`.
220    #[must_use]
221    #[allow(
222        clippy::too_many_arguments,
223        reason = "builder components have distinct lifecycle roles"
224    )]
225    pub(crate) fn new_from_builder(
226        kernel: NautilusKernel,
227        runner: AsyncRunner,
228        config: LiveNodeConfig,
229        handle: LiveNodeHandle,
230        mut exec_manager: ExecutionManager,
231        exec_clients: Vec<LiveExecutionClient>,
232        socket_registry: SocketReconnectRegistry,
233        cache_database_factory: Option<Box<dyn CacheDatabaseFactory>>,
234        external_msgbus: Option<ExternalMessageBusIngress>,
235        thread_owner: Rc<()>,
236    ) -> Self {
237        for client in &exec_clients {
238            exec_manager.register_submission_retention(client);
239        }
240
241        Self {
242            kernel,
243            runner: Some(runner),
244            config,
245            handle,
246            exec_manager,
247            exec_clients,
248            socket_registry,
249            cache_database_factory,
250            external_msgbus,
251            stream_processors: Vec::new(),
252            shutdown_deadline: None,
253            #[cfg(feature = "plugin")]
254            plugins: plugin::NodePlugins,
255            _thread_owner: thread_owner,
256        }
257    }
258
259    /// Creates a new [`LiveNodeBuilder`] for fluent configuration.
260    ///
261    /// # Errors
262    ///
263    /// Returns an error if the environment is invalid for live trading.
264    pub fn builder(
265        trader_id: TraderId,
266        environment: Environment,
267    ) -> anyhow::Result<LiveNodeBuilder> {
268        LiveNodeBuilder::new(trader_id, environment)
269    }
270
271    /// Creates a new [`LiveNode`] directly from a kernel name and optional configuration.
272    ///
273    /// This is a convenience method for creating a live node with a pre-configured
274    /// kernel configuration, bypassing the builder pattern. If no config is provided,
275    /// a default configuration will be used.
276    ///
277    /// # Errors
278    ///
279    /// Returns an error if kernel construction fails or another live node exists or is being
280    /// built on this thread.
281    pub fn build(name: String, config: Option<LiveNodeConfig>) -> anyhow::Result<Self> {
282        let thread_owner = Self::acquire_thread()?;
283        let config = config.unwrap_or_default();
284        validate_live_environment(config.environment())?;
285
286        config.validate_runtime_support()?;
287
288        if config.event_store.is_some() {
289            anyhow::bail!(
290                "LiveNodeConfig.event_store is set but LiveNode::build cannot install a factory; \
291                 use LiveNodeBuilder::with_event_store(...) instead"
292            );
293        }
294
295        let handle = LiveNodeHandle::new();
296        let runner = AsyncRunner::new();
297        runner.bind_senders_for_node(handle.clone());
298
299        let kernel = NautilusKernel::new(name, config.clone())?;
300        #[cfg(feature = "python")]
301        if let Some(controller) = config.controller.as_ref() {
302            Trader::add_controller_from_importable_config(&kernel.trader, controller)?;
303        }
304
305        #[cfg(not(feature = "python"))]
306        if let Some(controller) = config.controller.as_ref() {
307            anyhow::bail!(
308                "LiveNodeConfig.controller for importable controller '{}' requires the python feature",
309                controller.controller_path
310            );
311        }
312
313        let exec_manager_config =
314            ExecutionManagerConfig::from(&config.exec_engine).with_trader_id(config.trader_id);
315
316        let exec_manager = ExecutionManager::new(
317            kernel.clock.clone(),
318            kernel.cache.clone(),
319            exec_manager_config,
320        )?;
321
322        let node = Self {
323            kernel,
324            runner: Some(runner),
325            config,
326            handle,
327            exec_manager,
328            exec_clients: Vec::new(),
329            socket_registry: SocketReconnectRegistry::default(),
330            cache_database_factory: None,
331            external_msgbus: None,
332            stream_processors: Vec::new(),
333            shutdown_deadline: None,
334            #[cfg(feature = "plugin")]
335            plugins: plugin::NodePlugins,
336            _thread_owner: thread_owner,
337        };
338
339        node.load_configured_plugins()?;
340
341        log::info!("LiveNode built successfully with kernel config");
342
343        Ok(node)
344    }
345
346    fn acquire_thread() -> anyhow::Result<Rc<()>> {
347        NODE_THREAD_OWNER.with(|slot| {
348            let mut owner = slot.borrow_mut();
349            anyhow::ensure!(
350                owner.upgrade().is_none(),
351                "A LiveNode already exists or is being built on this thread; drop it before building another"
352            );
353            let token = Rc::new(());
354            *owner = Rc::downgrade(&token);
355            Ok(token)
356        })
357    }
358
359    /// Loads and registers plug-ins declared on the node config.
360    ///
361    /// # Errors
362    ///
363    /// Returns an error if plug-ins are configured without host-side support.
364    pub(crate) fn load_configured_plugins(&self) -> anyhow::Result<()> {
365        if self.config.plugins.is_empty() {
366            return Ok(());
367        }
368
369        anyhow::bail!(
370            "LiveNodeConfig.plugins requires host-side plug-in support; nautilus-plugin is the guest SDK only"
371        )
372    }
373
374    /// Loads and registers one plug-in instance.
375    ///
376    /// # Errors
377    ///
378    /// Returns an error if called. Dynamic plug-in hosting lives in the host-side integration.
379    #[expect(
380        clippy::needless_pass_by_value,
381        reason = "signature mirrors the host-enabled API"
382    )]
383    pub fn add_plugin(&mut self, config: PluginConfig) -> anyhow::Result<()> {
384        #[cfg(feature = "plugin")]
385        config.validate_runtime_support(self.config.plugins.len())?;
386        #[cfg(not(feature = "plugin"))]
387        let _ = config;
388
389        anyhow::bail!(
390            "LiveNode::add_plugin requires host-side plug-in support; nautilus-plugin is the guest SDK only"
391        )
392    }
393
394    /// Returns a thread-safe handle to control this node.
395    #[must_use]
396    pub fn handle(&self) -> LiveNodeHandle {
397        self.handle.clone()
398    }
399
400    /// Adds a callback for supported typed external messages.
401    ///
402    /// While the node's run loop services external ingress, processors run in registration order
403    /// before normal inbound streaming filters for supported typed JSON or MessagePack payloads.
404    /// Unsupported encodings do not reach processors. Rust callbacks receive the decoded
405    /// concrete value as [`Any`], allowing them to downcast to the concrete payload type.
406    ///
407    /// Python callbacks receive the serialized payload shape with an added `payload_type` field.
408    /// Native data replies use the shared response representation keyed by response kind.
409    /// Callback exceptions are logged, stop remaining processors, and skip internal republishing.
410    /// The Python wrapper returns an error if the node is no longer available for mutation.
411    ///
412    /// External egress is suppressed while processors run, so synchronous publications remain
413    /// local. During data-client connection, streaming-registered `DataResponse` messages with a
414    /// pending correlation handler bypass processors and can overtake messages buffered until
415    /// normal dispatch starts.
416    pub fn add_stream_processor<F>(&mut self, callback: F)
417    where
418        F: Fn(&dyn Any) + 'static,
419    {
420        self.add_stream_processor_with_mapping(move |message, _| {
421            callback(message);
422            Ok(())
423        });
424    }
425
426    pub(crate) fn add_stream_processor_with_mapping<F>(&mut self, callback: F)
427    where
428        F: Fn(&dyn Any, &serde_json::Value) -> anyhow::Result<()> + 'static,
429    {
430        self.stream_processors
431            .push(StreamProcessor(Box::new(callback)));
432    }
433
434    /// Starts the live node without entering a select loop.
435    ///
436    /// Connects clients, runs reconciliation, and starts the trader, but does
437    /// not consume the runner or drive channel receivers, so channel traffic arriving after
438    /// startup is never serviced. This is a building block for tests and embedding, not a
439    /// lifecycle: use [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) to run a node.
440    ///
441    /// # Errors
442    ///
443    /// Returns an error if startup fails.
444    pub async fn start(&mut self) -> anyhow::Result<()> {
445        if self.state().is_running() {
446            anyhow::bail!("Already running");
447        }
448
449        if self.external_msgbus.is_some() {
450            log::warn!(
451                "External message bus ingress is configured but LiveNode::start() does not service it; use LiveNode::run()"
452            );
453        }
454
455        self.prepare_cache().await?;
456
457        if let Some(runner) = self.runner.as_ref() {
458            runner.bind_senders_for_node(self.handle.clone());
459        }
460
461        self.handle.set_starting();
462
463        self.kernel.reset_shutdown_flag();
464        self.kernel.start_async().await;
465
466        if self.kernel.is_event_store_replay() {
467            log::info!(
468                "Event-store replay loaded; skipping live client connection and reconciliation",
469            );
470
471            if !self.finish_startup_replay().await? {
472                return Ok(());
473            }
474
475            return Ok(());
476        }
477
478        if self.kernel.is_event_store_replay_configured() {
479            self.abort_startup("Event-store replay did not start")
480                .await?;
481            return Ok(());
482        }
483
484        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
485
486        // Connect data clients first and flush instrument events into cache
487        if let Err(e) = self.connect_data_phase(connection_deadline).await {
488            return self
489                .abort_startup_with_error("Data client connection failed", e)
490                .await;
491        }
492
493        let (startup_system_events, startup_system_commands) =
494            if let Some(runner) = self.runner.as_mut() {
495                runner.flush_pending_data();
496                (
497                    runner.drain_pending_system_events(),
498                    runner.drain_pending_system_commands(),
499                )
500            } else {
501                (Vec::new(), Vec::new())
502            };
503
504        if let Err(e) = self.connect_exec_clients(connection_deadline).await {
505            return self
506                .abort_startup_with_error("Execution client connection failed", e)
507                .await;
508        }
509
510        if let Some(reason) = self.startup_abort_reason() {
511            self.abort_startup(reason).await?;
512            return Ok(());
513        }
514
515        match self.await_engines_connected(connection_deadline).await {
516            EngineConnectionStatus::Connected => {}
517            EngineConnectionStatus::TimedOut => {
518                return self
519                    .abort_startup_with_error(
520                        "Engine readiness timed out",
521                        anyhow::anyhow!("readiness timeout while waiting for engine connections"),
522                    )
523                    .await;
524            }
525            EngineConnectionStatus::StopRequested => {
526                self.abort_startup("Stop signal received during startup")
527                    .await?;
528                return Ok(());
529            }
530            EngineConnectionStatus::ShutdownRequested => {
531                self.abort_startup("Shutdown signal received during startup")
532                    .await?;
533                return Ok(());
534            }
535        }
536
537        if let Err(e) = self.perform_startup_reconciliation().await {
538            if let Err(finalize_err) = self.abort_startup("Startup reconciliation failed").await {
539                anyhow::bail!(
540                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
541                );
542            }
543
544            return Err(e);
545        }
546
547        if let Some(reason) = self.startup_abort_reason() {
548            self.abort_startup(reason).await?;
549            return Ok(());
550        }
551
552        if let Err(e) = self.kernel.start_trader() {
553            return self.abort_after_trader_start_failure(e).await;
554        }
555
556        #[cfg(feature = "plugin")]
557        if let Err(e) = self.plugins.start_controllers() {
558            return self.abort_after_trader_start_failure(e).await;
559        }
560
561        self.process_system_events(startup_system_events);
562        self.process_system_commands(startup_system_commands);
563
564        if !self.finish_startup_trader(None).await? {
565            return Ok(());
566        }
567
568        Ok(())
569    }
570
571    /// Stop the live node.
572    ///
573    /// This method stops the trader, waits for the configured grace period to allow
574    /// residual events to be processed, then finalizes the shutdown sequence.
575    ///
576    /// # Errors
577    ///
578    /// Returns an error if shutdown fails or retained submissions remain unresolved at the end
579    /// of `delay_post_stop`. Teardown still completes; subsequent events cannot clear this result.
580    pub async fn stop(&mut self) -> anyhow::Result<()> {
581        if !self.state().is_running() {
582            anyhow::bail!("Not running");
583        }
584
585        self.handle.set_shutting_down();
586
587        #[cfg(feature = "plugin")]
588        let controller_stop_result = self.plugins.stop_controllers();
589        #[cfg(not(feature = "plugin"))]
590        let controller_stop_result: anyhow::Result<()> = Ok(());
591
592        self.kernel.stop_trader();
593        let delay = self.kernel.delay_post_stop();
594        log::info!("Awaiting residual events ({delay:?})...");
595
596        let residual_events = self.process_runner_for(delay).await;
597        if residual_events > 0 {
598            log::debug!("Processed {residual_events} residual events during shutdown");
599        }
600
601        let stop_result = self.finalize_stop().await;
602        self.drain_runner_pending();
603
604        match (controller_stop_result, stop_result) {
605            (Ok(()), Ok(())) => Ok(()),
606            (Err(controller_err), Ok(())) => Err(controller_err),
607            (Ok(()), Err(stop_err)) => Err(stop_err),
608            (Err(controller_err), Err(stop_err)) => {
609                log::error!("Error stopping plug-in controllers: {controller_err}");
610                Err(stop_err)
611            }
612        }
613    }
614
615    /// Disposes the live node kernel and releases resources.
616    ///
617    /// Discards any retained runner messages and attempts callback cleanup. Logs latched callback
618    /// failures and cleanup rejection; externally retained work can prevent clearing.
619    pub fn dispose(&mut self) {
620        self.close_external_ingress();
621        self.handle.set_stopped();
622        self.kernel.dispose();
623        drop(self.runner.take());
624
625        if let Some(e) = actor::callback_failure() {
626            log::error!("Callback dispatch failed before disposal cleanup: {e}");
627        }
628
629        if let Err(e) = actor::clear_callbacks() {
630            log::error!("Failed to clear callback dispatch during disposal: {e}");
631        }
632    }
633
634    async fn process_runner_for(&mut self, duration: Duration) -> usize {
635        let Some(mut runner) = self.runner.take() else {
636            dst::time::sleep(duration).await;
637            return 0;
638        };
639
640        runner.bind_senders_for_node(self.handle.clone());
641        let deadline = dst::time::Instant::now() + duration;
642        let mut processed = 0;
643
644        loop {
645            tokio::select! {
646                biased;
647
648                () = dst::time::sleep_until(deadline) => break,
649                event = runner.recv() => {
650                    let Some(event) = event else {
651                        dst::time::sleep_until(deadline).await;
652                        break;
653                    };
654
655                    self.process_runner_event(event);
656                    processed += 1;
657                }
658            }
659        }
660
661        self.runner = Some(runner);
662        processed
663    }
664
665    fn drain_runner_pending(&mut self) -> usize {
666        let Some(mut runner) = self.runner.take() else {
667            return 0;
668        };
669
670        let processed = runner.poll_pending(|event| self.process_runner_event(event));
671        self.runner = Some(runner);
672        processed
673    }
674
675    fn process_runner_event(&mut self, event: PendingRunnerEvent) {
676        match event {
677            PendingRunnerEvent::TimeEvent(message) => {
678                let _ = AsyncRunner::handle_time_event(message);
679            }
680            PendingRunnerEvent::SystemEvent(event) => {
681                event.dispatch(|event| self.process_system_event(event));
682            }
683            PendingRunnerEvent::SystemCommand(command) => {
684                command.dispatch(|command| self.process_system_command(command));
685            }
686            PendingRunnerEvent::ExecEvent(event) => {
687                event.dispatch(|event| self.process_exec_event(event));
688            }
689            PendingRunnerEvent::ExecCommand(command) => self.process_exec_command(command),
690            PendingRunnerEvent::DataEvent(event) => AsyncRunner::dispatch_data_event(event),
691            PendingRunnerEvent::DataCommand(command) => AsyncRunner::handle_data_command(command),
692        }
693    }
694
695    fn process_system_events(&self, events: Vec<DispatchMessage<SystemEvent>>) {
696        for event in events {
697            event.dispatch(|event| self.process_system_event(event));
698        }
699    }
700
701    fn process_system_commands(&self, commands: Vec<DispatchMessage<SystemCommand>>) {
702        for command in commands {
703            command.dispatch(|command| self.process_system_command(command));
704        }
705    }
706
707    fn process_system_command(&self, command: SystemCommand) {
708        match command {
709            SystemCommand::ReconnectSocket(command) => {
710                self.process_socket_reconnect(command);
711            }
712        }
713    }
714
715    fn process_socket_reconnect(&self, command: ReconnectSocket) {
716        let outcome = if command.trader_id == self.config.trader_id {
717            Self::request_socket_reconnect(
718                self.socket_registry
719                    .get(command.client_id, command.endpoint),
720            )
721        } else {
722            SocketReconnectDispatchOutcome::InvalidTrader
723        };
724
725        if outcome == SocketReconnectDispatchOutcome::Accepted {
726            log::info!(
727                "Requested socket reconnect for client {} endpoint {}",
728                command.client_id,
729                command.endpoint
730            );
731        } else {
732            log::warn!(
733                "Rejected socket reconnect request for client {} endpoint {}: {outcome:?}",
734                command.client_id,
735                command.endpoint
736            );
737        }
738    }
739
740    fn request_socket_reconnect(lookup: SocketReconnectLookup) -> SocketReconnectDispatchOutcome {
741        match lookup {
742            SocketReconnectLookup::Handle(handle) => match handle.request_reconnect() {
743                ReconnectRequestOutcome::Accepted => SocketReconnectDispatchOutcome::Accepted,
744                ReconnectRequestOutcome::AlreadyReconnecting => {
745                    SocketReconnectDispatchOutcome::AlreadyReconnecting
746                }
747                ReconnectRequestOutcome::Disconnected => {
748                    SocketReconnectDispatchOutcome::Disconnected
749                }
750                ReconnectRequestOutcome::Closed => SocketReconnectDispatchOutcome::Closed,
751                ReconnectRequestOutcome::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
752            },
753            SocketReconnectLookup::ClientNotFound => SocketReconnectDispatchOutcome::UnknownClient,
754            SocketReconnectLookup::Unsupported => SocketReconnectDispatchOutcome::Unsupported,
755            SocketReconnectLookup::EndpointNotFound => {
756                SocketReconnectDispatchOutcome::UnknownEndpoint
757            }
758            SocketReconnectLookup::AmbiguousEndpoint => {
759                SocketReconnectDispatchOutcome::AmbiguousEndpoint
760            }
761        }
762    }
763
764    fn process_system_event(&self, event: SystemEvent) {
765        match event {
766            SystemEvent::SocketState(change) => self.publish_socket_state_change(change),
767        }
768    }
769
770    fn publish_socket_state_change(&self, change: SocketStateChange) {
771        let timestamp = self.kernel.generate_timestamp_ns();
772
773        let event = SocketStateChanged::new(
774            self.config.trader_id,
775            change.client_id,
776            change.venue,
777            change.endpoint,
778            change.state,
779            UUID4::new(),
780            timestamp,
781            timestamp,
782        );
783
784        msgbus::publish_any(
785            MessagingSwitchboard::socket_state_changed_topic(
786                event.client_id,
787                event.endpoint.as_str(),
788            ),
789            event.as_any(),
790        );
791    }
792
793    /// Awaits engine clients to connect with timeout.
794    ///
795    /// Returns the final connection wait status.
796    async fn await_engines_connected(
797        &self,
798        deadline: dst::time::Instant,
799    ) -> EngineConnectionStatus {
800        log::info!(
801            "Awaiting engine connections ({:?} timeout)...",
802            self.config.timeout_connection
803        );
804
805        let interval = Duration::from_millis(100);
806
807        loop {
808            if self.handle.should_stop() {
809                log::warn!("Stop signal received, aborting connection wait");
810                return EngineConnectionStatus::StopRequested;
811            }
812
813            if self.kernel.is_shutdown_requested() {
814                log::warn!("Shutdown signal received, aborting connection wait");
815                return EngineConnectionStatus::ShutdownRequested;
816            }
817
818            if self.kernel.check_engines_connected() {
819                log::info!("All engine clients connected");
820                return EngineConnectionStatus::Connected;
821            }
822
823            let now = dst::time::Instant::now();
824            if now >= deadline {
825                break;
826            }
827
828            dst::time::sleep(interval.min(deadline - now)).await;
829        }
830
831        self.log_connection_status();
832        EngineConnectionStatus::TimedOut
833    }
834
835    /// Awaits engine clients to disconnect with timeout.
836    ///
837    /// Returns an error with client status on timeout.
838    async fn await_engines_disconnected(&self, deadline: dst::time::Instant) -> anyhow::Result<()> {
839        log::info!(
840            "Awaiting engine disconnections ({:?} timeout)...",
841            self.config.timeout_disconnection
842        );
843
844        let timeout = self.config.timeout_disconnection;
845        let interval = Duration::from_millis(100);
846
847        loop {
848            if self.kernel.check_engines_disconnected() {
849                log::info!("All engine clients disconnected");
850                return Ok(());
851            }
852
853            let now = dst::time::Instant::now();
854            if now >= deadline {
855                break;
856            }
857
858            dst::time::sleep(interval.min(deadline - now)).await;
859        }
860
861        log::error!(
862            "Timed out ({:?}) waiting for engines to disconnect\n\
863             DataEngine.check_disconnected() == {}\n\
864             ExecEngine.check_disconnected() == {}",
865            timeout,
866            self.kernel.data_engine().check_disconnected(),
867            self.kernel.exec_engine().borrow().check_disconnected(),
868        );
869        anyhow::bail!("disconnect readiness timeout while waiting for engine disconnections")
870    }
871
872    fn log_connection_status(&self) {
873        let data_status = self.kernel.data_client_connection_status();
874        let exec_status = self.kernel.exec_client_connection_status();
875
876        let mut rows: Vec<ClientStatus> = Vec::new();
877
878        for (client_id, connected) in data_status {
879            rows.push(ClientStatus {
880                client: client_id.to_string(),
881                client_type: "Data",
882                connected,
883            });
884        }
885
886        for (client_id, connected) in exec_status {
887            rows.push(ClientStatus {
888                client: client_id.to_string(),
889                client_type: "Execution",
890                connected,
891            });
892        }
893
894        let table = render_client_statuses(rows);
895
896        log::warn!(
897            "Timed out ({:?}) waiting for engines to connect\n\n{table}\n\n\
898             DataEngine.check_connected() == {}\n\
899             ExecEngine.check_connected() == {}",
900            self.config.timeout_connection,
901            self.kernel.data_engine().check_connected(),
902            self.kernel.exec_engine().borrow().check_connected(),
903        );
904    }
905
906    /// Performs startup reconciliation to align internal state with venue state.
907    ///
908    /// This method queries each execution client for mass status (orders, fills, positions)
909    /// and reconciles any discrepancies with the local cache state.
910    ///
911    /// # Errors
912    ///
913    /// Returns an error if reconciliation fails or times out.
914    async fn perform_startup_reconciliation(&mut self) -> anyhow::Result<()> {
915        if !self.config.exec_engine.reconciliation {
916            log::info!("Startup reconciliation disabled");
917            self.kernel
918                .portfolio
919                .borrow_mut()
920                .initialize_wallet_orders()?;
921            return Ok(());
922        }
923
924        log_info!(
925            "Starting execution state reconciliation...",
926            color = LogColor::Blue
927        );
928
929        let lookback_mins = self
930            .config
931            .exec_engine
932            .reconciliation_lookback_mins
933            .map(u64::from);
934
935        let timeout = self.config.timeout_reconciliation;
936        let start = dst::time::Instant::now();
937        let client_ids = self.kernel.exec_engine.borrow().client_ids();
938
939        for client_id in client_ids {
940            let elapsed = start.elapsed();
941            if elapsed >= timeout {
942                anyhow::bail!("Startup reconciliation timeout reached");
943            }
944
945            let remaining = timeout
946                .checked_sub(elapsed)
947                .expect("elapsed checked against reconciliation timeout");
948
949            log_info!(
950                "Requesting mass status from {}...",
951                client_id,
952                color = LogColor::Blue
953            );
954
955            let client = self
956                .exec_clients
957                .iter()
958                .find(|client| client.client_id() == client_id)
959                .ok_or_else(|| anyhow::anyhow!("Execution client {client_id} has no live handle"))?
960                .clone();
961            let client = Box::new(client);
962            let client = ExecutionClientAdapter::new(client);
963
964            let mass_status_query = client.generate_mass_status(lookback_mins);
965
966            let mass_status_result = dst::time::timeout(remaining, mass_status_query)
967            .await
968            .map_err(|_| {
969                anyhow::anyhow!(
970                    "Startup reconciliation timeout reached while requesting mass status from {client_id}"
971                )
972            })?;
973
974            match mass_status_result {
975                Ok(Some(mass_status)) => {
976                    log_info!(
977                        "Reconciling ExecutionMassStatus for {}",
978                        client_id,
979                        color = LogColor::Blue
980                    );
981
982                    let result = self
983                        .exec_manager
984                        .reconcile_execution_mass_status(&mass_status, &self.kernel.exec_engine);
985
986                    anyhow::ensure!(
987                        self.kernel
988                            .exec_engine
989                            .borrow()
990                            .get_client(&client_id)
991                            .is_some(),
992                        "Execution client {client_id} disappeared during startup reconciliation",
993                    );
994
995                    anyhow::ensure!(
996                        result.unresolved_positions.is_empty(),
997                        "Unresolved positions during startup reconciliation for {client_id}: {}",
998                        result.unresolved_positions.join("; "),
999                    );
1000
1001                    if result.events.is_empty() {
1002                        log_info!(
1003                            "Reconciliation for {} succeeded",
1004                            client_id,
1005                            color = LogColor::Blue
1006                        );
1007                    } else {
1008                        log::info!(
1009                            color = LogColor::Blue as u8;
1010                            "Reconciliation for {} processed {} events",
1011                            client_id,
1012                            result.events.len()
1013                        );
1014                    }
1015
1016                    // Register external orders with execution clients for tracking
1017                    if !result.external_orders.is_empty() {
1018                        let exec_engine = self.kernel.exec_engine.borrow();
1019
1020                        let source_client = exec_engine.get_client(&client_id).ok_or_else(|| {
1021                            anyhow::anyhow!(
1022                                "Execution client {client_id} disappeared during startup reconciliation"
1023                            )
1024                        })?;
1025
1026                        for external in result.external_orders {
1027                            source_client.register_external_order(
1028                                external.client_order_id,
1029                                external.venue_order_id,
1030                                external.instrument_id,
1031                                external.strategy_id,
1032                                external.ts_init,
1033                            );
1034                        }
1035                    }
1036                }
1037                Ok(None) => {
1038                    log::warn!(
1039                        "No mass status available from {client_id} \
1040                         (likely adapter error when generating reports)"
1041                    );
1042                }
1043                Err(e) => {
1044                    return Err(e).context(format!("Failed to get mass status from {client_id}"));
1045                }
1046            }
1047        }
1048
1049        self.kernel.portfolio.borrow_mut().initialize_orders();
1050        self.kernel.portfolio.borrow_mut().initialize_positions();
1051        self.kernel
1052            .portfolio
1053            .borrow_mut()
1054            .initialize_wallet_orders()?;
1055
1056        let elapsed_secs = start.elapsed().as_secs_f64();
1057        log_info!(
1058            "Startup reconciliation completed in {:.2}s",
1059            elapsed_secs,
1060            color = LogColor::Blue
1061        );
1062
1063        Ok(())
1064    }
1065
1066    /// Run the live node with automatic shutdown handling.
1067    ///
1068    /// This method starts the node, runs indefinitely, and handles graceful shutdown
1069    /// on interrupt signals.
1070    ///
1071    /// # Thread Safety
1072    ///
1073    /// The event loop runs directly on the current thread (not spawned) because the
1074    /// msgbus uses thread-local storage. Endpoints registered by the kernel are only
1075    /// accessible from the same thread.
1076    ///
1077    /// # Shutdown Sequence
1078    ///
1079    /// 1. Signal received (SIGINT, SIGTERM, or handle stop).
1080    /// 2. Trader components stopped (triggers order cancellations, etc.).
1081    /// 3. Event loop continues processing residual events for the configured grace period.
1082    /// 4. Kernel finalized, clients disconnected, remaining events drained.
1083    ///
1084    /// # Errors
1085    ///
1086    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1087    /// with retained submissions unresolved at the end of `delay_post_stop`.
1088    pub async fn run(&mut self) -> anyhow::Result<()> {
1089        self.run_with_mode(NodeRunMode::Owned).await
1090    }
1091
1092    /// Run the live node under the given mode.
1093    ///
1094    /// [`NodeRunMode::Hosted`] leaves signal handling to the host application. Every other
1095    /// responsibility, including maintenance, reconciliation, external ingress, and the shutdown
1096    /// sequence, is identical across modes so that hosted and owned nodes cannot diverge.
1097    ///
1098    /// # Errors
1099    ///
1100    /// Returns an error if the node fails to start, encounters a runtime error, or shuts down
1101    /// with retained submissions unresolved at the end of `delay_post_stop`.
1102    pub async fn run_with_mode(&mut self, mode: NodeRunMode) -> anyhow::Result<()> {
1103        if self.state().is_running() {
1104            anyhow::bail!("Already running");
1105        }
1106
1107        if self.runner.is_none() {
1108            anyhow::bail!("Runner already consumed - run() called twice");
1109        }
1110
1111        self.prepare_cache().await?;
1112
1113        let Some(runner) = self.runner.take() else {
1114            anyhow::bail!("Runner already consumed - run() called twice");
1115        };
1116
1117        runner.bind_senders_for_node(self.handle.clone());
1118
1119        let AsyncRunnerChannels {
1120            mut time_evt_rx,
1121            mut system_evt_rx,
1122            mut system_cmd_rx,
1123            mut exec_evt_rx,
1124            mut exec_cmd_rx,
1125            mut data_evt_rx,
1126            mut data_cmd_rx,
1127        } = runner.take_channels();
1128
1129        log::info!("Event loop starting");
1130
1131        self.handle.set_starting();
1132        self.kernel.reset_shutdown_flag();
1133        self.kernel.start_async().await;
1134
1135        if self.kernel.is_event_store_replay() {
1136            log::info!(
1137                "Event-store replay loaded; skipping live client connection and reconciliation",
1138            );
1139
1140            if !self.finish_startup_replay().await? {
1141                return Ok(());
1142            }
1143
1144            return Ok(());
1145        }
1146
1147        if self.kernel.is_event_store_replay_configured() {
1148            self.abort_startup("Event-store replay did not start")
1149                .await?;
1150            return Ok(());
1151        }
1152
1153        let mut external_msgbus_rx = match self.take_external_ingress_receiver() {
1154            Ok(rx) => rx,
1155            Err(e) => {
1156                let result = self
1157                    .abort_startup("External message bus ingress failed to start")
1158                    .await;
1159                Self::drain_channels(
1160                    &mut time_evt_rx,
1161                    &mut system_evt_rx,
1162                    &mut system_cmd_rx,
1163                    &mut exec_evt_rx,
1164                    &mut exec_cmd_rx,
1165                    &mut data_evt_rx,
1166                    &mut data_cmd_rx,
1167                );
1168                log::info!("Event loop stopped");
1169
1170                if let Err(finalize_err) = result {
1171                    anyhow::bail!(
1172                        "failed to start external message bus ingress: {e}; failed to finalize startup abort: {finalize_err}"
1173                    );
1174                }
1175
1176                return Err(e);
1177            }
1178        };
1179
1180        let stop_handle = self.handle.clone();
1181        let mut pending = PendingEvents::default();
1182        let mut startup_system_events = Vec::new();
1183        let mut startup_system_commands = Vec::new();
1184        let connection_deadline = dst::time::Instant::now() + self.config.timeout_connection;
1185
1186        // Startup phase 1: Connect data clients and drain instrument events into cache.
1187        // This ensures the cache is populated before execution clients connect.
1188        let mut pending_external = VecDeque::new();
1189        let data_connect_result = drive_with_event_buffering(
1190            drive_with_external_ingress(
1191                self.connect_data_phase(connection_deadline),
1192                &mut external_msgbus_rx,
1193                &mut pending_external,
1194            ),
1195            &mut pending,
1196            &mut time_evt_rx,
1197            &mut system_evt_rx,
1198            &mut system_cmd_rx,
1199            &mut exec_evt_rx,
1200            &mut exec_cmd_rx,
1201            &mut data_evt_rx,
1202            &mut data_cmd_rx,
1203        )
1204        .await;
1205
1206        self.finish_external_ingress_connection(external_msgbus_rx.as_ref());
1207
1208        if let Err(e) = data_connect_result {
1209            flush_all_pending(
1210                &mut pending,
1211                &mut time_evt_rx,
1212                &mut system_evt_rx,
1213                &mut system_cmd_rx,
1214                &mut exec_evt_rx,
1215                &mut exec_cmd_rx,
1216                &mut data_evt_rx,
1217                &mut data_cmd_rx,
1218            );
1219            let result = self
1220                .abort_startup_with_error("Data client connection failed", e)
1221                .await;
1222            Self::drain_channels(
1223                &mut time_evt_rx,
1224                &mut system_evt_rx,
1225                &mut system_cmd_rx,
1226                &mut exec_evt_rx,
1227                &mut exec_cmd_rx,
1228                &mut data_evt_rx,
1229                &mut data_cmd_rx,
1230            );
1231            log::info!("Event loop stopped");
1232            return result;
1233        }
1234
1235        // Flush any data events still queued in the channel receivers that the
1236        // select loop did not capture before the connect future resolved, then
1237        // drain everything into cache.
1238        flush_pending_data(&mut pending, &mut data_evt_rx, &mut data_cmd_rx);
1239        startup_system_events.extend(pending.take_system_events());
1240        startup_system_commands.extend(pending.take_system_commands());
1241        debug_assert!(
1242            pending.data_evts.is_empty() && pending.data_cmds.is_empty(),
1243            "data must be drained into cache before exec clients connect",
1244        );
1245
1246        // Startup phase 2: Connect execution clients (instruments now in cache)
1247        let engine_connection_result = drive_with_event_buffering(
1248            self.connect_exec_phase(connection_deadline),
1249            &mut pending,
1250            &mut time_evt_rx,
1251            &mut system_evt_rx,
1252            &mut system_cmd_rx,
1253            &mut exec_evt_rx,
1254            &mut exec_cmd_rx,
1255            &mut data_evt_rx,
1256            &mut data_cmd_rx,
1257        )
1258        .await;
1259
1260        // Flush channel receivers and drain all remaining pending events
1261        flush_all_pending(
1262            &mut pending,
1263            &mut time_evt_rx,
1264            &mut system_evt_rx,
1265            &mut system_cmd_rx,
1266            &mut exec_evt_rx,
1267            &mut exec_cmd_rx,
1268            &mut data_evt_rx,
1269            &mut data_cmd_rx,
1270        );
1271        startup_system_events.extend(pending.take_system_events());
1272        startup_system_commands.extend(pending.take_system_commands());
1273        debug_assert!(
1274            pending.is_empty(),
1275            "all startup events must be processed before reconciliation",
1276        );
1277
1278        let engine_connection_status = match engine_connection_result {
1279            Ok(status) => status,
1280            Err(e) => {
1281                let result = self
1282                    .abort_startup_with_error("Execution client connection failed", e)
1283                    .await;
1284                Self::drain_channels(
1285                    &mut time_evt_rx,
1286                    &mut system_evt_rx,
1287                    &mut system_cmd_rx,
1288                    &mut exec_evt_rx,
1289                    &mut exec_cmd_rx,
1290                    &mut data_evt_rx,
1291                    &mut data_cmd_rx,
1292                );
1293                log::info!("Event loop stopped");
1294                return result;
1295            }
1296        };
1297
1298        if engine_connection_status == EngineConnectionStatus::TimedOut {
1299            let result = self
1300                .abort_startup_with_error(
1301                    "Engine readiness timed out",
1302                    anyhow::anyhow!("readiness timeout while waiting for engine connections"),
1303                )
1304                .await;
1305            Self::drain_channels(
1306                &mut time_evt_rx,
1307                &mut system_evt_rx,
1308                &mut system_cmd_rx,
1309                &mut exec_evt_rx,
1310                &mut exec_cmd_rx,
1311                &mut data_evt_rx,
1312                &mut data_cmd_rx,
1313            );
1314            log::info!("Event loop stopped");
1315            return result;
1316        }
1317
1318        if let Some(reason) = engine_connection_status
1319            .abort_reason()
1320            .or_else(|| self.startup_abort_reason())
1321        {
1322            self.abort_startup(reason).await?;
1323            Self::drain_channels(
1324                &mut time_evt_rx,
1325                &mut system_evt_rx,
1326                &mut system_cmd_rx,
1327                &mut exec_evt_rx,
1328                &mut exec_cmd_rx,
1329                &mut data_evt_rx,
1330                &mut data_cmd_rx,
1331            );
1332            log::info!("Event loop stopped");
1333            return Ok(());
1334        }
1335
1336        debug_assert_eq!(engine_connection_status, EngineConnectionStatus::Connected);
1337
1338        // Run reconciliation now that instruments are in cache and start trader
1339        if let Err(e) = self.perform_startup_reconciliation().await {
1340            let result = self.abort_startup("Startup reconciliation failed").await;
1341            Self::drain_channels(
1342                &mut time_evt_rx,
1343                &mut system_evt_rx,
1344                &mut system_cmd_rx,
1345                &mut exec_evt_rx,
1346                &mut exec_cmd_rx,
1347                &mut data_evt_rx,
1348                &mut data_cmd_rx,
1349            );
1350            log::info!("Event loop stopped");
1351
1352            if let Err(finalize_err) = result {
1353                anyhow::bail!(
1354                    "startup reconciliation failed: {e}; failed to finalize startup abort: {finalize_err}"
1355                );
1356            }
1357
1358            return Err(e);
1359        }
1360
1361        if let Some(reason) = self.startup_abort_reason() {
1362            let result = self.abort_startup(reason).await;
1363            Self::drain_channels(
1364                &mut time_evt_rx,
1365                &mut system_evt_rx,
1366                &mut system_cmd_rx,
1367                &mut exec_evt_rx,
1368                &mut exec_cmd_rx,
1369                &mut data_evt_rx,
1370                &mut data_cmd_rx,
1371            );
1372            log::info!("Event loop stopped");
1373            return result;
1374        }
1375
1376        if let Err(e) = self.kernel.start_trader() {
1377            let result = self.abort_after_trader_start_failure(e).await;
1378            Self::drain_channels(
1379                &mut time_evt_rx,
1380                &mut system_evt_rx,
1381                &mut system_cmd_rx,
1382                &mut exec_evt_rx,
1383                &mut exec_cmd_rx,
1384                &mut data_evt_rx,
1385                &mut data_cmd_rx,
1386            );
1387            log::info!("Event loop stopped");
1388            return result;
1389        }
1390
1391        #[cfg(feature = "plugin")]
1392        if let Err(e) = self.plugins.start_controllers() {
1393            let result = self.abort_after_trader_start_failure(e).await;
1394            Self::drain_channels(
1395                &mut time_evt_rx,
1396                &mut system_evt_rx,
1397                &mut system_cmd_rx,
1398                &mut exec_evt_rx,
1399                &mut exec_cmd_rx,
1400                &mut data_evt_rx,
1401                &mut data_cmd_rx,
1402            );
1403            log::info!("Event loop stopped");
1404            return result;
1405        }
1406
1407        self.process_system_events(startup_system_events);
1408        self.process_system_commands(startup_system_commands);
1409
1410        let finish_result = {
1411            let mut receivers = RunnerReceivers {
1412                time_evt: &mut time_evt_rx,
1413                system_evt: &mut system_evt_rx,
1414                system_cmd: &mut system_cmd_rx,
1415                data_evt: &mut data_evt_rx,
1416                data_cmd: &mut data_cmd_rx,
1417                exec_evt: &mut exec_evt_rx,
1418                exec_cmd: &mut exec_cmd_rx,
1419            };
1420
1421            self.finish_startup_trader(Some(&mut receivers)).await
1422        };
1423
1424        match finish_result {
1425            Ok(true) => {}
1426            result => {
1427                log::info!("Event loop stopped");
1428                return result.map(|_| ());
1429            }
1430        }
1431
1432        let exec_config = &self.config.exec_engine;
1433        let inflight_interval =
1434            Duration::from_millis(u64::from(exec_config.inflight_check_interval_ms));
1435
1436        let open_interval = exec_config
1437            .open_check_interval_secs
1438            .filter(|&s| s > 0.0)
1439            .map_or(Duration::ZERO, |secs| {
1440                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1441            });
1442
1443        let position_interval = exec_config
1444            .position_check_interval_secs
1445            .filter(|&s| s > 0.0)
1446            .map_or(Duration::ZERO, |secs| {
1447                Duration::from_nanos(secs_to_nanos_unchecked(secs))
1448            });
1449
1450        let has_clients = !self
1451            .kernel
1452            .exec_engine
1453            .borrow()
1454            .get_all_clients()
1455            .is_empty();
1456        let recon_enabled = has_clients
1457            && (!inflight_interval.is_zero()
1458                || !open_interval.is_zero()
1459                || !position_interval.is_zero());
1460
1461        let recon_min_interval = if recon_enabled {
1462            let mut intervals = Vec::new();
1463
1464            if !inflight_interval.is_zero() {
1465                intervals.push(inflight_interval);
1466            }
1467
1468            if !open_interval.is_zero() {
1469                intervals.push(open_interval);
1470            }
1471
1472            if !position_interval.is_zero() {
1473                intervals.push(position_interval);
1474            }
1475
1476            intervals
1477                .into_iter()
1478                .min()
1479                .unwrap_or(Duration::from_secs(1))
1480        } else {
1481            Duration::from_secs(1) // Unused, timer won't fire
1482        };
1483
1484        // `reconciliation_startup_delay_secs` is a post-reconciliation grace period:
1485        // startup reconciliation has already completed above, and this delay offsets
1486        // the first periodic tick to let the system stabilize before continuous checks
1487        // begin.
1488        let startup_delay = if self.config.exec_engine.reconciliation {
1489            Duration::from_secs_f64(exec_config.reconciliation_startup_delay_secs)
1490        } else {
1491            Duration::ZERO
1492        };
1493
1494        let recon_start = dst::time::Instant::now() + startup_delay;
1495
1496        let mut last_inflight_check = dst::time::Instant::now();
1497        let mut last_open_check = last_inflight_check;
1498        let mut last_position_check = last_inflight_check;
1499
1500        // Per-task `(interval, next_fire)` schedules dispatched by the
1501        // shared `maintenance_timer` below. See module docs for rationale.
1502        let far_future = Duration::from_hours(24 * 365 * 100);
1503
1504        let make_schedule = |opt_dur: Option<Duration>| -> (Duration, dst::time::Instant) {
1505            let dur = opt_dur.unwrap_or(far_future);
1506            (dur, recon_start + dur)
1507        };
1508
1509        let (recon_interval, mut recon_next) = make_schedule(if recon_enabled {
1510            Some(recon_min_interval)
1511        } else {
1512            None
1513        });
1514
1515        let (purge_orders_interval, mut purge_orders_next) = make_schedule(
1516            exec_config
1517                .purge_closed_orders_interval_mins
1518                .filter(|&m| m > 0)
1519                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1520        );
1521
1522        let (purge_positions_interval, mut purge_positions_next) = make_schedule(
1523            exec_config
1524                .purge_closed_positions_interval_mins
1525                .filter(|&m| m > 0)
1526                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1527        );
1528
1529        let (purge_account_interval, mut purge_account_next) = make_schedule(
1530            exec_config
1531                .purge_account_events_interval_mins
1532                .filter(|&m| m > 0)
1533                .map(|m| Duration::from_secs(mins_to_secs(u64::from(m)))),
1534        );
1535
1536        let (own_books_interval, mut own_books_next) = make_schedule(
1537            exec_config
1538                .own_books_audit_interval_secs
1539                .filter(|&s| s > 0.0)
1540                .map(Duration::from_secs_f64),
1541        );
1542
1543        let (prune_fills_interval, mut prune_fills_next) =
1544            make_schedule(Some(Duration::from_mins(1)));
1545
1546        let mut maintenance_timer = dst::time::interval(Duration::from_millis(100));
1547        maintenance_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1548
1549        // Stop-check timer is not subject to the reconciliation startup delay,
1550        // so shutdown signals remain responsive from the moment the node reaches
1551        // `Running`. Set `MissedTickBehavior::Skip` so backlog ticks do not fire
1552        // a burst after the select arm was suspended by other branches.
1553        let mut stop_check_timer = dst::time::interval(Duration::from_millis(100));
1554        stop_check_timer.set_missed_tick_behavior(dst::time::MissedTickBehavior::Skip);
1555
1556        // Running phase: runs until shutdown deadline expires
1557        let mut residual_events = 0usize;
1558        let mut open_order_report_task: Option<OpenOrderReportTask> = None;
1559        let mut targeted_order_report_task: Option<TargetedOrderReportTask> = None;
1560        let mut position_report_task: Option<PositionReportTask> = None;
1561
1562        // A hosted node never installs signal handlers, so these futures stay pending and their
1563        // listeners are never registered. Both arms resolve to the same type as the real listeners.
1564        let owns_signals = mode.owns_signals();
1565
1566        let ctrl_c = async move {
1567            if owns_signals {
1568                dst::signal::ctrl_c().await
1569            } else {
1570                std::future::pending::<std::io::Result<()>>().await
1571            }
1572        };
1573
1574        let terminate = async move {
1575            if owns_signals {
1576                dst::signal::terminate().await
1577            } else {
1578                std::future::pending::<std::io::Result<()>>().await
1579            }
1580        };
1581
1582        tokio::pin!(ctrl_c);
1583        tokio::pin!(terminate);
1584
1585        let metrics = self.handle.metrics.clone();
1586        let metrics_start = dst::time::Instant::now();
1587        metrics.reset();
1588
1589        let mut queue_monitor = self
1590            .config
1591            .queue_monitor
1592            .as_ref()
1593            .map(|config| QueueMonitor::new(config, metrics.snapshot()));
1594        let mut dispatches_since_yield = 0usize;
1595        let mut skip_time = false;
1596        let mut skip_data = false;
1597
1598        let dispatch_result = loop {
1599            let callbacks_pending = match drain_callbacks().await {
1600                Ok(pending) => pending,
1601                Err(e) => {
1602                    if self.state() == NodeState::Running {
1603                        self.initiate_shutdown();
1604                    }
1605
1606                    log::warn!(
1607                        "Skipping residual events and final buffered dispatch after callback failure"
1608                    );
1609
1610                    break Err(e);
1611                }
1612            };
1613
1614            let shutdown_deadline = self.shutdown_deadline;
1615            let is_shutting_down = self.state() == NodeState::ShuttingDown;
1616            let is_running = self.state() == NodeState::Running;
1617            let commands_empty = data_cmd_rx.is_empty();
1618            let poll_time = !skip_time || (data_evt_rx.is_empty() && commands_empty);
1619            let poll_data = !skip_data || commands_empty;
1620
1621            tokio::select! {
1622                biased;
1623
1624                // Signal branches first so they are always checked
1625                result = &mut ctrl_c, if is_running => {
1626                    match result {
1627                        Ok(()) => log::info!("Received SIGINT, shutting down"),
1628                        Err(e) => log::error!("Failed to listen for SIGINT: {e}"),
1629                    }
1630                    self.initiate_shutdown();
1631                }
1632                result = &mut terminate, if is_running => {
1633                    match result {
1634                        Ok(()) => log::info!("Received SIGTERM, shutting down"),
1635                        Err(e) => log::error!("Failed to listen for SIGTERM: {e}"),
1636                    }
1637                    self.initiate_shutdown();
1638                }
1639                _ = stop_check_timer.tick(), if is_running => {
1640                    if stop_handle.should_stop() {
1641                        log::info!("Received stop signal from handle");
1642                        self.initiate_shutdown();
1643                    } else if self.kernel.is_shutdown_requested() {
1644                        log::info!("Received ShutdownSystem command, shutting down");
1645                        self.initiate_shutdown();
1646                    }
1647                }
1648                () = async {
1649                    match shutdown_deadline {
1650                        Some(deadline) => dst::time::sleep_until(deadline).await,
1651                        None => std::future::pending::<()>().await,
1652                    }
1653                }, if self.state() == NodeState::ShuttingDown => {
1654                    break Ok(());
1655                }
1656                () = std::future::ready(()), if callbacks_pending => {},
1657                result = async {
1658                    match open_order_report_task.as_mut() {
1659                        Some(task) => task.future.as_mut().await,
1660                        None => std::future::pending::<ReportTaskOutcome<OpenOrderReportCollection>>().await,
1661                    }
1662                }, if open_order_report_task.is_some() => {
1663                    let maintenance_start = dst::time::Instant::now();
1664
1665                    drop(open_order_report_task.take());
1666
1667                    match result {
1668                        ReportTaskOutcome::Completed(result) => {
1669                            let result = result.finish();
1670                            self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1671                            let client_refs = self
1672                                .exec_clients
1673                                .iter()
1674                                .map(|client| client as &dyn ExecutionClient)
1675                                .collect::<Vec<_>>();
1676                            let reconciliation = self.exec_manager.reconcile_open_order_reports(
1677                                &result.check,
1678                                result.reports,
1679                                &result.queried_clients,
1680                                &result.failed_clients,
1681                                &client_refs,
1682                            );
1683                            self.process_reconciliation_events(&reconciliation.events);
1684                            if !reconciliation.targeted_queries.is_empty() {
1685                                if is_shutting_down {
1686                                    let planned_client_order_ids = reconciliation
1687                                        .targeted_queries
1688                                        .iter()
1689                                        .map(TargetedOrderQuery::client_order_id)
1690                                        .collect::<Vec<_>>();
1691                                    self.cleanup_cancelled_report_tasks(
1692                                        &planned_client_order_ids,
1693                                    );
1694                                } else {
1695                                    targeted_order_report_task = Some(
1696                                        self.start_targeted_order_report_check(
1697                                            reconciliation.targeted_queries,
1698                                        ),
1699                                    );
1700                                }
1701                            }
1702                        }
1703                        ReportTaskOutcome::TimedOut => {
1704                            self.cleanup_cancelled_report_tasks(&[]);
1705                            log::warn!(
1706                                "Open-order report collection expired after {:?}",
1707                                self.config.timeout_reconciliation,
1708                            );
1709                        }
1710                    }
1711                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1712                }
1713                result = async {
1714                    match targeted_order_report_task.as_mut() {
1715                        Some(task) => task.future.as_mut().await,
1716                        None => std::future::pending::<ReportTaskOutcome<()>>().await,
1717                    }
1718                }, if targeted_order_report_task.is_some() => {
1719                    let maintenance_start = dst::time::Instant::now();
1720
1721                    if let Some(task) = targeted_order_report_task.take() {
1722                        self.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
1723                        self.complete_targeted_order_report_check(task, &result);
1724                    }
1725                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1726                }
1727                result = async {
1728                    match position_report_task.as_mut() {
1729                        Some(task) => task.future.as_mut().await,
1730                        None => std::future::pending::<ReportTaskOutcome<PositionReportTaskResult>>().await,
1731                    }
1732                }, if position_report_task.is_some() => {
1733                    let maintenance_start = dst::time::Instant::now();
1734
1735                    drop(position_report_task.take());
1736
1737                    match result {
1738                        ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) => {
1739                            if is_shutting_down {
1740                                self.cleanup_cancelled_report_tasks(&[]);
1741                            } else {
1742                                let result = result.finish();
1743                                position_report_task = self.handle_position_report_result(result);
1744                            }
1745                        }
1746                        ReportTaskOutcome::Completed(PositionReportTaskResult::Fills { position_result, collection }) => {
1747                            let reports = collection.finish();
1748                            self.handle_position_fill_report_result(PositionFillReportResult {
1749                                position_result,
1750                                reports,
1751                            });
1752                        }
1753                        ReportTaskOutcome::TimedOut => {
1754                            self.cleanup_cancelled_report_tasks(&[]);
1755                            log::warn!(
1756                                "Position report collection expired after {:?}",
1757                                self.config.timeout_reconciliation,
1758                            );
1759                        }
1760                    }
1761                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1762                }
1763
1764                // Maintenance dispatcher (before event processing to avoid
1765                // starvation). See module docs for design rationale.
1766                _ = maintenance_timer.tick(), if is_running => {
1767                    let maintenance_start = dst::time::Instant::now();
1768                    metrics.publish_queue_depths(
1769                        RunnerChannelQueueDepths::from_receivers(
1770                            &time_evt_rx,
1771                            &exec_evt_rx,
1772                            &exec_cmd_rx,
1773                            &data_evt_rx,
1774                            &data_cmd_rx,
1775                        ),
1776                        metrics_start.elapsed(),
1777                    );
1778
1779                    if let Some(queue_monitor) = queue_monitor.as_mut() {
1780                        let transitions = queue_monitor.evaluate(metrics.snapshot());
1781                        self.publish_queue_state_transitions(&transitions);
1782                    }
1783
1784                    let mut now = dst::time::Instant::now();
1785
1786                    if recon_enabled && now >= recon_next {
1787                        let recon_intervals = ReconciliationCheckIntervals {
1788                            inflight: inflight_interval,
1789                            open: open_interval,
1790                            position: position_interval,
1791                        };
1792                        let mut recon_state = ReconciliationCheckState {
1793                            last_inflight_check: &mut last_inflight_check,
1794                            last_open_check: &mut last_open_check,
1795                            last_position_check: &mut last_position_check,
1796                            open_order_report_task: &mut open_order_report_task,
1797                            targeted_order_report_task: &mut targeted_order_report_task,
1798                            position_report_task: &mut position_report_task,
1799                        };
1800
1801                        self.run_reconciliation_checks(
1802                            now,
1803                            recon_intervals,
1804                            &mut recon_state,
1805                        );
1806
1807                        now = dst::time::Instant::now();
1808                        recon_next = now + recon_interval;
1809                    }
1810
1811                    if now >= purge_orders_next {
1812                        self.exec_manager.purge_closed_orders();
1813                        purge_orders_next = now + purge_orders_interval;
1814                    }
1815
1816                    if now >= purge_positions_next {
1817                        self.exec_manager.purge_closed_positions();
1818                        purge_positions_next = now + purge_positions_interval;
1819                    }
1820
1821                    if now >= purge_account_next {
1822                        self.exec_manager.purge_account_events();
1823                        purge_account_next = now + purge_account_interval;
1824                    }
1825
1826                    if now >= own_books_next {
1827                        self.kernel.cache().borrow_mut().audit_own_order_books();
1828                        own_books_next = now + own_books_interval;
1829                    }
1830
1831                    if now >= prune_fills_next {
1832                        self.exec_manager.prune_recent_fills_cache(60.0);
1833                        self.exec_manager.prune_processed_fills();
1834                        self.exec_manager.prune_order_local_activity();
1835                        prune_fills_next = now + prune_fills_interval;
1836                    }
1837
1838                    record_runner_maintenance(&metrics, maintenance_start, metrics_start);
1839                }
1840
1841                // Event processing branches. Exec commands and events are
1842                // ordered ahead of data events so a strategy action (cancel,
1843                // submit, etc.) is not delayed behind a market data backlog
1844                // when the biased select polls receivers each iteration.
1845                Some(handler) = time_evt_rx.recv(), if poll_time => {
1846                    skip_time = true;
1847                    let dispatch_start = dst::time::Instant::now();
1848                    let dispatched = AsyncRunner::handle_time_event(handler);
1849
1850                    if dispatched && is_shutting_down {
1851                        log::debug!("Residual time event");
1852                        residual_events += 1;
1853                    }
1854
1855                    if dispatched {
1856                        record_runner_dispatch(
1857                            &metrics,
1858                            SystemChannel::TimeEvents,
1859                            dispatch_start,
1860                            metrics_start,
1861                        );
1862                    }
1863                }
1864                Some(event) = system_evt_rx.recv() => {
1865                    skip_time = false;
1866
1867                    if is_shutting_down {
1868                        log::debug!("Residual system event: {event}");
1869                        residual_events += 1;
1870                    }
1871                    event.dispatch(|event| self.process_system_event(event));
1872                }
1873                Some(command) = system_cmd_rx.recv() => {
1874                    skip_time = false;
1875
1876                    if is_shutting_down {
1877                        log::debug!("Residual system command: {command}");
1878                        residual_events += 1;
1879                    }
1880                    command.dispatch(|command| self.process_system_command(command));
1881                }
1882                Some(evt) = exec_evt_rx.recv() => {
1883                    skip_time = false;
1884                    let dispatch_start = dst::time::Instant::now();
1885
1886                    if is_shutting_down {
1887                        log::debug!("Residual exec event: {evt}");
1888                        residual_events += 1;
1889                    }
1890
1891                    evt.dispatch(|evt| self.process_exec_event(evt));
1892                    record_runner_dispatch(
1893                        &metrics,
1894                        SystemChannel::ExecEvents,
1895                        dispatch_start,
1896                        metrics_start,
1897                    );
1898                }
1899                Some(cmd) = exec_cmd_rx.recv() => {
1900                    skip_time = false;
1901                    let dispatch_start = dst::time::Instant::now();
1902
1903                    if is_shutting_down {
1904                        log::debug!("Residual exec command: {cmd}");
1905                        residual_events += 1;
1906                    }
1907
1908                    self.process_exec_command(cmd);
1909                    record_runner_dispatch(
1910                        &metrics,
1911                        SystemChannel::ExecCommands,
1912                        dispatch_start,
1913                        metrics_start,
1914                    );
1915                }
1916                message = recv_external_msgbus_message(&mut external_msgbus_rx, &mut pending_external) => {
1917                    skip_time = false;
1918                    let external_msgbus_start = dst::time::Instant::now();
1919
1920                    match message {
1921                        Some(message) => {
1922                            if is_shutting_down {
1923                                log::debug!("Residual external message bus message: {message}");
1924                                residual_events += 1;
1925                            }
1926                            self.process_external_msgbus_message(&message);
1927                        }
1928                        None => {
1929                            log::info!("External message bus ingress closed");
1930                            external_msgbus_rx = None;
1931                            self.close_external_ingress();
1932                        }
1933                    }
1934
1935                    record_runner_external_msgbus(
1936                        &metrics,
1937                        external_msgbus_start,
1938                        metrics_start,
1939                    );
1940                }
1941                Some(evt) = data_evt_rx.recv(), if poll_data => {
1942                    skip_time = false;
1943                    skip_data = true;
1944                    let dispatch_start = dst::time::Instant::now();
1945
1946                    if is_shutting_down {
1947                        log::debug!("Residual data event: {evt:?}");
1948                        residual_events += 1;
1949                    }
1950                    AsyncRunner::dispatch_data_event(evt);
1951                    record_runner_dispatch(
1952                        &metrics,
1953                        SystemChannel::DataEvents,
1954                        dispatch_start,
1955                        metrics_start,
1956                    );
1957                }
1958                Some(cmd) = data_cmd_rx.recv() => {
1959                    skip_time = false;
1960                    skip_data = false;
1961                    let dispatch_start = dst::time::Instant::now();
1962
1963                    if is_shutting_down {
1964                        log::debug!("Residual data command: {cmd:?}");
1965                        residual_events += 1;
1966                    }
1967                    AsyncRunner::handle_data_command(cmd);
1968                    record_runner_dispatch(
1969                        &metrics,
1970                        SystemChannel::DataCommands,
1971                        dispatch_start,
1972                        metrics_start,
1973                    );
1974                }
1975            }
1976
1977            dispatches_since_yield += 1;
1978            if dispatches_since_yield >= DISPATCHES_PER_YIELD {
1979                dispatches_since_yield = 0;
1980                dst::task::yield_now().await;
1981            }
1982        };
1983
1984        if residual_events > 0 {
1985            log::debug!("Processed {residual_events} residual events during shutdown");
1986        }
1987
1988        self.cancel_report_tasks(
1989            &mut open_order_report_task,
1990            &mut targeted_order_report_task,
1991            &mut position_report_task,
1992            dispatch_result.is_ok(),
1993        );
1994
1995        drop(external_msgbus_rx.take());
1996        let _ = self.kernel.cache().borrow().check_residuals();
1997
1998        let stop_result = self.finalize_stop().await;
1999
2000        if let Err(e) = dispatch_result {
2001            if let Err(stop_err) = stop_result {
2002                log::error!("Failed to finalize node after callback failure: {stop_err}");
2003                // Python converts the outer display message, so preserve both diagnostics there.
2004                let diagnostic = format!("{e}; {stop_err}");
2005                return Err(anyhow::Error::new(e).context(diagnostic));
2006            }
2007
2008            return Err(e.into());
2009        }
2010
2011        // Handle events that arrived during finalize_stop
2012        Self::drain_channels(
2013            &mut time_evt_rx,
2014            &mut system_evt_rx,
2015            &mut system_cmd_rx,
2016            &mut exec_evt_rx,
2017            &mut exec_cmd_rx,
2018            &mut data_evt_rx,
2019            &mut data_cmd_rx,
2020        );
2021
2022        log::info!("Event loop stopped");
2023
2024        stop_result
2025    }
2026
2027    fn publish_queue_state_transitions(&self, transitions: &[QueueStateTransition]) {
2028        for transition in transitions {
2029            let topic = MessagingSwitchboard::queue_state_changed_topic(transition.channel);
2030            let timestamp = self.kernel.generate_timestamp_ns();
2031
2032            let event = QueueStateChanged::new(
2033                self.config.trader_id,
2034                transition.channel,
2035                transition.condition,
2036                transition.state,
2037                transition.queue_depth,
2038                transition.mean_dispatch_ns,
2039                UUID4::new(),
2040                timestamp,
2041                timestamp,
2042            );
2043
2044            msgbus::publish_any(topic, event.as_any());
2045        }
2046    }
2047
2048    #[expect(
2049        clippy::await_holding_refcell_ref,
2050        reason = "cache loading is serialized before the single-threaded live node starts"
2051    )]
2052    async fn prepare_cache(&mut self) -> anyhow::Result<()> {
2053        self.install_cache_database().await?;
2054
2055        let cache = self.kernel.cache();
2056        if !cache.borrow().has_backing() {
2057            return Ok(());
2058        }
2059
2060        if self
2061            .config
2062            .cache
2063            .as_ref()
2064            .is_some_and(|config| config.flush_on_start)
2065        {
2066            cache
2067                .borrow_mut()
2068                .flush_db()
2069                .context("Failed to flush persistent cache")?;
2070            return Ok(());
2071        }
2072
2073        if self.config.exec_engine.load_cache {
2074            self.kernel
2075                .exec_engine()
2076                .borrow_mut()
2077                .load_cache()
2078                .await
2079                .context("Failed to load persistent cache")?;
2080        }
2081
2082        Ok(())
2083    }
2084
2085    /// Returns whether a cache database backing is configured but not yet installed.
2086    #[must_use]
2087    pub const fn has_pending_cache_database(&self) -> bool {
2088        self.cache_database_factory.is_some()
2089    }
2090
2091    /// Constructs the configured cache database backing and installs it on the kernel cache.
2092    ///
2093    /// Construction is deferred to startup so the adapter is built inside the async runtime rather
2094    /// than by blocking the synchronous builder, and so the connection opens only when the node runs.
2095    async fn install_cache_database(&mut self) -> anyhow::Result<()> {
2096        let Some(factory) = self.cache_database_factory.as_ref() else {
2097            return Ok(());
2098        };
2099
2100        let config = self.config.cache.clone().unwrap_or_default();
2101
2102        // Cleared only after a successful install, so a failed startup can be retried rather than
2103        // silently starting without the backing the caller asked for.
2104        let database = factory
2105            .create(self.config.trader_id, self.kernel.instance_id, config)
2106            .await
2107            .context("failed to create cache database backing")?;
2108        self.kernel.cache().borrow_mut().set_database(database);
2109        self.cache_database_factory = None;
2110
2111        Ok(())
2112    }
2113
2114    fn take_external_ingress_receiver(
2115        &mut self,
2116    ) -> anyhow::Result<Option<tokio::sync::mpsc::Receiver<BusMessage>>> {
2117        let Some(external_ingress) = self.external_msgbus.as_mut() else {
2118            return Ok(None);
2119        };
2120
2121        let receiver = external_ingress.take_receiver()?;
2122        log::info!("External message bus ingress started");
2123        Ok(Some(receiver))
2124    }
2125
2126    fn republish_external_msgbus_message(message: &BusMessage) {
2127        if let Err(e) = msgbus::republish_external_message(message) {
2128            log::error!(
2129                "Failed to republish external message bus topic '{}': {e:#}",
2130                message.topic
2131            );
2132        }
2133    }
2134
2135    fn process_external_msgbus_message(&self, message: &BusMessage) {
2136        if self.stream_processors.is_empty() {
2137            Self::republish_external_msgbus_message(message);
2138            return;
2139        }
2140
2141        let mut process = |value: &dyn Any, mapping: &serde_json::Value| {
2142            for processor in &self.stream_processors {
2143                (processor.0)(value, mapping)?;
2144            }
2145
2146            Ok(())
2147        };
2148
2149        if let Err(e) = msgbus::process_external_typed_message(message, &mut process) {
2150            log::error!(
2151                "Failed to process external message bus topic '{}': {e:#}",
2152                message.topic
2153            );
2154        }
2155    }
2156
2157    fn finish_external_ingress_connection(
2158        &mut self,
2159        receiver: Option<&tokio::sync::mpsc::Receiver<BusMessage>>,
2160    ) {
2161        if receiver.is_none() {
2162            self.close_external_ingress();
2163        }
2164    }
2165
2166    fn close_external_ingress(&mut self) {
2167        if let Some(external_ingress) = self.external_msgbus.as_mut()
2168            && !external_ingress.is_closed()
2169        {
2170            external_ingress.close();
2171        }
2172    }
2173
2174    fn drain_exec_events_before_reconciliation(
2175        &mut self,
2176        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2177    ) {
2178        while let Ok(event) = exec_evt_rx.try_recv() {
2179            event.dispatch(|event| self.process_exec_event(event));
2180        }
2181    }
2182
2183    fn process_reconciliation_events(&mut self, events: &[OrderEventAny]) {
2184        if events.is_empty() {
2185            self.publish_submission_recovery_exhaustions();
2186            return;
2187        }
2188
2189        log::info!(
2190            "Processing {} reconciliation event{}",
2191            events.len(),
2192            if events.len() == 1 { "" } else { "s" }
2193        );
2194
2195        for event in events {
2196            self.exec_manager
2197                .record_local_activity(event.client_order_id());
2198            if let OrderEventAny::Filled(fill) = event {
2199                self.exec_manager
2200                    .record_position_activity(fill.instrument_id, fill.account_id);
2201            }
2202
2203            self.kernel.exec_engine.borrow_mut().process(event);
2204            self.exec_manager
2205                .confirm_submission_outcome(&event.client_order_id());
2206            if let OrderEventAny::Filled(fill) = event {
2207                self.exec_manager.commit_recent_fill_if_applied(fill);
2208            }
2209        }
2210
2211        self.publish_submission_recovery_exhaustions();
2212    }
2213
2214    fn publish_submission_recovery_exhaustions(&mut self) {
2215        for diagnostic in self.exec_manager.take_submission_recovery_exhaustions() {
2216            log::warn!(
2217                "Submission recovery exhausted for {}: {:?} after {} checks; submission remains unresolved pending venue evidence",
2218                diagnostic.client_order_id,
2219                diagnostic.source,
2220                diagnostic.retry_count,
2221            );
2222            msgbus::publish_any(
2223                MessagingSwitchboard::submission_recovery_exhausted_topic(),
2224                &diagnostic,
2225            );
2226        }
2227    }
2228
2229    fn process_exec_event(&mut self, event: ExecutionEvent) {
2230        let clear_closed_orders = matches!(
2231            &event,
2232            ExecutionEvent::Order(_) | ExecutionEvent::OrderCanceledBatch(_)
2233        );
2234        let closed_report_id = match &event {
2235            ExecutionEvent::Report(
2236                ExecutionReport::Order(report) | ExecutionReport::OrderWithFills(report, _),
2237            ) if report.order_status.is_closed() => report.client_order_id,
2238            _ => None,
2239        };
2240        let Some(order_ids) = self.observe_exec_event_before_dispatch(&event) else {
2241            return;
2242        };
2243
2244        self.dispatch_exec_event_and_commit_fill(event);
2245
2246        for client_order_id in &order_ids {
2247            self.exec_manager
2248                .confirm_submission_outcome(client_order_id);
2249            let is_closed = self
2250                .kernel
2251                .cache()
2252                .borrow()
2253                .order(client_order_id)
2254                .is_some_and(|order| order.is_closed());
2255
2256            if is_closed && (clear_closed_orders || closed_report_id == Some(*client_order_id)) {
2257                self.exec_manager
2258                    .clear_recon_tracking(client_order_id, true);
2259            }
2260        }
2261    }
2262
2263    fn process_exec_command(&mut self, message: DispatchMessage<TradingCommandMessage>) {
2264        message.dispatch_trading(|message| {
2265            if message.endpoint() == MessagingSwitchboard::exec_engine_execute() {
2266                self.observe_exec_command_before_dispatch(message.command());
2267            }
2268        });
2269    }
2270
2271    /// Dispatches a normal-ingress execution event, then commits a direct
2272    /// `OrderFilled` to the recent-fills dedup cache only once it is present on
2273    /// its canonical order.
2274    ///
2275    /// The fill candidate is captured from `&evt` BEFORE the value is moved into
2276    /// [`AsyncRunner::handle_exec_event`]; the gated commit runs AFTER dispatch,
2277    /// so a fill the execution engine rejects (unknown order, invalid
2278    /// transition) is never marked and its later `Fill` report stays eligible.
2279    fn dispatch_exec_event_and_commit_fill(&mut self, evt: ExecutionEvent) {
2280        let recent_fill_candidate = match &evt {
2281            ExecutionEvent::Order(OrderEventAny::Filled(fill)) => Some(fill.clone()),
2282            _ => None,
2283        };
2284
2285        AsyncRunner::handle_exec_event(evt);
2286
2287        if let Some(fill) = &recent_fill_candidate {
2288            self.exec_manager.commit_recent_fill_if_applied(fill);
2289        }
2290    }
2291
2292    async fn connect_data_phase(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2293        // A zero remaining budget still admits an immediately-ready connect (an
2294        // empty/ready client set completes on the first poll); a pending connect
2295        // fails closed on that same poll. This keeps a zero `timeout_connection`
2296        // - a supported "do not wait, but allow ready work" configuration -
2297        // working, while still bounding a hung connect.
2298        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2299        dst::time::timeout(remaining, self.kernel.connect_data_clients())
2300            .await
2301            .map_err(|_| anyhow::anyhow!("data-connect timeout"))?
2302    }
2303
2304    async fn connect_exec_clients(&mut self, deadline: dst::time::Instant) -> anyhow::Result<()> {
2305        let remaining = deadline.saturating_duration_since(dst::time::Instant::now());
2306        dst::time::timeout(remaining, self.kernel.connect_exec_clients())
2307            .await
2308            .map_err(|_| anyhow::anyhow!("exec-connect timeout"))?
2309    }
2310
2311    /// Connects execution clients and checks all engines are connected.
2312    ///
2313    /// Returns the final connection wait status.
2314    /// Must be called after data clients are connected and instrument events drained.
2315    async fn connect_exec_phase(
2316        &mut self,
2317        deadline: dst::time::Instant,
2318    ) -> anyhow::Result<EngineConnectionStatus> {
2319        self.connect_exec_clients(deadline).await?;
2320        Ok(self.await_engines_connected(deadline).await)
2321    }
2322
2323    fn startup_abort_reason(&self) -> Option<&'static str> {
2324        if self.handle.should_stop() {
2325            Some("Stop signal received during startup")
2326        } else if self.kernel.is_shutdown_requested() {
2327            Some("Shutdown signal received during startup")
2328        } else {
2329            None
2330        }
2331    }
2332
2333    async fn finish_startup_replay(&mut self) -> anyhow::Result<bool> {
2334        match self.handle.try_set_running() {
2335            RunningTransition::Entered => Ok(true),
2336            RunningTransition::StopRequested => {
2337                self.abort_startup("Stop signal received during startup")
2338                    .await?;
2339                Ok(false)
2340            }
2341            RunningTransition::Invalid(control) => {
2342                self.abort_startup_with_error(
2343                    "Invalid lifecycle state during startup",
2344                    anyhow::anyhow!(
2345                        "Invalid LiveNode control state {control:#04x} while entering Running"
2346                    ),
2347                )
2348                .await?;
2349                Ok(false)
2350            }
2351        }
2352    }
2353
2354    async fn finish_startup_trader(
2355        &mut self,
2356        receivers: Option<&mut RunnerReceivers<'_>>,
2357    ) -> anyhow::Result<bool> {
2358        match self.handle.try_set_running() {
2359            RunningTransition::Entered => Ok(true),
2360            RunningTransition::StopRequested => {
2361                self.abort_started_trader("Stop signal received during startup", receivers)
2362                    .await?;
2363                Ok(false)
2364            }
2365            RunningTransition::Invalid(control) => {
2366                let state_err = anyhow::anyhow!(
2367                    "Invalid LiveNode control state {control:#04x} while entering Running"
2368                );
2369
2370                match self
2371                    .abort_started_trader("Invalid lifecycle state during startup", receivers)
2372                    .await
2373                {
2374                    Ok(()) => Err(state_err),
2375                    Err(finalize_err) => {
2376                        anyhow::bail!(
2377                            "{state_err}; failed to finalize startup abort: {finalize_err}"
2378                        )
2379                    }
2380                }
2381            }
2382        }
2383    }
2384
2385    async fn abort_startup(&mut self, reason: &str) -> anyhow::Result<()> {
2386        log::info!("{reason}, aborting startup");
2387        self.handle.set_shutting_down();
2388        self.finalize_stop().await
2389    }
2390
2391    async fn abort_startup_with_error(
2392        &mut self,
2393        reason: &str,
2394        startup_err: anyhow::Error,
2395    ) -> anyhow::Result<()> {
2396        match self.abort_startup(reason).await {
2397            Ok(()) => Err(startup_err),
2398            Err(finalize_err) => {
2399                anyhow::bail!("{startup_err}; failed to finalize startup abort: {finalize_err}")
2400            }
2401        }
2402    }
2403
2404    async fn abort_started_trader(
2405        &mut self,
2406        reason: &str,
2407        mut receivers: Option<&mut RunnerReceivers<'_>>,
2408    ) -> anyhow::Result<()> {
2409        log::info!("{reason}, aborting startup");
2410        self.handle.set_shutting_down();
2411
2412        #[cfg(feature = "plugin")]
2413        let controller_stop_result = self.plugins.stop_controllers();
2414        #[cfg(not(feature = "plugin"))]
2415        let controller_stop_result: anyhow::Result<()> = Ok(());
2416
2417        let trader_stop_result = self.kernel.stop_trader_after_start_failure();
2418        let delay = self.kernel.delay_post_stop();
2419        log::info!("Awaiting residual events ({delay:?})...");
2420
2421        let residual_events = match receivers.as_mut() {
2422            Some(receivers) => self.process_receivers_for(delay, receivers).await,
2423            None => self.process_runner_for(delay).await,
2424        };
2425
2426        if residual_events > 0 {
2427            log::debug!("Processed {residual_events} residual events during shutdown");
2428        }
2429
2430        let finalize_result = self.finalize_stop().await;
2431
2432        if let Some(receivers) = receivers {
2433            Self::drain_channels(
2434                receivers.time_evt,
2435                receivers.system_evt,
2436                receivers.system_cmd,
2437                receivers.exec_evt,
2438                receivers.exec_cmd,
2439                receivers.data_evt,
2440                receivers.data_cmd,
2441            );
2442        } else {
2443            self.drain_runner_pending();
2444        }
2445
2446        let mut errors = Vec::new();
2447
2448        if let Err(e) = controller_stop_result {
2449            errors.push(format!("Failed to stop plug-in controllers: {e}"));
2450        }
2451
2452        if let Err(e) = trader_stop_result {
2453            errors.push(format!("Failed to stop trader: {e}"));
2454        }
2455
2456        if let Err(e) = finalize_result {
2457            errors.push(format!("Failed to finalize startup abort: {e}"));
2458        }
2459
2460        if errors.is_empty() {
2461            Ok(())
2462        } else {
2463            anyhow::bail!("{}", errors.join("; "))
2464        }
2465    }
2466
2467    async fn process_receivers_for(
2468        &mut self,
2469        duration: Duration,
2470        receivers: &mut RunnerReceivers<'_>,
2471    ) -> usize {
2472        let deadline = dst::time::Instant::now() + duration;
2473        let mut processed = 0;
2474
2475        loop {
2476            tokio::select! {
2477                biased;
2478
2479                () = dst::time::sleep_until(deadline) => break,
2480                Some(message) = receivers.time_evt.recv() => {
2481                    let _ = AsyncRunner::handle_time_event(message);
2482                    processed += 1;
2483                }
2484                Some(event) = receivers.system_evt.recv() => {
2485                    event.dispatch(|event| self.process_system_event(event));
2486                    processed += 1;
2487                }
2488                Some(command) = receivers.system_cmd.recv() => {
2489                    command.dispatch(|command| self.process_system_command(command));
2490                    processed += 1;
2491                }
2492                Some(event) = receivers.exec_evt.recv() => {
2493                    event.dispatch(|event| self.process_exec_event(event));
2494                    processed += 1;
2495                }
2496                Some(command) = receivers.exec_cmd.recv() => {
2497                    self.process_exec_command(command);
2498                    processed += 1;
2499                }
2500                Some(event) = receivers.data_evt.recv() => {
2501                    AsyncRunner::dispatch_data_event(event);
2502                    processed += 1;
2503                }
2504                Some(command) = receivers.data_cmd.recv() => {
2505                    AsyncRunner::handle_data_command(command);
2506                    processed += 1;
2507                }
2508            }
2509        }
2510
2511        processed
2512    }
2513
2514    async fn abort_after_trader_start_failure(
2515        &mut self,
2516        start_err: anyhow::Error,
2517    ) -> anyhow::Result<()> {
2518        log::info!("Trader startup failed, aborting startup");
2519        self.handle.set_shutting_down();
2520        let stop_result = self.kernel.stop_trader_after_start_failure();
2521        let finalize_result = self.finalize_stop().await;
2522
2523        match (stop_result, finalize_result) {
2524            (Ok(()), Ok(())) => Err(start_err),
2525            (Err(stop_err), Ok(())) => anyhow::bail!(
2526                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2527                 {stop_err}"
2528            ),
2529            (Ok(()), Err(finalize_err)) => anyhow::bail!(
2530                "Failed during trader startup: {start_err}; failed to finalize startup abort: \
2531                 {finalize_err}"
2532            ),
2533            (Err(stop_err), Err(finalize_err)) => anyhow::bail!(
2534                "Failed during trader startup: {start_err}; failed to stop partial trader start: \
2535                 {stop_err}; failed to finalize startup abort: {finalize_err}"
2536            ),
2537        }
2538    }
2539
2540    fn initiate_shutdown(&mut self) {
2541        #[cfg(feature = "plugin")]
2542        if let Err(e) = self.plugins.stop_controllers() {
2543            log::error!("Error stopping plug-in controllers: {e}");
2544        }
2545
2546        self.kernel.stop_trader();
2547        let delay = self.kernel.delay_post_stop();
2548        log::info!("Awaiting residual events ({delay:?})...");
2549
2550        self.shutdown_deadline = Some(dst::time::Instant::now() + delay);
2551        self.handle.set_shutting_down();
2552    }
2553
2554    async fn finalize_stop(&mut self) -> anyhow::Result<()> {
2555        // Capture recovery at the grace-period boundary, before disconnection can queue more
2556        // evidence. A later drain must not turn incomplete recovery into a successful shutdown.
2557        let unresolved_submissions = self.exec_manager.unresolved_submission_ids();
2558        self.close_external_ingress();
2559
2560        let timeout = self.config.timeout_disconnection;
2561        let deadline = dst::time::Instant::now() + timeout;
2562
2563        let mut errors = Vec::new();
2564        let (report_result, disconnect_result) = futures_util::join!(
2565            dst::time::timeout(
2566                timeout,
2567                Self::finish_report_tasks(&self.exec_clients, &mut errors)
2568            ),
2569            dst::time::timeout(timeout, self.kernel.disconnect_clients()),
2570        );
2571
2572        if report_result.is_err() {
2573            errors.push("report collection shutdown timeout".to_string());
2574        }
2575
2576        let disconnect_result = match disconnect_result {
2577            Ok(result) => result,
2578            Err(_) => Err(anyhow::anyhow!(
2579                "disconnect timeout while disconnecting clients"
2580            )),
2581        };
2582
2583        if let Err(ref e) = disconnect_result {
2584            log::error!("Error disconnecting clients: {e}");
2585        }
2586
2587        let readiness_result = self.await_engines_disconnected(deadline).await;
2588        let kernel_result = self.kernel.finalize_stop().await;
2589
2590        self.handle.set_stopped();
2591
2592        if !unresolved_submissions.is_empty() {
2593            errors.push(format!(
2594                "Submission recovery incomplete at shutdown: unresolved client order IDs {unresolved_submissions:?}"
2595            ));
2596        }
2597
2598        if let Err(e) = disconnect_result {
2599            errors.push(e.to_string());
2600        }
2601
2602        if let Err(e) = readiness_result {
2603            errors.push(format!("failed while awaiting engine disconnection: {e}"));
2604        }
2605
2606        if let Err(e) = kernel_result {
2607            errors.push(format!("failed while finalizing kernel shutdown: {e}"));
2608        }
2609
2610        if errors.is_empty() {
2611            Ok(())
2612        } else {
2613            anyhow::bail!("{}", errors.join("; "))
2614        }
2615    }
2616
2617    fn drain_channels(
2618        time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
2619        system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
2620        system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
2621        exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
2622        exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
2623            DispatchMessage<TradingCommandMessage>,
2624        >,
2625        data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
2626        data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
2627    ) {
2628        let pending = time_evt_rx.len()
2629            + system_evt_rx.len()
2630            + system_cmd_rx.len()
2631            + exec_evt_rx.len()
2632            + exec_cmd_rx.len()
2633            + data_evt_rx.len()
2634            + data_cmd_rx.len();
2635        log::debug!("Draining shutdown queues ({pending} events initially pending)");
2636
2637        let mut drained = 0;
2638        let log_progress = |processed: usize| {
2639            if processed.is_multiple_of(10_000) {
2640                log::debug!("Processed {processed} events during shutdown drain");
2641            }
2642        };
2643
2644        while let Ok(handler) = time_evt_rx.try_recv() {
2645            let _ = AsyncRunner::handle_time_event(handler);
2646            drained += 1;
2647            log_progress(drained);
2648        }
2649
2650        while system_evt_rx.try_recv().is_ok() {
2651            drained += 1;
2652            log_progress(drained);
2653        }
2654
2655        while system_cmd_rx.try_recv().is_ok() {
2656            drained += 1;
2657            log_progress(drained);
2658        }
2659
2660        while let Ok(evt) = data_evt_rx.try_recv() {
2661            AsyncRunner::dispatch_data_event(evt);
2662            drained += 1;
2663            log_progress(drained);
2664        }
2665
2666        while let Ok(cmd) = data_cmd_rx.try_recv() {
2667            AsyncRunner::handle_data_command(cmd);
2668            drained += 1;
2669            log_progress(drained);
2670        }
2671
2672        while let Ok(evt) = exec_evt_rx.try_recv() {
2673            AsyncRunner::dispatch_exec_event(evt);
2674            drained += 1;
2675            log_progress(drained);
2676        }
2677
2678        while let Ok(cmd) = exec_cmd_rx.try_recv() {
2679            AsyncRunner::handle_trading_command(cmd);
2680            drained += 1;
2681            log_progress(drained);
2682        }
2683
2684        log::debug!("Drained {drained} remaining events during shutdown");
2685    }
2686
2687    fn observe_exec_event_before_dispatch(
2688        &mut self,
2689        evt: &ExecutionEvent,
2690    ) -> Option<Vec<ClientOrderId>> {
2691        let mut order_ids = Vec::new();
2692
2693        match evt {
2694            ExecutionEvent::Order(order_evt) => {
2695                self.exec_manager.observe_order_event(order_evt);
2696                order_ids.push(order_evt.client_order_id());
2697            }
2698            ExecutionEvent::OrderSubmittedBatch(batch) => {
2699                for submitted in &batch.events {
2700                    self.exec_manager
2701                        .record_local_activity(submitted.client_order_id);
2702                }
2703            }
2704            ExecutionEvent::OrderAcceptedBatch(batch) => {
2705                for accepted in &batch.events {
2706                    self.exec_manager
2707                        .observe_order_event(&OrderEventAny::Accepted(*accepted));
2708                    order_ids.push(accepted.client_order_id);
2709                }
2710            }
2711            ExecutionEvent::OrderCanceledBatch(batch) => {
2712                for canceled in &batch.events {
2713                    self.exec_manager
2714                        .observe_order_event(&OrderEventAny::Canceled(*canceled));
2715                    order_ids.push(canceled.client_order_id);
2716                }
2717            }
2718            ExecutionEvent::Report(report) => {
2719                if let ExecutionReport::Fill(fill_report) = report
2720                    && self.exec_manager.is_fill_recently_processed(
2721                        fill_report.account_id,
2722                        fill_report.instrument_id,
2723                        fill_report.trade_id,
2724                    )
2725                {
2726                    log::debug!(
2727                        "Skipping recently processed fill report: {}",
2728                        fill_report.trade_id,
2729                    );
2730                    return None;
2731                }
2732
2733                self.exec_manager.observe_execution_report(report);
2734
2735                if let Some(client_order_id) = self.order_report_client_order_id(report) {
2736                    order_ids.push(client_order_id);
2737                }
2738            }
2739            ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {}
2740        }
2741
2742        Some(order_ids)
2743    }
2744
2745    fn order_report_client_order_id(&self, report: &ExecutionReport) -> Option<ClientOrderId> {
2746        match report {
2747            ExecutionReport::Order(order_report)
2748            | ExecutionReport::OrderWithFills(order_report, _) => {
2749                order_report.client_order_id.or_else(|| {
2750                    self.kernel
2751                        .cache
2752                        .borrow()
2753                        .client_order_id(&order_report.venue_order_id)
2754                        .copied()
2755                })
2756            }
2757            ExecutionReport::Fill(fill_report) => fill_report.client_order_id.or_else(|| {
2758                self.kernel
2759                    .cache
2760                    .borrow()
2761                    .client_order_id(&fill_report.venue_order_id)
2762                    .copied()
2763            }),
2764            _ => None,
2765        }
2766    }
2767
2768    fn observe_exec_command_before_dispatch(&mut self, cmd: &TradingCommand) {
2769        if matches!(
2770            cmd,
2771            TradingCommand::ModifyOrder(_)
2772                | TradingCommand::ModifyOrders(_)
2773                | TradingCommand::CancelOrder(_)
2774                | TradingCommand::CancelOrders(_)
2775        ) {
2776            let engine = self.kernel.exec_engine.borrow();
2777            let no_route = engine.find_client_for_command(cmd).is_none()
2778                && cmd.client_id().is_none_or(|client_id| {
2779                    engine
2780                        .config()
2781                        .external_clients
2782                        .as_ref()
2783                        .is_none_or(|clients| !clients.contains(&client_id))
2784                });
2785
2786            if no_route {
2787                return;
2788            }
2789        }
2790
2791        let submission_client_id = if matches!(
2792            cmd,
2793            TradingCommand::SubmitOrder(_) | TradingCommand::SubmitOrderList(_)
2794        ) {
2795            self.kernel
2796                .exec_engine
2797                .borrow()
2798                .find_client_for_command(cmd)
2799                .map(|client| client.client_id())
2800        } else {
2801            None
2802        };
2803
2804        match cmd {
2805            TradingCommand::SubmitOrder(submit) => {
2806                self.exec_manager.register_submission(
2807                    &submit.order_init,
2808                    submission_client_id.or(submit.client_id),
2809                );
2810            }
2811            TradingCommand::SubmitOrderList(submit) => {
2812                for order_init in &submit.order_inits {
2813                    self.exec_manager
2814                        .register_submission(order_init, submission_client_id.or(submit.client_id));
2815                }
2816            }
2817            TradingCommand::ModifyOrder(modify) => {
2818                self.exec_manager
2819                    .register_inflight(modify.client_order_id, modify.command_id);
2820            }
2821            TradingCommand::ModifyOrders(modify) => {
2822                for child in &modify.modifies {
2823                    self.exec_manager
2824                        .register_inflight(child.client_order_id, child.command_id);
2825                }
2826            }
2827            TradingCommand::CancelOrder(cancel) => {
2828                self.exec_manager
2829                    .register_inflight(cancel.client_order_id, cancel.command_id);
2830            }
2831            TradingCommand::CancelOrders(cancel) => {
2832                for child in &cancel.cancels {
2833                    self.exec_manager
2834                        .register_inflight(child.client_order_id, child.command_id);
2835                }
2836            }
2837            _ => {}
2838        }
2839    }
2840
2841    /// Gets the node's environment.
2842    #[must_use]
2843    pub fn environment(&self) -> Environment {
2844        self.kernel.environment()
2845    }
2846
2847    /// Gets a reference to the underlying kernel.
2848    #[must_use]
2849    pub const fn kernel(&self) -> &NautilusKernel {
2850        &self.kernel
2851    }
2852
2853    /// Gets an exclusive reference to the underlying kernel.
2854    #[must_use]
2855    pub const fn kernel_mut(&mut self) -> &mut NautilusKernel {
2856        &mut self.kernel
2857    }
2858
2859    /// Gets the node's trader ID.
2860    #[must_use]
2861    pub fn trader_id(&self) -> TraderId {
2862        self.kernel.trader_id()
2863    }
2864
2865    /// Gets the node's instance ID.
2866    #[must_use]
2867    pub const fn instance_id(&self) -> UUID4 {
2868        self.kernel.instance_id()
2869    }
2870
2871    /// Returns the current node state.
2872    #[must_use]
2873    pub fn state(&self) -> NodeState {
2874        self.handle.state()
2875    }
2876
2877    /// Checks if the live node is currently running.
2878    #[must_use]
2879    pub fn is_running(&self) -> bool {
2880        self.state().is_running()
2881    }
2882
2883    /// Sets the cache database adapter for persistence.
2884    ///
2885    /// This allows setting a database adapter (e.g., PostgreSQL, Redis) after the node
2886    /// is built but before it starts running. The database adapter is used to persist
2887    /// cache data for recovery and state management.
2888    ///
2889    /// # Errors
2890    ///
2891    /// Returns an error if the node is already running.
2892    pub fn set_cache_database(
2893        &mut self,
2894        database: Box<dyn CacheDatabaseAdapter>,
2895    ) -> anyhow::Result<()> {
2896        if self.state() != NodeState::Idle {
2897            anyhow::bail!(
2898                "Cannot set cache database while node is running, set it before running the node"
2899            );
2900        }
2901
2902        self.kernel.cache().borrow_mut().set_database(database);
2903        Ok(())
2904    }
2905
2906    /// Returns the execution manager.
2907    #[must_use]
2908    pub fn exec_manager(&self) -> &ExecutionManager {
2909        &self.exec_manager
2910    }
2911
2912    /// Returns a mutable reference to the execution manager.
2913    #[must_use]
2914    pub fn exec_manager_mut(&mut self) -> &mut ExecutionManager {
2915        &mut self.exec_manager
2916    }
2917
2918    /// Adds an actor to the trader.
2919    ///
2920    /// This method provides a high-level interface for adding actors to the underlying
2921    /// trader without requiring direct access to the kernel. Actors should be added
2922    /// after the node is built but before starting the node.
2923    ///
2924    /// # Errors
2925    ///
2926    /// Returns an error if:
2927    /// - The trader is not in a valid state for adding components.
2928    /// - An actor with the same ID is already registered.
2929    /// - The node is currently running.
2930    pub fn add_actor<T>(&mut self, actor: T) -> anyhow::Result<()>
2931    where
2932        T: DataActor + DataActorNative + Component + Actor + 'static,
2933    {
2934        if self.state() != NodeState::Idle {
2935            anyhow::bail!(
2936                "Cannot add actor while node is running, add actors before running the node"
2937            );
2938        }
2939
2940        self.kernel.trader.borrow_mut().add_actor(actor)
2941    }
2942
2943    /// Adds an actor to the live node using a factory function.
2944    ///
2945    /// The factory function is called at registration time to create the actor,
2946    /// avoiding cloning issues with non-cloneable actor types.
2947    ///
2948    /// # Errors
2949    ///
2950    /// Returns an error if:
2951    /// - The node is currently running.
2952    /// - The factory function fails to create the actor.
2953    /// - The underlying trader registration fails.
2954    pub fn add_actor_from_factory<F, T>(&mut self, factory: F) -> anyhow::Result<()>
2955    where
2956        F: FnOnce() -> anyhow::Result<T>,
2957        T: DataActor + DataActorNative + Component + Actor + 'static,
2958    {
2959        if self.state() != NodeState::Idle {
2960            anyhow::bail!(
2961                "Cannot add actor while node is running, add actors before running the node"
2962            );
2963        }
2964
2965        self.kernel
2966            .trader
2967            .borrow_mut()
2968            .add_actor_from_factory(factory)
2969    }
2970
2971    /// Adds a strategy to the trader.
2972    ///
2973    /// Strategies are registered in both the component registry (for lifecycle management)
2974    /// and the actor registry (for data callbacks via msgbus).
2975    ///
2976    /// # Errors
2977    ///
2978    /// Returns an error if:
2979    /// - The node is currently running.
2980    /// - A strategy with the same ID is already registered.
2981    /// - The configured external order instrument IDs repeat an instrument or the cache already
2982    ///   contains a requested claim.
2983    /// - The strategy configures one or more external order instrument IDs and the cache is already
2984    ///   borrowed.
2985    /// - The strategy configures an OMS type override and the execution engine is already borrowed.
2986    pub fn add_strategy<T>(&mut self, mut strategy: T) -> anyhow::Result<()>
2987    where
2988        T: Strategy + StrategyNative + DataActorNative + Component + Debug + 'static,
2989    {
2990        if self.state() != NodeState::Idle {
2991            anyhow::bail!(
2992                "Cannot add strategy while node is running, add strategies before running the node"
2993            );
2994        }
2995
2996        // Capture strategy-owned values before adding the strategy, which moves it
2997        let strategy_id = self
2998            .kernel
2999            .trader
3000            .borrow()
3001            .prepare_strategy_for_registration(&mut strategy)?;
3002        let oms_type = StrategyNative::strategy_core(&strategy).config.oms_type;
3003        let instrument_ids = strategy.external_order_instrument_ids().unwrap_or_default();
3004
3005        if !instrument_ids.is_empty() {
3006            self.register_external_order_claims(strategy_id, &instrument_ids)?;
3007        }
3008
3009        let mut exec_engine = match oms_type
3010            .map(|_| {
3011                self.kernel
3012                    .exec_engine
3013                    .try_borrow_mut()
3014                    .map_err(|e| anyhow::anyhow!("Cannot register OMS type: {e}"))
3015            })
3016            .transpose()
3017        {
3018            Ok(exec_engine) => exec_engine,
3019            Err(e) => {
3020                if !instrument_ids.is_empty()
3021                    && let Err(rollback_error) =
3022                        self.rollback_external_order_claims(strategy_id, &instrument_ids)
3023                {
3024                    anyhow::bail!(
3025                        "{e}; failed to roll back external order claims for {strategy_id}: {rollback_error}"
3026                    );
3027                }
3028
3029                return Err(e);
3030            }
3031        };
3032
3033        if let Err(add_error) = self.kernel.trader.borrow_mut().add_strategy(strategy) {
3034            drop(exec_engine);
3035
3036            if !instrument_ids.is_empty()
3037                && let Err(rollback_error) =
3038                    self.rollback_external_order_claims(strategy_id, &instrument_ids)
3039            {
3040                anyhow::bail!(
3041                    "Failed to add strategy {strategy_id}: {add_error}; failed to roll back external order claims: {rollback_error}"
3042                );
3043            }
3044
3045            return Err(add_error);
3046        }
3047
3048        if let Some(exec_engine) = &mut exec_engine
3049            && let Some(oms_type) = oms_type
3050        {
3051            exec_engine.register_oms_type(strategy_id, oms_type);
3052        }
3053
3054        Ok(())
3055    }
3056
3057    /// Registers external order claims in the shared cache.
3058    ///
3059    /// It can be called while the node is idle, after manual [`start`](Self::start) returns, or
3060    /// after the node stops. A running strategy can update its own claims through
3061    /// `Strategy::set_external_order_instrument_ids`.
3062    ///
3063    /// # Errors
3064    ///
3065    /// Returns an error if the cache is already borrowed, the request repeats an instrument, or any
3066    /// requested instrument already has a claim. The cache remains unchanged.
3067    pub fn register_external_order_claims(
3068        &self,
3069        strategy_id: StrategyId,
3070        instrument_ids: &[InstrumentId],
3071    ) -> anyhow::Result<()> {
3072        self.kernel
3073            .cache
3074            .try_borrow_mut()
3075            .map_err(|e| anyhow::anyhow!("Cannot register external order claims: {e}"))?
3076            .register_external_order_claims(strategy_id, instrument_ids)?;
3077
3078        if !instrument_ids.is_empty() {
3079            log::info!("Registered external order claims for {strategy_id}: {instrument_ids:?}");
3080        }
3081
3082        Ok(())
3083    }
3084
3085    /// Deregisters all external order claims owned by `strategy_id` from the shared cache.
3086    ///
3087    /// The operation is synchronous and can be called while the node is idle, after manual
3088    /// [`start`](Self::start) returns, or after the node stops. It cannot be called while
3089    /// [`run`](Self::run) or [`run_with_mode`](Self::run_with_mode) owns the node.
3090    ///
3091    /// # Errors
3092    ///
3093    /// Returns an error if the cache is already borrowed.
3094    pub fn deregister_external_order_claims(&self, strategy_id: StrategyId) -> anyhow::Result<()> {
3095        self.kernel
3096            .cache
3097            .try_borrow_mut()
3098            .map_err(|e| anyhow::anyhow!("Cannot deregister external order claims: {e}"))?
3099            .set_external_order_claims(strategy_id, &[])?;
3100
3101        Ok(())
3102    }
3103
3104    pub(crate) fn rollback_external_order_claims(
3105        &self,
3106        strategy_id: StrategyId,
3107        instrument_ids: &[InstrumentId],
3108    ) -> anyhow::Result<()> {
3109        let mut cache = self
3110            .kernel
3111            .cache
3112            .try_borrow_mut()
3113            .map_err(|e| anyhow::anyhow!("Cannot roll back external order claims: {e}"))?;
3114        let retained: Vec<_> = cache
3115            .external_order_claim_instrument_ids(Some(strategy_id))
3116            .into_iter()
3117            .filter(|instrument_id| !instrument_ids.contains(instrument_id))
3118            .collect();
3119        cache.set_external_order_claims(strategy_id, &retained)
3120    }
3121
3122    /// Adds an execution algorithm to the trader.
3123    ///
3124    /// Execution algorithms are registered in both the component registry (for lifecycle
3125    /// management) and the actor registry (for data callbacks via msgbus).
3126    ///
3127    /// # Errors
3128    ///
3129    /// Returns an error if:
3130    /// - The node is currently running.
3131    /// - An execution algorithm with the same ID is already registered.
3132    pub fn add_exec_algorithm<T>(&mut self, exec_algorithm: T) -> anyhow::Result<()>
3133    where
3134        T: ExecutionAlgorithm + ExecutionAlgorithmNative + Component + Debug + 'static,
3135    {
3136        if self.state() != NodeState::Idle {
3137            anyhow::bail!(
3138                "Cannot add exec algorithm while node is running, add exec algorithms before running the node"
3139            );
3140        }
3141
3142        self.kernel
3143            .trader
3144            .borrow_mut()
3145            .add_exec_algorithm(exec_algorithm)
3146    }
3147
3148    // Runs reconciliation sub-checks, each gated by its own interval.
3149    // Continuous checks only schedule venue work; they do not await venue I/O
3150    // in the event loop.
3151}
3152
3153#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3154enum SocketReconnectDispatchOutcome {
3155    Accepted,
3156    AlreadyReconnecting,
3157    Disconnected,
3158    Closed,
3159    Unsupported,
3160    InvalidTrader,
3161    UnknownClient,
3162    UnknownEndpoint,
3163    AmbiguousEndpoint,
3164}
3165
3166fn record_runner_dispatch(
3167    metrics: &RunnerMetrics,
3168    channel: SystemChannel,
3169    dispatch_start: dst::time::Instant,
3170    metrics_start: dst::time::Instant,
3171) {
3172    let dispatch_end = dst::time::Instant::now();
3173    metrics.record_dispatch(
3174        channel,
3175        dispatch_end.duration_since(dispatch_start),
3176        dispatch_end.duration_since(metrics_start),
3177    );
3178}
3179
3180fn record_runner_maintenance(
3181    metrics: &RunnerMetrics,
3182    work_start: dst::time::Instant,
3183    metrics_start: dst::time::Instant,
3184) {
3185    let work_end = dst::time::Instant::now();
3186    metrics.record_maintenance(
3187        work_end.duration_since(work_start),
3188        work_end.duration_since(metrics_start),
3189    );
3190}
3191
3192fn record_runner_external_msgbus(
3193    metrics: &RunnerMetrics,
3194    work_start: dst::time::Instant,
3195    metrics_start: dst::time::Instant,
3196) {
3197    let work_end = dst::time::Instant::now();
3198    metrics.record_external_msgbus(
3199        work_end.duration_since(work_start),
3200        work_end.duration_since(metrics_start),
3201    );
3202}
3203
3204async fn drive_with_external_ingress<F, T>(
3205    future: F,
3206    receiver: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3207    pending: &mut VecDeque<BusMessage>,
3208) -> T
3209where
3210    F: std::future::Future<Output = T>,
3211{
3212    tokio::pin!(future);
3213    let capacity = receiver
3214        .as_ref()
3215        .map_or(0, tokio::sync::mpsc::Receiver::max_capacity);
3216
3217    loop {
3218        tokio::select! {
3219            biased;
3220
3221            result = &mut future => return result,
3222            message = async { match receiver.as_mut() { Some(receiver) => receiver.recv().await, None => std::future::pending().await } }, if pending.len() < capacity => {
3223                let Some(message) = message else {
3224                    log::info!("External message bus ingress closed");
3225                    *receiver = None;
3226                    continue;
3227                };
3228
3229                match msgbus::republish_pending_external_response(&message) {
3230                    Ok(true) => {},
3231                    Ok(false) | Err(_) => pending.push_back(message),
3232                }
3233            }
3234        }
3235    }
3236}
3237
3238async fn recv_external_msgbus_message(
3239    rx: &mut Option<tokio::sync::mpsc::Receiver<BusMessage>>,
3240    pending: &mut VecDeque<BusMessage>,
3241) -> Option<BusMessage> {
3242    if let Some(message) = pending.pop_front() {
3243        return Some(message);
3244    }
3245
3246    match rx {
3247        Some(rx) => rx.recv().await,
3248        None => std::future::pending::<Option<BusMessage>>().await,
3249    }
3250}
3251
3252struct RunnerReceivers<'a> {
3253    time_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3254    system_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3255    system_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3256    exec_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3257    exec_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3258    data_evt: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3259    data_cmd: &'a mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3260}
3261
3262/// Flushes data events and commands from both `pending` and the channel receivers
3263/// into the cache, looping until no progress is made.
3264///
3265/// This closes the gap where `drive_with_event_buffering` exits as soon as its
3266/// driven future resolves (biased select), leaving items in the channel receivers
3267/// that were not captured into `pending`.
3268fn flush_pending_data(
3269    pending: &mut PendingEvents,
3270    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3271    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3272) {
3273    loop {
3274        let mut progressed = pending.drain_data();
3275
3276        while let Ok(evt) = data_evt_rx.try_recv() {
3277            AsyncRunner::dispatch_data_event(evt);
3278            progressed = true;
3279        }
3280
3281        while let Ok(cmd) = data_cmd_rx.try_recv() {
3282            AsyncRunner::handle_data_command(cmd);
3283            progressed = true;
3284        }
3285
3286        if !progressed {
3287            break;
3288        }
3289    }
3290}
3291
3292/// Flushes all channel receivers into `pending`, then drains everything.
3293///
3294/// Unlike [`flush_pending_data`] this is a single pass, not a drain-until-quiet
3295/// loop. Sufficient for phase 2 where the goal is to capture items the biased
3296/// select did not poll before the connect future resolved.
3297#[expect(
3298    clippy::too_many_arguments,
3299    reason = "all runner receivers are drained together"
3300)]
3301fn flush_all_pending(
3302    pending: &mut PendingEvents,
3303    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3304    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3305    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3306    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3307    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3308    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3309    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3310) {
3311    // Flush channel receivers into pending
3312    while let Ok(handler) = time_evt_rx.try_recv() {
3313        let _ = AsyncRunner::handle_time_event(handler);
3314    }
3315
3316    while let Ok(event) = system_evt_rx.try_recv() {
3317        pending.system_events.push(event);
3318    }
3319
3320    while let Ok(command) = system_cmd_rx.try_recv() {
3321        pending.system_commands.push(command);
3322    }
3323
3324    while let Ok(evt) = data_evt_rx.try_recv() {
3325        pending.data_evts.push(evt);
3326    }
3327
3328    while let Ok(cmd) = data_cmd_rx.try_recv() {
3329        pending.data_cmds.push(cmd);
3330    }
3331
3332    while let Ok(evt) = exec_evt_rx.try_recv() {
3333        pending.push_exec_event(evt);
3334    }
3335
3336    while let Ok(cmd) = exec_cmd_rx.try_recv() {
3337        pending.exec_cmds.push(cmd);
3338    }
3339
3340    pending.drain();
3341}
3342
3343/// Drives a future to completion while buffering channel events.
3344///
3345/// Time events are handled immediately. Account events are forwarded directly.
3346/// All other events are buffered in `pending` for later processing.
3347#[expect(
3348    clippy::too_many_arguments,
3349    reason = "startup buffering owns one future plus the pending state and all runner receivers"
3350)]
3351async fn drive_with_event_buffering<F: std::future::Future>(
3352    future: F,
3353    pending: &mut PendingEvents,
3354    time_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TimeEventMessage>>,
3355    system_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemEvent>>,
3356    system_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<SystemCommand>>,
3357    exec_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<ExecutionEvent>>,
3358    exec_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<TradingCommandMessage>>,
3359    data_evt_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataEvent>>,
3360    data_cmd_rx: &mut tokio::sync::mpsc::UnboundedReceiver<DispatchMessage<DataCommand>>,
3361) -> F::Output {
3362    tokio::pin!(future);
3363
3364    loop {
3365        tokio::select! {
3366            biased;
3367
3368            result = &mut future => {
3369                break result;
3370            }
3371            Some(handler) = time_evt_rx.recv() => {
3372                let _ = AsyncRunner::handle_time_event(handler);
3373            }
3374            Some(event) = system_evt_rx.recv() => {
3375                pending.system_events.push(event);
3376            }
3377            Some(command) = system_cmd_rx.recv() => {
3378                pending.system_commands.push(command);
3379            }
3380            Some(evt) = exec_evt_rx.recv() => {
3381                pending.push_exec_event(evt);
3382            }
3383            Some(cmd) = exec_cmd_rx.recv() => {
3384                pending.exec_cmds.push(cmd);
3385            }
3386            Some(evt) = data_evt_rx.recv() => {
3387                pending.data_evts.push(evt);
3388            }
3389            Some(cmd) = data_cmd_rx.recv() => {
3390                pending.data_cmds.push(cmd);
3391            }
3392        }
3393    }
3394}
3395
3396#[derive(Default)]
3397struct PendingEvents {
3398    system_events: Vec<DispatchMessage<SystemEvent>>,
3399    system_commands: Vec<DispatchMessage<SystemCommand>>,
3400    data_evts: Vec<DispatchMessage<DataEvent>>,
3401    data_cmds: Vec<DispatchMessage<DataCommand>>,
3402    exec_reports: Vec<DispatchMessage<ExecutionReport>>,
3403    order_evts: Vec<DispatchMessage<OrderEventAny>>,
3404    exec_cmds: Vec<DispatchMessage<TradingCommandMessage>>,
3405}
3406
3407impl PendingEvents {
3408    fn is_empty(&self) -> bool {
3409        self.system_events.is_empty()
3410            && self.system_commands.is_empty()
3411            && self.data_evts.is_empty()
3412            && self.data_cmds.is_empty()
3413            && self.exec_reports.is_empty()
3414            && self.order_evts.is_empty()
3415            && self.exec_cmds.is_empty()
3416    }
3417
3418    /// Drains only data events and commands into the cache.
3419    ///
3420    /// Returns `true` if any events or commands were drained.
3421    fn drain_data(&mut self) -> bool {
3422        let total = self.data_evts.len() + self.data_cmds.len();
3423
3424        if total > 0 {
3425            log::debug!(
3426                "Draining {total} data events/commands into cache \
3427                 (data_evts={}, data_cmds={})",
3428                self.data_evts.len(),
3429                self.data_cmds.len(),
3430            );
3431        }
3432
3433        for evt in self.data_evts.drain(..) {
3434            AsyncRunner::dispatch_data_event(evt);
3435        }
3436
3437        for cmd in self.data_cmds.drain(..) {
3438            AsyncRunner::handle_data_command(cmd);
3439        }
3440
3441        total > 0
3442    }
3443
3444    /// Drains all remaining pending events.
3445    fn drain(&mut self) {
3446        let total = self.data_evts.len()
3447            + self.data_cmds.len()
3448            + self.exec_reports.len()
3449            + self.order_evts.len()
3450            + self.exec_cmds.len();
3451
3452        if total > 0 {
3453            log::debug!(
3454                "Processing {total} events/commands queued during startup \
3455                 (data_evts={}, data_cmds={}, exec_reports={}, order_evts={}, exec_cmds={})",
3456                self.data_evts.len(),
3457                self.data_cmds.len(),
3458                self.exec_reports.len(),
3459                self.order_evts.len(),
3460                self.exec_cmds.len()
3461            );
3462        }
3463
3464        for evt in self.data_evts.drain(..) {
3465            AsyncRunner::dispatch_data_event(evt);
3466        }
3467
3468        for cmd in self.data_cmds.drain(..) {
3469            AsyncRunner::handle_data_command(cmd);
3470        }
3471
3472        for report in self.exec_reports.drain(..) {
3473            report
3474                .dispatch(|report| AsyncRunner::handle_exec_event(ExecutionEvent::Report(report)));
3475        }
3476
3477        for evt in self.order_evts.drain(..) {
3478            evt.dispatch(|evt| AsyncRunner::handle_exec_event(ExecutionEvent::Order(evt)));
3479        }
3480
3481        for cmd in self.exec_cmds.drain(..) {
3482            AsyncRunner::handle_trading_command(cmd);
3483        }
3484    }
3485
3486    fn push_exec_event(&mut self, event: DispatchMessage<ExecutionEvent>) {
3487        let rooted = event.is_rooted();
3488        event.dispatch(|event| {
3489            let owner = std::thread::current().id();
3490
3491            let order = |event| {
3492                if rooted {
3493                    DispatchMessage::new(event, owner)
3494                } else {
3495                    DispatchMessage::from(event)
3496                }
3497            };
3498
3499            // Account events are safe to process immediately. Reports and orders need
3500            // ExecEngine access, which may conflict with the driven startup future's borrow.
3501            match event {
3502                ExecutionEvent::Account(_) | ExecutionEvent::Data(_) => {
3503                    AsyncRunner::handle_exec_event(event);
3504                }
3505                ExecutionEvent::Report(report) => self.exec_reports.push(if rooted {
3506                    DispatchMessage::new(report, owner)
3507                } else {
3508                    report.into()
3509                }),
3510                ExecutionEvent::Order(event) => self.order_evts.push(order(event)),
3511                ExecutionEvent::OrderSubmittedBatch(batch) => {
3512                    for event in batch {
3513                        self.order_evts.push(order(OrderEventAny::Submitted(event)));
3514                    }
3515                }
3516                ExecutionEvent::OrderAcceptedBatch(batch) => {
3517                    for event in batch {
3518                        self.order_evts.push(order(OrderEventAny::Accepted(event)));
3519                    }
3520                }
3521                ExecutionEvent::OrderCanceledBatch(batch) => {
3522                    for event in batch {
3523                        self.order_evts.push(order(OrderEventAny::Canceled(event)));
3524                    }
3525                }
3526            }
3527        });
3528    }
3529
3530    fn take_system_events(&mut self) -> Vec<DispatchMessage<SystemEvent>> {
3531        std::mem::take(&mut self.system_events)
3532    }
3533
3534    fn take_system_commands(&mut self) -> Vec<DispatchMessage<SystemCommand>> {
3535        std::mem::take(&mut self.system_commands)
3536    }
3537}
3538
3539struct ClientStatus {
3540    client: String,
3541    client_type: &'static str,
3542    connected: bool,
3543}
3544
3545fn render_client_statuses(rows: Vec<ClientStatus>) -> String {
3546    let mut builder = Builder::with_capacity(rows.len() + 1, 3);
3547    builder.push_record(["Client", "Type", "Connected"]);
3548
3549    for row in rows {
3550        builder.push_record([
3551            row.client,
3552            row.client_type.to_string(),
3553            row.connected.to_string(),
3554        ]);
3555    }
3556
3557    builder.build().with(Style::rounded()).to_string()
3558}
3559
3560#[cfg(test)]
3561mod tests {
3562    mod report_acceptance;
3563    mod shutdown;
3564
3565    use std::{
3566        cell::{Cell, RefCell},
3567        fmt::Debug,
3568        rc::Rc,
3569        sync::{
3570            Arc,
3571            atomic::{AtomicBool, Ordering},
3572        },
3573    };
3574
3575    use bytes::Bytes;
3576    use indexmap::{IndexMap, IndexSet};
3577    use log::{Level, LevelFilter, Log, Metadata, Record};
3578    #[cfg(feature = "python")]
3579    use nautilus_common::runner::{
3580        SyncDataCommandSender, replace_data_cmd_sender, replace_exec_cmd_sender,
3581    };
3582    use nautilus_common::{
3583        actor::{
3584            self, CallbackDispatchError, DataActor, DataActorCore, data_actor::DataActorConfig,
3585        },
3586        cache::Cache,
3587        clock::{Clock, VirtualClock},
3588        enums::SerializationEncoding,
3589        live::{
3590            runner::{get_data_event_sender, get_exec_event_sender, get_system_event_sender},
3591            sender::DispatchSender,
3592        },
3593        logging::{logger::LoggerConfig, logging_sync_to_disk, writer::FileWriterConfig},
3594        messages::{
3595            data::{SubscribeCommand, SubscribeQuotes},
3596            execution::{
3597                GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
3598                GeneratePositionStatusReports, QueryAccount, SubmitOrder, SubmitOrderList,
3599                TradingCommand,
3600            },
3601            system::{
3602                QueueCondition, QueueState, ReconnectSocket, SocketState, SocketStateChanged,
3603            },
3604        },
3605        msgbus::{
3606            self, BusMessage, BusPayloadType, MessageBusBacking, MessageBusBackingFactory,
3607            MessageBusConfig, MessageBusExternalEgress, MessageBusExternalIngress,
3608            MessagingSwitchboard, ShareableMessageHandler, TypedHandler, TypedIntoHandler,
3609        },
3610        nautilus_actor,
3611        runner::{SyncTradingCommandSender, TradingCommandSender},
3612        testing::wait_until_async,
3613        timer::{TimeEvent, TimeEventCallback},
3614    };
3615    use nautilus_core::{Params, UUID4, UnixNanos};
3616    use nautilus_execution::{
3617        engine::{ExecutionEngine, SnapshotAnchorer, stubs::StubExecutionClient},
3618        reconciliation::{RECONCILIATION_ORDER_TAG, create_inferred_fill_for_qty},
3619    };
3620    use nautilus_model::{
3621        accounts::{AccountAny, MarginAccount},
3622        data::QuoteTick,
3623        enums::{
3624            AccountType, LiquiditySide, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
3625            TimeInForce,
3626        },
3627        events::{
3628            AccountState, OrderAccepted, OrderAcceptedBatch, OrderFilled, OrderUpdated,
3629            order::spec::{
3630                OrderAcceptedSpec, OrderPendingCancelSpec, OrderPendingUpdateSpec, OrderUpdatedSpec,
3631            },
3632        },
3633        identifiers::{
3634            AccountId, ActorId, ClientId, InstrumentId, OrderListId, PositionId, StrategyId,
3635            TradeId, TraderId, Venue, VenueOrderId,
3636        },
3637        instruments::{
3638            Instrument, InstrumentAny,
3639            stubs::{crypto_perpetual_ethusdt, currency_pair_btcusdt},
3640        },
3641        orders::{
3642            Order, OrderAny, OrderList, OrderTestBuilder,
3643            stubs::{OrderFilledTestBuilder, TestOrderEventStubs},
3644        },
3645        position::PositionReplayEvent,
3646        reports::{FillReport, PositionStatusReport},
3647        types::{AccountBalance, Currency, MarginBalance, Money, Price, Quantity},
3648    };
3649    use nautilus_system::{KernelEventStore, RegisteredComponents, event_store::EventStoreConfig};
3650    use nautilus_testkit::{
3651        cache::TestCacheDatabaseControl,
3652        components::{StateActor, StateStrategy},
3653    };
3654    use nautilus_trading::{
3655        nautilus_strategy,
3656        strategy::{config::StrategyConfig, core::StrategyCore},
3657    };
3658    use parking_lot::Mutex;
3659    use rstest::*;
3660    use rust_decimal::Decimal;
3661    use rust_decimal_macros::dec;
3662    use ustr::Ustr;
3663
3664    use super::{
3665        reconciliation::{
3666            PositionFillReportResult, PositionReportResult, reconciliation_check_due,
3667            request_position_fill_reports,
3668        },
3669        *,
3670    };
3671    use crate::{
3672        execution::{
3673            client::LiveExecutionClient,
3674            manager::{PositionFillReportQuery, ReportClientCoverage, SourcedOrderStatusReport},
3675            submission::{
3676                SubmissionRecoveryExhausted, SubmissionRecoveryPolicy, SubmissionRecoverySource,
3677            },
3678        },
3679        socket::SocketControl,
3680    };
3681
3682    struct ExternalIngressLogCapture {
3683        messages: Mutex<Vec<String>>,
3684    }
3685
3686    static EXTERNAL_INGRESS_LOG_CAPTURE: ExternalIngressLogCapture = ExternalIngressLogCapture {
3687        messages: Mutex::new(Vec::new()),
3688    };
3689
3690    #[derive(Debug)]
3691    enum FillReportClientOutcome {
3692        Reports(Vec<FillReport>),
3693        Failure,
3694    }
3695
3696    #[derive(Debug)]
3697    struct FillReportClient {
3698        client_id: ClientId,
3699        account_id: AccountId,
3700        venue: Venue,
3701        outcome: FillReportClientOutcome,
3702        commands: Rc<RefCell<Vec<GenerateFillReports>>>,
3703    }
3704
3705    #[async_trait::async_trait(?Send)]
3706    impl ExecutionClient for FillReportClient {
3707        fn is_connected(&self) -> bool {
3708            true
3709        }
3710
3711        fn client_id(&self) -> ClientId {
3712            self.client_id
3713        }
3714
3715        fn account_id(&self) -> AccountId {
3716            self.account_id
3717        }
3718
3719        fn venue(&self) -> Venue {
3720            self.venue
3721        }
3722
3723        fn oms_type(&self) -> OmsType {
3724            OmsType::Netting
3725        }
3726
3727        fn get_account(&self) -> Option<AccountAny> {
3728            None
3729        }
3730
3731        fn generate_account_state(
3732            &self,
3733            _balances: Vec<AccountBalance>,
3734            _margins: Vec<MarginBalance>,
3735            _reported: bool,
3736            _ts_event: UnixNanos,
3737            _info: Option<Params>,
3738        ) -> anyhow::Result<()> {
3739            Ok(())
3740        }
3741
3742        fn start(&mut self) -> anyhow::Result<()> {
3743            Ok(())
3744        }
3745
3746        fn stop(&mut self) -> anyhow::Result<()> {
3747            Ok(())
3748        }
3749
3750        async fn generate_order_status_reports(
3751            &self,
3752            _cmd: &GenerateOrderStatusReports,
3753        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3754            match self.outcome {
3755                FillReportClientOutcome::Failure => anyhow::bail!("order reports unavailable"),
3756                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3757            }
3758        }
3759
3760        async fn generate_position_status_reports(
3761            &self,
3762            _cmd: &GeneratePositionStatusReports,
3763        ) -> anyhow::Result<Vec<PositionStatusReport>> {
3764            match self.outcome {
3765                FillReportClientOutcome::Failure => anyhow::bail!("position reports unavailable"),
3766                FillReportClientOutcome::Reports(_) => Ok(Vec::new()),
3767            }
3768        }
3769
3770        async fn generate_fill_reports(
3771            &self,
3772            cmd: GenerateFillReports,
3773        ) -> anyhow::Result<Vec<FillReport>> {
3774            self.commands.borrow_mut().push(cmd);
3775
3776            match &self.outcome {
3777                FillReportClientOutcome::Reports(reports) => Ok(reports.clone()),
3778                FillReportClientOutcome::Failure => anyhow::bail!("fill reports unavailable"),
3779            }
3780        }
3781    }
3782
3783    #[derive(Debug)]
3784    struct OrderReportClient {
3785        client_id: ClientId,
3786        report: OrderStatusReport,
3787        targeted: bool,
3788        queries: Rc<RefCell<Vec<&'static str>>>,
3789    }
3790
3791    #[async_trait::async_trait(?Send)]
3792    impl ExecutionClient for OrderReportClient {
3793        fn is_connected(&self) -> bool {
3794            true
3795        }
3796
3797        fn client_id(&self) -> ClientId {
3798            self.client_id
3799        }
3800
3801        fn account_id(&self) -> AccountId {
3802            self.report.account_id
3803        }
3804
3805        fn venue(&self) -> Venue {
3806            self.report.instrument_id.venue
3807        }
3808
3809        fn oms_type(&self) -> OmsType {
3810            OmsType::Netting
3811        }
3812
3813        fn get_account(&self) -> Option<AccountAny> {
3814            None
3815        }
3816
3817        fn generate_account_state(
3818            &self,
3819            _balances: Vec<AccountBalance>,
3820            _margins: Vec<MarginBalance>,
3821            _reported: bool,
3822            _ts_event: UnixNanos,
3823            _info: Option<Params>,
3824        ) -> anyhow::Result<()> {
3825            Ok(())
3826        }
3827
3828        fn start(&mut self) -> anyhow::Result<()> {
3829            Ok(())
3830        }
3831
3832        fn stop(&mut self) -> anyhow::Result<()> {
3833            Ok(())
3834        }
3835
3836        async fn generate_order_status_reports(
3837            &self,
3838            _command: &GenerateOrderStatusReports,
3839        ) -> anyhow::Result<Vec<OrderStatusReport>> {
3840            self.queries.borrow_mut().push("bulk");
3841            Ok(if self.targeted {
3842                Vec::new()
3843            } else {
3844                vec![self.report.clone()]
3845            })
3846        }
3847
3848        async fn generate_order_status_report(
3849            &self,
3850            _command: &GenerateOrderStatusReport,
3851        ) -> anyhow::Result<Option<OrderStatusReport>> {
3852            self.queries.borrow_mut().push("targeted");
3853            Ok(Some(self.report.clone()))
3854        }
3855    }
3856
3857    #[derive(Debug)]
3858    struct StartupSocketActor {
3859        core: DataActorCore,
3860        received: Rc<RefCell<Vec<SocketStateChanged>>>,
3861    }
3862
3863    impl StartupSocketActor {
3864        fn new(received: Rc<RefCell<Vec<SocketStateChanged>>>) -> Self {
3865            Self {
3866                core: DataActorCore::new(DataActorConfig {
3867                    actor_id: Some(ActorId::from("SOCKET-STARTUP-ACTOR")),
3868                    ..Default::default()
3869                }),
3870                received,
3871            }
3872        }
3873    }
3874
3875    impl DataActor for StartupSocketActor {
3876        fn on_start(&mut self) -> anyhow::Result<()> {
3877            self.subscribe_socket_state(None, None, None);
3878            Ok(())
3879        }
3880
3881        fn on_socket_state(&mut self, event: &SocketStateChanged) -> anyhow::Result<()> {
3882            self.received.borrow_mut().push(event.clone());
3883            Ok(())
3884        }
3885    }
3886
3887    nautilus_actor!(StartupSocketActor);
3888
3889    impl Log for ExternalIngressLogCapture {
3890        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
3891            metadata.level() == Level::Error && metadata.target() == "nautilus_live::node"
3892        }
3893
3894        fn log(&self, record: &Record<'_>) {
3895            if self.enabled(record.metadata()) {
3896                self.messages.lock().push(record.args().to_string());
3897            }
3898        }
3899
3900        fn flush(&self) {}
3901    }
3902
3903    #[derive(Debug)]
3904    struct FailingTimerActor {
3905        core: DataActorCore,
3906        received: Rc<RefCell<Vec<u64>>>,
3907    }
3908
3909    nautilus_actor!(FailingTimerActor);
3910
3911    impl DataActor for FailingTimerActor {
3912        fn on_start(&mut self) -> anyhow::Result<()> {
3913            for timestamp in [17, 23] {
3914                let received = self.received.clone();
3915
3916                let callback = TimeEventCallback::RustLocal(Rc::new(move |_| {
3917                    received.borrow_mut().push(timestamp);
3918
3919                    if timestamp == 17 {
3920                        crate::dispatch::tests::latch_callback_failure();
3921                    }
3922                }));
3923
3924                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
3925                    TimeEvent::new(
3926                        "callback-failure".into(),
3927                        UUID4::new(),
3928                        timestamp.into(),
3929                        timestamp.into(),
3930                    ),
3931                    callback,
3932                ));
3933            }
3934
3935            Ok(())
3936        }
3937    }
3938
3939    #[rstest]
3940    #[tokio::test]
3941    async fn test_callback_failure_stops_later_live_events(
3942        #[values(false, true)] incomplete_submission: bool,
3943    ) {
3944        actor::clear_callbacks().unwrap();
3945
3946        let config = LiveNodeConfig {
3947            exec_engine: crate::config::LiveExecutionEngineConfig {
3948                reconciliation: false,
3949                submission_recovery_policy: SubmissionRecoveryPolicy::RetainUnresolved,
3950                ..Default::default()
3951            },
3952            timeout_connection: Duration::ZERO,
3953            timeout_reconciliation: Duration::ZERO,
3954            timeout_portfolio: Duration::ZERO,
3955            timeout_disconnection: Duration::ZERO,
3956            delay_post_stop: Duration::ZERO,
3957            timeout_shutdown: Duration::ZERO,
3958            ..Default::default()
3959        };
3960
3961        let mut node = LiveNode::build("CallbackFailureNode".to_string(), Some(config)).unwrap();
3962
3963        if incomplete_submission {
3964            let order = OrderTestBuilder::new(OrderType::Market)
3965                .client_order_id(ClientOrderId::from("O-CALLBACK-UNRESOLVED"))
3966                .instrument_id(InstrumentId::from("EUR/USD.SIM"))
3967                .quantity(Quantity::from("1"))
3968                .build();
3969            node.exec_manager
3970                .register_submission(order.init_event(), None);
3971        }
3972        let received = Rc::new(RefCell::new(Vec::new()));
3973        node.add_actor(FailingTimerActor {
3974            core: DataActorCore::new(DataActorConfig {
3975                actor_id: Some(ActorId::from("CALLBACK-FAILURE")),
3976                ..Default::default()
3977            }),
3978            received: received.clone(),
3979        })
3980        .unwrap();
3981
3982        let result = node.run_with_mode(NodeRunMode::Hosted).await;
3983
3984        let error = result.unwrap_err();
3985        assert!(error.to_string().contains("Callback delivery unwound"));
3986        if incomplete_submission {
3987            assert!(error.to_string().contains("O-CALLBACK-UNRESOLVED"));
3988        }
3989        let state = node.state();
3990        let trader_stopped = node.kernel.trader.borrow().is_stopped();
3991        let failure = actor::callback_failure();
3992        node.dispose();
3993
3994        assert_eq!(
3995            error.downcast_ref::<CallbackDispatchError>(),
3996            Some(&CallbackDispatchError::DeliveryUnwound)
3997        );
3998        assert_eq!(*received.borrow(), [17]);
3999        assert_eq!(state, NodeState::Stopped);
4000        assert!(trader_stopped);
4001        assert_eq!(failure, Some(CallbackDispatchError::DeliveryUnwound));
4002        assert_eq!(actor::callback_failure(), None);
4003        assert_eq!(actor::clear_callbacks(), Ok(()));
4004    }
4005
4006    #[rstest]
4007    fn test_render_client_statuses() {
4008        let rows = vec![
4009            ClientStatus {
4010                client: "BINANCE".to_string(),
4011                client_type: "Data",
4012                connected: true,
4013            },
4014            ClientStatus {
4015                client: "SIM".to_string(),
4016                client_type: "Execution",
4017                connected: false,
4018            },
4019        ];
4020
4021        let output = render_client_statuses(rows);
4022        let expected = "╭─────────┬───────────┬───────────╮\n\
4023│ Client  │ Type      │ Connected │\n\
4024├─────────┼───────────┼───────────┤\n\
4025│ BINANCE │ Data      │ true      │\n\
4026│ SIM     │ Execution │ false     │\n\
4027╰─────────┴───────────┴───────────╯";
4028
4029        assert_eq!(output, expected);
4030    }
4031
4032    #[rstest]
4033    fn test_republish_external_msgbus_message_logs_topic_and_error_chain() {
4034        log::set_logger(&EXTERNAL_INGRESS_LOG_CAPTURE).expect("test logger already installed");
4035        log::set_max_level(LevelFilter::Error);
4036        EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock().clear();
4037        let message = BusMessage::with_str_topic(
4038            "data.quotes.AUDUSD.SIM*",
4039            BusPayloadType::Custom(Ustr::from("UnregisteredCustomData")),
4040            Bytes::new(),
4041            SerializationEncoding::Json,
4042        );
4043
4044        LiveNode::republish_external_msgbus_message(&message);
4045
4046        assert_eq!(
4047            *EXTERNAL_INGRESS_LOG_CAPTURE.messages.lock(),
4048            vec![
4049                "Failed to republish external message bus topic 'data.quotes.AUDUSD.SIM*': invalid \
4050                 external message topic: Topic `value` contained invalid characters, was \
4051                 data.quotes.AUDUSD.SIM*"
4052                    .to_string()
4053            ],
4054        );
4055    }
4056
4057    #[rstest]
4058    #[tokio::test]
4059    async fn test_external_startup_prioritizes_completed_connection() {
4060        let message = BusMessage::with_str_topic(
4061            "data.trades.FIRST",
4062            BusPayloadType::TradeTick,
4063            bytes::Bytes::new(),
4064            SerializationEncoding::Json,
4065        );
4066        let (sender, receiver) = tokio::sync::mpsc::channel(1);
4067        sender.send(message.clone()).await.unwrap();
4068        let mut receiver = Some(receiver);
4069        let mut pending = VecDeque::new();
4070
4071        let result =
4072            drive_with_external_ingress(std::future::ready(17), &mut receiver, &mut pending).await;
4073        let queued = receiver.as_mut().unwrap().try_recv().unwrap();
4074
4075        assert_eq!(result, 17);
4076        assert!(pending.is_empty());
4077        assert_eq!(
4078            serde_json::to_value(queued).unwrap(),
4079            serde_json::to_value(message).unwrap()
4080        );
4081    }
4082
4083    #[rstest]
4084    #[tokio::test]
4085    async fn test_external_startup_dispatches_correlated_response_before_buffered_traffic() {
4086        use bytes::Bytes;
4087        use nautilus_common::messages::data::{
4088            CustomDataResponse, DataResponse, DataResponseEnvelope,
4089        };
4090        use nautilus_model::data::DataType;
4091
4092        let (egress, publications, _) = CapturingExternalEgress::new();
4093        let bus = Rc::new(RefCell::new(msgbus::MessageBus::default()));
4094        msgbus::set_message_bus(bus.clone());
4095        bus.borrow_mut()
4096            .set_external_egress_config(
4097                Box::new(egress),
4098                &MessageBusConfig {
4099                    encoding: SerializationEncoding::MsgPack,
4100                    external_streams: Some(vec!["external.test".into()]),
4101                    ..Default::default()
4102                },
4103            )
4104            .unwrap();
4105
4106        bus.borrow_mut()
4107            .add_streaming_type(BusPayloadType::DataResponse);
4108        let id = UUID4::new();
4109        let response = DataResponse::Data(CustomDataResponse::new(
4110            id,
4111            "FEED".into(),
4112            None,
4113            DataType::new("Startup", None, None),
4114            Bytes::from_static(b"connected"),
4115            Some(101.into()),
4116            Some(103.into()),
4117            107.into(),
4118            None,
4119        ));
4120        msgbus::publish_any("responses.data.FEED".into(), &response);
4121        let message = BusMessage::with_str_topic(
4122            "responses.data.FEED",
4123            BusPayloadType::DataResponse,
4124            publications.borrow()[0].payload.clone(),
4125            SerializationEncoding::MsgPack,
4126        );
4127        let (done, connected) = tokio::sync::oneshot::channel();
4128        let done = Rc::new(RefCell::new(Some(done)));
4129        msgbus::register_response_handler(
4130            &id,
4131            TypedHandler::from(move |envelope: &DataResponseEnvelope| {
4132                let Some(DataResponse::Data(reply)) = envelope.response() else {
4133                    panic!("expected custom reply");
4134                };
4135
4136                done.borrow_mut()
4137                    .take()
4138                    .unwrap()
4139                    .send(reply.clone())
4140                    .unwrap();
4141            }),
4142        );
4143
4144        msgbus::subscribe_any(
4145            message.topic.as_str().into(),
4146            ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
4147                msgbus::send_response(reply.correlation_id(), reply);
4148            }),
4149            None,
4150        );
4151
4152        let unrelated = BusMessage::with_str_topic(
4153            "data.trades.FIRST",
4154            BusPayloadType::TradeTick,
4155            Bytes::new(),
4156            SerializationEncoding::Json,
4157        );
4158        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4159        sender.send(unrelated.clone()).await.unwrap();
4160        sender.send(message).await.unwrap();
4161        let mut receiver = Some(receiver);
4162        let mut pending = VecDeque::new();
4163        let actual = tokio::time::timeout(
4164            Duration::from_secs(1),
4165            drive_with_external_ingress(connected, &mut receiver, &mut pending),
4166        )
4167        .await
4168        .unwrap()
4169        .unwrap();
4170
4171        assert_eq!(actual.correlation_id, id);
4172        assert_eq!(actual.client_id, ClientId::from("FEED"));
4173        assert_eq!(actual.venue, None);
4174        assert_eq!(actual.data_type, DataType::new("Startup", None, None));
4175        assert_eq!(
4176            actual.data.downcast_ref::<Bytes>(),
4177            Some(&Bytes::from_static(b"connected"))
4178        );
4179        assert_eq!(actual.start, Some(101.into()));
4180        assert_eq!(actual.end, Some(103.into()));
4181        assert_eq!(actual.ts_init, UnixNanos::from(107));
4182        assert_eq!(actual.params, None);
4183        assert_eq!(publications.borrow().len(), 1);
4184        assert_eq!(pending.len(), 1);
4185        assert_eq!(
4186            serde_json::to_value(pending.pop_front().unwrap()).unwrap(),
4187            serde_json::to_value(unrelated).unwrap()
4188        );
4189        assert!(receiver.as_ref().unwrap().is_empty());
4190    }
4191
4192    #[rstest]
4193    #[tokio::test]
4194    async fn test_external_startup_defers_unrelated_messages_with_bounded_fifo() {
4195        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4196        let mut receiver = Some(receiver);
4197
4198        let message = |name: &str| {
4199            BusMessage::with_str_topic(
4200                name,
4201                BusPayloadType::TradeTick,
4202                bytes::Bytes::new(),
4203                SerializationEncoding::Json,
4204            )
4205        };
4206
4207        let first = message("data.trades.FIRST");
4208        let second = message("data.trades.SECOND");
4209        let third = message("data.trades.THIRD");
4210        sender.send(first.clone()).await.unwrap();
4211        sender.send(second.clone()).await.unwrap();
4212        let (done, connected) = tokio::sync::oneshot::channel();
4213        let mut pending = VecDeque::new();
4214        let mut connection = Box::pin(drive_with_external_ingress(
4215            connected,
4216            &mut receiver,
4217            &mut pending,
4218        ));
4219        assert!(futures_util::poll!(&mut connection).is_pending());
4220        sender.send(third.clone()).await.unwrap();
4221        assert!(futures_util::poll!(&mut connection).is_pending());
4222        done.send(()).unwrap();
4223        connection.await.unwrap();
4224
4225        assert_eq!(pending.len(), 2);
4226        assert_eq!(receiver.as_ref().unwrap().len(), 1);
4227
4228        for expected in [first, second, third] {
4229            let received = recv_external_msgbus_message(&mut receiver, &mut pending)
4230                .await
4231                .unwrap();
4232            assert_eq!(
4233                serde_json::to_value(received).unwrap(),
4234                serde_json::to_value(expected).unwrap()
4235            );
4236        }
4237    }
4238
4239    #[rstest]
4240    #[tokio::test]
4241    async fn test_external_startup_retains_receiver_closure_for_lifecycle_cleanup() {
4242        let (sender, receiver) = tokio::sync::mpsc::channel(2);
4243        drop(sender);
4244        let mut receiver = Some(receiver);
4245        let mut pending = VecDeque::new();
4246        let (done, connected) = tokio::sync::oneshot::channel();
4247        let mut connection = Box::pin(drive_with_external_ingress(
4248            connected,
4249            &mut receiver,
4250            &mut pending,
4251        ));
4252        assert!(futures_util::poll!(&mut connection).is_pending());
4253        done.send(()).unwrap();
4254        connection.await.unwrap();
4255
4256        assert!(receiver.is_none());
4257        assert!(pending.is_empty());
4258    }
4259
4260    #[rstest]
4261    #[case(None, 2)]
4262    #[case(Some(SystemChannel::DataEvents), 1)]
4263    #[case(Some(SystemChannel::ExecCommands), 1)]
4264    fn test_publish_queue_state_transitions_reaches_typed_subscriber(
4265        #[case] channel: Option<SystemChannel>,
4266        #[case] expected_count: usize,
4267    ) {
4268        let config = LiveNodeConfig {
4269            trader_id: TraderId::from("QUEUE-001"),
4270            exec_engine: crate::config::LiveExecutionEngineConfig {
4271                reconciliation: false,
4272                ..Default::default()
4273            },
4274            ..Default::default()
4275        };
4276
4277        let node = LiveNode::build("QueuePublicationNode".to_string(), Some(config)).unwrap();
4278        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4279
4280        let handler = ShareableMessageHandler::from_typed({
4281            let received = received.clone();
4282            move |event: &QueueStateChanged| received.borrow_mut().push(event.clone())
4283        });
4284
4285        msgbus::subscribe_any(
4286            MessagingSwitchboard::queue_state_changed_pattern(channel),
4287            handler,
4288            None,
4289        );
4290
4291        let transitions = [
4292            QueueStateTransition {
4293                channel: SystemChannel::DataEvents,
4294                condition: QueueCondition::Backlogged,
4295                state: QueueState::Triggered,
4296                queue_depth: 17,
4297                mean_dispatch_ns: 23,
4298            },
4299            QueueStateTransition {
4300                channel: SystemChannel::ExecCommands,
4301                condition: QueueCondition::Slow,
4302                state: QueueState::Triggered,
4303                queue_depth: 17,
4304                mean_dispatch_ns: 23,
4305            },
4306        ];
4307
4308        node.publish_queue_state_transitions(&transitions);
4309
4310        let events = received.borrow();
4311        assert_eq!(events.len(), expected_count);
4312
4313        let expected = transitions
4314            .into_iter()
4315            .filter(|transition| channel.is_none_or(|channel| channel == transition.channel));
4316
4317        for (event, transition) in events.iter().zip(expected) {
4318            assert_eq!(event.trader_id, TraderId::from("QUEUE-001"));
4319            assert_eq!(event.channel, transition.channel);
4320            assert_eq!(event.condition, transition.condition);
4321            assert_eq!(event.state, transition.state);
4322            assert_eq!(event.queue_depth, transition.queue_depth);
4323            assert_eq!(event.mean_dispatch_ns, transition.mean_dispatch_ns);
4324            assert_ne!(event.event_id, UUID4::default());
4325            assert_ne!(event.ts_event, UnixNanos::default());
4326            assert_eq!(event.ts_init, event.ts_event);
4327        }
4328
4329        if channel.is_none() {
4330            assert_ne!(events[0].event_id, events[1].event_id);
4331        }
4332
4333        drop(events);
4334        msgbus::get_message_bus().borrow_mut().dispose();
4335    }
4336
4337    #[rstest]
4338    #[case(None, None)]
4339    #[case(Some(ClientId::from("BINANCE")), None)]
4340    #[case(None, Some("binance-futures-market-streams"))]
4341    #[case(
4342        Some(ClientId::from("BINANCE")),
4343        Some("binance-futures-market-streams")
4344    )]
4345    fn test_process_socket_state_change_reaches_typed_subscriber(
4346        #[case] client_id: Option<ClientId>,
4347        #[case] endpoint: Option<&str>,
4348    ) {
4349        let config = LiveNodeConfig {
4350            trader_id: TraderId::from("SOCKET-001"),
4351            exec_engine: crate::config::LiveExecutionEngineConfig {
4352                reconciliation: false,
4353                ..Default::default()
4354            },
4355            ..Default::default()
4356        };
4357
4358        let node = LiveNode::build("SocketPublicationNode".to_string(), Some(config)).unwrap();
4359        let received = Rc::new(RefCell::new(Vec::<SocketStateChanged>::new()));
4360
4361        let handler = ShareableMessageHandler::from_typed({
4362            let received = received.clone();
4363            move |event: &SocketStateChanged| received.borrow_mut().push(event.clone())
4364        });
4365
4366        msgbus::subscribe_any(
4367            MessagingSwitchboard::socket_state_changed_pattern(client_id, endpoint),
4368            handler,
4369            None,
4370        );
4371
4372        let change = SocketStateChange::new(
4373            ClientId::from("BINANCE"),
4374            Some(Venue::from("BINANCE")),
4375            Ustr::from("binance-futures-market-streams"),
4376            SocketState::Disconnected,
4377        );
4378
4379        node.process_system_event(SystemEvent::SocketState(change));
4380
4381        let events = received.borrow();
4382        assert_eq!(events.len(), 1);
4383        assert_eq!(events[0].trader_id, TraderId::from("SOCKET-001"));
4384        assert_eq!(events[0].client_id, ClientId::from("BINANCE"));
4385        assert_eq!(events[0].venue, Some(Venue::from("BINANCE")));
4386        assert_eq!(
4387            events[0].endpoint,
4388            Ustr::from("binance-futures-market-streams")
4389        );
4390        assert_eq!(events[0].state, SocketState::Disconnected);
4391        assert_ne!(events[0].event_id, UUID4::default());
4392        assert_ne!(events[0].ts_event, UnixNanos::default());
4393        assert_eq!(events[0].ts_init, events[0].ts_event);
4394        drop(events);
4395        msgbus::get_message_bus().borrow_mut().dispose();
4396    }
4397
4398    #[rstest]
4399    #[case::accepted(
4400        ReconnectRequestOutcome::Accepted,
4401        SocketReconnectDispatchOutcome::Accepted
4402    )]
4403    #[case::already_reconnecting(
4404        ReconnectRequestOutcome::AlreadyReconnecting,
4405        SocketReconnectDispatchOutcome::AlreadyReconnecting
4406    )]
4407    #[case::disconnected(
4408        ReconnectRequestOutcome::Disconnected,
4409        SocketReconnectDispatchOutcome::Disconnected
4410    )]
4411    #[case::closed(
4412        ReconnectRequestOutcome::Closed,
4413        SocketReconnectDispatchOutcome::Closed
4414    )]
4415    #[case::unsupported(
4416        ReconnectRequestOutcome::Unsupported,
4417        SocketReconnectDispatchOutcome::Unsupported
4418    )]
4419    fn test_request_socket_reconnect_maps_transport_outcome(
4420        #[case] transport: ReconnectRequestOutcome,
4421        #[case] expected: SocketReconnectDispatchOutcome,
4422    ) {
4423        let registry = SocketReconnectRegistry::default();
4424        let client_id = ClientId::from("TEST");
4425        let endpoint = Ustr::from("test-streams");
4426        let control = SocketControl::with_registry(client_id, None, endpoint, &registry);
4427        let _sink = control.sink();
4428        control.register(move || transport);
4429
4430        let outcome = LiveNode::request_socket_reconnect(registry.get(client_id, endpoint));
4431
4432        assert_eq!(outcome, expected);
4433    }
4434
4435    #[rstest]
4436    #[case::client_not_found(
4437        SocketReconnectLookup::ClientNotFound,
4438        SocketReconnectDispatchOutcome::UnknownClient
4439    )]
4440    #[case::unsupported(
4441        SocketReconnectLookup::Unsupported,
4442        SocketReconnectDispatchOutcome::Unsupported
4443    )]
4444    #[case::endpoint_not_found(
4445        SocketReconnectLookup::EndpointNotFound,
4446        SocketReconnectDispatchOutcome::UnknownEndpoint
4447    )]
4448    #[case::ambiguous(
4449        SocketReconnectLookup::AmbiguousEndpoint,
4450        SocketReconnectDispatchOutcome::AmbiguousEndpoint
4451    )]
4452    fn test_request_socket_reconnect_maps_lookup_failure(
4453        #[case] lookup: SocketReconnectLookup,
4454        #[case] expected: SocketReconnectDispatchOutcome,
4455    ) {
4456        assert_eq!(LiveNode::request_socket_reconnect(lookup), expected);
4457    }
4458
4459    #[rstest]
4460    fn test_system_dispatch_restores_context(#[values(false, true)] startup: bool) {
4461        let trader_id = TraderId::from("SOCKET-001");
4462
4463        let config = LiveNodeConfig {
4464            trader_id,
4465            ..Default::default()
4466        };
4467
4468        let mut node = LiveNode::build("SystemDispatchNode".to_string(), Some(config)).unwrap();
4469        let client_id = ClientId::from("TEST");
4470        let endpoint = Ustr::from("test-streams");
4471        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
4472        let sender = DispatchSender::new(tx);
4473        let events = sender.clone();
4474        msgbus::subscribe_any(
4475            MessagingSwitchboard::socket_state_changed_pattern(
4476                Some(client_id),
4477                Some(endpoint.as_str()),
4478            ),
4479            ShareableMessageHandler::from_typed(move |_: &SocketStateChanged| {
4480                events.send(17u32).unwrap();
4481            }),
4482            None,
4483        );
4484
4485        let control =
4486            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4487        let _sink = control.sink();
4488        control.register(move || {
4489            sender.send(23u32).unwrap();
4490            ReconnectRequestOutcome::Accepted
4491        });
4492
4493        let event = SystemEvent::SocketState(SocketStateChange::new(
4494            client_id,
4495            None,
4496            endpoint,
4497            SocketState::Connected,
4498        ));
4499        let command = SystemCommand::ReconnectSocket(ReconnectSocket::new(
4500            trader_id,
4501            client_id,
4502            endpoint,
4503            31.into(),
4504        ));
4505
4506        if startup {
4507            let mut pending = PendingEvents::default();
4508            pending.system_events.push(event.into());
4509            pending.system_commands.push(command.into());
4510            node.process_system_events(pending.take_system_events());
4511            node.process_system_commands(pending.take_system_commands());
4512            assert!(pending.is_empty());
4513        } else {
4514            node.process_runner_event(PendingRunnerEvent::SystemEvent(event.into()));
4515            node.process_runner_event(PendingRunnerEvent::SystemCommand(command.into()));
4516        }
4517
4518        let event_child = rx.try_recv().unwrap();
4519        let command_child = rx.try_recv().unwrap();
4520        assert!(event_child.is_rooted());
4521        assert!(command_child.is_rooted());
4522        assert_eq!(event_child.dispatch(|value| value), 17);
4523        assert_eq!(command_child.dispatch(|value| value), 23);
4524        assert!(rx.is_empty());
4525        msgbus::get_message_bus().borrow_mut().dispose();
4526    }
4527
4528    #[rstest]
4529    fn test_process_socket_reconnect_routes_only_matching_trader() {
4530        let trader_id = TraderId::from("SOCKET-001");
4531
4532        let config = LiveNodeConfig {
4533            trader_id,
4534            exec_engine: crate::config::LiveExecutionEngineConfig {
4535                reconciliation: false,
4536                ..Default::default()
4537            },
4538            ..Default::default()
4539        };
4540
4541        let node = LiveNode::build("SocketReconnectNode".to_string(), Some(config)).unwrap();
4542        let client_id = ClientId::from("TEST");
4543        let endpoint = Ustr::from("test-streams");
4544        let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
4545        let request_count = Arc::clone(&requests);
4546        let control =
4547            SocketControl::with_registry(client_id, None, endpoint, &node.socket_registry);
4548        let _sink = control.sink();
4549        control.register(move || {
4550            request_count.fetch_add(1, Ordering::SeqCst);
4551            ReconnectRequestOutcome::Accepted
4552        });
4553
4554        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4555            TraderId::from("OTHER-001"),
4556            client_id,
4557            endpoint,
4558            UnixNanos::default(),
4559        )));
4560        assert_eq!(requests.load(Ordering::SeqCst), 0);
4561
4562        node.process_system_command(SystemCommand::ReconnectSocket(ReconnectSocket::new(
4563            trader_id,
4564            client_id,
4565            endpoint,
4566            UnixNanos::default(),
4567        )));
4568        assert_eq!(requests.load(Ordering::SeqCst), 1);
4569    }
4570
4571    #[rstest]
4572    #[tokio::test]
4573    async fn test_start_publishes_socket_change_after_actor_subscribes() {
4574        let config = LiveNodeConfig {
4575            trader_id: TraderId::from("SOCKET-STARTUP-001"),
4576            exec_engine: crate::config::LiveExecutionEngineConfig {
4577                reconciliation: false,
4578                ..Default::default()
4579            },
4580            timeout_connection: Duration::ZERO,
4581            timeout_reconciliation: Duration::ZERO,
4582            timeout_portfolio: Duration::ZERO,
4583            timeout_disconnection: Duration::ZERO,
4584            delay_post_stop: Duration::ZERO,
4585            timeout_shutdown: Duration::ZERO,
4586            ..Default::default()
4587        };
4588
4589        let mut node = LiveNode::build("SocketStartupNode".to_string(), Some(config)).unwrap();
4590        let received = Rc::new(RefCell::new(Vec::new()));
4591        node.add_actor(StartupSocketActor::new(Rc::clone(&received)))
4592            .unwrap();
4593        node.runner.as_ref().unwrap().bind_senders();
4594
4595        let change = SocketStateChange::new(
4596            ClientId::from("BINANCE"),
4597            Some(Venue::from("BINANCE")),
4598            Ustr::from("binance-futures-market-streams"),
4599            SocketState::Connected,
4600        );
4601        get_system_event_sender()
4602            .send(SystemEvent::SocketState(change))
4603            .unwrap();
4604
4605        node.start().await.unwrap();
4606
4607        {
4608            let events = received.borrow();
4609            assert_eq!(events.len(), 1);
4610            assert_eq!(events[0].trader_id, TraderId::from("SOCKET-STARTUP-001"));
4611            assert_eq!(events[0].client_id, change.client_id);
4612            assert_eq!(events[0].venue, change.venue);
4613            assert_eq!(events[0].endpoint, change.endpoint);
4614            assert_eq!(events[0].state, change.state);
4615            assert_ne!(events[0].event_id, UUID4::default());
4616            assert_ne!(events[0].ts_event, UnixNanos::default());
4617            assert_eq!(events[0].ts_init, events[0].ts_event);
4618        }
4619
4620        node.stop().await.unwrap();
4621        node.dispose();
4622    }
4623
4624    #[rstest]
4625    #[tokio::test(flavor = "current_thread")]
4626    async fn test_run_publishes_queue_state_after_dispatch_sample() {
4627        let config = LiveNodeConfig {
4628            trader_id: TraderId::from("QUEUE-RUN-001"),
4629            queue_monitor: Some(crate::config::QueueMonitorConfig {
4630                queue_depth_trigger: usize::MAX,
4631                queue_depth_clear: 0,
4632                mean_dispatch_ns_trigger: 1,
4633                mean_dispatch_ns_clear: 0,
4634            }),
4635            exec_engine: crate::config::LiveExecutionEngineConfig {
4636                reconciliation: false,
4637                ..Default::default()
4638            },
4639            delay_post_stop: Duration::ZERO,
4640            ..Default::default()
4641        };
4642
4643        let mut node = LiveNode::build("QueueMonitorRunNode".to_string(), Some(config)).unwrap();
4644        let handle = node.handle();
4645        let received = Rc::new(RefCell::new(Vec::<QueueStateChanged>::new()));
4646
4647        let handler = ShareableMessageHandler::from_typed({
4648            let received = received.clone();
4649            let stop_handle = handle.clone();
4650
4651            move |event: &QueueStateChanged| {
4652                received.borrow_mut().push(event.clone());
4653                stop_handle.stop();
4654            }
4655        });
4656
4657        msgbus::subscribe_any(
4658            MessagingSwitchboard::queue_state_changed_pattern(None),
4659            handler,
4660            None,
4661        );
4662        let drive_handle = handle.clone();
4663
4664        let result = tokio::time::timeout(Duration::from_secs(5), async {
4665            let run = node.run();
4666            tokio::pin!(run);
4667
4668            let drive = async move {
4669                wait_until_async(
4670                    || async { drive_handle.is_running() },
4671                    Duration::from_secs(2),
4672                )
4673                .await;
4674                get_data_event_sender().send(stub_data_event()).unwrap();
4675            };
4676
4677            let (run_result, ()) = tokio::join!(run, drive);
4678            run_result
4679        })
4680        .await;
4681
4682        assert!(
4683            result.is_ok(),
4684            "queue state event should arrive before timeout"
4685        );
4686        assert!(result.unwrap().is_ok(), "run() should succeed");
4687        let events = received.borrow();
4688        assert_eq!(events.len(), 1);
4689        assert_eq!(events[0].trader_id, TraderId::from("QUEUE-RUN-001"));
4690        assert_eq!(events[0].channel, SystemChannel::DataEvents);
4691        assert_eq!(events[0].condition, QueueCondition::Slow);
4692        assert_eq!(events[0].state, QueueState::Triggered);
4693        assert_eq!(events[0].queue_depth, 0);
4694        assert!(events[0].mean_dispatch_ns > 0);
4695        assert_ne!(events[0].event_id, UUID4::default());
4696        assert_ne!(events[0].ts_event, UnixNanos::default());
4697        assert_eq!(events[0].ts_init, events[0].ts_event);
4698        drop(events);
4699        msgbus::get_message_bus().borrow_mut().dispose();
4700    }
4701
4702    #[rstest]
4703    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4704    async fn test_run_dispatch_progress_under_load(
4705        #[values(false, true)] execution: bool,
4706        #[values(false, true)] slow: bool,
4707    ) {
4708        let config = LiveNodeConfig {
4709            delay_post_stop: Duration::ZERO,
4710            ..Default::default()
4711        };
4712        let mut node = LiveNode::build("FairDispatchNode".to_string(), Some(config)).unwrap();
4713        let handle = node.handle();
4714        let stop = handle.clone();
4715        let callback = TimeEventCallback::RustLocal(Rc::new(move |event| {
4716            if slow {
4717                // Make housekeeping ready after each timer callback
4718                std::thread::sleep(Duration::from_millis(110));
4719            }
4720
4721            if event.ts_init == UnixNanos::from(8) {
4722                // Check progress before shutdown can drain the queues
4723                let snapshot = stop.metrics_snapshot();
4724                assert_eq!(snapshot.time_events.dispatched, 7);
4725
4726                if execution {
4727                    assert!((1..16).contains(&snapshot.exec_events.dispatched));
4728                    assert_eq!(snapshot.data_events.dispatched, 0);
4729                } else {
4730                    assert!(snapshot.data_events.dispatched > 0);
4731                    assert_eq!(snapshot.data_commands.dispatched, 1);
4732                }
4733                stop.stop();
4734            }
4735        }));
4736
4737        let drive = async {
4738            wait_until_async(|| async { handle.is_running() }, Duration::from_secs(2)).await;
4739
4740            for ts in (1..=16_u64).map(UnixNanos::from) {
4741                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
4742                    TimeEvent::new("fairness".into(), UUID4::new(), ts, ts),
4743                    callback.clone(),
4744                ));
4745                get_data_event_sender().send(stub_data_event()).unwrap();
4746
4747                if execution {
4748                    get_exec_event_sender().send(stub_exec_event()).unwrap();
4749                }
4750            }
4751            nautilus_common::runner::get_data_cmd_sender().execute(stub_data_command());
4752        };
4753        tokio::time::timeout(Duration::from_secs(5), async {
4754            let (result, ()) = tokio::join!(node.run(), drive);
4755            result.unwrap();
4756        })
4757        .await
4758        .unwrap();
4759        assert!(handle.should_stop());
4760    }
4761
4762    #[rstest]
4763    fn test_observe_exec_event_before_dispatch_skips_recent_fill_report() {
4764        let config = LiveNodeConfig {
4765            exec_engine: crate::config::LiveExecutionEngineConfig {
4766                reconciliation: false,
4767                ..Default::default()
4768            },
4769            ..Default::default()
4770        };
4771
4772        let mut node = LiveNode::build("FillSkipNode".to_string(), Some(config)).unwrap();
4773        let event = stub_exec_event();
4774        let account_id = AccountId::from("TEST-001");
4775        let instrument_id = InstrumentId::from("TEST.VENUE");
4776        let trade_id = TradeId::from("T-001");
4777
4778        let close_ids = node.observe_exec_event_before_dispatch(&event);
4779        assert_eq!(close_ids, Some(Vec::new()));
4780        assert!(
4781            !node
4782                .exec_manager
4783                .is_fill_recently_processed(account_id, instrument_id, trade_id)
4784        );
4785
4786        node.exec_manager
4787            .mark_fill_processed(account_id, instrument_id, trade_id);
4788
4789        let close_ids = node.observe_exec_event_before_dispatch(&event);
4790        assert_eq!(close_ids, None);
4791    }
4792
4793    #[rstest]
4794    #[case(false, false, OrderStatus::Canceled, 1)]
4795    #[case(false, true, OrderStatus::Accepted, 0)]
4796    #[case(true, false, OrderStatus::Canceled, 1)]
4797    #[case(true, true, OrderStatus::Accepted, 0)]
4798    fn test_process_exec_event_clears_terminal_activity_only_after_cached_order_closes(
4799        #[case] with_fills: bool,
4800        #[case] superseded: bool,
4801        #[case] expected_status: OrderStatus,
4802        #[case] expected_query_count: usize,
4803    ) {
4804        let config = LiveNodeConfig {
4805            exec_engine: crate::config::LiveExecutionEngineConfig {
4806                reconciliation: true,
4807                open_check_threshold_ms: 5_000,
4808                single_order_query_delay_ms: 0,
4809                ..Default::default()
4810            },
4811            ..Default::default()
4812        };
4813
4814        let mut node = LiveNode::build("TerminalReportNode".to_string(), Some(config)).unwrap();
4815        let client_order_id = ClientOrderId::from("O-TERMINAL-REPORT");
4816        let old_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-OLD");
4817        let new_venue_order_id = VenueOrderId::from("V-TERMINAL-REPORT-NEW");
4818        let account_id = AccountId::from("TEST-001");
4819        let client_id = ClientId::from("TEST");
4820        let instrument = crypto_perpetual_ethusdt();
4821        let instrument_id = instrument.id();
4822        let account = AccountAny::Margin(MarginAccount::new(
4823            AccountState::new(
4824                account_id,
4825                AccountType::Margin,
4826                vec![AccountBalance::new(
4827                    Money::from("1000000 USDT"),
4828                    Money::from("0 USDT"),
4829                    Money::from("1000000 USDT"),
4830                )],
4831                Vec::new(),
4832                true,
4833                UUID4::new(),
4834                UnixNanos::default(),
4835                UnixNanos::default(),
4836                Some(Currency::USDT()),
4837            ),
4838            true,
4839        ));
4840        node.kernel.cache.borrow_mut().add_account(account).unwrap();
4841        node.kernel
4842            .cache
4843            .borrow_mut()
4844            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
4845            .unwrap();
4846        insert_accepted_limit_order_in_node(
4847            &node,
4848            account_id,
4849            client_id,
4850            instrument_id,
4851            client_order_id,
4852            old_venue_order_id,
4853        );
4854
4855        if superseded {
4856            let order = node
4857                .kernel
4858                .cache
4859                .borrow()
4860                .order_owned(&client_order_id)
4861                .unwrap();
4862            let pending_update = OrderPendingUpdateSpec::builder()
4863                .trader_id(order.trader_id())
4864                .strategy_id(order.strategy_id())
4865                .instrument_id(order.instrument_id())
4866                .client_order_id(client_order_id)
4867                .account_id(account_id)
4868                .venue_order_id(old_venue_order_id)
4869                .build();
4870            node.kernel
4871                .cache
4872                .borrow_mut()
4873                .update_order(&OrderEventAny::PendingUpdate(pending_update))
4874                .unwrap();
4875            let order = node
4876                .kernel
4877                .cache
4878                .borrow()
4879                .order_owned(&client_order_id)
4880                .unwrap();
4881            let updated = OrderUpdatedSpec::builder()
4882                .trader_id(order.trader_id())
4883                .strategy_id(order.strategy_id())
4884                .instrument_id(order.instrument_id())
4885                .client_order_id(client_order_id)
4886                .quantity(order.quantity())
4887                .venue_order_id(new_venue_order_id)
4888                .account_id(account_id)
4889                .build();
4890            node.kernel
4891                .cache
4892                .borrow_mut()
4893                .update_order(&OrderEventAny::Updated(updated))
4894                .unwrap();
4895        }
4896
4897        let report = OrderStatusReport::new(
4898            account_id,
4899            instrument_id,
4900            Some(client_order_id),
4901            old_venue_order_id,
4902            OrderSide::Buy.into(),
4903            OrderType::Limit,
4904            TimeInForce::Gtc,
4905            OrderStatus::Canceled,
4906            Quantity::from("10.0"),
4907            Quantity::from("0.0"),
4908            UnixNanos::from(1_000),
4909            UnixNanos::from(2_000),
4910            UnixNanos::from(3_000),
4911            None,
4912        );
4913
4914        let report = if with_fills {
4915            ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
4916        } else {
4917            ExecutionReport::Order(Box::new(report))
4918        };
4919
4920        let event = ExecutionEvent::Report(report);
4921
4922        node.process_exec_event(event);
4923
4924        let order = node
4925            .kernel
4926            .cache
4927            .borrow()
4928            .order_owned(&client_order_id)
4929            .unwrap();
4930
4931        let expected_venue_order_id = if superseded {
4932            new_venue_order_id
4933        } else {
4934            old_venue_order_id
4935        };
4936
4937        assert_eq!(order.status(), expected_status);
4938        assert_eq!(order.venue_order_id(), Some(expected_venue_order_id));
4939
4940        if !superseded {
4941            let replacement = OrderTestBuilder::new(OrderType::Limit)
4942                .client_order_id(client_order_id)
4943                .instrument_id(instrument_id)
4944                .quantity(Quantity::from("20.0"))
4945                .price(Price::from("200.0"))
4946                .build();
4947            let submitted = TestOrderEventStubs::submitted(&replacement, account_id);
4948            node.kernel
4949                .cache
4950                .borrow_mut()
4951                .add_order(replacement, None, Some(client_id), true)
4952                .unwrap();
4953            let replacement = node
4954                .kernel
4955                .cache
4956                .borrow_mut()
4957                .update_order(&submitted)
4958                .unwrap();
4959            let accepted =
4960                TestOrderEventStubs::accepted(&replacement, account_id, old_venue_order_id);
4961            node.kernel
4962                .cache
4963                .borrow_mut()
4964                .update_order(&accepted)
4965                .unwrap();
4966        }
4967
4968        assert_eq!(
4969            node.exec_manager.check_open_order_queries().len(),
4970            expected_query_count,
4971        );
4972    }
4973
4974    #[rstest]
4975    fn test_rejected_direct_fill_stays_eligible_for_later_report() {
4976        let (mut node, mut fill_event, _) = recent_fill_test_fixture("RejectedDirectFillNode");
4977
4978        let OrderEventAny::Filled(fill) = &mut fill_event else {
4979            unreachable!();
4980        };
4981
4982        fill.client_order_id = ClientOrderId::from("O-UNKNOWN");
4983        fill.venue_order_id = VenueOrderId::from("V-UNKNOWN");
4984        let fill = fill.clone();
4985        let report_event = fill_report_event(&fill);
4986        let event = ExecutionEvent::Order(OrderEventAny::Filled(fill.clone()));
4987
4988        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
4989        let marked_before_dispatch = is_recent_fill(&node, &fill);
4990
4991        node.dispatch_exec_event_and_commit_fill(event);
4992
4993        let marked_after_dispatch = is_recent_fill(&node, &fill);
4994        let later_report_is_eligible = node
4995            .observe_exec_event_before_dispatch(&report_event)
4996            .is_some();
4997        assert_eq!(
4998            (
4999                marked_before_dispatch,
5000                marked_after_dispatch,
5001                later_report_is_eligible,
5002            ),
5003            (false, false, true),
5004        );
5005    }
5006
5007    #[rstest]
5008    fn test_applied_direct_fill_commits_and_skips_later_report() {
5009        let (mut node, fill_event, _) = recent_fill_test_fixture("AppliedDirectFillNode");
5010
5011        let OrderEventAny::Filled(fill) = &fill_event else {
5012            unreachable!();
5013        };
5014
5015        let fill = fill.clone();
5016        let report_event = fill_report_event(&fill);
5017        let event = ExecutionEvent::Order(fill_event);
5018
5019        assert!(node.observe_exec_event_before_dispatch(&event).is_some());
5020        assert!(!is_recent_fill(&node, &fill));
5021
5022        node.dispatch_exec_event_and_commit_fill(event);
5023
5024        assert!(is_recent_fill(&node, &fill));
5025        assert_eq!(node.observe_exec_event_before_dispatch(&report_event), None);
5026    }
5027
5028    #[rstest]
5029    fn test_canonical_duplicate_fill_counts_as_applied() {
5030        let (mut node, fill_event, _) = recent_fill_test_fixture("DuplicateDirectFillNode");
5031
5032        let OrderEventAny::Filled(fill) = &fill_event else {
5033            unreachable!();
5034        };
5035
5036        let mut fill = fill.clone();
5037        node.kernel
5038            .cache
5039            .borrow_mut()
5040            .update_order(&fill_event)
5041            .unwrap();
5042        fill.client_order_id = ClientOrderId::from("O-DUPLICATE-UNKNOWN");
5043
5044        node.exec_manager.commit_recent_fill_if_applied(&fill);
5045
5046        assert!(is_recent_fill(&node, &fill));
5047    }
5048
5049    #[rstest]
5050    #[case(false)]
5051    #[case(true)]
5052    fn test_continuous_reconciliation_commits_only_applied_fill(#[case] applied: bool) {
5053        let (mut node, mut fill_event, _) = recent_fill_test_fixture(if applied {
5054            "AppliedContinuousFillNode"
5055        } else {
5056            "RejectedContinuousFillNode"
5057        });
5058
5059        if !applied {
5060            let OrderEventAny::Filled(fill) = &mut fill_event else {
5061                unreachable!();
5062            };
5063
5064            fill.client_order_id = ClientOrderId::from("O-CONTINUOUS-UNKNOWN");
5065            fill.venue_order_id = VenueOrderId::from("V-CONTINUOUS-UNKNOWN");
5066        }
5067
5068        let OrderEventAny::Filled(fill) = &fill_event else {
5069            unreachable!();
5070        };
5071
5072        let fill = fill.clone();
5073
5074        node.process_reconciliation_events(&[fill_event]);
5075
5076        assert_eq!(is_recent_fill(&node, &fill), applied);
5077    }
5078
5079    #[rstest]
5080    fn test_recent_fill_commit_requires_account_and_instrument_match() {
5081        let (mut node, fill_event, _) = recent_fill_test_fixture("MismatchedDirectFillNode");
5082        node.kernel
5083            .cache
5084            .borrow_mut()
5085            .update_order(&fill_event)
5086            .unwrap();
5087
5088        let OrderEventAny::Filled(fill) = fill_event else {
5089            unreachable!();
5090        };
5091
5092        let mut account_mismatch = fill.clone();
5093        account_mismatch.account_id = AccountId::from("OTHER-001");
5094        let mut instrument_mismatch = fill;
5095        instrument_mismatch.instrument_id = InstrumentId::from("OTHER.VENUE");
5096
5097        node.exec_manager
5098            .commit_recent_fill_if_applied(&account_mismatch);
5099        node.exec_manager
5100            .commit_recent_fill_if_applied(&instrument_mismatch);
5101
5102        assert!(!is_recent_fill(&node, &account_mismatch));
5103        assert!(!is_recent_fill(&node, &instrument_mismatch));
5104    }
5105
5106    #[rstest]
5107    fn test_applied_inferred_fill_remains_recently_processed() {
5108        let (mut node, _, instrument) = recent_fill_test_fixture("InferredFillNode");
5109        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
5110        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
5111        let account_id = AccountId::from("TEST-001");
5112        let order = node
5113            .kernel
5114            .cache
5115            .borrow()
5116            .order_owned(&client_order_id)
5117            .unwrap();
5118        let report = OrderStatusReport::new(
5119            account_id,
5120            instrument.id(),
5121            Some(client_order_id),
5122            venue_order_id,
5123            OrderSide::Buy.into(),
5124            OrderType::Limit,
5125            TimeInForce::Gtc,
5126            OrderStatus::PartiallyFilled,
5127            Quantity::from("10.0"),
5128            Quantity::from("1.0"),
5129            UnixNanos::from(1_000),
5130            UnixNanos::from(1_000),
5131            UnixNanos::from(1_000),
5132            None,
5133        )
5134        .with_avg_px(dec!(100.0));
5135        let inferred = create_inferred_fill_for_qty(
5136            &order,
5137            &report,
5138            &account_id,
5139            &instrument,
5140            Quantity::from("1.0"),
5141            UnixNanos::from(1_000),
5142            None,
5143        )
5144        .unwrap();
5145
5146        let OrderEventAny::Filled(fill) = &inferred else {
5147            unreachable!();
5148        };
5149
5150        let fill = fill.clone();
5151
5152        node.process_reconciliation_events(&[inferred]);
5153
5154        assert!(fill.reconciliation);
5155        assert!(is_recent_fill(&node, &fill));
5156    }
5157
5158    #[rstest]
5159    fn test_drain_exec_events_before_reconciliation_applies_real_fill_first() {
5160        let (mut node, fill_event, instrument) = recent_fill_test_fixture("DeferredFillNode");
5161
5162        let OrderEventAny::Filled(fill) = fill_event else {
5163            unreachable!();
5164        };
5165
5166        let account_id = fill.account_id;
5167        let venue_order_id = fill.venue_order_id;
5168        let client_id = ClientId::from("TEST-EVIDENCE");
5169        node.kernel
5170            .exec_engine
5171            .borrow_mut()
5172            .register_client(Box::new(StubExecutionClient::new(
5173                client_id,
5174                account_id,
5175                instrument.id().venue,
5176                OmsType::Netting,
5177                None,
5178            )))
5179            .unwrap();
5180
5181        let report = FillReport::new(
5182            account_id,
5183            instrument.id(),
5184            venue_order_id,
5185            fill.trade_id,
5186            fill.order_side,
5187            fill.last_qty,
5188            fill.last_px,
5189            fill.commission.unwrap(),
5190            fill.liquidity_side,
5191            Some(fill.client_order_id),
5192            None,
5193            fill.ts_event,
5194            fill.ts_init,
5195            None,
5196        );
5197        let (exec_evt_tx, mut exec_evt_rx) =
5198            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
5199        exec_evt_tx
5200            .send(ExecutionEvent::Report(ExecutionReport::Fill(Box::new(report))).into())
5201            .unwrap();
5202
5203        node.drain_exec_events_before_reconciliation(&mut exec_evt_rx);
5204
5205        assert!(exec_evt_rx.try_recv().is_err());
5206        let order = node
5207            .kernel
5208            .cache
5209            .borrow()
5210            .order_owned(&fill.client_order_id)
5211            .unwrap();
5212        assert_eq!(order.filled_qty(), fill.last_qty);
5213        assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5214
5215        let report = OrderStatusReport::new(
5216            account_id,
5217            instrument.id(),
5218            Some(fill.client_order_id),
5219            venue_order_id,
5220            OrderSide::Buy.into(),
5221            OrderType::Limit,
5222            TimeInForce::Gtc,
5223            OrderStatus::PartiallyFilled,
5224            Quantity::from("10.0"),
5225            fill.last_qty,
5226            UnixNanos::from(1_000),
5227            UnixNanos::from(1_000),
5228            UnixNanos::from(1_000),
5229            None,
5230        )
5231        .with_avg_px(dec!(100.0));
5232        let engine = node.kernel.exec_engine.borrow();
5233        let clients = engine.get_all_clients();
5234        let check = node
5235            .exec_manager
5236            .prepare_open_order_report_check(UUID4::new(), &clients);
5237        let reconciliation = node.exec_manager.reconcile_open_order_reports(
5238            &check,
5239            vec![SourcedOrderStatusReport { client_id, report }],
5240            &IndexSet::new(),
5241            &IndexSet::new(),
5242            &clients,
5243        );
5244        drop(engine);
5245        node.process_reconciliation_events(&reconciliation.events);
5246
5247        let order = node
5248            .kernel
5249            .cache
5250            .borrow()
5251            .order_owned(&fill.client_order_id)
5252            .unwrap();
5253        assert_eq!(order.filled_qty(), fill.last_qty);
5254        assert_eq!(order.trade_ids(), &[&fill.trade_id]);
5255    }
5256
5257    #[rstest]
5258    #[tokio::test]
5259    async fn test_request_position_fill_reports_keeps_failed_query_unsuccessful() {
5260        let client_id = ClientId::from("POSITION-FILLS");
5261        let account_id = AccountId::from("POSITION-FILLS-001");
5262        let instrument_id = crypto_perpetual_ethusdt().id();
5263        let commands = Rc::new(RefCell::new(Vec::new()));
5264
5265        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5266            client_id,
5267            account_id,
5268            venue: instrument_id.venue,
5269            outcome: FillReportClientOutcome::Failure,
5270            commands: commands.clone(),
5271        }));
5272
5273        let command = GenerateFillReports::new(
5274            UUID4::new(),
5275            UnixNanos::from(2_000),
5276            Some(instrument_id),
5277            None,
5278            Some(UnixNanos::from(1_000)),
5279            Some(UnixNanos::from(2_000)),
5280            None,
5281            None,
5282        );
5283
5284        let result = request_position_fill_reports(
5285            vec![client],
5286            vec![PositionFillReportQuery {
5287                key: (instrument_id, account_id),
5288                client_id,
5289                command: command.clone(),
5290            }],
5291        )
5292        .await
5293        .finish();
5294
5295        assert_eq!(*commands.borrow(), vec![command]);
5296        assert!(result.is_empty());
5297    }
5298
5299    #[rstest]
5300    #[tokio::test]
5301    async fn test_request_position_fill_reports_accepts_scoped_response() {
5302        let client_id = ClientId::from("POSITION-FILLS");
5303        let account_id = AccountId::from("POSITION-FILLS-001");
5304        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5305        let instrument_id = instrument.id();
5306
5307        let report_b = FillReport::new(
5308            account_id,
5309            instrument_id,
5310            VenueOrderId::from("V-POSITION-FILLS"),
5311            TradeId::from("T-POSITION-FILLS-B"),
5312            OrderSide::Buy,
5313            Quantity::from("1.0"),
5314            Price::from("100.0"),
5315            Money::zero(instrument.quote_currency()),
5316            LiquiditySide::Taker,
5317            Some(ClientOrderId::from("O-POSITION-FILLS")),
5318            None,
5319            UnixNanos::from(1_500),
5320            UnixNanos::from(2_000),
5321            None,
5322        );
5323        let mut report_a = report_b.clone();
5324        report_a.trade_id = TradeId::from("T-POSITION-FILLS-A");
5325        let commands = Rc::new(RefCell::new(Vec::new()));
5326
5327        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5328            client_id,
5329            account_id,
5330            venue: instrument_id.venue,
5331            outcome: FillReportClientOutcome::Reports(vec![report_b.clone(), report_a.clone()]),
5332            commands,
5333        }));
5334
5335        let command = GenerateFillReports::new(
5336            UUID4::new(),
5337            UnixNanos::from(2_000),
5338            Some(instrument_id),
5339            None,
5340            Some(UnixNanos::from(1_000)),
5341            Some(UnixNanos::from(2_000)),
5342            None,
5343            None,
5344        );
5345        let key = (instrument_id, account_id);
5346
5347        let result = request_position_fill_reports(
5348            vec![client],
5349            vec![PositionFillReportQuery {
5350                key,
5351                client_id,
5352                command,
5353            }],
5354        )
5355        .await
5356        .finish();
5357
5358        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5359        assert_eq!(result, IndexMap::from([(key, vec![report_a, report_b])]));
5360    }
5361
5362    #[rstest]
5363    #[case("OTHER-001", "ETHUSDT-PERP.BINANCE", 1_500, "1.0")]
5364    #[case("POSITION-FILLS-001", "BTCUSDT-PERP.BINANCE", 1_500, "1.0")]
5365    #[case("POSITION-FILLS-001", "ETHUSDT-PERP.BINANCE", 1_500, "0.0")]
5366    #[tokio::test]
5367    async fn test_request_position_fill_reports_rejects_out_of_scope_response(
5368        #[case] report_account: &str,
5369        #[case] report_instrument: &str,
5370        #[case] ts_event: u64,
5371        #[case] quantity: &str,
5372    ) {
5373        let client_id = ClientId::from("POSITION-FILLS");
5374        let account_id = AccountId::from("POSITION-FILLS-001");
5375        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5376        let instrument_id = instrument.id();
5377
5378        let report = FillReport::new(
5379            AccountId::from(report_account),
5380            InstrumentId::from(report_instrument),
5381            VenueOrderId::from("V-POSITION-FILLS"),
5382            TradeId::from("T-POSITION-FILLS"),
5383            OrderSide::Buy,
5384            Quantity::from(quantity),
5385            Price::from("100.0"),
5386            Money::zero(instrument.quote_currency()),
5387            LiquiditySide::Taker,
5388            Some(ClientOrderId::from("O-POSITION-FILLS")),
5389            None,
5390            UnixNanos::from(ts_event),
5391            UnixNanos::from(2_000),
5392            None,
5393        );
5394
5395        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5396            client_id,
5397            account_id,
5398            venue: instrument_id.venue,
5399            outcome: FillReportClientOutcome::Reports(vec![report]),
5400            commands: Rc::new(RefCell::new(Vec::new())),
5401        }));
5402
5403        let command = GenerateFillReports::new(
5404            UUID4::new(),
5405            UnixNanos::from(2_000),
5406            Some(instrument_id),
5407            None,
5408            Some(UnixNanos::from(1_000)),
5409            Some(UnixNanos::from(2_000)),
5410            None,
5411            None,
5412        );
5413        let key = (instrument_id, account_id);
5414
5415        let result = request_position_fill_reports(
5416            vec![client],
5417            vec![PositionFillReportQuery {
5418                key,
5419                client_id,
5420                command,
5421            }],
5422        )
5423        .await
5424        .finish();
5425
5426        assert!(result.is_empty());
5427    }
5428
5429    #[rstest]
5430    #[case(999)]
5431    #[case(2_001)]
5432    #[tokio::test]
5433    async fn test_request_position_fill_reports_filters_time_window_superset(
5434        #[case] ts_event: u64,
5435    ) {
5436        let client_id = ClientId::from("POSITION-FILLS");
5437        let account_id = AccountId::from("POSITION-FILLS-001");
5438        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
5439        let instrument_id = instrument.id();
5440
5441        let report = FillReport::new(
5442            account_id,
5443            instrument_id,
5444            VenueOrderId::from("V-POSITION-FILLS"),
5445            TradeId::from("T-POSITION-FILLS"),
5446            OrderSide::Buy,
5447            Quantity::from("1.0"),
5448            Price::from("100.0"),
5449            Money::zero(instrument.quote_currency()),
5450            LiquiditySide::Taker,
5451            Some(ClientOrderId::from("O-POSITION-FILLS")),
5452            None,
5453            UnixNanos::from(ts_event),
5454            UnixNanos::from(2_000),
5455            None,
5456        );
5457
5458        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5459            client_id,
5460            account_id,
5461            venue: instrument_id.venue,
5462            outcome: FillReportClientOutcome::Reports(vec![report]),
5463            commands: Rc::new(RefCell::new(Vec::new())),
5464        }));
5465
5466        let command = GenerateFillReports::new(
5467            UUID4::new(),
5468            UnixNanos::from(2_000),
5469            Some(instrument_id),
5470            None,
5471            Some(UnixNanos::from(1_000)),
5472            Some(UnixNanos::from(2_000)),
5473            None,
5474            None,
5475        );
5476        let key = (instrument_id, account_id);
5477
5478        let result = request_position_fill_reports(
5479            vec![client],
5480            vec![PositionFillReportQuery {
5481                key,
5482                client_id,
5483                command,
5484            }],
5485        )
5486        .await
5487        .finish();
5488
5489        assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5490        assert_eq!(result, IndexMap::from([(key, Vec::new())]));
5491    }
5492
5493    #[rstest]
5494    #[case::query_error(false)]
5495    #[case::missing_client(true)]
5496    #[tokio::test]
5497    async fn test_position_fill_collection_discards_partial_failure(
5498        #[case] missing_client: bool,
5499        #[values(false, true)] failure_first: bool,
5500        position_fill_query: (FillReport, PositionFillReportQuery),
5501    ) {
5502        let (report, query) = position_fill_query;
5503        let failed_client_id = ClientId::from("FAILED-FILLS");
5504        let healthy_key = (report.instrument_id, AccountId::from("HEALTHY-001"));
5505        let mut healthy_report = report.clone();
5506        healthy_report.account_id = healthy_key.1;
5507        let healthy_client_id = ClientId::from("HEALTHY-FILLS");
5508        let empty_key = (report.instrument_id, AccountId::from("EMPTY-001"));
5509        let empty_client_id = ClientId::from("EMPTY-FILLS");
5510        let mut clients = vec![
5511            LiveExecutionClient::new(Box::new(FillReportClient {
5512                client_id: query.client_id,
5513                account_id: query.key.1,
5514                venue: query.key.0.venue,
5515                outcome: FillReportClientOutcome::Reports(vec![report]),
5516                commands: Rc::new(RefCell::new(Vec::new())),
5517            })),
5518            LiveExecutionClient::new(Box::new(FillReportClient {
5519                client_id: healthy_client_id,
5520                account_id: healthy_key.1,
5521                venue: healthy_key.0.venue,
5522                outcome: FillReportClientOutcome::Reports(vec![healthy_report.clone()]),
5523                commands: Rc::new(RefCell::new(Vec::new())),
5524            })),
5525            LiveExecutionClient::new(Box::new(FillReportClient {
5526                client_id: empty_client_id,
5527                account_id: empty_key.1,
5528                venue: empty_key.0.venue,
5529                outcome: FillReportClientOutcome::Reports(Vec::new()),
5530                commands: Rc::new(RefCell::new(Vec::new())),
5531            })),
5532        ];
5533
5534        if !missing_client {
5535            clients.push(LiveExecutionClient::new(Box::new(FillReportClient {
5536                client_id: failed_client_id,
5537                account_id: query.key.1,
5538                venue: query.key.0.venue,
5539                outcome: FillReportClientOutcome::Failure,
5540                commands: Rc::new(RefCell::new(Vec::new())),
5541            })));
5542        }
5543
5544        let failed_query = PositionFillReportQuery {
5545            client_id: failed_client_id,
5546            key: query.key,
5547            command: query.command.clone(),
5548        };
5549
5550        let healthy_query = PositionFillReportQuery {
5551            key: healthy_key,
5552            client_id: healthy_client_id,
5553            command: query.command.clone(),
5554        };
5555
5556        let empty_query = PositionFillReportQuery {
5557            key: empty_key,
5558            client_id: empty_client_id,
5559            command: query.command.clone(),
5560        };
5561
5562        let mut queries = vec![query, failed_query];
5563
5564        if failure_first {
5565            queries.reverse();
5566        }
5567
5568        queries.insert(0, empty_query);
5569        queries.push(healthy_query);
5570
5571        let result = request_position_fill_reports(clients, queries)
5572            .await
5573            .finish();
5574
5575        assert_eq!(
5576            result.keys().copied().collect::<Vec<_>>(),
5577            vec![empty_key, healthy_key]
5578        );
5579        assert_eq!(
5580            result,
5581            IndexMap::from([(empty_key, Vec::new()), (healthy_key, vec![healthy_report])])
5582        );
5583    }
5584
5585    #[rstest]
5586    #[case::same_economics("metadata", true)]
5587    #[case::venue_order("venue_order", false)]
5588    #[case::side("side", false)]
5589    #[case::quantity("quantity", false)]
5590    #[case::price("price", false)]
5591    #[case::commission("commission", false)]
5592    #[case::liquidity("liquidity", false)]
5593    #[case::average("average", false)]
5594    #[case::event_time("event_time", false)]
5595    #[case::client_order("client_order", false)]
5596    #[case::position("position", false)]
5597    #[tokio::test]
5598    async fn test_position_fill_collection_validates_duplicate_economics(
5599        #[case] changed: &str,
5600        #[case] accepted: bool,
5601        position_fill_query: (FillReport, PositionFillReportQuery),
5602    ) {
5603        let (report, query) = position_fill_query;
5604        let mut duplicate = report.clone();
5605
5606        match changed {
5607            "metadata" => {
5608                duplicate.report_id = UUID4::new();
5609                duplicate.ts_init = UnixNanos::from(3_000);
5610            }
5611            "venue_order" => duplicate.venue_order_id = VenueOrderId::from("OTHER-ORDER"),
5612            "side" => duplicate.order_side = OrderSide::Sell,
5613            "quantity" => duplicate.last_qty = Quantity::from("2.0"),
5614            "price" => duplicate.last_px = Price::from("101.0"),
5615            "commission" => duplicate.commission = Money::from("0.25 USDT"),
5616            "liquidity" => duplicate.liquidity_side = LiquiditySide::Maker,
5617            "average" => duplicate.avg_px = Some(dec!(100.5)),
5618            "event_time" => duplicate.ts_event = UnixNanos::from(1_501),
5619            "client_order" => duplicate.client_order_id = None,
5620            "position" => duplicate.venue_position_id = Some(PositionId::from("OTHER-POSITION")),
5621            _ => unreachable!(),
5622        }
5623
5624        let key = query.key;
5625
5626        let client = LiveExecutionClient::new(Box::new(FillReportClient {
5627            client_id: query.client_id,
5628            account_id: key.1,
5629            venue: key.0.venue,
5630            outcome: FillReportClientOutcome::Reports(vec![report.clone(), duplicate]),
5631            commands: Rc::new(RefCell::new(Vec::new())),
5632        }));
5633
5634        let result = request_position_fill_reports(vec![client], vec![query])
5635            .await
5636            .finish();
5637
5638        if accepted {
5639            assert_eq!(result.keys().copied().collect::<Vec<_>>(), vec![key]);
5640            assert_eq!(result, IndexMap::from([(key, vec![report])]));
5641        } else {
5642            assert!(result.is_empty());
5643        }
5644    }
5645
5646    #[fixture]
5647    fn position_fill_query() -> (FillReport, PositionFillReportQuery) {
5648        let account_id = AccountId::from("POSITION-FILLS-001");
5649        let instrument_id = crypto_perpetual_ethusdt().id();
5650
5651        let report = FillReport::new(
5652            account_id,
5653            instrument_id,
5654            VenueOrderId::from("V-POSITION-FILLS"),
5655            TradeId::from("T-POSITION-FILLS"),
5656            OrderSide::Buy,
5657            Quantity::from("1.0"),
5658            Price::from("100.0"),
5659            Money::from("0.10 USDT"),
5660            LiquiditySide::Taker,
5661            Some(ClientOrderId::from("O-POSITION-FILLS")),
5662            None,
5663            UnixNanos::from(1_500),
5664            UnixNanos::from(2_000),
5665            None,
5666        );
5667
5668        let query = PositionFillReportQuery {
5669            key: (instrument_id, account_id),
5670            client_id: ClientId::from("POSITION-FILLS"),
5671            command: GenerateFillReports::new(
5672                UUID4::new(),
5673                UnixNanos::from(2_000),
5674                Some(instrument_id),
5675                None,
5676                Some(UnixNanos::from(1_000)),
5677                Some(UnixNanos::from(2_000)),
5678                None,
5679                None,
5680            ),
5681        };
5682
5683        (report, query)
5684    }
5685
5686    #[rstest]
5687    #[case::failed_query(false)]
5688    #[case::local_activity(true)]
5689    fn test_position_fallback_preserves_deferred_venue_only_retries(#[case] local_activity: bool) {
5690        let (mut node, report, _) =
5691            position_fill_test_fixture("PositionRetryRetentionNode", Quantity::from("1.0"));
5692        let active_key = (report.instrument_id, report.account_id);
5693        let deferred_key = (report.instrument_id, AccountId::from("SECOND-001"));
5694        let deferred_reports = vec![
5695            PositionStatusReport::new(
5696                deferred_key.1,
5697                deferred_key.0,
5698                PositionSide::Long,
5699                Quantity::from("2.0"),
5700                UnixNanos::from(1_000),
5701                UnixNanos::from(1_000),
5702                None,
5703                None,
5704                Some(dec!(100.0)),
5705            ),
5706            PositionStatusReport::new(
5707                deferred_key.1,
5708                deferred_key.0,
5709                PositionSide::Short,
5710                Quantity::from("1.0"),
5711                UnixNanos::from(1_000),
5712                UnixNanos::from(1_000),
5713                None,
5714                None,
5715                Some(dec!(100.0)),
5716            ),
5717        ];
5718        let mut check = node
5719            .exec_manager
5720            .prepare_position_report_check(UUID4::new(), &[]);
5721        check.client_coverage.clear();
5722        let events = node.exec_manager.reconcile_position_reports(
5723            &check,
5724            deferred_reports.clone(),
5725            &IndexSet::new(),
5726            &IndexSet::new(),
5727        );
5728        assert!(events.is_empty());
5729        assert_eq!(
5730            node.exec_manager.position_recon_retry_count(&deferred_key),
5731            1
5732        );
5733
5734        let mut position_result = position_report_result(&node, report);
5735        position_result.reports.extend(deferred_reports);
5736        position_result.check.client_coverage.insert(
5737            deferred_key,
5738            ReportClientCoverage::Resolved(IndexSet::from([ClientId::from("SECOND")])),
5739        );
5740        position_result
5741            .check
5742            .activity_revisions
5743            .insert(deferred_key, 0);
5744        position_result
5745            .queried_clients
5746            .insert(ClientId::from("SECOND"));
5747        let mut reports = IndexMap::from([(active_key, Vec::new())]);
5748        if local_activity {
5749            reports.insert(deferred_key, Vec::new());
5750            node.exec_manager
5751                .record_position_activity(deferred_key.0, deferred_key.1);
5752        }
5753
5754        node.handle_position_fill_report_result(PositionFillReportResult {
5755            position_result,
5756            reports,
5757        });
5758
5759        assert_eq!(
5760            node.exec_manager.position_recon_retry_count(&deferred_key),
5761            1
5762        );
5763        {
5764            let cache = node.kernel.cache.borrow();
5765            let positions =
5766                cache.positions_open(None, Some(&active_key.0), None, Some(&active_key.1), None);
5767            assert_eq!(
5768                positions
5769                    .iter()
5770                    .map(|position| position.quantity)
5771                    .collect::<Vec<_>>(),
5772                vec![Quantity::from("1.0"), Quantity::from("1.0")],
5773            );
5774            assert_eq!(
5775                cache
5776                    .positions_open(
5777                        None,
5778                        Some(&deferred_key.0),
5779                        None,
5780                        Some(&deferred_key.1),
5781                        None,
5782                    )
5783                    .len(),
5784                0
5785            );
5786        }
5787
5788        let mut fresh_check = node
5789            .exec_manager
5790            .prepare_position_report_check(UUID4::new(), &[]);
5791        node.exec_manager.plan_position_fill_reports(
5792            &mut fresh_check,
5793            &[],
5794            &IndexSet::new(),
5795            &IndexSet::new(),
5796            &[],
5797        );
5798
5799        assert_eq!(
5800            node.exec_manager.position_recon_retry_count(&deferred_key),
5801            0
5802        );
5803    }
5804
5805    #[rstest]
5806    fn test_position_fill_report_result_applies_authoritative_fill_without_synthetic_order() {
5807        let (mut node, venue_report, fill_report) =
5808            position_fill_test_fixture("AuthoritativePositionFillNode", Quantity::from("1.0"));
5809        let key = (venue_report.instrument_id, venue_report.account_id);
5810        let revision = node.exec_manager.position_activity_revision(&key);
5811        let position_result = position_report_result(&node, venue_report);
5812
5813        node.handle_position_fill_report_result(PositionFillReportResult {
5814            position_result,
5815            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5816        });
5817
5818        let cache = node.kernel.cache.borrow();
5819        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
5820        assert_eq!(
5821            positions
5822                .iter()
5823                .map(|position| position.quantity)
5824                .sum::<Quantity>(),
5825            Quantity::from("2.0")
5826        );
5827        assert_eq!(
5828            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5829            1
5830        );
5831        drop(positions);
5832        drop(cache);
5833        assert!(
5834            node.exec_manager
5835                .position_contains_fill_report(&fill_report)
5836        );
5837        assert_eq!(
5838            node.exec_manager.position_activity_revision(&key),
5839            revision + 1
5840        );
5841    }
5842
5843    #[rstest]
5844    fn test_position_fill_prefix_rechecks_history_after_dispatch_callback() {
5845        let (mut node, venue_report, mut first) =
5846            position_fill_test_fixture("PositionPrefixCallbackNode", Quantity::from("0.5"));
5847        first.commission = Money::from("0.13 USDT");
5848        let key = (first.instrument_id, first.account_id);
5849        let client_order_id = first.client_order_id.unwrap();
5850        let position_result = position_report_result(&node, venue_report.clone());
5851        node.handle_position_fill_report_result(PositionFillReportResult {
5852            position_result,
5853            reports: IndexMap::from([(key, vec![first.clone()])]),
5854        });
5855
5856        let mut second = first.clone();
5857        second.trade_id = TradeId::from("T-PREFIX-SECOND");
5858        second.last_qty = Quantity::from("0.75");
5859        second.commission = Money::from("0.29 USDT");
5860        let mut third = first.clone();
5861        third.trade_id = TradeId::from("T-PREFIX-THIRD");
5862        third.last_qty = Quantity::from("0.25");
5863        third.commission = Money::from("0.07 USDT");
5864        let revision = node.exec_manager.position_activity_revision(&key);
5865        let received = Rc::new(RefCell::new(Vec::new()));
5866        let cache = node.kernel.cache.clone();
5867        let callback_received = received.clone();
5868        let second_trade_id = second.trade_id;
5869
5870        let handler = TypedHandler::from(move |event: &OrderEventAny| {
5871            let OrderEventAny::Filled(fill) = event else {
5872                return;
5873            };
5874
5875            callback_received.borrow_mut().push(fill.trade_id);
5876            if fill.trade_id == second_trade_id {
5877                let mut cache = cache.borrow_mut();
5878                let position_id = *cache.position_id(&fill.client_order_id).unwrap();
5879                let mut position = cache.position_mut(&position_id).unwrap();
5880                position.replay_events.retain(|event| !matches!(event, PositionReplayEvent::Filled(fill) if fill.trade_id == second_trade_id));
5881            }
5882        });
5883
5884        let strategy_id = node
5885            .kernel
5886            .cache
5887            .borrow()
5888            .order(&client_order_id)
5889            .unwrap()
5890            .strategy_id();
5891        let topic = nautilus_common::msgbus::switchboard::get_event_order_topic(strategy_id);
5892        msgbus::subscribe_order_events(topic.into(), handler.clone(), None);
5893        let position_result = position_report_result(&node, venue_report);
5894        node.handle_position_fill_report_result(PositionFillReportResult {
5895            position_result,
5896            reports: IndexMap::from([(
5897                key,
5898                vec![first.clone(), first.clone(), second.clone(), third.clone()],
5899            )]),
5900        });
5901
5902        msgbus::unsubscribe_order_events(topic.into(), &handler);
5903        let cache = node.kernel.cache.borrow();
5904        let order = cache.order(&client_order_id).unwrap();
5905        let position_id = cache.position_id(&client_order_id).unwrap();
5906        let position = cache.position_ref(position_id).unwrap();
5907
5908        assert_eq!(*received.borrow(), vec![second.trade_id]);
5909        assert_eq!(order.filled_qty(), Quantity::from("2.25"));
5910        assert_eq!(order.events().len(), 6);
5911        assert_eq!(
5912            order.trade_ids(),
5913            vec![
5914                &TradeId::from("T-POSITION-INITIAL"),
5915                &first.trade_id,
5916                &second.trade_id
5917            ]
5918        );
5919        assert_eq!(
5920            order.commissions().values().copied().collect::<Vec<_>>(),
5921            vec![Money::from("0.42 USDT")]
5922        );
5923        assert_eq!(position.quantity, Quantity::from("2.25"));
5924        assert_eq!(position.commissions(), vec![Money::from("0.42 USDT")]);
5925        assert_eq!(position.replay_events.len(), 2);
5926        assert_eq!(
5927            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
5928            1
5929        );
5930        assert_eq!(
5931            node.exec_manager.position_activity_revision(&key),
5932            revision + 1
5933        );
5934        assert!(node.exec_manager.position_contains_fill_report(&first));
5935        assert!(!node.exec_manager.position_contains_fill_report(&second));
5936        assert!(!node.exec_manager.position_contains_fill_report(&third));
5937        assert!(
5938            !node
5939                .exec_manager
5940                .is_fill_recently_processed(key.1, key.0, second.trade_id)
5941        );
5942        assert!(
5943            !node
5944                .exec_manager
5945                .is_fill_recently_processed(key.1, key.0, third.trade_id)
5946        );
5947    }
5948
5949    #[rstest]
5950    fn test_position_fill_report_match_ignores_adapter_timestamp_source() {
5951        let (mut node, venue_report, fill_report) =
5952            position_fill_test_fixture("PositionFillTimestampNode", Quantity::from("1.0"));
5953        let key = (venue_report.instrument_id, venue_report.account_id);
5954        let position_result = position_report_result(&node, venue_report);
5955
5956        node.handle_position_fill_report_result(PositionFillReportResult {
5957            position_result,
5958            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
5959        });
5960
5961        let mut rest_report = fill_report;
5962        rest_report.ts_event = UnixNanos::from(2_000);
5963
5964        assert!(
5965            node.exec_manager
5966                .position_contains_fill_report(&rest_report)
5967        );
5968    }
5969
5970    #[rstest]
5971    fn test_position_fill_report_result_falls_back_when_order_contains_inferred_fill() {
5972        let (mut node, venue_report, fill_report) =
5973            position_fill_test_fixture("InferredPositionFillNode", Quantity::from("1.0"));
5974        let key = (venue_report.instrument_id, venue_report.account_id);
5975        let client_order_id = fill_report.client_order_id.unwrap();
5976        apply_inferred_position_fill(&mut node, &fill_report);
5977
5978        let venue_report = PositionStatusReport::new(
5979            key.1,
5980            key.0,
5981            PositionSide::Long,
5982            Quantity::from("3.0"),
5983            venue_report.ts_last,
5984            venue_report.ts_init,
5985            None,
5986            None,
5987            venue_report.avg_px_open,
5988        );
5989        let mut later_fill_report = fill_report.clone();
5990        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
5991        later_fill_report.ts_event = UnixNanos::from(1_001);
5992        let position_result = position_report_result(&node, venue_report);
5993
5994        node.handle_position_fill_report_result(PositionFillReportResult {
5995            position_result,
5996            reports: IndexMap::from([(key, vec![fill_report.clone(), later_fill_report.clone()])]),
5997        });
5998
5999        let cache = node.kernel.cache.borrow();
6000        let order = cache.order(&client_order_id).unwrap();
6001        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6002        assert_eq!(order.filled_qty(), Quantity::from("2.0"));
6003        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6004        assert!(!order.trade_ids().contains(&&later_fill_report.trade_id));
6005        assert_eq!(
6006            positions
6007                .iter()
6008                .map(|position| position.quantity)
6009                .sum::<Quantity>(),
6010            Quantity::from("3.0")
6011        );
6012        assert_eq!(
6013            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6014            2
6015        );
6016    }
6017
6018    #[rstest]
6019    fn test_position_fill_report_result_falls_back_when_reconciled_position_includes_fill() {
6020        let (mut node, _, _) =
6021            position_fill_test_fixture("ReconciledPositionFillNode", Quantity::from("1.0"));
6022        let account_id = AccountId::from("TEST-001");
6023        let instrument = InstrumentAny::CurrencyPair(currency_pair_btcusdt());
6024        let key = (instrument.id(), account_id);
6025        let commission = Money::zero(instrument.quote_currency());
6026        node.kernel
6027            .cache
6028            .borrow_mut()
6029            .add_instrument(instrument.clone())
6030            .unwrap();
6031
6032        let order = OrderTestBuilder::new(OrderType::Market)
6033            .strategy_id(StrategyId::external())
6034            .instrument_id(instrument.id())
6035            .client_order_id(ClientOrderId::from("O-RECONCILED"))
6036            .quantity(Quantity::from("1.000000"))
6037            .tags(vec![Ustr::from(RECONCILIATION_ORDER_TAG)])
6038            .build();
6039        let submitted = TestOrderEventStubs::submitted(&order, account_id);
6040        let accepted =
6041            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-RECONCILED"));
6042        node.kernel
6043            .cache
6044            .borrow_mut()
6045            .add_order(
6046                order.clone(),
6047                None,
6048                Some(ClientId::from("POSITION-FILLS")),
6049                false,
6050            )
6051            .unwrap();
6052        node.process_reconciliation_events(&[submitted, accepted]);
6053        let order = node
6054            .kernel
6055            .cache
6056            .borrow()
6057            .order_owned(&order.client_order_id())
6058            .unwrap();
6059        let opening_fill = TestOrderEventStubs::filled(
6060            &order,
6061            &instrument,
6062            Some(TradeId::from("T-RECONCILED")),
6063            None,
6064            Some(Price::from("100.00")),
6065            Some(Quantity::from("1.000000")),
6066            Some(LiquiditySide::Taker),
6067            Some(commission),
6068            Some(UnixNanos::from(2_000)),
6069            Some(account_id),
6070        );
6071        node.process_reconciliation_events(&[opening_fill]);
6072
6073        // A real fill between the position report query and reconciliation stays off the position
6074        let held_fill_report = FillReport::new(
6075            account_id,
6076            instrument.id(),
6077            VenueOrderId::from("V-HELD"),
6078            TradeId::from("T-HELD"),
6079            OrderSide::Buy,
6080            Quantity::from("1.000000"),
6081            Price::from("100.00"),
6082            commission,
6083            LiquiditySide::Taker,
6084            None,
6085            None,
6086            UnixNanos::from(1_000),
6087            UnixNanos::from(1_000),
6088            None,
6089        );
6090        node.kernel
6091            .exec_engine
6092            .borrow_mut()
6093            .reconcile_fill_report(&held_fill_report);
6094
6095        let venue_report = PositionStatusReport::new(
6096            account_id,
6097            instrument.id(),
6098            PositionSide::Long,
6099            Quantity::from("2.000000"),
6100            UnixNanos::from(3_000),
6101            UnixNanos::from(3_000),
6102            None,
6103            None,
6104            Some(dec!(100.00)),
6105        );
6106        let position_result = position_report_result(&node, venue_report);
6107
6108        node.handle_position_fill_report_result(PositionFillReportResult {
6109            position_result,
6110            reports: IndexMap::from([(key, vec![held_fill_report.clone()])]),
6111        });
6112
6113        let cache = node.kernel.cache.borrow();
6114        let held_order = cache.order(&ClientOrderId::from("V-HELD")).unwrap();
6115        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6116        assert_eq!(held_order.filled_qty(), Quantity::from("1.000000"));
6117        assert_eq!(
6118            positions
6119                .iter()
6120                .map(|position| position.quantity)
6121                .sum::<Quantity>(),
6122            Quantity::from("2.000000")
6123        );
6124        assert!(
6125            positions
6126                .iter()
6127                .all(|position| !position.trade_ids.contains(&held_fill_report.trade_id))
6128        );
6129    }
6130
6131    #[rstest]
6132    fn test_position_fill_report_validates_hedge_identity_before_inferred_fallback() {
6133        let (mut node, _, mut fill_report) =
6134            position_fill_test_fixture("InferredHedgeIdentityNode", Quantity::from("1.0"));
6135        apply_inferred_position_fill(&mut node, &fill_report);
6136        let conflicting_position_id = PositionId::from("P-POSITION-CONFLICT");
6137        fill_report.venue_position_id = Some(conflicting_position_id);
6138
6139        let error = node
6140            .exec_manager
6141            .prepare_position_fill_report(&mut fill_report, &[])
6142            .unwrap_err();
6143
6144        assert!(
6145            error.to_string().contains(&format!(
6146                "position ID {conflicting_position_id} conflicts with cached order position"
6147            )),
6148            "{error:#}"
6149        );
6150    }
6151
6152    #[rstest]
6153    #[case::not_applied_exactly("100.0", "O-POSITION-FILLS")]
6154    #[case::preparation_error("1.0", "O-POSITION-CONFLICT")]
6155    fn test_position_fill_report_result_falls_back_after_unapplied_fill_expires(
6156        #[case] authoritative_qty: &str,
6157        #[case] client_order_id: &str,
6158    ) {
6159        let (mut node, venue_report, mut fill_report) = position_fill_test_fixture(
6160            "UnappliedPositionFillNode",
6161            Quantity::from(authoritative_qty),
6162        );
6163        fill_report.client_order_id = Some(ClientOrderId::from(client_order_id));
6164        let reports = [fill_report.clone()];
6165
6166        let quantities = (0..2)
6167            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6168            .collect::<Vec<_>>();
6169
6170        assert_eq!(
6171            quantities,
6172            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6173        );
6174        assert!(
6175            !node
6176                .exec_manager
6177                .position_contains_fill_report(&fill_report)
6178        );
6179    }
6180
6181    #[rstest]
6182    fn test_position_fill_report_result_expires_one_unapplied_fill_per_check() {
6183        let (mut node, venue_report, fill_report) =
6184            position_fill_test_fixture("UnappliedPositionFillsNode", Quantity::from("100.0"));
6185        let mut later_fill_report = fill_report.clone();
6186        later_fill_report.trade_id = TradeId::from("T-POSITION-AUTHORITATIVE-LATER");
6187        later_fill_report.ts_event = UnixNanos::from(1_001);
6188        let reports = [fill_report.clone(), later_fill_report.clone()];
6189
6190        let quantities = (0..3)
6191            .map(|_| position_quantity_after_check(&mut node, &venue_report, &reports))
6192            .collect::<Vec<_>>();
6193
6194        assert_eq!(
6195            quantities,
6196            vec![
6197                Quantity::from("1.0"),
6198                Quantity::from("1.0"),
6199                Quantity::from("2.0"),
6200            ]
6201        );
6202        assert!(
6203            !node
6204                .exec_manager
6205                .position_contains_fill_report(&fill_report)
6206        );
6207        assert!(
6208            !node
6209                .exec_manager
6210                .position_contains_fill_report(&later_fill_report)
6211        );
6212    }
6213
6214    #[rstest]
6215    #[case::within_grace(60_000, true)]
6216    #[case::after_grace(0, false)]
6217    fn test_position_fill_report_result_retries_refused_fill_only_within_grace(
6218        #[case] position_check_threshold_ms: u32,
6219        #[case] expected_applied: bool,
6220    ) {
6221        let (mut node, venue_report, fill_report) = position_fill_test_fixture_with_threshold(
6222            "TransientPositionFillNode",
6223            Quantity::from("1.0"),
6224            position_check_threshold_ms,
6225        );
6226        let mut conflicting_report = fill_report.clone();
6227        conflicting_report.client_order_id = Some(ClientOrderId::from("O-POSITION-CONFLICT"));
6228
6229        let quantities = [conflicting_report, fill_report.clone()]
6230            .into_iter()
6231            .map(|report| position_quantity_after_check(&mut node, &venue_report, &[report]))
6232            .collect::<Vec<_>>();
6233
6234        assert_eq!(
6235            quantities,
6236            vec![Quantity::from("1.0"), Quantity::from("2.0")]
6237        );
6238        assert_eq!(
6239            node.exec_manager
6240                .position_contains_fill_report(&fill_report),
6241            expected_applied
6242        );
6243    }
6244
6245    #[rstest]
6246    fn test_position_fill_report_result_synthesizes_only_residual_after_fresh_report() {
6247        let (mut node, venue_report, fill_report) =
6248            position_fill_test_fixture("ResidualPositionFillNode", Quantity::from("0.5"));
6249        let key = (venue_report.instrument_id, venue_report.account_id);
6250        let first_result = position_report_result(&node, venue_report.clone());
6251
6252        node.handle_position_fill_report_result(PositionFillReportResult {
6253            position_result: first_result,
6254            reports: IndexMap::from([(key, vec![fill_report])]),
6255        });
6256
6257        let fresh_result = position_report_result(&node, venue_report);
6258        node.handle_position_fill_report_result(PositionFillReportResult {
6259            position_result: fresh_result,
6260            reports: IndexMap::from([(key, Vec::new())]),
6261        });
6262
6263        let cache = node.kernel.cache.borrow();
6264        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6265        assert_eq!(
6266            positions
6267                .iter()
6268                .map(|position| position.quantity)
6269                .sum::<Quantity>(),
6270            Quantity::from("2.0")
6271        );
6272        assert_eq!(
6273            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6274            2
6275        );
6276    }
6277
6278    #[rstest]
6279    #[case::below_limit(63)]
6280    #[case::at_limit(64)]
6281    #[case::above_limit(65)]
6282    fn test_position_fill_dispatch_limit_resumes_without_duplicate_economics(#[case] count: usize) {
6283        let (mut node, venue_report, fill_report) =
6284            position_fill_test_fixture("PositionFillLimitNode", Quantity::from("0.1"));
6285        let expected_quantity = dec!(1) + Decimal::new(i64::try_from(count).unwrap(), 1);
6286
6287        let venue_report = PositionStatusReport::new(
6288            venue_report.account_id,
6289            venue_report.instrument_id,
6290            PositionSide::Long,
6291            Quantity::from_decimal_dp(expected_quantity, 1).unwrap(),
6292            venue_report.ts_last,
6293            venue_report.ts_init,
6294            None,
6295            None,
6296            venue_report.avg_px_open,
6297        );
6298        let key = (venue_report.instrument_id, venue_report.account_id);
6299        let client_order_id = fill_report.client_order_id.unwrap();
6300
6301        let reports = (0..count)
6302            .map(|index| {
6303                let mut report = fill_report.clone();
6304                report.trade_id = TradeId::from(format!("T-LIMIT-{index:03}"));
6305                report.ts_event = UnixNanos::from(1_000 + index as u64);
6306                report
6307            })
6308            .collect::<Vec<_>>();
6309
6310        let position_result = position_report_result(&node, venue_report.clone());
6311
6312        node.handle_position_fill_report_result(PositionFillReportResult {
6313            position_result,
6314            reports: IndexMap::from([(key, reports.clone())]),
6315        });
6316
6317        {
6318            let cache = node.kernel.cache.borrow();
6319            let order = cache.order(&client_order_id).unwrap();
6320            let applied = count.min(64);
6321            assert_eq!(
6322                order.filled_qty().as_decimal(),
6323                dec!(1) + Decimal::new(i64::try_from(applied).unwrap(), 1)
6324            );
6325            assert_eq!(order.trade_ids().len(), applied + 1);
6326
6327            for (index, report) in reports.iter().enumerate() {
6328                assert_eq!(
6329                    order.trade_ids().contains(&&report.trade_id),
6330                    index < applied
6331                );
6332            }
6333
6334            assert_eq!(
6335                cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6336                1
6337            );
6338        }
6339
6340        // A fresh cycle replays the full response; only the deferred fill may change exposure
6341        let position_result = position_report_result(&node, venue_report);
6342        node.handle_position_fill_report_result(PositionFillReportResult {
6343            position_result,
6344            reports: IndexMap::from([(key, reports)]),
6345        });
6346
6347        let cache = node.kernel.cache.borrow();
6348        let order = cache.order(&client_order_id).unwrap();
6349        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6350        assert_eq!(order.filled_qty().as_decimal(), expected_quantity);
6351        assert_eq!(order.trade_ids().len(), count + 1);
6352        assert_eq!(positions.len(), 1);
6353        assert_eq!(positions[0].quantity.as_decimal(), expected_quantity);
6354        assert_eq!(positions[0].side, PositionSide::Long);
6355        assert_eq!(
6356            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6357            1
6358        );
6359    }
6360
6361    #[rstest]
6362    #[case::client_order(0)]
6363    #[case::venue_order(1)]
6364    #[case::side(2)]
6365    #[case::position(3)]
6366    fn test_position_fill_conflict_blocks_remaining_fills_and_fallback(#[case] conflict: u8) {
6367        let (mut node, venue_report, valid_report) =
6368            position_fill_test_fixture("ConflictingPositionFillNode", Quantity::from("1.0"));
6369        let key = (venue_report.instrument_id, venue_report.account_id);
6370        let client_order_id = valid_report.client_order_id.unwrap();
6371        let revision = node.exec_manager.position_activity_revision(&key);
6372        let position_result = position_report_result(&node, venue_report);
6373        let mut conflicting_report = valid_report.clone();
6374        conflicting_report.trade_id = TradeId::from("T-CONFLICTING-FILL");
6375
6376        match conflict {
6377            0 => conflicting_report.client_order_id = Some(ClientOrderId::from("O-CONFLICT")),
6378            1 => conflicting_report.venue_order_id = VenueOrderId::from("V-CONFLICT"),
6379            2 => conflicting_report.order_side = OrderSide::Sell,
6380            3 => conflicting_report.venue_position_id = Some(PositionId::from("P-CONFLICT")),
6381            _ => unreachable!(),
6382        }
6383
6384        node.handle_position_fill_report_result(PositionFillReportResult {
6385            position_result,
6386            reports: IndexMap::from([(key, vec![conflicting_report, valid_report])]),
6387        });
6388
6389        let cache = node.kernel.cache.borrow();
6390        let order = cache.order(&client_order_id).unwrap();
6391        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6392        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6393        assert_eq!(
6394            order.trade_ids(),
6395            vec![&TradeId::from("T-POSITION-INITIAL")]
6396        );
6397        assert_eq!(positions.len(), 1);
6398        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6399        assert_eq!(
6400            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6401            1
6402        );
6403        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
6404    }
6405
6406    #[rstest]
6407    fn test_position_fill_report_failure_does_not_trigger_synthetic_fallback() {
6408        let (mut node, venue_report, _) =
6409            position_fill_test_fixture("FailedPositionFillNode", Quantity::from("1.0"));
6410        let key = (venue_report.instrument_id, venue_report.account_id);
6411        let position_result = position_report_result(&node, venue_report);
6412
6413        node.handle_position_fill_report_result(PositionFillReportResult {
6414            position_result,
6415            reports: IndexMap::new(),
6416        });
6417
6418        let cache = node.kernel.cache.borrow();
6419        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6420        assert_eq!(positions.len(), 1);
6421        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6422        assert_eq!(
6423            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6424            1
6425        );
6426    }
6427
6428    #[rstest]
6429    fn test_position_fill_report_result_falls_back_for_unattributable_hedge_fill() {
6430        let (mut node, mut venue_report, mut fill_report) =
6431            position_fill_test_fixture("UnattributedHedgeFillNode", Quantity::from("1.0"));
6432        node.kernel
6433            .exec_engine
6434            .borrow_mut()
6435            .register_oms_type(StrategyId::from("EXTERNAL"), OmsType::Hedging);
6436        let key = (venue_report.instrument_id, venue_report.account_id);
6437
6438        let position_id = {
6439            let cache = node.kernel.cache.borrow();
6440            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6441            assert_eq!(positions.len(), 1);
6442            positions[0].id
6443        };
6444
6445        venue_report.venue_position_id = Some(position_id);
6446        fill_report.client_order_id = None;
6447        fill_report.venue_order_id = VenueOrderId::from("V-POSITION-EXTERNAL");
6448        let position_result = position_report_result(&node, venue_report);
6449
6450        node.handle_position_fill_report_result(PositionFillReportResult {
6451            position_result,
6452            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6453        });
6454
6455        let cache = node.kernel.cache.borrow();
6456        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6457        assert_eq!(positions.len(), 1);
6458        assert_eq!(positions[0].id, position_id);
6459        assert_eq!(positions[0].quantity, Quantity::from("2.0"));
6460        assert_eq!(
6461            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6462            2
6463        );
6464        drop(positions);
6465        drop(cache);
6466        assert!(
6467            !node
6468                .exec_manager
6469                .position_contains_fill_report(&fill_report)
6470        );
6471    }
6472
6473    #[rstest]
6474    fn test_bundled_fill_report_invalidates_prepared_position_reconciliation() {
6475        let (mut node, venue_report, fill_report) =
6476            position_fill_test_fixture("BundledPositionActivityNode", Quantity::from("1.0"));
6477        let key = (venue_report.instrument_id, venue_report.account_id);
6478        let client_order_id = fill_report.client_order_id.unwrap();
6479        let position_result = position_report_result(&node, venue_report);
6480        let revision = node.exec_manager.position_activity_revision(&key);
6481
6482        let order_report = OrderStatusReport::new(
6483            key.1,
6484            key.0,
6485            Some(client_order_id),
6486            fill_report.venue_order_id,
6487            OrderSide::Buy.into(),
6488            OrderType::Limit,
6489            TimeInForce::Gtc,
6490            OrderStatus::PartiallyFilled,
6491            Quantity::from("10.0"),
6492            Quantity::from("2.0"),
6493            UnixNanos::from(1_000),
6494            UnixNanos::from(1_000),
6495            UnixNanos::from(2_000),
6496            None,
6497        );
6498        let event = ExecutionEvent::Report(ExecutionReport::OrderWithFills(
6499            Box::new(order_report),
6500            vec![fill_report.clone()],
6501        ));
6502
6503        assert_eq!(
6504            node.observe_exec_event_before_dispatch(&event),
6505            Some(vec![client_order_id])
6506        );
6507
6508        assert_eq!(
6509            node.exec_manager.position_activity_revision(&key),
6510            revision + 1
6511        );
6512        assert!(
6513            !node
6514                .exec_manager
6515                .position_report_check_is_current(&position_result.check, &key)
6516        );
6517
6518        node.handle_position_fill_report_result(PositionFillReportResult {
6519            position_result,
6520            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6521        });
6522
6523        let cache = node.kernel.cache.borrow();
6524        let order = cache.order(&client_order_id).unwrap();
6525        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6526        assert_eq!(order.filled_qty(), Quantity::from("1.0"));
6527        assert!(!order.trade_ids().contains(&&fill_report.trade_id));
6528        assert_eq!(positions.len(), 1);
6529        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6530        assert_eq!(positions[0].side, PositionSide::Long);
6531        assert_eq!(
6532            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6533            1
6534        );
6535    }
6536
6537    #[rstest]
6538    fn test_position_fill_report_result_defers_after_local_position_activity() {
6539        let (mut node, venue_report, fill_report) =
6540            position_fill_test_fixture("StalePositionFillNode", Quantity::from("1.0"));
6541        let key = (venue_report.instrument_id, venue_report.account_id);
6542        let position_result = position_report_result(&node, venue_report);
6543        node.exec_manager.record_position_activity(key.0, key.1);
6544
6545        node.handle_position_fill_report_result(PositionFillReportResult {
6546            position_result,
6547            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
6548        });
6549
6550        let cache = node.kernel.cache.borrow();
6551        let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
6552        assert_eq!(positions.len(), 1);
6553        assert_eq!(positions[0].quantity, Quantity::from("1.0"));
6554        assert_eq!(
6555            cache.orders_total_count(None, Some(&key.0), None, Some(&key.1), None),
6556            1
6557        );
6558        drop(positions);
6559        drop(cache);
6560        assert!(
6561            !node
6562                .exec_manager
6563                .position_contains_fill_report(&fill_report)
6564        );
6565    }
6566
6567    #[rstest]
6568    fn test_observe_exec_event_before_dispatch_accepted_batch_stamps_local_activity() {
6569        let config = LiveNodeConfig {
6570            exec_engine: crate::config::LiveExecutionEngineConfig {
6571                reconciliation: true,
6572                open_check_threshold_ms: 5_000,
6573                single_order_query_delay_ms: 0,
6574                ..Default::default()
6575            },
6576            ..Default::default()
6577        };
6578
6579        let mut node = LiveNode::build("AcceptedBatchNode".to_string(), Some(config)).unwrap();
6580        let account_id = AccountId::from("TEST-ACCEPTED-BATCH-001");
6581        let client_id = ClientId::from("TEST-ACCEPTED-BATCH");
6582        let instrument = crypto_perpetual_ethusdt();
6583        let instrument_id = instrument.id();
6584        let client_order_id = ClientOrderId::from("O-ACCEPTED-BATCH");
6585        let venue_order_id = VenueOrderId::from("V-ACCEPTED-BATCH");
6586
6587        node.kernel
6588            .cache
6589            .borrow_mut()
6590            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6591            .unwrap();
6592        insert_accepted_limit_order_in_node(
6593            &node,
6594            account_id,
6595            client_id,
6596            instrument_id,
6597            client_order_id,
6598            venue_order_id,
6599        );
6600
6601        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
6602
6603        let accepted = OrderAcceptedSpec::builder()
6604            .instrument_id(instrument_id)
6605            .client_order_id(client_order_id)
6606            .venue_order_id(venue_order_id)
6607            .account_id(account_id)
6608            .build();
6609        let event = ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![accepted]));
6610
6611        let close_ids = node.observe_exec_event_before_dispatch(&event);
6612
6613        assert_eq!(close_ids, Some(vec![client_order_id]));
6614        assert!(node.exec_manager.check_open_order_queries().is_empty());
6615    }
6616
6617    #[rstest]
6618    #[cfg_attr(
6619        not(all(feature = "simulation", madsim)),
6620        tokio::test(start_paused = true)
6621    )]
6622    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6623    async fn test_batch_cancel_command_registers_each_child_for_inflight_timeout() {
6624        use nautilus_common::messages::execution::{BatchCancelOrders, CancelOrder};
6625        use nautilus_model::{
6626            events::order::spec::OrderPendingCancelSpec, identifiers::ClientOrderId,
6627        };
6628
6629        let config = LiveNodeConfig {
6630            exec_engine: crate::config::LiveExecutionEngineConfig {
6631                reconciliation: true,
6632                inflight_check_threshold_ms: 100,
6633                inflight_check_retries: 1,
6634                ..Default::default()
6635            },
6636            ..Default::default()
6637        };
6638
6639        let mut node = LiveNode::build("BatchCancelNode".to_string(), Some(config)).unwrap();
6640        let trader_id = TraderId::from("TESTER-001");
6641        let strategy_id = StrategyId::from("S-BATCH-CANCEL");
6642        let account_id = AccountId::from("TEST-001");
6643        let instrument = crypto_perpetual_ethusdt();
6644        let instrument_id = instrument.id();
6645        node.kernel
6646            .exec_engine
6647            .borrow_mut()
6648            .register_client(Box::new(StubExecutionClient::new(
6649                ClientId::from("STUB"),
6650                account_id,
6651                instrument_id.venue,
6652                OmsType::Netting,
6653                None,
6654            )))
6655            .unwrap();
6656        node.kernel
6657            .exec_engine
6658            .borrow_mut()
6659            .set_default_client(ClientId::from("STUB"))
6660            .unwrap();
6661        let child_ids = [
6662            ClientOrderId::from("O-BATCH-CANCEL-1"),
6663            ClientOrderId::from("O-BATCH-CANCEL-2"),
6664        ];
6665        node.kernel
6666            .cache
6667            .borrow_mut()
6668            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6669            .unwrap();
6670
6671        for client_order_id in child_ids {
6672            let order = OrderTestBuilder::new(OrderType::Limit)
6673                .trader_id(trader_id)
6674                .strategy_id(strategy_id)
6675                .client_order_id(client_order_id)
6676                .instrument_id(instrument_id)
6677                .quantity(Quantity::from("10.0"))
6678                .price(Price::from("100.0"))
6679                .build();
6680            let venue_order_id = VenueOrderId::from(format!("V-{client_order_id}").as_str());
6681            // Model the production batch-cancel path: each child is accepted, then
6682            // moved to PendingCancel, so an inflight timeout must emit a Canceled
6683            // event (not a Submitted-order rejection).
6684            let submitted = TestOrderEventStubs::submitted(&order, account_id);
6685            let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
6686            let pending_cancel = OrderEventAny::PendingCancel(
6687                OrderPendingCancelSpec::builder()
6688                    .trader_id(trader_id)
6689                    .strategy_id(strategy_id)
6690                    .instrument_id(instrument_id)
6691                    .client_order_id(client_order_id)
6692                    .account_id(account_id)
6693                    .venue_order_id(venue_order_id)
6694                    .build(),
6695            );
6696            let mut cache = node.kernel.cache.borrow_mut();
6697            cache.add_order(order, None, None, false).unwrap();
6698            cache.update_order(&submitted).unwrap();
6699            cache.update_order(&accepted).unwrap();
6700            cache.update_order(&pending_cancel).unwrap();
6701        }
6702
6703        let cancels = child_ids
6704            .into_iter()
6705            .map(|client_order_id| {
6706                CancelOrder::new(
6707                    trader_id,
6708                    None,
6709                    strategy_id,
6710                    instrument_id,
6711                    client_order_id,
6712                    None,
6713                    UUID4::new(),
6714                    UnixNanos::default(),
6715                    None,
6716                    None,
6717                )
6718            })
6719            .collect();
6720
6721        let command = TradingCommand::CancelOrders(BatchCancelOrders::new(
6722            trader_id,
6723            None,
6724            strategy_id,
6725            instrument_id,
6726            cancels,
6727            UUID4::new(),
6728            UnixNanos::default(),
6729            None,
6730            None,
6731        ));
6732
6733        node.observe_exec_command_before_dispatch(&command);
6734        advance_clock(Duration::from_millis(101)).await;
6735        let result = node.exec_manager.check_inflight_orders();
6736        let timed_out_ids = result
6737            .events
6738            .iter()
6739            .map(OrderEventAny::client_order_id)
6740            .collect::<IndexSet<_>>();
6741
6742        assert_eq!(timed_out_ids, IndexSet::from(child_ids));
6743        assert_eq!(result.events.len(), child_ids.len());
6744        assert!(
6745            result
6746                .events
6747                .iter()
6748                .all(|event| matches!(event, OrderEventAny::CancelRejected(_))),
6749            "batch-cancel children must time out as CancelRejected events",
6750        );
6751    }
6752
6753    #[rstest]
6754    #[cfg_attr(
6755        not(all(feature = "simulation", madsim)),
6756        tokio::test(start_paused = true)
6757    )]
6758    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6759    async fn test_risk_bound_command_does_not_register_inflight() {
6760        let config = LiveNodeConfig {
6761            exec_engine: crate::config::LiveExecutionEngineConfig {
6762                reconciliation: true,
6763                inflight_check_threshold_ms: 100,
6764                inflight_check_retries: 2,
6765                ..Default::default()
6766            },
6767            ..Default::default()
6768        };
6769
6770        let mut node = LiveNode::build("RiskBoundNode".to_string(), Some(config)).unwrap();
6771        msgbus::register_trading_command_endpoint(
6772            MessagingSwitchboard::risk_engine_execute(),
6773            TypedIntoHandler::from(|_: TradingCommand| {}),
6774        );
6775        let instrument = crypto_perpetual_ethusdt();
6776        let instrument_id = instrument.id();
6777        let order = OrderTestBuilder::new(OrderType::Limit)
6778            .trader_id(node.trader_id())
6779            .strategy_id(StrategyId::from("S-RISK-DENIED"))
6780            .instrument_id(instrument_id)
6781            .side(OrderSide::Buy)
6782            .quantity(Quantity::from("1.000"))
6783            .price(Price::from("100.00"))
6784            .build();
6785        let client_order_id = order.client_order_id();
6786
6787        {
6788            let mut cache = node.kernel.cache.borrow_mut();
6789            cache
6790                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
6791                .unwrap();
6792            cache.add_order(order.clone(), None, None, false).unwrap();
6793        }
6794
6795        let submit_order = SubmitOrder::new(
6796            order.trader_id(),
6797            None,
6798            order.strategy_id(),
6799            instrument_id,
6800            client_order_id,
6801            order.init_event().clone(),
6802            None,
6803            None,
6804            None,
6805            UUID4::new(),
6806            UnixNanos::default(),
6807            None,
6808        );
6809        node.process_exec_command(
6810            TradingCommandMessage::new(
6811                MessagingSwitchboard::risk_engine_execute(),
6812                TradingCommand::SubmitOrder(submit_order),
6813            )
6814            .into(),
6815        );
6816
6817        advance_clock(Duration::from_millis(101)).await;
6818        let result = node.exec_manager.check_inflight_orders();
6819        let status = node
6820            .kernel
6821            .cache
6822            .borrow()
6823            .order(&client_order_id)
6824            .unwrap()
6825            .status();
6826
6827        assert_eq!(status, OrderStatus::Initialized);
6828        assert_eq!(
6829            node.exec_manager.recon_check_retry_count(&client_order_id),
6830            0
6831        );
6832        assert!(result.events.is_empty());
6833        assert!(result.queries.is_empty());
6834    }
6835
6836    #[rstest]
6837    #[cfg_attr(
6838        not(all(feature = "simulation", madsim)),
6839        tokio::test(start_paused = true)
6840    )]
6841    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6842    async fn test_external_cancel_command_still_registers_inflight() {
6843        use nautilus_common::messages::execution::CancelOrder;
6844        let external = ClientId::from("EXTERNAL");
6845
6846        let config = LiveNodeConfig {
6847            exec_engine: crate::config::LiveExecutionEngineConfig {
6848                reconciliation: true,
6849                inflight_check_threshold_ms: 100,
6850                inflight_check_retries: 2,
6851                external_clients: Some(vec![external]),
6852                ..Default::default()
6853            },
6854            ..Default::default()
6855        };
6856
6857        let mut node = LiveNode::build("ExternalCancelNode".to_string(), Some(config)).unwrap();
6858        let instrument = crypto_perpetual_ethusdt();
6859        let mut order = OrderTestBuilder::new(OrderType::Limit)
6860            .trader_id(node.trader_id())
6861            .strategy_id(StrategyId::from("S-EXTERNAL"))
6862            .instrument_id(instrument.id())
6863            .quantity(Quantity::from("9.003"))
6864            .price(Price::from("113.47"))
6865            .build();
6866        let venue = VenueOrderId::from("V-EXTERNAL");
6867        order
6868            .apply(TestOrderEventStubs::accepted(
6869                &order,
6870                AccountId::from("SIM-001"),
6871                venue,
6872            ))
6873            .unwrap();
6874        let id = order.client_order_id();
6875        let command = TradingCommand::CancelOrder(CancelOrder::new(
6876            order.trader_id(),
6877            Some(external),
6878            order.strategy_id(),
6879            order.instrument_id(),
6880            id,
6881            Some(venue),
6882            UUID4::new(),
6883            17.into(),
6884            None,
6885            None,
6886        ));
6887        {
6888            let mut cache = node.kernel.cache.borrow_mut();
6889            cache.add_instrument(instrument.into()).unwrap();
6890            cache
6891                .add_order(order.clone(), None, Some(external), false)
6892                .unwrap();
6893        }
6894
6895        node.process_exec_command(
6896            TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
6897        );
6898        advance_clock(Duration::from_millis(101)).await;
6899        let result = node.exec_manager.check_inflight_orders();
6900
6901        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
6902            panic!("expected external query");
6903        };
6904
6905        assert_eq!(
6906            (query.client_order_id, query.venue_order_id, query.client_id),
6907            (id, Some(venue), Some(external))
6908        );
6909        assert!(result.events.is_empty());
6910        assert_eq!(
6911            serde_json::to_value(node.kernel.cache.borrow().order(&id).unwrap().clone()).unwrap(),
6912            serde_json::to_value(&order).unwrap()
6913        );
6914    }
6915
6916    #[rstest]
6917    #[case::modify(false, false)]
6918    #[case::batch_modify(false, true)]
6919    #[case::cancel(true, false)]
6920    #[case::batch_cancel(true, true)]
6921    #[cfg_attr(
6922        not(all(feature = "simulation", madsim)),
6923        tokio::test(start_paused = true)
6924    )]
6925    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
6926    async fn test_native_command_denial_does_not_leave_inflight_tracking(
6927        #[case] cancel: bool,
6928        #[case] batch: bool,
6929        #[values(false, true)] old_pending: bool,
6930    ) {
6931        use nautilus_common::messages::execution::{
6932            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
6933        };
6934        use nautilus_model::events::order::spec::{OrderModifyDeniedSpec, OrderPendingUpdateSpec};
6935
6936        let config = LiveNodeConfig {
6937            exec_engine: crate::config::LiveExecutionEngineConfig {
6938                reconciliation: true,
6939                inflight_check_threshold_ms: 100,
6940                inflight_check_retries: 2,
6941                ..Default::default()
6942            },
6943            ..Default::default()
6944        };
6945
6946        let mut node = LiveNode::build("LocalDenialNode".to_string(), Some(config)).unwrap();
6947        let instrument = crypto_perpetual_ethusdt();
6948        let mut order = OrderTestBuilder::new(OrderType::Limit)
6949            .trader_id(node.trader_id())
6950            .strategy_id(StrategyId::from("S-LOCAL-DENIED"))
6951            .instrument_id(instrument.id())
6952            .side(OrderSide::Buy)
6953            .quantity(Quantity::from("9.003"))
6954            .price(Price::from("113.47"))
6955            .build();
6956        let account_id = AccountId::from("SIM-001");
6957        let venue_id = VenueOrderId::from("V-LOCAL-DENIED");
6958        order
6959            .apply(TestOrderEventStubs::accepted(&order, account_id, venue_id))
6960            .unwrap();
6961        let id = order.client_order_id();
6962        let old_cause = UUID4::new();
6963
6964        if old_pending {
6965            let mut marker = OrderPendingUpdateSpec::builder()
6966                .trader_id(order.trader_id())
6967                .strategy_id(order.strategy_id())
6968                .instrument_id(order.instrument_id())
6969                .client_order_id(id)
6970                .venue_order_id(venue_id)
6971                .account_id(account_id)
6972                .build();
6973            marker.causation_id = Some(old_cause);
6974            order.apply(OrderEventAny::PendingUpdate(marker)).unwrap();
6975            node.exec_manager.register_inflight(id, old_cause);
6976        }
6977
6978        let expected_status = order.status();
6979        let events_before = order.event_count();
6980        {
6981            let mut cache = node.kernel.cache.borrow_mut();
6982            cache.add_instrument(instrument.into()).unwrap();
6983            cache.add_order(order.clone(), None, None, false).unwrap();
6984        }
6985
6986        let cause = UUID4::new();
6987
6988        let command = if cancel {
6989            let command = CancelOrder::new(
6990                order.trader_id(),
6991                None,
6992                order.strategy_id(),
6993                order.instrument_id(),
6994                id,
6995                Some(venue_id),
6996                cause,
6997                17.into(),
6998                None,
6999                None,
7000            );
7001
7002            if batch {
7003                TradingCommand::CancelOrders(BatchCancelOrders::new(
7004                    order.trader_id(),
7005                    None,
7006                    order.strategy_id(),
7007                    order.instrument_id(),
7008                    vec![command],
7009                    UUID4::new(),
7010                    19.into(),
7011                    None,
7012                    None,
7013                ))
7014            } else {
7015                TradingCommand::CancelOrder(command)
7016            }
7017        } else {
7018            let command = ModifyOrder::new(
7019                order.trader_id(),
7020                None,
7021                order.strategy_id(),
7022                order.instrument_id(),
7023                id,
7024                Some(venue_id),
7025                Some(Quantity::from("8.001")),
7026                None,
7027                None,
7028                cause,
7029                17.into(),
7030                None,
7031                None,
7032            );
7033
7034            if batch {
7035                TradingCommand::ModifyOrders(BatchModifyOrders::new(
7036                    order.trader_id(),
7037                    None,
7038                    order.strategy_id(),
7039                    order.instrument_id(),
7040                    vec![command],
7041                    UUID4::new(),
7042                    19.into(),
7043                    None,
7044                    None,
7045                ))
7046            } else {
7047                TradingCommand::ModifyOrder(command)
7048            }
7049        };
7050
7051        node.process_exec_command(
7052            TradingCommandMessage::new(MessagingSwitchboard::exec_engine_execute(), command).into(),
7053        );
7054        {
7055            let cache = node.kernel.cache.borrow();
7056            let cached = cache.order(&id).unwrap();
7057            match cached.last_event() {
7058                OrderEventAny::ModifyDenied(event) if !cancel => {
7059                    assert_eq!(event.causation_id, Some(cause));
7060                }
7061                OrderEventAny::CancelDenied(event) if cancel => {
7062                    assert_eq!(event.causation_id, Some(cause));
7063                }
7064                other => panic!("expected native command denial, was {other:?}"),
7065            }
7066
7067            assert_eq!(cached.status(), expected_status);
7068            assert_eq!(cached.quantity(), order.quantity());
7069            assert_eq!(cached.price(), order.price());
7070            assert_eq!(cached.filled_qty(), order.filled_qty());
7071            assert_eq!(cached.event_count(), events_before + 1);
7072        }
7073
7074        advance_clock(Duration::from_millis(101)).await;
7075        let result = node.exec_manager.check_inflight_orders();
7076        assert_eq!(result.queries.len(), usize::from(old_pending));
7077        assert!(result.events.is_empty());
7078
7079        if old_pending {
7080            let refusal = OrderModifyDeniedSpec::builder()
7081                .trader_id(order.trader_id())
7082                .strategy_id(order.strategy_id())
7083                .instrument_id(order.instrument_id())
7084                .client_order_id(id)
7085                .venue_order_id(venue_id)
7086                .account_id(account_id)
7087                .causation_id(old_cause)
7088                .build();
7089            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::ModifyDenied(refusal)));
7090        }
7091
7092        assert_eq!(node.exec_manager.recon_check_retry_count(&id), 0);
7093    }
7094
7095    #[rstest]
7096    #[cfg_attr(
7097        not(all(feature = "simulation", madsim)),
7098        tokio::test(start_paused = true)
7099    )]
7100    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7101    async fn test_risk_approved_command_registers_inflight() {
7102        let config = LiveNodeConfig {
7103            risk_engine: crate::config::LiveRiskEngineConfig {
7104                bypass: true,
7105                ..Default::default()
7106            },
7107            exec_engine: crate::config::LiveExecutionEngineConfig {
7108                reconciliation: true,
7109                inflight_check_threshold_ms: 100,
7110                inflight_check_retries: 2,
7111                ..Default::default()
7112            },
7113            ..Default::default()
7114        };
7115
7116        let mut node = LiveNode::build("RiskApprovedNode".to_string(), Some(config)).unwrap();
7117        msgbus::register_trading_command_endpoint(
7118            MessagingSwitchboard::exec_engine_execute(),
7119            TypedIntoHandler::from(|_: TradingCommand| {}),
7120        );
7121        let instrument = crypto_perpetual_ethusdt();
7122        let instrument_id = instrument.id();
7123        let order = OrderTestBuilder::new(OrderType::Limit)
7124            .trader_id(node.trader_id())
7125            .strategy_id(StrategyId::from("S-RISK-APPROVED"))
7126            .instrument_id(instrument_id)
7127            .side(OrderSide::Buy)
7128            .quantity(Quantity::from("1.000"))
7129            .price(Price::from("100.00"))
7130            .build();
7131        let client_order_id = order.client_order_id();
7132
7133        {
7134            let mut cache = node.kernel.cache.borrow_mut();
7135            cache
7136                .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7137                .unwrap();
7138            cache.add_order(order.clone(), None, None, false).unwrap();
7139        }
7140
7141        let submit_order = SubmitOrder::new(
7142            order.trader_id(),
7143            None,
7144            order.strategy_id(),
7145            instrument_id,
7146            client_order_id,
7147            order.init_event().clone(),
7148            None,
7149            None,
7150            None,
7151            UUID4::new(),
7152            UnixNanos::default(),
7153            None,
7154        );
7155        node.process_exec_command(
7156            TradingCommandMessage::new(
7157                MessagingSwitchboard::risk_engine_execute(),
7158                TradingCommand::SubmitOrder(submit_order),
7159            )
7160            .into(),
7161        );
7162
7163        advance_clock(Duration::from_millis(101)).await;
7164        let result = node.exec_manager.check_inflight_orders();
7165
7166        let [TradingCommand::QueryOrder(query)] = result.queries.as_slice() else {
7167            panic!("expected one query order command");
7168        };
7169
7170        assert_eq!(query.client_order_id, client_order_id);
7171        assert_eq!(
7172            node.exec_manager.recon_check_retry_count(&client_order_id),
7173            1
7174        );
7175        assert!(result.events.is_empty());
7176    }
7177
7178    #[rstest]
7179    fn test_live_node_builder_clock_factory_drives_kernel_clock() {
7180        let calls = Rc::new(Cell::new(0usize));
7181        let calls_in_factory = calls.clone();
7182        let sentinel = UnixNanos::from(123_456_789_u64);
7183
7184        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7185            .unwrap()
7186            .with_reconciliation(false)
7187            .with_clock_factory(move || {
7188                calls_in_factory.set(calls_in_factory.get() + 1);
7189                let mut clock = VirtualClock::new();
7190                clock.advance_time(sentinel, true);
7191                Rc::new(RefCell::new(clock)) as Rc<RefCell<dyn Clock>>
7192            })
7193            .build()
7194            .unwrap();
7195
7196        assert_eq!(node.kernel().clock().borrow().timestamp_ns(), sentinel);
7197        assert_eq!(calls.get(), 1);
7198    }
7199
7200    #[derive(Debug)]
7201    struct ReplayKernelEventStore {
7202        fail_restore: bool,
7203    }
7204
7205    impl KernelEventStore for ReplayKernelEventStore {
7206        fn restore_parent_cache(
7207            &mut self,
7208            _instance_id: UUID4,
7209            _cache: &mut Cache,
7210        ) -> anyhow::Result<()> {
7211            if self.fail_restore {
7212                anyhow::bail!("replay restore failed");
7213            }
7214
7215            Ok(())
7216        }
7217
7218        fn open(
7219            &mut self,
7220            _instance_id: UUID4,
7221            _components: &RegisteredComponents,
7222            _environment: Environment,
7223        ) -> anyhow::Result<()> {
7224            Ok(())
7225        }
7226
7227        fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
7228            None
7229        }
7230
7231        fn seal(&mut self, _ts_init: UnixNanos) {}
7232
7233        fn run_id(&self) -> Option<&str> {
7234            Some("replay-child")
7235        }
7236
7237        fn parent_run_id(&self) -> Option<&str> {
7238            Some("seed-run")
7239        }
7240
7241        fn is_event_store_replay_configured(&self) -> bool {
7242            true
7243        }
7244
7245        fn is_halted(&self) -> bool {
7246            false
7247        }
7248    }
7249
7250    #[derive(Debug)]
7251    struct TestStrategy {
7252        core: StrategyCore,
7253    }
7254
7255    impl TestStrategy {
7256        fn new(config: StrategyConfig) -> Self {
7257            Self {
7258                core: StrategyCore::new(config),
7259            }
7260        }
7261    }
7262
7263    impl DataActor for TestStrategy {}
7264
7265    nautilus_strategy!(TestStrategy, {
7266        fn external_order_instrument_ids(&self) -> Option<Vec<InstrumentId>> {
7267            self.core.config.external_order_instrument_ids.clone()
7268        }
7269    });
7270
7271    fn live_node_with_replay_store(fail_restore: bool) -> LiveNode {
7272        // load_state must be true: the kernel rejects event-store replay otherwise,
7273        // and LiveNodeConfig defaults it to false.
7274        let builder = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
7275            .unwrap()
7276            .with_exec_engine_config(crate::config::LiveExecutionEngineConfig {
7277                reconciliation: false,
7278                ..Default::default()
7279            })
7280            .with_load_state(true)
7281            .with_name("TestKernel")
7282            .with_event_store(move |_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
7283                Ok(Box::new(ReplayKernelEventStore { fail_restore }) as Box<dyn KernelEventStore>)
7284            });
7285
7286        builder.build().unwrap()
7287    }
7288
7289    #[rstest]
7290    fn test_add_strategy_registers_external_order_claims_with_manager_and_engine() {
7291        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7292            .unwrap()
7293            .with_reconciliation(false)
7294            .with_delay_post_stop_secs(0)
7295            .with_timeout_connection(1)
7296            .build()
7297            .unwrap();
7298        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7299        let strategy_id = StrategyId::from("CLAIMS-001");
7300
7301        node.add_strategy(TestStrategy::new(StrategyConfig {
7302            strategy_id: Some(strategy_id),
7303            external_order_instrument_ids: Some(vec![instrument_id]),
7304            ..Default::default()
7305        }))
7306        .unwrap();
7307
7308        assert_eq!(
7309            node.exec_manager.get_external_order_claim(&instrument_id),
7310            Some(strategy_id)
7311        );
7312
7313        {
7314            let exec_engine = node.kernel().exec_engine.borrow();
7315            assert_eq!(
7316                exec_engine.get_external_order_claim(&instrument_id),
7317                Some(strategy_id)
7318            );
7319        }
7320    }
7321
7322    #[rstest]
7323    fn test_register_external_order_claims_after_build_is_visible_to_manager_and_engine() {
7324        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7325            .unwrap()
7326            .with_reconciliation(false)
7327            .build()
7328            .unwrap();
7329        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7330        let strategy_id = StrategyId::from("CLAIMS-001");
7331
7332        node.register_external_order_claims(strategy_id, &[instrument_id])
7333            .unwrap();
7334
7335        assert_eq!(
7336            node.exec_manager.get_external_order_claim(&instrument_id),
7337            Some(strategy_id)
7338        );
7339        assert_eq!(
7340            node.kernel
7341                .exec_engine
7342                .borrow()
7343                .get_external_order_claim(&instrument_id),
7344            Some(strategy_id)
7345        );
7346    }
7347
7348    #[rstest]
7349    #[tokio::test]
7350    async fn test_register_external_order_claims_while_running_is_visible_to_manager_and_engine() {
7351        let mut node = live_node_with_replay_store(false);
7352        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7353        let strategy_id = StrategyId::from("CLAIMS-001");
7354
7355        node.start().await.unwrap();
7356        assert_eq!(node.state(), NodeState::Running);
7357
7358        node.register_external_order_claims(strategy_id, &[instrument_id])
7359            .unwrap();
7360
7361        assert_eq!(
7362            node.exec_manager.get_external_order_claim(&instrument_id),
7363            Some(strategy_id)
7364        );
7365        assert_eq!(
7366            node.kernel
7367                .exec_engine
7368                .borrow()
7369                .get_external_order_claim(&instrument_id),
7370            Some(strategy_id)
7371        );
7372    }
7373
7374    #[rstest]
7375    fn test_register_external_order_claims_conflicting_batch_leaves_new_claims_absent() {
7376        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7377            .unwrap()
7378            .with_reconciliation(false)
7379            .build()
7380            .unwrap();
7381        let existing_instrument = InstrumentId::from("AUDUSD.SIM");
7382        let new_instruments = [
7383            InstrumentId::from("EURUSD.SIM"),
7384            InstrumentId::from("GBPUSD.SIM"),
7385        ];
7386        let existing_strategy_id = StrategyId::from("CLAIMS-001");
7387        let new_strategy_id = StrategyId::from("CLAIMS-002");
7388        node.register_external_order_claims(existing_strategy_id, &[existing_instrument])
7389            .unwrap();
7390
7391        let result = node.register_external_order_claims(
7392            new_strategy_id,
7393            &[new_instruments[0], existing_instrument, new_instruments[1]],
7394        );
7395
7396        assert!(result.is_err());
7397        assert_eq!(
7398            node.exec_manager
7399                .get_external_order_claim(&existing_instrument),
7400            Some(existing_strategy_id)
7401        );
7402
7403        for instrument_id in new_instruments {
7404            assert_eq!(
7405                node.exec_manager.get_external_order_claim(&instrument_id),
7406                None
7407            );
7408            assert_eq!(
7409                node.kernel
7410                    .exec_engine
7411                    .borrow()
7412                    .get_external_order_claim(&instrument_id),
7413                None
7414            );
7415        }
7416    }
7417
7418    #[rstest]
7419    fn test_deregister_external_order_claims_allows_successor_to_claim() {
7420        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7421            .unwrap()
7422            .with_reconciliation(false)
7423            .build()
7424            .unwrap();
7425        let instruments = [
7426            InstrumentId::from("AUDUSD.SIM"),
7427            InstrumentId::from("EURUSD.SIM"),
7428        ];
7429        let first_strategy_id = StrategyId::from("CLAIMS-001");
7430        let successor_strategy_id = StrategyId::from("CLAIMS-002");
7431        node.register_external_order_claims(first_strategy_id, &instruments)
7432            .unwrap();
7433
7434        node.deregister_external_order_claims(first_strategy_id)
7435            .unwrap();
7436        node.register_external_order_claims(successor_strategy_id, &instruments)
7437            .unwrap();
7438
7439        for instrument_id in instruments {
7440            assert_eq!(
7441                node.exec_manager.get_external_order_claim(&instrument_id),
7442                Some(successor_strategy_id)
7443            );
7444            assert_eq!(
7445                node.kernel
7446                    .exec_engine
7447                    .borrow()
7448                    .get_external_order_claim(&instrument_id),
7449                Some(successor_strategy_id)
7450            );
7451        }
7452    }
7453
7454    #[rstest]
7455    fn test_deregister_external_order_claims_without_claims_is_idempotent() {
7456        let node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7457            .unwrap()
7458            .with_reconciliation(false)
7459            .build()
7460            .unwrap();
7461        let strategy_id = StrategyId::from("CLAIMS-001");
7462
7463        node.deregister_external_order_claims(strategy_id).unwrap();
7464        node.deregister_external_order_claims(strategy_id).unwrap();
7465    }
7466
7467    #[rstest]
7468    fn test_add_strategy_rejects_duplicate_external_order_claim_without_overwriting() {
7469        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7470            .unwrap()
7471            .with_reconciliation(false)
7472            .with_delay_post_stop_secs(0)
7473            .with_timeout_connection(1)
7474            .build()
7475            .unwrap();
7476        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7477        let strategy_id = StrategyId::from("CLAIMS-001");
7478        let duplicate_strategy_id = StrategyId::from("OTHER-002");
7479
7480        node.add_strategy(TestStrategy::new(StrategyConfig {
7481            strategy_id: Some(strategy_id),
7482            external_order_instrument_ids: Some(vec![instrument_id]),
7483            ..Default::default()
7484        }))
7485        .unwrap();
7486
7487        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7488            strategy_id: Some(duplicate_strategy_id),
7489            external_order_instrument_ids: Some(vec![instrument_id]),
7490            ..Default::default()
7491        }));
7492
7493        assert!(result.is_err());
7494        assert!(
7495            result
7496                .unwrap_err()
7497                .to_string()
7498                .contains("already exists for CLAIMS-001")
7499        );
7500        assert_eq!(
7501            node.exec_manager.get_external_order_claim(&instrument_id),
7502            Some(strategy_id)
7503        );
7504
7505        {
7506            let exec_engine = node.kernel().exec_engine.borrow();
7507            assert_eq!(
7508                exec_engine.get_external_order_claim(&instrument_id),
7509                Some(strategy_id)
7510            );
7511        }
7512    }
7513
7514    #[rstest]
7515    fn test_add_strategy_rejects_repeated_external_order_claim_without_registering() {
7516        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7517            .unwrap()
7518            .with_reconciliation(false)
7519            .with_delay_post_stop_secs(0)
7520            .with_timeout_connection(1)
7521            .build()
7522            .unwrap();
7523        let instrument_id = InstrumentId::from("AUDUSD.SIM");
7524        let strategy_id = StrategyId::from("CLAIMS-001");
7525
7526        let result = node.add_strategy(TestStrategy::new(StrategyConfig {
7527            strategy_id: Some(strategy_id),
7528            external_order_instrument_ids: Some(vec![instrument_id, instrument_id]),
7529            ..Default::default()
7530        }));
7531
7532        assert!(result.is_err());
7533        assert!(
7534            result
7535                .unwrap_err()
7536                .to_string()
7537                .contains("appears more than once for CLAIMS-001")
7538        );
7539        assert_eq!(
7540            node.exec_manager.get_external_order_claim(&instrument_id),
7541            None
7542        );
7543
7544        {
7545            let exec_engine = node.kernel().exec_engine.borrow();
7546            assert_eq!(exec_engine.get_external_order_claim(&instrument_id), None);
7547        }
7548    }
7549
7550    #[rstest]
7551    fn test_add_strategy_failure_restores_external_order_claims() {
7552        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7553            .unwrap()
7554            .with_reconciliation(false)
7555            .with_delay_post_stop_secs(0)
7556            .with_timeout_connection(1)
7557            .build()
7558            .unwrap();
7559        let existing_instrument_id = InstrumentId::from("AUDUSD.SIM");
7560        let configured_instrument_id = InstrumentId::from("EURUSD.SIM");
7561        let strategy_id = StrategyId::from("CLAIMS-001");
7562        node.register_external_order_claims(strategy_id, &[existing_instrument_id])
7563            .unwrap();
7564
7565        let mut strategy = TestStrategy::new(StrategyConfig {
7566            strategy_id: Some(strategy_id),
7567            external_order_instrument_ids: Some(vec![configured_instrument_id]),
7568            ..Default::default()
7569        });
7570
7571        strategy
7572            .core
7573            .register(
7574                node.trader_id(),
7575                node.kernel.clock(),
7576                node.kernel.cache.clone(),
7577                node.kernel.portfolio.clone(),
7578            )
7579            .unwrap();
7580
7581        let result = node.add_strategy(strategy);
7582
7583        assert!(result.is_err());
7584        assert!(
7585            result
7586                .unwrap_err()
7587                .to_string()
7588                .contains("already registered with trader")
7589        );
7590        assert_eq!(
7591            node.exec_manager
7592                .get_external_order_claim(&existing_instrument_id),
7593            Some(strategy_id)
7594        );
7595        assert_eq!(
7596            node.kernel
7597                .exec_engine
7598                .borrow()
7599                .get_external_order_claim(&existing_instrument_id),
7600            Some(strategy_id)
7601        );
7602        assert_eq!(
7603            node.exec_manager
7604                .get_external_order_claim(&configured_instrument_id),
7605            None
7606        );
7607        assert_eq!(
7608            node.kernel
7609                .exec_engine
7610                .borrow()
7611                .get_external_order_claim(&configured_instrument_id),
7612            None
7613        );
7614    }
7615
7616    #[rstest]
7617    fn test_add_strategy_without_claims_or_oms_type_does_not_require_engine_borrow() {
7618        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7619            .unwrap()
7620            .with_reconciliation(false)
7621            .build()
7622            .unwrap();
7623        let exec_engine = node.kernel.exec_engine.clone();
7624        let _engine_borrow = exec_engine.borrow_mut();
7625
7626        node.add_strategy(TestStrategy::new(StrategyConfig {
7627            strategy_id: Some(StrategyId::from("NOCLAIMS-001")),
7628            ..Default::default()
7629        }))
7630        .unwrap();
7631    }
7632
7633    #[rstest]
7634    fn test_add_strategy_registers_configured_hedging_oms_type() {
7635        let mut node = LiveNode::builder(TraderId::from("TESTER-001"), Environment::Sandbox)
7636            .unwrap()
7637            .with_reconciliation(false)
7638            .with_delay_post_stop_secs(0)
7639            .with_timeout_connection(1)
7640            .build()
7641            .unwrap();
7642        let strategy_id = StrategyId::from("FUNDING_ARBITRAGE-001");
7643
7644        node.add_strategy(TestStrategy::new(StrategyConfig {
7645            strategy_id: Some(strategy_id),
7646            oms_type: Some(OmsType::Hedging),
7647            ..Default::default()
7648        }))
7649        .unwrap();
7650
7651        let instrument = crypto_perpetual_ethusdt();
7652        let instrument_id = instrument.id();
7653        let client_id = ClientId::from("STUB");
7654
7655        node.kernel
7656            .cache
7657            .borrow_mut()
7658            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
7659            .unwrap();
7660        node.kernel
7661            .exec_engine
7662            .borrow_mut()
7663            .register_client(Box::new(StubExecutionClient::new(
7664                client_id,
7665                AccountId::from("TEST-ACCOUNT"),
7666                instrument_id.venue,
7667                OmsType::Netting,
7668                None,
7669            )))
7670            .unwrap();
7671
7672        let order = OrderTestBuilder::new(OrderType::Market)
7673            .trader_id(node.trader_id())
7674            .strategy_id(strategy_id)
7675            .instrument_id(instrument_id)
7676            .quantity(Quantity::from("1.000"))
7677            .build();
7678        let position_id = PositionId::new("CUSTOM-POSITION-001");
7679
7680        node.kernel
7681            .cache
7682            .borrow_mut()
7683            .add_order(order.clone(), Some(position_id), Some(client_id), true)
7684            .unwrap();
7685
7686        let submit_order = SubmitOrder::new(
7687            order.trader_id(),
7688            Some(client_id),
7689            strategy_id,
7690            instrument_id,
7691            order.client_order_id(),
7692            order.init_event().clone(),
7693            order.exec_algorithm_id(),
7694            Some(position_id),
7695            None,
7696            UUID4::new(),
7697            UnixNanos::default(),
7698            None,
7699        );
7700
7701        node.kernel
7702            .exec_engine
7703            .borrow()
7704            .execute(TradingCommand::SubmitOrder(submit_order));
7705
7706        let exec_engine = node.kernel.exec_engine.borrow();
7707        let cache = exec_engine.cache().borrow();
7708        let cached_order = cache
7709            .order(&order.client_order_id())
7710            .expect("Order should be cached");
7711
7712        assert_eq!(cached_order.status(), OrderStatus::Initialized);
7713    }
7714
7715    #[cfg(all(feature = "simulation", madsim))]
7716    async fn advance_clock(d: Duration) {
7717        madsim::time::advance(d);
7718        madsim::task::yield_now().await;
7719    }
7720
7721    #[cfg(not(all(feature = "simulation", madsim)))]
7722    async fn advance_clock(d: Duration) {
7723        tokio::time::advance(d).await;
7724    }
7725
7726    #[rstest]
7727    #[case::single_default(SubmissionRecoveryPolicy::ResolveLocally, false, None, false)]
7728    #[case::list_default(SubmissionRecoveryPolicy::ResolveLocally, true, None, false)]
7729    #[case::single_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, false, None, false)]
7730    #[case::list_diagnostics(SubmissionRecoveryPolicy::RetainUnresolved, true, None, false)]
7731    #[case::unapplied_cancel_default(
7732        SubmissionRecoveryPolicy::ResolveLocally,
7733        false,
7734        Some(OrderStatus::PendingCancel),
7735        false
7736    )]
7737    #[case::unapplied_modify_default(
7738        SubmissionRecoveryPolicy::ResolveLocally,
7739        false,
7740        Some(OrderStatus::PendingUpdate),
7741        false
7742    )]
7743    #[case::unapplied_cancel_diagnostics(
7744        SubmissionRecoveryPolicy::RetainUnresolved,
7745        false,
7746        Some(OrderStatus::PendingCancel),
7747        false
7748    )]
7749    #[case::unapplied_modify_diagnostics(
7750        SubmissionRecoveryPolicy::RetainUnresolved,
7751        false,
7752        Some(OrderStatus::PendingUpdate),
7753        false
7754    )]
7755    #[case::first_fill_cancel_default(
7756        SubmissionRecoveryPolicy::ResolveLocally,
7757        false,
7758        Some(OrderStatus::PendingCancel),
7759        true
7760    )]
7761    #[case::first_fill_modify_default(
7762        SubmissionRecoveryPolicy::ResolveLocally,
7763        false,
7764        Some(OrderStatus::PendingUpdate),
7765        true
7766    )]
7767    #[case::first_fill_cancel_diagnostics(
7768        SubmissionRecoveryPolicy::RetainUnresolved,
7769        false,
7770        Some(OrderStatus::PendingCancel),
7771        true
7772    )]
7773    #[case::first_fill_modify_diagnostics(
7774        SubmissionRecoveryPolicy::RetainUnresolved,
7775        false,
7776        Some(OrderStatus::PendingUpdate),
7777        true
7778    )]
7779    #[cfg_attr(
7780        not(all(feature = "simulation", madsim)),
7781        tokio::test(start_paused = true)
7782    )]
7783    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
7784    async fn test_submission_registry_publishes_after_native_resolution(
7785        #[case] policy: SubmissionRecoveryPolicy,
7786        #[case] order_list: bool,
7787        #[case] pending_status: Option<OrderStatus>,
7788        #[case] first_fill: bool,
7789    ) {
7790        let config = LiveNodeConfig {
7791            exec_engine: crate::config::LiveExecutionEngineConfig {
7792                inflight_check_threshold_ms: 100,
7793                inflight_check_retries: 2,
7794                submission_recovery_policy: policy,
7795                ..Default::default()
7796            },
7797            ..Default::default()
7798        };
7799        let mut node = LiveNode::build("SubmissionRegistryNode".to_string(), Some(config)).unwrap();
7800        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
7801        let client_id = ClientId::from("TEST-SUBMISSION");
7802        let account_id = AccountId::from("TEST-001");
7803        let strategy_id = StrategyId::from("SUBMISSION-001");
7804        let account = AccountAny::Margin(MarginAccount::new(
7805            AccountState::new(
7806                account_id,
7807                AccountType::Margin,
7808                vec![AccountBalance::new(
7809                    Money::from("1000000 USDT"),
7810                    Money::from("0 USDT"),
7811                    Money::from("1000000 USDT"),
7812                )],
7813                Vec::new(),
7814                true,
7815                UUID4::new(),
7816                UnixNanos::default(),
7817                UnixNanos::default(),
7818                Some(Currency::USDT()),
7819            ),
7820            true,
7821        ));
7822        node.kernel.cache.borrow_mut().add_account(account).unwrap();
7823        node.kernel
7824            .cache
7825            .borrow_mut()
7826            .add_instrument(instrument.clone())
7827            .unwrap();
7828        let orders: Vec<_> = (0..if order_list { 2 } else { 1 })
7829            .map(|index| {
7830                OrderTestBuilder::new(OrderType::Limit)
7831                    .trader_id(node.trader_id())
7832                    .strategy_id(strategy_id)
7833                    .client_order_id(ClientOrderId::from(
7834                        format!("O-SUBMISSION-{index}").as_str(),
7835                    ))
7836                    .instrument_id(instrument.id())
7837                    .quantity(Quantity::from("1.000"))
7838                    .price(Price::from("100.00"))
7839                    .build()
7840            })
7841            .collect();
7842        let command = if order_list {
7843            TradingCommand::SubmitOrderList(SubmitOrderList::new(
7844                node.trader_id(),
7845                Some(client_id),
7846                strategy_id,
7847                OrderList::new(
7848                    OrderListId::from("OL-SUBMISSION"),
7849                    instrument.id(),
7850                    strategy_id,
7851                    orders.iter().map(Order::client_order_id).collect(),
7852                    UnixNanos::default(),
7853                ),
7854                orders
7855                    .iter()
7856                    .map(|order| order.init_event().clone())
7857                    .collect(),
7858                None,
7859                None,
7860                None,
7861                UUID4::new(),
7862                UnixNanos::default(),
7863                None,
7864            ))
7865        } else {
7866            TradingCommand::SubmitOrder(SubmitOrder::from_order(
7867                &orders[0],
7868                node.trader_id(),
7869                Some(client_id),
7870                None,
7871                UUID4::new(),
7872                UnixNanos::default(),
7873            ))
7874        };
7875        node.observe_exec_command_before_dispatch(&command);
7876
7877        for order in &orders {
7878            let submitted = TestOrderEventStubs::submitted(order, account_id);
7879            node.kernel
7880                .cache
7881                .borrow_mut()
7882                .add_order(order.clone(), None, Some(client_id), false)
7883                .unwrap();
7884            node.process_exec_event(ExecutionEvent::Order(submitted));
7885        }
7886
7887        let expected_status = match pending_status {
7888            Some(status) if !first_fill => status,
7889            Some(_) => OrderStatus::Canceled,
7890            None if policy == SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
7891            None => OrderStatus::Rejected,
7892        };
7893        let diagnostics = Rc::new(RefCell::new(Vec::new()));
7894        let handler = ShareableMessageHandler::from_typed({
7895            let diagnostics = diagnostics.clone();
7896            let cache = node.kernel.cache.clone();
7897            move |diagnostic: &SubmissionRecoveryExhausted| {
7898                let cache = cache.borrow_mut();
7899                let order = cache.order(&diagnostic.client_order_id).unwrap();
7900                assert_eq!(order.status(), expected_status);
7901                assert!(diagnostic.ts_event >= order.ts_last());
7902                diagnostics.borrow_mut().push(diagnostic.clone());
7903            }
7904        });
7905        let topic = MessagingSwitchboard::submission_recovery_exhausted_topic();
7906        msgbus::subscribe_any(topic.into(), handler.clone(), None);
7907        let queries = Rc::new(RefCell::new(Vec::new()));
7908        msgbus::register_trading_command_endpoint(
7909            MessagingSwitchboard::exec_engine_execute(),
7910            TypedIntoHandler::from({
7911                let queries = queries.clone();
7912                move |command: TradingCommand| queries.borrow_mut().push(command)
7913            }),
7914        );
7915
7916        let mut last_inflight_check = dst::time::Instant::now();
7917        let mut last_open_check = last_inflight_check;
7918        let mut last_position_check = last_inflight_check;
7919        let mut open_order_report_task = None;
7920        let mut targeted_order_report_task = None;
7921        let mut position_report_task = None;
7922
7923        for check in 1..=3 {
7924            advance_clock(Duration::from_millis(101)).await;
7925            node.run_reconciliation_checks(
7926                dst::time::Instant::now(),
7927                ReconciliationCheckIntervals {
7928                    inflight: Duration::from_millis(100),
7929                    open: Duration::ZERO,
7930                    position: Duration::ZERO,
7931                },
7932                &mut ReconciliationCheckState {
7933                    last_inflight_check: &mut last_inflight_check,
7934                    last_open_check: &mut last_open_check,
7935                    last_position_check: &mut last_position_check,
7936                    open_order_report_task: &mut open_order_report_task,
7937                    targeted_order_report_task: &mut targeted_order_report_task,
7938                    position_report_task: &mut position_report_task,
7939                },
7940            );
7941
7942            if check == 1 {
7943                assert!(diagnostics.borrow().is_empty());
7944
7945                if let Some(pending_status) = pending_status {
7946                    let order = &orders[0];
7947                    let pending = if pending_status == OrderStatus::PendingCancel {
7948                        OrderEventAny::PendingCancel(
7949                            OrderPendingCancelSpec::builder()
7950                                .trader_id(node.trader_id())
7951                                .strategy_id(strategy_id)
7952                                .instrument_id(instrument.id())
7953                                .client_order_id(order.client_order_id())
7954                                .account_id(account_id)
7955                                .build(),
7956                        )
7957                    } else {
7958                        OrderEventAny::PendingUpdate(
7959                            OrderPendingUpdateSpec::builder()
7960                                .trader_id(node.trader_id())
7961                                .strategy_id(strategy_id)
7962                                .instrument_id(instrument.id())
7963                                .client_order_id(order.client_order_id())
7964                                .account_id(account_id)
7965                                .build(),
7966                        )
7967                    };
7968                    node.process_exec_event(ExecutionEvent::Order(pending));
7969                    node.exec_manager
7970                        .register_inflight(order.client_order_id(), UUID4::new());
7971
7972                    if first_fill {
7973                        let cached = node
7974                            .kernel
7975                            .cache
7976                            .borrow()
7977                            .order(&order.client_order_id())
7978                            .unwrap()
7979                            .clone();
7980                        let OrderEventAny::Filled(mut fill) =
7981                            OrderFilledTestBuilder::new(&cached, &instrument)
7982                                .last_qty(Quantity::from("0.400"))
7983                                .last_px(Price::from("100.00"))
7984                                .commission(Money::from("0 USDT"))
7985                                .without_position_id()
7986                                .build()
7987                        else {
7988                            unreachable!()
7989                        };
7990                        fill.venue_order_id = VenueOrderId::from("V-SUBMISSION");
7991                        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
7992                    } else {
7993                        let report = OrderStatusReport::new(
7994                            account_id,
7995                            instrument.id(),
7996                            Some(order.client_order_id()),
7997                            VenueOrderId::from("V-SUBMISSION"),
7998                            Some(order.order_side()),
7999                            order.order_type(),
8000                            order.time_in_force(),
8001                            OrderStatus::Submitted,
8002                            order.quantity(),
8003                            Quantity::zero(3),
8004                            UnixNanos::default(),
8005                            UnixNanos::default(),
8006                            UnixNanos::default(),
8007                            None,
8008                        )
8009                        .with_price(Price::from("100.00"));
8010                        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(
8011                            Box::new(report),
8012                        )));
8013                    }
8014                    let cache = node.kernel.cache.borrow();
8015                    let cached = cache.order(&order.client_order_id()).unwrap();
8016                    assert_eq!(cached.status(), pending_status);
8017                    assert_eq!(
8018                        cached.filled_qty(),
8019                        if first_fill {
8020                            Quantity::from("0.400")
8021                        } else {
8022                            Quantity::zero(3)
8023                        }
8024                    );
8025                    assert!(
8026                        !cached
8027                            .events()
8028                            .iter()
8029                            .any(|event| matches!(event, OrderEventAny::Accepted(_)))
8030                    );
8031                }
8032            }
8033        }
8034
8035        assert_eq!(
8036            queries.borrow().len(),
8037            orders.len()
8038                + usize::from(first_fill && policy == SubmissionRecoveryPolicy::ResolveLocally)
8039        );
8040
8041        for (command, order) in queries.borrow().iter().zip(orders.iter().cycle()) {
8042            assert!(matches!(command, TradingCommand::QueryOrder(query)
8043                if query.client_order_id == order.client_order_id()
8044                && query.client_id == Some(client_id)));
8045            assert_eq!(
8046                node.kernel
8047                    .cache
8048                    .borrow()
8049                    .order(&order.client_order_id())
8050                    .unwrap()
8051                    .status(),
8052                if first_fill && policy == SubmissionRecoveryPolicy::RetainUnresolved {
8053                    pending_status.unwrap()
8054                } else {
8055                    expected_status
8056                }
8057            );
8058        }
8059        let diagnostics = diagnostics.borrow();
8060        assert_eq!(
8061            diagnostics.len(),
8062            if policy == SubmissionRecoveryPolicy::RetainUnresolved && pending_status.is_none() {
8063                orders.len()
8064            } else {
8065                0
8066            }
8067        );
8068
8069        for (diagnostic, order) in diagnostics.iter().zip(&orders) {
8070            assert_eq!(diagnostic.trader_id, node.trader_id());
8071            assert_eq!(diagnostic.client_id, Some(client_id));
8072            assert_eq!(diagnostic.strategy_id, strategy_id);
8073            assert_eq!(diagnostic.instrument_id, instrument.id());
8074            assert_eq!(diagnostic.client_order_id, order.client_order_id());
8075            assert_eq!(diagnostic.source, SubmissionRecoverySource::Inflight);
8076            assert_eq!(diagnostic.retry_count, 2);
8077        }
8078        msgbus::unsubscribe_any(topic.into(), &handler);
8079        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8080    }
8081
8082    #[derive(Debug)]
8083    struct SubmissionCallbackStrategy {
8084        core: StrategyCore,
8085        cancel: bool,
8086        batch: bool,
8087        callback: &'static str,
8088        client_id: ClientId,
8089    }
8090
8091    impl DataActor for SubmissionCallbackStrategy {}
8092
8093    impl SubmissionCallbackStrategy {
8094        fn queue_command(&mut self, client_order_id: ClientOrderId) {
8095            match (self.cancel, self.batch) {
8096                (true, true) => {
8097                    self.cancel_orders(vec![client_order_id], Some(self.client_id), None)
8098                        .unwrap();
8099                }
8100                (true, false) => {
8101                    self.cancel_order(client_order_id, Some(self.client_id), None)
8102                        .unwrap();
8103                }
8104                (false, true) => {
8105                    self.modify_orders(
8106                        vec![(client_order_id, None, Some(Price::from("101.00")), None)],
8107                        Some(self.client_id),
8108                        None,
8109                    )
8110                    .unwrap();
8111                }
8112                (false, false) => self
8113                    .modify_order(
8114                        client_order_id,
8115                        None,
8116                        Some(Price::from("101.00")),
8117                        None,
8118                        Some(self.client_id),
8119                        None,
8120                    )
8121                    .unwrap(),
8122            }
8123        }
8124    }
8125
8126    nautilus_strategy!(SubmissionCallbackStrategy, {
8127        fn on_order_accepted(&mut self, event: OrderAccepted) {
8128            if self.callback == "accepted" {
8129                self.queue_command(event.client_order_id);
8130            }
8131        }
8132
8133        fn on_order_updated(&mut self, event: OrderUpdated) {
8134            if self.callback == "updated" {
8135                self.queue_command(event.client_order_id);
8136            }
8137        }
8138    });
8139
8140    #[rstest]
8141    #[case::native(false, "accepted")]
8142    #[case::report(true, "accepted")]
8143    #[case::updated(false, "updated")]
8144    #[cfg_attr(
8145        not(all(feature = "simulation", madsim)),
8146        tokio::test(start_paused = true)
8147    )]
8148    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8149    async fn test_submission_registry_ack_callback_starts_command_budget_at_dispatch(
8150        #[values(
8151            SubmissionRecoveryPolicy::ResolveLocally,
8152            SubmissionRecoveryPolicy::RetainUnresolved
8153        )]
8154        policy: SubmissionRecoveryPolicy,
8155        #[values(false, true)] cancel: bool,
8156        #[values(false, true)] batch: bool,
8157        #[case] report: bool,
8158        #[case] callback: &'static str,
8159    ) {
8160        let config = LiveNodeConfig {
8161            exec_engine: crate::config::LiveExecutionEngineConfig {
8162                inflight_check_threshold_ms: 100,
8163                inflight_check_retries: 2,
8164                submission_recovery_policy: policy,
8165                ..Default::default()
8166            },
8167            ..Default::default()
8168        };
8169        let mut node = LiveNode::build("SubmissionCallbackNode".to_string(), Some(config)).unwrap();
8170        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8171        let account_id = AccountId::from("TEST-001");
8172        let strategy_id = StrategyId::from("CALLBACK-001");
8173        let client_id = ClientId::from("TEST-CALLBACK");
8174        let account = AccountAny::Margin(MarginAccount::new(
8175            AccountState::new(
8176                account_id,
8177                AccountType::Margin,
8178                vec![AccountBalance::new(
8179                    Money::from("1000000 USDT"),
8180                    Money::from("0 USDT"),
8181                    Money::from("1000000 USDT"),
8182                )],
8183                Vec::new(),
8184                true,
8185                UUID4::new(),
8186                UnixNanos::default(),
8187                UnixNanos::default(),
8188                Some(Currency::USDT()),
8189            ),
8190            true,
8191        ));
8192        node.kernel.cache.borrow_mut().add_account(account).unwrap();
8193        node.kernel
8194            .cache
8195            .borrow_mut()
8196            .add_instrument(instrument.clone())
8197            .unwrap();
8198        node.kernel
8199            .exec_engine
8200            .borrow_mut()
8201            .register_client(Box::new(StubExecutionClient::new(
8202                client_id,
8203                account_id,
8204                instrument.id().venue,
8205                OmsType::Netting,
8206                None,
8207            )))
8208            .unwrap();
8209        node.add_strategy(SubmissionCallbackStrategy {
8210            core: StrategyCore::new(StrategyConfig {
8211                strategy_id: Some(strategy_id),
8212                ..Default::default()
8213            }),
8214            cancel,
8215            batch,
8216            callback,
8217            client_id,
8218        })
8219        .unwrap();
8220        node.kernel.trader.borrow_mut().initialize().unwrap();
8221        node.kernel.start_trader().unwrap();
8222        let runner = node.runner.take().unwrap();
8223        runner.bind_senders_for_node(node.handle.clone());
8224        let mut channels = runner.take_channels();
8225        let order = OrderTestBuilder::new(OrderType::Limit)
8226            .trader_id(node.trader_id())
8227            .strategy_id(strategy_id)
8228            .client_order_id(ClientOrderId::from("O-CALLBACK"))
8229            .instrument_id(instrument.id())
8230            .quantity(Quantity::from("1.000"))
8231            .price(Price::from("100.00"))
8232            .build();
8233        let client_order_id = order.client_order_id();
8234        let submit = TradingCommand::SubmitOrder(SubmitOrder::from_order(
8235            &order,
8236            node.trader_id(),
8237            Some(client_id),
8238            None,
8239            UUID4::new(),
8240            UnixNanos::default(),
8241        ));
8242        node.observe_exec_command_before_dispatch(&submit);
8243        node.kernel
8244            .cache
8245            .borrow_mut()
8246            .add_order(order.clone(), None, Some(client_id), false)
8247            .unwrap();
8248        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
8249            &order, account_id,
8250        )));
8251        advance_clock(Duration::from_millis(101)).await;
8252        let first = node.exec_manager.check_inflight_orders();
8253        assert_eq!(first.queries.len(), 1);
8254        assert!(first.events.is_empty());
8255
8256        advance_clock(Duration::from_millis(101)).await;
8257        let accepted =
8258            TestOrderEventStubs::accepted(&order, account_id, VenueOrderId::from("V-CALLBACK"));
8259
8260        if callback == "updated" {
8261            let mut update = OrderUpdatedSpec::builder()
8262                .trader_id(node.trader_id())
8263                .strategy_id(strategy_id)
8264                .instrument_id(instrument.id())
8265                .client_order_id(client_order_id)
8266                .account_id(account_id)
8267                .quantity(order.quantity())
8268                .price(order.price().unwrap())
8269                .build();
8270            update.venue_order_id = Some(VenueOrderId::from("V-CALLBACK"));
8271            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
8272        } else if report {
8273            let accepted = OrderStatusReport::new(
8274                account_id,
8275                instrument.id(),
8276                Some(client_order_id),
8277                VenueOrderId::from("V-CALLBACK"),
8278                Some(order.order_side()),
8279                order.order_type(),
8280                order.time_in_force(),
8281                OrderStatus::Accepted,
8282                order.quantity(),
8283                Quantity::zero(3),
8284                UnixNanos::default(),
8285                UnixNanos::default(),
8286                UnixNanos::default(),
8287                None,
8288            )
8289            .with_price(Price::from("100.00"));
8290            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
8291                accepted,
8292            ))));
8293        } else {
8294            node.process_exec_event(ExecutionEvent::Order(accepted));
8295        }
8296
8297        let pending_status = if cancel {
8298            OrderStatus::PendingCancel
8299        } else {
8300            OrderStatus::PendingUpdate
8301        };
8302        assert_eq!(channels.exec_cmd_rx.len(), 1);
8303        assert_eq!(
8304            node.kernel
8305                .cache
8306                .borrow()
8307                .order(&client_order_id)
8308                .unwrap()
8309                .status(),
8310            pending_status
8311        );
8312
8313        // Maintenance may win the runner selection before the queued command dispatches
8314        for _ in 0..3 {
8315            advance_clock(Duration::from_millis(101)).await;
8316            let maintenance = node.exec_manager.check_inflight_orders();
8317            assert!(maintenance.events.is_empty());
8318            assert!(maintenance.queries.is_empty());
8319            assert!(
8320                node.exec_manager
8321                    .take_submission_recovery_exhaustions()
8322                    .is_empty()
8323            );
8324        }
8325
8326        let command = channels.exec_cmd_rx.try_recv().unwrap();
8327        node.process_exec_command(command);
8328        assert!(channels.exec_cmd_rx.is_empty());
8329        let immediate = node.exec_manager.check_inflight_orders();
8330        assert!(immediate.events.is_empty());
8331        assert!(immediate.queries.is_empty());
8332        advance_clock(Duration::from_millis(101)).await;
8333        let first_command_check = node.exec_manager.check_inflight_orders();
8334        assert!(first_command_check.events.is_empty());
8335        assert_eq!(first_command_check.queries.len(), 1);
8336        advance_clock(Duration::from_millis(101)).await;
8337        let exhausted = node.exec_manager.check_inflight_orders();
8338        assert!(exhausted.queries.is_empty());
8339        assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
8340        assert!(
8341            node.exec_manager
8342                .take_submission_recovery_exhaustions()
8343                .is_empty()
8344        );
8345        node.process_reconciliation_events(&exhausted.events);
8346
8347        let expected_status = match policy {
8348            SubmissionRecoveryPolicy::ResolveLocally => OrderStatus::Canceled,
8349            SubmissionRecoveryPolicy::RetainUnresolved if callback == "updated" => {
8350                OrderStatus::Submitted
8351            }
8352            SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Accepted,
8353        };
8354
8355        assert_eq!(
8356            node.kernel
8357                .cache
8358                .borrow()
8359                .order(&client_order_id)
8360                .unwrap()
8361                .status(),
8362            expected_status
8363        );
8364    }
8365
8366    #[cfg_attr(
8367        not(all(feature = "simulation", madsim)),
8368        tokio::test(start_paused = true)
8369    )]
8370    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8371    async fn test_reconciliation_check_due_uses_monotonic_elapsed_time() {
8372        let last = dst::time::Instant::now();
8373        let interval = Duration::from_millis(100);
8374
8375        assert!(!reconciliation_check_due(last, last, Duration::ZERO));
8376        assert!(!reconciliation_check_due(last, last, interval));
8377
8378        advance_clock(Duration::from_millis(99)).await;
8379        let before_interval = dst::time::Instant::now();
8380        assert!(!reconciliation_check_due(before_interval, last, interval));
8381
8382        advance_clock(Duration::from_millis(1)).await;
8383        let at_interval = dst::time::Instant::now();
8384        assert!(reconciliation_check_due(at_interval, last, interval));
8385
8386        assert!(!reconciliation_check_due(last, at_interval, interval));
8387    }
8388
8389    #[cfg_attr(
8390        not(all(feature = "simulation", madsim)),
8391        tokio::test(start_paused = true)
8392    )]
8393    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
8394    async fn test_run_reconciliation_checks_does_not_publish_open_order_queries() {
8395        let config = LiveNodeConfig {
8396            exec_engine: crate::config::LiveExecutionEngineConfig {
8397                reconciliation: true,
8398                open_check_interval_secs: Some(1.0),
8399                position_check_interval_secs: Some(1.0),
8400                max_single_order_queries_per_cycle: 5,
8401                ..Default::default()
8402            },
8403            ..Default::default()
8404        };
8405
8406        let mut node =
8407            LiveNode::build("ReconciliationFallbackNode".to_string(), Some(config)).unwrap();
8408        let client_id = ClientId::from("TEST-QUERY");
8409        let account_id = AccountId::from("TEST-QUERY-001");
8410
8411        let trading_commands = Rc::new(RefCell::new(Vec::new()));
8412        msgbus::register_trading_command_endpoint(
8413            MessagingSwitchboard::exec_engine_execute(),
8414            TypedIntoHandler::from({
8415                let trading_commands = trading_commands.clone();
8416                move |command: TradingCommand| {
8417                    trading_commands.borrow_mut().push(command);
8418                }
8419            }),
8420        );
8421
8422        let venue_order_id = VenueOrderId::from("V-NODE-QUERY-001");
8423        let instrument = crypto_perpetual_ethusdt();
8424        let instrument_id = instrument.id();
8425        let client_order_id = ClientOrderId::from("O-NODE-QUERY-001");
8426
8427        node.kernel
8428            .cache
8429            .borrow_mut()
8430            .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
8431            .unwrap();
8432        insert_accepted_limit_order_in_node(
8433            &node,
8434            account_id,
8435            client_id,
8436            instrument_id,
8437            client_order_id,
8438            venue_order_id,
8439        );
8440
8441        let last = dst::time::Instant::now();
8442        advance_clock(Duration::from_nanos(1)).await;
8443        let now = dst::time::Instant::now();
8444        let mut last_inflight_check = last;
8445        let mut last_open_check = last;
8446        let mut last_position_check = last;
8447        let mut open_order_report_task = None;
8448        let mut targeted_order_report_task = None;
8449        let mut position_report_task = None;
8450
8451        node.run_reconciliation_checks(
8452            now,
8453            ReconciliationCheckIntervals {
8454                inflight: Duration::ZERO,
8455                open: Duration::from_nanos(1),
8456                position: Duration::ZERO,
8457            },
8458            &mut ReconciliationCheckState {
8459                last_inflight_check: &mut last_inflight_check,
8460                last_open_check: &mut last_open_check,
8461                last_position_check: &mut last_position_check,
8462                open_order_report_task: &mut open_order_report_task,
8463                targeted_order_report_task: &mut targeted_order_report_task,
8464                position_report_task: &mut position_report_task,
8465            },
8466        );
8467
8468        let commands = trading_commands.borrow();
8469
8470        assert!(commands.is_empty());
8471        assert!(open_order_report_task.is_none());
8472        assert!(targeted_order_report_task.is_none());
8473        assert!(position_report_task.is_none());
8474
8475        ExecutionEngine::register_msgbus_handlers(&node.kernel.exec_engine);
8476    }
8477
8478    #[rstest]
8479    #[tokio::test]
8480    async fn test_recurring_report_failure_preserves_client_coverage(
8481        #[values(false, true)] positions: bool,
8482        #[values(false, true)] failure_first: bool,
8483    ) {
8484        let mut node = LiveNode::build("ReportFailureNode".to_string(), None).unwrap();
8485        let failed_id = ClientId::from("FAILED");
8486        let healthy_id = ClientId::from("HEALTHY");
8487
8488        for (client_id, outcome) in [
8489            (failed_id, FillReportClientOutcome::Failure),
8490            (healthy_id, FillReportClientOutcome::Reports(Vec::new())),
8491        ] {
8492            node.exec_clients
8493                .push(LiveExecutionClient::new(Box::new(FillReportClient {
8494                    client_id,
8495                    account_id: AccountId::from("TEST-001"),
8496                    venue: Venue::from("BINANCE"),
8497                    outcome,
8498                    commands: Rc::new(RefCell::new(Vec::new())),
8499                })));
8500        }
8501
8502        if !failure_first {
8503            node.exec_clients.reverse();
8504        }
8505
8506        let last = dst::time::Instant::now();
8507        let now = last + Duration::from_secs(1);
8508        let mut last_inflight = last;
8509        let mut last_open = last;
8510        let mut last_position = last;
8511        let mut open_task = None;
8512        let mut targeted_task = None;
8513        let mut position_task = None;
8514
8515        node.run_reconciliation_checks(
8516            now,
8517            ReconciliationCheckIntervals {
8518                inflight: Duration::ZERO,
8519                open: if positions {
8520                    Duration::ZERO
8521                } else {
8522                    Duration::from_secs(1)
8523                },
8524                position: if positions {
8525                    Duration::from_secs(1)
8526                } else {
8527                    Duration::ZERO
8528                },
8529            },
8530            &mut ReconciliationCheckState {
8531                last_inflight_check: &mut last_inflight,
8532                last_open_check: &mut last_open,
8533                last_position_check: &mut last_position,
8534                open_order_report_task: &mut open_task,
8535                targeted_order_report_task: &mut targeted_task,
8536                position_report_task: &mut position_task,
8537            },
8538        );
8539
8540        let (queried_clients, failed_clients) = if positions {
8541            let ReportTaskOutcome::Completed(PositionReportTaskResult::Positions(result)) =
8542                position_task.unwrap().future.await
8543            else {
8544                panic!("position collection should complete despite one client failure");
8545            };
8546
8547            let result = result.finish();
8548            assert!(result.reports.is_empty());
8549            assert!(open_task.is_none());
8550            (result.queried_clients, result.failed_clients)
8551        } else {
8552            let ReportTaskOutcome::Completed(result) = open_task.unwrap().future.await else {
8553                panic!("order collection should complete despite one client failure");
8554            };
8555
8556            let result = result.finish();
8557
8558            assert!(result.reports.is_empty());
8559            assert!(position_task.is_none());
8560            (result.queried_clients, result.failed_clients)
8561        };
8562
8563        assert_eq!(queried_clients, IndexSet::from([failed_id, healthy_id]));
8564        assert_eq!(failed_clients, IndexSet::from([failed_id]));
8565        assert!(targeted_task.is_none());
8566    }
8567
8568    #[rstest]
8569    #[case::inflight(true, false)]
8570    #[case::open(false, false)]
8571    #[case::positions(false, true)]
8572    #[tokio::test]
8573    async fn test_reconciliation_shutdown_preserves_pending_checks(
8574        #[case] inflight: bool,
8575        #[case] positions: bool,
8576    ) {
8577        let mut node = LiveNode::build("ShutdownChecksNode".to_string(), None).unwrap();
8578        node.handle.set_shutting_down();
8579        let last = dst::time::Instant::now();
8580        let now = last + Duration::from_secs(1);
8581        let mut last_inflight = last;
8582        let mut last_open = last;
8583        let mut last_position = last;
8584        let mut open_task = None;
8585        let mut targeted_task = None;
8586        let mut position_task = None;
8587
8588        node.run_reconciliation_checks(
8589            now,
8590            ReconciliationCheckIntervals {
8591                inflight: if inflight {
8592                    Duration::from_secs(1)
8593                } else {
8594                    Duration::ZERO
8595                },
8596                open: if !inflight && !positions {
8597                    Duration::from_secs(1)
8598                } else {
8599                    Duration::ZERO
8600                },
8601                position: if positions {
8602                    Duration::from_secs(1)
8603                } else {
8604                    Duration::ZERO
8605                },
8606            },
8607            &mut ReconciliationCheckState {
8608                last_inflight_check: &mut last_inflight,
8609                last_open_check: &mut last_open,
8610                last_position_check: &mut last_position,
8611                open_order_report_task: &mut open_task,
8612                targeted_order_report_task: &mut targeted_task,
8613                position_report_task: &mut position_task,
8614            },
8615        );
8616
8617        assert_eq!(
8618            (last_inflight, last_open, last_position),
8619            (last, last, last)
8620        );
8621        assert!(open_task.is_none());
8622        assert!(targeted_task.is_none());
8623        assert!(position_task.is_none());
8624    }
8625
8626    fn insert_accepted_limit_order_in_node(
8627        node: &LiveNode,
8628        account_id: AccountId,
8629        client_id: ClientId,
8630        instrument_id: InstrumentId,
8631        client_order_id: ClientOrderId,
8632        venue_order_id: VenueOrderId,
8633    ) {
8634        let order = OrderTestBuilder::new(OrderType::Limit)
8635            .client_order_id(client_order_id)
8636            .instrument_id(instrument_id)
8637            .quantity(Quantity::from("10.0"))
8638            .price(Price::from("100.0"))
8639            .build();
8640        let submitted = TestOrderEventStubs::submitted(&order, account_id);
8641        node.kernel
8642            .cache
8643            .borrow_mut()
8644            .add_order(order, None, Some(client_id), false)
8645            .unwrap();
8646        let order = node
8647            .kernel
8648            .cache
8649            .borrow_mut()
8650            .update_order(&submitted)
8651            .unwrap();
8652        let accepted = TestOrderEventStubs::accepted(&order, account_id, venue_order_id);
8653        node.kernel
8654            .cache
8655            .borrow_mut()
8656            .update_order(&accepted)
8657            .unwrap();
8658    }
8659
8660    fn recent_fill_test_fixture(name: &str) -> (LiveNode, OrderEventAny, InstrumentAny) {
8661        let config = LiveNodeConfig {
8662            exec_engine: crate::config::LiveExecutionEngineConfig {
8663                reconciliation: true,
8664                ..Default::default()
8665            },
8666            ..Default::default()
8667        };
8668
8669        let node = LiveNode::build(name.to_string(), Some(config)).unwrap();
8670        let account_id = AccountId::from("TEST-001");
8671        let client_id = ClientId::from("TEST-RECENT-FILL");
8672        let client_order_id = ClientOrderId::from("O-RECENT-FILL");
8673        let venue_order_id = VenueOrderId::from("V-RECENT-FILL");
8674        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8675        node.kernel
8676            .cache
8677            .borrow_mut()
8678            .add_instrument(instrument.clone())
8679            .unwrap();
8680        insert_accepted_limit_order_in_node(
8681            &node,
8682            account_id,
8683            client_id,
8684            instrument.id(),
8685            client_order_id,
8686            venue_order_id,
8687        );
8688        let order = node
8689            .kernel
8690            .cache
8691            .borrow()
8692            .order_owned(&client_order_id)
8693            .unwrap();
8694        let fill = TestOrderEventStubs::filled(
8695            &order,
8696            &instrument,
8697            Some(TradeId::from("T-RECENT-FILL")),
8698            None,
8699            Some(Price::from("100.0")),
8700            Some(Quantity::from("1.0")),
8701            Some(LiquiditySide::Taker),
8702            None,
8703            None,
8704            Some(account_id),
8705        );
8706
8707        (node, fill, instrument)
8708    }
8709
8710    fn apply_inferred_position_fill(node: &mut LiveNode, fill_report: &FillReport) {
8711        let client_order_id = fill_report.client_order_id.unwrap();
8712        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8713        let order = node
8714            .kernel
8715            .cache
8716            .borrow()
8717            .order_owned(&client_order_id)
8718            .unwrap();
8719        let order_report = OrderStatusReport::new(
8720            fill_report.account_id,
8721            fill_report.instrument_id,
8722            Some(client_order_id),
8723            fill_report.venue_order_id,
8724            OrderSide::Buy.into(),
8725            OrderType::Limit,
8726            TimeInForce::Gtc,
8727            OrderStatus::PartiallyFilled,
8728            Quantity::from("10.0"),
8729            Quantity::from("2.0"),
8730            UnixNanos::from(1_000),
8731            UnixNanos::from(1_000),
8732            UnixNanos::from(1_000),
8733            None,
8734        )
8735        .with_avg_px(dec!(100.0));
8736        let inferred = create_inferred_fill_for_qty(
8737            &order,
8738            &order_report,
8739            &fill_report.account_id,
8740            &instrument,
8741            Quantity::from("1.0"),
8742            UnixNanos::from(1_000),
8743            None,
8744        )
8745        .unwrap();
8746
8747        node.process_reconciliation_events(&[inferred]);
8748    }
8749
8750    #[rstest]
8751    #[case::rejected(false, "10.0", "10.0", "1.0")]
8752    #[case::dust(true, "9.004", "10.004", "10.004")]
8753    #[case::price_improvement(true, "12.5", "13.5", "13.5")]
8754    fn test_position_check_overfill_raises_before_fill_when_client_allows(
8755        #[case] allow: bool,
8756        #[case] fill_qty: &str,
8757        #[case] expected_qty: &str,
8758        #[case] expected_filled: &str,
8759    ) {
8760        let (mut node, venue_report, mut fill_report) =
8761            position_fill_test_fixture("PositionOverfillRaise", Quantity::from(fill_qty));
8762        fill_report.commission = Money::from("1.25 USDT");
8763        let key = (fill_report.instrument_id, fill_report.account_id);
8764
8765        let client = StubExecutionClient::new(
8766            ClientId::from("POSITION-FILLS"),
8767            AccountId::from("TEST-001"),
8768            fill_report.instrument_id.venue,
8769            OmsType::Netting,
8770            None,
8771        );
8772
8773        let client = if allow {
8774            client.with_allows_reconciliation_overfill()
8775        } else {
8776            client
8777        };
8778
8779        node.kernel
8780            .exec_engine
8781            .borrow_mut()
8782            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8783            .unwrap();
8784
8785        let position_result = position_report_result(&node, venue_report);
8786        node.handle_position_fill_report_result(PositionFillReportResult {
8787            position_result,
8788            reports: IndexMap::from([(key, vec![fill_report.clone()])]),
8789        });
8790
8791        let order = node
8792            .kernel
8793            .cache
8794            .borrow()
8795            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8796            .unwrap();
8797        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8798        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8799        let applied = order
8800            .trade_ids()
8801            .iter()
8802            .any(|trade_id| trade_id.as_str() == "T-POSITION-AUTHORITATIVE");
8803        assert_eq!(applied, allow);
8804        let revision = node.exec_manager.position_activity_revision(&key);
8805
8806        if !allow {
8807            return;
8808        }
8809
8810        assert_eq!(revision, 2);
8811
8812        {
8813            let cache = node.kernel.cache.borrow();
8814            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8815            assert_eq!(positions.len(), 1);
8816            assert_eq!(positions[0].quantity, Quantity::from(expected_qty));
8817            assert_eq!(positions[0].commissions(), vec![Money::from("1.25 USDT")]);
8818            assert_eq!(positions[0].trade_ids.len(), 2);
8819        }
8820
8821        let replay_report = position_report_result(
8822            &node,
8823            PositionStatusReport::new(
8824                key.1,
8825                key.0,
8826                PositionSide::Long,
8827                Quantity::from(expected_qty),
8828                UnixNanos::from(2_000),
8829                UnixNanos::from(2_000),
8830                None,
8831                None,
8832                Some(dec!(100.0)),
8833            ),
8834        );
8835        node.handle_position_fill_report_result(PositionFillReportResult {
8836            position_result: replay_report,
8837            reports: IndexMap::from([(key, vec![fill_report])]),
8838        });
8839
8840        let order = node
8841            .kernel
8842            .cache
8843            .borrow()
8844            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8845            .unwrap();
8846        assert_eq!(order.quantity(), Quantity::from(expected_qty));
8847        assert_eq!(order.filled_qty(), Quantity::from(expected_filled));
8848        assert_eq!(node.exec_manager.position_activity_revision(&key), revision);
8849    }
8850
8851    #[rstest]
8852    fn test_position_check_applies_fitting_fill_then_overfill_in_one_cycle() {
8853        let (mut node, venue_report, mut overfill) =
8854            position_fill_test_fixture("PositionSplitOverfill", Quantity::from("2.0"));
8855        overfill.commission = Money::from("0.20 USDT");
8856        let key = (overfill.instrument_id, overfill.account_id);
8857        let client = StubExecutionClient::new(
8858            ClientId::from("POSITION-FILLS"),
8859            AccountId::from("TEST-001"),
8860            overfill.instrument_id.venue,
8861            OmsType::Netting,
8862            None,
8863        )
8864        .with_allows_reconciliation_overfill();
8865        node.kernel
8866            .exec_engine
8867            .borrow_mut()
8868            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8869            .unwrap();
8870
8871        let mut fitting = overfill.clone();
8872        fitting.trade_id = TradeId::from("T-POSITION-FITTING");
8873        fitting.last_qty = Quantity::from("8.0");
8874        fitting.commission = Money::from("0.80 USDT");
8875        let position_result = position_report_result(&node, venue_report);
8876        node.handle_position_fill_report_result(PositionFillReportResult {
8877            position_result,
8878            reports: IndexMap::from([(key, vec![fitting, overfill])]),
8879        });
8880
8881        let order = node
8882            .kernel
8883            .cache
8884            .borrow()
8885            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8886            .unwrap();
8887        assert_eq!(order.quantity(), Quantity::from("11.0"));
8888        assert_eq!(order.filled_qty(), Quantity::from("11.0"));
8889        assert_eq!(order.trade_ids().len(), 3);
8890        assert_eq!(node.exec_manager.position_activity_revision(&key), 3);
8891        {
8892            let cache = node.kernel.cache.borrow();
8893            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8894            assert_eq!(positions.len(), 1);
8895            assert_eq!(positions[0].quantity, Quantity::from("11.0"));
8896            assert_eq!(positions[0].commissions(), vec![Money::from("1.00 USDT")]);
8897        }
8898    }
8899
8900    #[rstest]
8901    fn test_position_check_second_overfill_does_not_raise_filled_order() {
8902        let (mut node, venue_report, mut first) =
8903            position_fill_test_fixture("PositionSecondOverfill", Quantity::from("12.0"));
8904        first.commission = Money::from("1.20 USDT");
8905        let key = (first.instrument_id, first.account_id);
8906        let client = StubExecutionClient::new(
8907            ClientId::from("POSITION-FILLS"),
8908            AccountId::from("TEST-001"),
8909            first.instrument_id.venue,
8910            OmsType::Netting,
8911            None,
8912        )
8913        .with_allows_reconciliation_overfill();
8914        node.kernel
8915            .exec_engine
8916            .borrow_mut()
8917            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8918            .unwrap();
8919
8920        let mut second = first.clone();
8921        second.trade_id = TradeId::from("T-POSITION-LATER");
8922        second.last_qty = Quantity::from("1.0");
8923        second.commission = Money::from("0.10 USDT");
8924        second.ts_event = UnixNanos::from(1_001);
8925        let position_result = position_report_result(&node, venue_report);
8926        node.handle_position_fill_report_result(PositionFillReportResult {
8927            position_result,
8928            reports: IndexMap::from([(key, vec![first, second])]),
8929        });
8930
8931        let order = node
8932            .kernel
8933            .cache
8934            .borrow()
8935            .order_owned(&ClientOrderId::from("O-POSITION-FILLS"))
8936            .unwrap();
8937        assert_eq!(order.quantity(), Quantity::from("13.0"));
8938        assert_eq!(order.filled_qty(), Quantity::from("13.0"));
8939        assert_eq!(order.status(), OrderStatus::Filled);
8940        assert_eq!(order.trade_ids().len(), 2);
8941        assert!(
8942            order
8943                .trade_ids()
8944                .iter()
8945                .all(|trade_id| trade_id.as_str() != "T-POSITION-LATER")
8946        );
8947        {
8948            let cache = node.kernel.cache.borrow();
8949            let positions = cache.positions_open(None, Some(&key.0), None, Some(&key.1), None);
8950            assert_eq!(positions.len(), 1);
8951            assert_eq!(positions[0].quantity, Quantity::from("13.0"));
8952            assert_eq!(positions[0].commissions(), vec![Money::from("1.20 USDT")]);
8953        }
8954    }
8955
8956    #[rstest]
8957    fn test_allow_overfills_applies_fill_without_raising_quantity() {
8958        let config = LiveNodeConfig {
8959            exec_engine: crate::config::LiveExecutionEngineConfig {
8960                reconciliation: true,
8961                allow_overfills: true,
8962                ..Default::default()
8963            },
8964            ..Default::default()
8965        };
8966
8967        let node = LiveNode::build("AllowOverfillNoRaise".to_string(), Some(config)).unwrap();
8968        let account_id = AccountId::from("TEST-001");
8969        let client_id = ClientId::from("POSITION-FILLS");
8970        let client_order_id = ClientOrderId::from("O-ALLOW");
8971        let venue_order_id = VenueOrderId::from("V-ALLOW");
8972        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
8973        node.kernel
8974            .cache
8975            .borrow_mut()
8976            .add_instrument(instrument.clone())
8977            .unwrap();
8978        insert_accepted_limit_order_in_node(
8979            &node,
8980            account_id,
8981            client_id,
8982            instrument.id(),
8983            client_order_id,
8984            venue_order_id,
8985        );
8986        let client = StubExecutionClient::new(
8987            client_id,
8988            account_id,
8989            instrument.id().venue,
8990            OmsType::Netting,
8991            None,
8992        )
8993        .with_allows_reconciliation_overfill();
8994        node.kernel
8995            .exec_engine
8996            .borrow_mut()
8997            .register_client(Box::new(LiveExecutionClient::new(Box::new(client))))
8998            .unwrap();
8999
9000        let fill = FillReport::new(
9001            account_id,
9002            instrument.id(),
9003            venue_order_id,
9004            TradeId::from("T-ALLOW"),
9005            OrderSide::Buy,
9006            Quantity::from("12.0"),
9007            Price::from("100.0"),
9008            Money::from("1.00 USDT"),
9009            LiquiditySide::Taker,
9010            Some(client_order_id),
9011            None,
9012            UnixNanos::from(1_000),
9013            UnixNanos::from(1_000),
9014            None,
9015        );
9016        node.kernel
9017            .exec_engine
9018            .borrow_mut()
9019            .reconcile_fill_report(&fill);
9020
9021        let order = node
9022            .kernel
9023            .cache
9024            .borrow()
9025            .order_owned(&client_order_id)
9026            .unwrap();
9027        assert_eq!(order.quantity(), Quantity::from("10.0"));
9028        assert_eq!(order.filled_qty(), Quantity::from("12.0"));
9029        assert_eq!(order.trade_ids().len(), 1);
9030    }
9031
9032    fn position_fill_test_fixture(
9033        name: &str,
9034        authoritative_qty: Quantity,
9035    ) -> (LiveNode, PositionStatusReport, FillReport) {
9036        position_fill_test_fixture_with_threshold(name, authoritative_qty, 0)
9037    }
9038
9039    fn position_fill_test_fixture_with_threshold(
9040        name: &str,
9041        authoritative_qty: Quantity,
9042        position_check_threshold_ms: u32,
9043    ) -> (LiveNode, PositionStatusReport, FillReport) {
9044        let config = LiveNodeConfig {
9045            exec_engine: crate::config::LiveExecutionEngineConfig {
9046                reconciliation: true,
9047                position_check_threshold_ms,
9048                ..Default::default()
9049            },
9050            ..Default::default()
9051        };
9052
9053        let mut node = LiveNode::build(name.to_string(), Some(config)).unwrap();
9054        let account_id = AccountId::from("TEST-001");
9055        let client_id = ClientId::from("POSITION-FILLS");
9056        let client_order_id = ClientOrderId::from("O-POSITION-FILLS");
9057        let venue_order_id = VenueOrderId::from("V-POSITION-FILLS");
9058        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
9059        let account = AccountAny::Margin(MarginAccount::new(
9060            AccountState::new(
9061                account_id,
9062                AccountType::Margin,
9063                vec![AccountBalance::new(
9064                    Money::from("1000000 USDT"),
9065                    Money::from("0 USDT"),
9066                    Money::from("1000000 USDT"),
9067                )],
9068                Vec::new(),
9069                true,
9070                UUID4::new(),
9071                UnixNanos::default(),
9072                UnixNanos::default(),
9073                Some(Currency::USDT()),
9074            ),
9075            true,
9076        ));
9077        node.kernel.cache.borrow_mut().add_account(account).unwrap();
9078        node.kernel
9079            .cache
9080            .borrow_mut()
9081            .add_instrument(instrument.clone())
9082            .unwrap();
9083        insert_accepted_limit_order_in_node(
9084            &node,
9085            account_id,
9086            client_id,
9087            instrument.id(),
9088            client_order_id,
9089            venue_order_id,
9090        );
9091        let order = node
9092            .kernel
9093            .cache
9094            .borrow()
9095            .order_owned(&client_order_id)
9096            .unwrap();
9097        let mut initial_fill = TestOrderEventStubs::filled(
9098            &order,
9099            &instrument,
9100            Some(TradeId::from("T-POSITION-INITIAL")),
9101            None,
9102            Some(Price::from("100.0")),
9103            Some(Quantity::from("1.0")),
9104            Some(LiquiditySide::Taker),
9105            None,
9106            None,
9107            Some(account_id),
9108        );
9109
9110        let OrderEventAny::Filled(fill) = &mut initial_fill else {
9111            unreachable!();
9112        };
9113
9114        fill.commission = Some(Money::zero(instrument.quote_currency()));
9115        node.process_reconciliation_events(&[initial_fill]);
9116
9117        let ts_event = UnixNanos::from(1_000);
9118
9119        let fill_report = FillReport::new(
9120            account_id,
9121            instrument.id(),
9122            venue_order_id,
9123            TradeId::from("T-POSITION-AUTHORITATIVE"),
9124            OrderSide::Buy,
9125            authoritative_qty,
9126            Price::from("100.0"),
9127            Money::zero(instrument.quote_currency()),
9128            LiquiditySide::Taker,
9129            Some(client_order_id),
9130            None,
9131            ts_event,
9132            ts_event,
9133            None,
9134        );
9135
9136        let venue_report = PositionStatusReport::new(
9137            account_id,
9138            instrument.id(),
9139            PositionSide::Long,
9140            Quantity::from("2.0"),
9141            ts_event,
9142            ts_event,
9143            None,
9144            None,
9145            Some(dec!(100.0)),
9146        );
9147
9148        (node, venue_report, fill_report)
9149    }
9150
9151    fn position_report_result(
9152        node: &LiveNode,
9153        report: PositionStatusReport,
9154    ) -> PositionReportResult {
9155        let client_id = ClientId::from("POSITION-FILLS");
9156        let key = (report.instrument_id, report.account_id);
9157        let mut check = node
9158            .exec_manager
9159            .prepare_position_report_check(UUID4::new(), &[]);
9160        check.client_coverage.insert(
9161            key,
9162            ReportClientCoverage::Resolved(IndexSet::from([client_id])),
9163        );
9164
9165        PositionReportResult {
9166            check,
9167            reports: vec![report],
9168            queried_clients: IndexSet::from([client_id]),
9169            failed_clients: IndexSet::new(),
9170        }
9171    }
9172
9173    fn position_quantity_after_check(
9174        node: &mut LiveNode,
9175        venue_report: &PositionStatusReport,
9176        reports: &[FillReport],
9177    ) -> Quantity {
9178        let key = (venue_report.instrument_id, venue_report.account_id);
9179        let position_result = position_report_result(node, venue_report.clone());
9180        node.handle_position_fill_report_result(PositionFillReportResult {
9181            position_result,
9182            reports: IndexMap::from([(key, reports.to_vec())]),
9183        });
9184
9185        node.kernel
9186            .cache
9187            .borrow()
9188            .positions_open(None, Some(&key.0), None, Some(&key.1), None)
9189            .iter()
9190            .map(|position| position.quantity)
9191            .sum()
9192    }
9193
9194    fn fill_report_event(fill: &OrderFilled) -> ExecutionEvent {
9195        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
9196            fill.account_id,
9197            fill.instrument_id,
9198            fill.venue_order_id,
9199            fill.trade_id,
9200            fill.order_side,
9201            fill.last_qty,
9202            fill.last_px,
9203            fill.commission
9204                .unwrap_or_else(|| Money::zero(fill.currency)),
9205            fill.liquidity_side,
9206            Some(fill.client_order_id),
9207            fill.position_id,
9208            fill.ts_event,
9209            fill.ts_init,
9210            None,
9211        ))))
9212    }
9213
9214    fn is_recent_fill(node: &LiveNode, fill: &OrderFilled) -> bool {
9215        node.exec_manager.is_fill_recently_processed(
9216            fill.account_id,
9217            fill.instrument_id,
9218            fill.trade_id,
9219        )
9220    }
9221
9222    #[rstest]
9223    #[case(0, NodeState::Idle)]
9224    #[case(1, NodeState::Starting)]
9225    #[case(2, NodeState::Running)]
9226    #[case(3, NodeState::ShuttingDown)]
9227    #[case(4, NodeState::Stopped)]
9228    fn test_node_state_from_u8_valid(#[case] value: u8, #[case] expected: NodeState) {
9229        assert_eq!(NodeState::from_u8(value), expected);
9230    }
9231
9232    #[rstest]
9233    #[case(5)]
9234    #[case(255)]
9235    #[should_panic(expected = "Invalid NodeState value")]
9236    fn test_node_state_from_u8_invalid_panics(#[case] value: u8) {
9237        let _ = NodeState::from_u8(value);
9238    }
9239
9240    #[rstest]
9241    fn test_node_state_roundtrip() {
9242        for state in [
9243            NodeState::Idle,
9244            NodeState::Starting,
9245            NodeState::Running,
9246            NodeState::ShuttingDown,
9247            NodeState::Stopped,
9248        ] {
9249            assert_eq!(NodeState::from_u8(state.as_u8()), state);
9250        }
9251    }
9252
9253    #[rstest]
9254    fn test_node_state_is_running_only_for_running() {
9255        assert!(!NodeState::Idle.is_running());
9256        assert!(!NodeState::Starting.is_running());
9257        assert!(NodeState::Running.is_running());
9258        assert!(!NodeState::ShuttingDown.is_running());
9259        assert!(!NodeState::Stopped.is_running());
9260    }
9261
9262    #[rstest]
9263    #[tokio::test]
9264    async fn test_await_engines_connected_returns_stop_requested() {
9265        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9266        let handle = node.handle();
9267
9268        handle.stop();
9269
9270        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9271        let status = node.await_engines_connected(deadline).await;
9272
9273        assert_eq!(status, EngineConnectionStatus::StopRequested);
9274        assert!(handle.should_stop());
9275    }
9276
9277    #[rstest]
9278    #[tokio::test]
9279    async fn test_await_engines_connected_returns_shutdown_requested() {
9280        let node = LiveNode::build("TestNode".to_string(), None).unwrap();
9281
9282        node.kernel().shutdown_flag().set(true);
9283
9284        let deadline = dst::time::Instant::now() + Duration::from_secs(1);
9285        let status = node.await_engines_connected(deadline).await;
9286
9287        assert_eq!(status, EngineConnectionStatus::ShutdownRequested);
9288    }
9289
9290    #[rstest]
9291    #[tokio::test]
9292    async fn test_start_stop_request_aborts_startup_without_running() {
9293        let config = LiveNodeConfig {
9294            exec_engine: crate::config::LiveExecutionEngineConfig {
9295                reconciliation: false,
9296                ..Default::default()
9297            },
9298            timeout_disconnection: Duration::from_millis(50),
9299            ..Default::default()
9300        };
9301
9302        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9303        let handle = node.handle();
9304
9305        handle.stop();
9306        node.start().await.unwrap();
9307
9308        assert_eq!(handle.state(), NodeState::Stopped);
9309        assert!(handle.should_stop());
9310        assert!(!handle.is_running());
9311    }
9312
9313    #[rstest]
9314    #[tokio::test(start_paused = true)]
9315    async fn test_stop_processes_residual_exec_event_during_grace_period() {
9316        let config = LiveNodeConfig {
9317            exec_engine: crate::config::LiveExecutionEngineConfig {
9318                reconciliation: false,
9319                ..Default::default()
9320            },
9321            timeout_connection: Duration::ZERO,
9322            timeout_reconciliation: Duration::ZERO,
9323            timeout_portfolio: Duration::ZERO,
9324            timeout_disconnection: Duration::ZERO,
9325            delay_post_stop: Duration::from_millis(20),
9326            timeout_shutdown: Duration::ZERO,
9327            ..Default::default()
9328        };
9329
9330        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9331        let order = OrderTestBuilder::new(OrderType::Market)
9332            .instrument_id(InstrumentId::from("EUR/USD.SIM"))
9333            .quantity(Quantity::from("2"))
9334            .build();
9335        let client_order_id = order.client_order_id();
9336        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-STOP-001"));
9337
9338        node.kernel
9339            .cache()
9340            .borrow_mut()
9341            .add_order(order, None, None, false)
9342            .unwrap();
9343
9344        node.start().await.unwrap();
9345        let exec_event_sender = get_exec_event_sender();
9346
9347        let send_residual = tokio::spawn(async move {
9348            tokio::time::sleep(Duration::from_millis(1)).await;
9349            exec_event_sender
9350                .send(ExecutionEvent::Order(submitted))
9351                .unwrap();
9352        });
9353
9354        node.stop().await.unwrap();
9355        send_residual.await.unwrap();
9356
9357        assert_eq!(
9358            node.kernel
9359                .cache()
9360                .borrow()
9361                .order(&client_order_id)
9362                .unwrap()
9363                .status(),
9364            OrderStatus::Submitted
9365        );
9366
9367        node.dispose();
9368    }
9369
9370    #[rstest]
9371    #[tokio::test]
9372    async fn test_live_state_persistence_loads_before_start_and_saves_after_stop() {
9373        let actor_id = ActorId::from("LIVE-STATE-ACTOR");
9374        let strategy_id = StrategyId::from("LIVE-STATE-STRATEGY-001");
9375        let actor_load = IndexMap::from([("actor-load".to_string(), b"actor-loaded".to_vec())]);
9376        let strategy_load =
9377            IndexMap::from([("strategy-load".to_string(), b"strategy-loaded".to_vec())]);
9378        let actor_save = IndexMap::from([("actor-save".to_string(), b"actor-saved".to_vec())]);
9379        let strategy_save =
9380            IndexMap::from([("strategy-save".to_string(), b"strategy-saved".to_vec())]);
9381        let (database, control) = TestCacheDatabaseControl::create();
9382        control.set_actor_state(actor_id, &actor_load);
9383        control.set_strategy_state(strategy_id, &strategy_load);
9384
9385        let config = LiveNodeConfig {
9386            load_state: true,
9387            save_state: true,
9388            exec_engine: crate::config::LiveExecutionEngineConfig {
9389                reconciliation: false,
9390                ..Default::default()
9391            },
9392            timeout_connection: Duration::ZERO,
9393            timeout_reconciliation: Duration::ZERO,
9394            timeout_portfolio: Duration::ZERO,
9395            timeout_disconnection: Duration::ZERO,
9396            delay_post_stop: Duration::ZERO,
9397            timeout_shutdown: Duration::ZERO,
9398            ..Default::default()
9399        };
9400
9401        let mut node = LiveNode::build("StatePersistenceNode".to_string(), Some(config)).unwrap();
9402        node.set_cache_database(Box::new(database)).unwrap();
9403        node.add_actor(StateActor::new(
9404            actor_id,
9405            control.clone(),
9406            actor_save.clone(),
9407        ))
9408        .unwrap();
9409        node.add_strategy(StateStrategy::new(
9410            strategy_id,
9411            control.clone(),
9412            strategy_save.clone(),
9413        ))
9414        .unwrap();
9415
9416        node.start().await.unwrap();
9417        node.stop().await.unwrap();
9418        node.dispose();
9419
9420        assert_eq!(
9421            control.events(),
9422            vec![
9423                "actor.load:LIVE-STATE-ACTOR",
9424                "actor.on_load",
9425                "strategy.load:LIVE-STATE-STRATEGY-001",
9426                "strategy.on_load",
9427                "actor.on_start",
9428                "strategy.on_start",
9429                "actor.on_stop",
9430                "strategy.on_stop",
9431                "actor.on_save",
9432                "actor.update:LIVE-STATE-ACTOR",
9433                "strategy.on_save",
9434                "strategy.update:LIVE-STATE-STRATEGY-001",
9435                "database.close",
9436            ]
9437        );
9438        assert_eq!(control.actor_state(&actor_id), Some(actor_save));
9439        assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
9440        assert_eq!(node.state(), NodeState::Stopped);
9441    }
9442
9443    #[rstest]
9444    #[tokio::test]
9445    async fn test_live_state_persistence_reports_callback_errors_after_shutdown() {
9446        let actor_id = ActorId::from("LIVE-FAIL-SAVE-ACTOR");
9447        let strategy_id = StrategyId::from("LIVE-FAIL-SAVE-STRATEGY-001");
9448        let (database, control) = TestCacheDatabaseControl::create();
9449
9450        let config = LiveNodeConfig {
9451            save_state: true,
9452            exec_engine: crate::config::LiveExecutionEngineConfig {
9453                reconciliation: false,
9454                ..Default::default()
9455            },
9456            timeout_connection: Duration::ZERO,
9457            timeout_reconciliation: Duration::ZERO,
9458            timeout_portfolio: Duration::ZERO,
9459            timeout_disconnection: Duration::ZERO,
9460            delay_post_stop: Duration::ZERO,
9461            timeout_shutdown: Duration::ZERO,
9462            ..Default::default()
9463        };
9464
9465        let mut node =
9466            LiveNode::build("StatePersistenceErrorNode".to_string(), Some(config)).unwrap();
9467        node.set_cache_database(Box::new(database)).unwrap();
9468        node.add_actor(
9469            StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save(),
9470        )
9471        .unwrap();
9472        node.add_strategy(
9473            StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save(),
9474        )
9475        .unwrap();
9476
9477        node.start().await.unwrap();
9478        let error = node.stop().await.unwrap_err();
9479        node.dispose();
9480
9481        assert_eq!(
9482            error.to_string(),
9483            "failed while finalizing kernel shutdown: Failed to save component state: actor \
9484             LIVE-FAIL-SAVE-ACTOR callback: test actor on_save failure; strategy \
9485             LIVE-FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
9486        );
9487        assert_eq!(
9488            control.events(),
9489            vec![
9490                "actor.on_start",
9491                "strategy.on_start",
9492                "actor.on_stop",
9493                "strategy.on_stop",
9494                "actor.on_save",
9495                "strategy.on_save",
9496                "database.close",
9497            ]
9498        );
9499        assert_eq!(node.state(), NodeState::Stopped);
9500    }
9501
9502    #[rstest]
9503    #[tokio::test]
9504    async fn test_stop_drains_queued_exec_event_after_zero_grace() {
9505        let config = LiveNodeConfig {
9506            exec_engine: crate::config::LiveExecutionEngineConfig {
9507                reconciliation: false,
9508                ..Default::default()
9509            },
9510            timeout_connection: Duration::ZERO,
9511            timeout_reconciliation: Duration::ZERO,
9512            timeout_portfolio: Duration::ZERO,
9513            timeout_disconnection: Duration::ZERO,
9514            delay_post_stop: Duration::ZERO,
9515            timeout_shutdown: Duration::ZERO,
9516            ..Default::default()
9517        };
9518
9519        let mut node = LiveNode::build("TestNode".to_string(), Some(config)).unwrap();
9520        let order = OrderTestBuilder::new(OrderType::Market)
9521            .instrument_id(InstrumentId::from("GBP/USD.SIM"))
9522            .quantity(Quantity::from("3"))
9523            .build();
9524        let client_order_id = order.client_order_id();
9525        let submitted = TestOrderEventStubs::submitted(&order, AccountId::from("POLL-DRAIN-001"));
9526
9527        node.kernel
9528            .cache()
9529            .borrow_mut()
9530            .add_order(order, None, None, false)
9531            .unwrap();
9532
9533        node.start().await.unwrap();
9534        get_exec_event_sender()
9535            .send(ExecutionEvent::Order(submitted))
9536            .unwrap();
9537
9538        node.stop().await.unwrap();
9539
9540        assert_eq!(
9541            node.kernel
9542                .cache()
9543                .borrow()
9544                .order(&client_order_id)
9545                .unwrap()
9546                .status(),
9547            OrderStatus::Submitted
9548        );
9549
9550        node.dispose();
9551    }
9552
9553    #[rstest]
9554    #[case::manual(false)]
9555    #[case::integrated(true)]
9556    fn test_shutdown_drain_logs_progress(
9557        #[case] integrated: bool,
9558        #[values(0, 9_999, 10_000, 20_001)] pending: usize,
9559    ) {
9560        struct DrainLogCapture(Mutex<Vec<String>>);
9561
9562        impl Log for DrainLogCapture {
9563            fn enabled(&self, metadata: &Metadata<'_>) -> bool {
9564                metadata.level() == Level::Debug
9565            }
9566
9567            fn log(&self, record: &Record<'_>) {
9568                assert_eq!(record.level(), Level::Debug);
9569                self.0.lock().push(record.args().to_string());
9570            }
9571
9572            fn flush(&self) {}
9573        }
9574
9575        static LOGS: DrainLogCapture = DrainLogCapture(Mutex::new(Vec::new()));
9576        log::set_logger(&LOGS).unwrap();
9577        log::set_max_level(LevelFilter::Debug);
9578
9579        let mut runner = AsyncRunner::new();
9580        runner.bind_senders();
9581        let event = TimeEvent::new("drain".into(), UUID4::new(), 0.into(), 0.into());
9582
9583        for _ in 0..pending {
9584            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9585                event.clone(),
9586                TimeEventCallback::from(|_: TimeEvent| {}),
9587            ));
9588        }
9589
9590        if integrated {
9591            let mut channels = runner.take_channels();
9592            LiveNode::drain_channels(
9593                &mut channels.time_evt_rx,
9594                &mut channels.system_evt_rx,
9595                &mut channels.system_cmd_rx,
9596                &mut channels.exec_evt_rx,
9597                &mut channels.exec_cmd_rx,
9598                &mut channels.data_evt_rx,
9599                &mut channels.data_cmd_rx,
9600            );
9601        } else {
9602            assert_eq!(runner.poll_pending(|_| {}), pending);
9603        }
9604
9605        let mut expected = vec![format!(
9606            "Draining shutdown queues ({pending} events initially pending)"
9607        )];
9608        expected.extend(
9609            (10_000..=pending)
9610                .step_by(10_000)
9611                .map(|processed| format!("Processed {processed} events during shutdown drain")),
9612        );
9613        expected.push(format!(
9614            "Drained {pending} remaining events during shutdown"
9615        ));
9616        assert_eq!(*LOGS.0.lock(), expected);
9617    }
9618
9619    #[rstest]
9620    #[tokio::test]
9621    async fn test_start_event_store_replay_skips_live_connections() {
9622        let mut node = live_node_with_replay_store(false);
9623        let handle = node.handle();
9624
9625        node.start().await.unwrap();
9626
9627        assert_eq!(handle.state(), NodeState::Running);
9628        assert!(handle.is_running());
9629        assert!(node.kernel.is_event_store_replay());
9630        assert!(node.runner.is_some());
9631    }
9632
9633    #[rstest]
9634    #[tokio::test]
9635    async fn test_start_event_store_replay_preserves_stop_request() {
9636        let mut node = live_node_with_replay_store(false);
9637        let handle = node.handle();
9638        handle.stop();
9639
9640        node.start().await.unwrap();
9641
9642        assert_eq!(handle.state(), NodeState::Stopped);
9643        assert!(handle.should_stop());
9644        assert!(node.kernel.is_event_store_replay());
9645        assert!(node.runner.is_some());
9646    }
9647
9648    #[rstest]
9649    #[tokio::test]
9650    async fn test_start_event_store_replay_config_failure_aborts_startup() {
9651        let mut node = live_node_with_replay_store(true);
9652        let handle = node.handle();
9653
9654        node.start().await.unwrap();
9655
9656        assert_eq!(handle.state(), NodeState::Stopped);
9657        assert!(!handle.is_running());
9658        assert!(node.kernel.is_event_store_replay_configured());
9659        assert!(!node.kernel.is_event_store_replay());
9660        assert!(node.runner.is_some());
9661    }
9662
9663    #[rstest]
9664    #[tokio::test]
9665    async fn test_run_event_store_replay_consumes_runner_and_stops_before_connections() {
9666        let mut node = live_node_with_replay_store(false);
9667        let handle = node.handle();
9668
9669        node.run().await.unwrap();
9670
9671        assert_eq!(handle.state(), NodeState::Running);
9672        assert!(handle.is_running());
9673        assert!(node.kernel.is_event_store_replay());
9674        assert!(node.runner.is_none());
9675    }
9676
9677    #[rstest]
9678    #[tokio::test]
9679    async fn test_run_event_store_replay_preserves_stop_request() {
9680        let mut node = live_node_with_replay_store(false);
9681        let handle = node.handle();
9682        handle.stop();
9683
9684        node.run().await.unwrap();
9685
9686        assert_eq!(handle.state(), NodeState::Stopped);
9687        assert!(handle.should_stop());
9688        assert!(node.kernel.is_event_store_replay());
9689        assert!(node.runner.is_none());
9690    }
9691
9692    #[rstest]
9693    #[tokio::test]
9694    async fn test_run_event_store_replay_config_failure_aborts_startup() {
9695        let mut node = live_node_with_replay_store(true);
9696        let handle = node.handle();
9697
9698        node.run().await.unwrap();
9699
9700        assert_eq!(handle.state(), NodeState::Stopped);
9701        assert!(!handle.is_running());
9702        assert!(node.kernel.is_event_store_replay_configured());
9703        assert!(!node.kernel.is_event_store_replay());
9704        assert!(node.runner.is_none());
9705    }
9706
9707    #[rstest]
9708    fn test_build_rejects_event_store_config_without_factory() {
9709        let config = LiveNodeConfig {
9710            event_store: Some(EventStoreConfig::default()),
9711            exec_engine: crate::config::LiveExecutionEngineConfig {
9712                reconciliation: false,
9713                ..Default::default()
9714            },
9715            ..Default::default()
9716        };
9717
9718        let err = LiveNodeBuilder::from_config(config)
9719            .expect("builder")
9720            .build()
9721            .expect_err("should reject event_store config without factory");
9722
9723        assert!(
9724            err.to_string().contains("with_event_store"),
9725            "error message should mention with_event_store, was: {err}"
9726        );
9727    }
9728
9729    #[rstest]
9730    fn test_direct_build_rejects_event_store_config() {
9731        let config = LiveNodeConfig {
9732            event_store: Some(EventStoreConfig::default()),
9733            exec_engine: crate::config::LiveExecutionEngineConfig {
9734                reconciliation: false,
9735                ..Default::default()
9736            },
9737            ..Default::default()
9738        };
9739
9740        let err = LiveNode::build("TestNode".to_string(), Some(config))
9741            .expect_err("LiveNode::build should reject event_store config");
9742
9743        assert!(
9744            err.to_string().contains("with_event_store"),
9745            "error message should mention with_event_store, was: {err}"
9746        );
9747    }
9748
9749    #[rstest]
9750    fn test_dispose_before_start_is_idempotent() {
9751        let mut node = LiveNode::build("TestNode".to_string(), None).unwrap();
9752        node.add_strategy(TestStrategy::new(StrategyConfig {
9753            strategy_id: Some(StrategyId::from("DISPOSAL-001")),
9754            ..Default::default()
9755        }))
9756        .unwrap();
9757
9758        node.dispose();
9759        node.dispose();
9760
9761        assert!(node.kernel.trader().borrow().is_disposed());
9762        assert_eq!(node.kernel.trader().borrow().component_count(), 0);
9763        assert_eq!(node.state(), NodeState::Stopped);
9764    }
9765
9766    #[rstest]
9767    fn test_dispose_releases_retained_callback_roots(
9768        #[values(false, true)] fatal: bool,
9769        #[values(false, true)] external: bool,
9770    ) {
9771        actor::clear_callbacks().unwrap();
9772        let directory =
9773            std::env::temp_dir().join(format!("nautilus-callback-disposal-{}", UUID4::new()));
9774
9775        let config = LiveNodeConfig {
9776            logging: LoggerConfig {
9777                fileout_level: LevelFilter::Info,
9778                file_config: Some(FileWriterConfig {
9779                    directory: Some(directory.to_str().unwrap().to_string()),
9780                    file_name: Some("disposal".to_string()),
9781                    ..Default::default()
9782                }),
9783                ..Default::default()
9784            },
9785            ..Default::default()
9786        };
9787
9788        let mut node = LiveNode::build("CallbackDisposalNode".to_string(), Some(config)).unwrap();
9789        let received = Rc::new(RefCell::new(Vec::new()));
9790        let observed = received.clone();
9791
9792        let retained = DispatchMessage::from(()).dispatch(|()| {
9793            nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9794                TimeEvent::new("disposal".into(), UUID4::new(), 17.into(), 23.into()),
9795                TimeEventCallback::RustLocal(Rc::new(move |_| {
9796                    observed.borrow_mut().push("delivered");
9797                })),
9798            ));
9799
9800            external.then(|| DispatchMessage::new((), std::thread::current().id()))
9801        });
9802
9803        if let Some(retained) = &retained {
9804            assert!(retained.is_rooted());
9805        }
9806
9807        assert_eq!(actor::clear_callbacks(), Err(CallbackDispatchError::Active));
9808
9809        if fatal {
9810            crate::dispatch::tests::latch_callback_failure();
9811        }
9812
9813        node.dispose();
9814
9815        assert!(node.runner.is_none());
9816        assert!(received.borrow().is_empty());
9817        assert_eq!(Rc::strong_count(&received), 1);
9818        assert_eq!(
9819            actor::callback_failure(),
9820            (fatal && external).then_some(CallbackDispatchError::DeliveryUnwound)
9821        );
9822        assert_eq!(
9823            actor::clear_callbacks(),
9824            if external {
9825                Err(CallbackDispatchError::Active)
9826            } else {
9827                Ok(())
9828            }
9829        );
9830
9831        logging_sync_to_disk().unwrap();
9832        let output = std::fs::read_to_string(directory.join("disposal.log")).unwrap();
9833        drop(retained);
9834        node.dispose();
9835
9836        assert_eq!(actor::callback_failure(), None);
9837        assert_eq!(actor::clear_callbacks(), Ok(()));
9838        drop(node);
9839        std::fs::remove_dir_all(directory).unwrap();
9840
9841        let errors: Vec<_> = output
9842            .lines()
9843            .filter(|line| line.contains("[ERROR]"))
9844            .filter_map(|line| {
9845                line.split_once(".nautilus_live::node: ")
9846                    .map(|(_, text)| text)
9847            })
9848            .collect();
9849
9850        let mut expected = Vec::new();
9851
9852        if fatal {
9853            expected.push(
9854                "Callback dispatch failed before disposal cleanup: Callback delivery unwound",
9855            );
9856        }
9857
9858        if external {
9859            expected.push(
9860                "Failed to clear callback dispatch during disposal: Callback work or access is still active",
9861            );
9862        }
9863
9864        assert_eq!(errors, expected);
9865    }
9866
9867    #[tokio::test]
9868    async fn test_dispose_releases_stop_generated_callback_roots() {
9869        actor::clear_callbacks().unwrap();
9870
9871        let config = LiveNodeConfig {
9872            exec_engine: crate::config::LiveExecutionEngineConfig {
9873                reconciliation: false,
9874                ..Default::default()
9875            },
9876            timeout_connection: Duration::ZERO,
9877            timeout_reconciliation: Duration::ZERO,
9878            timeout_portfolio: Duration::ZERO,
9879            timeout_shutdown: Duration::ZERO,
9880            ..Default::default()
9881        };
9882
9883        let mut node =
9884            LiveNode::build("StopCallbackDisposalNode".to_string(), Some(config)).unwrap();
9885        let received = Rc::new(RefCell::new(Vec::new()));
9886        node.add_actor(StopCallbackActor {
9887            core: DataActorCore::new(DataActorConfig {
9888                actor_id: Some(ActorId::from("STOP-CALLBACK")),
9889                ..Default::default()
9890            }),
9891            received: received.clone(),
9892        })
9893        .unwrap();
9894
9895        node.start().await.unwrap();
9896
9897        assert!(node.kernel.trader().borrow().is_running());
9898        assert_eq!(actor::clear_callbacks(), Ok(()));
9899
9900        node.dispose();
9901
9902        assert_eq!(*received.borrow(), ["stop", "queued"]);
9903        assert_eq!(Rc::strong_count(&received), 1);
9904        assert!(node.runner.is_none());
9905        assert!(node.kernel.trader().borrow().is_disposed());
9906        assert_eq!(node.state(), NodeState::Stopped);
9907        assert_eq!(actor::clear_callbacks(), Ok(()));
9908    }
9909
9910    #[derive(Debug)]
9911    struct StopCallbackActor {
9912        core: DataActorCore,
9913        received: Rc<RefCell<Vec<&'static str>>>,
9914    }
9915
9916    nautilus_actor!(StopCallbackActor);
9917
9918    impl DataActor for StopCallbackActor {
9919        fn on_stop(&mut self) -> anyhow::Result<()> {
9920            self.received.borrow_mut().push("stop");
9921            let received = self.received.clone();
9922            DispatchMessage::from(()).dispatch(|()| {
9923                nautilus_common::runner::get_time_event_sender().send(TimeEventMessage::new(
9924                    TimeEvent::new("stop-disposal".into(), UUID4::new(), 31.into(), 37.into()),
9925                    TimeEventCallback::RustLocal(Rc::new(move |_| {
9926                        received.borrow_mut().push("delivered");
9927                    })),
9928                ));
9929            });
9930
9931            let clear_result = actor::clear_callbacks();
9932            anyhow::ensure!(
9933                clear_result == Err(CallbackDispatchError::Active),
9934                "Expected active callback roots during stop, received {clear_result:?}"
9935            );
9936            self.received.borrow_mut().push("queued");
9937            Ok(())
9938        }
9939    }
9940
9941    #[rstest]
9942    fn test_handle_initial_state() {
9943        let handle = LiveNodeHandle::new();
9944
9945        assert_eq!(handle.state(), NodeState::Idle);
9946        assert!(!handle.should_stop());
9947        assert!(!handle.is_running());
9948    }
9949
9950    #[rstest]
9951    fn test_handle_initial_metrics_snapshot_is_zero() {
9952        let handle = LiveNodeHandle::new();
9953
9954        assert_eq!(handle.metrics_snapshot(), RunnerMetricsSnapshot::default());
9955    }
9956
9957    #[rstest]
9958    fn test_record_runner_dispatch_updates_selected_channel() {
9959        let metrics = RunnerMetrics::default();
9960        let dispatch_start = dst::time::Instant::now();
9961        let metrics_start = dispatch_start
9962            .checked_sub(Duration::from_micros(1))
9963            .expect("test instant should support a one-microsecond lookback");
9964
9965        record_runner_dispatch(
9966            &metrics,
9967            SystemChannel::DataCommands,
9968            dispatch_start,
9969            metrics_start,
9970        );
9971        let snapshot = metrics.snapshot();
9972
9973        assert_eq!(snapshot.time_events.dispatched, 0);
9974        assert_eq!(snapshot.exec_events.dispatched, 0);
9975        assert_eq!(snapshot.exec_commands.dispatched, 0);
9976        assert_eq!(snapshot.data_events.dispatched, 0);
9977        assert_eq!(snapshot.data_commands.dispatched, 1);
9978        assert_eq!(
9979            snapshot.data_commands.last_dispatch_at_ns,
9980            snapshot.elapsed_ns
9981        );
9982        assert_eq!(
9983            snapshot.data_commands.dispatch_busy_ns,
9984            snapshot.dispatch_busy_ns
9985        );
9986        assert!(snapshot.dispatch_busy_ns < snapshot.elapsed_ns);
9987    }
9988
9989    #[rstest]
9990    fn test_handle_stop_sets_flag() {
9991        let handle = LiveNodeHandle::new();
9992
9993        handle.stop();
9994
9995        assert!(handle.should_stop());
9996    }
9997
9998    #[rstest]
9999    fn test_handle_stop_blocks_running_transition() {
10000        let handle = LiveNodeHandle::new();
10001        handle.set_starting();
10002        handle.stop();
10003
10004        let transition = handle.try_set_running();
10005
10006        assert_eq!(transition, RunningTransition::StopRequested);
10007        assert_eq!(handle.state(), NodeState::Starting);
10008        assert!(handle.should_stop());
10009        assert!(!handle.is_running());
10010    }
10011
10012    #[rstest]
10013    fn test_handle_stop_after_running_transition_remains_pending() {
10014        let handle = LiveNodeHandle::new();
10015        handle.set_starting();
10016
10017        let transition = handle.try_set_running();
10018        handle.stop();
10019
10020        assert_eq!(transition, RunningTransition::Entered);
10021        assert_eq!(handle.state(), NodeState::Running);
10022        assert!(handle.should_stop());
10023        assert!(handle.is_running());
10024    }
10025
10026    #[rstest]
10027    fn test_handle_node_state_transitions() {
10028        let handle = LiveNodeHandle::new();
10029        assert_eq!(handle.state(), NodeState::Idle);
10030
10031        handle.set_starting();
10032        assert_eq!(handle.state(), NodeState::Starting);
10033        assert!(!handle.is_running());
10034
10035        assert_eq!(handle.try_set_running(), RunningTransition::Entered);
10036        assert_eq!(handle.state(), NodeState::Running);
10037        assert!(handle.is_running());
10038
10039        handle.set_shutting_down();
10040        assert_eq!(handle.state(), NodeState::ShuttingDown);
10041        assert!(!handle.is_running());
10042
10043        handle.set_stopped();
10044        assert_eq!(handle.state(), NodeState::Stopped);
10045        assert!(!handle.is_running());
10046    }
10047
10048    #[rstest]
10049    fn test_handle_clone_shares_state_bidirectionally() {
10050        let handle1 = LiveNodeHandle::new();
10051        let handle2 = handle1.clone();
10052
10053        handle1.set_starting();
10054        let transition = handle2.try_set_running();
10055        handle1.stop();
10056
10057        assert_eq!(transition, RunningTransition::Entered);
10058        assert_eq!(handle1.state(), NodeState::Running);
10059        assert!(handle2.should_stop());
10060    }
10061
10062    #[rstest]
10063    fn test_handle_stop_flag_survives_non_running_state_changes() {
10064        let handle = LiveNodeHandle::new();
10065
10066        handle.set_starting();
10067        handle.stop();
10068        handle.set_shutting_down();
10069        handle.set_stopped();
10070
10071        assert_eq!(handle.state(), NodeState::Stopped);
10072        assert!(handle.should_stop());
10073    }
10074
10075    #[rstest]
10076    fn test_builder_creation() {
10077        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10078
10079        assert!(result.is_ok());
10080    }
10081
10082    #[rstest]
10083    fn test_stream_processor_receives_unregistered_typed_payload() {
10084        let mut node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10085            .unwrap()
10086            .build()
10087            .unwrap();
10088        let command = SubscribeCommand::Quotes(SubscribeQuotes::new(
10089            InstrumentId::from("AUD/USD.SIM"),
10090            Some(ClientId::from("EXTERNAL")),
10091            Some(Venue::from("SIM")),
10092            UUID4::from("00000000-0000-4000-8000-000000000001"),
10093            UnixNanos::from(1),
10094            None,
10095            None,
10096        ));
10097        let expected = serde_json::to_value(&command).unwrap();
10098        let received = Rc::new(RefCell::new(Vec::new()));
10099        let steps = Rc::new(RefCell::new(Vec::new()));
10100        let received_processor = received.clone();
10101        let first_steps = steps.clone();
10102        node.add_stream_processor(move |message| {
10103            let command = message
10104                .downcast_ref::<SubscribeCommand>()
10105                .expect("processor must receive the decoded concrete command");
10106            received_processor
10107                .borrow_mut()
10108                .push(serde_json::to_value(command).unwrap());
10109            first_steps.borrow_mut().push(1);
10110        });
10111
10112        let second_steps = steps.clone();
10113        node.add_stream_processor(move |_| second_steps.borrow_mut().push(2));
10114        let republished = Rc::new(RefCell::new(Vec::new()));
10115        let republished_handler = republished.clone();
10116        let subscriber_steps = steps.clone();
10117        msgbus::subscribe_any(
10118            "external.test".into(),
10119            ShareableMessageHandler::from_typed(move |command: &SubscribeCommand| {
10120                republished_handler
10121                    .borrow_mut()
10122                    .push(serde_json::to_value(command).unwrap());
10123                subscriber_steps.borrow_mut().push(3);
10124            }),
10125            None,
10126        );
10127
10128        let message = BusMessage::with_str_topic(
10129            "external.test",
10130            BusPayloadType::SubscribeCommand,
10131            Bytes::from(serde_json::to_vec(&command).unwrap()),
10132            SerializationEncoding::Json,
10133        );
10134
10135        assert!(
10136            !msgbus::get_message_bus()
10137                .borrow()
10138                .is_streaming_type(BusPayloadType::SubscribeCommand)
10139        );
10140        node.process_external_msgbus_message(&message);
10141
10142        assert_eq!(*received.borrow(), vec![expected.clone()]);
10143        assert_eq!(*steps.borrow(), vec![1, 2]);
10144        assert!(republished.borrow().is_empty());
10145
10146        received.borrow_mut().clear();
10147        steps.borrow_mut().clear();
10148        msgbus::get_message_bus()
10149            .borrow_mut()
10150            .add_streaming_type(BusPayloadType::SubscribeCommand);
10151        node.process_external_msgbus_message(&message);
10152
10153        assert_eq!(*received.borrow(), vec![expected.clone()]);
10154        assert_eq!(*republished.borrow(), vec![expected]);
10155        assert_eq!(*steps.borrow(), vec![1, 2, 3]);
10156        msgbus::get_message_bus().borrow_mut().dispose();
10157    }
10158
10159    #[rstest]
10160    fn test_builder_rejects_backtest() {
10161        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Backtest);
10162
10163        assert!(result.is_err());
10164        assert!(result.unwrap_err().to_string().contains("Backtest"));
10165    }
10166
10167    #[rstest]
10168    fn test_builder_accepts_live_environment() {
10169        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live);
10170
10171        assert!(result.is_ok());
10172    }
10173
10174    #[rstest]
10175    fn test_builder_accepts_sandbox_environment() {
10176        let result = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox);
10177
10178        assert!(result.is_ok());
10179    }
10180
10181    #[rstest]
10182    fn test_builder_fluent_api_chaining() {
10183        let builder = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
10184            .unwrap()
10185            .with_name("TestNode")
10186            .with_instance_id(UUID4::new())
10187            .with_load_state(false)
10188            .with_save_state(true)
10189            .with_timeout_connection(30)
10190            .with_timeout_reconciliation(60)
10191            .with_reconciliation(true)
10192            .with_reconciliation_lookback_mins(120)
10193            .with_timeout_portfolio(10)
10194            .with_timeout_disconnection_secs(5)
10195            .with_delay_post_stop_secs(3)
10196            .with_delay_shutdown_secs(10);
10197
10198        assert_eq!(builder.name(), "TestNode");
10199    }
10200
10201    #[rstest]
10202    fn test_builder_with_external_msgbus_egress_uses_configured_encoding() {
10203        let (external_egress, publications, closed) = CapturingExternalEgress::new();
10204
10205        let msgbus_config = MessageBusConfig {
10206            encoding: SerializationEncoding::Json,
10207            ..Default::default()
10208        };
10209
10210        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10211            .unwrap()
10212            .with_msgbus_config(msgbus_config)
10213            .with_external_msgbus_egress(Box::new(external_egress))
10214            .build()
10215            .expect("node builds with external message bus egress");
10216        let quote = QuoteTick::default();
10217
10218        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10219
10220        let publications = publications.borrow();
10221        assert_eq!(publications.len(), 1);
10222        assert_eq!(publications[0].topic, "data.quotes.TEST");
10223        assert_eq!(
10224            serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10225                .expect("JSON payload must decode as QuoteTick"),
10226            quote
10227        );
10228        drop(publications);
10229
10230        msgbus::get_message_bus().borrow_mut().dispose();
10231        assert!(closed.get());
10232        drop(node);
10233    }
10234
10235    #[rstest]
10236    #[tokio::test(flavor = "current_thread")]
10237    async fn test_builder_with_external_msgbus_factory_installs_egress_and_ingress() {
10238        let quote = QuoteTick::default();
10239        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10240        let publications = Arc::new(Mutex::new(Vec::new()));
10241        let closed = Arc::new(AtomicBool::new(false));
10242        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), Some(rx));
10243
10244        let msgbus_config = MessageBusConfig {
10245            external_streams: Some(vec!["stream".to_string()]),
10246            ..Default::default()
10247        };
10248
10249        let config = LiveNodeConfig {
10250            environment: Environment::Sandbox,
10251            msgbus: Some(msgbus_config),
10252            exec_engine: crate::config::LiveExecutionEngineConfig {
10253                reconciliation: false,
10254                ..Default::default()
10255            },
10256            delay_post_stop: Duration::ZERO,
10257            timeout_connection: Duration::from_millis(500),
10258            timeout_disconnection: Duration::from_millis(500),
10259            ..Default::default()
10260        };
10261
10262        let mut node = LiveNodeBuilder::from_config(config)
10263            .unwrap()
10264            .with_external_msgbus_factory(Box::new(factory))
10265            .build()
10266            .expect("node builds with external message bus factory");
10267
10268        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10269        {
10270            let publications = publications.lock();
10271            assert_eq!(publications.len(), 1);
10272            assert_eq!(publications[0].topic, "data.quotes.TEST");
10273            assert_eq!(
10274                serde_json::from_slice::<QuoteTick>(&publications[0].payload)
10275                    .expect("JSON payload must decode as QuoteTick"),
10276                quote
10277            );
10278        }
10279
10280        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10281        let handle = node.handle();
10282
10283        // Stopping from `drive` rather than here, so the node cannot finish `run` before the
10284        // republished quote is observed.
10285        let handler = TypedHandler::from({
10286            let received = received.clone();
10287            move |quote: &QuoteTick| {
10288                received.borrow_mut().push(*quote);
10289            }
10290        });
10291
10292        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10293        msgbus::get_message_bus()
10294            .borrow_mut()
10295            .add_streaming_type(BusPayloadType::QuoteTick);
10296
10297        let payload =
10298            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
10299        let message = BusMessage::with_str_topic(
10300            "data.quotes.TEST",
10301            BusPayloadType::QuoteTick,
10302            payload,
10303            SerializationEncoding::Json,
10304        );
10305
10306        tokio::time::timeout(Duration::from_secs(30), async {
10307            let run = node.run();
10308            tokio::pin!(run);
10309
10310            let drive = async {
10311                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10312
10313                tx.send(message)
10314                    .await
10315                    .expect("external ingress receiver should be open");
10316
10317                wait_until_async(
10318                    || async { received.borrow().len() == 1 },
10319                    Duration::from_secs(10),
10320                )
10321                .await;
10322                assert_eq!(*received.borrow(), vec![quote]);
10323                handle.stop();
10324            };
10325
10326            tokio::select! {
10327                biased;
10328
10329                () = drive => {}
10330                result = &mut run => {
10331                    panic!("node stopped before factory ingress was republished: {result:?}");
10332                }
10333            }
10334
10335            run.await.expect("node should stop cleanly");
10336        })
10337        .await
10338        .expect("live node should republish factory ingress and stop before timeout");
10339
10340        assert_eq!(handle.state(), NodeState::Stopped);
10341        assert!(closed.load(Ordering::Relaxed));
10342        msgbus::get_message_bus().borrow_mut().dispose();
10343    }
10344
10345    #[rstest]
10346    #[tokio::test(flavor = "current_thread")]
10347    async fn test_builder_with_external_msgbus_factory_without_streams_runs_without_ingress() {
10348        let quote = QuoteTick::default();
10349        let publications = Arc::new(Mutex::new(Vec::new()));
10350        let closed = Arc::new(AtomicBool::new(false));
10351        let factory = CapturingBackingFactory::new(publications.clone(), closed.clone(), None);
10352
10353        let config = LiveNodeConfig {
10354            environment: Environment::Sandbox,
10355            msgbus: Some(MessageBusConfig::default()),
10356            exec_engine: crate::config::LiveExecutionEngineConfig {
10357                reconciliation: false,
10358                ..Default::default()
10359            },
10360            delay_post_stop: Duration::ZERO,
10361            timeout_connection: Duration::from_millis(500),
10362            timeout_disconnection: Duration::from_millis(500),
10363            ..Default::default()
10364        };
10365
10366        let mut node = LiveNodeBuilder::from_config(config)
10367            .unwrap()
10368            .with_external_msgbus_factory(Box::new(factory))
10369            .build()
10370            .expect("node builds with egress-only message bus factory");
10371        let handle = node.handle();
10372
10373        msgbus::publish_quote("data.quotes.TEST".into(), &quote);
10374        {
10375            let publications = publications.lock();
10376            assert_eq!(publications.len(), 1);
10377            assert_eq!(publications[0].topic, "data.quotes.TEST");
10378        }
10379
10380        tokio::time::timeout(Duration::from_secs(30), async {
10381            let run = node.run();
10382            tokio::pin!(run);
10383
10384            let drive = async {
10385                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10386                handle.stop();
10387            };
10388
10389            tokio::select! {
10390                biased;
10391
10392                () = drive => {}
10393                result = &mut run => {
10394                    panic!("node stopped before egress-only factory run was observed: {result:?}");
10395                }
10396            }
10397
10398            run.await.expect("node should stop cleanly");
10399        })
10400        .await
10401        .expect("live node should run without external ingress before timeout");
10402
10403        assert_eq!(handle.state(), NodeState::Stopped);
10404        msgbus::get_message_bus().borrow_mut().dispose();
10405        assert!(closed.load(Ordering::Relaxed));
10406    }
10407
10408    #[rstest]
10409    fn test_builder_with_external_msgbus_factory_rejects_injected_surfaces() {
10410        let (external_egress, _publications, _closed) = CapturingExternalEgress::new();
10411
10412        let egress_factory = CapturingBackingFactory::new(
10413            Arc::new(Mutex::new(Vec::new())),
10414            Arc::new(AtomicBool::new(false)),
10415            None,
10416        );
10417        let egress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10418            .unwrap()
10419            .with_external_msgbus_factory(Box::new(egress_factory))
10420            .with_external_msgbus_egress(Box::new(external_egress))
10421            .build()
10422            .expect_err("builder should reject factory plus injected egress");
10423
10424        assert!(
10425            egress_error
10426                .to_string()
10427                .contains("cannot be combined with injected egress or ingress")
10428        );
10429
10430        let (_tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10431
10432        let ingress_factory = CapturingBackingFactory::new(
10433            Arc::new(Mutex::new(Vec::new())),
10434            Arc::new(AtomicBool::new(false)),
10435            None,
10436        );
10437        let ingress = CapturingExternalIngress::new(rx, Rc::new(Cell::new(false)));
10438        let ingress_error = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10439            .unwrap()
10440            .with_external_msgbus_factory(Box::new(ingress_factory))
10441            .with_external_ingress(Box::new(ingress))
10442            .build()
10443            .expect_err("builder should reject factory plus injected ingress");
10444
10445        assert!(
10446            ingress_error
10447                .to_string()
10448                .contains("cannot be combined with injected egress or ingress")
10449        );
10450    }
10451
10452    #[rstest]
10453    #[tokio::test(flavor = "current_thread")]
10454    async fn test_run_republishes_external_ingress_on_local_msgbus() {
10455        let quote = QuoteTick::default();
10456        let received = Rc::new(RefCell::new(Vec::<QuoteTick>::new()));
10457        let payload =
10458            Bytes::from(serde_json::to_vec(&quote).expect("QuoteTick should serialize as JSON"));
10459        let message = BusMessage::with_str_topic(
10460            "data.quotes.TEST",
10461            BusPayloadType::QuoteTick,
10462            payload,
10463            SerializationEncoding::Json,
10464        );
10465        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10466        let closed = Rc::new(Cell::new(false));
10467        let ingress = CapturingExternalIngress::new(rx, closed.clone());
10468
10469        let config = LiveNodeConfig {
10470            environment: Environment::Sandbox,
10471            exec_engine: crate::config::LiveExecutionEngineConfig {
10472                reconciliation: false,
10473                ..Default::default()
10474            },
10475            delay_post_stop: Duration::ZERO,
10476            timeout_connection: Duration::from_millis(500),
10477            timeout_disconnection: Duration::from_millis(500),
10478            ..Default::default()
10479        };
10480
10481        let mut node = LiveNodeBuilder::from_config(config)
10482            .unwrap()
10483            .with_external_ingress(Box::new(ingress))
10484            .build()
10485            .expect("node builds with external message bus ingress");
10486        let handle = node.handle();
10487
10488        let handler = TypedHandler::from({
10489            let received = received.clone();
10490            move |quote: &QuoteTick| {
10491                received.borrow_mut().push(*quote);
10492            }
10493        });
10494
10495        msgbus::subscribe_quotes("data.quotes.*".into(), handler, None);
10496        msgbus::get_message_bus()
10497            .borrow_mut()
10498            .add_streaming_type(BusPayloadType::QuoteTick);
10499
10500        tokio::time::timeout(Duration::from_secs(30), async {
10501            let run = node.run();
10502            tokio::pin!(run);
10503
10504            let drive = async {
10505                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10506
10507                tx.send(message)
10508                    .await
10509                    .expect("external ingress receiver should be open");
10510
10511                wait_until_async(
10512                    || async { received.borrow().len() == 1 },
10513                    Duration::from_secs(10),
10514                )
10515                .await;
10516                assert_eq!(*received.borrow(), vec![quote]);
10517                handle.stop();
10518            };
10519
10520            tokio::select! {
10521                biased;
10522
10523                () = drive => {}
10524                result = &mut run => {
10525                    panic!("node stopped before external message was republished: {result:?}");
10526                }
10527            }
10528
10529            run.await.expect("node should stop cleanly");
10530        })
10531        .await
10532        .expect("live node should republish ingress and stop before timeout");
10533
10534        assert_eq!(handle.state(), NodeState::Stopped);
10535        assert!(closed.get());
10536        msgbus::get_message_bus().borrow_mut().dispose();
10537    }
10538
10539    #[rstest]
10540    #[tokio::test(flavor = "current_thread")]
10541    async fn test_run_closes_external_ingress_when_receiver_closes() {
10542        let (tx, rx) = tokio::sync::mpsc::channel::<BusMessage>(1);
10543        let closed = Rc::new(Cell::new(false));
10544        let ingress = CapturingExternalIngress::new(rx, closed.clone());
10545
10546        let config = LiveNodeConfig {
10547            environment: Environment::Sandbox,
10548            exec_engine: crate::config::LiveExecutionEngineConfig {
10549                reconciliation: false,
10550                ..Default::default()
10551            },
10552            delay_post_stop: Duration::ZERO,
10553            timeout_connection: Duration::from_millis(500),
10554            timeout_disconnection: Duration::from_millis(500),
10555            ..Default::default()
10556        };
10557
10558        let mut node = LiveNodeBuilder::from_config(config)
10559            .unwrap()
10560            .with_external_ingress(Box::new(ingress))
10561            .build()
10562            .expect("node builds with external message bus ingress");
10563        let handle = node.handle();
10564
10565        tokio::time::timeout(Duration::from_secs(30), async {
10566            let run = node.run();
10567            tokio::pin!(run);
10568
10569            let drive = async {
10570                wait_until_async(|| async { handle.is_running() }, Duration::from_secs(10)).await;
10571
10572                drop(tx);
10573
10574                wait_until_async(|| async { closed.get() }, Duration::from_secs(10)).await;
10575                assert!(
10576                    handle.is_running(),
10577                    "node should keep running after ingress closes"
10578                );
10579                handle.stop();
10580            };
10581
10582            tokio::select! {
10583                biased;
10584
10585                () = drive => {}
10586                result = &mut run => {
10587                    panic!("node stopped before ingress close was observed: {result:?}");
10588                }
10589            }
10590
10591            run.await.expect("node should stop cleanly");
10592        })
10593        .await
10594        .expect("live node should close ingress and stop before timeout");
10595
10596        assert_eq!(handle.state(), NodeState::Stopped);
10597    }
10598
10599    #[rstest]
10600    #[tokio::test(flavor = "current_thread")]
10601    async fn test_run_aborts_startup_when_external_ingress_receiver_unavailable() {
10602        let closed = Rc::new(Cell::new(false));
10603        let ingress = FailingExternalIngress::new(closed.clone());
10604
10605        let config = LiveNodeConfig {
10606            environment: Environment::Sandbox,
10607            exec_engine: crate::config::LiveExecutionEngineConfig {
10608                reconciliation: false,
10609                ..Default::default()
10610            },
10611            delay_post_stop: Duration::ZERO,
10612            timeout_connection: Duration::from_millis(500),
10613            timeout_disconnection: Duration::from_millis(500),
10614            ..Default::default()
10615        };
10616
10617        let mut node = LiveNodeBuilder::from_config(config)
10618            .unwrap()
10619            .with_external_ingress(Box::new(ingress))
10620            .build()
10621            .expect("node builds with external message bus ingress");
10622        let handle = node.handle();
10623
10624        let err = node.run().await.expect_err("run should fail");
10625
10626        assert!(
10627            err.to_string()
10628                .contains("external ingress receiver unavailable")
10629        );
10630        assert_eq!(handle.state(), NodeState::Stopped);
10631        assert!(closed.get());
10632    }
10633
10634    #[rstest]
10635    fn test_node_build_rejects_existing_node_preserving_account_delivery(
10636        #[values(false, true)] first_builder: bool,
10637        #[values(false, true)] second_builder: bool,
10638    ) {
10639        let config = LiveNodeConfig {
10640            trader_id: TraderId::from("PROBE-001"),
10641            ..Default::default()
10642        };
10643
10644        let mut node = if first_builder {
10645            LiveNodeBuilder::from_config(config)
10646                .unwrap()
10647                .build()
10648                .unwrap()
10649        } else {
10650            LiveNode::build("Original".to_string(), Some(config)).unwrap()
10651        };
10652
10653        let bus = msgbus::get_message_bus();
10654
10655        let ExecutionEvent::Account(before) = stub_account_event() else {
10656            unreachable!()
10657        };
10658
10659        get_exec_event_sender()
10660            .send(ExecutionEvent::Account(before.clone()))
10661            .unwrap();
10662        assert_eq!(node.drain_runner_pending(), 1);
10663        assert_eq!(
10664            node.kernel
10665                .cache
10666                .borrow()
10667                .account(&before.account_id)
10668                .unwrap()
10669                .last_event(),
10670            Some(before)
10671        );
10672
10673        let result = if second_builder {
10674            LiveNodeBuilder::from_config(LiveNodeConfig::default())
10675                .unwrap()
10676                .build()
10677        } else {
10678            LiveNode::build("Extra".to_string(), None)
10679        };
10680
10681        assert_eq!(
10682            result.unwrap_err().to_string(),
10683            "A LiveNode already exists or is being built on this thread; drop it before building another"
10684        );
10685
10686        let ExecutionEvent::Account(after) = stub_account_event() else {
10687            unreachable!()
10688        };
10689
10690        get_exec_event_sender()
10691            .send(ExecutionEvent::Account(after.clone()))
10692            .unwrap();
10693        assert_eq!(node.drain_runner_pending(), 1);
10694        assert_eq!(node.trader_id(), TraderId::from("PROBE-001"));
10695        assert!(Rc::ptr_eq(&bus, &msgbus::get_message_bus()));
10696        assert_eq!(
10697            node.kernel
10698                .cache
10699                .borrow()
10700                .account(&after.account_id)
10701                .unwrap()
10702                .last_event(),
10703            Some(after)
10704        );
10705    }
10706
10707    #[rstest]
10708    fn test_node_build_rejects_reentry_and_releases_thread_after_factory_failure() {
10709        let result = LiveNodeBuilder::new(TraderId::default(), Environment::Live)
10710            .unwrap()
10711            .with_event_store(|_instance_id: UUID4, _clock: Rc<RefCell<dyn Clock>>| {
10712                let error = LiveNode::build("Reentrant".to_string(), None).unwrap_err();
10713                assert!(
10714                    error
10715                        .to_string()
10716                        .contains("already exists or is being built")
10717                );
10718                anyhow::bail!("Event store construction failed")
10719            })
10720            .build();
10721
10722        assert_eq!(
10723            result.unwrap_err().to_string(),
10724            "Event store construction failed"
10725        );
10726
10727        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10728        assert_eq!(node.state(), NodeState::Idle);
10729    }
10730
10731    #[rstest]
10732    fn test_node_build_allows_separate_threads() {
10733        let node = LiveNode::build("First".to_string(), None).unwrap();
10734
10735        let other_state = std::thread::spawn(|| {
10736            let other = LiveNode::build("OtherThread".to_string(), None).unwrap();
10737            other.state()
10738        })
10739        .join()
10740        .unwrap();
10741
10742        assert_eq!(node.state(), NodeState::Idle);
10743        assert_eq!(other_state, NodeState::Idle);
10744    }
10745
10746    #[rstest]
10747    fn test_node_build_releases_thread_after_drop() {
10748        let mut node = LiveNode::build("First".to_string(), None).unwrap();
10749        node.dispose();
10750        assert!(LiveNode::build("WhileRetained".to_string(), None).is_err());
10751        drop(node);
10752
10753        let replacement = LiveNode::build("Replacement".to_string(), None).unwrap();
10754        assert_eq!(replacement.state(), NodeState::Idle);
10755    }
10756
10757    #[rstest]
10758    fn test_node_build_releases_thread_after_failure(#[values(false, true)] builder: bool) {
10759        let config = LiveNodeConfig {
10760            event_store: Some(EventStoreConfig::default()),
10761            ..Default::default()
10762        };
10763
10764        let result = if builder {
10765            LiveNodeBuilder::from_config(config).unwrap().build()
10766        } else {
10767            LiveNode::build("Failure".to_string(), Some(config))
10768        };
10769
10770        assert!(result.unwrap_err().to_string().contains("factory"));
10771
10772        let node = LiveNode::build("Retry".to_string(), None).unwrap();
10773        assert_eq!(node.state(), NodeState::Idle);
10774    }
10775
10776    #[cfg(feature = "python")]
10777    #[rstest]
10778    fn test_node_build_and_initial_state() {
10779        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10780            .unwrap()
10781            .with_name("TestNode")
10782            .build()
10783            .unwrap();
10784
10785        assert_eq!(node.state(), NodeState::Idle);
10786        assert!(!node.is_running());
10787        assert_eq!(node.environment(), Environment::Sandbox);
10788        assert_eq!(node.trader_id(), TraderId::from("TRADER-001"));
10789    }
10790
10791    #[cfg(feature = "python")]
10792    #[rstest]
10793    fn test_node_build_replaces_stale_runner_senders() {
10794        replace_data_cmd_sender(Arc::new(SyncDataCommandSender));
10795        replace_exec_cmd_sender(Arc::new(SyncTradingCommandSender));
10796
10797        let first = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10798            .unwrap()
10799            .with_name("FirstNode")
10800            .build()
10801            .unwrap();
10802
10803        assert_eq!(first.state(), NodeState::Idle);
10804        drop(first);
10805
10806        let second = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10807            .unwrap()
10808            .with_name("SecondNode")
10809            .build()
10810            .unwrap();
10811
10812        assert_eq!(second.state(), NodeState::Idle);
10813        assert!(!second.is_running());
10814    }
10815
10816    #[cfg(feature = "python")]
10817    #[rstest]
10818    fn test_node_handle_reflects_node_state() {
10819        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Sandbox)
10820            .unwrap()
10821            .with_name("TestNode")
10822            .build()
10823            .unwrap();
10824
10825        let handle = node.handle();
10826
10827        assert_eq!(handle.state(), NodeState::Idle);
10828        assert!(!handle.is_running());
10829    }
10830
10831    #[rstest]
10832    fn test_pending_drain_data_returns_false_when_empty() {
10833        let mut pending = PendingEvents::default();
10834
10835        assert!(!pending.drain_data());
10836    }
10837
10838    #[rstest]
10839    fn test_pending_drain_data_returns_true_when_non_empty() {
10840        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10841
10842        let mut pending = PendingEvents::default();
10843        pending.data_evts.push(
10844            DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10845                .into(),
10846        );
10847
10848        assert!(pending.drain_data());
10849        assert!(pending.data_evts.is_empty());
10850    }
10851
10852    fn stub_data_event() -> DataEvent {
10853        use nautilus_model::instruments::{InstrumentAny, stubs::crypto_perpetual_ethusdt};
10854
10855        DataEvent::Data(InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt()).into())
10856    }
10857
10858    fn stub_data_command() -> DataCommand {
10859        use nautilus_common::messages::data::{SubscribeCommand, subscribe::SubscribeInstruments};
10860        use nautilus_core::{UUID4, UnixNanos};
10861        use nautilus_model::identifiers::Venue;
10862
10863        DataCommand::Subscribe(SubscribeCommand::Instruments(SubscribeInstruments::new(
10864            None,
10865            Venue::from("TEST"),
10866            UUID4::new(),
10867            UnixNanos::default(),
10868            None,
10869            None,
10870        )))
10871    }
10872
10873    fn stub_system_command() -> SystemCommand {
10874        SystemCommand::ReconnectSocket(ReconnectSocket::new(
10875            TraderId::from("TRADER-001"),
10876            ClientId::from("POLYMARKET"),
10877            Ustr::from("polymarket-market-streams"),
10878            UnixNanos::default(),
10879        ))
10880    }
10881
10882    #[rstest]
10883    fn test_flush_pending_data_drains_events_and_commands() {
10884        let (evt_tx, mut evt_rx) =
10885            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10886        let (cmd_tx, mut cmd_rx) =
10887            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10888
10889        let mut pending = PendingEvents::default();
10890
10891        // Pre-load pending (items captured by the select loop)
10892        pending.data_evts.push((stub_data_event()).into());
10893        pending.data_cmds.push(stub_data_command().into());
10894
10895        // Pre-load channels (items missed by the select loop)
10896        evt_tx.send((stub_data_event()).into()).unwrap();
10897        cmd_tx.send(stub_data_command().into()).unwrap();
10898
10899        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10900
10901        assert!(pending.data_evts.is_empty());
10902        assert!(pending.data_cmds.is_empty());
10903        assert!(evt_rx.try_recv().is_err());
10904        assert!(cmd_rx.try_recv().is_err());
10905    }
10906
10907    #[rstest]
10908    fn test_flush_pending_data_drains_mixed_sources() {
10909        let (evt_tx, mut evt_rx) =
10910            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
10911        let (cmd_tx, mut cmd_rx) =
10912            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
10913
10914        let mut pending = PendingEvents::default();
10915
10916        // First pass: pending has an event, channel has a command
10917        pending.data_evts.push((stub_data_event()).into());
10918        cmd_tx.send(stub_data_command().into()).unwrap();
10919
10920        // Second pass: channel has items that simulate arrival during first drain
10921        evt_tx.send((stub_data_event()).into()).unwrap();
10922        evt_tx.send((stub_data_event()).into()).unwrap();
10923        cmd_tx.send(stub_data_command().into()).unwrap();
10924
10925        flush_pending_data(&mut pending, &mut evt_rx, &mut cmd_rx);
10926
10927        assert!(pending.data_evts.is_empty());
10928        assert!(pending.data_cmds.is_empty());
10929        assert!(evt_rx.try_recv().is_err());
10930        assert!(cmd_rx.try_recv().is_err());
10931    }
10932
10933    #[rstest]
10934    fn test_pending_system_events_stay_separate_from_data() {
10935        let mut pending = PendingEvents::default();
10936
10937        let change = SocketStateChange::new(
10938            ClientId::from("BINANCE"),
10939            Some(Venue::from("BINANCE")),
10940            ustr::Ustr::from("binance-futures-market-streams"),
10941            SocketState::Connected,
10942        );
10943
10944        pending
10945            .system_events
10946            .push(SystemEvent::SocketState(change).into());
10947        let system_events = pending
10948            .take_system_events()
10949            .into_iter()
10950            .map(|message| message.dispatch(|value| value))
10951            .collect::<Vec<_>>();
10952
10953        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
10954        assert!(pending.is_empty());
10955    }
10956
10957    #[rstest]
10958    fn test_pending_system_commands_stay_separate_from_data() {
10959        let mut pending = PendingEvents::default();
10960        let command = stub_system_command();
10961
10962        pending.system_commands.push(command.into());
10963        let system_commands = pending
10964            .take_system_commands()
10965            .into_iter()
10966            .map(|message| message.dispatch(|value| value))
10967            .collect::<Vec<_>>();
10968
10969        assert_eq!(system_commands, vec![command]);
10970        assert!(pending.is_empty());
10971    }
10972
10973    fn stub_time_event_handler() -> TimeEventMessage {
10974        use std::rc::Rc;
10975
10976        use nautilus_common::{
10977            runner::TimeEventMessage,
10978            timer::{TimeEvent, TimeEventCallback},
10979        };
10980        use nautilus_core::{UUID4, UnixNanos};
10981        use ustr::Ustr;
10982
10983        TimeEventMessage::new(
10984            TimeEvent::new(
10985                Ustr::from("test-timer"),
10986                UUID4::new(),
10987                UnixNanos::default(),
10988                UnixNanos::default(),
10989            ),
10990            TimeEventCallback::RustLocal(Rc::new(|_| {})),
10991        )
10992    }
10993
10994    fn stub_trading_command_message() -> TradingCommandMessage {
10995        use nautilus_common::messages::execution::query::QueryAccount;
10996        use nautilus_core::{UUID4, UnixNanos};
10997        use nautilus_model::identifiers::AccountId;
10998
10999        TradingCommandMessage::new(
11000            MessagingSwitchboard::exec_engine_execute(),
11001            TradingCommand::QueryAccount(QueryAccount::new(
11002                TraderId::from("TESTER-001"),
11003                None,
11004                AccountId::from("TEST-001"),
11005                UUID4::new(),
11006                UnixNanos::default(),
11007                None,
11008                None, // correlation_id
11009            )),
11010        )
11011    }
11012
11013    fn stub_exec_event() -> ExecutionEvent {
11014        use nautilus_model::{
11015            enums::{LiquiditySide, OrderSide},
11016            identifiers::{AccountId, InstrumentId, TradeId, VenueOrderId},
11017            reports::FillReport,
11018            types::{Money, Price, Quantity},
11019        };
11020
11021        ExecutionEvent::Report(ExecutionReport::Fill(Box::new(FillReport::new(
11022            AccountId::from("TEST-001"),
11023            InstrumentId::from("TEST.VENUE"),
11024            VenueOrderId::from("V-001"),
11025            TradeId::from("T-001"),
11026            OrderSide::Buy,
11027            Quantity::from("1.0"),
11028            Price::from("100.0"),
11029            Money::from("0.01 USD"),
11030            LiquiditySide::Maker,
11031            None,
11032            None,
11033            nautilus_core::UnixNanos::default(),
11034            nautilus_core::UnixNanos::default(),
11035            None,
11036        ))))
11037    }
11038
11039    #[rstest]
11040    fn test_flush_all_pending_drains_buffered_channels() {
11041        let (time_tx, mut time_rx) =
11042            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11043        let (system_evt_tx, mut system_evt_rx) =
11044            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11045        let (system_cmd_tx, mut system_cmd_rx) =
11046            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11047        let (data_evt_tx, mut data_evt_rx) =
11048            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11049        let (data_cmd_tx, mut data_cmd_rx) =
11050            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11051        let (exec_evt_tx, mut exec_evt_rx) =
11052            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11053        let (exec_cmd_tx, mut exec_cmd_rx) =
11054            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11055
11056        let mut pending = PendingEvents::default();
11057
11058        // Pre-load pending with data items
11059        pending.data_evts.push((stub_data_event()).into());
11060        pending.data_cmds.push(stub_data_command().into());
11061
11062        // Pre-load all channel types
11063        time_tx.send((stub_time_event_handler()).into()).unwrap();
11064
11065        let change = SocketStateChange::new(
11066            ClientId::from("BINANCE"),
11067            Some(Venue::from("BINANCE")),
11068            Ustr::from("binance-futures-market-streams"),
11069            SocketState::Connected,
11070        );
11071        system_evt_tx
11072            .send((SystemEvent::SocketState(change)).into())
11073            .unwrap();
11074        system_cmd_tx.send((stub_system_command()).into()).unwrap();
11075        data_evt_tx.send((stub_data_event()).into()).unwrap();
11076        data_cmd_tx.send(stub_data_command().into()).unwrap();
11077        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11078        exec_cmd_tx
11079            .send(stub_trading_command_message().into())
11080            .unwrap();
11081
11082        flush_all_pending(
11083            &mut pending,
11084            &mut time_rx,
11085            &mut system_evt_rx,
11086            &mut system_cmd_rx,
11087            &mut exec_evt_rx,
11088            &mut exec_cmd_rx,
11089            &mut data_evt_rx,
11090            &mut data_cmd_rx,
11091        );
11092
11093        let system_events = pending
11094            .take_system_events()
11095            .into_iter()
11096            .map(|message| message.dispatch(|value| value))
11097            .collect::<Vec<_>>();
11098        let system_commands = pending
11099            .take_system_commands()
11100            .into_iter()
11101            .map(|message| message.dispatch(|value| value))
11102            .collect::<Vec<_>>();
11103        assert_eq!(system_events, vec![SystemEvent::SocketState(change)]);
11104        assert_eq!(system_commands, vec![stub_system_command()]);
11105        assert!(pending.data_evts.is_empty());
11106        assert!(pending.data_cmds.is_empty());
11107        assert!(pending.exec_reports.is_empty());
11108        assert!(pending.exec_cmds.is_empty());
11109        assert!(pending.order_evts.is_empty());
11110        assert!(time_rx.try_recv().is_err());
11111        assert!(system_evt_rx.try_recv().is_err());
11112        assert!(system_cmd_rx.try_recv().is_err());
11113        assert!(data_evt_rx.try_recv().is_err());
11114        assert!(data_cmd_rx.try_recv().is_err());
11115        assert!(exec_evt_rx.try_recv().is_err());
11116        assert!(exec_cmd_rx.try_recv().is_err());
11117    }
11118
11119    fn stub_order_event() -> ExecutionEvent {
11120        use nautilus_model::events::order::spec::OrderSubmittedSpec;
11121
11122        ExecutionEvent::Order(OrderEventAny::Submitted(
11123            OrderSubmittedSpec::builder().build(),
11124        ))
11125    }
11126
11127    fn stub_account_event() -> ExecutionEvent {
11128        use nautilus_core::{UUID4, UnixNanos};
11129        use nautilus_model::{
11130            enums::AccountType, events::account::state::AccountState, identifiers::AccountId,
11131        };
11132
11133        ExecutionEvent::Account(AccountState::new(
11134            AccountId::from("TEST-001"),
11135            AccountType::Cash,
11136            vec![],
11137            vec![],
11138            true,
11139            UUID4::new(),
11140            UnixNanos::default(),
11141            UnixNanos::default(),
11142            None,
11143        ))
11144    }
11145
11146    #[rstest]
11147    fn test_flush_all_pending_routes_order_event_to_order_evts() {
11148        let (_time_tx, mut time_rx) =
11149            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11150        let (_system_evt_tx, mut system_evt_rx) =
11151            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11152        let (_system_cmd_tx, mut system_cmd_rx) =
11153            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11154        let (_data_evt_tx, mut data_evt_rx) =
11155            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11156        let (_data_cmd_tx, mut data_cmd_rx) =
11157            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11158        let (exec_evt_tx, mut exec_evt_rx) =
11159            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11160        let (_exec_cmd_tx, mut exec_cmd_rx) =
11161            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11162
11163        let mut pending = PendingEvents::default();
11164
11165        exec_evt_tx.send((stub_order_event()).into()).unwrap();
11166        exec_evt_tx.send((stub_exec_event()).into()).unwrap();
11167
11168        flush_all_pending(
11169            &mut pending,
11170            &mut time_rx,
11171            &mut system_evt_rx,
11172            &mut system_cmd_rx,
11173            &mut exec_evt_rx,
11174            &mut exec_cmd_rx,
11175            &mut data_evt_rx,
11176            &mut data_cmd_rx,
11177        );
11178
11179        // Both order and report events are drained by pending.drain()
11180        assert!(pending.order_evts.is_empty());
11181        assert!(pending.exec_reports.is_empty());
11182        assert!(exec_evt_rx.try_recv().is_err());
11183    }
11184
11185    #[rstest]
11186    fn test_flush_all_pending_routes_account_event_immediately() {
11187        let (_time_tx, mut time_rx) =
11188            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11189        let (_system_evt_tx, mut system_evt_rx) =
11190            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11191        let (_system_cmd_tx, mut system_cmd_rx) =
11192            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11193        let (_data_evt_tx, mut data_evt_rx) =
11194            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11195        let (_data_cmd_tx, mut data_cmd_rx) =
11196            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11197        let (exec_evt_tx, mut exec_evt_rx) =
11198            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11199        let (_exec_cmd_tx, mut exec_cmd_rx) =
11200            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11201
11202        let mut pending = PendingEvents::default();
11203
11204        exec_evt_tx.send((stub_account_event()).into()).unwrap();
11205
11206        flush_all_pending(
11207            &mut pending,
11208            &mut time_rx,
11209            &mut system_evt_rx,
11210            &mut system_cmd_rx,
11211            &mut exec_evt_rx,
11212            &mut exec_cmd_rx,
11213            &mut data_evt_rx,
11214            &mut data_cmd_rx,
11215        );
11216
11217        // Account events are forwarded immediately, never buffered in pending
11218        assert!(pending.exec_reports.is_empty());
11219        assert!(pending.order_evts.is_empty());
11220        assert!(pending.exec_cmds.is_empty());
11221        assert!(exec_evt_rx.try_recv().is_err());
11222    }
11223
11224    #[rstest]
11225    fn test_pending_is_empty_when_default() {
11226        let pending = PendingEvents::default();
11227
11228        assert!(pending.is_empty());
11229    }
11230
11231    #[rstest]
11232    fn test_pending_is_empty_false_with_data_evt() {
11233        let mut pending = PendingEvents::default();
11234        pending.data_evts.push((stub_data_event()).into());
11235
11236        assert!(!pending.is_empty());
11237    }
11238
11239    #[rstest]
11240    fn test_pending_is_empty_false_with_data_cmd() {
11241        let mut pending = PendingEvents::default();
11242        pending.data_cmds.push(stub_data_command().into());
11243
11244        assert!(!pending.is_empty());
11245    }
11246
11247    #[rstest]
11248    fn test_pending_is_empty_false_with_exec_cmd() {
11249        let mut pending = PendingEvents::default();
11250        pending
11251            .exec_cmds
11252            .push(stub_trading_command_message().into());
11253
11254        assert!(!pending.is_empty());
11255    }
11256
11257    #[rstest]
11258    fn test_pending_drain_preserves_trading_command_target() {
11259        std::thread::spawn(|| {
11260            msgbus::get_message_bus().borrow_mut().dispose();
11261            let risk_commands = Rc::new(RefCell::new(Vec::new()));
11262            let exec_commands = Rc::new(RefCell::new(Vec::new()));
11263
11264            let risk_commands_handler = risk_commands.clone();
11265            msgbus::register_trading_command_endpoint(
11266                MessagingSwitchboard::risk_engine_execute(),
11267                TypedIntoHandler::from(move |command: TradingCommand| {
11268                    risk_commands_handler.borrow_mut().push(command);
11269                }),
11270            );
11271
11272            let exec_commands_handler = exec_commands.clone();
11273            msgbus::register_trading_command_endpoint(
11274                MessagingSwitchboard::exec_engine_execute(),
11275                TypedIntoHandler::from(move |command: TradingCommand| {
11276                    exec_commands_handler.borrow_mut().push(command);
11277                }),
11278            );
11279
11280            let mut pending = PendingEvents::default();
11281            pending.exec_cmds.push(
11282                TradingCommandMessage::new(
11283                    MessagingSwitchboard::risk_engine_execute(),
11284                    TradingCommand::QueryAccount(QueryAccount::new(
11285                        TraderId::from("TESTER-001"),
11286                        None,
11287                        AccountId::from("TEST-001"),
11288                        UUID4::new(),
11289                        UnixNanos::default(),
11290                        None,
11291                        None,
11292                    )),
11293                )
11294                .into(),
11295            );
11296
11297            pending.drain();
11298
11299            assert!(pending.is_empty());
11300            assert_eq!(risk_commands.borrow().len(), 1);
11301            assert!(matches!(
11302                &risk_commands.borrow()[0],
11303                TradingCommand::QueryAccount(_)
11304            ));
11305            assert_eq!(exec_commands.borrow().as_slice(), &[]);
11306        })
11307        .join()
11308        .unwrap();
11309    }
11310
11311    #[rstest]
11312    fn test_pending_is_empty_false_with_exec_report() {
11313        let mut pending = PendingEvents::default();
11314
11315        if let ExecutionEvent::Report(report) = stub_exec_event() {
11316            pending.exec_reports.push((report).into());
11317        }
11318
11319        assert!(!pending.is_empty());
11320    }
11321
11322    #[rstest]
11323    fn test_pending_is_empty_false_with_order_evt() {
11324        let mut pending = PendingEvents::default();
11325
11326        if let ExecutionEvent::Order(order_evt) = stub_order_event() {
11327            pending.order_evts.push((order_evt).into());
11328        }
11329
11330        assert!(!pending.is_empty());
11331    }
11332
11333    fn stub_submitted_batch_event() -> ExecutionEvent {
11334        use nautilus_model::{
11335            events::{OrderSubmittedBatch, order::spec::OrderSubmittedSpec},
11336            identifiers::ClientOrderId,
11337        };
11338
11339        let events = vec![
11340            OrderSubmittedSpec::builder()
11341                .client_order_id(ClientOrderId::from("O-001"))
11342                .build(),
11343            OrderSubmittedSpec::builder()
11344                .client_order_id(ClientOrderId::from("O-002"))
11345                .build(),
11346        ];
11347
11348        ExecutionEvent::OrderSubmittedBatch(OrderSubmittedBatch::new(events))
11349    }
11350
11351    fn stub_canceled_batch_event() -> ExecutionEvent {
11352        use nautilus_model::{
11353            events::{OrderCanceledBatch, order::spec::OrderCanceledSpec},
11354            identifiers::ClientOrderId,
11355        };
11356
11357        let events = vec![
11358            OrderCanceledSpec::builder()
11359                .client_order_id(ClientOrderId::from("O-001"))
11360                .build(),
11361            OrderCanceledSpec::builder()
11362                .client_order_id(ClientOrderId::from("O-002"))
11363                .build(),
11364        ];
11365
11366        ExecutionEvent::OrderCanceledBatch(OrderCanceledBatch::new(events))
11367    }
11368
11369    #[rstest]
11370    fn test_flush_all_pending_buffers_submitted_batch_as_individual_events() {
11371        let (_time_tx, mut time_rx) =
11372            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11373        let (_system_evt_tx, mut system_evt_rx) =
11374            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11375        let (_system_cmd_tx, mut system_cmd_rx) =
11376            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11377        let (_data_evt_tx, mut data_evt_rx) =
11378            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11379        let (_data_cmd_tx, mut data_cmd_rx) =
11380            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11381        let (exec_evt_tx, mut exec_evt_rx) =
11382            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11383        let (_exec_cmd_tx, mut exec_cmd_rx) =
11384            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11385
11386        let mut pending = PendingEvents::default();
11387
11388        exec_evt_tx
11389            .send((stub_submitted_batch_event()).into())
11390            .unwrap();
11391
11392        flush_all_pending(
11393            &mut pending,
11394            &mut time_rx,
11395            &mut system_evt_rx,
11396            &mut system_cmd_rx,
11397            &mut exec_evt_rx,
11398            &mut exec_cmd_rx,
11399            &mut data_evt_rx,
11400            &mut data_cmd_rx,
11401        );
11402
11403        // Batch should be unpacked into individual Submitted events then drained
11404        assert!(pending.order_evts.is_empty());
11405        assert!(exec_evt_rx.try_recv().is_err());
11406    }
11407
11408    #[rstest]
11409    fn test_flush_all_pending_buffers_canceled_batch_as_individual_events() {
11410        let (_time_tx, mut time_rx) =
11411            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TimeEventMessage>>();
11412        let (_system_evt_tx, mut system_evt_rx) =
11413            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemEvent>>();
11414        let (_system_cmd_tx, mut system_cmd_rx) =
11415            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<SystemCommand>>();
11416        let (_data_evt_tx, mut data_evt_rx) =
11417            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataEvent>>();
11418        let (_data_cmd_tx, mut data_cmd_rx) =
11419            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<DataCommand>>();
11420        let (exec_evt_tx, mut exec_evt_rx) =
11421            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11422        let (_exec_cmd_tx, mut exec_cmd_rx) =
11423            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<TradingCommandMessage>>();
11424
11425        let mut pending = PendingEvents::default();
11426
11427        exec_evt_tx
11428            .send((stub_canceled_batch_event()).into())
11429            .unwrap();
11430
11431        flush_all_pending(
11432            &mut pending,
11433            &mut time_rx,
11434            &mut system_evt_rx,
11435            &mut system_cmd_rx,
11436            &mut exec_evt_rx,
11437            &mut exec_cmd_rx,
11438            &mut data_evt_rx,
11439            &mut data_cmd_rx,
11440        );
11441
11442        // Batch should be unpacked into individual Canceled events then drained
11443        assert!(pending.order_evts.is_empty());
11444        assert!(exec_evt_rx.try_recv().is_err());
11445    }
11446
11447    #[rstest]
11448    fn test_flush_all_pending_expands_batch_into_order_evts_before_drain() {
11449        use nautilus_model::identifiers::ClientOrderId;
11450
11451        let (exec_evt_tx, mut exec_evt_rx) =
11452            tokio::sync::mpsc::unbounded_channel::<DispatchMessage<ExecutionEvent>>();
11453
11454        exec_evt_tx
11455            .send((stub_canceled_batch_event()).into())
11456            .unwrap();
11457
11458        let mut pending = PendingEvents::default();
11459
11460        while let Ok(evt) = exec_evt_rx.try_recv() {
11461            pending.push_exec_event(evt);
11462        }
11463
11464        let events: Vec<_> = pending
11465            .order_evts
11466            .drain(..)
11467            .map(|event| event.dispatch(|event| event))
11468            .collect();
11469        assert_eq!(events.len(), 2);
11470        assert!(
11471            matches!(&events[0], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-001"))
11472        );
11473        assert!(
11474            matches!(&events[1], OrderEventAny::Canceled(c) if c.client_order_id == ClientOrderId::from("O-002"))
11475        );
11476    }
11477
11478    #[rstest]
11479    #[cfg_attr(
11480        not(all(feature = "simulation", madsim)),
11481        tokio::test(start_paused = true)
11482    )]
11483    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11484    async fn test_submission_confirmation_prevents_duplicate_tracking(
11485        #[values(false, true)] report_confirmation: bool,
11486        #[values(false, true)] exhausted: bool,
11487    ) {
11488        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
11489        let client_order_id = order.client_order_id();
11490        let account_id = AccountId::from("TEST-001");
11491        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11492        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11493            OrderPendingUpdateSpec::builder()
11494                .trader_id(order.trader_id())
11495                .strategy_id(order.strategy_id())
11496                .instrument_id(order.instrument_id())
11497                .client_order_id(client_order_id)
11498                .account_id(account_id)
11499                .build(),
11500        )));
11501        advance_clock(Duration::from_millis(101)).await;
11502        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11503
11504        if exhausted {
11505            advance_clock(Duration::from_millis(101)).await;
11506            let result = node.exec_manager.check_inflight_orders();
11507            assert!(result.events.is_empty());
11508            assert!(result.queries.is_empty());
11509            assert_eq!(
11510                node.exec_manager
11511                    .take_submission_recovery_exhaustions()
11512                    .len(),
11513                1
11514            );
11515        }
11516
11517        if report_confirmation {
11518            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11519                OrderStatusReport::new(
11520                    account_id,
11521                    order.instrument_id(),
11522                    Some(client_order_id),
11523                    venue_order_id,
11524                    Some(order.order_side()),
11525                    order.order_type(),
11526                    order.time_in_force(),
11527                    OrderStatus::Submitted,
11528                    order.quantity(),
11529                    Quantity::zero(3),
11530                    UnixNanos::default(),
11531                    UnixNanos::default(),
11532                    UnixNanos::default(),
11533                    None,
11534                )
11535                .with_price(order.price().unwrap()),
11536            ))));
11537            node.exec_manager
11538                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11539            advance_clock(Duration::from_millis(101)).await;
11540            let result = node.exec_manager.check_inflight_orders();
11541            assert!(result.queries.is_empty());
11542            assert!(result.events.is_empty());
11543        }
11544
11545        let mut update = OrderUpdatedSpec::builder()
11546            .trader_id(order.trader_id())
11547            .strategy_id(order.strategy_id())
11548            .instrument_id(order.instrument_id())
11549            .client_order_id(client_order_id)
11550            .account_id(account_id)
11551            .quantity(order.quantity())
11552            .price(order.price().unwrap())
11553            .build();
11554
11555        if !report_confirmation {
11556            update.venue_order_id = Some(venue_order_id);
11557        }
11558        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11559        assert_eq!(
11560            node.kernel
11561                .cache
11562                .borrow()
11563                .order(&client_order_id)
11564                .unwrap()
11565                .status(),
11566            OrderStatus::Submitted
11567        );
11568
11569        for _ in 0..3 {
11570            node.exec_manager
11571                .register_submission(order.init_event(), Some(ClientId::from("OTHER")));
11572            advance_clock(Duration::from_millis(101)).await;
11573            let result = node.exec_manager.check_inflight_orders();
11574            assert!(result.queries.is_empty());
11575            assert!(result.events.is_empty());
11576            assert!(
11577                node.exec_manager
11578                    .take_submission_recovery_exhaustions()
11579                    .is_empty()
11580            );
11581        }
11582    }
11583
11584    #[rstest]
11585    #[cfg_attr(
11586        not(all(feature = "simulation", madsim)),
11587        tokio::test(start_paused = true)
11588    )]
11589    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11590    async fn test_submission_report_confirmation_preserves_later_command_budget(
11591        #[values(false, true)] cancel: bool,
11592        #[values(false, true)] later_fill: bool,
11593        #[values(false, true)] repeated_report: bool,
11594        #[values(
11595            SubmissionRecoveryPolicy::ResolveLocally,
11596            SubmissionRecoveryPolicy::RetainUnresolved
11597        )]
11598        policy: SubmissionRecoveryPolicy,
11599    ) {
11600        let (mut node, order, instrument) = venue_evidence_node(policy);
11601        let client_order_id = order.client_order_id();
11602        let account_id = AccountId::from("TEST-001");
11603        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11604        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11605            OrderPendingUpdateSpec::builder()
11606                .trader_id(order.trader_id())
11607                .strategy_id(order.strategy_id())
11608                .instrument_id(order.instrument_id())
11609                .client_order_id(client_order_id)
11610                .account_id(account_id)
11611                .build(),
11612        )));
11613        node.exec_manager
11614            .register_inflight(client_order_id, UUID4::new());
11615        advance_clock(Duration::from_millis(101)).await;
11616        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11617        let report = ExecutionReport::Order(Box::new(
11618            OrderStatusReport::new(
11619                account_id,
11620                order.instrument_id(),
11621                Some(client_order_id),
11622                venue_order_id,
11623                Some(order.order_side()),
11624                order.order_type(),
11625                order.time_in_force(),
11626                OrderStatus::Submitted,
11627                order.quantity(),
11628                Quantity::zero(3),
11629                UnixNanos::default(),
11630                UnixNanos::default(),
11631                UnixNanos::default(),
11632                None,
11633            )
11634            .with_price(order.price().unwrap()),
11635        ));
11636        node.process_exec_event(ExecutionEvent::Report(report.clone()));
11637        advance_clock(Duration::from_millis(101)).await;
11638        let result = node.exec_manager.check_inflight_orders();
11639        assert!(result.events.is_empty());
11640        assert!(result.queries.is_empty());
11641
11642        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(
11643            OrderUpdatedSpec::builder()
11644                .trader_id(order.trader_id())
11645                .strategy_id(order.strategy_id())
11646                .instrument_id(order.instrument_id())
11647                .client_order_id(client_order_id)
11648                .account_id(account_id)
11649                .quantity(order.quantity())
11650                .price(order.price().unwrap())
11651                .build(),
11652        )));
11653        let pending = if cancel {
11654            OrderEventAny::PendingCancel(
11655                OrderPendingCancelSpec::builder()
11656                    .trader_id(order.trader_id())
11657                    .strategy_id(order.strategy_id())
11658                    .instrument_id(order.instrument_id())
11659                    .client_order_id(client_order_id)
11660                    .account_id(account_id)
11661                    .build(),
11662            )
11663        } else {
11664            OrderEventAny::PendingUpdate(
11665                OrderPendingUpdateSpec::builder()
11666                    .trader_id(order.trader_id())
11667                    .strategy_id(order.strategy_id())
11668                    .instrument_id(order.instrument_id())
11669                    .client_order_id(client_order_id)
11670                    .account_id(account_id)
11671                    .build(),
11672            )
11673        };
11674        node.process_exec_event(ExecutionEvent::Order(pending));
11675        node.exec_manager
11676            .register_inflight(client_order_id, UUID4::new());
11677
11678        if repeated_report {
11679            node.process_exec_event(ExecutionEvent::Report(report));
11680        }
11681        advance_clock(Duration::from_millis(50)).await;
11682
11683        if later_fill {
11684            let cached = node
11685                .kernel
11686                .cache
11687                .borrow()
11688                .order_owned(&client_order_id)
11689                .unwrap();
11690            let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
11691                .last_qty(Quantity::from("0.400"))
11692                .last_px(order.price().unwrap())
11693                .commission(Money::from("0 USDT"))
11694                .without_position_id()
11695                .build()
11696            else {
11697                unreachable!()
11698            };
11699            fill.venue_order_id = venue_order_id;
11700            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
11701        }
11702        let early = node.exec_manager.check_inflight_orders();
11703        assert!(early.events.is_empty());
11704        assert!(early.queries.is_empty());
11705        advance_clock(Duration::from_millis(51)).await;
11706        let first = node.exec_manager.check_inflight_orders();
11707        assert_eq!(first.queries.len(), usize::from(!repeated_report));
11708        assert!(first.events.is_empty());
11709        advance_clock(Duration::from_millis(101)).await;
11710        let second = node.exec_manager.check_inflight_orders();
11711        assert!(second.queries.is_empty());
11712        if repeated_report {
11713            assert!(second.events.is_empty());
11714        } else {
11715            assert_command_timeout_event(&second.events, client_order_id, policy, cancel);
11716        }
11717        assert!(
11718            node.exec_manager
11719                .take_submission_recovery_exhaustions()
11720                .is_empty()
11721        );
11722    }
11723
11724    #[rstest]
11725    #[cfg_attr(
11726        not(all(feature = "simulation", madsim)),
11727        tokio::test(start_paused = true)
11728    )]
11729    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11730    async fn test_confirmed_submission_command_dispatch_starts_own_budget(
11731        #[values(false, true)] report_confirmation: bool,
11732        #[values(false, true)] cancel: bool,
11733        #[values(false, true)] batch: bool,
11734        #[values(false, true)] pending_before_dispatch: bool,
11735        #[values(false, true)] missing_order_check: bool,
11736        #[values(
11737            SubmissionRecoveryPolicy::ResolveLocally,
11738            SubmissionRecoveryPolicy::RetainUnresolved
11739        )]
11740        policy: SubmissionRecoveryPolicy,
11741    ) {
11742        let (mut node, order, instrument) = venue_evidence_node_with_config(
11743            crate::config::LiveExecutionEngineConfig {
11744                reconciliation: true,
11745                inflight_check_threshold_ms: 100,
11746                inflight_check_retries: 2,
11747                open_check_threshold_ms: 5_000,
11748                open_check_open_only: !missing_order_check,
11749                open_check_lookback_mins: None,
11750                open_check_missing_retries: 5,
11751                single_order_query_delay_ms: 0,
11752                submission_recovery_policy: policy,
11753                ..Default::default()
11754            },
11755            OrderType::Limit,
11756            true,
11757        );
11758        let client_order_id = order.client_order_id();
11759        let client_id = ClientId::from("TEST-EVIDENCE");
11760        let account_id = AccountId::from("TEST-001");
11761        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11762        node.kernel
11763            .exec_engine
11764            .borrow_mut()
11765            .register_client(Box::new(StubExecutionClient::new(
11766                client_id,
11767                account_id,
11768                instrument.id().venue,
11769                OmsType::Netting,
11770                None,
11771            )))
11772            .unwrap();
11773
11774        // Consume part of the original submission budget before venue confirmation
11775        advance_clock(Duration::from_millis(101)).await;
11776        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
11777
11778        if report_confirmation {
11779            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
11780                OrderPendingUpdateSpec::builder()
11781                    .trader_id(order.trader_id())
11782                    .strategy_id(order.strategy_id())
11783                    .instrument_id(order.instrument_id())
11784                    .client_order_id(client_order_id)
11785                    .account_id(account_id)
11786                    .build(),
11787            )));
11788            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
11789                OrderStatusReport::new(
11790                    account_id,
11791                    order.instrument_id(),
11792                    Some(client_order_id),
11793                    venue_order_id,
11794                    Some(order.order_side()),
11795                    order.order_type(),
11796                    order.time_in_force(),
11797                    OrderStatus::Submitted,
11798                    order.quantity(),
11799                    Quantity::zero(3),
11800                    UnixNanos::default(),
11801                    UnixNanos::default(),
11802                    UnixNanos::default(),
11803                    None,
11804                )
11805                .with_price(order.price().unwrap()),
11806            ))));
11807        }
11808
11809        let mut update = OrderUpdatedSpec::builder()
11810            .trader_id(order.trader_id())
11811            .strategy_id(order.strategy_id())
11812            .instrument_id(order.instrument_id())
11813            .client_order_id(client_order_id)
11814            .account_id(account_id)
11815            .quantity(order.quantity())
11816            .price(order.price().unwrap())
11817            .build();
11818
11819        if !report_confirmation {
11820            update.venue_order_id = Some(venue_order_id);
11821        }
11822        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(update)));
11823        assert_eq!(
11824            node.kernel
11825                .cache
11826                .borrow()
11827                .order(&client_order_id)
11828                .unwrap()
11829                .status(),
11830            OrderStatus::Submitted,
11831        );
11832        advance_clock(Duration::from_millis(101)).await;
11833        let confirmed = node.exec_manager.check_inflight_orders();
11834        assert!(confirmed.queries.is_empty());
11835        assert!(confirmed.events.is_empty());
11836
11837        let pending = if cancel {
11838            OrderEventAny::PendingCancel(
11839                OrderPendingCancelSpec::builder()
11840                    .trader_id(order.trader_id())
11841                    .strategy_id(order.strategy_id())
11842                    .instrument_id(order.instrument_id())
11843                    .client_order_id(client_order_id)
11844                    .account_id(account_id)
11845                    .build(),
11846            )
11847        } else {
11848            OrderEventAny::PendingUpdate(
11849                OrderPendingUpdateSpec::builder()
11850                    .trader_id(order.trader_id())
11851                    .strategy_id(order.strategy_id())
11852                    .instrument_id(order.instrument_id())
11853                    .client_order_id(client_order_id)
11854                    .account_id(account_id)
11855                    .build(),
11856            )
11857        };
11858        let command = venue_evidence_command(&order, cancel, batch);
11859
11860        if pending_before_dispatch {
11861            node.process_exec_event(ExecutionEvent::Order(pending.clone()));
11862        }
11863        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11864            MessagingSwitchboard::exec_engine_execute(),
11865            command,
11866        )));
11867        advance_clock(Duration::from_millis(50)).await;
11868
11869        if missing_order_check {
11870            let engine = node.kernel.exec_engine.borrow();
11871            let clients = engine.get_all_clients();
11872            let check = node
11873                .exec_manager
11874                .prepare_open_order_report_check(UUID4::new(), &clients);
11875            assert_eq!(check.filtered_orders.len(), 1);
11876            let missing = node.exec_manager.reconcile_open_order_reports(
11877                &check,
11878                Vec::new(),
11879                &IndexSet::from([client_id]),
11880                &IndexSet::new(),
11881                &clients,
11882            );
11883            assert!(missing.events.is_empty());
11884            assert!(missing.targeted_queries.is_empty());
11885            assert_eq!(
11886                node.exec_manager.recon_check_retry_count(&client_order_id),
11887                1
11888            );
11889        }
11890        let early = node.exec_manager.check_inflight_orders();
11891        assert!(early.queries.is_empty());
11892        assert!(early.events.is_empty());
11893
11894        if !pending_before_dispatch {
11895            // The queued pending event must not be needed to register the command,
11896            // nor shift its timeout start from dispatch to event application.
11897            assert_eq!(
11898                node.kernel
11899                    .cache
11900                    .borrow()
11901                    .order(&client_order_id)
11902                    .unwrap()
11903                    .status(),
11904                OrderStatus::Submitted,
11905            );
11906            node.process_exec_event(ExecutionEvent::Order(pending));
11907        }
11908
11909        if policy == SubmissionRecoveryPolicy::RetainUnresolved {
11910            node.exec_manager
11911                .register_submission(order.init_event(), Some(client_id));
11912        }
11913        advance_clock(Duration::from_millis(51)).await;
11914        let first = node.exec_manager.check_inflight_orders();
11915        assert_eq!(first.queries.len(), 1);
11916        assert!(
11917            matches!(&first.queries[0], TradingCommand::QueryOrder(query)
11918            if query.client_order_id == client_order_id)
11919        );
11920        assert!(first.events.is_empty());
11921        advance_clock(Duration::from_millis(101)).await;
11922        let exhausted = node.exec_manager.check_inflight_orders();
11923        assert!(exhausted.queries.is_empty());
11924        assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
11925        assert!(
11926            node.exec_manager
11927                .take_submission_recovery_exhaustions()
11928                .is_empty()
11929        );
11930        node.process_reconciliation_events(&exhausted.events);
11931
11932        let expected_status = match policy {
11933            SubmissionRecoveryPolicy::ResolveLocally => OrderStatus::Canceled,
11934            SubmissionRecoveryPolicy::RetainUnresolved => OrderStatus::Submitted,
11935        };
11936
11937        assert_eq!(
11938            node.kernel
11939                .cache
11940                .borrow()
11941                .order(&client_order_id)
11942                .unwrap()
11943                .status(),
11944            expected_status,
11945        );
11946    }
11947
11948    #[rstest]
11949    #[cfg_attr(
11950        not(all(feature = "simulation", madsim)),
11951        tokio::test(start_paused = true)
11952    )]
11953    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
11954    async fn test_submission_command_report_cleanup_matches_default(
11955        #[values(OrderStatus::Submitted, OrderStatus::PartiallyFilled)] report_status: OrderStatus,
11956        #[values(
11957            SubmissionRecoveryPolicy::ResolveLocally,
11958            SubmissionRecoveryPolicy::RetainUnresolved
11959        )]
11960        policy: SubmissionRecoveryPolicy,
11961    ) {
11962        let (mut node, order, instrument) = venue_evidence_node(policy);
11963        let client_order_id = order.client_order_id();
11964        let client_id = ClientId::from("TEST-EVIDENCE");
11965        let account_id = AccountId::from("TEST-001");
11966        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
11967        node.kernel
11968            .exec_engine
11969            .borrow_mut()
11970            .register_client(Box::new(StubExecutionClient::new(
11971                client_id,
11972                account_id,
11973                instrument.id().venue,
11974                OmsType::Netting,
11975                None,
11976            )))
11977            .unwrap();
11978        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
11979            &order,
11980            account_id,
11981            venue_order_id,
11982        )));
11983        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
11984            MessagingSwitchboard::exec_engine_execute(),
11985            venue_evidence_command(&order, true, false),
11986        )));
11987        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
11988            OrderPendingCancelSpec::builder()
11989                .trader_id(order.trader_id())
11990                .strategy_id(order.strategy_id())
11991                .instrument_id(order.instrument_id())
11992                .client_order_id(client_order_id)
11993                .account_id(account_id)
11994                .venue_order_id(venue_order_id)
11995                .build(),
11996        )));
11997        advance_clock(Duration::from_millis(101)).await;
11998        let first = node.exec_manager.check_inflight_orders();
11999        assert_eq!(first.queries.len(), 1);
12000        assert!(first.events.is_empty());
12001
12002        let filled_qty = if report_status == OrderStatus::PartiallyFilled {
12003            Quantity::from("0.400")
12004        } else {
12005            Quantity::zero(3)
12006        };
12007
12008        let report = OrderStatusReport::new(
12009            account_id,
12010            order.instrument_id(),
12011            Some(client_order_id),
12012            venue_order_id,
12013            Some(order.order_side()),
12014            order.order_type(),
12015            order.time_in_force(),
12016            report_status,
12017            order.quantity(),
12018            filled_qty,
12019            UnixNanos::default(),
12020            UnixNanos::default(),
12021            UnixNanos::default(),
12022            None,
12023        )
12024        .with_price(order.price().unwrap())
12025        .with_avg_px(dec!(100.00));
12026        node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12027            report,
12028        ))));
12029        advance_clock(Duration::from_millis(101)).await;
12030        let exhausted = node.exec_manager.check_inflight_orders();
12031
12032        assert!(exhausted.queries.is_empty());
12033        assert!(exhausted.events.is_empty());
12034        assert!(
12035            node.exec_manager
12036                .take_submission_recovery_exhaustions()
12037                .is_empty()
12038        );
12039        let cache = node.kernel.cache.borrow();
12040        let cached = cache.order_ref(&client_order_id).unwrap();
12041        assert_eq!(cached.status(), OrderStatus::PendingCancel);
12042        assert_eq!(cached.filled_qty(), filled_qty);
12043    }
12044
12045    #[rstest]
12046    #[case::accepted_modify(false, "accepted")]
12047    #[case::accepted_cancel(true, "accepted")]
12048    #[case::accepted_batch_modify(false, "accepted_batch")]
12049    #[case::accepted_batch_cancel(true, "accepted_batch")]
12050    #[case::updated_cancel(true, "updated")]
12051    #[case::triggered_cancel(true, "triggered")]
12052    #[cfg_attr(
12053        not(all(feature = "simulation", madsim)),
12054        tokio::test(start_paused = true)
12055    )]
12056    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12057    async fn test_submission_acknowledgement_retires_overlapping_command_recovery(
12058        #[case] cancel: bool,
12059        #[case] evidence: &str,
12060        #[values(false, true)] batch: bool,
12061        #[values(false, true)] report_first: bool,
12062        #[values(
12063            SubmissionRecoveryPolicy::ResolveLocally,
12064            SubmissionRecoveryPolicy::RetainUnresolved
12065        )]
12066        policy: SubmissionRecoveryPolicy,
12067    ) {
12068        use nautilus_model::events::order::spec::OrderTriggeredSpec;
12069
12070        let triggered = evidence == "triggered";
12071
12072        let order_type = if triggered {
12073            OrderType::StopLimit
12074        } else {
12075            OrderType::Limit
12076        };
12077        let (mut node, order, instrument) = venue_evidence_node_with_order_type(policy, order_type);
12078        let client_order_id = order.client_order_id();
12079        let account_id = AccountId::from("TEST-001");
12080        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12081        node.kernel
12082            .exec_engine
12083            .borrow_mut()
12084            .register_client(Box::new(StubExecutionClient::new(
12085                ClientId::from("TEST-EVIDENCE"),
12086                account_id,
12087                instrument.id().venue,
12088                OmsType::Netting,
12089                None,
12090            )))
12091            .unwrap();
12092        advance_clock(Duration::from_millis(101)).await;
12093        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12094
12095        if triggered || report_first {
12096            // Evidence can overtake a different, queued command intent
12097            let previous_pending = if cancel {
12098                OrderEventAny::PendingUpdate(
12099                    OrderPendingUpdateSpec::builder()
12100                        .trader_id(order.trader_id())
12101                        .strategy_id(order.strategy_id())
12102                        .instrument_id(order.instrument_id())
12103                        .client_order_id(client_order_id)
12104                        .account_id(account_id)
12105                        .build(),
12106                )
12107            } else {
12108                OrderEventAny::PendingCancel(
12109                    OrderPendingCancelSpec::builder()
12110                        .trader_id(order.trader_id())
12111                        .strategy_id(order.strategy_id())
12112                        .instrument_id(order.instrument_id())
12113                        .client_order_id(client_order_id)
12114                        .account_id(account_id)
12115                        .build(),
12116                )
12117            };
12118            node.process_exec_event(ExecutionEvent::Order(previous_pending));
12119        }
12120        advance_clock(Duration::from_millis(10)).await;
12121        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12122            MessagingSwitchboard::exec_engine_execute(),
12123            venue_evidence_command(&order, cancel, batch),
12124        )));
12125
12126        if report_first {
12127            let report = OrderStatusReport::new(
12128                account_id,
12129                instrument.id(),
12130                Some(client_order_id),
12131                venue_order_id,
12132                Some(order.order_side()),
12133                order.order_type(),
12134                order.time_in_force(),
12135                OrderStatus::Submitted,
12136                order.quantity(),
12137                Quantity::zero(3),
12138                UnixNanos::default(),
12139                UnixNanos::default(),
12140                UnixNanos::default(),
12141                None,
12142            )
12143            .with_price(order.price().unwrap());
12144            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12145                report,
12146            ))));
12147            assert!(
12148                node.kernel
12149                    .cache
12150                    .borrow()
12151                    .order(&client_order_id)
12152                    .unwrap()
12153                    .venue_order_id()
12154                    .is_none()
12155            );
12156        }
12157        advance_clock(Duration::from_millis(40)).await;
12158        let acknowledgement = if triggered {
12159            OrderEventAny::Triggered(
12160                OrderTriggeredSpec::builder()
12161                    .trader_id(order.trader_id())
12162                    .strategy_id(order.strategy_id())
12163                    .instrument_id(order.instrument_id())
12164                    .client_order_id(client_order_id)
12165                    .account_id(account_id)
12166                    .venue_order_id(venue_order_id)
12167                    .build(),
12168            )
12169        } else if evidence == "updated" {
12170            OrderEventAny::Updated(
12171                OrderUpdatedSpec::builder()
12172                    .trader_id(order.trader_id())
12173                    .strategy_id(order.strategy_id())
12174                    .instrument_id(order.instrument_id())
12175                    .client_order_id(client_order_id)
12176                    .account_id(account_id)
12177                    .venue_order_id(venue_order_id)
12178                    .quantity(order.quantity())
12179                    .price(order.price().unwrap())
12180                    .build(),
12181            )
12182        } else {
12183            TestOrderEventStubs::accepted(&order, account_id, venue_order_id)
12184        };
12185
12186        if evidence == "accepted_batch" {
12187            let OrderEventAny::Accepted(accepted) = acknowledgement else {
12188                unreachable!()
12189            };
12190            node.process_exec_event(ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(
12191                vec![accepted],
12192            )));
12193        } else {
12194            node.process_exec_event(ExecutionEvent::Order(acknowledgement));
12195        }
12196        assert_eq!(
12197            node.kernel
12198                .cache
12199                .borrow()
12200                .order(&client_order_id)
12201                .unwrap()
12202                .status(),
12203            match evidence {
12204                "triggered" => OrderStatus::Triggered,
12205                "updated" => OrderStatus::Submitted,
12206                _ => OrderStatus::Accepted,
12207            }
12208        );
12209        advance_clock(Duration::from_millis(20)).await;
12210        let pending = if cancel {
12211            OrderEventAny::PendingCancel(
12212                OrderPendingCancelSpec::builder()
12213                    .trader_id(order.trader_id())
12214                    .strategy_id(order.strategy_id())
12215                    .instrument_id(order.instrument_id())
12216                    .client_order_id(client_order_id)
12217                    .account_id(account_id)
12218                    .build(),
12219            )
12220        } else {
12221            OrderEventAny::PendingUpdate(
12222                OrderPendingUpdateSpec::builder()
12223                    .trader_id(order.trader_id())
12224                    .strategy_id(order.strategy_id())
12225                    .instrument_id(order.instrument_id())
12226                    .client_order_id(client_order_id)
12227                    .account_id(account_id)
12228                    .build(),
12229            )
12230        };
12231        node.process_exec_event(ExecutionEvent::Order(pending));
12232        let early = node.exec_manager.check_inflight_orders();
12233        assert!(early.queries.is_empty());
12234        assert!(early.events.is_empty());
12235        advance_clock(Duration::from_millis(41)).await;
12236        let first = node.exec_manager.check_inflight_orders();
12237        // The default does not clear inflight tracking on Triggered events.
12238        let recover_command =
12239            triggered && !report_first && policy == SubmissionRecoveryPolicy::ResolveLocally;
12240        assert_eq!(first.queries.len(), usize::from(recover_command));
12241        assert!(first.events.is_empty());
12242        advance_clock(Duration::from_millis(101)).await;
12243        let exhausted = node.exec_manager.check_inflight_orders();
12244        assert!(exhausted.queries.is_empty());
12245        assert!(
12246            node.exec_manager
12247                .take_submission_recovery_exhaustions()
12248                .is_empty()
12249        );
12250
12251        if recover_command {
12252            assert!(
12253                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12254                if event.client_order_id == client_order_id)
12255            );
12256        } else {
12257            assert!(exhausted.events.is_empty());
12258        }
12259    }
12260
12261    #[rstest]
12262    #[case::submission_only(None, false)]
12263    #[case::modify(Some(false), false)]
12264    #[case::cancel(Some(true), false)]
12265    #[case::batch_modify(Some(false), true)]
12266    #[case::batch_cancel(Some(true), true)]
12267    #[cfg_attr(
12268        not(all(feature = "simulation", madsim)),
12269        tokio::test(start_paused = true)
12270    )]
12271    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12272    async fn test_unconfirmed_submission_bulk_confirmation_before_registration(
12273        #[case] cancel: Option<bool>,
12274        #[case] batch: bool,
12275        #[values(false, true)] missing_order_check: bool,
12276        #[values(
12277            SubmissionRecoveryPolicy::ResolveLocally,
12278            SubmissionRecoveryPolicy::RetainUnresolved
12279        )]
12280        policy: SubmissionRecoveryPolicy,
12281    ) {
12282        let (mut node, order, instrument) = venue_evidence_node_with_config(
12283            crate::config::LiveExecutionEngineConfig {
12284                reconciliation: true,
12285                inflight_check_threshold_ms: 100,
12286                inflight_check_retries: 2,
12287                open_check_threshold_ms: 1,
12288                open_check_open_only: false,
12289                open_check_lookback_mins: None,
12290                open_check_missing_retries: 5,
12291                single_order_query_delay_ms: 0,
12292                submission_recovery_policy: policy,
12293                ..Default::default()
12294            },
12295            OrderType::Limit,
12296            false,
12297        );
12298        let client_order_id = order.client_order_id();
12299        let client_id = ClientId::from("TEST-EVIDENCE");
12300        let account_id = AccountId::from("TEST-001");
12301        node.kernel
12302            .exec_engine
12303            .borrow_mut()
12304            .register_client(Box::new(StubExecutionClient::new(
12305                client_id,
12306                account_id,
12307                instrument.id().venue,
12308                OmsType::Netting,
12309                None,
12310            )))
12311            .unwrap();
12312        let queries = Rc::new(RefCell::new(Vec::new()));
12313        let client = OrderReportClient {
12314            client_id,
12315            report: OrderStatusReport::new(
12316                account_id,
12317                instrument.id(),
12318                Some(client_order_id),
12319                VenueOrderId::from("V-VENUE-EVIDENCE"),
12320                Some(order.order_side()),
12321                order.order_type(),
12322                order.time_in_force(),
12323                OrderStatus::Submitted,
12324                order.quantity(),
12325                Quantity::zero(3),
12326                UnixNanos::default(),
12327                UnixNanos::default(),
12328                UnixNanos::default(),
12329                None,
12330            )
12331            .with_price(order.price().unwrap()),
12332            targeted: false,
12333            queries: queries.clone(),
12334        };
12335        advance_clock(Duration::from_millis(2)).await;
12336        let events = node.exec_manager.check_open_orders(&[&client]).await;
12337        assert!(events.is_empty());
12338        assert_eq!(*queries.borrow(), vec!["bulk"]);
12339        assert_eq!(
12340            node.kernel
12341                .cache
12342                .borrow()
12343                .order(&client_order_id)
12344                .unwrap()
12345                .status(),
12346            OrderStatus::Submitted
12347        );
12348        advance_clock(Duration::from_millis(2)).await;
12349
12350        if missing_order_check {
12351            let engine = node.kernel.exec_engine.borrow();
12352            let clients = engine.get_all_clients();
12353            let check = node
12354                .exec_manager
12355                .prepare_open_order_report_check(UUID4::new(), &clients);
12356            assert_eq!(check.filtered_orders.len(), 1);
12357            let missing = node.exec_manager.reconcile_open_order_reports(
12358                &check,
12359                Vec::new(),
12360                &IndexSet::from([client_id]),
12361                &IndexSet::new(),
12362                &clients,
12363            );
12364            assert!(missing.events.is_empty());
12365            assert!(missing.targeted_queries.is_empty());
12366            assert_eq!(
12367                node.exec_manager.recon_check_retry_count(&client_order_id),
12368                1
12369            );
12370        }
12371        node.exec_manager
12372            .register_submission(order.init_event(), Some(client_id));
12373        advance_clock(Duration::from_millis(101)).await;
12374        let duplicate = node.exec_manager.check_inflight_orders();
12375        assert!(duplicate.events.is_empty());
12376        assert_eq!(
12377            duplicate.queries.len(),
12378            usize::from(policy == SubmissionRecoveryPolicy::ResolveLocally)
12379        );
12380
12381        if let Some(cancel) = cancel {
12382            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12383                MessagingSwitchboard::exec_engine_execute(),
12384                venue_evidence_command(&order, cancel, batch),
12385            )));
12386            let pending = if cancel {
12387                OrderEventAny::PendingCancel(
12388                    OrderPendingCancelSpec::builder()
12389                        .trader_id(order.trader_id())
12390                        .strategy_id(order.strategy_id())
12391                        .instrument_id(order.instrument_id())
12392                        .client_order_id(client_order_id)
12393                        .account_id(account_id)
12394                        .build(),
12395                )
12396            } else {
12397                OrderEventAny::PendingUpdate(
12398                    OrderPendingUpdateSpec::builder()
12399                        .trader_id(order.trader_id())
12400                        .strategy_id(order.strategy_id())
12401                        .instrument_id(order.instrument_id())
12402                        .client_order_id(client_order_id)
12403                        .account_id(account_id)
12404                        .build(),
12405                )
12406            };
12407            node.process_exec_event(ExecutionEvent::Order(pending));
12408        }
12409        advance_clock(Duration::from_millis(101)).await;
12410        let first = node.exec_manager.check_inflight_orders();
12411        assert_eq!(first.queries.len(), usize::from(cancel.is_some()));
12412
12413        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12414            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12415                if event.client_order_id == client_order_id));
12416        } else {
12417            assert!(first.events.is_empty());
12418        }
12419        node.process_reconciliation_events(&first.events);
12420        advance_clock(Duration::from_millis(101)).await;
12421        let exhausted = node.exec_manager.check_inflight_orders();
12422        assert!(exhausted.queries.is_empty());
12423
12424        if let Some(cancel) = cancel {
12425            assert_command_timeout_event(&exhausted.events, client_order_id, policy, cancel);
12426        } else {
12427            assert!(exhausted.events.is_empty());
12428        }
12429        assert!(
12430            node.exec_manager
12431                .take_submission_recovery_exhaustions()
12432                .is_empty()
12433        );
12434    }
12435
12436    fn assert_command_timeout_event(
12437        events: &[OrderEventAny],
12438        client_order_id: ClientOrderId,
12439        policy: SubmissionRecoveryPolicy,
12440        cancel: bool,
12441    ) {
12442        let [event] = events else {
12443            panic!("Expected one command timeout event, was {events:?}");
12444        };
12445
12446        let is_expected = match (policy, cancel) {
12447            (SubmissionRecoveryPolicy::ResolveLocally, _) => {
12448                matches!(event, OrderEventAny::Canceled(_))
12449            }
12450            (SubmissionRecoveryPolicy::RetainUnresolved, true) => {
12451                matches!(event, OrderEventAny::CancelRejected(_))
12452            }
12453            (SubmissionRecoveryPolicy::RetainUnresolved, false) => {
12454                matches!(event, OrderEventAny::ModifyRejected(_))
12455            }
12456        };
12457
12458        assert!(is_expected, "Unexpected command timeout event {event:?}");
12459        assert_eq!(event.client_order_id(), client_order_id);
12460    }
12461
12462    #[rstest]
12463    #[case::submission_only(None, false)]
12464    #[case::modify(Some(false), false)]
12465    #[case::cancel(Some(true), false)]
12466    #[case::batch_modify(Some(false), true)]
12467    #[case::batch_cancel(Some(true), true)]
12468    #[cfg_attr(
12469        not(all(feature = "simulation", madsim)),
12470        tokio::test(start_paused = true)
12471    )]
12472    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12473    async fn test_unconfirmed_submission_query_report_confirms_before_exhaustion(
12474        #[case] cancel: Option<bool>,
12475        #[case] batch: bool,
12476        #[values(false, true)] targeted: bool,
12477        #[values(
12478            SubmissionRecoveryPolicy::ResolveLocally,
12479            SubmissionRecoveryPolicy::RetainUnresolved
12480        )]
12481        policy: SubmissionRecoveryPolicy,
12482    ) {
12483        let (mut node, order, instrument) = venue_evidence_node_with_config(
12484            crate::config::LiveExecutionEngineConfig {
12485                reconciliation: true,
12486                inflight_check_threshold_ms: 100,
12487                inflight_check_retries: 2,
12488                open_check_threshold_ms: 1,
12489                open_check_open_only: false,
12490                open_check_lookback_mins: None,
12491                open_check_missing_retries: 1,
12492                single_order_query_delay_ms: 0,
12493                submission_recovery_policy: policy,
12494                ..Default::default()
12495            },
12496            OrderType::Limit,
12497            true,
12498        );
12499        let client_order_id = order.client_order_id();
12500        let client_id = ClientId::from("TEST-EVIDENCE");
12501        let account_id = AccountId::from("TEST-001");
12502        node.kernel
12503            .exec_engine
12504            .borrow_mut()
12505            .register_client(Box::new(StubExecutionClient::new(
12506                client_id,
12507                account_id,
12508                instrument.id().venue,
12509                OmsType::Netting,
12510                None,
12511            )))
12512            .unwrap();
12513        advance_clock(Duration::from_millis(101)).await;
12514        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12515        advance_clock(Duration::from_millis(10)).await;
12516
12517        if let Some(cancel) = cancel {
12518            node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12519                MessagingSwitchboard::exec_engine_execute(),
12520                venue_evidence_command(&order, cancel, batch),
12521            )));
12522            let pending = if cancel {
12523                OrderEventAny::PendingCancel(
12524                    OrderPendingCancelSpec::builder()
12525                        .trader_id(order.trader_id())
12526                        .strategy_id(order.strategy_id())
12527                        .instrument_id(order.instrument_id())
12528                        .client_order_id(client_order_id)
12529                        .account_id(account_id)
12530                        .build(),
12531                )
12532            } else {
12533                OrderEventAny::PendingUpdate(
12534                    OrderPendingUpdateSpec::builder()
12535                        .trader_id(order.trader_id())
12536                        .strategy_id(order.strategy_id())
12537                        .instrument_id(order.instrument_id())
12538                        .client_order_id(client_order_id)
12539                        .account_id(account_id)
12540                        .build(),
12541                )
12542            };
12543            node.process_exec_event(ExecutionEvent::Order(pending));
12544        }
12545        let queries = Rc::new(RefCell::new(Vec::new()));
12546        let client = OrderReportClient {
12547            client_id,
12548            report: OrderStatusReport::new(
12549                account_id,
12550                order.instrument_id(),
12551                Some(client_order_id),
12552                VenueOrderId::from("V-VENUE-EVIDENCE"),
12553                Some(order.order_side()),
12554                order.order_type(),
12555                order.time_in_force(),
12556                OrderStatus::Submitted,
12557                order.quantity(),
12558                Quantity::zero(3),
12559                UnixNanos::default(),
12560                UnixNanos::default(),
12561                UnixNanos::default(),
12562                None,
12563            )
12564            .with_price(order.price().unwrap()),
12565            targeted,
12566            queries: queries.clone(),
12567        };
12568        advance_clock(Duration::from_millis(40)).await;
12569        let events = node.exec_manager.check_open_orders(&[&client]).await;
12570        assert!(events.is_empty());
12571        assert_eq!(
12572            *queries.borrow(),
12573            if targeted {
12574                vec!["bulk", "targeted"]
12575            } else {
12576                vec!["bulk"]
12577            }
12578        );
12579        node.process_reconciliation_events(&events);
12580        advance_clock(Duration::from_millis(61)).await;
12581        let first = node.exec_manager.check_inflight_orders();
12582        let recover_command =
12583            cancel.is_some() && policy == SubmissionRecoveryPolicy::ResolveLocally;
12584        assert_eq!(first.queries.len(), usize::from(recover_command));
12585
12586        if cancel.is_none() && policy == SubmissionRecoveryPolicy::ResolveLocally {
12587            assert!(matches!(&first.events[..], [OrderEventAny::Rejected(event)]
12588                if event.client_order_id == client_order_id));
12589        } else {
12590            assert!(first.events.is_empty());
12591        }
12592        node.process_reconciliation_events(&first.events);
12593        advance_clock(Duration::from_millis(101)).await;
12594        let exhausted = node.exec_manager.check_inflight_orders();
12595        assert!(exhausted.queries.is_empty());
12596
12597        if recover_command {
12598            assert!(
12599                matches!(&exhausted.events[..], [OrderEventAny::Canceled(event)]
12600                if event.client_order_id == client_order_id)
12601            );
12602        } else {
12603            assert!(exhausted.events.is_empty());
12604        }
12605        assert!(
12606            node.exec_manager
12607                .take_submission_recovery_exhaustions()
12608                .is_empty()
12609        );
12610    }
12611
12612    #[rstest]
12613    #[case::local_update_before_pending("updated")]
12614    #[case::modify_rejected_after_pending("modify_rejected")]
12615    #[case::cancel_rejected_after_pending("cancel_rejected")]
12616    #[case::modify_rejected_after_cancel_pending("modify_rejected_after_cancel_pending")]
12617    #[cfg_attr(
12618        not(all(feature = "simulation", madsim)),
12619        tokio::test(start_paused = true)
12620    )]
12621    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12622    async fn test_unconfirmed_submission_completed_command_does_not_revive_budget(
12623        #[case] completion: &str,
12624        #[values(
12625            SubmissionRecoveryPolicy::ResolveLocally,
12626            SubmissionRecoveryPolicy::RetainUnresolved
12627        )]
12628        policy: SubmissionRecoveryPolicy,
12629    ) {
12630        use nautilus_model::events::order::spec::{
12631            OrderCancelRejectedSpec, OrderModifyRejectedSpec,
12632        };
12633
12634        let (mut node, order, instrument) = venue_evidence_node(policy);
12635        let client_order_id = order.client_order_id();
12636        let client_id = ClientId::from("TEST-EVIDENCE");
12637        let account_id = AccountId::from("TEST-001");
12638        let cancel = completion == "cancel_rejected";
12639        let later_cancel_pending = completion == "modify_rejected_after_cancel_pending";
12640        node.kernel
12641            .exec_engine
12642            .borrow_mut()
12643            .register_client(Box::new(StubExecutionClient::new(
12644                client_id,
12645                account_id,
12646                instrument.id().venue,
12647                OmsType::Netting,
12648                None,
12649            )))
12650            .unwrap();
12651        node.process_exec_command(DispatchMessage::from(TradingCommandMessage::new(
12652            MessagingSwitchboard::exec_engine_execute(),
12653            venue_evidence_command(&order, cancel, false),
12654        )));
12655        advance_clock(Duration::from_millis(10)).await;
12656
12657        if completion != "updated" {
12658            let pending = if cancel {
12659                OrderEventAny::PendingCancel(
12660                    OrderPendingCancelSpec::builder()
12661                        .trader_id(order.trader_id())
12662                        .strategy_id(order.strategy_id())
12663                        .instrument_id(order.instrument_id())
12664                        .client_order_id(client_order_id)
12665                        .account_id(account_id)
12666                        .build(),
12667                )
12668            } else {
12669                OrderEventAny::PendingUpdate(
12670                    OrderPendingUpdateSpec::builder()
12671                        .trader_id(order.trader_id())
12672                        .strategy_id(order.strategy_id())
12673                        .instrument_id(order.instrument_id())
12674                        .client_order_id(client_order_id)
12675                        .account_id(account_id)
12676                        .build(),
12677                )
12678            };
12679            node.process_exec_event(ExecutionEvent::Order(pending));
12680        }
12681
12682        if later_cancel_pending {
12683            // The cancel is pending in native state, but has not dispatched
12684            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12685                OrderPendingCancelSpec::builder()
12686                    .trader_id(order.trader_id())
12687                    .strategy_id(order.strategy_id())
12688                    .instrument_id(order.instrument_id())
12689                    .client_order_id(client_order_id)
12690                    .account_id(account_id)
12691                    .build(),
12692            )));
12693        }
12694        advance_clock(Duration::from_millis(10)).await;
12695        let completed = match completion {
12696            "updated" => OrderEventAny::Updated(
12697                OrderUpdatedSpec::builder()
12698                    .trader_id(order.trader_id())
12699                    .strategy_id(order.strategy_id())
12700                    .instrument_id(order.instrument_id())
12701                    .client_order_id(client_order_id)
12702                    .account_id(account_id)
12703                    .quantity(order.quantity())
12704                    .price(Price::from("101.00"))
12705                    .build(),
12706            ),
12707            "modify_rejected" | "modify_rejected_after_cancel_pending" => {
12708                OrderEventAny::ModifyRejected(
12709                    OrderModifyRejectedSpec::builder()
12710                        .trader_id(order.trader_id())
12711                        .strategy_id(order.strategy_id())
12712                        .instrument_id(order.instrument_id())
12713                        .client_order_id(client_order_id)
12714                        .account_id(account_id)
12715                        .build(),
12716                )
12717            }
12718            "cancel_rejected" => OrderEventAny::CancelRejected(
12719                OrderCancelRejectedSpec::builder()
12720                    .trader_id(order.trader_id())
12721                    .strategy_id(order.strategy_id())
12722                    .instrument_id(order.instrument_id())
12723                    .client_order_id(client_order_id)
12724                    .account_id(account_id)
12725                    .build(),
12726            ),
12727            _ => unreachable!(),
12728        };
12729        node.process_exec_event(ExecutionEvent::Order(completed));
12730        assert_eq!(
12731            node.kernel
12732                .cache
12733                .borrow()
12734                .order(&client_order_id)
12735                .unwrap()
12736                .status(),
12737            if later_cancel_pending {
12738                OrderStatus::PendingCancel
12739            } else {
12740                OrderStatus::Submitted
12741            }
12742        );
12743
12744        // A newer pending event is queued, but that command has not dispatched
12745        advance_clock(Duration::from_millis(10)).await;
12746
12747        if !later_cancel_pending {
12748            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingUpdate(
12749                OrderPendingUpdateSpec::builder()
12750                    .trader_id(order.trader_id())
12751                    .strategy_id(order.strategy_id())
12752                    .instrument_id(order.instrument_id())
12753                    .client_order_id(client_order_id)
12754                    .account_id(account_id)
12755                    .build(),
12756            )));
12757        }
12758        advance_clock(Duration::from_millis(10)).await;
12759        let cached = node
12760            .kernel
12761            .cache
12762            .borrow()
12763            .order_owned(&client_order_id)
12764            .unwrap();
12765        let OrderEventAny::Filled(mut fill) = OrderFilledTestBuilder::new(&cached, &instrument)
12766            .last_qty(Quantity::from("0.400"))
12767            .last_px(order.price().unwrap())
12768            .commission(Money::from("0 USDT"))
12769            .without_position_id()
12770            .build()
12771        else {
12772            unreachable!()
12773        };
12774        fill.venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12775        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Filled(fill)));
12776
12777        for _ in 0..3 {
12778            advance_clock(Duration::from_millis(101)).await;
12779            let result = node.exec_manager.check_inflight_orders();
12780            assert!(result.queries.is_empty());
12781            assert!(result.events.is_empty());
12782            assert!(
12783                node.exec_manager
12784                    .take_submission_recovery_exhaustions()
12785                    .is_empty()
12786            );
12787        }
12788        assert_eq!(
12789            node.kernel
12790                .cache
12791                .borrow()
12792                .order(&client_order_id)
12793                .unwrap()
12794                .status(),
12795            if later_cancel_pending {
12796                OrderStatus::PendingCancel
12797            } else {
12798                OrderStatus::PendingUpdate
12799            }
12800        );
12801    }
12802
12803    #[rstest]
12804    #[case::wrong_account("account")]
12805    #[case::wrong_instrument("instrument")]
12806    #[case::conflicting_report_venue("report_venue")]
12807    #[case::unapplied_venue_update("update_venue")]
12808    #[case::local_update("local_update")]
12809    #[cfg_attr(
12810        not(all(feature = "simulation", madsim)),
12811        tokio::test(start_paused = true)
12812    )]
12813    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
12814    async fn test_submission_confirmation_requires_matching_venue_evidence(#[case] invalid: &str) {
12815        let (mut node, order, _) = venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
12816        let client_order_id = order.client_order_id();
12817        let account_id = AccountId::from("TEST-001");
12818        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12819
12820        if invalid.ends_with("venue") {
12821            insert_accepted_limit_order_in_node(
12822                &node,
12823                account_id,
12824                ClientId::from("TEST-EVIDENCE"),
12825                order.instrument_id(),
12826                ClientOrderId::from("O-OTHER"),
12827                venue_order_id,
12828            );
12829        }
12830        node.process_exec_event(ExecutionEvent::Order(OrderEventAny::PendingCancel(
12831            OrderPendingCancelSpec::builder()
12832                .trader_id(order.trader_id())
12833                .strategy_id(order.strategy_id())
12834                .instrument_id(order.instrument_id())
12835                .client_order_id(client_order_id)
12836                .account_id(account_id)
12837                .build(),
12838        )));
12839        advance_clock(Duration::from_millis(101)).await;
12840        assert_eq!(node.exec_manager.check_inflight_orders().queries.len(), 1);
12841
12842        if invalid == "update_venue" || invalid == "local_update" {
12843            let mut updated = OrderUpdatedSpec::builder()
12844                .trader_id(order.trader_id())
12845                .strategy_id(order.strategy_id())
12846                .instrument_id(order.instrument_id())
12847                .client_order_id(client_order_id)
12848                .account_id(account_id)
12849                .quantity(order.quantity())
12850                .price(order.price().unwrap())
12851                .build();
12852
12853            if invalid == "update_venue" {
12854                updated.venue_order_id = Some(venue_order_id);
12855            }
12856            node.process_exec_event(ExecutionEvent::Order(OrderEventAny::Updated(updated)));
12857        } else {
12858            let report = OrderStatusReport::new(
12859                if invalid == "account" {
12860                    AccountId::from("OTHER-001")
12861                } else {
12862                    account_id
12863                },
12864                if invalid == "instrument" {
12865                    InstrumentId::from("OTHER.BINANCE")
12866                } else {
12867                    order.instrument_id()
12868                },
12869                Some(client_order_id),
12870                venue_order_id,
12871                Some(order.order_side()),
12872                order.order_type(),
12873                order.time_in_force(),
12874                OrderStatus::Submitted,
12875                order.quantity(),
12876                Quantity::zero(3),
12877                UnixNanos::default(),
12878                UnixNanos::default(),
12879                UnixNanos::default(),
12880                None,
12881            )
12882            .with_price(order.price().unwrap());
12883            node.process_exec_event(ExecutionEvent::Report(ExecutionReport::Order(Box::new(
12884                report,
12885            ))));
12886        }
12887        assert!(
12888            node.kernel
12889                .cache
12890                .borrow()
12891                .order(&client_order_id)
12892                .unwrap()
12893                .venue_order_id()
12894                .is_none()
12895        );
12896        advance_clock(Duration::from_millis(101)).await;
12897        let result = node.exec_manager.check_inflight_orders();
12898        assert!(result.events.is_empty());
12899        assert!(result.queries.is_empty());
12900        let cached = node
12901            .kernel
12902            .cache
12903            .borrow()
12904            .order_owned(&client_order_id)
12905            .unwrap();
12906        assert_eq!(cached.status(), OrderStatus::PendingCancel);
12907        let diagnostics = node.exec_manager.take_submission_recovery_exhaustions();
12908        assert_eq!(diagnostics.len(), 1);
12909        assert_eq!(diagnostics[0].client_order_id, client_order_id);
12910    }
12911
12912    fn venue_evidence_command(order: &OrderAny, cancel: bool, batch: bool) -> TradingCommand {
12913        use nautilus_common::messages::execution::{
12914            BatchCancelOrders, BatchModifyOrders, CancelOrder, ModifyOrder,
12915        };
12916
12917        let client_id = ClientId::from("TEST-EVIDENCE");
12918        let client_order_id = order.client_order_id();
12919        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
12920
12921        if cancel {
12922            let cancel = CancelOrder::new(
12923                order.trader_id(),
12924                Some(client_id),
12925                order.strategy_id(),
12926                order.instrument_id(),
12927                client_order_id,
12928                Some(venue_order_id),
12929                UUID4::new(),
12930                UnixNanos::default(),
12931                None,
12932                None,
12933            );
12934
12935            if batch {
12936                TradingCommand::CancelOrders(BatchCancelOrders::new(
12937                    order.trader_id(),
12938                    Some(client_id),
12939                    order.strategy_id(),
12940                    order.instrument_id(),
12941                    vec![cancel],
12942                    UUID4::new(),
12943                    UnixNanos::default(),
12944                    None,
12945                    None,
12946                ))
12947            } else {
12948                TradingCommand::CancelOrder(cancel)
12949            }
12950        } else {
12951            let modify = ModifyOrder::new(
12952                order.trader_id(),
12953                Some(client_id),
12954                order.strategy_id(),
12955                order.instrument_id(),
12956                client_order_id,
12957                Some(venue_order_id),
12958                None,
12959                Some(Price::from("101.00")),
12960                None,
12961                UUID4::new(),
12962                UnixNanos::default(),
12963                None,
12964                None,
12965            );
12966
12967            if batch {
12968                TradingCommand::ModifyOrders(BatchModifyOrders::new(
12969                    order.trader_id(),
12970                    Some(client_id),
12971                    order.strategy_id(),
12972                    order.instrument_id(),
12973                    vec![modify],
12974                    UUID4::new(),
12975                    UnixNanos::default(),
12976                    None,
12977                    None,
12978                ))
12979            } else {
12980                TradingCommand::ModifyOrder(modify)
12981            }
12982        }
12983    }
12984
12985    fn venue_evidence_node(
12986        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12987    ) -> (LiveNode, OrderAny, InstrumentAny) {
12988        venue_evidence_node_with_order_type(policy, OrderType::Limit)
12989    }
12990
12991    fn venue_evidence_node_with_order_type(
12992        policy: crate::execution::submission::SubmissionRecoveryPolicy,
12993        order_type: OrderType,
12994    ) -> (LiveNode, OrderAny, InstrumentAny) {
12995        venue_evidence_node_with_config(
12996            crate::config::LiveExecutionEngineConfig {
12997                reconciliation: true,
12998                inflight_check_threshold_ms: 100,
12999                inflight_check_retries: 2,
13000                open_check_threshold_ms: 5_000,
13001                single_order_query_delay_ms: 0,
13002                submission_recovery_policy: policy,
13003                ..Default::default()
13004            },
13005            order_type,
13006            true,
13007        )
13008    }
13009
13010    fn venue_evidence_node_with_config(
13011        exec_engine: crate::config::LiveExecutionEngineConfig,
13012        order_type: OrderType,
13013        register_submission: bool,
13014    ) -> (LiveNode, OrderAny, InstrumentAny) {
13015        let config = LiveNodeConfig {
13016            exec_engine,
13017            ..Default::default()
13018        };
13019        let mut node = LiveNode::build("VenueEvidenceNode".to_string(), Some(config)).unwrap();
13020        let instrument = InstrumentAny::CryptoPerpetual(crypto_perpetual_ethusdt());
13021        let account_id = AccountId::from("TEST-001");
13022        let client_id = ClientId::from("TEST-EVIDENCE");
13023        let account = AccountAny::Margin(MarginAccount::new(
13024            AccountState::new(
13025                account_id,
13026                AccountType::Margin,
13027                vec![AccountBalance::new(
13028                    Money::from("1000000 USDT"),
13029                    Money::from("0 USDT"),
13030                    Money::from("1000000 USDT"),
13031                )],
13032                Vec::new(),
13033                true,
13034                UUID4::new(),
13035                UnixNanos::default(),
13036                UnixNanos::default(),
13037                Some(Currency::USDT()),
13038            ),
13039            true,
13040        ));
13041        node.kernel.cache.borrow_mut().add_account(account).unwrap();
13042        node.kernel
13043            .cache
13044            .borrow_mut()
13045            .add_instrument(instrument.clone())
13046            .unwrap();
13047        let mut builder = OrderTestBuilder::new(order_type);
13048        builder
13049            .trader_id(node.trader_id())
13050            .client_order_id(ClientOrderId::from("O-VENUE-EVIDENCE"))
13051            .instrument_id(instrument.id())
13052            .quantity(Quantity::from("1.000"))
13053            .price(Price::from("100.00"));
13054
13055        if order_type == OrderType::StopLimit {
13056            builder.trigger_price(Price::from("110.00"));
13057        }
13058        let order = builder.build();
13059        let command = TradingCommand::SubmitOrder(SubmitOrder::from_order(
13060            &order,
13061            node.trader_id(),
13062            Some(client_id),
13063            None,
13064            UUID4::new(),
13065            UnixNanos::default(),
13066        ));
13067
13068        if register_submission {
13069            node.observe_exec_command_before_dispatch(&command);
13070        }
13071        node.kernel
13072            .cache
13073            .borrow_mut()
13074            .add_order(order.clone(), None, Some(client_id), false)
13075            .unwrap();
13076        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::submitted(
13077            &order, account_id,
13078        )));
13079        (node, order, instrument)
13080    }
13081
13082    #[rstest]
13083    #[case::partial("0.400", OrderStatus::PartiallyFilled)]
13084    #[case::full("1.000", OrderStatus::Filled)]
13085    #[tokio::test(start_paused = true)]
13086    async fn test_retained_submission_resolves_late_fill(
13087        #[case] quantity: &str,
13088        #[case] expected_status: OrderStatus,
13089    ) {
13090        let (mut node, order, instrument) =
13091            venue_evidence_node(SubmissionRecoveryPolicy::RetainUnresolved);
13092        let client_order_id = order.client_order_id();
13093
13094        for _ in 0..2 {
13095            advance_clock(Duration::from_millis(101)).await;
13096            assert!(node.exec_manager.check_inflight_orders().events.is_empty());
13097        }
13098
13099        assert_eq!(
13100            node.exec_manager
13101                .take_submission_recovery_exhaustions()
13102                .len(),
13103            1
13104        );
13105        let fill = TestOrderEventStubs::filled(
13106            &order,
13107            &instrument,
13108            Some(TradeId::from("T-LATE-RETAINED")),
13109            None,
13110            Some(Price::from("100.00")),
13111            Some(Quantity::from(quantity)),
13112            Some(LiquiditySide::Taker),
13113            None,
13114            None,
13115            Some(AccountId::from("TEST-001")),
13116        );
13117        node.process_exec_event(ExecutionEvent::Order(fill));
13118        advance_clock(Duration::from_millis(101)).await;
13119        let result = node.exec_manager.check_inflight_orders();
13120        let cache = node.kernel.cache.borrow();
13121        let cached = cache.order_ref(&client_order_id).unwrap();
13122        assert_eq!(cached.status(), expected_status);
13123        assert_eq!(cached.filled_qty(), Quantity::from(quantity));
13124        assert_eq!(cached.trade_ids(), vec![&TradeId::from("T-LATE-RETAINED")]);
13125        assert!(result.events.is_empty());
13126        assert!(result.queries.is_empty());
13127        assert!(
13128            node.exec_manager
13129                .take_submission_recovery_exhaustions()
13130                .is_empty()
13131        );
13132    }
13133
13134    #[rstest]
13135    #[case::submitted_report("order")]
13136    #[case::submitted_report_with_fills("order_with_fills")]
13137    #[case::venue_update("updated")]
13138    #[cfg_attr(
13139        not(all(feature = "simulation", madsim)),
13140        tokio::test(start_paused = true)
13141    )]
13142    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13143    async fn test_venue_evidence_stops_submission_timeout(
13144        #[case] evidence: &str,
13145        #[values(
13146            OrderStatus::Submitted,
13147            OrderStatus::PendingCancel,
13148            OrderStatus::PendingUpdate
13149        )]
13150        status: OrderStatus,
13151        #[values(
13152            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13153            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13154        )]
13155        policy: crate::execution::submission::SubmissionRecoveryPolicy,
13156    ) {
13157        let (mut node, order, _) = venue_evidence_node(policy);
13158        let client_order_id = order.client_order_id();
13159        let account_id = AccountId::from("TEST-001");
13160        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13161        let pending = match status {
13162            OrderStatus::PendingCancel => Some(OrderEventAny::PendingCancel(
13163                OrderPendingCancelSpec::builder()
13164                    .trader_id(order.trader_id())
13165                    .strategy_id(order.strategy_id())
13166                    .instrument_id(order.instrument_id())
13167                    .client_order_id(client_order_id)
13168                    .account_id(account_id)
13169                    .build(),
13170            )),
13171            OrderStatus::PendingUpdate => Some(OrderEventAny::PendingUpdate(
13172                OrderPendingUpdateSpec::builder()
13173                    .trader_id(order.trader_id())
13174                    .strategy_id(order.strategy_id())
13175                    .instrument_id(order.instrument_id())
13176                    .client_order_id(client_order_id)
13177                    .account_id(account_id)
13178                    .build(),
13179            )),
13180            _ => None,
13181        };
13182
13183        if let Some(pending) = pending {
13184            node.process_exec_event(ExecutionEvent::Order(pending));
13185        }
13186        advance_clock(Duration::from_millis(101)).await;
13187        let first = node.exec_manager.check_inflight_orders();
13188        assert_eq!(first.queries.len(), 1);
13189        assert!(first.events.is_empty());
13190        advance_clock(Duration::from_millis(94)).await;
13191
13192        let event = if evidence == "updated" {
13193            ExecutionEvent::Order(OrderEventAny::Updated(
13194                OrderUpdatedSpec::builder()
13195                    .trader_id(order.trader_id())
13196                    .strategy_id(order.strategy_id())
13197                    .instrument_id(order.instrument_id())
13198                    .client_order_id(client_order_id)
13199                    .account_id(account_id)
13200                    .venue_order_id(venue_order_id)
13201                    .quantity(order.quantity())
13202                    .price(Price::from("101.00"))
13203                    .build(),
13204            ))
13205        } else {
13206            let report = OrderStatusReport::new(
13207                account_id,
13208                order.instrument_id(),
13209                Some(client_order_id),
13210                venue_order_id,
13211                Some(order.order_side()),
13212                order.order_type(),
13213                order.time_in_force(),
13214                OrderStatus::Submitted,
13215                order.quantity(),
13216                Quantity::zero(3),
13217                UnixNanos::default(),
13218                UnixNanos::default(),
13219                UnixNanos::default(),
13220                None,
13221            )
13222            .with_price(Price::from("100.00"));
13223            ExecutionEvent::Report(if evidence == "order" {
13224                ExecutionReport::Order(Box::new(report))
13225            } else {
13226                ExecutionReport::OrderWithFills(Box::new(report), Vec::new())
13227            })
13228        };
13229        node.process_exec_event(event);
13230
13231        let expected_status = if evidence == "updated" && status != OrderStatus::PendingCancel {
13232            OrderStatus::Submitted
13233        } else if status == OrderStatus::Submitted {
13234            OrderStatus::Accepted
13235        } else {
13236            status
13237        };
13238        assert_eq!(
13239            node.kernel
13240                .cache
13241                .borrow()
13242                .order(&client_order_id)
13243                .unwrap()
13244                .status(),
13245            expected_status,
13246        );
13247
13248        for _ in 0..3 {
13249            advance_clock(Duration::from_millis(101)).await;
13250            let result = node.exec_manager.check_inflight_orders();
13251            assert!(result.queries.is_empty());
13252            assert!(result.events.is_empty());
13253        }
13254        assert_eq!(
13255            node.exec_manager.recon_check_retry_count(&client_order_id),
13256            0
13257        );
13258        assert!(node.exec_manager.check_open_order_queries().is_empty());
13259        advance_clock(Duration::from_secs(5)).await;
13260        assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13261    }
13262
13263    #[rstest]
13264    #[case::fill("fill", false)]
13265    #[case::terminal_order("order", true)]
13266    #[case::terminal_order_with_fills("order_with_fills", true)]
13267    #[case::working_order_for_closed_cache("order", false)]
13268    #[case::working_order_with_fills_for_closed_cache("order_with_fills", false)]
13269    #[cfg_attr(
13270        not(all(feature = "simulation", madsim)),
13271        tokio::test(start_paused = true)
13272    )]
13273    #[cfg_attr(all(feature = "simulation", madsim), madsim::test)]
13274    async fn test_report_cleanup_preserves_original_settling_predicate(
13275        #[case] report_kind: &str,
13276        #[case] terminal_report: bool,
13277        #[values(false, true)] explicit_client_id: bool,
13278        #[values(
13279            crate::execution::submission::SubmissionRecoveryPolicy::ResolveLocally,
13280            crate::execution::submission::SubmissionRecoveryPolicy::RetainUnresolved
13281        )]
13282        policy: crate::execution::submission::SubmissionRecoveryPolicy,
13283    ) {
13284        let (mut node, order, instrument) = venue_evidence_node(policy);
13285        let client_order_id = order.client_order_id();
13286        let account_id = AccountId::from("TEST-001");
13287        let client_id = ClientId::from("TEST-EVIDENCE");
13288        let venue_order_id = VenueOrderId::from("V-VENUE-EVIDENCE");
13289        node.process_exec_event(ExecutionEvent::Order(TestOrderEventStubs::accepted(
13290            &order,
13291            account_id,
13292            venue_order_id,
13293        )));
13294        let accepted = node
13295            .kernel
13296            .cache
13297            .borrow()
13298            .order_owned(&client_order_id)
13299            .unwrap();
13300        let OrderEventAny::Filled(fill) = OrderFilledTestBuilder::new(&accepted, &instrument)
13301            .last_qty(accepted.quantity())
13302            .last_px(Price::from("100.00"))
13303            .commission(Money::from("0 USDT"))
13304            .without_position_id()
13305            .build()
13306        else {
13307            unreachable!()
13308        };
13309        let mut fill_report = FillReport::new(
13310            account_id,
13311            instrument.id(),
13312            venue_order_id,
13313            fill.trade_id,
13314            fill.order_side,
13315            fill.last_qty,
13316            fill.last_px,
13317            fill.commission.unwrap(),
13318            fill.liquidity_side,
13319            Some(client_order_id),
13320            None,
13321            fill.ts_event,
13322            fill.ts_init,
13323            None,
13324        );
13325
13326        if !explicit_client_id {
13327            fill_report.client_order_id = None;
13328        }
13329        let event = if report_kind == "fill" {
13330            ExecutionReport::Fill(Box::new(fill_report))
13331        } else {
13332            if !terminal_report {
13333                node.kernel
13334                    .cache
13335                    .borrow_mut()
13336                    .update_order(&OrderEventAny::Filled(fill))
13337                    .unwrap();
13338            }
13339            let report = OrderStatusReport::new(
13340                account_id,
13341                instrument.id(),
13342                explicit_client_id.then_some(client_order_id),
13343                venue_order_id,
13344                Some(order.order_side()),
13345                order.order_type(),
13346                order.time_in_force(),
13347                if terminal_report {
13348                    OrderStatus::Filled
13349                } else {
13350                    OrderStatus::Accepted
13351                },
13352                order.quantity(),
13353                order.quantity(),
13354                UnixNanos::default(),
13355                UnixNanos::default(),
13356                UnixNanos::default(),
13357                None,
13358            )
13359            .with_price(Price::from("100.00"))
13360            .with_avg_px(dec!(100.00));
13361
13362            if report_kind == "order" {
13363                ExecutionReport::Order(Box::new(report))
13364            } else {
13365                ExecutionReport::OrderWithFills(Box::new(report), vec![fill_report])
13366            }
13367        };
13368        node.process_exec_event(ExecutionEvent::Report(event));
13369        assert_eq!(
13370            node.kernel
13371                .cache
13372                .borrow()
13373                .order(&client_order_id)
13374                .unwrap()
13375                .status(),
13376            OrderStatus::Filled
13377        );
13378
13379        // Replace only cached state, without an observer stamp, to expose the existing grace gate
13380        node.kernel
13381            .cache
13382            .borrow_mut()
13383            .add_order(order.clone(), None, Some(client_id), true)
13384            .unwrap();
13385        node.kernel
13386            .cache
13387            .borrow_mut()
13388            .update_order(&TestOrderEventStubs::submitted(&order, account_id))
13389            .unwrap();
13390        node.kernel
13391            .cache
13392            .borrow_mut()
13393            .update_order(&TestOrderEventStubs::accepted(
13394                &order,
13395                account_id,
13396                venue_order_id,
13397            ))
13398            .unwrap();
13399        let should_clear = terminal_report && explicit_client_id;
13400        assert_eq!(
13401            node.exec_manager.check_open_order_queries().len(),
13402            usize::from(should_clear)
13403        );
13404
13405        if !should_clear {
13406            advance_clock(Duration::from_secs(5)).await;
13407            assert_eq!(node.exec_manager.check_open_order_queries().len(), 1);
13408        }
13409    }
13410
13411    #[rstest]
13412    #[case(stub_order_event, 1)]
13413    #[case(stub_submitted_batch_event, 2)]
13414    #[case(stub_accepted_batch_event, 2)]
13415    #[case(stub_canceled_batch_event, 2)]
13416    fn test_pending_events_preserve_roots_when_splitting_batches(
13417        #[values(false, true)] rooted: bool,
13418        #[case] make_event: fn() -> ExecutionEvent,
13419        #[case] orders: usize,
13420    ) {
13421        let runner = AsyncRunner::new();
13422        runner.bind_senders();
13423        let mut channels = runner.take_channels();
13424
13425        let send = move || {
13426            get_data_event_sender().send(stub_data_event()).unwrap();
13427            get_exec_event_sender().send(stub_exec_event()).unwrap();
13428            get_exec_event_sender().send(make_event()).unwrap();
13429        };
13430
13431        if rooted {
13432            msgbus::register_trading_command_endpoint(
13433                MessagingSwitchboard::risk_engine_execute(),
13434                TypedIntoHandler::from(move |_| send()),
13435            );
13436            TradingCommandSender::execute(
13437                &SyncTradingCommandSender,
13438                TradingCommandMessage::new(
13439                    MessagingSwitchboard::risk_engine_execute(),
13440                    TradingCommand::QueryAccount(QueryAccount::new(
13441                        "TRADER-001".into(),
13442                        None,
13443                        "SIM-001".into(),
13444                        UUID4::new(),
13445                        37.into(),
13446                        None,
13447                        None,
13448                    )),
13449                ),
13450            );
13451            nautilus_common::runner::drain_trading_cmd_queue();
13452        } else {
13453            send();
13454        }
13455
13456        let mut pending = PendingEvents::default();
13457        pending
13458            .data_evts
13459            .push(channels.data_evt_rx.try_recv().unwrap());
13460        while let Ok(event) = channels.exec_evt_rx.try_recv() {
13461            pending.push_exec_event(event);
13462        }
13463
13464        assert_eq!(pending.data_evts.len(), 1);
13465        assert_eq!(pending.exec_reports.len(), 1);
13466        assert_eq!(pending.order_evts.len(), orders);
13467        assert_eq!(pending.data_evts[0].is_rooted(), rooted);
13468        assert_eq!(pending.exec_reports[0].is_rooted(), rooted);
13469        assert!(
13470            pending
13471                .order_evts
13472                .iter()
13473                .all(|event| event.is_rooted() == rooted)
13474        );
13475
13476        let results = std::thread::spawn(move || {
13477            let mut results = Vec::new();
13478
13479            for event in pending.data_evts {
13480                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13481                    || event.dispatch(drop),
13482                )));
13483            }
13484
13485            for event in pending.exec_reports {
13486                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13487                    || event.dispatch(drop),
13488                )));
13489            }
13490
13491            for event in pending.order_evts {
13492                results.push(std::panic::catch_unwind(std::panic::AssertUnwindSafe(
13493                    || event.dispatch(drop),
13494                )));
13495            }
13496
13497            results
13498        })
13499        .join()
13500        .unwrap();
13501
13502        assert_eq!(results.len(), orders + 2);
13503
13504        for result in results {
13505            if rooted {
13506                let error = result.unwrap_err();
13507                assert!(
13508                    error
13509                        .downcast_ref::<String>()
13510                        .unwrap()
13511                        .contains("command context dispatched outside its owner thread")
13512                );
13513            } else {
13514                result.unwrap();
13515            }
13516        }
13517    }
13518
13519    fn stub_accepted_batch_event() -> ExecutionEvent {
13520        ExecutionEvent::OrderAcceptedBatch(OrderAcceptedBatch::new(vec![
13521            OrderAcceptedSpec::builder()
13522                .client_order_id(ClientOrderId::from("O-017"))
13523                .build(),
13524            OrderAcceptedSpec::builder()
13525                .client_order_id(ClientOrderId::from("O-023"))
13526                .build(),
13527        ]))
13528    }
13529
13530    #[derive(Debug)]
13531    struct CapturedEgressMessage {
13532        topic: String,
13533        payload: Bytes,
13534    }
13535
13536    type CapturedEgressMessages = Rc<RefCell<Vec<CapturedEgressMessage>>>;
13537    type SharedClosed = Rc<Cell<bool>>;
13538
13539    #[derive(Debug)]
13540    struct CapturingExternalIngress {
13541        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13542        closed: SharedClosed,
13543    }
13544
13545    impl CapturingExternalIngress {
13546        fn new(rx: tokio::sync::mpsc::Receiver<BusMessage>, closed: SharedClosed) -> Self {
13547            Self {
13548                rx: Some(rx),
13549                closed,
13550            }
13551        }
13552    }
13553
13554    impl MessageBusExternalIngress for CapturingExternalIngress {
13555        fn is_closed(&self) -> bool {
13556            self.closed.get()
13557        }
13558
13559        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13560            self.rx
13561                .take()
13562                .ok_or_else(|| anyhow::anyhow!("external ingress receiver already taken"))
13563        }
13564
13565        fn close(&mut self) {
13566            self.closed.set(true);
13567        }
13568    }
13569
13570    #[derive(Debug)]
13571    struct FailingExternalIngress {
13572        closed: SharedClosed,
13573    }
13574
13575    impl FailingExternalIngress {
13576        fn new(closed: SharedClosed) -> Self {
13577            Self { closed }
13578        }
13579    }
13580
13581    impl MessageBusExternalIngress for FailingExternalIngress {
13582        fn is_closed(&self) -> bool {
13583            self.closed.get()
13584        }
13585
13586        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13587            anyhow::bail!("external ingress receiver unavailable")
13588        }
13589
13590        fn close(&mut self) {
13591            self.closed.set(true);
13592        }
13593    }
13594
13595    struct CapturingExternalEgress {
13596        publications: CapturedEgressMessages,
13597        closed: SharedClosed,
13598    }
13599
13600    impl CapturingExternalEgress {
13601        fn new() -> (Self, CapturedEgressMessages, SharedClosed) {
13602            let publications = Rc::new(RefCell::new(Vec::new()));
13603            let closed = Rc::new(Cell::new(false));
13604            (
13605                Self {
13606                    publications: publications.clone(),
13607                    closed: closed.clone(),
13608                },
13609                publications,
13610                closed,
13611            )
13612        }
13613    }
13614
13615    impl MessageBusExternalEgress for CapturingExternalEgress {
13616        fn is_closed(&self) -> bool {
13617            self.closed.get()
13618        }
13619
13620        fn publish(&self, message: BusMessage) {
13621            self.publications.borrow_mut().push(CapturedEgressMessage {
13622                topic: message.topic.to_string(),
13623                payload: message.payload,
13624            });
13625        }
13626
13627        fn close(&mut self) {
13628            self.closed.set(true);
13629        }
13630    }
13631
13632    struct CapturingBackingFactory {
13633        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13634        closed: Arc<AtomicBool>,
13635        rx: Mutex<Option<tokio::sync::mpsc::Receiver<BusMessage>>>,
13636    }
13637
13638    impl CapturingBackingFactory {
13639        fn new(
13640            publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13641            closed: Arc<AtomicBool>,
13642            rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13643        ) -> Self {
13644            Self {
13645                publications,
13646                closed,
13647                rx: Mutex::new(rx),
13648            }
13649        }
13650    }
13651
13652    impl Debug for CapturingBackingFactory {
13653        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
13654            f.debug_struct(stringify!(CapturingBackingFactory))
13655                .finish_non_exhaustive()
13656        }
13657    }
13658
13659    impl MessageBusBackingFactory for CapturingBackingFactory {
13660        fn create(
13661            &self,
13662            _trader_id: TraderId,
13663            _instance_id: UUID4,
13664            _config: MessageBusConfig,
13665        ) -> anyhow::Result<Box<dyn MessageBusBacking>> {
13666            let rx = self.rx.lock().take();
13667            Ok(Box::new(CapturingBacking {
13668                publications: self.publications.clone(),
13669                closed: self.closed.clone(),
13670                rx,
13671            }))
13672        }
13673    }
13674
13675    struct CapturingBacking {
13676        publications: Arc<Mutex<Vec<CapturedEgressMessage>>>,
13677        closed: Arc<AtomicBool>,
13678        rx: Option<tokio::sync::mpsc::Receiver<BusMessage>>,
13679    }
13680
13681    impl MessageBusBacking for CapturingBacking {
13682        fn is_closed(&self) -> bool {
13683            self.closed.load(Ordering::Relaxed)
13684        }
13685
13686        fn publish(&self, message: BusMessage) {
13687            self.publications.lock().push(CapturedEgressMessage {
13688                topic: message.topic.to_string(),
13689                payload: message.payload,
13690            });
13691        }
13692
13693        fn take_receiver(&mut self) -> anyhow::Result<tokio::sync::mpsc::Receiver<BusMessage>> {
13694            self.rx
13695                .take()
13696                .ok_or_else(|| anyhow::anyhow!("external ingress receiver unavailable"))
13697        }
13698
13699        fn close(&mut self) {
13700            self.closed.store(true, Ordering::Relaxed);
13701        }
13702    }
13703}